Merge PR #163 into 12.0

Signed-off-by dreispt
This commit is contained in:
OCA-git-bot
2022-08-11 07:17:49 +00:00
13 changed files with 543 additions and 0 deletions

97
pglogical/README.rst Normal file
View File

@@ -0,0 +1,97 @@
=========
pglogical
=========
.. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!! This file is generated by oca-gen-addon-readme !!
!! changes will be overwritten. !!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png
:target: https://odoo-community.org/page/development-status
:alt: Alpha
.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png
:target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
:alt: License: AGPL-3
.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fserver--backend-lightgray.png?logo=github
:target: https://github.com/OCA/server-backend/tree/12.0/pglogical
:alt: OCA/server-backend
.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png
:target: https://translation.odoo-community.org/projects/server-backend-12-0/server-backend-12-0-pglogical
:alt: Translate me on Weblate
.. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png
:target: https://runbot.odoo-community.org/runbot/253/12.0
:alt: Try me on Runbot
|badge1| |badge2| |badge3| |badge4| |badge5|
This module supports replication of an Odoo database with `pglogical <https://github.com/2ndQuadrant/pglogical>`__ by also passing DDL to the replicas. You can configure which replica sets to pass DDL to.
.. IMPORTANT::
This is an alpha version, the data model and design can change at any time without warning.
Only for development or testing purpose, do not use in production.
`More details on development status <https://odoo-community.org/page/development-status>`_
**Table of contents**
.. contents::
:local:
Configuration
=============
To configure this module, you need to:
#. add section ``[pglogical]`` to your odoo configuration file
#. add value ``replication_sets = set1,set2,set3,etc`` in this section
#. add ``pglogical`` to your list of server wide modules
Example::
[pglogical]
replication_sets = ddl_sql
Usage
=====
If configured correctly, this module is completely transparent for users.
Bug Tracker
===========
Bugs are tracked on `GitHub Issues <https://github.com/OCA/server-backend/issues>`_.
In case of trouble, please check there if your issue has already been reported.
If you spotted it first, help us smashing it by providing a detailed and welcomed
`feedback <https://github.com/OCA/server-backend/issues/new?body=module:%20pglogical%0Aversion:%2012.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
Do not contact contributors directly about support or help with technical issues.
Credits
=======
Authors
~~~~~~~
* Hunki Enterprises BV
Contributors
~~~~~~~~~~~~
* Holger Brunn <mail@hunki-enterprises.com> (https://hunki-enterprises.com)
Maintainers
~~~~~~~~~~~
This module is maintained by the OCA.
.. image:: https://odoo-community.org/logo.png
:alt: Odoo Community Association
:target: https://odoo-community.org
OCA, or the Odoo Community Association, is a nonprofit organization whose
mission is to support the collaborative development of Odoo features and
promote its widespread use.
This module is part of the `OCA/server-backend <https://github.com/OCA/server-backend/tree/12.0/pglogical>`_ project on GitHub.
You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

3
pglogical/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
from .hooks import post_load

16
pglogical/__manifest__.py Normal file
View File

@@ -0,0 +1,16 @@
# Copyright 2022 Hunki Enterprises BV
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
{
"name": "pglogical",
"summary": "Support for replicating Odoo's database",
"version": "12.0.1.0.0",
"development_status": "Alpha",
"category": "Tools",
"website": "https://github.com/OCA/server-backend",
"author": "Hunki Enterprises BV, Odoo Community Association (OCA)",
"license": "AGPL-3",
"post_load": "post_load",
'external_dependencies': {
'python': ['sqlparse'],
},
}

138
pglogical/hooks.py Normal file
View File

@@ -0,0 +1,138 @@
# Copyright 2022 Hunki Enterprises BV
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
import logging
from odoo.sql_db import Cursor
from odoo.tools import config
try:
import sqlparse
except ImportError:
sqlparse = None
SECTION_NAME = "pglogical"
DDL_KEYWORDS = set(["CREATE", "ALTER", "DROP", "TRUNCATE"])
QUALIFY_KEYWORDS = DDL_KEYWORDS | set(["FROM", "INHERITS", "JOIN"])
NO_QUALIFY_KEYWORDS = set(["AS", "COLUMN", "ON", "RETURNS", "SELECT"])
TEMPORARY = set(["TEMP", "TEMPORARY"])
def schema_qualify(tokens, temp_tables, keywords=None, schema="public"):
"""
Add tokens to add a schema to objects if there's none, but record and
exclude temporary tables
"""
Identifier = sqlparse.sql.Identifier
Name = sqlparse.tokens.Name
Punctuation = sqlparse.tokens.Punctuation
Token = sqlparse.sql.Token
Statement = sqlparse.sql.Statement
Function = sqlparse.sql.Function
Parenthesis = sqlparse.sql.Parenthesis
keywords = list(keywords or [])
for token in tokens.tokens:
if token.is_keyword:
keywords.append(token.normalized)
continue
elif token.is_whitespace:
continue
elif token.__class__ == Identifier and not token.is_wildcard():
str_token = str(token)
needs_qualification = "." not in str_token
# qualify tokens that are direct children of a statement as in ALTER TABLE
if token.parent.__class__ == Statement:
pass
# or of an expression parsed as function as in CREATE TABLE table
# but not within brackets
if token.parent.__class__ == Function:
needs_qualification &= not token.within(Parenthesis)
elif token.parent.__class__ == Parenthesis:
needs_qualification &= (
keywords and (keywords[-1] in QUALIFY_KEYWORDS) or False
)
# but not if the identifier is ie a column name
if set(keywords) & NO_QUALIFY_KEYWORDS:
needs_qualification &= (
keywords and (keywords[-1] in QUALIFY_KEYWORDS) or False
)
# and also not if this is a temporary table
if needs_qualification:
if len(keywords) > 1 and keywords[-2] in TEMPORARY:
needs_qualification = False
temp_tables.append(str_token)
elif str_token in temp_tables:
needs_qualification = False
temp_tables.remove(str_token)
if needs_qualification:
token.insert_before(0, Token(Punctuation, "."))
token.insert_before(0, Token(Name, schema))
keywords = []
elif token.is_group:
schema_qualify(token, temp_tables, keywords=keywords, schema=schema)
return tokens.tokens
def post_load():
"""
Patch cursor to funnel DDL through pglogical.replicate_ddl_command if configured to
do so
"""
_logger = logging.getLogger("odoo.addons.pglogical")
if SECTION_NAME not in config.misc:
_logger.info("%s section missing in config, not doing anything", SECTION_NAME)
return
replication_sets = list(
filter(None, config.misc[SECTION_NAME].get("replication_sets", "").split(","))
)
if not replication_sets:
_logger.error("no replication sets defined, not doing anything")
return
if not sqlparse:
_logger.error("DDL replication not supported - sqlparse is not available")
return
if config["test_enable"]:
_logger.info("test mode enabled, not doing anything")
return
_logger.info("patching cursor to intercept ddl")
execute_org = Cursor.execute
def execute(self, query, params=None, log_exceptions=None):
"""Wrap DDL in pglogical.replicate_ddl_command"""
# short circuit statements that must be as fast as possible
if query[:6] not in ("SELECT", "UPDATE"):
temp_tables = getattr(self, "__temp_tables", [])
parsed_queries = sqlparse.parse(query)
if any(
parsed_query.get_type() in DDL_KEYWORDS
for parsed_query in parsed_queries
) and not any(
token.is_keyword and token.normalized in
# don't replicate constraints, triggers, indexes
("CONSTRAINT", "TRIGGER", "INDEX")
for parsed in parsed_queries
for token in parsed.tokens[1:]
):
qualified_query = "".join(
"".join(
str(token)
for token in schema_qualify(
parsed_query,
temp_tables,
)
)
for parsed_query in parsed_queries
)
mogrified = self.mogrify(qualified_query, params).decode("utf8")
query = "SELECT pglogical.replicate_ddl_command(%s, %s)"
params = (mogrified, execute.replication_sets)
setattr(self, "__temp_tables", temp_tables)
return execute.origin(self, query, params=params, log_exceptions=log_exceptions)
execute.origin = execute_org
execute.replication_sets = replication_sets
Cursor.execute = execute

View File

@@ -0,0 +1,10 @@
To configure this module, you need to:
#. add section ``[pglogical]`` to your odoo configuration file
#. add value ``replication_sets = set1,set2,set3,etc`` in this section
#. add ``pglogical`` to your list of server wide modules
Example::
[pglogical]
replication_sets = ddl_sql

View File

@@ -0,0 +1 @@
* Holger Brunn <mail@hunki-enterprises.com> (https://hunki-enterprises.com)

View File

@@ -0,0 +1 @@
This module supports replication of an Odoo database with `pglogical <https://github.com/2ndQuadrant/pglogical>`__ by also passing DDL to the replicas. You can configure which replica sets to pass DDL to.

View File

@@ -0,0 +1 @@
If configured correctly, this module is completely transparent for users.

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.2 KiB

View File

@@ -0,0 +1,124 @@
<section class="oe_container">
<div class="oe_row oe_spaced">
<div class="oe_span12">
<h2 class="oe_slogan">Module name</h2>
<p>This module was written to extend the functionality of ... to support ... and allow you to ...</p>
</div>
</div>
</section>
<section class="oe_container oe_dark">
<div class="oe_row oe_spaced">
<div class="oe_span12">
<h2 class="oe_slogan">Installation</h2>
</div>
<div class="oe_span6">
<p class="oe_mt32">To install this module, you need to:
<ul>
<li>...</li>
</ul>
</p>
</div>
<div class="oe_span6">
<div class="oe_demo oe_picture oe_screenshot">
<a href="https://www.odoo.com/saas_master/demo?lang=en_US&module=crm">
<img src="crm_sc_01.png">
</a>
</div>
</div>
</div>
</section>
<section class="oe_container">
<div class="oe_row oe_spaced">
<div class="oe_span12">
<h2 class="oe_slogan">Configuration</h2>
</div>
<div class="oe_span6">
<p class="oe_mt32">To configure this module, you need to:
<ul>
<li>...</li>
</ul>
</p>
</div>
<div class="oe_span6">
<div class="oe_demo oe_picture oe_screenshot">
<a href="https://www.odoo.com/saas_master/demo?lang=en_US&module=crm">
<img src="crm_sc_01.png">
</a>
</div>
</div>
</div>
</section>
<section class="oe_container oe_dark">
<div class="oe_row oe_spaced">
<div class="oe_span12">
<h2 class="oe_slogan">Usage</h2>
</div>
<div class="oe_span6">
<p class="oe_mt32">To use this module, you need to:
<ul>
<li>...</li>
</ul>
</p>
<p class="oe_mt32">For further information, please visit:
<ul>
<li><a href="https://www.odoo.com/forum/help-1">https://www.odoo.com/forum/help-1</a></li>
</ul>
</p>
</div>
<div class="oe_span6">
<div class="oe_demo oe_picture oe_screenshot">
<a href="https://www.odoo.com/saas_master/demo?lang=en_US&module=crm">
<img src="crm_sc_01.png">
</a>
</div>
</div>
</div>
</section>
<section class="oe_container">
<div class="oe_row oe_spaced">
<div class="oe_span12">
<h2 class="oe_slogan">Known issues / Roadmap</h2>
</div>
<div class="oe_span6">
<p class="oe_mt32">
<ul>
<li>...</li>
</ul>
</p>
</div>
<div class="oe_span6">
<div class="oe_demo oe_picture oe_screenshot">
<a href="https://www.odoo.com/saas_master/demo?lang=en_US&module=crm">
<img src="crm_sc_01.png">
</a>
</div>
</div>
</div>
</section>
<section class="oe_container oe_dark">
<div class="oe_row">
<div class="oe_span12">
<h2 class="oe_slogan">Credits</h2>
</div>
<div class="oe_span12">
<h3>Contributors</h3>
<ul>
<li>Firstname Lastname &lt;<a href="mailto:email.address@example.com">email.address@example.com</a>&gt;</li>
</ul>
</div>
<div class="oe_span12">
<h3>Maintainer</h3>
<p>
This module is maintained by the OCA.<br/>
OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.<br/>
To contribute to this module, please visit <a href="http://odoo-community.org">http://odoo-community.org</a>.<br/>
<a href="http://odoo-community.org"><img class="oe_picture oe_centered" src="http://odoo-community.org/logo.png"></a>
</p>
</div>
</div>
</section>

View File

@@ -0,0 +1,3 @@
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
from . import test_pglogical

View File

@@ -0,0 +1,148 @@
# Copyright 2022 Hunki Enterprises BV
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
import mock
from contextlib import contextmanager
from odoo.sql_db import Cursor
from odoo.tests.common import TransactionCase
from odoo.tools.config import config
from ..hooks import post_load, schema_qualify, sqlparse
class TestPglogical(TransactionCase):
@contextmanager
def _config(self, misc, test_enable=False):
"""
Temporarily change the config to test_enable=False and impose a custom misc
section
"""
original_misc = config.misc
config.misc = misc
config["test_enable"] = test_enable
yield
config["test_enable"] = True
config.misc = original_misc
def test_configuration(self):
"""Test we react correctly to misconfigurations"""
with self._config(
dict(pglogical=dict(replication_sets="nothing")), True
), self.assertLogs("odoo.addons.pglogical") as log:
post_load()
self.assertEqual(
log.output,
["INFO:odoo.addons.pglogical:test mode enabled, not doing anything"],
)
with self._config({}), self.assertLogs("odoo.addons.pglogical") as log:
post_load()
self.assertEqual(
log.output,
[
"INFO:odoo.addons.pglogical:pglogical section missing in config, "
"not doing anything"
],
)
with self._config(dict(pglogical={"hello": "world"})), self.assertLogs(
"odoo.addons.pglogical"
) as log:
post_load()
self.assertEqual(
log.output,
[
"ERROR:odoo.addons.pglogical:no replication sets defined, "
"not doing anything"
],
)
with self._config(
dict(pglogical={"replication_sets": "ddl_sql"})
), self.assertLogs("odoo.addons.pglogical") as log, mock.patch(
"odoo.addons.pglogical.hooks.sqlparse"
) as mock_sqlparse:
mock_sqlparse.__bool__.return_value = False
post_load()
self.assertEqual(
log.output,
[
"ERROR:odoo.addons.pglogical:"
"DDL replication not supported - sqlparse is not available"
],
)
def test_patching(self):
"""Test patching the cursor succeeds"""
with self._config(dict(pglogical=dict(replication_sets="set1,set2"))):
try:
post_load()
self.assertTrue(getattr(Cursor.execute, "origin", False))
with mock.patch.object(self.env.cr, "_obj") as mock_cursor:
self.env.cr.execute("ALTER TABLE test ADD COLUMN test varchar")
self.assertIn(
"pglogical.replicate_ddl_command",
mock_cursor.execute.call_args[0][0],
)
with mock.patch.object(self.env.cr, "_obj") as mock_cursor:
self.env.cr.execute(
"ALTER TABLE test ADD CONSTRAINT test unique(id)"
)
self.assertNotIn(
"pglogical.replicate_ddl_command",
mock_cursor.execute.call_args[0][0],
)
with mock.patch.object(self.env.cr, "_obj") as mock_cursor:
self.env.cr.execute("SELECT * from test")
self.assertNotIn(
"pglogical.replicate_ddl_command",
mock_cursor.execute.call_args[0][0],
)
finally:
Cursor.execute = getattr(Cursor.execute, "origin", Cursor.execute)
def test_schema_qualify(self):
"""Test that schema qualifications are the only changes"""
temp_tables = []
for statement in (
"create table if not exists testtable",
"drop table testtable",
"alter table testtable",
"""create table
testtable
(col1 int, col2 int); select * from testtable""",
"alter table testschema.test drop column somecol",
" DROP view if exists testtable",
"truncate table testtable",
"""CREATE FUNCTION testtable(integer, integer) RETURNS integer
AS 'select $1 + $2;'
LANGUAGE SQL
IMMUTABLE
RETURNS NULL ON NULL INPUT""",
"drop table",
"alter table 'test'",
'ALTER TABLE "testtable" ADD COLUMN "test_field" double precision',
'CREATE TEMP TABLE "temptable" (col1 char) INHERITS (testtable)',
'DROP TABLE "temptable"',
"create view testtable as select col1, col2 from testtable join "
"testtable test1 on col3=test1.col4)",
'CREATE TABLE public."ir_model" (id SERIAL NOT NULL, PRIMARY KEY(id))',
):
qualified_query = "".join(
"".join(
str(token) for token in schema_qualify(parsed_query, temp_tables)
)
for parsed_query in sqlparse.parse(statement)
)
self.assertEqual(
qualified_query,
statement.replace("testtable", "public.testtable").replace(
'"public.testtable"', 'public."testtable"'
),
)

View File

@@ -1,3 +1,4 @@
sqlalchemy
mysqlclient==2.0.1
pymssql
sqlparse