mirror of
https://github.com/OCA/server-backend.git
synced 2025-02-18 09:52:42 +02:00
[IMP] base_external_dbsource: Refactor & Split by source
* Heavily refactor code for reusability * Split all sources into independent modules * Add more test coverage * Add CRUD methods * Add iterator execute return to roadmap OCA Transbot updated translations from Transifex
This commit is contained in:
Dave Lasley
committed by
Andrea Cattalani
Andrea Cattalani
parent
144ee73398
commit
879cc23bd3
@@ -1,70 +1,50 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2011 Daniel Reis
|
||||
# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl).
|
||||
# Copyright 2016 LasLabs Inc.
|
||||
# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html).
|
||||
|
||||
import os
|
||||
import logging
|
||||
import psycopg2
|
||||
|
||||
from odoo import models, fields, api, _
|
||||
from odoo.exceptions import Warning as UserError
|
||||
import odoo.tools as tools
|
||||
from contextlib import contextmanager
|
||||
|
||||
from odoo import _, api, fields, models, tools
|
||||
|
||||
from ..exceptions import ConnectionFailedError, ConnectionSuccessError
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
CONNECTORS = []
|
||||
|
||||
try:
|
||||
import sqlalchemy
|
||||
CONNECTORS.append(('sqlite', 'SQLite'))
|
||||
try:
|
||||
import pymssql
|
||||
CONNECTORS.append(('mssql', 'Microsoft SQL Server'))
|
||||
assert pymssql
|
||||
except (ImportError, AssertionError):
|
||||
_logger.info('MS SQL Server not available. Please install "pymssql"\
|
||||
python package.')
|
||||
try:
|
||||
import MySQLdb
|
||||
CONNECTORS.append(('mysql', 'MySQL'))
|
||||
assert MySQLdb
|
||||
except (ImportError, AssertionError):
|
||||
_logger.info('MySQL not available. Please install "mysqldb"\
|
||||
python package.')
|
||||
except:
|
||||
_logger.info('SQL Alchemy not available. Please install "slqalchemy"\
|
||||
python package.')
|
||||
try:
|
||||
import pyodbc
|
||||
CONNECTORS.append(('pyodbc', 'ODBC'))
|
||||
except:
|
||||
_logger.info('ODBC libraries not available. Please install "unixodbc"\
|
||||
and "python-pyodbc" packages.')
|
||||
|
||||
try:
|
||||
import cx_Oracle
|
||||
CONNECTORS.append(('cx_Oracle', 'Oracle'))
|
||||
except:
|
||||
_logger.info('Oracle libraries not available. Please install "cx_Oracle"\
|
||||
python package.')
|
||||
|
||||
try:
|
||||
import fdb
|
||||
CONNECTORS.append(('fdb', 'Firebird'))
|
||||
except:
|
||||
_logger.info('Firebird libraries not available. Please install "fdb"\
|
||||
python package.')
|
||||
|
||||
CONNECTORS.append(('postgresql', 'PostgreSQL'))
|
||||
|
||||
|
||||
class BaseExternalDbsource(models.Model):
|
||||
""" It provides logic for connection to an external data source
|
||||
|
||||
Classes implementing this interface must provide the following methods
|
||||
suffixed with the adapter type. See the method definitions and examples
|
||||
for more information:
|
||||
* ``connection_open_*``
|
||||
* ``connection_close_*``
|
||||
* ``execute_*``
|
||||
|
||||
Optional methods for adapters to implement:
|
||||
* ``remote_browse_*``
|
||||
* ``remote_create_*``
|
||||
* ``remote_delete_*``
|
||||
* ``remote_search_*``
|
||||
* ``remote_update_*``
|
||||
"""
|
||||
|
||||
_name = "base.external.dbsource"
|
||||
_description = 'External Database Sources'
|
||||
|
||||
name = fields.Char('Datasource name', required=True, size=64)
|
||||
CONNECTORS = [
|
||||
('postgresql', 'PostgreSQL'),
|
||||
]
|
||||
# This is appended to the conn string if pass declared but not detected.
|
||||
# Children should declare PWD_STRING_CONNECTOR (such as PWD_STRING_FBD)
|
||||
# to allow for override.
|
||||
PWD_STRING = 'PWD=%s;'
|
||||
|
||||
name = fields.Char('Datasource name', required=True, size=64)
|
||||
conn_string = fields.Text('Connection string', help="""
|
||||
Sample connection strings:
|
||||
- Microsoft SQL Server:
|
||||
@@ -72,64 +52,99 @@ class BaseExternalDbsource(models.Model):
|
||||
- MySQL: mysql://user:%s@server:port/dbname
|
||||
- ODBC: DRIVER={FreeTDS};SERVER=server.address;Database=mydb;UID=sa
|
||||
- ORACLE: username/%s@//server.address:port/instance
|
||||
- FireBird: host=localhost;database=mydatabase.gdb;user=sysdba;password=%s;
|
||||
port=3050;charset=utf8
|
||||
- PostgreSQL:
|
||||
dbname='template1' user='dbuser' host='localhost' port='5432' \
|
||||
password=%s
|
||||
- SQLite: sqlite:///test.db
|
||||
- Elasticsearch: https://user:%s@localhost:9200
|
||||
""")
|
||||
|
||||
conn_string_full = fields.Text(
|
||||
readonly=True,
|
||||
compute='_compute_conn_string_full',
|
||||
)
|
||||
password = fields.Char('Password', size=40)
|
||||
client_cert = fields.Text()
|
||||
client_key = fields.Text()
|
||||
ca_certs = fields.Char(
|
||||
help='Path to CA Certs file on server.',
|
||||
)
|
||||
connector = fields.Selection(
|
||||
CONNECTORS, 'Connector', required=True,
|
||||
help="If a connector is missing from the list, check the server "
|
||||
"log to confirm that the required components were detected.",
|
||||
)
|
||||
|
||||
connector = fields.Selection(CONNECTORS, 'Connector', required=True,
|
||||
help="If a connector is missing from the\
|
||||
list, check the server log to confirm\
|
||||
that the required components were\
|
||||
detected.")
|
||||
current_table = None
|
||||
|
||||
@api.multi
|
||||
def conn_open(self):
|
||||
"""The connection is open here."""
|
||||
@api.depends('conn_string', 'password')
|
||||
def _compute_conn_string_full(self):
|
||||
for record in self:
|
||||
if record.password:
|
||||
if '%s' not in record.conn_string:
|
||||
pwd_string = getattr(
|
||||
record,
|
||||
'PWD_STRING_%s' % record.connector.upper(),
|
||||
record.PWD_STRING,
|
||||
)
|
||||
record.conn_string += pwd_string
|
||||
record.conn_string_full = record.conn_string % record.password
|
||||
else:
|
||||
record.conn_string_full = record.conn_string
|
||||
|
||||
self.ensure_one()
|
||||
# Get dbsource record
|
||||
# Build the full connection string
|
||||
connStr = self.conn_string
|
||||
if self.password:
|
||||
if '%s' not in self.conn_string:
|
||||
connStr += ';PWD=%s'
|
||||
connStr = connStr % self.password
|
||||
|
||||
# Try to connect
|
||||
if self.connector == 'cx_Oracle':
|
||||
os.environ['NLS_LANG'] = 'AMERICAN_AMERICA.UTF8'
|
||||
conn = cx_Oracle.connect(connStr)
|
||||
elif self.connector == 'pyodbc':
|
||||
conn = pyodbc.connect(connStr)
|
||||
elif self.connector in ('sqlite', 'mysql', 'mssql'):
|
||||
conn = sqlalchemy.create_engine(connStr).connect()
|
||||
elif self.connector == 'fdb':
|
||||
kwargs = dict([x.split('=') for x in connStr.split(';')])
|
||||
conn = fdb.connect(**kwargs)
|
||||
elif self.connector == 'postgresql':
|
||||
conn = psycopg2.connect(connStr)
|
||||
|
||||
return conn
|
||||
# Interface
|
||||
|
||||
@api.multi
|
||||
def execute(self, sqlquery, sqlparams=None, metadata=False,
|
||||
context=None):
|
||||
"""Executes SQL and returns a list of rows.
|
||||
def change_table(self, name):
|
||||
""" Change the table that is used for CRUD operations """
|
||||
self.current_table = name
|
||||
|
||||
"sqlparams" can be a dict of values, that can be referenced in
|
||||
the SQL statement using "%(key)s" or, in the case of Oracle,
|
||||
@api.multi
|
||||
def connection_close(self, connection):
|
||||
""" It closes the connection to the data source.
|
||||
|
||||
This method calls adapter method of this same name, suffixed with
|
||||
the adapter type.
|
||||
"""
|
||||
|
||||
method = self._get_adapter_method('connection_close')
|
||||
return method(connection)
|
||||
|
||||
@api.multi
|
||||
@contextmanager
|
||||
def connection_open(self):
|
||||
""" It provides a context manager for the data source.
|
||||
|
||||
This method calls adapter method of this same name, suffixed with
|
||||
the adapter type.
|
||||
"""
|
||||
|
||||
method = self._get_adapter_method('connection_open')
|
||||
try:
|
||||
connection = method()
|
||||
yield connection
|
||||
finally:
|
||||
try:
|
||||
self.connection_close(connection)
|
||||
except:
|
||||
_logger.exception('Connection close failure.')
|
||||
|
||||
@api.multi
|
||||
def execute(
|
||||
self, query=None, execute_params=None, metadata=False, **kwargs
|
||||
):
|
||||
""" Executes a query and returns a list of rows.
|
||||
|
||||
"execute_params" can be a dict of values, that can be referenced
|
||||
in the SQL statement using "%(key)s" or, in the case of Oracle,
|
||||
":key".
|
||||
Example:
|
||||
sqlquery = "select * from mytable where city = %(city)s and
|
||||
query = "SELECT * FROM mytable WHERE city = %(city)s AND
|
||||
date > %(dt)s"
|
||||
params = {'city': 'Lisbon',
|
||||
'dt': datetime.datetime(2000, 12, 31)}
|
||||
execute_params = {
|
||||
'city': 'Lisbon',
|
||||
'dt': datetime.datetime(2000, 12, 31),
|
||||
}
|
||||
|
||||
If metadata=True, it will instead return a dict containing the
|
||||
rows list and the columns list, in the format:
|
||||
@@ -137,52 +152,202 @@ class BaseExternalDbsource(models.Model):
|
||||
, 'rows': [ (a0, b0, ...), (a1, b1, ...), ...] }
|
||||
"""
|
||||
|
||||
rows, cols = list(), list()
|
||||
for obj in self:
|
||||
conn = obj.conn_open()
|
||||
if obj.connector in ["sqlite", "mysql", "mssql"]:
|
||||
# using sqlalchemy
|
||||
cur = conn.execute(sqlquery, sqlparams)
|
||||
if metadata:
|
||||
cols = cur.keys()
|
||||
rows = [r for r in cur]
|
||||
# Old API compatibility
|
||||
if not query:
|
||||
try:
|
||||
query = kwargs['sqlquery']
|
||||
except KeyError:
|
||||
raise TypeError(_('query is a required argument'))
|
||||
if not execute_params:
|
||||
try:
|
||||
execute_params = kwargs['sqlparams']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
elif obj.connector in ["fdb"]:
|
||||
# using other db connectors
|
||||
cur = conn.cursor()
|
||||
for key in sqlparams:
|
||||
sqlquery = sqlquery.replace('%%(%s)s' % key,
|
||||
str(sqlparams[key]))
|
||||
method = self._get_adapter_method('execute')
|
||||
rows, cols = method(query, execute_params, metadata)
|
||||
|
||||
cur.execute(sqlquery)
|
||||
rows = cur.fetchall()
|
||||
else:
|
||||
# using other db connectors
|
||||
cur = conn.cursor()
|
||||
cur.execute(sqlquery, sqlparams)
|
||||
if metadata:
|
||||
cols = [d[0] for d in cur.description]
|
||||
rows = cur.fetchall()
|
||||
conn.close()
|
||||
if metadata:
|
||||
return{'cols': cols, 'rows': rows}
|
||||
return {'cols': cols, 'rows': rows}
|
||||
else:
|
||||
return rows
|
||||
|
||||
@api.multi
|
||||
def connection_test(self):
|
||||
"""Test of connection."""
|
||||
self.ensure_one()
|
||||
conn = False
|
||||
try:
|
||||
conn = self.conn_open()
|
||||
except Exception as e:
|
||||
raise UserError(_("Connection test failed: \
|
||||
Here is what we got instead:\n %s") % tools.ustr(e))
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
""" It tests the connection
|
||||
|
||||
# TODO: if OK a (wizard) message box should be displayed
|
||||
raise UserError(_("Connection test succeeded: \
|
||||
Everything seems properly set up!"))
|
||||
Raises:
|
||||
ConnectionSuccessError: On connection success
|
||||
ConnectionFailedError: On connection failed
|
||||
"""
|
||||
|
||||
for obj in self:
|
||||
try:
|
||||
with self.connection_open():
|
||||
pass
|
||||
except Exception as e:
|
||||
raise ConnectionFailedError(_(
|
||||
"Connection test failed:\n"
|
||||
"Here is what we got instead:\n%s"
|
||||
) % tools.ustr(e))
|
||||
raise ConnectionSuccessError(_(
|
||||
"Connection test succeeded:\n"
|
||||
"Everything seems properly set up!",
|
||||
))
|
||||
|
||||
@api.multi
|
||||
def remote_browse(self, record_ids, *args, **kwargs):
|
||||
""" It browses for and returns the records from remote by ID
|
||||
|
||||
This method calls adapter method of this same name, suffixed with
|
||||
the adapter type.
|
||||
|
||||
Args:
|
||||
record_ids: (list) List of remote IDs to browse.
|
||||
*args: Positional arguments to be passed to adapter method.
|
||||
**kwargs: Keyword arguments to be passed to adapter method.
|
||||
Returns:
|
||||
(iter) Iterator of record mappings that match the ID.
|
||||
"""
|
||||
|
||||
assert self.current_table
|
||||
method = self._get_adapter_method('remote_browse')
|
||||
return method(record_ids, *args, **kwargs)
|
||||
|
||||
@api.multi
|
||||
def remote_create(self, vals, *args, **kwargs):
|
||||
""" It creates a record on the remote data source.
|
||||
|
||||
This method calls adapter method of this same name, suffixed with
|
||||
the adapter type.
|
||||
|
||||
Args:
|
||||
vals: (dict) Values to use for creation.
|
||||
*args: Positional arguments to be passed to adapter method.
|
||||
**kwargs: Keyword arguments to be passed to adapter method.
|
||||
Returns:
|
||||
(mapping) A mapping of the record that was created.
|
||||
"""
|
||||
|
||||
assert self.current_table
|
||||
method = self._get_adapter_method('remote_create')
|
||||
return method(vals, *args, **kwargs)
|
||||
|
||||
@api.multi
|
||||
def remote_delete(self, record_ids, *args, **kwargs):
|
||||
""" It deletes records by ID on remote
|
||||
|
||||
This method calls adapter method of this same name, suffixed with
|
||||
the adapter type.
|
||||
|
||||
Args:
|
||||
record_ids: (list) List of remote IDs to delete.
|
||||
*args: Positional arguments to be passed to adapter method.
|
||||
**kwargs: Keyword arguments to be passed to adapter method.
|
||||
Returns:
|
||||
(iter) Iterator of bools indicating delete status.
|
||||
"""
|
||||
|
||||
assert self.current_table
|
||||
method = self._get_adapter_method('remote_delete')
|
||||
return method(record_ids, *args, **kwargs)
|
||||
|
||||
@api.multi
|
||||
def remote_search(self, query, *args, **kwargs):
|
||||
""" It searches the remote for the query.
|
||||
|
||||
This method calls adapter method of this same name, suffixed with
|
||||
the adapter type.
|
||||
|
||||
Args:
|
||||
query: (mixed) Query domain as required by the adapter.
|
||||
*args: Positional arguments to be passed to adapter method.
|
||||
**kwargs: Keyword arguments to be passed to adapter method.
|
||||
Returns:
|
||||
(iter) Iterator of record mappings that match query.
|
||||
"""
|
||||
|
||||
assert self.current_table
|
||||
method = self._get_adapter_method('remote_search')
|
||||
return method(query, *args, **kwargs)
|
||||
|
||||
@api.multi
|
||||
def remote_update(self, record_ids, vals, *args, **kwargs):
|
||||
""" It updates the remote records with the vals
|
||||
|
||||
This method calls adapter method of this same name, suffixed with
|
||||
the adapter type.
|
||||
|
||||
Args:
|
||||
record_ids: (list) List of remote IDs to delete.
|
||||
*args: Positional arguments to be passed to adapter method.
|
||||
**kwargs: Keyword arguments to be passed to adapter method.
|
||||
Returns:
|
||||
(iter) Iterator of record mappings that were updated.
|
||||
"""
|
||||
|
||||
assert self.current_table
|
||||
method = self._get_adapter_method('remote_update')
|
||||
return method(record_ids, vals, *args, **kwargs)
|
||||
|
||||
# Adapters
|
||||
|
||||
def connection_close_postgresql(self, connection):
|
||||
return connection.close()
|
||||
|
||||
def connection_open_postgresql(self):
|
||||
return psycopg2.connect(self.conn_string)
|
||||
|
||||
def execute_postgresql(self, query, params, metadata):
|
||||
return self._execute_generic(query, params, metadata)
|
||||
|
||||
def _execute_generic(self, query, params, metadata):
|
||||
with self.connection_open() as connection:
|
||||
cur = connection.cursor()
|
||||
cur.execute(query, params)
|
||||
cols = []
|
||||
if metadata:
|
||||
cols = [d[0] for d in cur.description]
|
||||
rows = cur.fetchall()
|
||||
return rows, cols
|
||||
|
||||
# Compatibility & Private
|
||||
|
||||
@api.multi
|
||||
def conn_open(self):
|
||||
""" It opens and returns a connection to the remote data source.
|
||||
|
||||
This method calls adapter method of this same name, suffixed with
|
||||
the adapter type.
|
||||
|
||||
Deprecate:
|
||||
This method has been replaced with ``connection_open``.
|
||||
"""
|
||||
|
||||
with self.connection_open() as connection:
|
||||
return connection
|
||||
|
||||
def _get_adapter_method(self, method_prefix):
|
||||
""" It returns the connector adapter method for ``method_prefix``.
|
||||
|
||||
Args:
|
||||
method_prefix: (str) Prefix of adapter method (such as
|
||||
``connection_open``).
|
||||
Raises:
|
||||
NotImplementedError: When the method is not found
|
||||
Returns:
|
||||
(instancemethod)
|
||||
"""
|
||||
|
||||
self.ensure_one()
|
||||
method = '%s_%s' % (method_prefix, self.connector)
|
||||
|
||||
try:
|
||||
return getattr(self, method)
|
||||
except AttributeError:
|
||||
raise NotImplementedError(_(
|
||||
'"%s" method not found, check that all assets are installed '
|
||||
'for the %s connector type.'
|
||||
)) % (
|
||||
method, self.connector,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user