Initial commit of connector_walmart for Odoo 11.0 (using beta version of connector_ecommerce)

connector_walmart/components/__init__.py

@@ -0,0 +1,6 @@
+from . import api
+from . import backend_adapter
+from . import binder
+from . import importer
+from . import exporter
+from . import mapper

connector_walmart/components/api/__init__.py

@@ -0,0 +1 @@
+from . import walmart

connector_walmart/components/api/walmart.py

@@ -0,0 +1,407 @@
+# -*- coding: utf-8 -*-
+
+# BSD License
+#
+# Copyright (c) 2016, Fulfil.IO Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice, this
+#   list of conditions and the following disclaimer in the documentation and/or
+#   other materials provided with the distribution.
+#
+# * Neither the name of Fulfil nor the names of its
+#   contributors may be used to endorse or promote products derived from this
+#   software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+# OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# © 2017 Hibou Corp. - Extended and converted to v3/JSON
+
+
+import requests
+import base64
+import time
+from uuid import uuid4
+# from lxml import etree
+# from lxml.builder import E, ElementMaker
+from json import dumps, loads
+
+from Crypto.Hash import SHA256
+from Crypto.PublicKey import RSA
+from Crypto.Signature import PKCS1_v1_5
+
+try:
+    from urllib.parse import urlencode
+except ImportError:
+    from urllib import urlencode
+
+
+class Walmart(object):
+
+    def __init__(self, consumer_id, channel_type, private_key):
+        self.base_url = 'https://marketplace.walmartapis.com/v3/%s'
+        self.consumer_id = consumer_id
+        self.channel_type = channel_type
+        self.private_key = private_key
+        self.session = requests.Session()
+        self.session.headers['Accept'] = 'application/json'
+        self.session.headers['WM_SVC.NAME'] = 'Walmart Marketplace'
+        self.session.headers['WM_CONSUMER.ID'] = self.consumer_id
+        self.session.headers['WM_CONSUMER.CHANNEL.TYPE'] = self.channel_type
+
+    @property
+    def items(self):
+        return Items(connection=self)
+
+    @property
+    def inventory(self):
+        return Inventory(connection=self)
+
+    @property
+    def prices(self):
+        return Prices(connection=self)
+
+    @property
+    def orders(self):
+        return Orders(connection=self)
+
+    def get_sign(self, url, method, timestamp):
+        return self.sign_data(
+            '\n'.join([self.consumer_id, url, method, timestamp]) + '\n'
+        )
+
+    def sign_data(self, data):
+        rsakey = RSA.importKey(base64.b64decode(self.private_key))
+        signer = PKCS1_v1_5.new(rsakey)
+        digest = SHA256.new()
+        digest.update(data.encode('utf-8'))
+        sign = signer.sign(digest)
+        return base64.b64encode(sign)
+
+    def get_headers(self, url, method):
+        timestamp = str(int(round(time.time() * 1000)))
+        headers = {
+            'WM_SEC.AUTH_SIGNATURE': self.get_sign(url, method, timestamp),
+            'WM_SEC.TIMESTAMP': timestamp,
+            'WM_QOS.CORRELATION_ID': str(uuid4()),
+        }
+        if method in ('POST', ):
+            headers['Content-Type'] = 'application/json'
+        return headers
+
+    def send_request(self, method, url, params=None, body=None):
+        encoded_url = url
+        if params:
+            encoded_url += '?%s' % urlencode(params)
+        headers = self.get_headers(encoded_url, method)
+
+        if method == 'GET':
+            return loads(self.session.get(url, params=params, headers=headers).text)
+        elif method == 'PUT':
+            return loads(self.session.put(url, params=params, headers=headers).text)
+        elif method == 'POST':
+            return loads(self.session.post(url, data=body, headers=headers).text)
+
+
+class Resource(object):
+    """
+    A base class for all Resources to extend
+    """
+
+    def __init__(self, connection):
+        self.connection = connection
+
+    @property
+    def url(self):
+        return self.connection.base_url % self.path
+
+    def all(self, **kwargs):
+        return self.connection.send_request(
+            method='GET', url=self.url, params=kwargs)
+
+    def get(self, id):
+        url = self.url + '/%s' % id
+        return self.connection.send_request(method='GET', url=url)
+
+    def update(self, **kwargs):
+        return self.connection.send_request(
+            method='PUT', url=self.url, params=kwargs)
+
+    # def bulk_update(self, items):
+    #     url = self.connection.base_url % 'feeds?feedType=%s' % self.feedType
+    #     return self.connection.send_request(
+    #         method='POST', url=url, data=self.get_payload(items))
+
+
+class Items(Resource):
+    """
+    Get all items
+    """
+
+    path = 'items'
+
+
+class Inventory(Resource):
+    """
+    Retreives inventory of an item
+    """
+
+    path = 'inventory'
+    feedType = 'inventory'
+
+    def get_payload(self, items):
+        return etree.tostring(
+            E.InventoryFeed(
+                E.InventoryHeader(E('version', '1.4')),
+                *[E(
+                    'inventory',
+                    E('sku', item['sku']),
+                    E(
+                        'quantity',
+                        E('unit', 'EACH'),
+                        E('amount', item['quantity']),
+                    )
+                ) for item in items],
+                xmlns='http://walmart.com/'
+            )
+        )
+
+
+class Prices(Resource):
+    """
+    Retreives price of an item
+    """
+
+    path = 'prices'
+    feedType = 'price'
+
+    def get_payload(self, items):
+        # root = ElementMaker(
+        #     nsmap={'gmp': 'http://walmart.com/'}
+        # )
+        # return etree.tostring(
+        #     root.PriceFeed(
+        #         E.PriceHeader(E('version', '1.5')),
+        #         *[E.Price(
+        #             E(
+        #                 'itemIdentifier',
+        #                 E('sku', item['sku'])
+        #             ),
+        #             E(
+        #                 'pricingList',
+        #                 E(
+        #                     'pricing',
+        #                     E(
+        #                         'currentPrice',
+        #                         E(
+        #                             'value',
+        #                             **{
+        #                                 'currency': item['currenctCurrency'],
+        #                                 'amount': item['currenctPrice']
+        #                             }
+        #                         )
+        #                     ),
+        #                     E('currentPriceType', item['priceType']),
+        #                     E(
+        #                         'comparisonPrice',
+        #                         E(
+        #                             'value',
+        #                             **{
+        #                                 'currency': item['comparisonCurrency'],
+        #                                 'amount': item['comparisonPrice']
+        #                             }
+        #                         )
+        #                     ),
+        #                     E(
+        #                         'priceDisplayCode',
+        #                         **{
+        #                             'submapType': item['displayCode']
+        #                         }
+        #                     ),
+        #                 )
+        #             )
+        #         ) for item in items]
+        #     ), xml_declaration=True, encoding='utf-8'
+        # )
+        payload = {}
+        return
+
+
+class Orders(Resource):
+    """
+    Retrieves Order details
+    """
+
+    path = 'orders'
+
+    def all(self, **kwargs):
+        next_cursor = kwargs.pop('nextCursor', '')
+        return self.connection.send_request(
+            method='GET', url=self.url + next_cursor, params=kwargs)
+
+    def released(self, **kwargs):
+        next_cursor = kwargs.pop('nextCursor', '')
+        url = self.url + '/released'
+        return self.connection.send_request(
+            method='GET', url=url + next_cursor, params=kwargs)
+
+    def acknowledge(self, id):
+        url = self.url + '/%s/acknowledge' % id
+        return self.connection.send_request(method='POST', url=url)
+
+    def cancel(self, id, lines):
+        url = self.url + '/%s/cancel' % id
+        return self.connection.send_request(
+            method='POST', url=url, body=self.get_cancel_payload(lines))
+
+    def get_cancel_payload(self, lines):
+        """
+        {
+          "orderCancellation": {
+            "orderLines": {
+              "orderLine": [
+                {
+                  "lineNumber": "1",
+                  "orderLineStatuses": {
+                    "orderLineStatus": [
+                      {
+                        "status": "Cancelled",
+                        "cancellationReason": "CUSTOMER_REQUESTED_SELLER_TO_CANCEL",
+                        "statusQuantity": {
+                          "unitOfMeasurement": "EA",
+                          "amount": "1"
+                        }
+                      }
+                    ]
+                  }
+                }
+              ]
+            }
+          }
+        }
+        :param lines:
+        :return: string
+        """
+        payload = {
+            'orderCancellation': {
+                'orderLines': [{
+                    'lineNumber': line['number'],
+                    'orderLineStatuses': {
+                        'orderLineStatus': [
+                            {
+                                'status': 'Cancelled',
+                                'cancellationReason': 'CUSTOMER_REQUESTED_SELLER_TO_CANCEL',
+                                'statusQuantity': {
+                                    'unitOfMeasurement': 'EA',
+                                    'amount': line['amount'],
+                                }
+                            }
+                        ]
+                    }
+                } for line in lines]
+            }
+        }
+        return dumps(payload)
+
+
+
+    def ship(self, id, lines):
+        url = self.url + '/%s/shipping' % id
+        return self.connection.send_request(
+            method='POST',
+            url=url,
+            body=self.get_ship_payload(lines)
+        )
+
+    def get_ship_payload(self, lines):
+        """
+
+        :param lines: list[ dict(number, amount, carrier, methodCode, trackingNumber, trackingUrl) ]
+        :return:
+        """
+        """
+        {
+          "orderShipment": {
+            "orderLines": {
+              "orderLine": [
+                {
+                  "lineNumber": "1",
+                  "orderLineStatuses": {
+                    "orderLineStatus": [
+                      {
+                        "status": "Shipped",
+                        "statusQuantity": {
+                          "unitOfMeasurement": "EA",
+                          "amount": "1"
+                        },
+                        "trackingInfo": {
+                          "shipDateTime": 1488480443000,
+                          "carrierName": {
+                            "otherCarrier": null,
+                            "carrier": "UPS"
+                          },
+                          "methodCode": "Express",
+                          "trackingNumber": "12345",
+                          "trackingURL": "www.fedex.com"
+                        }
+                      }
+                    ]
+                  }
+                }
+              ]
+            }
+          }
+        }
+        :param lines:
+        :return:
+        """
+
+        payload = {
+            "orderShipment": {
+                "orderLines": {
+                    "orderLine": [
+                        {
+                            "lineNumber": str(line['number']),
+                            "orderLineStatuses": {
+                                "orderLineStatus": [
+                                    {
+                                        "status": "Shipped",
+                                        "statusQuantity": {
+                                            "unitOfMeasurement": "EA",
+                                            "amount": str(line['amount'])
+                                        },
+                                        "trackingInfo": {
+                                            "shipDateTime": line['shipDateTime'],
+                                            "carrierName": {
+                                                "otherCarrier": None,
+                                                "carrier": line['carrier']
+                                            },
+                                            "methodCode": line['methodCode'],
+                                            "trackingNumber": line['trackingNumber'],
+                                            "trackingURL": line['trackingUrl']
+                                        }
+                                    }
+                                ]
+                            }
+                        }
+                    for line in lines]
+                }
+            }
+        }
+
+        return dumps(payload)

connector_walmart/components/backend_adapter.py

@@ -0,0 +1,67 @@
+# © 2017,2018 Hibou Corp.
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+from odoo.addons.component.core import AbstractComponent
+from odoo.addons.queue_job.exception import RetryableJobError
+from odoo.addons.connector.exception import NetworkRetryableError
+from .api.walmart import Walmart
+from logging import getLogger
+from lxml import etree
+
+_logger = getLogger(__name__)
+
+
+class BaseWalmartConnectorComponent(AbstractComponent):
+    """ Base Walmart Connector Component
+
+    All components of this connector should inherit from it.
+    """
+    _name = 'base.walmart.connector'
+    _inherit = 'base.connector'
+    _collection = 'walmart.backend'
+
+
+class WalmartAdapter(AbstractComponent):
+
+    _name = 'walmart.adapter'
+    _inherit = ['base.backend.adapter', 'base.walmart.connector']
+
+    _walmart_model = None
+
+    def search(self, filters=None):
+        """ Search records according to some criterias
+        and returns a list of ids """
+        raise NotImplementedError
+
+    def read(self, id, attributes=None):
+        """ Returns the information of a record """
+        raise NotImplementedError
+
+    def search_read(self, filters=None):
+        """ Search records according to some criterias
+        and returns their information"""
+        raise NotImplementedError
+
+    def create(self, data):
+        """ Create a record on the external system """
+        raise NotImplementedError
+
+    def write(self, id, data):
+        """ Update records on the external system """
+        raise NotImplementedError
+
+    def delete(self, id):
+        """ Delete a record on the external system """
+        raise NotImplementedError
+
+    @property
+    def api_instance(self):
+        try:
+            walmart_api = getattr(self.work, 'walmart_api')
+        except AttributeError:
+            raise AttributeError(
+                'You must provide a walmart_api attribute with a '
+                'Walmart instance to be able to use the '
+                'Backend Adapter.'
+            )
+        return walmart_api

connector_walmart/components/binder.py

@@ -0,0 +1,22 @@
+# © 2017,2018 Hibou Corp.
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+from odoo.addons.component.core import Component
+
+
+class WalmartModelBinder(Component):
+    """ Bind records and give odoo/walmart ids correspondence
+
+    Binding models are models called ``walmart.{normal_model}``,
+    like ``walmart.sale.order`` or ``walmart.product.product``.
+    They are ``_inherits`` of the normal models and contains
+    the Walmart ID, the ID of the Walmart Backend and the additional
+    fields belonging to the Walmart instance.
+    """
+    _name = 'walmart.binder'
+    _inherit = ['base.binder', 'base.walmart.connector']
+    _apply_on = [
+        'walmart.sale.order',
+        'walmart.sale.order.line',
+        'walmart.stock.picking',
+    ]

connector_walmart/components/exporter.py

@@ -0,0 +1,313 @@
+# © 2017,2018 Hibou Corp.
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+import logging
+
+from contextlib import contextmanager
+from datetime import datetime
+
+import psycopg2
+
+import odoo
+from odoo import _
+from odoo.addons.component.core import AbstractComponent
+from odoo.addons.connector.exception import (IDMissingInBackend,
+                                             RetryableJobError)
+
+_logger = logging.getLogger(__name__)
+
+
+
+
+class WalmartBaseExporter(AbstractComponent):
+    """ Base exporter for Walmart """
+
+    _name = 'walmart.base.exporter'
+    _inherit = ['base.exporter', 'base.walmart.connector']
+    _usage = 'record.exporter'
+
+    def __init__(self, working_context):
+        super(WalmartBaseExporter, self).__init__(working_context)
+        self.binding = None
+        self.external_id = None
+
+    def run(self, binding, *args, **kwargs):
+        """ Run the synchronization
+
+        :param binding: binding record to export
+        """
+        self.binding = binding
+        self.external_id = self.binder.to_external(self.binding)
+
+        result = self._run(*args, **kwargs)
+
+        self.binder.bind(self.external_id, self.binding)
+        # Commit so we keep the external ID when there are several
+        # exports (due to dependencies) and one of them fails.
+        # The commit will also release the lock acquired on the binding
+        # record
+        if not odoo.tools.config['test_enable']:
+            self.env.cr.commit()  # noqa
+
+        self._after_export()
+        return result
+
+    def _run(self):
+        """ Flow of the synchronization, implemented in inherited classes"""
+        raise NotImplementedError
+
+    def _after_export(self):
+        """ Can do several actions after exporting a record to Walmart """
+        pass
+
+
+class WalmartExporter(AbstractComponent):
+    """ A common flow for the exports to Walmart """
+
+    _name = 'walmart.exporter'
+    _inherit = 'walmart.base.exporter'
+
+    def __init__(self, working_context):
+        super(WalmartExporter, self).__init__(working_context)
+        self.binding = None
+
+    def _lock(self):
+        """ Lock the binding record.
+
+        Lock the binding record so we are sure that only one export
+        job is running for this record if concurrent jobs have to export the
+        same record.
+
+        When concurrent jobs try to export the same record, the first one
+        will lock and proceed, the others will fail to lock and will be
+        retried later.
+
+        This behavior works also when the export becomes multilevel
+        with :meth:`_export_dependencies`. Each level will set its own lock
+        on the binding record it has to export.
+
+        """
+        sql = ("SELECT id FROM %s WHERE ID = %%s FOR UPDATE NOWAIT" %
+               self.model._table)
+        try:
+            self.env.cr.execute(sql, (self.binding.id, ),
+                                log_exceptions=False)
+        except psycopg2.OperationalError:
+            _logger.info('A concurrent job is already exporting the same '
+                         'record (%s with id %s). Job delayed later.',
+                         self.model._name, self.binding.id)
+            raise RetryableJobError(
+                'A concurrent job is already exporting the same record '
+                '(%s with id %s). The job will be retried later.' %
+                (self.model._name, self.binding.id))
+
+    def _has_to_skip(self):
+        """ Return True if the export can be skipped """
+        return False
+
+    @contextmanager
+    def _retry_unique_violation(self):
+        """ Context manager: catch Unique constraint error and retry the
+        job later.
+
+        When we execute several jobs workers concurrently, it happens
+        that 2 jobs are creating the same record at the same time (binding
+        record created by :meth:`_export_dependency`), resulting in:
+
+            IntegrityError: duplicate key value violates unique
+            constraint "walmart_product_product_odoo_uniq"
+            DETAIL:  Key (backend_id, odoo_id)=(1, 4851) already exists.
+
+        In that case, we'll retry the import just later.
+
+        .. warning:: The unique constraint must be created on the
+                     binding record to prevent 2 bindings to be created
+                     for the same Walmart record.
+
+        """
+        try:
+            yield
+        except psycopg2.IntegrityError as err:
+            if err.pgcode == psycopg2.errorcodes.UNIQUE_VIOLATION:
+                raise RetryableJobError(
+                    'A database error caused the failure of the job:\n'
+                    '%s\n\n'
+                    'Likely due to 2 concurrent jobs wanting to create '
+                    'the same record. The job will be retried later.' % err)
+            else:
+                raise
+
+    def _export_dependency(self, relation, binding_model,
+                           component_usage='record.exporter',
+                           binding_field='walmart_bind_ids',
+                           binding_extra_vals=None):
+        """
+        Export a dependency. The exporter class is a subclass of
+        ``WalmartExporter``. If a more precise class need to be defined,
+        it can be passed to the ``exporter_class`` keyword argument.
+
+        .. warning:: a commit is done at the end of the export of each
+                     dependency. The reason for that is that we pushed a record
+                     on the backend and we absolutely have to keep its ID.
+
+                     So you *must* take care not to modify the Odoo
+                     database during an export, excepted when writing
+                     back the external ID or eventually to store
+                     external data that we have to keep on this side.
+
+                     You should call this method only at the beginning
+                     of the exporter synchronization,
+                     in :meth:`~._export_dependencies`.
+
+        :param relation: record to export if not already exported
+        :type relation: :py:class:`odoo.models.BaseModel`
+        :param binding_model: name of the binding model for the relation
+        :type binding_model: str | unicode
+        :param component_usage: 'usage' to look for to find the Component to
+                                for the export, by default 'record.exporter'
+        :type exporter: str | unicode
+        :param binding_field: name of the one2many field on a normal
+                              record that points to the binding record
+                              (default: walmart_bind_ids).
+                              It is used only when the relation is not
+                              a binding but is a normal record.
+        :type binding_field: str | unicode
+        :binding_extra_vals:  In case we want to create a new binding
+                              pass extra values for this binding
+        :type binding_extra_vals: dict
+        """
+        if not relation:
+            return
+        rel_binder = self.binder_for(binding_model)
+        # wrap is typically True if the relation is for instance a
+        # 'product.product' record but the binding model is
+        # 'walmart.product.product'
+        wrap = relation._name != binding_model
+
+        if wrap and hasattr(relation, binding_field):
+            domain = [('odoo_id', '=', relation.id),
+                      ('backend_id', '=', self.backend_record.id)]
+            binding = self.env[binding_model].search(domain)
+            if binding:
+                assert len(binding) == 1, (
+                    'only 1 binding for a backend is '
+                    'supported in _export_dependency')
+            # we are working with a unwrapped record (e.g.
+            # product.category) and the binding does not exist yet.
+            # Example: I created a product.product and its binding
+            # walmart.product.product and we are exporting it, but we need to
+            # create the binding for the product.category on which it
+            # depends.
+            else:
+                bind_values = {'backend_id': self.backend_record.id,
+                               'odoo_id': relation.id}
+                if binding_extra_vals:
+                    bind_values.update(binding_extra_vals)
+                # If 2 jobs create it at the same time, retry
+                # one later. A unique constraint (backend_id,
+                # odoo_id) should exist on the binding model
+                with self._retry_unique_violation():
+                    binding = (self.env[binding_model]
+                               .with_context(connector_no_export=True)
+                               .sudo()
+                               .create(bind_values))
+                    # Eager commit to avoid having 2 jobs
+                    # exporting at the same time. The constraint
+                    # will pop if an other job already created
+                    # the same binding. It will be caught and
+                    # raise a RetryableJobError.
+                    if not odoo.tools.config['test_enable']:
+                        self.env.cr.commit()  # noqa
+        else:
+            # If walmart_bind_ids does not exist we are typically in a
+            # "direct" binding (the binding record is the same record).
+            # If wrap is True, relation is already a binding record.
+            binding = relation
+
+        if not rel_binder.to_external(binding):
+            exporter = self.component(usage=component_usage,
+                                      model_name=binding_model)
+            exporter.run(binding)
+
+    def _export_dependencies(self):
+        """ Export the dependencies for the record"""
+        return
+
+    def _map_data(self):
+        """ Returns an instance of
+        :py:class:`~odoo.addons.connector.components.mapper.MapRecord`
+
+        """
+        return self.mapper.map_record(self.binding)
+
+    def _validate_create_data(self, data):
+        """ Check if the values to import are correct
+
+        Pro-actively check before the ``Model.create`` if some fields
+        are missing or invalid
+
+        Raise `InvalidDataError`
+        """
+        return
+
+    def _validate_update_data(self, data):
+        """ Check if the values to import are correct
+
+        Pro-actively check before the ``Model.update`` if some fields
+        are missing or invalid
+
+        Raise `InvalidDataError`
+        """
+        return
+
+    def _create_data(self, map_record, fields=None, **kwargs):
+        """ Get the data to pass to :py:meth:`_create` """
+        return map_record.values(for_create=True, fields=fields, **kwargs)
+
+    def _create(self, data):
+        """ Create the Walmart record """
+        # special check on data before export
+        self._validate_create_data(data)
+        return self.backend_adapter.create(data)
+
+    def _update_data(self, map_record, fields=None, **kwargs):
+        """ Get the data to pass to :py:meth:`_update` """
+        return map_record.values(fields=fields, **kwargs)
+
+    def _update(self, data):
+        """ Update an Walmart record """
+        assert self.external_id
+        # special check on data before export
+        self._validate_update_data(data)
+        self.backend_adapter.write(self.external_id, data)
+
+    def _run(self, fields=None):
+        """ Flow of the synchronization, implemented in inherited classes"""
+        assert self.binding
+
+        if not self.external_id:
+            fields = None  # should be created with all the fields
+
+        if self._has_to_skip():
+            return
+
+        # export the missing linked resources
+        self._export_dependencies()
+
+        # prevent other jobs to export the same record
+        # will be released on commit (or rollback)
+        self._lock()
+
+        map_record = self._map_data()
+
+        if self.external_id:
+            record = self._update_data(map_record, fields=fields)
+            if not record:
+                return _('Nothing to export.')
+            self._update(record)
+        else:
+            record = self._create_data(map_record, fields=fields)
+            if not record:
+                return _('Nothing to export.')
+            self.external_id = self._create(record)
+        return _('Record exported with ID %s on Walmart.') % self.external_id

connector_walmart/components/importer.py

@@ -0,0 +1,324 @@
+# © 2017,2018 Hibou Corp.
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+"""
+
+Importers for Walmart.
+
+An import can be skipped if the last sync date is more recent than
+the last update in Walmart.
+
+They should call the ``bind`` method if the binder even if the records
+are already bound, to update the last sync date.
+
+"""
+
+import logging
+from odoo import fields, _
+from odoo.addons.component.core import AbstractComponent, Component
+from odoo.addons.connector.exception import IDMissingInBackend
+from odoo.addons.queue_job.exception import NothingToDoJob
+
+
+_logger = logging.getLogger(__name__)
+
+
+class WalmartImporter(AbstractComponent):
+    """ Base importer for Walmart """
+
+    _name = 'walmart.importer'
+    _inherit = ['base.importer', 'base.walmart.connector']
+    _usage = 'record.importer'
+
+    def __init__(self, work_context):
+        super(WalmartImporter, self).__init__(work_context)
+        self.external_id = None
+        self.walmart_record = None
+
+    def _get_walmart_data(self):
+        """ Return the raw Walmart data for ``self.external_id`` """
+        return self.backend_adapter.read(self.external_id)
+
+    def _before_import(self):
+        """ Hook called before the import, when we have the Walmart
+        data"""
+
+    def _is_uptodate(self, binding):
+        """Return True if the import should be skipped because
+        it is already up-to-date in Odoo"""
+        assert self.walmart_record
+        if not self.walmart_record.get('updated_at'):
+            return  # no update date on Walmart, always import it.
+        if not binding:
+            return  # it does not exist so it should not be skipped
+        sync = binding.sync_date
+        if not sync:
+            return
+        from_string = fields.Datetime.from_string
+        sync_date = from_string(sync)
+        walmart_date = from_string(self.walmart_record['updated_at'])
+        # if the last synchronization date is greater than the last
+        # update in walmart, we skip the import.
+        # Important: at the beginning of the exporters flows, we have to
+        # check if the walmart_date is more recent than the sync_date
+        # and if so, schedule a new import. If we don't do that, we'll
+        # miss changes done in Walmart
+        return walmart_date < sync_date
+
+    def _import_dependency(self, external_id, binding_model,
+                           importer=None, always=False):
+        """ Import a dependency.
+
+        The importer class is a class or subclass of
+        :class:`WalmartImporter`. A specific class can be defined.
+
+        :param external_id: id of the related binding to import
+        :param binding_model: name of the binding model for the relation
+        :type binding_model: str | unicode
+        :param importer_component: component to use for import
+                                   By default: 'importer'
+        :type importer_component: Component
+        :param always: if True, the record is updated even if it already
+                       exists, note that it is still skipped if it has
+                       not been modified on Walmart since the last
+                       update. When False, it will import it only when
+                       it does not yet exist.
+        :type always: boolean
+        """
+        if not external_id:
+            return
+        binder = self.binder_for(binding_model)
+        if always or not binder.to_internal(external_id):
+            if importer is None:
+                importer = self.component(usage='record.importer',
+                                          model_name=binding_model)
+            try:
+                importer.run(external_id)
+            except NothingToDoJob:
+                _logger.info(
+                    'Dependency import of %s(%s) has been ignored.',
+                    binding_model._name, external_id
+                )
+
+    def _import_dependencies(self):
+        """ Import the dependencies for the record
+
+        Import of dependencies can be done manually or by calling
+        :meth:`_import_dependency` for each dependency.
+        """
+        return
+
+    def _map_data(self):
+        """ Returns an instance of
+        :py:class:`~odoo.addons.connector.components.mapper.MapRecord`
+
+        """
+        return self.mapper.map_record(self.walmart_record)
+
+    def _validate_data(self, data):
+        """ Check if the values to import are correct
+
+        Pro-actively check before the ``_create`` or
+        ``_update`` if some fields are missing or invalid.
+
+        Raise `InvalidDataError`
+        """
+        return
+
+    def _must_skip(self):
+        """ Hook called right after we read the data from the backend.
+
+        If the method returns a message giving a reason for the
+        skipping, the import will be interrupted and the message
+        recorded in the job (if the import is called directly by the
+        job, not by dependencies).
+
+        If it returns None, the import will continue normally.
+
+        :returns: None | str | unicode
+        """
+        return
+
+    def _get_binding(self):
+        return self.binder.to_internal(self.external_id)
+
+    def _create_data(self, map_record, **kwargs):
+        return map_record.values(for_create=True, **kwargs)
+
+    def _create(self, data):
+        """ Create the OpenERP record """
+        # special check on data before import
+        self._validate_data(data)
+        model = self.model.with_context(connector_no_export=True)
+        binding = model.create(data)
+        _logger.debug('%d created from walmart %s', binding, self.external_id)
+        return binding
+
+    def _update_data(self, map_record, **kwargs):
+        return map_record.values(**kwargs)
+
+    def _update(self, binding, data):
+        """ Update an OpenERP record """
+        # special check on data before import
+        self._validate_data(data)
+        binding.with_context(connector_no_export=True).write(data)
+        _logger.debug('%d updated from walmart %s', binding, self.external_id)
+        return
+
+    def _after_import(self, binding):
+        """ Hook called at the end of the import """
+        return
+
+    def run(self, external_id, force=False):
+        """ Run the synchronization
+
+        :param external_id: identifier of the record on Walmart
+        """
+        self.external_id = external_id
+        lock_name = 'import({}, {}, {}, {})'.format(
+            self.backend_record._name,
+            self.backend_record.id,
+            self.work.model_name,
+            external_id,
+        )
+
+        try:
+            self.walmart_record = self._get_walmart_data()
+        except IDMissingInBackend:
+            return _('Record does no longer exist in Walmart')
+
+        skip = self._must_skip()
+        if skip:
+            return skip
+
+        binding = self._get_binding()
+
+        if not force and self._is_uptodate(binding):
+            return _('Already up-to-date.')
+
+        # Keep a lock on this import until the transaction is committed
+        # The lock is kept since we have detected that the informations
+        # will be updated into Odoo
+        self.advisory_lock_or_retry(lock_name)
+        self._before_import()
+
+        # import the missing linked resources
+        self._import_dependencies()
+
+        map_record = self._map_data()
+
+        if binding:
+            record = self._update_data(map_record)
+            self._update(binding, record)
+        else:
+            record = self._create_data(map_record)
+            binding = self._create(record)
+
+        self.binder.bind(self.external_id, binding)
+
+        self._after_import(binding)
+
+
+class BatchImporter(AbstractComponent):
+    """ The role of a BatchImporter is to search for a list of
+    items to import, then it can either import them directly or delay
+    the import of each item separately.
+    """
+
+    _name = 'walmart.batch.importer'
+    _inherit = ['base.importer', 'base.walmart.connector']
+    _usage = 'batch.importer'
+
+    def run(self, filters=None):
+        """ Run the synchronization """
+        record_ids = self.backend_adapter.search(filters)
+        for record_id in record_ids:
+            self._import_record(record_id)
+
+    def _import_record(self, external_id):
+        """ Import a record directly or delay the import of the record.
+
+        Method to implement in sub-classes.
+        """
+        raise NotImplementedError
+
+
+class DirectBatchImporter(AbstractComponent):
+    """ Import the records directly, without delaying the jobs. """
+
+    _name = 'walmart.direct.batch.importer'
+    _inherit = 'walmart.batch.importer'
+
+    def _import_record(self, external_id):
+        """ Import the record directly """
+        self.model.import_record(self.backend_record, external_id)
+
+
+class DelayedBatchImporter(AbstractComponent):
+    """ Delay import of the records """
+
+    _name = 'walmart.delayed.batch.importer'
+    _inherit = 'walmart.batch.importer'
+
+    def _import_record(self, external_id, job_options=None, **kwargs):
+        """ Delay the import of the records"""
+        delayable = self.model.with_delay(**job_options or {})
+        delayable.import_record(self.backend_record, external_id, **kwargs)
+
+
+# class SimpleRecordImporter(Component):
+#     """ Import one Walmart Website """
+#
+#     _name = 'walmart.simple.record.importer'
+#     _inherit = 'walmart.importer'
+#     _apply_on = [
+#         'walmart.res.partner.category',
+#     ]
+
+
+# class TranslationImporter(Component):
+#     """ Import translations for a record.
+#
+#     Usually called from importers, in ``_after_import``.
+#     For instance from the products and products' categories importers.
+#     """
+#
+#     _name = 'walmart.translation.importer'
+#     _inherit = 'walmart.importer'
+#     _usage = 'translation.importer'
+#
+#     def _get_walmart_data(self, storeview_id=None):
+#         """ Return the raw Walmart data for ``self.external_id`` """
+#         return self.backend_adapter.read(self.external_id, storeview_id)
+#
+#     def run(self, external_id, binding, mapper=None):
+#         self.external_id = external_id
+#         storeviews = self.env['walmart.storeview'].search(
+#             [('backend_id', '=', self.backend_record.id)]
+#         )
+#         default_lang = self.backend_record.default_lang_id
+#         lang_storeviews = [sv for sv in storeviews
+#                            if sv.lang_id and sv.lang_id != default_lang]
+#         if not lang_storeviews:
+#             return
+#
+#         # find the translatable fields of the model
+#         fields = self.model.fields_get()
+#         translatable_fields = [field for field, attrs in fields.items()
+#                                if attrs.get('translate')]
+#
+#         if mapper is None:
+#             mapper = self.mapper
+#         else:
+#             mapper = self.component_by_name(mapper)
+#
+#         for storeview in lang_storeviews:
+#             lang_record = self._get_walmart_data(storeview.external_id)
+#             map_record = mapper.map_record(lang_record)
+#             record = map_record.values()
+#
+#             data = dict((field, value) for field, value in record.items()
+#                         if field in translatable_fields)
+#
+#             binding.with_context(connector_no_export=True,
+#                                  lang=storeview.lang_id.code).write(data)

connector_walmart/components/mapper.py

@@ -0,0 +1,16 @@
+# © 2017,2018 Hibou Corp.
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+from odoo.addons.component.core import AbstractComponent
+
+
+class WalmartImportMapper(AbstractComponent):
+    _name = 'walmart.import.mapper'
+    _inherit = ['base.walmart.connector', 'base.import.mapper']
+    _usage = 'import.mapper'
+
+
+class WalmartExportMapper(AbstractComponent):
+    _name = 'walmart.export.mapper'
+    _inherit = ['base.walmart.connector', 'base.export.mapper']
+    _usage = 'export.mapper'