base_import_match: Add conditionals to fields when importing.

This patch allows to import res.partner records by VAT when is_company==True by default.

base_import_match/README.rst

@@ -9,8 +9,8 @@ Base Import Match
 By default, when importing data (like CSV import) with the ``base_import``
 module, Odoo follows this rule:
 
-#. If you import the XMLID of a record, make an **update**.
+- If you import the XMLID of a record, make an **update**.
-#. If you do not, **create** a new record.
+- If you do not, **create** a new record.
 
 This module allows you to set additional rules to match if a given import is an
 update or a new record.
@@ -21,21 +21,31 @@ name, VAT, email, etc.
 
 After installing this module, the import logic will be changed to:
 
-#. If you import the XMLID of a record, make an **update**.
+- If you import the XMLID of a record, make an **update**.
-#. If you do not:
+- If you do not:
-   #. If there are import match rules for the model you are importing:
+  - If there are import match rules for the model you are importing:
-       #. Discard the rules that require fields you are not importing.
+      - Discard the rules that require fields you are not importing.
-       #. Traverse the remaining rules one by one in order to find a match in
+      - Traverse the remaining rules one by one in order to find a match in
-          the database.
+        the database.
-          #. If one match is found:
+          - Skip the rule if it requires a special condition that is not
-             #. Stop traversing the rest of valid rules.
+            satisfied.
-             #. **Update** that record.
+          - If one match is found:
-          #. If zero or multiple matches are found:
+            - Stop traversing the rest of valid rules.
-             #. Continue with the next rule.
+            - **Update** that record.
-          #. If all rules are exhausted and no single match is found:
+          - If zero or multiple matches are found:
-             #. **Create** a new record.
+            - Continue with the next rule.
-   #. If there are no match rules for your model:
+          - If all rules are exhausted and no single match is found:
-      #. **Create** a new record.
+            - **Create** a new record.
+  - If there are no match rules for your model:
+    - **Create** a new record.
+
+By default 2 rules are installed for production instances:
+
+- One rule that will allow you to update companies based on their VAT, when
+  ``is_company`` is ``True``.
+- One rule that will allow you to update users based on their login.
+
+In demo instances there are more examples.
 
 Configuration
 =============
@@ -46,6 +56,12 @@ To configure this module, you need to:
 #. *Create*.
 #. Choose a *Model*.
 #. Choose the *Fields* that conform an unique key in that model.
+#. If the rule must be used only for certain imported values, check
+   *Conditional* and enter the **exact string** that is going to be imported
+   in *Imported value*.
+   #. Keep in mind that the match here is evaluated as a case sensitive
+      **text string** always. If you enter e.g. ``True``, it will match that
+      string, but will not match ``1`` or ``true``.
 #. *Save*.
 
 In that list view, you can sort rules by drag and drop.
@@ -63,15 +79,11 @@ To use this module, you need to:
    :alt: Try me on Runbot
    :target: https://runbot.odoo-community.org/runbot/149/8.0
 
-Roadmap / Known Issues
+Known Issues / Roadmap
 ======================
 
-* Add a filter to let you apply some rules only to incoming imports that match
+* Add a setting to throw an error when multiple matches are found, instead of
-  a given criteria (like a domain, but for import data).
+  falling back to creation of new record.
-* Matching by VAT for ``res.partner`` records will only work when the partner
-  has no contacts, because otherwise Odoo reflects the parent company's VAT in
-  the contact, and that results in multiple matches. Fixing the above point
-  should make this work.
 
 Bug Tracker
 ===========

base_import_match/__openerp__.py

@@ -16,6 +16,8 @@
         "base_import",
     ],
     "data": [
+        "security/ir.model.access.csv",
+        "data/base_import_match.yml",
         "views/base_import_match_view.xml",
     ],
     "demo": [

base_import_match/data/base_import_match.yml

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# © 2016 Grupo ESOC Ingeniería de Servicios, S.L.U. - Jairo Llopis
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+- !context {noupdate: True}
+
+# Match partners by VAT when is_company is True
+- !record {id: res_partner_vat, model: base_import.match}:
+    model_id: base.model_res_partner
+    sequence: 10
+
+- !record {id: res_partner_vat_vat, model: base_import.match.field}:
+    match_id: res_partner_vat
+    field_id: base.field_res_partner_vat
+
+- !record {id: res_partner_vat_is_company, model: base_import.match.field}:
+    match_id: res_partner_vat
+    field_id: base.field_res_partner_is_company
+    conditional: True
+    imported_value: "True"
+
+# Match users by login
+- !record {id: res_users_login, model: base_import.match}:
+    model_id: base.model_res_users
+    sequence: 50
+
+- !record {id: res_users_login_login, model: base_import.match.field}:
+    match_id: res_users_login
+    field_id: base.field_res_users_login

base_import_match/demo/base_import_match.yml

@@ -4,34 +4,37 @@
 
 - !context {noupdate: True}
 
-- !record {id: res_partner_vat, model: base_import.match}:
+# Match partners by name, parent_id and is_company
-    model_id: base.model_res_partner
-    sequence: 10
-    field_ids:
-        - base.field_res_partner_vat
-
 - !record {id: res_partner_parent_name_is_company, model: base_import.match}:
     model_id: base.model_res_partner
     sequence: 20
-    field_ids:
-        - base.field_res_partner_name
-        - base.field_res_partner_parent_id
-        - base.field_res_partner_is_company
 
+- !record {id: res_partner_parent_name_is_company_name, model: base_import.match.field}:
+    match_id: res_partner_parent_name_is_company
+    field_id: base.field_res_partner_name
+
+- !record {id: res_partner_parent_name_is_company_parent, model: base_import.match.field}:
+    match_id: res_partner_parent_name_is_company
+    field_id: base.field_res_partner_parent_id
+
+- !record {id: res_partner_parent_name_is_company_is_company, model: base_import.match.field}:
+    match_id: res_partner_parent_name_is_company
+    field_id: base.field_res_partner_is_company
+
+# Match partner by email
 - !record {id: res_partner_email, model: base_import.match}:
     model_id: base.model_res_partner
     sequence: 30
-    field_ids:
-        - base.field_res_partner_email
 
+- !record {id: res_partner_email_email, model: base_import.match.field}:
+    match_id: res_partner_email
+    field_id: base.field_res_partner_email
+
+# Match partner by name
 - !record {id: res_partner_name, model: base_import.match}:
     model_id: base.model_res_partner
     sequence: 40
-    field_ids:
-        - base.field_res_partner_name
 
-- !record {id: res_users_login, model: base_import.match}:
+- !record {id: res_partner_name_name, model: base_import.match.field}:
-    model_id: base.model_res_users
+    match_id: res_partner_name
-    sequence: 50
+    field_id: base.field_res_partner_name
-    field_ids:
-        - base.field_res_users_login

base_import_match/models/base_import.py

@@ -3,7 +3,6 @@
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
 
 from openerp import api, fields, models
-from openerp.exceptions import except_orm as ValueError  # TODO remove in v9
 from openerp import SUPERUSER_ID  # TODO remove in v10
 
 
@@ -11,9 +10,6 @@ class BaseImportMatch(models.Model):
     _name = "base_import.match"
     _description = "Deduplicate settings prior to CSV imports."
     _order = "sequence, name"
-    _sql_constraints = [
-        ("name_unique", "UNIQUE(name)", "Duplicated match!"),
-    ]
 
     name = fields.Char(
         compute="_compute_name",
@@ -31,17 +27,17 @@ class BaseImportMatch(models.Model):
         related="model_id.model",
         store=True,
         index=True)
-    field_ids = fields.Many2many(
+    field_ids = fields.One2many(
-        "ir.model.fields",
+        comodel_name="base_import.match.field",
+        inverse_name="match_id",
         string="Fields",
         required=True,
-        domain="[('model_id', '=', model_id)]",
         help="Fields that will define an unique key.")
 
     @api.multi
     @api.onchange("model_id")
     def _onchange_model_id(self):
-        self.field_ids = False
+        self.field_ids.unlink()
 
     @api.model
     def create(self, vals):
@@ -89,7 +85,70 @@ class BaseImportMatch(models.Model):
         for s in self:
             s.name = "{}: {}".format(
                 s.model_id.display_name,
-                " + ".join(s.field_ids.mapped("display_name")))
+                " + ".join(
+                    s.field_ids.mapped(
+                        lambda r: (
+                            str(r.field_id.name) +
+                            (" ({})".format(r.imported_value)
+                             if r.conditional else "")))))
+
+    @api.model
+    def _match_find(self, model, converted_row, imported_row):
+        """Find a update target for the given row.
+
+        This will traverse by order all match rules that can be used with the
+        imported data, and return a match for the first rule that returns a
+        single result.
+
+        :param openerp.models.Model model:
+            Model object that is being imported.
+
+        :param dict converted_row:
+            Row converted to Odoo api format, like the 3rd value that
+            :meth:`openerp.models.Model._convert_records` returns.
+
+        :param dict imported_row:
+            Row as it is being imported, in format::
+
+                {
+                    "field_name": "string value",
+                    "other_field": "True",
+                    ...
+                }
+
+        :return openerp.models.Model:
+            Return a dataset with one single match if it was found, or an
+            empty dataset if none or multiple matches were found.
+        """
+        # Get usable rules to perform matches
+        usable = self._usable_for_load(model._name, converted_row.keys())
+
+        # Traverse usable combinations
+        for combination in usable:
+            combination_valid = True
+            domain = list()
+
+            for field in combination.field_ids:
+                # Check imported value if it is a conditional field
+                if field.conditional:
+                    # Invalid combinations are skipped
+                    if imported_row[field.name] != field.imported_value:
+                        combination_valid = False
+                        break
+
+                domain.append((field.name, "=", converted_row[field.name]))
+
+            if not combination_valid:
+                continue
+
+            match = model.search(domain)
+
+            # When a single match is found, stop searching
+            if len(match) == 1:
+                return match
+
+        # Return an empty match if none or multiple was found
+        return model
 
     @api.model
     def _load_wrapper(self):
@@ -105,53 +164,37 @@ class BaseImportMatch(models.Model):
             """
             newdata = list()
 
+            # Data conversion to ORM format
+            import_fields = map(models.fix_import_export_id_paths, fields)
+            converted_data = self._convert_records(
+                self._extract_records(import_fields, data))
+
             # Mock Odoo to believe the user is importing the ID field
             if "id" not in fields:
                 fields.append("id")
+                import_fields.append(["id"])
 
-            # Needed to work with relational fields
+            # Needed to match with converted data field names
-            clean_fields = [
+            clean_fields = [f[0] for f in import_fields]
-                models.fix_import_export_id_paths(f)[0] for f in fields]
 
-            # Get usable rules to perform matches
+            for dbid, xmlid, record, info in converted_data:
-            usable = self.env["base_import.match"]._usable_for_load(
+                row = dict(zip(clean_fields, data[info["record"]]))
-                self._name, clean_fields)
-
-            for row in (dict(zip(clean_fields, r)) for r in data):
-                # All rows need an ID
-                if "id" not in row:
-                    row["id"] = u""
-
-                # Skip rows with ID, they do not need all this
-                elif row["id"]:
-                    continue
-
-                # Store records that match a combination
                 match = self
-                for combination in usable:
-                    match |= self.search(
-                        [(field.name, "=", row[field.name])
-                         for field in combination.field_ids])
 
-                    # When a single match is found, stop searching
+                if xmlid:
-                    if len(match) != 1:
+                    # Skip rows with ID, they do not need all this
-                        break
+                    row["id"] = xmlid
+                elif dbid:
+                    # Find the xmlid for this dbid
+                    match = self.browse(dbid)
+                else:
+                    # Store records that match a combination
+                    match = self.env["base_import.match"]._match_find(
+                        self, record, row)
 
-                # Only one record should have been found
+                # Give a valid XMLID to this row if a match was found
-                try:
+                row["id"] = (match._BaseModel__export_xml_id()
-                    match.ensure_one()
+                             if match else row.get("id", u""))
-
-                # You hit this because...
-                # a. No match. Odoo must create a new record.
-                # b. Multiple matches. No way to know which is the right
-                #    one, so we let Odoo create a new record or raise
-                #    the corresponding exception.
-                # In any case, we must do nothing.
-                except ValueError:
-                    continue
-
-                # Give a valid XMLID to this row
-                row["id"] = match._BaseModel__export_xml_id()
 
                 # Store the modified row, in the same order as fields
                 newdata.append(tuple(row[f] for f in clean_fields))
@@ -218,3 +261,37 @@ class BaseImportMatch(models.Model):
                 result += record
 
         return result
+
+
+class BaseImportMatchField(models.Model):
+    _name = "base_import.match.field"
+    _description = "Field import match definition"
+
+    name = fields.Char(
+        related="field_id.name")
+    field_id = fields.Many2one(
+        comodel_name="ir.model.fields",
+        string="Field",
+        required=True,
+        ondelete="cascade",
+        domain="[('model_id', '=', model_id)]",
+        help="Field that will be part of an unique key.")
+    match_id = fields.Many2one(
+        comodel_name="base_import.match",
+        string="Match",
+        required=True)
+    model_id = fields.Many2one(
+        related="match_id.model_id")
+    conditional = fields.Boolean(
+        help="Enable if you want to use this field only in some conditions.")
+    imported_value = fields.Char(
+        help="If the imported value is not this, the whole matching rule will "
+             "be discarded. Be careful, this data is always treated as a "
+             "string, and comparison is case-sensitive so if you set 'True', "
+             "it will NOT match '1' nor 'true', only EXACTLY 'True'.")
+
+    @api.multi
+    @api.onchange("field_id", "match_id", "conditional", "imported_value")
+    def _onchange_match_id_name(self):
+        """Update match name."""
+        self.mapped("match_id")._compute_name()

base_import_match/security/ir.model.access.csv

@@ -0,0 +1,5 @@
+id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink
+access_base_import_match,Access base_import.match,model_base_import_match,base.group_user,1,0,0,0
+access_base_import_match_field,Access base_import.match.field,model_base_import_match_field,base.group_user,1,0,0,0
+write_base_import_match,Write base_import.match,model_base_import_match,base.group_system,1,1,1,1
+write_base_import_match_field,Write base_import.match.field,model_base_import_match_field,base.group_system,1,1,1,1

base_import_match/tests/import_data/res_partner_vat.csv

@@ -1,2 +1,2 @@
-name,vat
+name,vat,is_company
-Federal Changed,BE0477472701
+Agrolait Changed,BE0477472701,True

base_import_match/tests/test_import.py

@@ -27,12 +27,12 @@ class ImportCase(TransactionCase):
 
     def test_res_partner_vat(self):
         """Change name based on VAT."""
-        federal = self.env.ref("base.res_partner_26")
+        agrolait = self.env.ref("base.res_partner_2")
-        federal.vat = "BE0477472701"
+        agrolait.vat = "BE0477472701"
         record = self._base_import_record("res.partner", "res_partner_vat")
-        record.do(["name", "vat"], OPTIONS)
+        record.do(["name", "vat", "is_company"], OPTIONS)
-        federal.env.invalidate_all()
+        agrolait.env.invalidate_all()
-        self.assertEqual(federal.name, "Federal Changed")
+        self.assertEqual(agrolait.name, "Agrolait Changed")
 
     def test_res_partner_parent_name_is_company(self):
         """Change email based on parent_id, name and is_company."""

base_import_match/views/base_import_match_view.xml

@@ -16,12 +16,22 @@
                 </h1>
                 <group>
                     <field name="model_id"/>
-                    <field
+                    <field name="field_ids">
-                        name="field_ids"
+                        <tree  editable="bottom">
-                        widget="many2many_tags"
+                            <field name="field_id"
-                        options="{
+                                   options="{'no_create': True}"/>
-                            'no_create': True,
+                            <field name="match_id" invisible="True"/>
-                        }"/>
+                            <field name="model_id" invisible="True"/>
+                            <field name="conditional"/>
+                            <field
+                                name="imported_value"
+                                attrs="{
+                                    'readonly': [
+                                        ('conditional', '=', False),
+                                    ],
+                                }"/>
+                        </tree>
+                    </field>
                     <field name="sequence"/>
                 </group>
             </sheet>