[10.0] Better parse camt groupped transactions (#131)

* correctly parse entries with multiple transactions

Our bank does it.  Care has been taken not to break cases where
Ntry/NtryDtls/TxDtls isn't used: if any piece of information isn't found
on these nodes, the matching information from the Ntry is used.  Most
often the entry has a sum or a summary and the transaction details are
more precise and specific, so it makes sense to use their contents
rather than its.

* more complete unit tests for camt parser

This checks the whole output data of the parser against its expected
output.

* add a test case that checks TxDtls decoding

* special case for Ntrys without TxDtls

* CO-12 - Date Import BVR

* FIX tests

* CO-1232 Fixed issues with CAMT parser tests

account_bank_statement_import_camt/models/parser.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 """Class to parse camt files."""
 # © 2013-2016 Therp BV <http://therp.nl>
+# Copyright 2017 Open Net Sàrl
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
 import re
 from lxml import etree
@@ -46,7 +47,7 @@ class CamtParser(models.AbstractModel):
                 break
 
     def parse_transaction_details(self, ns, node, transaction):
-        """Parse transaction details (message, party, account...)."""
+        """Parse TxDtls node."""
         # message
         self.add_value_from_node(
             ns, node, [
@@ -64,9 +65,13 @@ class CamtParser(models.AbstractModel):
             ns, node, [
                 './ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref',
                 './ns:Refs/ns:EndToEndId',
+                './ns:Ntry/ns:AcctSvcrRef'
             ],
             transaction, 'ref'
         )
+        amount = self.parse_amount(ns, node)
+        if amount != 0.0:
+            transaction['amount'] = amount
         # remote party values
         party_type = 'Dbtr'
         party_type_node = node.xpath(
@@ -107,10 +112,11 @@ class CamtParser(models.AbstractModel):
                     ns, account_node[0], './ns:Othr/ns:Id', transaction,
                     'account_number'
                 )
+        transaction['data'] = etree.tostring(node)
 
-    def parse_transaction(self, ns, node):
-        """Parse transaction (entry) node."""
-        transaction = {}
+    def parse_entry(self, ns, node):
+        """Parse an Ntry node and yield transactions"""
+        transaction = {'name': '/', 'amount': 0}  # fallback defaults
         self.add_value_from_node(
             ns, node, './ns:BkTxCd/ns:Prtry/ns:Cd', transaction,
             'transfer_type'
@@ -121,27 +127,30 @@ class CamtParser(models.AbstractModel):
             ns, node, './ns:BookgDt/ns:Dt', transaction, 'execution_date')
         self.add_value_from_node(
             ns, node, './ns:ValDt/ns:Dt', transaction, 'value_date')
+        amount = self.parse_amount(ns, node)
+        if amount != 0.0:
+            transaction['amount'] = amount
+        self.add_value_from_node(
+            ns, node, './ns:AddtlNtryInf', transaction, 'name')
+        self.add_value_from_node(
+            ns, node, [
+                './ns:NtryDtls/ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref',
+                './ns:NtryDtls/ns:Btch/ns:PmtInfId',
+                './ns:NtryDtls/ns:TxDtls/ns:Refs/ns:AcctSvcrRef'
+            ],
+            transaction, 'ref'
+        )
 
-        transaction['amount'] = self.parse_amount(ns, node)
-
-        details_node = node.xpath(
+        details_nodes = node.xpath(
             './ns:NtryDtls/ns:TxDtls', namespaces={'ns': ns})
-        if details_node:
-            self.parse_transaction_details(ns, details_node[0], transaction)
-        if not transaction.get('name'):
-            self.add_value_from_node(
-                ns, node, './ns:AddtlNtryInf', transaction, 'name')
-        if not transaction.get('name'):
-            transaction['name'] = '/'
-        if not transaction.get('ref'):
-            self.add_value_from_node(
-                ns, node, [
-                    './ns:NtryDtls/ns:Btch/ns:PmtInfId',
-                ],
-                transaction, 'ref'
-            )
-        transaction['data'] = etree.tostring(node)
-        return transaction
+        if len(details_nodes) == 0:
+            yield transaction
+            return
+        transaction_base = transaction
+        for node in details_nodes:
+            transaction = transaction_base.copy()
+            self.parse_transaction_details(ns, node, transaction)
+            yield transaction
 
     def get_balance_amounts(self, ns, node):
         """Return opening and closing balance.
@@ -193,12 +202,15 @@ class CamtParser(models.AbstractModel):
             ns, node, './ns:Acct/ns:Ccy', result, 'currency')
         result['balance_start'], result['balance_end_real'] = (
             self.get_balance_amounts(ns, node))
-        transaction_nodes = node.xpath('./ns:Ntry', namespaces={'ns': ns})
-        result['transactions'] = []
-        for entry_node in transaction_nodes:
-            transaction = self.parse_transaction(ns, entry_node)
-            if transaction:
-                result['transactions'].append(transaction)
+        entry_nodes = node.xpath('./ns:Ntry', namespaces={'ns': ns})
+        transactions = []
+        for entry_node in entry_nodes:
+            transactions.extend(self.parse_entry(ns, entry_node))
+        result['transactions'] = transactions
+        result['date'] = sorted(transactions,
+                                key=lambda x: x['date'],
+                                reverse=True
+                                )[0]['date']
         return result
 
     def check_version(self, ns, root):