From aa0696e3fdedb3dbd6259e287576548951836733 Mon Sep 17 00:00:00 2001
From: Stefan Rijnhart <stefan@therp.nl>
Date: Sat, 14 Sep 2013 12:10:48 +0200
Subject: [PATCH 01/14] [FIX] Error in object attribute

---
 account_banking/parsers/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/account_banking/parsers/models.py b/account_banking/parsers/models.py
index a8a0c3014..8f8effb6d 100644
--- a/account_banking/parsers/models.py
+++ b/account_banking/parsers/models.py
@@ -34,7 +34,7 @@ class mem_bank_statement(object):
     # Lock attributes to enable parsers to trigger non-conformity faults
     __slots__ = [
         'start_balance','end_balance', 'date', 'local_account',
-        'local_currency', 'id', 'statements'
+        'local_currency', 'id', 'transactions'
     ]
     def __init__(self, *args, **kwargs):
         super(mem_bank_statement, self).__init__(*args, **kwargs)

From 9e10ad4c79ce058e6bc0e56b7d58660e80e52843 Mon Sep 17 00:00:00 2001
From: Stefan Rijnhart <stefan@therp.nl>
Date: Sat, 14 Sep 2013 12:11:13 +0200
Subject: [PATCH 02/14] [ADD] Parse CAMT053 structure

---
 account_banking_camt/__init__.py    |   1 +
 account_banking_camt/__openerp__.py |  32 ++++++
 account_banking_camt/camt.py        | 153 ++++++++++++++++++++++++++++
 3 files changed, 186 insertions(+)
 create mode 100644 account_banking_camt/__init__.py
 create mode 100644 account_banking_camt/__openerp__.py
 create mode 100644 account_banking_camt/camt.py

diff --git a/account_banking_camt/__init__.py b/account_banking_camt/__init__.py
new file mode 100644
index 000000000..f8b1d5b3a
--- /dev/null
+++ b/account_banking_camt/__init__.py
@@ -0,0 +1 @@
+import camt
diff --git a/account_banking_camt/__openerp__.py b/account_banking_camt/__openerp__.py
new file mode 100644
index 000000000..6c0d332b0
--- /dev/null
+++ b/account_banking_camt/__openerp__.py
@@ -0,0 +1,32 @@
+##############################################################################
+#
+#    Copyright (C) 2013 Therp BV (<http://therp.nl>)
+#    All Rights Reserved
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+{
+    'name': 'CAMT Format Bank Statements Import',
+    'version': '0.1',
+    'license': 'AGPL-3',
+    'author': 'Therp BV',
+    'website': 'https://launchpad.net/account-banking',
+    'category': 'Account Banking',
+    'depends': ['account_banking'],
+    'description': '''
+Module to import SEPA CAMT Format bank statement files
+    ''',
+    'installable': True,
+}
diff --git a/account_banking_camt/camt.py b/account_banking_camt/camt.py
new file mode 100644
index 000000000..fe31279e2
--- /dev/null
+++ b/account_banking_camt/camt.py
@@ -0,0 +1,153 @@
+from lxml import etree
+from datetime import datetime
+from account_banking.parsers import models
+from account_banking.parsers.convert import str2date
+from account_banking.sepa import postalcode
+from tools.translate import _
+
+bt = models.mem_bank_transaction
+
+class transaction(models.mem_bank_transaction):
+
+    def __init__(self, values, *args, **kwargs):
+        super(transaction, self).__init__(*args, **kwargs)
+        for attr in self.attrnames:
+            if attr in values:
+                setattr(self, attr, values['attr'])
+
+class parser(models.parser):
+    code = 'CAMT'
+    country_code = 'NL'
+    name = _('Generic CAMT Format')
+    doc = _('''\
+CAMT Format parser
+''')
+
+    def tag(self, node):
+        """
+        Return the tag of a node, stripped from its namespace
+        """
+        return node.tag[len(self.ns):]
+
+    def assert_tag(self, node, expected):
+        """
+        Get node's stripped tag and compare with expected
+        """
+        assert self.tag(node) == expected, (
+            _("Expected tag '%s', got '%s' instead") %
+            (self.tag(node), expected))
+        
+    def xpath(self, node, expr):
+        """
+        Wrap namespaces argument into call to Element.xpath():
+
+        self.xpath(node, './ns:Acct/ns:Id')
+        """
+        return node.xpath(expr, namespaces={'ns': self.ns[1:-1]})
+
+    def get_balance_type_node(self, node, balance_type):
+        """
+        :param node: BkToCstmrStmt/Stmt/Bal node
+        :param balance type: one of 'OPBD', 'PRCD', 'ITBD', 'CLBD'
+        """
+        code_expr = './ns:Bal/ns:Tp/ns:CdOrPrtry/ns:Cd[text()="%s"]/../../..' % balance_type
+        return self.xpath(node, code_expr)
+    
+    def parse_amount(self, node):
+        """
+        Parse an element that contains both Amount and CreditDebitIndicator
+        
+        :return: signed amount
+        :returntype: float
+        """
+        sign = -1 if node.find(self.ns + 'CdtDbtInd').text == 'CRDT' else 1
+        return sign * float(node.find(self.ns + 'Amt').text)
+        
+    def get_start_balance(self, node):
+        """
+        Find the (only) balance node with code OpeningBalance, or
+        the only one with code 'PreviousClosingBalance'
+        or the first balance node with code InterimBalance in
+        the case of preceeding pagination.
+
+        :param node: BkToCstmrStmt/Stmt/Bal node
+        """
+        nodes = (
+            self.get_balance_type_node(node, 'OPBD') or
+            self.get_balance_type_node(node, 'PRCD') or
+            self.get_balance_type_node(node, 'ITBD'))
+        return self.parse_amount(nodes[0])
+
+    def get_end_balance(self, node):
+        """
+        Find the (only) balance node with code ClosingBalance, or
+        the second (and last) balance node with code InterimBalance in
+        the case of continued pagination.
+
+        :param node: BkToCstmrStmt/Stmt/Bal node
+        """
+        nodes = (
+            self.get_balance_type_node(node, 'CLBD') or
+            self.get_balance_type_node(node, 'ITBD'))
+        return self.parse_amount(nodes[-1])
+
+    def parse_Stmt(self, node):
+        statement = models.mem_bank_statement()
+        statement.id = node.find(self.ns + 'Id').text
+        statement.local_account = (
+            self.xpath(node, './ns:Acct/ns:Id/ns:IBAN')[0].text
+            if self.xpath(node, './ns:Acct/ns:Id/ns:IBAN')
+            else self.xpath(node, './ns:Acct/ns:Id/ns:Othr/ns:Id')[0].text)
+        statement.local_currency = self.xpath(node, './ns:Acct/ns:Ccy')[0].text
+        statement.start_balance = self.get_start_balance(node)
+        statement.end_balance = self.get_end_balance(node)
+        print "Number of Ntry in statement: %s" % len(self.xpath(node, '.ns:Ntry'))
+        for Ntry in self.xpath(node, '.ns:Ntry'):
+            for transaction_detail in self.parse_Ntry(Ntry):
+                statement.transactions.append(
+                    transaction(transaction_detail))
+        return statement
+
+    def get_entry_description(self, node):
+        """
+        :param node: Ntry node
+        """
+        codes = self.xpath(node, './ns:BxTxCd/ns:Prtry/ns:Cd')
+        if codes:
+            return codes[0].text
+        return False
+
+    def parse_Ntry(self, node):
+        entry_description = self.get_entry_description(node)
+        entry_details = {
+            'effective_date': self.xpath(node, './ns:BookgDt/ns:Dt')[0].text,
+            'transaction_date': self.xpath(node, './ns:ValDt/ns:Dt')[0].text,
+            'transfer_type': bt.ORDER,
+            'transferred_amount': self.parse_amount(node)
+            }
+        amount_sign = -1 if node.find(self.ns + 'CdtDbtInd').text == 'CRDT' else 1
+        transaction_details = []
+        print "  NUmber of NtryDtls in Ntry with code %s: %s" % (
+            entry_description, len(self.xpath(node, './ns:NtryDtls')))
+        for NtryDtl in self.xpath(node, './ns:NtryDtls'):
+            # Todo: process Btch tag on entry-detail level
+            print "    NUmber of TxDtls in NtryDtl: %s" % len(self.xpath(node, './ns:TxDtls'))
+            continue
+            for TxDtl in self.xpath(NtryDtl, './ns:TxDtls'):
+                transaction_details.append(
+                    self.parse_TxDtl(TxDtl, entry_details, amount_sign))
+        return transaction_details
+
+    def parse_TxDtl(self, node, entry_values, amount_sign):
+        transaction_values = dict(entry_values)
+        amount = amount_sign * float(node.find(self.ns + 'Amt').text)
+
+
+    def parse(self, cr, data):
+        root = etree.fromstring(data)
+        self.ns = root.tag[:root.tag.index("}") + 1]
+        self.assert_tag(root[0][0], 'GrpHdr')
+        statements = []
+        for node in root[0][1:]:
+            statements.append(self.parse_Stmt(node))
+        return statements

From ac003bd6d87be65bb6753b81905282963bea7d4f Mon Sep 17 00:00:00 2001
From: Stefan Rijnhart <stefan@therp.nl>
Date: Sat, 14 Sep 2013 12:32:59 +0200
Subject: [PATCH 03/14] [FIX] Only get TxDtls if there is only one

---
 account_banking_camt/camt.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/account_banking_camt/camt.py b/account_banking_camt/camt.py
index fe31279e2..0bc94fb11 100644
--- a/account_banking_camt/camt.py
+++ b/account_banking_camt/camt.py
@@ -101,8 +101,8 @@ CAMT Format parser
         statement.local_currency = self.xpath(node, './ns:Acct/ns:Ccy')[0].text
         statement.start_balance = self.get_start_balance(node)
         statement.end_balance = self.get_end_balance(node)
-        print "Number of Ntry in statement: %s" % len(self.xpath(node, '.ns:Ntry'))
-        for Ntry in self.xpath(node, '.ns:Ntry'):
+        print "Number of Ntry in statement: %s" % len(self.xpath(node, './ns:Ntry'))
+        for Ntry in self.xpath(node, './ns:Ntry'):
             for transaction_detail in self.parse_Ntry(Ntry):
                 statement.transactions.append(
                     transaction(transaction_detail))
@@ -112,7 +112,7 @@ CAMT Format parser
         """
         :param node: Ntry node
         """
-        codes = self.xpath(node, './ns:BxTxCd/ns:Prtry/ns:Cd')
+        codes = self.xpath(node, './ns:BkTxCd/ns:Prtry/ns:Cd')
         if codes:
             return codes[0].text
         return False
@@ -130,12 +130,16 @@ CAMT Format parser
         print "  NUmber of NtryDtls in Ntry with code %s: %s" % (
             entry_description, len(self.xpath(node, './ns:NtryDtls')))
         for NtryDtl in self.xpath(node, './ns:NtryDtls'):
+            TxDtls = self.xpath(NtryDtl, './ns:TxDtls')
             # Todo: process Btch tag on entry-detail level
-            print "    NUmber of TxDtls in NtryDtl: %s" % len(self.xpath(node, './ns:TxDtls'))
+            print "    NUmber of TxDtls in NtryDtl: %s" % len(TxDtls)
             continue
-            for TxDtl in self.xpath(NtryDtl, './ns:TxDtls'):
+            if len(TxDtls) == 1:
                 transaction_details.append(
-                    self.parse_TxDtl(TxDtl, entry_details, amount_sign))
+                    self.parse_TxDtl(TxDtls[0], entry_details, amount_sign))
+            else:
+                transaction_details.append(
+                   transaction(entry_details))
         return transaction_details
 
     def parse_TxDtl(self, node, entry_values, amount_sign):

From 21f2e288e371248e2410a9dbef95e95ae2e8281f Mon Sep 17 00:00:00 2001
From: Stefan Rijnhart <stefan@therp.nl>
Date: Wed, 18 Sep 2013 16:28:03 +0200
Subject: [PATCH 04/14] [IMP] Various parsing

---
 account_banking_camt/camt.py | 76 ++++++++++++++++++++++++++++++------
 1 file changed, 65 insertions(+), 11 deletions(-)

diff --git a/account_banking_camt/camt.py b/account_banking_camt/camt.py
index 0bc94fb11..783c71cd7 100644
--- a/account_banking_camt/camt.py
+++ b/account_banking_camt/camt.py
@@ -1,5 +1,6 @@
 from lxml import etree
 from datetime import datetime
+import re
 from account_banking.parsers import models
 from account_banking.parsers.convert import str2date
 from account_banking.sepa import postalcode
@@ -11,9 +12,8 @@ class transaction(models.mem_bank_transaction):
 
     def __init__(self, values, *args, **kwargs):
         super(transaction, self).__init__(*args, **kwargs)
-        for attr in self.attrnames:
-            if attr in values:
-                setattr(self, attr, values['attr'])
+        for attr in values:
+                setattr(self, attr, values[attr])
 
 class parser(models.parser):
     code = 'CAMT'
@@ -45,6 +45,17 @@ CAMT Format parser
         """
         return node.xpath(expr, namespaces={'ns': self.ns[1:-1]})
 
+    def find(self, node, expr):
+        """
+        Like xpath(), but return first result if any or else False
+        
+        Return None to test nodes for being truesy
+        """
+        result = node.xpath(expr, namespaces={'ns': self.ns[1:-1]})
+        if result:
+            return result[0]
+        return None
+
     def get_balance_type_node(self, node, balance_type):
         """
         :param node: BkToCstmrStmt/Stmt/Bal node
@@ -133,19 +144,62 @@ CAMT Format parser
             TxDtls = self.xpath(NtryDtl, './ns:TxDtls')
             # Todo: process Btch tag on entry-detail level
             print "    NUmber of TxDtls in NtryDtl: %s" % len(TxDtls)
-            continue
             if len(TxDtls) == 1:
-                transaction_details.append(
-                    self.parse_TxDtl(TxDtls[0], entry_details, amount_sign))
+                vals = self.parse_TxDtl(TxDtls[0], entry_details, amount_sign)
             else:
-                transaction_details.append(
-                   transaction(entry_details))
+                vals = entry_details
+            print vals
+            transaction_details.append(transaction(vals))
         return transaction_details
 
-    def parse_TxDtl(self, node, entry_values, amount_sign):
-        transaction_values = dict(entry_values)
-        amount = amount_sign * float(node.find(self.ns + 'Amt').text)
+    def get_party_values(self, TxDtl):
+        """
+        Determine to get either the debtor or creditor party node
+        and extract the available data from it
+        """
+        vals = {}
+        party_type = self.find(
+            TxDtl, '../../ns:CdtDbtInd').text == 'CRDT' and 'Dbtr' or 'Cdtr'
+        party_node = self.find(TxDtl, './ns:RltdPties/ns:%s' % party_type)
+        account_node = self.find(TxDtl, './ns:RltdPties/ns:%sAcct/ns:Id' % party_type)
+        bic_node = self.find(
+            TxDtl,
+            './ns:RltdAgts/ns:%sAgt/ns:FinInstnId/ns:BIC' % party_type)
+        if party_node is not None:
+            name_node = self.find(party_node, './ns:Nm')
+            vals['remote_owner'] = name_node.text if name_node is not None else False
+            country_node = self.find(party_node, './ns:PstlAdr/ns:Ctry')
+            vals['remote_owner_country'] = (
+                country_node.text if country_node is not None else False)
+            address_node = self.find(party_node, './ns:AdrLine')
+            vals['remote_owner_address'] = (
+                address_node.text if address_node is not None else False)
+        if account_node is not None:
+            iban_node = self.find(account_node, './ns:IBAN')
+            if iban_node is not None:
+                vals['remote_account'] = iban_node.text
+                if bic_node is not None:
+                    vals['remote_iban'] = bic_node.text
+            else:
+                domestic_node = self.find(account_node, './ns:Othr/ns:Id')
+                vals['remote_account'] = (
+                    domestic_node.text if domestic_node is not None else False)
+        return vals
 
+    def parse_TxDtl(self, TxDtl, entry_values, amount_sign):
+        vals = dict(entry_values)
+        # amount = amount_sign * float(node.find(self.ns + 'Amt').text)
+        unstructured = self.xpath(TxDtl, './ns:RmtInf/ns:Ustrd')
+        if unstructured:
+            vals['message'] = ' '.join([x.text for x in unstructured])
+        structured = self.find(TxDtl, './ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref')
+        if structured is not None:
+            vals['reference'] = structured.text
+        else:
+            if vals['message'] and re.match('^[^\s]$', vals['message']):
+                vals['reference'] = vals['message']
+        vals.update(self.get_party_values(TxDtl))
+        return vals
 
     def parse(self, cr, data):
         root = etree.fromstring(data)

From 484d59a309c53dcd8c504c59eab7b590788abe59 Mon Sep 17 00:00:00 2001
From: Stefan Rijnhart <stefan@therp.nl>
Date: Wed, 18 Sep 2013 17:14:11 +0200
Subject: [PATCH 05/14] [FIX] Remove unused imports [FIX] Translation is not
 available (no context) [FIX] Assign statement date [FIX] Effective vs.
 execution date confusion [FIX] Remove debug statements [FIX] Don't say
 'TxDtl' when processing TxDtls node [FIX] Xpath of address line [ADD] Some
 docstrings

---
 account_banking_camt/camt.py | 69 ++++++++++++++++++++++--------------
 1 file changed, 42 insertions(+), 27 deletions(-)

diff --git a/account_banking_camt/camt.py b/account_banking_camt/camt.py
index 783c71cd7..7687aeb1b 100644
--- a/account_banking_camt/camt.py
+++ b/account_banking_camt/camt.py
@@ -3,8 +3,6 @@ from datetime import datetime
 import re
 from account_banking.parsers import models
 from account_banking.parsers.convert import str2date
-from account_banking.sepa import postalcode
-from tools.translate import _
 
 bt = models.mem_bank_transaction
 
@@ -15,13 +13,16 @@ class transaction(models.mem_bank_transaction):
         for attr in values:
                 setattr(self, attr, values[attr])
 
+    def is_valid(self):
+        return not self.error_message
+
 class parser(models.parser):
     code = 'CAMT'
     country_code = 'NL'
-    name = _('Generic CAMT Format')
-    doc = _('''\
+    name = 'Generic CAMT Format'
+    doc = '''\
 CAMT Format parser
-''')
+'''
 
     def tag(self, node):
         """
@@ -34,9 +35,9 @@ CAMT Format parser
         Get node's stripped tag and compare with expected
         """
         assert self.tag(node) == expected, (
-            _("Expected tag '%s', got '%s' instead") %
+            "Expected tag '%s', got '%s' instead" %
             (self.tag(node), expected))
-        
+
     def xpath(self, node, expr):
         """
         Wrap namespaces argument into call to Element.xpath():
@@ -103,6 +104,9 @@ CAMT Format parser
         return self.parse_amount(nodes[-1])
 
     def parse_Stmt(self, node):
+        """
+        Parse a single Stmt node
+        """
         statement = models.mem_bank_statement()
         statement.id = node.find(self.ns + 'Id').text
         statement.local_account = (
@@ -112,9 +116,15 @@ CAMT Format parser
         statement.local_currency = self.xpath(node, './ns:Acct/ns:Ccy')[0].text
         statement.start_balance = self.get_start_balance(node)
         statement.end_balance = self.get_end_balance(node)
-        print "Number of Ntry in statement: %s" % len(self.xpath(node, './ns:Ntry'))
+        number = 0
         for Ntry in self.xpath(node, './ns:Ntry'):
             for transaction_detail in self.parse_Ntry(Ntry):
+                if number == 0:
+                    # Take the statement date from the first transaction
+                    statement.date = str2date(
+                        transaction_detail['execution_date'], "%Y-%m-%d")
+                number += 1
+                transaction_detail['id'] = str(number).zfill(4)
                 statement.transactions.append(
                     transaction(transaction_detail))
         return statement
@@ -129,41 +139,40 @@ CAMT Format parser
         return False
 
     def parse_Ntry(self, node):
+        """
+        :param node: Ntry node
+        """
         entry_description = self.get_entry_description(node)
         entry_details = {
-            'effective_date': self.xpath(node, './ns:BookgDt/ns:Dt')[0].text,
-            'transaction_date': self.xpath(node, './ns:ValDt/ns:Dt')[0].text,
+            'execution_date': self.xpath(node, './ns:BookgDt/ns:Dt')[0].text,
+            'effective_date': self.xpath(node, './ns:ValDt/ns:Dt')[0].text,
             'transfer_type': bt.ORDER,
             'transferred_amount': self.parse_amount(node)
             }
         amount_sign = -1 if node.find(self.ns + 'CdtDbtInd').text == 'CRDT' else 1
         transaction_details = []
-        print "  NUmber of NtryDtls in Ntry with code %s: %s" % (
-            entry_description, len(self.xpath(node, './ns:NtryDtls')))
         for NtryDtl in self.xpath(node, './ns:NtryDtls'):
             TxDtls = self.xpath(NtryDtl, './ns:TxDtls')
             # Todo: process Btch tag on entry-detail level
-            print "    NUmber of TxDtls in NtryDtl: %s" % len(TxDtls)
             if len(TxDtls) == 1:
-                vals = self.parse_TxDtl(TxDtls[0], entry_details, amount_sign)
+                vals = self.parse_TxDtls(TxDtls[0], entry_details, amount_sign)
             else:
                 vals = entry_details
-            print vals
-            transaction_details.append(transaction(vals))
+            transaction_details.append(vals)
         return transaction_details
 
-    def get_party_values(self, TxDtl):
+    def get_party_values(self, TxDtls):
         """
         Determine to get either the debtor or creditor party node
         and extract the available data from it
         """
         vals = {}
         party_type = self.find(
-            TxDtl, '../../ns:CdtDbtInd').text == 'CRDT' and 'Dbtr' or 'Cdtr'
-        party_node = self.find(TxDtl, './ns:RltdPties/ns:%s' % party_type)
-        account_node = self.find(TxDtl, './ns:RltdPties/ns:%sAcct/ns:Id' % party_type)
+            TxDtls, '../../ns:CdtDbtInd').text == 'CRDT' and 'Dbtr' or 'Cdtr'
+        party_node = self.find(TxDtls, './ns:RltdPties/ns:%s' % party_type)
+        account_node = self.find(TxDtls, './ns:RltdPties/ns:%sAcct/ns:Id' % party_type)
         bic_node = self.find(
-            TxDtl,
+            TxDtls,
             './ns:RltdAgts/ns:%sAgt/ns:FinInstnId/ns:BIC' % party_type)
         if party_node is not None:
             name_node = self.find(party_node, './ns:Nm')
@@ -171,7 +180,7 @@ CAMT Format parser
             country_node = self.find(party_node, './ns:PstlAdr/ns:Ctry')
             vals['remote_owner_country'] = (
                 country_node.text if country_node is not None else False)
-            address_node = self.find(party_node, './ns:AdrLine')
+            address_node = self.find(party_node, './ns:PstlAdr/ns:AdrLine')
             vals['remote_owner_address'] = (
                 address_node.text if address_node is not None else False)
         if account_node is not None:
@@ -186,22 +195,28 @@ CAMT Format parser
                     domestic_node.text if domestic_node is not None else False)
         return vals
 
-    def parse_TxDtl(self, TxDtl, entry_values, amount_sign):
+    def parse_TxDtls(self, TxDtls, entry_values, amount_sign):
+        """
+        Parse a single TxDtls node
+        """
         vals = dict(entry_values)
         # amount = amount_sign * float(node.find(self.ns + 'Amt').text)
-        unstructured = self.xpath(TxDtl, './ns:RmtInf/ns:Ustrd')
+        unstructured = self.xpath(TxDtls, './ns:RmtInf/ns:Ustrd')
         if unstructured:
             vals['message'] = ' '.join([x.text for x in unstructured])
-        structured = self.find(TxDtl, './ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref')
+        structured = self.find(TxDtls, './ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref')
         if structured is not None:
             vals['reference'] = structured.text
         else:
-            if vals['message'] and re.match('^[^\s]$', vals['message']):
+            if vals['message'] and re.match('^[^\s]+$', vals['message']):
                 vals['reference'] = vals['message']
-        vals.update(self.get_party_values(TxDtl))
+        vals.update(self.get_party_values(TxDtls))
         return vals
 
     def parse(self, cr, data):
+        """
+        Parse a CAMT053 XML file
+        """
         root = etree.fromstring(data)
         self.ns = root.tag[:root.tag.index("}") + 1]
         self.assert_tag(root[0][0], 'GrpHdr')

From 9e6648b091b26ce555747497a0ea0a06f49ff0fb Mon Sep 17 00:00:00 2001
From: Stefan Rijnhart <stefan@therp.nl>
Date: Wed, 18 Sep 2013 17:27:30 +0200
Subject: [PATCH 06/14] [ADD] CAMT version check

---
 account_banking_camt/__openerp__.py |  7 ++++---
 account_banking_camt/camt.py        | 17 +++++++++++++++++
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/account_banking_camt/__openerp__.py b/account_banking_camt/__openerp__.py
index 6c0d332b0..e7950c991 100644
--- a/account_banking_camt/__openerp__.py
+++ b/account_banking_camt/__openerp__.py
@@ -22,11 +22,12 @@
     'version': '0.1',
     'license': 'AGPL-3',
     'author': 'Therp BV',
-    'website': 'https://launchpad.net/account-banking',
-    'category': 'Account Banking',
+    'website': 'https://launchpad.net/banking-addons',
+    'category': 'Banking addons',
     'depends': ['account_banking'],
     'description': '''
-Module to import SEPA CAMT Format bank statement files
+Module to import SEPA CAMT.053 Format bank statement files. Based
+on the Banking addons framework.
     ''',
     'installable': True,
 }
diff --git a/account_banking_camt/camt.py b/account_banking_camt/camt.py
index 7687aeb1b..0e44a2f5c 100644
--- a/account_banking_camt/camt.py
+++ b/account_banking_camt/camt.py
@@ -1,6 +1,7 @@
 from lxml import etree
 from datetime import datetime
 import re
+from openerp.osv.orm import except_orm
 from account_banking.parsers import models
 from account_banking.parsers.convert import str2date
 
@@ -213,12 +214,28 @@ CAMT Format parser
         vals.update(self.get_party_values(TxDtls))
         return vals
 
+    def check_version(self):
+        """
+        Sanity check the document's namespace
+        """
+        if not self.ns.startswith('{urn:iso:std:iso:20022:tech:xsd:camt.'):
+            raise except_orm(
+                "Error",
+                "This does not seem to be a CAMT format bank statement.")
+
+        if not self.ns.startswith('{urn:iso:std:iso:20022:tech:xsd:camt.053.'):
+            raise except_orm(
+                "Error",
+                "Only CAMT.053 is supported at the moment.")
+        return True
+
     def parse(self, cr, data):
         """
         Parse a CAMT053 XML file
         """
         root = etree.fromstring(data)
         self.ns = root.tag[:root.tag.index("}") + 1]
+        self.check_version()
         self.assert_tag(root[0][0], 'GrpHdr')
         statements = []
         for node in root[0][1:]:

From b700f5e3f902abee2b7498b2cf08fbed9e2cc5a5 Mon Sep 17 00:00:00 2001
From: Stefan Rijnhart <stefan@therp.nl>
Date: Thu, 19 Sep 2013 17:28:32 +0200
Subject: [PATCH 07/14] [FIX] Don't mind multiple NtryDtls entries [FIX] Mind
 missing NtryDtls entry [RFR] Remove singleton [FIX] Mind missing message

---
 account_banking_camt/camt.py | 40 +++++++++++++++---------------------
 1 file changed, 17 insertions(+), 23 deletions(-)

diff --git a/account_banking_camt/camt.py b/account_banking_camt/camt.py
index 0e44a2f5c..d5017b6c7 100644
--- a/account_banking_camt/camt.py
+++ b/account_banking_camt/camt.py
@@ -119,15 +119,15 @@ CAMT Format parser
         statement.end_balance = self.get_end_balance(node)
         number = 0
         for Ntry in self.xpath(node, './ns:Ntry'):
-            for transaction_detail in self.parse_Ntry(Ntry):
-                if number == 0:
-                    # Take the statement date from the first transaction
-                    statement.date = str2date(
-                        transaction_detail['execution_date'], "%Y-%m-%d")
-                number += 1
-                transaction_detail['id'] = str(number).zfill(4)
-                statement.transactions.append(
-                    transaction(transaction_detail))
+            transaction_detail = self.parse_Ntry(Ntry)
+            if number == 0:
+                # Take the statement date from the first transaction
+                statement.date = str2date(
+                    transaction_detail['execution_date'], "%Y-%m-%d")
+            number += 1
+            transaction_detail['id'] = str(number).zfill(4)
+            statement.transactions.append(
+                transaction(transaction_detail))
         return statement
 
     def get_entry_description(self, node):
@@ -150,17 +150,12 @@ CAMT Format parser
             'transfer_type': bt.ORDER,
             'transferred_amount': self.parse_amount(node)
             }
-        amount_sign = -1 if node.find(self.ns + 'CdtDbtInd').text == 'CRDT' else 1
-        transaction_details = []
-        for NtryDtl in self.xpath(node, './ns:NtryDtls'):
-            TxDtls = self.xpath(NtryDtl, './ns:TxDtls')
-            # Todo: process Btch tag on entry-detail level
-            if len(TxDtls) == 1:
-                vals = self.parse_TxDtls(TxDtls[0], entry_details, amount_sign)
-            else:
-                vals = entry_details
-            transaction_details.append(vals)
-        return transaction_details
+        TxDtls = self.xpath(node, './ns:NtryDtls/ns:TxDtls')
+        if len(TxDtls) == 1:
+            vals = self.parse_TxDtls(TxDtls[0], entry_details)
+        else:
+            vals = entry_details
+        return vals
 
     def get_party_values(self, TxDtls):
         """
@@ -196,12 +191,11 @@ CAMT Format parser
                     domestic_node.text if domestic_node is not None else False)
         return vals
 
-    def parse_TxDtls(self, TxDtls, entry_values, amount_sign):
+    def parse_TxDtls(self, TxDtls, entry_values):
         """
         Parse a single TxDtls node
         """
         vals = dict(entry_values)
-        # amount = amount_sign * float(node.find(self.ns + 'Amt').text)
         unstructured = self.xpath(TxDtls, './ns:RmtInf/ns:Ustrd')
         if unstructured:
             vals['message'] = ' '.join([x.text for x in unstructured])
@@ -209,7 +203,7 @@ CAMT Format parser
         if structured is not None:
             vals['reference'] = structured.text
         else:
-            if vals['message'] and re.match('^[^\s]+$', vals['message']):
+            if vals.get('message') and re.match('^[^\s]+$', vals['message']):
                 vals['reference'] = vals['message']
         vals.update(self.get_party_values(TxDtls))
         return vals

From 525b3059a4ad7198e6d97f76da0c92dfb37725cf Mon Sep 17 00:00:00 2001
From: Stefan Rijnhart <stefan@therp.nl>
Date: Thu, 19 Sep 2013 20:44:10 +0200
Subject: [PATCH 08/14] [FIX] Prefix statement id with local account number for
 proper duplicate detection [FIX] Address storage format [FIX] BIC directive

---
 account_banking_camt/camt.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/account_banking_camt/camt.py b/account_banking_camt/camt.py
index d5017b6c7..8b9c435a2 100644
--- a/account_banking_camt/camt.py
+++ b/account_banking_camt/camt.py
@@ -109,11 +109,13 @@ CAMT Format parser
         Parse a single Stmt node
         """
         statement = models.mem_bank_statement()
-        statement.id = node.find(self.ns + 'Id').text
         statement.local_account = (
             self.xpath(node, './ns:Acct/ns:Id/ns:IBAN')[0].text
             if self.xpath(node, './ns:Acct/ns:Id/ns:IBAN')
             else self.xpath(node, './ns:Acct/ns:Id/ns:Othr/ns:Id')[0].text)
+        statement.id = "%s-%s" % (
+            statement.local_account,
+            node.find(self.ns + 'Id').text)
         statement.local_currency = self.xpath(node, './ns:Acct/ns:Ccy')[0].text
         statement.start_balance = self.get_start_balance(node)
         statement.end_balance = self.get_end_balance(node)
@@ -177,14 +179,14 @@ CAMT Format parser
             vals['remote_owner_country'] = (
                 country_node.text if country_node is not None else False)
             address_node = self.find(party_node, './ns:PstlAdr/ns:AdrLine')
-            vals['remote_owner_address'] = (
-                address_node.text if address_node is not None else False)
+            if address_node is not None:
+                vals['remote_owner_address'] = [address_node.text]
         if account_node is not None:
             iban_node = self.find(account_node, './ns:IBAN')
             if iban_node is not None:
                 vals['remote_account'] = iban_node.text
                 if bic_node is not None:
-                    vals['remote_iban'] = bic_node.text
+                    vals['remote_bank_bic'] = bic_node.text
             else:
                 domestic_node = self.find(account_node, './ns:Othr/ns:Id')
                 vals['remote_account'] = (

From 9af5471cf74b692109b431bdcc98674cc0e96593 Mon Sep 17 00:00:00 2001
From: Stefan Rijnhart <stefan@therp.nl>
Date: Thu, 19 Sep 2013 21:08:22 +0200
Subject: [PATCH 09/14] [FIX] Remove unused import [RFR] Proper hook to get
 transfer type

---
 account_banking_camt/camt.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/account_banking_camt/camt.py b/account_banking_camt/camt.py
index 8b9c435a2..6349dccf4 100644
--- a/account_banking_camt/camt.py
+++ b/account_banking_camt/camt.py
@@ -1,5 +1,4 @@
 from lxml import etree
-from datetime import datetime
 import re
 from openerp.osv.orm import except_orm
 from account_banking.parsers import models
@@ -132,24 +131,27 @@ CAMT Format parser
                 transaction(transaction_detail))
         return statement
 
-    def get_entry_description(self, node):
+    def get_transfer_type(self, node):
         """
+        Map entry descriptions to transfer types. To extend with
+        proper mapping from BkTxCd/Domn/Cd/Fmly/Cd to transfer types
+        if we can get our hands on real life samples.
+
+        For now, leave as a hook for bank specific overrides to map
+        properietary codes from BkTxCd/Prtry/Cd.
+
         :param node: Ntry node
         """
-        codes = self.xpath(node, './ns:BkTxCd/ns:Prtry/ns:Cd')
-        if codes:
-            return codes[0].text
-        return False
+        return bt.ORDER
 
     def parse_Ntry(self, node):
         """
         :param node: Ntry node
         """
-        entry_description = self.get_entry_description(node)
         entry_details = {
             'execution_date': self.xpath(node, './ns:BookgDt/ns:Dt')[0].text,
             'effective_date': self.xpath(node, './ns:ValDt/ns:Dt')[0].text,
-            'transfer_type': bt.ORDER,
+            'transfer_type': self.get_transfer_type(node),
             'transferred_amount': self.parse_amount(node)
             }
         TxDtls = self.xpath(node, './ns:NtryDtls/ns:TxDtls')

From 55c9528204a4ad18c5bbe5d0b5f8e03f9c2bfa26 Mon Sep 17 00:00:00 2001
From: Stefan Rijnhart <stefan@therp.nl>
Date: Sat, 21 Sep 2013 16:06:20 +0200
Subject: [PATCH 10/14] [ADD] Copyright notice

---
 account_banking_camt/camt.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/account_banking_camt/camt.py b/account_banking_camt/camt.py
index 6349dccf4..e59f6e81a 100644
--- a/account_banking_camt/camt.py
+++ b/account_banking_camt/camt.py
@@ -1,3 +1,24 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    Copyright (C) 2013 Therp BV (<http://therp.nl>)
+#    All Rights Reserved
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
 from lxml import etree
 import re
 from openerp.osv.orm import except_orm

From 3501ac0ce9a8909aa44a536f16ccc323e64080ab Mon Sep 17 00:00:00 2001
From: Stefan Rijnhart <stefan@therp.nl>
Date: Tue, 24 Sep 2013 15:31:01 +0200
Subject: [PATCH 11/14] [IMP] Always set message as reference if no structured
 reference [RFR] Pep8, unused import

---
 account_banking_camt/camt.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/account_banking_camt/camt.py b/account_banking_camt/camt.py
index e59f6e81a..58e539bd3 100644
--- a/account_banking_camt/camt.py
+++ b/account_banking_camt/camt.py
@@ -20,7 +20,6 @@
 ##############################################################################
 
 from lxml import etree
-import re
 from openerp.osv.orm import except_orm
 from account_banking.parsers import models
 from account_banking.parsers.convert import str2date
@@ -32,7 +31,7 @@ class transaction(models.mem_bank_transaction):
     def __init__(self, values, *args, **kwargs):
         super(transaction, self).__init__(*args, **kwargs)
         for attr in values:
-                setattr(self, attr, values[attr])
+            setattr(self, attr, values[attr])
 
     def is_valid(self):
         return not self.error_message
@@ -191,13 +190,15 @@ CAMT Format parser
         party_type = self.find(
             TxDtls, '../../ns:CdtDbtInd').text == 'CRDT' and 'Dbtr' or 'Cdtr'
         party_node = self.find(TxDtls, './ns:RltdPties/ns:%s' % party_type)
-        account_node = self.find(TxDtls, './ns:RltdPties/ns:%sAcct/ns:Id' % party_type)
+        account_node = self.find(
+            TxDtls, './ns:RltdPties/ns:%sAcct/ns:Id' % party_type)
         bic_node = self.find(
             TxDtls,
             './ns:RltdAgts/ns:%sAgt/ns:FinInstnId/ns:BIC' % party_type)
         if party_node is not None:
             name_node = self.find(party_node, './ns:Nm')
-            vals['remote_owner'] = name_node.text if name_node is not None else False
+            vals['remote_owner'] = (
+                name_node.text if name_node is not None else False)
             country_node = self.find(party_node, './ns:PstlAdr/ns:Ctry')
             vals['remote_owner_country'] = (
                 country_node.text if country_node is not None else False)
@@ -224,11 +225,12 @@ CAMT Format parser
         unstructured = self.xpath(TxDtls, './ns:RmtInf/ns:Ustrd')
         if unstructured:
             vals['message'] = ' '.join([x.text for x in unstructured])
-        structured = self.find(TxDtls, './ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref')
+        structured = self.find(
+            TxDtls, './ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref')
         if structured is not None:
             vals['reference'] = structured.text
         else:
-            if vals.get('message') and re.match('^[^\s]+$', vals['message']):
+            if vals.get('message'):
                 vals['reference'] = vals['message']
         vals.update(self.get_party_values(TxDtls))
         return vals

From a0abfeb15dd97d3d66d368d6e3a822138a7f0650 Mon Sep 17 00:00:00 2001
From: Stefan Rijnhart <stefan@therp.nl>
Date: Mon, 30 Sep 2013 22:25:23 +0200
Subject: [PATCH 12/14] [FIX] As per spec, an increase is marked CRDT, a
 decrease DBIT

---
 account_banking_camt/camt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/account_banking_camt/camt.py b/account_banking_camt/camt.py
index 58e539bd3..d39b05ffe 100644
--- a/account_banking_camt/camt.py
+++ b/account_banking_camt/camt.py
@@ -92,7 +92,7 @@ CAMT Format parser
         :return: signed amount
         :returntype: float
         """
-        sign = -1 if node.find(self.ns + 'CdtDbtInd').text == 'CRDT' else 1
+        sign = -1 if node.find(self.ns + 'CdtDbtInd').text == 'DBIT' else 1
         return sign * float(node.find(self.ns + 'Amt').text)
         
     def get_start_balance(self, node):

From 9c8337fe315d9fc4bce518b19d13d98c3bc3d3ee Mon Sep 17 00:00:00 2001
From: Stefan Rijnhart <stefan@therp.nl>
Date: Wed, 2 Oct 2013 12:20:26 +0200
Subject: [PATCH 13/14] [FIX] Move line names overflow when using full IBAN and
 CAMT statement   	identifier as bank statement identification in OpenERP

---
 account_banking/parsers/models.py | 28 ++++++++++++++++++++++++++++
 account_banking_camt/camt.py      | 24 ++++++++++++++++++------
 2 files changed, 46 insertions(+), 6 deletions(-)

diff --git a/account_banking/parsers/models.py b/account_banking/parsers/models.py
index 8f8effb6d..477b4536a 100644
--- a/account_banking/parsers/models.py
+++ b/account_banking/parsers/models.py
@@ -19,6 +19,7 @@
 #
 ##############################################################################
 
+import re
 from tools.translate import _
 
 class mem_bank_statement(object):
@@ -353,6 +354,33 @@ class parser(object):
             name = "%s-%d" % (base, suffix)
         return name
 
+    def get_unique_account_identifier(self, cr, account):
+        """
+        Get an identifier for a local bank account, based on the last
+        characters of the account number with minimum length 3.
+        The identifier should be unique amongst the company accounts
+
+        Presumably, the bank account is one of the company accounts
+        itself but importing bank statements for non-company accounts
+        is not prevented anywhere else in the system so the 'account'
+        param being a company account is not enforced here either.
+        """
+        def normalize(account_no):
+            return re.sub('\s', '', account_no)
+
+        account = normalize(account)
+        cr.execute(
+            """SELECT acc_number FROM res_partner_bank
+               WHERE company_id IS NOT NULL""")
+        accounts = [normalize(row[0]) for row in cr.fetchall()]
+        tail_length = 3
+        while tail_length <= len(account):
+            tail = account[-tail_length:]
+            if len([acc for acc in accounts if acc.endswith(tail)]) < 2:
+                return tail
+            tail_length += 1
+        return account
+
     def parse(self, cr, data):
         '''
         Parse data.
diff --git a/account_banking_camt/camt.py b/account_banking_camt/camt.py
index d39b05ffe..7384a036b 100644
--- a/account_banking_camt/camt.py
+++ b/account_banking_camt/camt.py
@@ -123,18 +123,30 @@ CAMT Format parser
             self.get_balance_type_node(node, 'ITBD'))
         return self.parse_amount(nodes[-1])
 
-    def parse_Stmt(self, node):
+    def parse_Stmt(self, cr, node):
         """
-        Parse a single Stmt node
+        Parse a single Stmt node.
+
+        Be sure to craft a unique, but short enough statement identifier,
+        as it is used as the basis of the generated move lines' names
+        which overflow when using the full IBAN and CAMT statement id.
         """
         statement = models.mem_bank_statement()
         statement.local_account = (
             self.xpath(node, './ns:Acct/ns:Id/ns:IBAN')[0].text
             if self.xpath(node, './ns:Acct/ns:Id/ns:IBAN')
             else self.xpath(node, './ns:Acct/ns:Id/ns:Othr/ns:Id')[0].text)
-        statement.id = "%s-%s" % (
-            statement.local_account,
-            node.find(self.ns + 'Id').text)
+
+        identifier = node.find(self.ns + 'Id').text
+        if identifier.upper().startswith('CAMT053'):
+            identifier = identifier[7:]
+        statement.id = self.get_unique_statement_id(
+            cr, "%s-%s" % (
+                self.get_unique_account_identifier(
+                    cr, statement.local_account),
+                identifier)
+            )
+
         statement.local_currency = self.xpath(node, './ns:Acct/ns:Ccy')[0].text
         statement.start_balance = self.get_start_balance(node)
         statement.end_balance = self.get_end_balance(node)
@@ -260,5 +272,5 @@ CAMT Format parser
         self.assert_tag(root[0][0], 'GrpHdr')
         statements = []
         for node in root[0][1:]:
-            statements.append(self.parse_Stmt(node))
+            statements.append(self.parse_Stmt(cr, node))
         return statements

From c127768fc980694cf1ba883a14cb97f71c3707da Mon Sep 17 00:00:00 2001
From: Stefan Rijnhart <stefan@therp.nl>
Date: Fri, 11 Oct 2013 14:20:14 +0200
Subject: [PATCH 14/14] [IMP] Take EndToEndId into account when no structured
 reference

---
 account_banking_camt/camt.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/account_banking_camt/camt.py b/account_banking_camt/camt.py
index 7384a036b..2ec31439f 100644
--- a/account_banking_camt/camt.py
+++ b/account_banking_camt/camt.py
@@ -239,6 +239,8 @@ CAMT Format parser
             vals['message'] = ' '.join([x.text for x in unstructured])
         structured = self.find(
             TxDtls, './ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref')
+        if structured is None or not structured.text:
+            structured = self.find(TxDtls, './ns:Refs/ns:EndToEndId') 
         if structured is not None:
             vals['reference'] = structured.text
         else: