From 26fac7bfa76a4ab7246c5f19d5c9f6593cf52a99 Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <rportier@therp.nl>
Date: Sat, 13 Jun 2015 01:20:18 +0200
Subject: [PATCH] [ENH] Support zip files.

---
 .../models/account_bank_statement_import.py   |  33 ++++++++++++++++--
 .../test_files/test-camt053.xml               |  30 ++++++++--------
 .../test_files/test-camt053.zip               | Bin 0 -> 3111 bytes
 .../tests/test_import_bank_statement.py       |  11 ++++--
 .../mt940.py                                  |   2 +-
 .../test_files/test-camt053.zip               | Bin 0 -> 3111 bytes
 6 files changed, 55 insertions(+), 21 deletions(-)
 create mode 100644 account_bank_statement_import_camt/test_files/test-camt053.zip
 create mode 100644 bank_statement_parse_camt/test_files/test-camt053.zip

diff --git a/account_bank_statement_import/models/account_bank_statement_import.py b/account_bank_statement_import/models/account_bank_statement_import.py
index 9a8b9136..58d07c53 100644
--- a/account_bank_statement_import/models/account_bank_statement_import.py
+++ b/account_bank_statement_import/models/account_bank_statement_import.py
@@ -2,6 +2,8 @@
 """Framework for importing bank statement files."""
 import logging
 import base64
+from StringIO import StringIO
+from zipfile import ZipFile, BadZipfile  # BadZipFile in Python >= 3.2
 
 from openerp import api, models, fields
 from openerp.tools.translate import _
@@ -52,7 +54,7 @@ class AccountBankStatementImport(models.TransientModel):
 
     @api.multi
     def import_file(self):
-        """ Process the file chosen in the wizard, create bank statement(s) and
+        """Process the file chosen in the wizard, create bank statement(s) and
         go to reconciliation."""
         self.ensure_one()
         data_file = base64.b64decode(self.data_file)
@@ -71,13 +73,38 @@ class AccountBankStatementImport(models.TransientModel):
             'type': 'ir.actions.client',
         }
 
+    @api.model
+    def _parse_all_files(self, data_file):
+        """Parse one file or multiple files from zip-file.
+
+        Return array of statements for further processing.
+        """
+        statements = []
+        files = [data_file]
+        try:
+            with ZipFile(StringIO(data_file), 'r') as archive:
+                files = [
+                    archive.read(filename) for filename in archive.namelist()
+                    if not filename.endswith('/')
+                    ]
+        except BadZipfile:
+            pass
+        # Parse the file(s)
+        for import_file in files:
+            # The appropriate implementation module(s) returns the statements.
+            # Actually we don't care wether all the files have the same
+            # format. Although unlikely you might mix mt940 and camt files
+            # in one zipfile.
+            statements += self._parse_file(import_file)
+        return statements
+
     @api.model
     def _import_file(self, data_file):
         """ Create bank statement(s) from file."""
         # The appropriate implementation module returns the required data
         statement_ids = []
         notifications = []
-        parse_result = self._parse_file(data_file)
+        parse_result = self._parse_all_files(data_file)
         # Check for old version result, with separate currency and account
         if isinstance(parse_result, tuple) and len(parse_result) == 3:
             (currency_code, account_number, statements) = parse_result
@@ -125,7 +152,7 @@ class AccountBankStatementImport(models.TransientModel):
         return self._create_bank_statement(stmt_vals)
 
     @api.model
-    def _parse_file(self, data_file):
+    def _parse_file(self, dummy_data_file):
         """ Each module adding a file support must extends this method. It
         processes the file if it can, returns super otherwise, resulting in a
         chain of responsability.
diff --git a/account_bank_statement_import_camt/test_files/test-camt053.xml b/account_bank_statement_import_camt/test_files/test-camt053.xml
index 1d74d81b..4a2a0473 100644
--- a/account_bank_statement_import_camt/test_files/test-camt053.xml
+++ b/account_bank_statement_import_camt/test_files/test-camt053.xml
@@ -2,15 +2,15 @@
     <BkToCstmrStmt>
         <GrpHdr>
             <MsgId>TESTBANK/NL/1420561226673</MsgId>
-            <CreDtTm>2013-01-06T16:20:26.673Z</CreDtTm>
+            <CreDtTm>2014-01-06T16:20:26.673Z</CreDtTm>
         </GrpHdr>
         <Stmt>
             <Id>1234Test/1</Id>
             <LglSeqNb>2</LglSeqNb>
-            <CreDtTm>2013-01-06T16:20:26.673Z</CreDtTm>
+            <CreDtTm>2014-01-06T16:20:26.673Z</CreDtTm>
             <FrToDt>
-                <FrDtTm>2013-01-05T00:00:00.000Z</FrDtTm>
-                <ToDtTm>2013-01-05T23:59:59.999Z</ToDtTm>
+                <FrDtTm>2014-01-05T00:00:00.000Z</FrDtTm>
+                <ToDtTm>2014-01-05T23:59:59.999Z</ToDtTm>
             </FrToDt>
             <Acct>
                 <Id>
@@ -32,7 +32,7 @@
                 <Amt Ccy="EUR">15568.27</Amt>
                 <CdtDbtInd>CRDT</CdtDbtInd>
                 <Dt>
-                    <Dt>2013-01-05</Dt>
+                    <Dt>2014-01-05</Dt>
                 </Dt>
             </Bal>
             <Bal>
@@ -44,7 +44,7 @@
                 <Amt Ccy="EUR">15121.12</Amt>
                 <CdtDbtInd>CRDT</CdtDbtInd>
                 <Dt>
-                    <Dt>2013-01-05</Dt>
+                    <Dt>2014-01-05</Dt>
                 </Dt>
             </Bal>
             <Ntry>
@@ -52,10 +52,10 @@
                 <CdtDbtInd>DBIT</CdtDbtInd>
                 <Sts>BOOK</Sts>
                 <BookgDt>
-                    <Dt>2013-01-05</Dt>
+                    <Dt>2014-01-05</Dt>
                 </BookgDt>
                 <ValDt>
-                    <Dt>2013-01-05</Dt>
+                    <Dt>2014-01-05</Dt>
                 </ValDt>
                 <BkTxCd>
                     <Domn>
@@ -104,9 +104,9 @@
                             </CdtrAgt>
                         </RltdAgts>
                         <RmtInf>
-                            <Ustrd>Insurance policy 857239PERIOD 01.01.2013 - 31.12.2013</Ustrd>
+                            <Ustrd>Insurance policy 857239PERIOD 01.01.2014 - 31.12.2014</Ustrd>
                         </RmtInf>
-                        <AddtlTxInf>MKB Insurance 859239PERIOD 01.01.2013 - 31.12.2013</AddtlTxInf>
+                        <AddtlTxInf>MKB Insurance 859239PERIOD 01.01.2014 - 31.12.2014</AddtlTxInf>
                     </TxDtls>
                 </NtryDtls>
             </Ntry>
@@ -116,10 +116,10 @@
                 <RvslInd>true</RvslInd>
                 <Sts>BOOK</Sts>
                 <BookgDt>
-                    <Dt>2013-01-05</Dt>
+                    <Dt>2014-01-05</Dt>
                 </BookgDt>
                 <ValDt>
-                    <Dt>2013-01-05</Dt>
+                    <Dt>2014-01-05</Dt>
                 </ValDt>
                 <BkTxCd>
                     <Domn>
@@ -182,10 +182,10 @@
                 <CdtDbtInd>CRDT</CdtDbtInd>
                 <Sts>BOOK</Sts>
                 <BookgDt>
-                    <Dt>2013-01-05</Dt>
+                    <Dt>2014-01-05</Dt>
                 </BookgDt>
                 <ValDt>
-                    <Dt>2013-01-05</Dt>
+                    <Dt>2014-01-05</Dt>
                 </ValDt>
                 <BkTxCd>
                     <Domn>
@@ -202,7 +202,7 @@
                 <NtryDtls>
                     <TxDtls>
                         <Refs>
-                            <InstrId>INNDNL2U20130105000217200000708</InstrId>
+                            <InstrId>INNDNL2U20140105000217200000708</InstrId>
                             <EndToEndId>115</EndToEndId>
                         </Refs>
                         <AmtDtls>
diff --git a/account_bank_statement_import_camt/test_files/test-camt053.zip b/account_bank_statement_import_camt/test_files/test-camt053.zip
new file mode 100644
index 0000000000000000000000000000000000000000..ccf5b3c2ba72ad03cc137edcf2ee55f67b7a2952
GIT binary patch
literal 3111
zcmeH}=T{S08pZ=5FccF7R764P14xk;5EP_{0RmY=5O4w{AVr!$C=oHlAWJhyReBE!
zAqJ!<ML=3G(xoXyK_C$fO&!PxF7ED`-JPBJ19s1xd!BpW_setcr{D7+tbu&u007_z
z0GPk;y!Wsd2MYoK4$A`oVt``++?5cVnmfiHr>U#$s;Q+O;qQmC7X$#CFp;S53XKp3
z@bIyK0Kng`kJ*<4=k&#UW|xqT(@Tov5YUyorl#QtnL>GL823Dkl9(M6fc0cI%(|Ni
z6Q^GLcu71wsM2}%?CvjH!8b!s<&9$V%#|&2<ZU+x4lkelsJ@AUmdg;7B*ccxhUN-5
zijby6yeWwKg10+0y4PY=?AGzegyYneT*WLqu|~D|1%fBU^|_yGn{dzewB`y3Xw16I
zsbi7NVKR=R@}@fQOEy8i2tN<66f;t(CB5_~Y;b&?$*>9fp7$*Z6H3|gs9r@Iab`}w
zn70^AGxt!43sw<D8nGjAhkqGr<@iHX3XH$e^=3FK!Vdi@`Dlxw0!g>21&ugEeu5!s
zB=aSvS|O$Bs(_1W`3N!7kW{?C+9uvd?a-AuvRv(>X!goeIdlV;b{mCeyP1vK$0=9p
zhvGziT|Owbw0q_obsli0rv4Sr^-c(wMC`04QcZhDwWKHp@XFHQlHEbhsWMKSBZZOw
zA}>C4#%A3<RyqHC|H{C84g2~7)+?~mq)y4Cdmmth4$b-^<%mXU%Z@!EmLZ0p`Y4D`
zCiV;*CS`>WrP-sj`HDO4lA-;!yrBVX&3%#eOMKMU@+IV(nv7lhYS%{;$3{6@D@-?8
zj%$0W-f~sSJ^sQi^P9;MO$$IZFznJn;)6b5Ft;O?XIp3Wvr?Mxgx4Co_F1d*^z6>W
z16A~WLf`!H5cIp~GA4ZCR<HxixmC8@9s0l}W$J@Uyl>{`G`WEw$zws*gec}$;&fZM
zo5o5(B`9uG=!A2Ina*WDMms&g&>^6!qHt5;<KPny{Q3tsB)WHBGwU*37mC@IBYi!E
zLkn#5C8<djf(2^~=*c31_a%f12huPtP3I)}RkuK~2~;ejF11*qTcPxE{7Lr=Z@!#4
zBC4e+Nb+Lrx!Cp?MuLa8Bx0hz)-z#Qpuazc%LuA;LteK^$NNS;7f0#g+M;y&uGw@b
zqtC<LiD^_`n2=|*w%K?AKGPR+_skJHwVze7Mqk4XKIH1(U(~55KW!X-U48`AKn1=R
zm~q>GqTVR1>*Ec+Z)I*wlOs-1v{7k(SD7cx&3TX<f17Z=oQAMO<2Q*_vAvs?Y5TO5
zdnUz3DAP62sX1D4&=d;r$S{^BfqD{I(G=vG&{u;I(|ZJecC)IfhR<1ZetNo2@Z!-n
zbscZPH@Efb-lA*$?4R91Mv2&H@Ca>>4UUU|z4aL*H}d*MEMlY`)`QpML{Lh>UZ>Y$
z#RRHb?<f$ZHdvy)BAhLfa=_8?hkke6oG};X-8z!G$D?Y;F%R+%@q)82y&tH+30NY1
z@~1LeV?{^FvTvb9=Bts?k0rCK<l^3O*d#@QIvnh3mG+qJG+63d!hMO~SQGVUrkBcL
z<K5ux-KyR-?X%YLqPCI2At5EA(MZ(0`;k5*L)6{=Tn$v!qMu5Ik@Yi=@hpbR5-!7h
zvI(&u(04GDF!)ncP6)-oE6YMSA(s^fOMwi=929H?RpGCF)w{a3ku9?02YT3DQ#~fz
zo=^uaYCQK!#+@T8c1V_F5=U-kxm*|*%++A&xlGRvbo`+EJ~^b=4%S|A!a6W$nM=&*
z84bJ+^J(2m9R}Y#<FNY-fscU2D^P2gFql%(41HIHnl4`_Wwd*7vC{ZJf)+vcQ+xG@
zDEIwIbI4}W+2SB^^`%vO7R#=iG>tB<n73Ze7>{p^?Uws=MQd|l>2$Th%nhp<L6f@v
zFRe>47tF`Rx7B!RoxbGo{<z9%;2S)@PzCrD9k#o6OsB5-t*Rs>x9Ijt)~YQ+V444%
zPzs&KG$xTJ)T2(<N6-wu4uhEyZ&rCfpWT5Ve#N!=9<EiO9E$rN;#&JxTyO26`n&$;
za1H+ruD3;fiKWCw^&vj^y+E7wUhHgB#8it@dQW1ZQar<I|AKuwe7$sj2)$zYt+C+@
zHesNE;CM7JJOiY<bYXy2%(C03$=NvEDMf0{Yxl8Uq)((W&~-qKq7k+UGw<AA_VC(S
zdni#WsOP}zOKrF>Qz^8|QRB}mXgGS2uVz8>+^ZoN75^~j6MqhVW51<$wUzWDyF|J1
zirME@jT#|^OHAAz3aV0H+~Jxv^`tk-VY6f-Rl*g=@Y-F-Ba{e9gU80nOv-XmG14&C
zV<gElSr<s}nDKZW?iPybi*uak&oqTCQ8Ex78q&tDBAzoAb{A27`{Tf7r)TsH1z$Za
z>(E)5%on5Y=I7h_Gab0duA42{ST<Q-`SuY*GRv@QLb@e9NkLXnbo&f;B@<GsFONp?
z73)D5Y{*gm_an{Y1vWHHL3{ZKclT;2;b->}8V<9on@o5adK;Lj4$TmL;iTZRlaNcE
zF1&6LTv;hka~iWF8};~H>r~tMthOl-2An0f&<V6dCmd9xy;0yHziJtQj>eCUPY+h!
z>c3ZOHlMoCRY`(}DnQqY19eBIViuA`HhKm_r5I4{dR+8rdB$SfS*>Oa_D7-8&b&W)
z{U@*guX)YGD-Hakf-SLEya527J(Uu_PrqBYLI0@RzhO7R`d`NX9R>ex{Ae#A{?quM
b!LVoiJE4HE2JwIY3-8{k*?V1uzi)p7q~E=A

literal 0
HcmV?d00001

diff --git a/account_bank_statement_import_camt/tests/test_import_bank_statement.py b/account_bank_statement_import_camt/tests/test_import_bank_statement.py
index 8a1694f0..d14aca96 100644
--- a/account_bank_statement_import_camt/tests/test_import_bank_statement.py
+++ b/account_bank_statement_import_camt/tests/test_import_bank_statement.py
@@ -31,11 +31,10 @@ class TestImport(TestStatementFile):
             {
                 'remote_account': 'NL46ABNA0499998748',
                 'transferred_amount': -754.25,
-                'value_date': '2013-01-05',
+                'value_date': '2014-01-05',
                 'ref': '435005714488-ABNO33052620',
             },
         ]
-        # statement name is account number + '-' + date of last 62F line:
         self._test_statement_import(
             'account_bank_statement_import_camt', 'test-camt053.xml',
             '1234Test/1',
@@ -43,3 +42,11 @@ class TestImport(TestStatementFile):
             start_balance=15568.27, end_balance=15121.12,
             transactions=transactions
         )
+
+    def test_zip_import(self):
+        """Test import of multiple statements from zip file."""
+        self._test_statement_import(
+            'account_bank_statement_import_camt', 'test-camt053.zip',
+            '1234Test/2',  # Only name of first statement
+            local_account='NL77ABNA0574908765',
+        )
diff --git a/account_bank_statement_import_mt940_base/mt940.py b/account_bank_statement_import_mt940_base/mt940.py
index 8677bc7c..9ddd9cef 100644
--- a/account_bank_statement_import_mt940_base/mt940.py
+++ b/account_bank_statement_import_mt940_base/mt940.py
@@ -111,7 +111,7 @@ class MT940(object):
         This in fact uses the ING syntax, override in others."""
         self.mt940_type = 'General'
         self.header_lines = 3  # Number of lines to skip
-        self.header_regex = '^0000 01INGBNL2AXXXX|^{1'
+        self.header_regex = '^0000 01INGBNL2AXXXX|^{1'  # Start of header
         self.footer_regex = '^-}$|^-XXX$'  # Stop processing on seeing this
         self.tag_regex = '^:[0-9]{2}[A-Z]*:'  # Start of new tag
         self.current_statement = None
diff --git a/bank_statement_parse_camt/test_files/test-camt053.zip b/bank_statement_parse_camt/test_files/test-camt053.zip
new file mode 100644
index 0000000000000000000000000000000000000000..ccf5b3c2ba72ad03cc137edcf2ee55f67b7a2952
GIT binary patch
literal 3111
zcmeH}=T{S08pZ=5FccF7R764P14xk;5EP_{0RmY=5O4w{AVr!$C=oHlAWJhyReBE!
zAqJ!<ML=3G(xoXyK_C$fO&!PxF7ED`-JPBJ19s1xd!BpW_setcr{D7+tbu&u007_z
z0GPk;y!Wsd2MYoK4$A`oVt``++?5cVnmfiHr>U#$s;Q+O;qQmC7X$#CFp;S53XKp3
z@bIyK0Kng`kJ*<4=k&#UW|xqT(@Tov5YUyorl#QtnL>GL823Dkl9(M6fc0cI%(|Ni
z6Q^GLcu71wsM2}%?CvjH!8b!s<&9$V%#|&2<ZU+x4lkelsJ@AUmdg;7B*ccxhUN-5
zijby6yeWwKg10+0y4PY=?AGzegyYneT*WLqu|~D|1%fBU^|_yGn{dzewB`y3Xw16I
zsbi7NVKR=R@}@fQOEy8i2tN<66f;t(CB5_~Y;b&?$*>9fp7$*Z6H3|gs9r@Iab`}w
zn70^AGxt!43sw<D8nGjAhkqGr<@iHX3XH$e^=3FK!Vdi@`Dlxw0!g>21&ugEeu5!s
zB=aSvS|O$Bs(_1W`3N!7kW{?C+9uvd?a-AuvRv(>X!goeIdlV;b{mCeyP1vK$0=9p
zhvGziT|Owbw0q_obsli0rv4Sr^-c(wMC`04QcZhDwWKHp@XFHQlHEbhsWMKSBZZOw
zA}>C4#%A3<RyqHC|H{C84g2~7)+?~mq)y4Cdmmth4$b-^<%mXU%Z@!EmLZ0p`Y4D`
zCiV;*CS`>WrP-sj`HDO4lA-;!yrBVX&3%#eOMKMU@+IV(nv7lhYS%{;$3{6@D@-?8
zj%$0W-f~sSJ^sQi^P9;MO$$IZFznJn;)6b5Ft;O?XIp3Wvr?Mxgx4Co_F1d*^z6>W
z16A~WLf`!H5cIp~GA4ZCR<HxixmC8@9s0l}W$J@Uyl>{`G`WEw$zws*gec}$;&fZM
zo5o5(B`9uG=!A2Ina*WDMms&g&>^6!qHt5;<KPny{Q3tsB)WHBGwU*37mC@IBYi!E
zLkn#5C8<djf(2^~=*c31_a%f12huPtP3I)}RkuK~2~;ejF11*qTcPxE{7Lr=Z@!#4
zBC4e+Nb+Lrx!Cp?MuLa8Bx0hz)-z#Qpuazc%LuA;LteK^$NNS;7f0#g+M;y&uGw@b
zqtC<LiD^_`n2=|*w%K?AKGPR+_skJHwVze7Mqk4XKIH1(U(~55KW!X-U48`AKn1=R
zm~q>GqTVR1>*Ec+Z)I*wlOs-1v{7k(SD7cx&3TX<f17Z=oQAMO<2Q*_vAvs?Y5TO5
zdnUz3DAP62sX1D4&=d;r$S{^BfqD{I(G=vG&{u;I(|ZJecC)IfhR<1ZetNo2@Z!-n
zbscZPH@Efb-lA*$?4R91Mv2&H@Ca>>4UUU|z4aL*H}d*MEMlY`)`QpML{Lh>UZ>Y$
z#RRHb?<f$ZHdvy)BAhLfa=_8?hkke6oG};X-8z!G$D?Y;F%R+%@q)82y&tH+30NY1
z@~1LeV?{^FvTvb9=Bts?k0rCK<l^3O*d#@QIvnh3mG+qJG+63d!hMO~SQGVUrkBcL
z<K5ux-KyR-?X%YLqPCI2At5EA(MZ(0`;k5*L)6{=Tn$v!qMu5Ik@Yi=@hpbR5-!7h
zvI(&u(04GDF!)ncP6)-oE6YMSA(s^fOMwi=929H?RpGCF)w{a3ku9?02YT3DQ#~fz
zo=^uaYCQK!#+@T8c1V_F5=U-kxm*|*%++A&xlGRvbo`+EJ~^b=4%S|A!a6W$nM=&*
z84bJ+^J(2m9R}Y#<FNY-fscU2D^P2gFql%(41HIHnl4`_Wwd*7vC{ZJf)+vcQ+xG@
zDEIwIbI4}W+2SB^^`%vO7R#=iG>tB<n73Ze7>{p^?Uws=MQd|l>2$Th%nhp<L6f@v
zFRe>47tF`Rx7B!RoxbGo{<z9%;2S)@PzCrD9k#o6OsB5-t*Rs>x9Ijt)~YQ+V444%
zPzs&KG$xTJ)T2(<N6-wu4uhEyZ&rCfpWT5Ve#N!=9<EiO9E$rN;#&JxTyO26`n&$;
za1H+ruD3;fiKWCw^&vj^y+E7wUhHgB#8it@dQW1ZQar<I|AKuwe7$sj2)$zYt+C+@
zHesNE;CM7JJOiY<bYXy2%(C03$=NvEDMf0{Yxl8Uq)((W&~-qKq7k+UGw<AA_VC(S
zdni#WsOP}zOKrF>Qz^8|QRB}mXgGS2uVz8>+^ZoN75^~j6MqhVW51<$wUzWDyF|J1
zirME@jT#|^OHAAz3aV0H+~Jxv^`tk-VY6f-Rl*g=@Y-F-Ba{e9gU80nOv-XmG14&C
zV<gElSr<s}nDKZW?iPybi*uak&oqTCQ8Ex78q&tDBAzoAb{A27`{Tf7r)TsH1z$Za
z>(E)5%on5Y=I7h_Gab0duA42{ST<Q-`SuY*GRv@QLb@e9NkLXnbo&f;B@<GsFONp?
z73)D5Y{*gm_an{Y1vWHHL3{ZKclT;2;b->}8V<9on@o5adK;Lj4$TmL;iTZRlaNcE
zF1&6LTv;hka~iWF8};~H>r~tMthOl-2An0f&<V6dCmd9xy;0yHziJtQj>eCUPY+h!
z>c3ZOHlMoCRY`(}DnQqY19eBIViuA`HhKm_r5I4{dR+8rdB$SfS*>Oa_D7-8&b&W)
z{U@*guX)YGD-Hakf-SLEya527J(Uu_PrqBYLI0@RzhO7R`d`NX9R>ex{Ae#A{?quM
b!LVoiJE4HE2JwIY3-8{k*?V1uzi)p7q~E=A

literal 0
HcmV?d00001