From 301e70f4b72f8fd1fdac0272af12426f2625b7f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Felipe=20Mil=C3=A9o?= Date: Fri, 10 May 2024 20:37:42 -0300 Subject: [PATCH 1/2] [IMP][account_statement_import_sheet_file] Empty lines with spaces, or with simbols eg: USD/$ --- .../account_statement_import_sheet_parser.py | 12 +- .../sample_statement_en_empty_values.xlsx | Bin 0 -> 6501 bytes ...est_account_statement_import_sheet_file.py | 140 ++++++++++++++++++ 3 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 account_statement_import_sheet_file/tests/fixtures/sample_statement_en_empty_values.xlsx diff --git a/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py b/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py index cfe9069e..babd4286 100644 --- a/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py @@ -4,6 +4,7 @@ import itertools import logging +import re from collections.abc import Iterable from datetime import datetime from decimal import Decimal @@ -459,9 +460,16 @@ class AccountStatementImportSheetParser(models.TransientModel): if isinstance(value, Decimal): return value elif isinstance(value, float): - return Decimal(value) - value = value or "0" + return Decimal(str(value)) thousands, decimal = mapping._get_float_separators() + # Remove all characters except digits, thousands separator, + # decimal separator, and signs + value = ( + re.sub( + r"[^\d\-+" + re.escape(thousands) + re.escape(decimal) + "]+", "", value + ) + or "0" + ) value = value.replace(thousands, "") value = value.replace(decimal, ".") return Decimal(value) diff --git a/account_statement_import_sheet_file/tests/fixtures/sample_statement_en_empty_values.xlsx b/account_statement_import_sheet_file/tests/fixtures/sample_statement_en_empty_values.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..7b9a2193eb8df879b7b8f2498463ede2936dddab GIT binary patch literal 6501 zcmaKQ1z41A6YkO>Ai1P;$P!DJAl;oxce^Z&q)173mxQErOLw=VO9>(%ASwvbXYu>b zf#3gp&UIPMb-2~JNdo1GAme7a$sHI zU{o{;)7A|`Y-2R{E$SV)a`FW8jph0E;|a)WKqW!WF{%njqaVeG-*~&mDqwCODl1MQ zY?<#_07S%IIU8zwa9vr1eh$*zi^O-b*2!CVf!rFT46)~Rkf);Xdd$84gf@Z3eNaIu z-=|2R6(8F&1k-cH()ZoS>Omq7DHeyEGF}SJ=dXjn_l?msmx&`gsi_3b?>;*_njN8z zwc-bHSi5>F)A`CNeSJ;zfZJFhLNEJbk+}rrFXJ&g}I8sfJxO zEqyJgx*3wCKfChJhx^N<%B1^JY}(IEjpKx-_c;*ayG_1j9e+D4&|-oQOEQ z9-aG8l%)Bg6e>&B0~WP~0FqdWur@Yn$fB7MI)*AllCpAlqLuaoMhfX6ab;Zx8F+K? z<5TFvM^-2)z~U*5*Bx)bu{3VI1r|o*c-^)fdm2~%lX7WVA(>Gd@|}xU(foyyOLlba z_Q3{Ux$+AlNz*sV$?&A(E?H{qQGX6X6Cuj%afdZ6DAz9FS zX2xctj30T-D1RK~l4_00AQ-X)Rd0t25UONl`hVdC@_y>r8Qd>B_Zr4_eQW%DfrDY* zS0y-0DVCPWR2mdkm4<(sntsF$wsG(K5Yz1LsaWDvTS&#B4p{V66H4 zl~-MkGq5s#ax#M&g~E*2^enH0@|kQ;^?8nBgvF{`rtQF-`o>(biUUJMb;tX~r&;vC z4tS=Zq0fTSUSA>r08pgAWe)Z~nd9Q>`OMPgCU5q24WStvgubhlRZH{bu{`pk$XbR% zg>6c+=?V!AA_69zrPXmxA0~}OUmzpe*4WwB7w8vjefuf0MH$W6RN?V>g1MV+BfGq}FqEb6U{yRhYx5nd;Jv7qDUJ<^ zLP31E&o_~pB(>sxx_b386N|mkqzcKk$I&F)TqumfW6%_LUIsECzf8fIM*vw(C{%H8qJrbMlTi+Qxi%bZPMy+msKnUx}om#@ zKm}py1sf!!a@M^(VJskM&Y`yyG?H({H&~ZTZ;Zk~qbwy5??>@eoACm?@7tRLGs)9F zs5!a!ve)G_SkXc^vC#Wvco-%hBR(*ryR>o?*|d>>GQLoMQHzHQusOvRZ68M(MpT!y zGnznDv{azl2JzL~HsCFN;P6OW6tz-T=>^y-FtvPZ&S>=iUVLV#vMomi6i4m~Vx~CtF)9by0 zBdSU`*D-r3H1Z-ASrp;suvLk3w{gJ_nkDJq4jbJqO}p4wTDrP`|EX=iZ#ZvG-|s3H z6*Gd_&4DX@;wPQjmNpk0%Cj#M0?;g9sxNqi1ow1(LJrx?2 zamrPr>7x^3SUsK*auj|#3004>u=q5dFgCsx5Xbyl zJU`_Ni1%aR!v>OeQZ;aD**;281Nfzzs_J#sGKW$)j>Y5QgC2y4AA-hNi<^Tebi%)0RgKWkP zQ;JY3a6N1S?Vk=Yzm4^0)^BMwC9$;A)NX#c#o!Pu2iZm6+T-Q4$xa{y+ccLrvbo}h zDAH5=8pVbj@P~8To0FJCL-4Ar;8gqYdK@tW9w~rv$pfv846EUnk;}@ zYQta-*~(mqgu@N`SX-Ls-J%^$m-vcQn?JJF9vnom8BH$Zb1fx?9w31>m}F|mHO#9p z8*HPlCzmsm=fzMm%dfgSNCaTyB)Y`08`$I0#2PuXBJ55e$GKIHMmz@pCDA7=dhQ5Z znfc1h7MK9p#g+Ep$Rc+(3^p2i2z@U_N&)LkgT~G|={i#!q)kn4xu_U9-DhHV>{L0^EuYpw3k+mN`QN!a*W|6BF-|V!Z;J_(G1%|yvdQ0)452*=O|v)jdgpP%7@r1 z>-ZPiLcZj^^Rw?{ztv)>4bMX59m2@iVe_cU{05VyV~YI6)Vd+N300W25~{Hb7Yy}h zBYm~cV&CY!!dTPgO}=PlvzU}XEuto{CTtb4Sda)|qQ2~vUvGRBsW-Td{ArMs2^vGv z82Y`0b|Bj0_&8SnURq{AWajpLu`{EpqPdd`%}(=yn3?jq6X_1SSg~4+{6`TbwG~r8 zXTJdMH@e4$9bfhF>U@)lQfm%MP?fAuGv!9xb^olqRIXI@vx!IK!aU~P4jzR!t5VFj zg^l5(&}fn5LfIwHrcNVS({JO1s36YDl&4%dlScyNYQ-V}*#kCWxjUScYBqVl0*%<~ z6^jk*#+)ho=>{S$q%JeF3VD326(nmDyK-Bw#>@o;+zuymMyV)Tf_JzqeoBFQ5=gHY z-%dTMB}*&lMbe>5nC&vnmK7QPa-154n#h`Wz4=p$D^RgI(l25D=|1-YgzD3akfjlv zBejV&%!zu%rMFDzt=vqcMsDw+jxk^deF4?Bo;q8HbYS7mFb|vZd#3lUko3nHgBO5j~K{Sj|-Gh2;A)VM61IK#y zX6a|jfF>>H9+yu=obQ+C2!2Vd-hZOiX&TF&N~q9Z9Ci@n^L*qLhAU1xby=o{lh@FG z??NF0%A|?As!`Qd>V63}!}JF*c1M=G@lFGSiZh!HmO0 z0xY~q;9LBH7-F~MeCoXSzQDny>}~~d+fiEvBAyi5lE-FlQ@BkoT=gPZXo?Et0kek) zrSHpFi4b&Teo#ED?2G5{^ZzpHq3d2RV$iS{WR<4#%2C40oa9UBhoz`frX zg?0ov-4xjrrG2nKH(VL`T7%C6W7gP;^1$mJ6a!YS=C^l2fB2&@O}e~tW!VSu1{u|F zr(V1OClrUx2%|RG)a6(8EY50z@oLAXUzoZ^X z+;!#`AlFK-%649AamCMxJaReG{A;|=wcsfZeOg;1ldF25tx%TPYCZ|Gp~txll}0PU zL<|L;dBMtvW|4NdXku6G9y@10E%7X4pRD~kGfm;+QXR6&-6VJZ%H>Ig-G-i*seWqC z&}fClV<|)hQd`S>&eE3J24)+M{n29J7b)ZQyCT8)0LFL_VX}pPP?XuncAB! z63>!jSq|?vK5Ue(_rV((BO9DZL6ib&zXRkQ`Xp6G;2--Nbc#{tIra7v;N(dsRW{oA z7@CffAO%xetfm#o6ULCpw(KJkO7JJT^Q!NiB9Kp^vyUOEKvO;w@|95Sa8Rg+e}#rR zLc5We5U~uknRdrPy+{GhMWr&762CyTTm01hT)LbU^dXKCNF&?KReKq9 zPB8@-xeX15YIg3tm{yFX(mUyG(*e5tUyyjqWpvWWrZ82kv$r(6$S~fNEa_0kV=#dy zH2AfnYy}@dzDh)$xlIqh?5^nE=ewbt3K7>rnviXK!JZvXgqjATR_d>8RX~!HitS} z-Uz*5jS=`H1Gn{%DDEI^;|QVVgBGZm2}JV{(TRJUkC#5$5~_c8A|RgtEbfI^TMM86 zusGHRcHxw1CS~IT`{)5|ZvVO=f?aKS>YYyu=qW&C3E*?2mE_&jl&<|{JMn3Hl@L>5tc363*ryLavwCJs8XwsA z@Krq1t7Tf-z{;!}&gn2!ypnrvfSv`t?A8qYcu^g9RC1OW?AUZ9Ys@nE8STJ{X5LMU z(&aju2UEAk41N~v`Ubk6odd;H{(Q^9FRskb zuy2Ux)Tt}N7QRogzwQ&9uW$Fs)Y0)~o3dh+q1`tFr*k6h-g;%*#sD?4I@=EzwvSwG z8bOK%gNa}UX3(`C0?(T8>%I9o^K*t>`L}z3$8z?1JYa>ymmhWHYVD`2Pm;mvKw9nq zX8RmzS#%p^s=-%{I6jHEHBub8(&r@0m>ZoZIl2!uU~k4v+N?N}{9f&I$l+8b$B-%Y zYU}cm6r#)c4=xjhviHu?ToHpWjeyr?ftTr{BNuUMk{PtRDPYfzo(G^8Kt~#5Zw$oCEl$!2k7k`0L+^@Fp&O zNvMOXrGu-nmZzhoi{VXNZAl{zzq>OZ=}(X(lIG%4&`W8m5nzd{VVXsJQ}6x7knUBF zOto+uvhdKSy2qn!)>dYM-!0fmO;Gxs83?$-L+Al%rI91FG|<|jZ^=sifzKPJOpr#C z_-G1T&Z0FklBrXp5>dnA3`n~u9F)a*6^Y9LULq7LlUnj<)4}83u}&8NL46(e=LQ>2 zJ<@_0Uo(Vb6wu!{uQR)l6h2nacS8liXbgt-a+)3Wb%Tb529gb6Z#yvbfM@`AAM2}D z!|%Ddh8k#UhXEUW!ht5kk=|B_1luhKK{6sI?`8C!9BHQY8I(xRh#Ujchh4HI3a1Oi zs08FgpX~C?L&$dskE76q_gA9_c07n1 z&rdd2@utr99J)H(NiB+M-WRwjWJW=s`SqZ2mS1ab{Lagf(a8)sAN9v{_0Pq3`~RDf+O3(wrvmWBw*$66m)>om zZw3UnHVzN(KhOQ!xZqERcLnH8=Y4DaXn#7q?Zf}{a#yU~h}BzL#QN>!U!?1we(s8n z8)<+j`u|g#|6G2T25*YQ ztv$oPv;03LxSqZtV^H_V(fbk2wEya5vlk+kp=pj{grssVSnsGY0^`f?ovi K^b-c&-2ETyu-R7t literal 0 HcmV?d00001 diff --git a/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py b/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py index 08caf065..16c7dd89 100644 --- a/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py +++ b/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py @@ -3,7 +3,9 @@ # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). from base64 import b64encode +from decimal import Decimal from os import path +from unittest.mock import Mock from odoo import fields from odoo.exceptions import UserError @@ -45,6 +47,12 @@ class TestAccountStatementImportSheetFile(common.TransactionCase): "account_type": "asset_current", } ) + self.parser = self.env["account.statement.import.sheet.parser"] + # Mock the mapping object to return predefined separators + self.mock_mapping_comma_dot = Mock() + self.mock_mapping_comma_dot._get_float_separators.return_value = (",", ".") + self.mock_mapping_dot_comma = Mock() + self.mock_mapping_dot_comma._get_float_separators.return_value = (".", ",") def _data_file(self, filename, encoding=None): mode = "rt" if encoding else "rb" @@ -538,3 +546,135 @@ class TestAccountStatementImportSheetFile(common.TransactionCase): line2 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 2") self.assertEqual(line2.amount, 1525.00) self.assertEqual(line2.amount_currency, 1000.00) + + def test_import_xlsx_empty_values(self): + sample_statement_map_empty_values = ( + self.AccountStatementImportSheetMapping.create( + { + "name": "Sample Statement with empty values", + "amount_type": "distinct_credit_debit", + "float_decimal_sep": "comma", + "delimiter": "n/a", + "no_header": 0, + "footer_lines_skip_count": 1, + "amount_inverse_sign": 0, + "header_lines_skip_count": 1, + "quotechar": '"', + "float_thousands_sep": "dot", + "reference_column": "REF", + "description_column": "DESCRIPTION", + "amount_credit_column": "DEBIT", + "amount_debit_column": "CREDIT", + "balance_column": "BALANCE", + "timestamp_format": "%d/%m/%Y", + "timestamp_column": "DATE", + } + ) + ) + journal = self.AccountJournal.create( + { + "name": "Bank 2", + "type": "bank", + "code": "BAN2", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + data = self._data_file("fixtures/sample_statement_en_empty_values.xlsx") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/sample_statement_en_empty_values.xlsx", + "statement_file": data, + "sheet_mapping_id": sample_statement_map_empty_values.id, + } + ) + wizard.with_context( + account_statement_import_sheet_file_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 3) + + def test_parse_decimal(self): + # Define a series of test cases + test_cases = [ + ( + "1,234.56", + Decimal("1234.56"), + self.mock_mapping_comma_dot, + ), # standard case with thousands separator + ( + "1,234,567.89", + Decimal("1234567.89"), + self.mock_mapping_comma_dot, + ), # multiple thousands separators + ( + "-1,234.56", + Decimal("-1234.56"), + self.mock_mapping_comma_dot, + ), # negative value + ( + "$1,234.56", + Decimal("1234.56"), + self.mock_mapping_comma_dot, + ), # prefixed with currency symbol + ( + "1,234.56 USD", + Decimal("1234.56"), + self.mock_mapping_comma_dot, + ), # suffixed with currency code + ( + " 1,234.56 ", + Decimal("1234.56"), + self.mock_mapping_comma_dot, + ), # leading and trailing spaces + ( + "not a number", + Decimal("0"), + self.mock_mapping_comma_dot, + ), # non-numeric input + (" ", Decimal("0"), self.mock_mapping_comma_dot), # empty string + ("", Decimal("0"), self.mock_mapping_comma_dot), # empty space + ("USD", Decimal("0"), self.mock_mapping_comma_dot), # empty dolar + ( + "12,34.56", + Decimal("1234.56"), + self.mock_mapping_comma_dot, + ), # unusual thousand separator placement + ( + "1234,567.89", + Decimal("1234567.89"), + self.mock_mapping_comma_dot, + ), # missing one separator + ( + "1234.567,89", + Decimal("1234567.89"), + self.mock_mapping_dot_comma, + ), # inverted separators + ] + + for value, expected, mock_mapping in test_cases: + with self.subTest(value=value): + result = self.parser._parse_decimal(value, mock_mapping) + self.assertEqual(result, expected, f"Failed for value: {value}") + + def test_decimal_and_float_inputs(self): + # Test direct Decimal and float inputs + self.assertEqual( + self.parser._parse_decimal( + Decimal("-1234.56"), self.mock_mapping_comma_dot + ), + Decimal("-1234.56"), + ) + self.assertEqual( + self.parser._parse_decimal(Decimal("1234.56"), self.mock_mapping_comma_dot), + Decimal("1234.56"), + ) + self.assertEqual( + self.parser._parse_decimal(-1234.56, self.mock_mapping_comma_dot), + Decimal("-1234.56"), + ) + self.assertEqual( + self.parser._parse_decimal(1234.56, self.mock_mapping_comma_dot), + Decimal("1234.56"), + ) From 9f853779adb71c16475f4b965b0eb6e63a611795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Felipe=20Mil=C3=A9o?= Date: Sun, 12 May 2024 19:44:55 -0300 Subject: [PATCH 2/2] [REF][account_statement_import_sheet_file] Remove decimal --- .../account_statement_import_sheet_parser.py | 19 +++---- ...est_account_statement_import_sheet_file.py | 50 +++++++++---------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py b/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py index babd4286..4697525c 100644 --- a/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_sheet_file/models/account_statement_import_sheet_parser.py @@ -4,6 +4,7 @@ import itertools import logging +import math import re from collections.abc import Iterable from datetime import datetime @@ -84,8 +85,8 @@ class AccountStatementImportSheetParser(models.TransientModel): balance_end = last_line["balance"] data.update( { - "balance_start": float(balance_start), - "balance_end_real": float(balance_end), + "balance_start": balance_start, + "balance_end_real": balance_end, } ) transactions = list( @@ -337,14 +338,14 @@ class AccountStatementImportSheetParser(models.TransientModel): balance = None if debit_credit is not None: - amount = amount.copy_abs() + amount = abs(amount) if debit_credit == mapping.debit_value: amount = -amount if original_amount: - original_amount = self._parse_decimal( - original_amount, mapping - ).copy_sign(amount) + original_amount = math.copysign( + self._parse_decimal(original_amount, mapping), amount + ) else: original_amount = 0.0 if mapping.amount_inverse_sign: @@ -458,9 +459,9 @@ class AccountStatementImportSheetParser(models.TransientModel): @api.model def _parse_decimal(self, value, mapping): if isinstance(value, Decimal): - return value + return float(value) elif isinstance(value, float): - return Decimal(str(value)) + return value thousands, decimal = mapping._get_float_separators() # Remove all characters except digits, thousands separator, # decimal separator, and signs @@ -472,4 +473,4 @@ class AccountStatementImportSheetParser(models.TransientModel): ) value = value.replace(thousands, "") value = value.replace(decimal, ".") - return Decimal(value) + return float(value) diff --git a/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py b/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py index 16c7dd89..4c767609 100644 --- a/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py +++ b/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py @@ -600,55 +600,55 @@ class TestAccountStatementImportSheetFile(common.TransactionCase): test_cases = [ ( "1,234.56", - Decimal("1234.56"), + 1234.56, self.mock_mapping_comma_dot, ), # standard case with thousands separator ( "1,234,567.89", - Decimal("1234567.89"), + 1234567.89, self.mock_mapping_comma_dot, ), # multiple thousands separators ( "-1,234.56", - Decimal("-1234.56"), + -1234.56, self.mock_mapping_comma_dot, ), # negative value ( "$1,234.56", - Decimal("1234.56"), + 1234.56, self.mock_mapping_comma_dot, ), # prefixed with currency symbol ( "1,234.56 USD", - Decimal("1234.56"), + 1234.56, self.mock_mapping_comma_dot, ), # suffixed with currency code ( " 1,234.56 ", - Decimal("1234.56"), + 1234.56, self.mock_mapping_comma_dot, ), # leading and trailing spaces ( "not a number", - Decimal("0"), + 0, self.mock_mapping_comma_dot, ), # non-numeric input - (" ", Decimal("0"), self.mock_mapping_comma_dot), # empty string - ("", Decimal("0"), self.mock_mapping_comma_dot), # empty space - ("USD", Decimal("0"), self.mock_mapping_comma_dot), # empty dolar + (" ", 0, self.mock_mapping_comma_dot), # empty string + ("", 0, self.mock_mapping_comma_dot), # empty space + ("USD", 0, self.mock_mapping_comma_dot), # empty dolar ( "12,34.56", - Decimal("1234.56"), + 1234.56, self.mock_mapping_comma_dot, ), # unusual thousand separator placement ( "1234,567.89", - Decimal("1234567.89"), + 1234567.89, self.mock_mapping_comma_dot, ), # missing one separator ( "1234.567,89", - Decimal("1234567.89"), + 1234567.89, self.mock_mapping_dot_comma, ), # inverted separators ] @@ -660,21 +660,21 @@ class TestAccountStatementImportSheetFile(common.TransactionCase): def test_decimal_and_float_inputs(self): # Test direct Decimal and float inputs - self.assertEqual( - self.parser._parse_decimal( - Decimal("-1234.56"), self.mock_mapping_comma_dot - ), - Decimal("-1234.56"), - ) - self.assertEqual( - self.parser._parse_decimal(Decimal("1234.56"), self.mock_mapping_comma_dot), - Decimal("1234.56"), - ) self.assertEqual( self.parser._parse_decimal(-1234.56, self.mock_mapping_comma_dot), - Decimal("-1234.56"), + -1234.56, ) self.assertEqual( self.parser._parse_decimal(1234.56, self.mock_mapping_comma_dot), - Decimal("1234.56"), + 1234.56, + ) + self.assertEqual( + self.parser._parse_decimal( + Decimal("-1234.56"), self.mock_mapping_comma_dot + ), + -1234.56, + ) + self.assertEqual( + self.parser._parse_decimal(Decimal("1234.56"), self.mock_mapping_comma_dot), + 1234.56, )