From ee63c3e133fc86b5b583edb6973042d925f9a8a7 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 15 Feb 2013 12:02:14 +0100 Subject: [PATCH 1/8] [IMP] return error messages --- .../parser/file_parser.py | 57 +++++++++++++++---- .../parser/generic_file_parser.py | 9 ++- 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/account_statement_base_import/parser/file_parser.py b/account_statement_base_import/parser/file_parser.py index 139d3721..f3bd76b6 100644 --- a/account_statement_base_import/parser/file_parser.py +++ b/account_statement_base_import/parser/file_parser.py @@ -17,8 +17,8 @@ # along with this program. If not, see . # ############################################################################## - from openerp.tools.translate import _ +from openerp.osv.osv import except_osv import tempfile import datetime from parser import BankStatementImportParser @@ -34,7 +34,8 @@ class FileParser(BankStatementImportParser): Generic abstract class for defining parser for .csv or .xls file format. """ - def __init__(self, parse_name, keys_to_validate=[], ftype='csv', convertion_dict=None, header=None, *args, **kwargs): + def __init__(self, parse_name, keys_to_validate=[], ftype='csv', convertion_dict=None, + header=None, *args, **kwargs): """ :param char: parse_name : The name of the parser :param list: keys_to_validate : contain the key that need to be present in the file @@ -54,7 +55,8 @@ class FileParser(BankStatementImportParser): if ftype in ('csv', 'xls'): self.ftype = ftype else: - raise Exception(_('Invalide file type %s. please use csv or xls') % (ftype)) + raise except_osv(_('User Error'), + _('Invalide file type %s. please use csv or xls') % (ftype)) self.keys_to_validate = keys_to_validate self.convertion_dict = convertion_dict self.fieldnames = header @@ -99,7 +101,8 @@ class FileParser(BankStatementImportParser): parsed_cols = self.result_row_list[0].keys() for col in self.keys_to_validate: if col not in parsed_cols: - raise Exception(_('Column %s not present in file') % (col)) + raise except_osv(_('Invalid data'), + _('Column %s not present in file') % (col)) return True def _post(self, *args, **kwargs): @@ -149,11 +152,27 @@ class FileParser(BankStatementImportParser): for line in result_set: for rule in conversion_rules: if conversion_rules[rule] == datetime.datetime: - date_string = line[rule].split(' ')[0] - line[rule] = datetime.datetime.strptime(date_string, - '%Y-%m-%d') + try: + date_string = line[rule].split(' ')[0] + line[rule] = datetime.datetime.strptime(date_string, + '%Y-%m-%d') + except ValueError, err: + raise except_osv(_('Invalid data'), + _("Date format is not valid." + " It should be YYYY-MM-DD for column: %s" + " value: %s" + " \n line: %s \n Detail: %s") % (rule, line[rule], + line, repr(err))) else: - line[rule] = conversion_rules[rule](line[rule]) + try: + line[rule] = conversion_rules[rule](line[rule]) + except Exception, err: + raise except_osv(_('Invalid data'), + _("Value %s of column % is not valid." + "\n line: %s " + "\n Detail: %s") % (line[rule], rule, + line, repr(err))) + return result_set def _from_xls(self, result_set, conversion_rules): @@ -164,10 +183,26 @@ class FileParser(BankStatementImportParser): for line in result_set: for rule in conversion_rules: if conversion_rules[rule] == datetime.datetime: - t_tuple = xlrd.xldate_as_tuple(line[rule], self._datemode) - line[rule] = datetime.datetime(*t_tuple) + try: + t_tuple = xlrd.xldate_as_tuple(line[rule], self._datemode) + line[rule] = datetime.datetime(*t_tuple) + except Exception, err: + raise except_osv(_('Invalid data'), + _("Date format is not valid. You should modify the cell formatting" + " of your column to be of type date." + " For column: %s" + " value: %s" + " \n line: %s \n Detail: %s") % (rule, line[rule], + line, repr(err))) else: - line[rule] = conversion_rules[rule](line[rule]) + try: + line[rule] = conversion_rules[rule](line[rule]) + except Exception, err: + raise except_osv(_('Invalid data'), + _("Value %s of column % is not valid." + "\n line: %s " + "\n Detail: %s") % (line[rule], rule, + line, repr(err))) return result_set def _cast_rows(self, *args, **kwargs): diff --git a/account_statement_base_import/parser/generic_file_parser.py b/account_statement_base_import/parser/generic_file_parser.py index 84ddc3c6..d2e04e7d 100644 --- a/account_statement_base_import/parser/generic_file_parser.py +++ b/account_statement_base_import/parser/generic_file_parser.py @@ -29,6 +29,11 @@ try: except: raise Exception(_('Please install python lib xlrd')) +def float_or_zero(val): + """ Convertion function used to manage + empty string into float usecase""" + return val and float(val) or 0.0 + class GenericFileParser(FileParser): """ @@ -42,8 +47,8 @@ class GenericFileParser(FileParser): 'ref': unicode, 'label': unicode, 'date': datetime.datetime, - 'amount': float, - 'commission_amount': float + 'amount': float_or_zero, + 'commission_amount': float_or_zero } # Order of cols does not matter but first row of the file has to be header keys_to_validate = ['ref', 'label', 'date', 'amount', 'commission_amount'] From 858812167d33d792f5cee1be4ddaadea17d8ccc1 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 15 Feb 2013 12:02:32 +0100 Subject: [PATCH 2/8] [IMP] error message english readability --- .../parser/file_parser.py | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/account_statement_base_import/parser/file_parser.py b/account_statement_base_import/parser/file_parser.py index f3bd76b6..58da1a3b 100644 --- a/account_statement_base_import/parser/file_parser.py +++ b/account_statement_base_import/parser/file_parser.py @@ -157,22 +157,25 @@ class FileParser(BankStatementImportParser): line[rule] = datetime.datetime.strptime(date_string, '%Y-%m-%d') except ValueError, err: - raise except_osv(_('Invalid data'), - _("Date format is not valid." - " It should be YYYY-MM-DD for column: %s" - " value: %s" - " \n line: %s \n Detail: %s") % (rule, line[rule], - line, repr(err))) + raise except_osv(_("Date format is not valid."), + _(" It should be YYYY-MM-DD for column: %s" + " value: %s \n \n" + " \n Please check the line with ref: %s" + " \n \n Detail: %s") % (rule, + line.get(rule, _('Missing')), + line.get('ref', line), + repr(err))) else: try: line[rule] = conversion_rules[rule](line[rule]) except Exception, err: raise except_osv(_('Invalid data'), - _("Value %s of column % is not valid." - "\n line: %s " - "\n Detail: %s") % (line[rule], rule, - line, repr(err))) - + _("Value %s of column %s is not valid." + "\n Please check the line with ref %s:" + "\n \n Detail: %s") % (line.get(rule, _('Missing')), + rule, + line.get('ref', line), + repr(err))) return result_set def _from_xls(self, result_set, conversion_rules): @@ -187,22 +190,26 @@ class FileParser(BankStatementImportParser): t_tuple = xlrd.xldate_as_tuple(line[rule], self._datemode) line[rule] = datetime.datetime(*t_tuple) except Exception, err: - raise except_osv(_('Invalid data'), - _("Date format is not valid. You should modify the cell formatting" - " of your column to be of type date." - " For column: %s" + raise except_osv(_("Date format is not valid"), + _("Please modify the cell formatting to date format" + " for column: %s" " value: %s" - " \n line: %s \n Detail: %s") % (rule, line[rule], - line, repr(err))) + "\n Please check the line with ref: %s" + "\n \n Detail: %s") % (rule, + line.get(rule, _('Missing')), + line.get('ref', line), + repr(err))) else: try: line[rule] = conversion_rules[rule](line[rule]) except Exception, err: raise except_osv(_('Invalid data'), - _("Value %s of column % is not valid." - "\n line: %s " - "\n Detail: %s") % (line[rule], rule, - line, repr(err))) + _("Value %s of column %s is not valid." + "\n Please check the line with ref %s:"y + "\n \n Detail: %s") % (line.get(rule, _('Missing')), + rule, + line.get('ref', line), + repr(err))) return result_set def _cast_rows(self, *args, **kwargs): From cae935236d7299cbf08ff1e248873884977c5734 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 15 Feb 2013 12:02:52 +0100 Subject: [PATCH 3/8] [IMP] using with statement --- .../parser/file_parser.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/account_statement_base_import/parser/file_parser.py b/account_statement_base_import/parser/file_parser.py index 58da1a3b..5ee1bbfd 100644 --- a/account_statement_base_import/parser/file_parser.py +++ b/account_statement_base_import/parser/file_parser.py @@ -131,17 +131,13 @@ class FileParser(BankStatementImportParser): wb_file.write(self.filebuffer) # We ensure that cursor is at beginig of file wb_file.seek(0) - wb = xlrd.open_workbook(wb_file.name) - self._datemode = wb.datemode - sheet = wb.sheet_by_index(0) - header = sheet.row_values(0) - res = [] - for rownum in range(1, sheet.nrows): - res.append(dict(zip(header, sheet.row_values(rownum)))) - try: - wb_file.close() - except Exception, e: - pass # file is already closed + with xlrd.open_workbook(wb_file.name) as wb: + self._datemode = wb.datemode + sheet = wb.sheet_by_index(0) + header = sheet.row_values(0) + res = [] + for rownum in range(1, sheet.nrows): + res.append(dict(zip(header, sheet.row_values(rownum)))) return res def _from_csv(self, result_set, conversion_rules): From 648d81dd1c9bfc77218cddda5606a641e3d1158b Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 15 Feb 2013 16:29:59 +0100 Subject: [PATCH 4/8] [TYPO] --- account_statement_base_import/parser/file_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/account_statement_base_import/parser/file_parser.py b/account_statement_base_import/parser/file_parser.py index 5ee1bbfd..15e43139 100644 --- a/account_statement_base_import/parser/file_parser.py +++ b/account_statement_base_import/parser/file_parser.py @@ -201,7 +201,7 @@ class FileParser(BankStatementImportParser): except Exception, err: raise except_osv(_('Invalid data'), _("Value %s of column %s is not valid." - "\n Please check the line with ref %s:"y + "\n Please check the line with ref %s:" "\n \n Detail: %s") % (line.get(rule, _('Missing')), rule, line.get('ref', line), From c292adb42a84556140c82b7c4d8140693b8b9241 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 26 Feb 2013 09:14:18 +0100 Subject: [PATCH 5/8] [FIX] coding standard and typo + mutable default value --- .../parser/file_parser.py | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/account_statement_base_import/parser/file_parser.py b/account_statement_base_import/parser/file_parser.py index 15e43139..fb7c0f19 100644 --- a/account_statement_base_import/parser/file_parser.py +++ b/account_statement_base_import/parser/file_parser.py @@ -34,13 +34,13 @@ class FileParser(BankStatementImportParser): Generic abstract class for defining parser for .csv or .xls file format. """ - def __init__(self, parse_name, keys_to_validate=[], ftype='csv', convertion_dict=None, + def __init__(self, parse_name, keys_to_validate=None, ftype='csv', conversion_dict=None, header=None, *args, **kwargs): """ :param char: parse_name : The name of the parser :param list: keys_to_validate : contain the key that need to be present in the file :param char ftype: extension of the file (could be csv or xls) - :param: convertion_dict : keys and type to convert of every column in the file like + :param: conversion_dict : keys and type to convert of every column in the file like { 'ref': unicode, 'label': unicode, @@ -50,15 +50,14 @@ class FileParser(BankStatementImportParser): } :param list: header : specify header fields if the csv file has no header """ - super(FileParser, self).__init__(parse_name, *args, **kwargs) if ftype in ('csv', 'xls'): self.ftype = ftype else: raise except_osv(_('User Error'), - _('Invalide file type %s. please use csv or xls') % (ftype)) - self.keys_to_validate = keys_to_validate - self.convertion_dict = convertion_dict + _('Invalid file type %s. Please use csv or xls') % ftype) + self.keys_to_validate = keys_to_validate if keys_to_validate is not None else [] + self.conversion_dict = conversion_dict self.fieldnames = header self._datemode = 0 # used only for xls documents, # 0 means Windows mode (1900 based dates). @@ -102,7 +101,7 @@ class FileParser(BankStatementImportParser): for col in self.keys_to_validate: if col not in parsed_cols: raise except_osv(_('Invalid data'), - _('Column %s not present in file') % (col)) + _('Column %s not present in file') % col) return True def _post(self, *args, **kwargs): @@ -152,7 +151,7 @@ class FileParser(BankStatementImportParser): date_string = line[rule].split(' ')[0] line[rule] = datetime.datetime.strptime(date_string, '%Y-%m-%d') - except ValueError, err: + except ValueError as err: raise except_osv(_("Date format is not valid."), _(" It should be YYYY-MM-DD for column: %s" " value: %s \n \n" @@ -164,7 +163,7 @@ class FileParser(BankStatementImportParser): else: try: line[rule] = conversion_rules[rule](line[rule]) - except Exception, err: + except Exception as err: raise except_osv(_('Invalid data'), _("Value %s of column %s is not valid." "\n Please check the line with ref %s:" @@ -185,7 +184,7 @@ class FileParser(BankStatementImportParser): try: t_tuple = xlrd.xldate_as_tuple(line[rule], self._datemode) line[rule] = datetime.datetime(*t_tuple) - except Exception, err: + except Exception as err: raise except_osv(_("Date format is not valid"), _("Please modify the cell formatting to date format" " for column: %s" @@ -198,7 +197,7 @@ class FileParser(BankStatementImportParser): else: try: line[rule] = conversion_rules[rule](line[rule]) - except Exception, err: + except Exception as err: raise except_osv(_('Invalid data'), _("Value %s of column %s is not valid." "\n Please check the line with ref %s:" @@ -210,9 +209,9 @@ class FileParser(BankStatementImportParser): def _cast_rows(self, *args, **kwargs): """ - Convert the self.result_row_list using the self.convertion_dict providen. + Convert the self.result_row_list using the self.conversion_dict providen. We call here _from_xls or _from_csv depending on the self.ftype variable. """ func = getattr(self, '_from_%s' % self.ftype) - res = func(self.result_row_list, self.convertion_dict) + res = func(self.result_row_list, self.conversion_dict) return res From 22e49ae4b6f5e464dfbff92f9f531ed2f714042f Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 26 Feb 2013 09:19:44 +0100 Subject: [PATCH 6/8] [FIX] typo convertion/conversion in file_parser derived classes --- account_statement_base_import/parser/generic_file_parser.py | 6 +++--- .../parser/transactionid_file_parser.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/account_statement_base_import/parser/generic_file_parser.py b/account_statement_base_import/parser/generic_file_parser.py index d2e04e7d..46109abf 100644 --- a/account_statement_base_import/parser/generic_file_parser.py +++ b/account_statement_base_import/parser/generic_file_parser.py @@ -30,7 +30,7 @@ except: raise Exception(_('Please install python lib xlrd')) def float_or_zero(val): - """ Convertion function used to manage + """ Conversion function used to manage empty string into float usecase""" return val and float(val) or 0.0 @@ -43,7 +43,7 @@ class GenericFileParser(FileParser): """ def __init__(self, parse_name, ftype='csv'): - convertion_dict = { + conversion_dict = { 'ref': unicode, 'label': unicode, 'date': datetime.datetime, @@ -52,7 +52,7 @@ class GenericFileParser(FileParser): } # Order of cols does not matter but first row of the file has to be header keys_to_validate = ['ref', 'label', 'date', 'amount', 'commission_amount'] - super(GenericFileParser, self).__init__(parse_name, keys_to_validate=keys_to_validate, ftype=ftype, convertion_dict=convertion_dict) + super(GenericFileParser, self).__init__(parse_name, keys_to_validate=keys_to_validate, ftype=ftype, conversion_dict=conversion_dict) @classmethod def parser_for(cls, parser_name): diff --git a/account_statement_transactionid_import/parser/transactionid_file_parser.py b/account_statement_transactionid_import/parser/transactionid_file_parser.py index a4c07845..b584c2ba 100644 --- a/account_statement_transactionid_import/parser/transactionid_file_parser.py +++ b/account_statement_transactionid_import/parser/transactionid_file_parser.py @@ -30,7 +30,7 @@ class TransactionIDFileParser(FileParser): """ def __init__(self, parse_name, ftype='csv'): - convertion_dict = { + conversion_dict = { 'transaction_id': unicode, 'label': unicode, 'date': datetime.datetime, @@ -39,7 +39,8 @@ class TransactionIDFileParser(FileParser): } # Order of cols does not matter but first row of the file has to be header keys_to_validate = ['transaction_id', 'label', 'date', 'amount', 'commission_amount'] - super(TransactionIDFileParser, self).__init__(parse_name, keys_to_validate=keys_to_validate, ftype=ftype, convertion_dict=convertion_dict) + super(TransactionIDFileParser, self).__init__(parse_name, keys_to_validate=keys_to_validate, + ftype=ftype, conversion_dict=conversion_dict) @classmethod def parser_for(cls, parser_name): From 02902e30eed5402cca521f537f8e296296f3a7b0 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 26 Feb 2013 09:22:25 +0100 Subject: [PATCH 7/8] [FIX] coding style in return statement --- account_statement_base_import/parser/generic_file_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/account_statement_base_import/parser/generic_file_parser.py b/account_statement_base_import/parser/generic_file_parser.py index 46109abf..a2ba7a42 100644 --- a/account_statement_base_import/parser/generic_file_parser.py +++ b/account_statement_base_import/parser/generic_file_parser.py @@ -32,7 +32,7 @@ except: def float_or_zero(val): """ Conversion function used to manage empty string into float usecase""" - return val and float(val) or 0.0 + return float(val) if val else 0.0 class GenericFileParser(FileParser): From 74c6caaf6be546867480a2efa23d0a46cc31d647 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 26 Feb 2013 09:27:03 +0100 Subject: [PATCH 8/8] [TYPO] missing return carriage --- account_statement_base_import/parser/file_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/account_statement_base_import/parser/file_parser.py b/account_statement_base_import/parser/file_parser.py index fb7c0f19..938863a3 100644 --- a/account_statement_base_import/parser/file_parser.py +++ b/account_statement_base_import/parser/file_parser.py @@ -50,6 +50,7 @@ class FileParser(BankStatementImportParser): } :param list: header : specify header fields if the csv file has no header """ + super(FileParser, self).__init__(parse_name, *args, **kwargs) if ftype in ('csv', 'xls'): self.ftype = ftype