From d6edc8d4ab22a3ca02be6c22ed586ca3432a1ebb Mon Sep 17 00:00:00 2001 From: Jared Kipe Date: Tue, 2 Jul 2019 13:30:36 -0700 Subject: [PATCH 1/5] Initial commit of `attachment_minio` for 11.0 --- attachment_minio/__init__.py | 2 + attachment_minio/__manifest__.py | 51 +++++++++++ attachment_minio/models/__init__.py | 1 + attachment_minio/models/ir_attachment.py | 103 +++++++++++++++++++++++ attachment_minio/s3uri.py | 22 +++++ 5 files changed, 179 insertions(+) create mode 100755 attachment_minio/__init__.py create mode 100755 attachment_minio/__manifest__.py create mode 100644 attachment_minio/models/__init__.py create mode 100644 attachment_minio/models/ir_attachment.py create mode 100644 attachment_minio/s3uri.py diff --git a/attachment_minio/__init__.py b/attachment_minio/__init__.py new file mode 100755 index 00000000..899bcc97 --- /dev/null +++ b/attachment_minio/__init__.py @@ -0,0 +1,2 @@ +from . import models + diff --git a/attachment_minio/__manifest__.py b/attachment_minio/__manifest__.py new file mode 100755 index 00000000..8bd4e98b --- /dev/null +++ b/attachment_minio/__manifest__.py @@ -0,0 +1,51 @@ +{ + "name": "Attachment MinIO", + "version": "11.0.1.0.0", + "depends": [ + "base_attachment_object_storage", + ], + "author": "Hibou Corp.", + "license": "AGPL-3", + "description": """ +# Use MinIO (or Amazon S3) for Attachment/filestore + +MinIO provides S3 API compatible storage to scale out without a shared filesystem like NFS. + + +## Setup details + +Before installing this app, you should add several System Parameters. + +Key : Example Value : Default Value + +ir_attachment.location.host : minio.yourdomain.com : _ + +ir_attachment.location.bucket : odoo_prod : _ + +ir_attachment.location.region : us-west-1 : us-west-1 + +ir_attachment.location.access_key : odoo : _ + +ir_attachment.location.secret_key : 123456 : _ + +ir_attachment.location.secure : 1 : _ + + +In general, they should all be specified other than "region" (if you are not using AWS S3) +and "secure" which should be set if the "host" needs to be accessed over SSL/TLS. + +Install `attachment_minio` and during the installation `base_attachment_object_storage` should move +your existing filestore attachment files into the database or object storage. + """, + "summary": "", + "website": "", + "category": 'Tools', + "auto_install": False, + "installable": True, + "application": False, + "external_dependencies": { + "python": [ + "minio", + ], + }, +} diff --git a/attachment_minio/models/__init__.py b/attachment_minio/models/__init__.py new file mode 100644 index 00000000..aaf38a16 --- /dev/null +++ b/attachment_minio/models/__init__.py @@ -0,0 +1 @@ +from . import ir_attachment diff --git a/attachment_minio/models/ir_attachment.py b/attachment_minio/models/ir_attachment.py new file mode 100644 index 00000000..ba987899 --- /dev/null +++ b/attachment_minio/models/ir_attachment.py @@ -0,0 +1,103 @@ +import io +import base64 +import logging +from minio import Minio +from minio.error import NoSuchKey + +from odoo import api, exceptions, models, tools +from ..s3uri import S3Uri + +_logger = logging.getLogger(__name__) + + +class MinioAttachment(models.Model): + _inherit = 'ir.attachment' + + @api.model + def _get_minio_client(self): + params = self.env['ir.config_parameter'].sudo() + host = params.get_param('ir_attachment.location.host') + region = params.get_param('ir_attachment.location.region', 'us-west-1') + access_key = params.get_param('ir_attachment.location.access_key') + secret_key = params.get_param('ir_attachment.location.secret_key') + secure = params.get_param('ir_attachment.location.secure') + if not all((host, access_key, secret_key)): + raise exceptions.UserError('Incorrect configuration of attachment_minio.') + return Minio(host, + access_key=access_key, + secret_key=secret_key, + region=region, + secure=bool(secure)) + + @api.model + def _get_minio_bucket(self, client, name=None): + params = self.env['ir.config_parameter'].sudo() + bucket = name or params.get_param('ir_attachment.location.bucket') + if not bucket: + raise exceptions.UserError('Incorrect configuration of attachment_minio -- Missing bucket.') + if not client.bucket_exists(bucket): + client.make_bucket(bucket) + return bucket + + @api.model + def _get_minio_key(self, sha): + # scatter files across 256 dirs + # This mirrors Odoo's own object storage so that it is easier to migrate + # to or from external storage. + fname = sha[:2] + '/' + sha + return fname + + @api.model + def _get_minio_fname(self, bucket, key): + return 's3://%s/%s' % (bucket, key) + + # core API methods from base_attachment_object_storage + + def _get_stores(self): + res = super(MinioAttachment, self)._get_stores() + res.append('s3') + return res + + @api.model + def _store_file_read(self, fname, bin_size=False): + if fname.startswith('s3://'): + client = self._get_minio_client() + s3uri = S3Uri(fname) + bucket = self._get_minio_bucket(client, name=s3uri.bucket()) + try: + response = client.get_object(bucket, s3uri.item()) + return base64.b64encode(response.read()) + except NoSuchKey: + _logger.info('attachment "%s" missing from remote object storage', (fname, )) + return '' + return super(MinioAttachment, self)._store_file_read(fname, bin_size=bin_size) + + @api.model + def _store_file_write(self, key, bin_data): + if self._storage() == 's3': + client = self._get_minio_client() + bucket = self._get_minio_bucket(client) + minio_key = self._get_minio_key(key) + client.put_object(bucket, minio_key, io.BytesIO(bin_data), len(bin_data)) + return self._get_minio_fname(bucket, minio_key) + return super(MinioAttachment, self)._store_file_write(key, bin_data) + + @api.model + def _store_file_delete(self, fname): + if fname.startswith('s3://'): + client = self._get_minio_client() + try: + s3uri = S3Uri(fname) + except ValueError: + # Cannot delete unparsable file + return True + bucket_name = s3uri.bucket() + if bucket_name == self._get_s3_bucket(client): + try: + client.remove_object(bucket_name, s3uri.item()) + except NoSuchKey: + _logger.info('unable to remove missing attachment "%s" from remote object storage', (fname, )) + else: + _logger.info('skip delete "%s" because of bucket-mismatch', (fname, )) + return + return super(MinioAttachment, self)._store_file_delete(fname) diff --git a/attachment_minio/s3uri.py b/attachment_minio/s3uri.py new file mode 100644 index 00000000..f94df79e --- /dev/null +++ b/attachment_minio/s3uri.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import re + + +class S3Uri(object): + + _url_re = re.compile("^s3:///*([^/]*)/?(.*)", re.IGNORECASE | re.UNICODE) + + def __init__(self, uri): + match = self._url_re.match(uri) + if not match: + raise ValueError("%s: is not a valid S3 URI" % (uri,)) + self._bucket, self._item = match.groups() + + def bucket(self): + return self._bucket + + def item(self): + return self._item From 3aa2819880efe2bef8c70b01419d5ef027fe1fcc Mon Sep 17 00:00:00 2001 From: Jared Kipe Date: Tue, 2 Jul 2019 14:44:30 -0700 Subject: [PATCH 2/5] IMP `attachment_minio` improve requested parameters and bucket creation per testing. --- attachment_minio/__manifest__.py | 4 +++- attachment_minio/models/ir_attachment.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/attachment_minio/__manifest__.py b/attachment_minio/__manifest__.py index 8bd4e98b..89d7402b 100755 --- a/attachment_minio/__manifest__.py +++ b/attachment_minio/__manifest__.py @@ -18,9 +18,11 @@ Before installing this app, you should add several System Parameters. Key : Example Value : Default Value +ir_attachment.location : s3 : _ + ir_attachment.location.host : minio.yourdomain.com : _ -ir_attachment.location.bucket : odoo_prod : _ +ir_attachment.location.bucket : odoo-prod : _ ir_attachment.location.region : us-west-1 : us-west-1 diff --git a/attachment_minio/models/ir_attachment.py b/attachment_minio/models/ir_attachment.py index ba987899..8e5dbba5 100644 --- a/attachment_minio/models/ir_attachment.py +++ b/attachment_minio/models/ir_attachment.py @@ -36,7 +36,8 @@ class MinioAttachment(models.Model): if not bucket: raise exceptions.UserError('Incorrect configuration of attachment_minio -- Missing bucket.') if not client.bucket_exists(bucket): - client.make_bucket(bucket) + region = params.get_param('ir_attachment.location.region', 'us-west-1') + client.make_bucket(bucket, region) return bucket @api.model From d2fc7748fe89fced0120dd1a04345e7c3877c023 Mon Sep 17 00:00:00 2001 From: Jared Kipe Date: Sun, 7 Jul 2019 06:37:17 -0700 Subject: [PATCH 3/5] FIX `attachment_minio` Cannot delete attachments. --- attachment_minio/__manifest__.py | 7 +++++-- attachment_minio/models/ir_attachment.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/attachment_minio/__manifest__.py b/attachment_minio/__manifest__.py index 89d7402b..5b7c2f32 100755 --- a/attachment_minio/__manifest__.py +++ b/attachment_minio/__manifest__.py @@ -7,12 +7,15 @@ "author": "Hibou Corp.", "license": "AGPL-3", "description": """ -# Use MinIO (or Amazon S3) for Attachment/filestore +################################################# +Use MinIO (or Amazon S3) for Attachment/filestore +################################################# MinIO provides S3 API compatible storage to scale out without a shared filesystem like NFS. -## Setup details +Setup details +############# Before installing this app, you should add several System Parameters. diff --git a/attachment_minio/models/ir_attachment.py b/attachment_minio/models/ir_attachment.py index 8e5dbba5..599c3d83 100644 --- a/attachment_minio/models/ir_attachment.py +++ b/attachment_minio/models/ir_attachment.py @@ -93,7 +93,7 @@ class MinioAttachment(models.Model): # Cannot delete unparsable file return True bucket_name = s3uri.bucket() - if bucket_name == self._get_s3_bucket(client): + if bucket_name == self._get_minio_bucket(client): try: client.remove_object(bucket_name, s3uri.item()) except NoSuchKey: From 9b27b97efa813a54ba74240fbec8dab0256549ed Mon Sep 17 00:00:00 2001 From: Jared Kipe Date: Thu, 10 Oct 2019 15:50:12 -0700 Subject: [PATCH 4/5] MIG `attachment_minio` to 12.0 --- attachment_minio/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attachment_minio/__manifest__.py b/attachment_minio/__manifest__.py index 5b7c2f32..cdf2d0e3 100755 --- a/attachment_minio/__manifest__.py +++ b/attachment_minio/__manifest__.py @@ -1,6 +1,6 @@ { "name": "Attachment MinIO", - "version": "11.0.1.0.0", + "version": "12.0.1.0.0", "depends": [ "base_attachment_object_storage", ], From bc7366a5fe68fe1d2955ec29eb2e8a9d159d2e6d Mon Sep 17 00:00:00 2001 From: Jared Kipe Date: Fri, 11 Oct 2019 13:55:05 -0700 Subject: [PATCH 5/5] IMP `attachment_minio` Allow configuration via config file and improve documentation. --- attachment_minio/README.md | 52 +++++++++++++++++++++ attachment_minio/__manifest__.py | 57 ++++++++++++++++-------- attachment_minio/models/ir_attachment.py | 14 +++--- 3 files changed, 98 insertions(+), 25 deletions(-) create mode 100644 attachment_minio/README.md diff --git a/attachment_minio/README.md b/attachment_minio/README.md new file mode 100644 index 00000000..1e32a8e0 --- /dev/null +++ b/attachment_minio/README.md @@ -0,0 +1,52 @@ +# Use MinIO (or Amazon S3) for Attachment/filestore + +MinIO provides S3 API compatible storage to scale out without a shared filesystem like NFS. + +This module will store the bucket used in the attachment database object, thus allowing +you to retain read-only access to the filestore by simply overriding the bucket. + +## Setup details + +Before installing this app, you should add several System Parameters (the most important of +which is `ir_attachment.location`), OR set them through the config file as described later. + +**The in database System Parameters will act as overrides to the Config File versions.** + +| Key | Example Value | Default Value | +|-----------------------------------|---------------|---------------| +| ir_attachment.location | s3 | | +| ir_attachment.location.host | minio:9000 | | +| ir_attachment.location.bucket | odoo | | +| ir_attachment.location.region | us-west-1 | us-west-1 | +| ir_attachment.location.access_key | minio | | +| ir_attachment.location.secret_key | minio_secret | | +| ir_attachment.location.secure | 1 | | + +**Config File:** + +``` +attachment_minio_host = minio:9000 +attachment_minio_region = us-west-1 +attachment_minio_access_key = minio +attachment_minio_secret_key = minio_secret +attachment_minio_bucket = odoo +attachment_minio_secure = False +``` + +In general, they should all be specified other than "region" (if you are not using AWS S3) +and "secure" which should be set if the "host" needs to be accessed over SSL/TLS. + +Install `attachment_minio` and during the installation `base_attachment_object_storage` should move +your existing filestore attachment files into the database or object storage. + +For example, you can run a shell command like the following to set the parameter: + +``` +env['ir.config_parameter'].set_param('ir_attachment.location', 's3') +# If already installed... +# env['ir.attachment'].force_storage() +env.cr.commit() +``` + +If `attachment_minio` is not already installed, you can then install it and the migration +should be noted in the logs. **Ensure that the timeouts are long enough that the migration can finish.** diff --git a/attachment_minio/__manifest__.py b/attachment_minio/__manifest__.py index cdf2d0e3..973a8b1d 100755 --- a/attachment_minio/__manifest__.py +++ b/attachment_minio/__manifest__.py @@ -7,40 +7,59 @@ "author": "Hibou Corp.", "license": "AGPL-3", "description": """ -################################################# -Use MinIO (or Amazon S3) for Attachment/filestore -################################################# +# Use MinIO (or Amazon S3) for Attachment/filestore MinIO provides S3 API compatible storage to scale out without a shared filesystem like NFS. +This module will store the bucket used in the attachment database object, thus allowing +you to retain read-only access to the filestore by simply overriding the bucket. -Setup details -############# +## Setup details -Before installing this app, you should add several System Parameters. +Before installing this app, you should add several System Parameters (the most important of +which is `ir_attachment.location`), OR set them through the config file as described later. -Key : Example Value : Default Value +**The in database System Parameters will act as overrides to the Config File versions.** -ir_attachment.location : s3 : _ +| Key | Example Value | Default Value | +|-----------------------------------|---------------|---------------| +| ir_attachment.location | s3 | | +| ir_attachment.location.host | minio:9000 | | +| ir_attachment.location.bucket | odoo | | +| ir_attachment.location.region | us-west-1 | us-west-1 | +| ir_attachment.location.access_key | minio | | +| ir_attachment.location.secret_key | minio_secret | | +| ir_attachment.location.secure | 1 | | -ir_attachment.location.host : minio.yourdomain.com : _ - -ir_attachment.location.bucket : odoo-prod : _ - -ir_attachment.location.region : us-west-1 : us-west-1 - -ir_attachment.location.access_key : odoo : _ - -ir_attachment.location.secret_key : 123456 : _ - -ir_attachment.location.secure : 1 : _ +**Config File:** +``` +attachment_minio_host = minio:9000 +attachment_minio_region = us-west-1 +attachment_minio_access_key = minio +attachment_minio_secret_key = minio_secret +attachment_minio_bucket = odoo +attachment_minio_secure = False +``` In general, they should all be specified other than "region" (if you are not using AWS S3) and "secure" which should be set if the "host" needs to be accessed over SSL/TLS. Install `attachment_minio` and during the installation `base_attachment_object_storage` should move your existing filestore attachment files into the database or object storage. + +For example, you can run a shell command like the following to set the parameter: + +``` +env['ir.config_parameter'].set_param('ir_attachment.location', 's3') +# If already installed... +# env['ir.attachment'].force_storage() +env.cr.commit() +``` + +If `attachment_minio` is not already installed, you can then install it and the migration +should be noted in the logs. **Ensure that the timeouts are long enough that the migration can finish.** + """, "summary": "", "website": "", diff --git a/attachment_minio/models/ir_attachment.py b/attachment_minio/models/ir_attachment.py index 599c3d83..08122566 100644 --- a/attachment_minio/models/ir_attachment.py +++ b/attachment_minio/models/ir_attachment.py @@ -15,12 +15,13 @@ class MinioAttachment(models.Model): @api.model def _get_minio_client(self): + config = tools.config params = self.env['ir.config_parameter'].sudo() - host = params.get_param('ir_attachment.location.host') - region = params.get_param('ir_attachment.location.region', 'us-west-1') - access_key = params.get_param('ir_attachment.location.access_key') - secret_key = params.get_param('ir_attachment.location.secret_key') - secure = params.get_param('ir_attachment.location.secure') + host = params.get_param('ir_attachment.location.host') or config.get('attachment_minio_host') + region = params.get_param('ir_attachment.location.region') or config.get('attachment_minio_region', 'us-west-1') + access_key = params.get_param('ir_attachment.location.access_key') or config.get('attachment_minio_access_key') + secret_key = params.get_param('ir_attachment.location.secret_key') or config.get('attachment_minio_secret_key') + secure = params.get_param('ir_attachment.location.secure') or config.get('attachment_minio_secure') if not all((host, access_key, secret_key)): raise exceptions.UserError('Incorrect configuration of attachment_minio.') return Minio(host, @@ -31,8 +32,9 @@ class MinioAttachment(models.Model): @api.model def _get_minio_bucket(self, client, name=None): + config = tools.config params = self.env['ir.config_parameter'].sudo() - bucket = name or params.get_param('ir_attachment.location.bucket') + bucket = name or params.get_param('ir_attachment.location.bucket') or config.get('attachment_minio_bucket') if not bucket: raise exceptions.UserError('Incorrect configuration of attachment_minio -- Missing bucket.') if not client.bucket_exists(bucket):