diff --git a/attachment_minio/README.md b/attachment_minio/README.md new file mode 100644 index 00000000..3a81269c --- /dev/null +++ b/attachment_minio/README.md @@ -0,0 +1,60 @@ +# Use MinIO (or Amazon S3) for Attachment/filestore + +MinIO provides S3 API compatible storage to scale out without a shared filesystem like NFS. + +This module will store the bucket used in the attachment database object, thus allowing +you to retain read-only access to the filestore by simply overriding the bucket. + +## Setup details + +Before installing this app, you should add several System Parameters (the most important of +which is `ir_attachment.location`), OR set them through the config file as described later. + +**The in database System Parameters will act as overrides to the Config File versions.** + +| Key | Example Value | Default Value | +|-----------------------------------|---------------|---------------| +| ir_attachment.location | s3 | | +| ir_attachment.location.host | minio:9000 | | +| ir_attachment.location.bucket | odoo | | +| ir_attachment.location.region | us-west-1 | us-west-1 | +| ir_attachment.location.access_key | minio | | +| ir_attachment.location.secret_key | minio_secret | | +| ir_attachment.location.secure | 1 | | + +**Config File:** + +``` +attachment_minio_host = minio:9000 +attachment_minio_region = us-west-1 +attachment_minio_access_key = minio +attachment_minio_secret_key = minio_secret +attachment_minio_bucket = odoo +attachment_minio_secure = False +``` + +In general, they should all be specified other than "region" (if you are not using AWS S3) +and "secure" which should be set if the "host" needs to be accessed over SSL/TLS. + +Install `attachment_minio` and during the installation `base_attachment_object_storage` should move +your existing filestore attachment files into the database or object storage. + +For example, you can run a shell command like the following to set the parameter: + +``` +env['ir.config_parameter'].set_param('ir_attachment.location', 's3') +# If already installed... +# env['ir.attachment'].force_storage() +env.cr.commit() +``` + +If `attachment_minio` is not already installed, you can then install it and the migration +should be noted in the logs. **Ensure that the timeouts are long enough that the migration can finish.** + +### Base Setup + +This module utilizes `base_attachment_object_storage` + +The System Parameter `ir_attachment.storage.force.database` can be customized to +force storage of files in the database. See the documentation of the module +`base_attachment_object_storage`. diff --git a/attachment_minio/__init__.py b/attachment_minio/__init__.py new file mode 100755 index 00000000..899bcc97 --- /dev/null +++ b/attachment_minio/__init__.py @@ -0,0 +1,2 @@ +from . import models + diff --git a/attachment_minio/__manifest__.py b/attachment_minio/__manifest__.py new file mode 100755 index 00000000..495041dc --- /dev/null +++ b/attachment_minio/__manifest__.py @@ -0,0 +1,75 @@ +{ + "name": "Attachment MinIO", + "version": "14.0.1.0.0", + "depends": [ + "base_attachment_object_storage", + ], + "author": "Hibou Corp.", + "license": "AGPL-3", + "description": """ +# Use MinIO (or Amazon S3) for Attachment/filestore + +MinIO provides S3 API compatible storage to scale out without a shared filesystem like NFS. + +This module will store the bucket used in the attachment database object, thus allowing +you to retain read-only access to the filestore by simply overriding the bucket. + +## Setup details + +Before installing this app, you should add several System Parameters (the most important of +which is `ir_attachment.location`), OR set them through the config file as described later. + +**The in database System Parameters will act as overrides to the Config File versions.** + +| Key | Example Value | Default Value | +|-----------------------------------|---------------|---------------| +| ir_attachment.location | s3 | | +| ir_attachment.location.host | minio:9000 | | +| ir_attachment.location.bucket | odoo | | +| ir_attachment.location.region | us-west-1 | us-west-1 | +| ir_attachment.location.access_key | minio | | +| ir_attachment.location.secret_key | minio_secret | | +| ir_attachment.location.secure | 1 | | + +**Config File:** + +``` +attachment_minio_host = minio:9000 +attachment_minio_region = us-west-1 +attachment_minio_access_key = minio +attachment_minio_secret_key = minio_secret +attachment_minio_bucket = odoo +attachment_minio_secure = False +``` + +In general, they should all be specified other than "region" (if you are not using AWS S3) +and "secure" which should be set if the "host" needs to be accessed over SSL/TLS. + +Install `attachment_minio` and during the installation `base_attachment_object_storage` should move +your existing filestore attachment files into the database or object storage. + +For example, you can run a shell command like the following to set the parameter: + +``` +env['ir.config_parameter'].set_param('ir_attachment.location', 's3') +# If already installed... +# env['ir.attachment'].force_storage() +env.cr.commit() +``` + +If `attachment_minio` is not already installed, you can then install it and the migration +should be noted in the logs. **Ensure that the timeouts are long enough that the migration can finish.** + + """, + "summary": "", + "website": "", + "category": 'Tools', + "auto_install": False, + "installable": True, + "application": False, + "external_dependencies": { + "python": [ + "minio", + ], + }, +} diff --git a/attachment_minio/migrations/14.0.0.0.1/post-migration.py b/attachment_minio/migrations/14.0.0.0.1/post-migration.py new file mode 100644 index 00000000..b8a62e3a --- /dev/null +++ b/attachment_minio/migrations/14.0.0.0.1/post-migration.py @@ -0,0 +1,67 @@ +# Copyright 2020 Hibou Corp. +# Copyright 2016-2019 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import logging + +from contextlib import closing + +import odoo + +_logger = logging.getLogger(__name__) + + +def migrate(cr, version): + if not version: + return + cr.execute(""" + SELECT value FROM ir_config_parameter + WHERE key = 'ir_attachment.location' + """) + row = cr.fetchone() + + if row[0] == 's3': + uid = odoo.SUPERUSER_ID + registry = odoo.modules.registry.Registry(cr.dbname) + new_cr = registry.cursor() + with closing(new_cr): + with odoo.api.Environment.manage(): + env = odoo.api.Environment(new_cr, uid, {}) + store_local = env['ir.attachment'].search( + [('store_fname', '=like', 's3://%'), + '|', ('res_model', '=', 'ir.ui.view'), + ('res_field', 'in', ['image_small', + 'image_medium', + 'web_icon_data', + # image.mixin sizes + # image_128 is essentially image_medium + 'image_128', + # depending on use case, these may need migrated/moved + # 'image_256', + # 'image_512', + ]) + ], + ) + + _logger.info( + 'Moving %d attachments from S3 to DB for fast access', + len(store_local) + ) + for attachment_id in store_local.ids: + # force re-storing the document, will move + # it from the object storage to the database + + # This is a trick to avoid having the 'datas' function + # fields computed for every attachment on each + # iteration of the loop. The former issue being that + # it reads the content of the file of ALL the + # attachments on each loop. + try: + env.clear() + attachment = env['ir.attachment'].browse(attachment_id) + _logger.info('Moving attachment %s (id: %s)', + attachment.name, attachment.id) + attachment.write({'datas': attachment.datas}) + new_cr.commit() + except: + new_cr.rollback() diff --git a/attachment_minio/models/__init__.py b/attachment_minio/models/__init__.py new file mode 100644 index 00000000..aaf38a16 --- /dev/null +++ b/attachment_minio/models/__init__.py @@ -0,0 +1 @@ +from . import ir_attachment diff --git a/attachment_minio/models/ir_attachment.py b/attachment_minio/models/ir_attachment.py new file mode 100644 index 00000000..2d963f8f --- /dev/null +++ b/attachment_minio/models/ir_attachment.py @@ -0,0 +1,111 @@ +# Copyright 2020 Hibou Corp. +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import io +import logging +from minio import Minio +from minio.error import NoSuchKey + +from odoo import api, exceptions, models, tools +from ..s3uri import S3Uri + +_logger = logging.getLogger(__name__) + + +class MinioAttachment(models.Model): + _inherit = 'ir.attachment' + + @api.model + def _get_minio_client(self): + config = tools.config + params = self.env['ir.config_parameter'].sudo() + host = params.get_param('ir_attachment.location.host') or config.get('attachment_minio_host') + region = params.get_param('ir_attachment.location.region') or config.get('attachment_minio_region', 'us-west-1') + access_key = params.get_param('ir_attachment.location.access_key') or config.get('attachment_minio_access_key') + secret_key = params.get_param('ir_attachment.location.secret_key') or config.get('attachment_minio_secret_key') + secure = params.get_param('ir_attachment.location.secure') or config.get('attachment_minio_secure') + if not all((host, access_key, secret_key)): + raise exceptions.UserError('Incorrect configuration of attachment_minio.') + return Minio(host, + access_key=access_key, + secret_key=secret_key, + region=region, + secure=bool(secure)) + + @api.model + def _get_minio_bucket(self, client, name=None): + config = tools.config + params = self.env['ir.config_parameter'].sudo() + bucket = name or params.get_param('ir_attachment.location.bucket') or config.get('attachment_minio_bucket') + if not bucket: + raise exceptions.UserError('Incorrect configuration of attachment_minio -- Missing bucket.') + if not client.bucket_exists(bucket): + region = params.get_param('ir_attachment.location.region', 'us-west-1') + client.make_bucket(bucket, region) + return bucket + + @api.model + def _get_minio_key(self, sha): + # scatter files across 256 dirs + # This mirrors Odoo's own object storage so that it is easier to migrate + # to or from external storage. + fname = sha[:2] + '/' + sha + return fname + + @api.model + def _get_minio_fname(self, bucket, key): + return 's3://%s/%s' % (bucket, key) + + # core API methods from base_attachment_object_storage + + def _get_stores(self): + res = super(MinioAttachment, self)._get_stores() + res.append('s3') + return res + + @api.model + def _store_file_read(self, fname, bin_size=False): + if fname.startswith('s3://'): + client = self._get_minio_client() + s3uri = S3Uri(fname) + bucket = self._get_minio_bucket(client, name=s3uri.bucket()) + try: + response = client.get_object(bucket, s3uri.item()) + return response.read() + except NoSuchKey: + _logger.info('attachment "%s" missing from remote object storage', (fname, )) + return '' + return super(MinioAttachment, self)._store_file_read(fname, bin_size=bin_size) + + @api.model + def _store_file_write(self, key, bin_data): + if self._storage() == 's3': + client = self._get_minio_client() + bucket = self._get_minio_bucket(client) + minio_key = self._get_minio_key(key) + with io.BytesIO(bin_data) as bin_data_io: + client.put_object(bucket, minio_key, bin_data_io, len(bin_data), + content_type=self.mimetype) + return self._get_minio_fname(bucket, minio_key) + return super(MinioAttachment, self)._store_file_write(key, bin_data) + + @api.model + def _store_file_delete(self, fname): + if fname.startswith('s3://'): + client = self._get_minio_client() + try: + s3uri = S3Uri(fname) + except ValueError: + # Cannot delete unparsable file + return True + bucket_name = s3uri.bucket() + if bucket_name == self._get_minio_bucket(client): + try: + client.remove_object(bucket_name, s3uri.item()) + except NoSuchKey: + _logger.info('unable to remove missing attachment "%s" from remote object storage', (fname, )) + else: + _logger.info('skip delete "%s" because of bucket-mismatch', (fname, )) + return + return super(MinioAttachment, self)._store_file_delete(fname) diff --git a/attachment_minio/s3uri.py b/attachment_minio/s3uri.py new file mode 100644 index 00000000..f94df79e --- /dev/null +++ b/attachment_minio/s3uri.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import re + + +class S3Uri(object): + + _url_re = re.compile("^s3:///*([^/]*)/?(.*)", re.IGNORECASE | re.UNICODE) + + def __init__(self, uri): + match = self._url_re.match(uri) + if not match: + raise ValueError("%s: is not a valid S3 URI" % (uri,)) + self._bucket, self._item = match.groups() + + def bucket(self): + return self._bucket + + def item(self): + return self._item