Merge branch 'mig/14.0/attachment_minio' into '14.0'

mig/14.0/attachment_minio into 14.0

See merge request hibou-io/hibou-odoo/suite!823

attachment_minio/README.md

@@ -0,0 +1,60 @@
+# Use MinIO (or Amazon S3) for Attachment/filestore
+
+MinIO provides S3 API compatible storage to scale out without a shared filesystem like NFS.
+
+This module will store the bucket used in the attachment database object, thus allowing 
+you to retain read-only access to the filestore by simply overriding the bucket.
+
+## Setup details
+
+Before installing this app, you should add several System Parameters (the most important of
+which is `ir_attachment.location`), OR set them through the config file as described later.
+
+**The in database System Parameters will act as overrides to the Config File versions.**
+
+| Key                               | Example Value | Default Value |
+|-----------------------------------|---------------|---------------|
+| ir_attachment.location            | s3            |               |
+| ir_attachment.location.host       | minio:9000    |               |
+| ir_attachment.location.bucket     | odoo          |               |
+| ir_attachment.location.region     | us-west-1     | us-west-1     |
+| ir_attachment.location.access_key | minio         |               |
+| ir_attachment.location.secret_key | minio_secret  |               |
+| ir_attachment.location.secure     | 1             |               |
+
+**Config File:**
+
+```
+attachment_minio_host = minio:9000
+attachment_minio_region = us-west-1
+attachment_minio_access_key = minio
+attachment_minio_secret_key = minio_secret
+attachment_minio_bucket = odoo
+attachment_minio_secure = False
+```
+
+In general, they should all be specified other than "region" (if you are not using AWS S3) 
+and "secure" which should be set if the "host" needs to be accessed over SSL/TLS.
+
+Install `attachment_minio` and during the installation `base_attachment_object_storage` should move 
+your existing filestore attachment files into the database or object storage.
+
+For example, you can run a shell command like the following to set the parameter:
+
+```
+env['ir.config_parameter'].set_param('ir_attachment.location', 's3')
+# If already installed...
+# env['ir.attachment'].force_storage()
+env.cr.commit()
+```
+
+If `attachment_minio` is not already installed, you can then install it and the migration 
+should be noted in the logs.  **Ensure that the timeouts are long enough that the migration can finish.**
+
+### Base Setup
+
+This module utilizes `base_attachment_object_storage`
+
+The System Parameter `ir_attachment.storage.force.database` can be customized to 
+force storage of files in the database. See the documentation of the module 
+`base_attachment_object_storage`.

attachment_minio/__init__.py

@@ -0,0 +1,2 @@
+from . import models
+

attachment_minio/__manifest__.py

@@ -0,0 +1,75 @@
+{
+    "name": "Attachment MinIO",
+    "version": "14.0.1.0.0",
+    "depends": [
+        "base_attachment_object_storage",
+    ],
+    "author": "Hibou Corp.",
+    "license": "AGPL-3",
+    "description": """
+# Use MinIO (or Amazon S3) for Attachment/filestore
+
+MinIO provides S3 API compatible storage to scale out without a shared filesystem like NFS.
+
+This module will store the bucket used in the attachment database object, thus allowing 
+you to retain read-only access to the filestore by simply overriding the bucket.
+
+## Setup details
+
+Before installing this app, you should add several System Parameters (the most important of
+which is `ir_attachment.location`), OR set them through the config file as described later.
+
+**The in database System Parameters will act as overrides to the Config File versions.**
+
+| Key                               | Example Value | Default Value |
+|-----------------------------------|---------------|---------------|
+| ir_attachment.location            | s3            |               |
+| ir_attachment.location.host       | minio:9000    |               |
+| ir_attachment.location.bucket     | odoo          |               |
+| ir_attachment.location.region     | us-west-1     | us-west-1     |
+| ir_attachment.location.access_key | minio         |               |
+| ir_attachment.location.secret_key | minio_secret  |               |
+| ir_attachment.location.secure     | 1             |               |
+
+**Config File:**
+
+```
+attachment_minio_host = minio:9000
+attachment_minio_region = us-west-1
+attachment_minio_access_key = minio
+attachment_minio_secret_key = minio_secret
+attachment_minio_bucket = odoo
+attachment_minio_secure = False
+```
+
+In general, they should all be specified other than "region" (if you are not using AWS S3) 
+and "secure" which should be set if the "host" needs to be accessed over SSL/TLS.
+
+Install `attachment_minio` and during the installation `base_attachment_object_storage` should move 
+your existing filestore attachment files into the database or object storage.
+
+For example, you can run a shell command like the following to set the parameter:
+
+```
+env['ir.config_parameter'].set_param('ir_attachment.location', 's3')
+# If already installed...
+# env['ir.attachment'].force_storage()
+env.cr.commit()
+```
+
+If `attachment_minio` is not already installed, you can then install it and the migration 
+should be noted in the logs.  **Ensure that the timeouts are long enough that the migration can finish.**
+
+    """,
+    "summary": "",
+    "website": "",
+    "category": 'Tools',
+    "auto_install": False,
+    "installable": True,
+    "application": False,
+    "external_dependencies": {
+        "python": [
+            "minio",
+        ],
+    },
+}

attachment_minio/migrations/14.0.0.0.1/post-migration.py

@@ -0,0 +1,67 @@
+# Copyright 2020 Hibou Corp.
+# Copyright 2016-2019 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import logging
+
+from contextlib import closing
+
+import odoo
+
+_logger = logging.getLogger(__name__)
+
+
+def migrate(cr, version):
+    if not version:
+        return
+    cr.execute("""
+        SELECT value FROM ir_config_parameter
+        WHERE key = 'ir_attachment.location'
+    """)
+    row = cr.fetchone()
+
+    if row[0] == 's3':
+        uid = odoo.SUPERUSER_ID
+        registry = odoo.modules.registry.Registry(cr.dbname)
+        new_cr = registry.cursor()
+        with closing(new_cr):
+            with odoo.api.Environment.manage():
+                env = odoo.api.Environment(new_cr, uid, {})
+                store_local = env['ir.attachment'].search(
+                    [('store_fname', '=like', 's3://%'),
+                     '|', ('res_model', '=', 'ir.ui.view'),
+                          ('res_field', 'in', ['image_small',
+                                               'image_medium',
+                                               'web_icon_data',
+                                               # image.mixin sizes
+                                               # image_128 is essentially image_medium
+                                               'image_128',
+                                               # depending on use case, these may need migrated/moved
+                                               # 'image_256',
+                                               # 'image_512',
+                                               ])
+                     ],
+                )
+
+                _logger.info(
+                    'Moving %d attachments from S3 to DB for fast access',
+                    len(store_local)
+                )
+                for attachment_id in store_local.ids:
+                    # force re-storing the document, will move
+                    # it from the object storage to the database
+
+                    # This is a trick to avoid having the 'datas' function
+                    # fields computed for every attachment on each
+                    # iteration of the loop.  The former issue being that
+                    # it reads the content of the file of ALL the
+                    # attachments on each loop.
+                    try:
+                        env.clear()
+                        attachment = env['ir.attachment'].browse(attachment_id)
+                        _logger.info('Moving attachment %s (id: %s)',
+                                     attachment.name, attachment.id)
+                        attachment.write({'datas': attachment.datas})
+                        new_cr.commit()
+                    except:
+                        new_cr.rollback()

attachment_minio/models/__init__.py

@@ -0,0 +1 @@
+from . import ir_attachment

attachment_minio/models/ir_attachment.py

@@ -0,0 +1,111 @@
+# Copyright 2020 Hibou Corp.
+# Copyright 2016 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import io
+import logging
+from minio import Minio
+from minio.error import NoSuchKey
+
+from odoo import api, exceptions, models, tools
+from ..s3uri import S3Uri
+
+_logger = logging.getLogger(__name__)
+
+
+class MinioAttachment(models.Model):
+    _inherit = 'ir.attachment'
+
+    @api.model
+    def _get_minio_client(self):
+        config = tools.config
+        params = self.env['ir.config_parameter'].sudo()
+        host = params.get_param('ir_attachment.location.host') or config.get('attachment_minio_host')
+        region = params.get_param('ir_attachment.location.region') or config.get('attachment_minio_region', 'us-west-1')
+        access_key = params.get_param('ir_attachment.location.access_key') or config.get('attachment_minio_access_key')
+        secret_key = params.get_param('ir_attachment.location.secret_key') or config.get('attachment_minio_secret_key')
+        secure = params.get_param('ir_attachment.location.secure') or config.get('attachment_minio_secure')
+        if not all((host, access_key, secret_key)):
+            raise exceptions.UserError('Incorrect configuration of attachment_minio.')
+        return Minio(host,
+                           access_key=access_key,
+                           secret_key=secret_key,
+                           region=region,
+                           secure=bool(secure))
+
+    @api.model
+    def _get_minio_bucket(self, client, name=None):
+        config = tools.config
+        params = self.env['ir.config_parameter'].sudo()
+        bucket = name or params.get_param('ir_attachment.location.bucket') or config.get('attachment_minio_bucket')
+        if not bucket:
+            raise exceptions.UserError('Incorrect configuration of attachment_minio -- Missing bucket.')
+        if not client.bucket_exists(bucket):
+            region = params.get_param('ir_attachment.location.region', 'us-west-1')
+            client.make_bucket(bucket, region)
+        return bucket
+
+    @api.model
+    def _get_minio_key(self, sha):
+        # scatter files across 256 dirs
+        # This mirrors Odoo's own object storage so that it is easier to migrate
+        # to or from external storage.
+        fname = sha[:2] + '/' + sha
+        return fname
+
+    @api.model
+    def _get_minio_fname(self, bucket, key):
+        return 's3://%s/%s' % (bucket, key)
+
+    # core API methods from base_attachment_object_storage
+
+    def _get_stores(self):
+        res = super(MinioAttachment, self)._get_stores()
+        res.append('s3')
+        return res
+
+    @api.model
+    def _store_file_read(self, fname, bin_size=False):
+        if fname.startswith('s3://'):
+            client = self._get_minio_client()
+            s3uri = S3Uri(fname)
+            bucket = self._get_minio_bucket(client, name=s3uri.bucket())
+            try:
+                response = client.get_object(bucket, s3uri.item())
+                return response.read()
+            except NoSuchKey:
+                _logger.info('attachment "%s" missing from remote object storage', (fname, ))
+            return ''
+        return super(MinioAttachment, self)._store_file_read(fname, bin_size=bin_size)
+
+    @api.model
+    def _store_file_write(self, key, bin_data):
+        if self._storage() == 's3':
+            client = self._get_minio_client()
+            bucket = self._get_minio_bucket(client)
+            minio_key = self._get_minio_key(key)
+            with io.BytesIO(bin_data) as bin_data_io:
+                client.put_object(bucket, minio_key, bin_data_io, len(bin_data),
+                                  content_type=self.mimetype)
+            return self._get_minio_fname(bucket, minio_key)
+        return super(MinioAttachment, self)._store_file_write(key, bin_data)
+
+    @api.model
+    def _store_file_delete(self, fname):
+        if fname.startswith('s3://'):
+            client = self._get_minio_client()
+            try:
+                s3uri = S3Uri(fname)
+            except ValueError:
+                # Cannot delete unparsable file
+                return True
+            bucket_name = s3uri.bucket()
+            if bucket_name == self._get_minio_bucket(client):
+                try:
+                    client.remove_object(bucket_name, s3uri.item())
+                except NoSuchKey:
+                    _logger.info('unable to remove missing attachment "%s" from remote object storage', (fname, ))
+            else:
+                _logger.info('skip delete "%s" because of bucket-mismatch', (fname, ))
+            return
+        return super(MinioAttachment, self)._store_file_delete(fname)

attachment_minio/s3uri.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import re
+
+
+class S3Uri(object):
+
+    _url_re = re.compile("^s3:///*([^/]*)/?(.*)", re.IGNORECASE | re.UNICODE)
+
+    def __init__(self, uri):
+        match = self._url_re.match(uri)
+        if not match:
+            raise ValueError("%s: is not a valid S3 URI" % (uri,))
+        self._bucket, self._item = match.groups()
+
+    def bucket(self):
+        return self._bucket
+
+    def item(self):
+        return self._item