audrius/suite
1
0
Fork 1
mirror of https://gitlab.com/hibou-io/hibou-odoo/suite.git synced 2025-01-20 12:37:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'mig/14.0/attachment_minio' into '14.0'

Browse Source 
mig/14.0/attachment_minio into 14.0

See merge request hibou-io/hibou-odoo/suite!823
This commit is contained in:
Jared Kipe
2021-03-21 20:24:13 +00:00
parent df40a152af a717f9c4bf
commit 2d3c211063
7 changed files with 338 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
attachment_minio/README.md Normal file
View File

@@ -0,0 +1,60 @@
# Use MinIO (or Amazon S3) for Attachment/filestore
MinIO provides S3 API compatible storage to scale out without a shared filesystem like NFS.
This module will store the bucket used in the attachment database object, thus allowing
you to retain read-only access to the filestore by simply overriding the bucket.
## Setup details
Before installing this app, you should add several System Parameters (the most important of
which is `ir_attachment.location`), OR set them through the config file as described later.
**The in database System Parameters will act as overrides to the Config File versions.**
| Key | Example Value | Default Value |
|-----------------------------------|---------------|---------------|
| ir_attachment.location | s3 | |
| ir_attachment.location.host | minio:9000 | |
| ir_attachment.location.bucket | odoo | |
| ir_attachment.location.region | us-west-1 | us-west-1 |
| ir_attachment.location.access_key | minio | |
| ir_attachment.location.secret_key | minio_secret | |
| ir_attachment.location.secure | 1 | |
**Config File:**
```
attachment_minio_host = minio:9000
attachment_minio_region = us-west-1
attachment_minio_access_key = minio
attachment_minio_secret_key = minio_secret
attachment_minio_bucket = odoo
attachment_minio_secure = False
```
In general, they should all be specified other than "region" (if you are not using AWS S3)
and "secure" which should be set if the "host" needs to be accessed over SSL/TLS.
Install `attachment_minio` and during the installation `base_attachment_object_storage` should move
your existing filestore attachment files into the database or object storage.
For example, you can run a shell command like the following to set the parameter:
```
env['ir.config_parameter'].set_param('ir_attachment.location', 's3')
# If already installed...
# env['ir.attachment'].force_storage()
env.cr.commit()
```
If `attachment_minio` is not already installed, you can then install it and the migration
should be noted in the logs. **Ensure that the timeouts are long enough that the migration can finish.**
### Base Setup
This module utilizes `base_attachment_object_storage`
The System Parameter `ir_attachment.storage.force.database` can be customized to
force storage of files in the database. See the documentation of the module
`base_attachment_object_storage`.

2
attachment_minio/__init__.py Executable file
View File

@@ -0,0 +1,2 @@
from . import models

75
attachment_minio/__manifest__.py Executable file
View File

@@ -0,0 +1,75 @@
{
"name": "Attachment MinIO",
"version": "14.0.1.0.0",
"depends": [
"base_attachment_object_storage",
],
"author": "Hibou Corp.",
"license": "AGPL-3",
"description": """
# Use MinIO (or Amazon S3) for Attachment/filestore
MinIO provides S3 API compatible storage to scale out without a shared filesystem like NFS.
This module will store the bucket used in the attachment database object, thus allowing
you to retain read-only access to the filestore by simply overriding the bucket.
## Setup details
Before installing this app, you should add several System Parameters (the most important of
which is `ir_attachment.location`), OR set them through the config file as described later.
**The in database System Parameters will act as overrides to the Config File versions.**
| Key | Example Value | Default Value |
|-----------------------------------|---------------|---------------|
| ir_attachment.location | s3 | |
| ir_attachment.location.host | minio:9000 | |
| ir_attachment.location.bucket | odoo | |
| ir_attachment.location.region | us-west-1 | us-west-1 |
| ir_attachment.location.access_key | minio | |
| ir_attachment.location.secret_key | minio_secret | |
| ir_attachment.location.secure | 1 | |
**Config File:**
```
attachment_minio_host = minio:9000
attachment_minio_region = us-west-1
attachment_minio_access_key = minio
attachment_minio_secret_key = minio_secret
attachment_minio_bucket = odoo
attachment_minio_secure = False
```
In general, they should all be specified other than "region" (if you are not using AWS S3)
and "secure" which should be set if the "host" needs to be accessed over SSL/TLS.
Install `attachment_minio` and during the installation `base_attachment_object_storage` should move
your existing filestore attachment files into the database or object storage.
For example, you can run a shell command like the following to set the parameter:
```
env['ir.config_parameter'].set_param('ir_attachment.location', 's3')
# If already installed...
# env['ir.attachment'].force_storage()
env.cr.commit()
```
If `attachment_minio` is not already installed, you can then install it and the migration
should be noted in the logs. **Ensure that the timeouts are long enough that the migration can finish.**
""",
"summary": "",
"website": "",
"category": 'Tools',
"auto_install": False,
"installable": True,
"application": False,
"external_dependencies": {
"python": [
"minio",
],
},
}

67
attachment_minio/migrations/14.0.0.0.1/post-migration.py Normal file
View File

@@ -0,0 +1,67 @@
# Copyright 2020 Hibou Corp.
# Copyright 2016-2019 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
import logging
from contextlib import closing
import odoo
_logger = logging.getLogger(__name__)
def migrate(cr, version):
if not version:
return
cr.execute("""
SELECT value FROM ir_config_parameter
WHERE key = 'ir_attachment.location'
""")
row = cr.fetchone()
if row[0] == 's3':
uid = odoo.SUPERUSER_ID
registry = odoo.modules.registry.Registry(cr.dbname)
new_cr = registry.cursor()
with closing(new_cr):
with odoo.api.Environment.manage():
env = odoo.api.Environment(new_cr, uid, {})
store_local = env['ir.attachment'].search(
[('store_fname', '=like', 's3://%'),
'|', ('res_model', '=', 'ir.ui.view'),
('res_field', 'in', ['image_small',
'image_medium',
'web_icon_data',
# image.mixin sizes
# image_128 is essentially image_medium
'image_128',
# depending on use case, these may need migrated/moved
# 'image_256',
# 'image_512',
])
],
)
_logger.info(
'Moving %d attachments from S3 to DB for fast access',
len(store_local)
)
for attachment_id in store_local.ids:
# force re-storing the document, will move
# it from the object storage to the database
# This is a trick to avoid having the 'datas' function
# fields computed for every attachment on each
# iteration of the loop. The former issue being that
# it reads the content of the file of ALL the
# attachments on each loop.
try:
env.clear()
attachment = env['ir.attachment'].browse(attachment_id)
_logger.info('Moving attachment %s (id: %s)',
attachment.name, attachment.id)
attachment.write({'datas': attachment.datas})
new_cr.commit()
except:
new_cr.rollback()

1
attachment_minio/models/__init__.py Normal file
View File

@@ -0,0 +1 @@
from . import ir_attachment

111
attachment_minio/models/ir_attachment.py Normal file
View File

@@ -0,0 +1,111 @@
# Copyright 2020 Hibou Corp.
# Copyright 2016 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
import io
import logging
from minio import Minio
from minio.error import NoSuchKey
from odoo import api, exceptions, models, tools
from ..s3uri import S3Uri
_logger = logging.getLogger(__name__)
class MinioAttachment(models.Model):
_inherit = 'ir.attachment'
@api.model
def _get_minio_client(self):
config = tools.config
params = self.env['ir.config_parameter'].sudo()
host = params.get_param('ir_attachment.location.host') or config.get('attachment_minio_host')
region = params.get_param('ir_attachment.location.region') or config.get('attachment_minio_region', 'us-west-1')
access_key = params.get_param('ir_attachment.location.access_key') or config.get('attachment_minio_access_key')
secret_key = params.get_param('ir_attachment.location.secret_key') or config.get('attachment_minio_secret_key')
secure = params.get_param('ir_attachment.location.secure') or config.get('attachment_minio_secure')
if not all((host, access_key, secret_key)):
raise exceptions.UserError('Incorrect configuration of attachment_minio.')
return Minio(host,
access_key=access_key,
secret_key=secret_key,
region=region,
secure=bool(secure))
@api.model
def _get_minio_bucket(self, client, name=None):
config = tools.config
params = self.env['ir.config_parameter'].sudo()
bucket = name or params.get_param('ir_attachment.location.bucket') or config.get('attachment_minio_bucket')
if not bucket:
raise exceptions.UserError('Incorrect configuration of attachment_minio -- Missing bucket.')
if not client.bucket_exists(bucket):
region = params.get_param('ir_attachment.location.region', 'us-west-1')
client.make_bucket(bucket, region)
return bucket
@api.model
def _get_minio_key(self, sha):
# scatter files across 256 dirs
# This mirrors Odoo's own object storage so that it is easier to migrate
# to or from external storage.
fname = sha[:2] + '/' + sha
return fname
@api.model
def _get_minio_fname(self, bucket, key):
return 's3://%s/%s' % (bucket, key)
# core API methods from base_attachment_object_storage
def _get_stores(self):
res = super(MinioAttachment, self)._get_stores()
res.append('s3')
return res
@api.model
def _store_file_read(self, fname, bin_size=False):
if fname.startswith('s3://'):
client = self._get_minio_client()
s3uri = S3Uri(fname)
bucket = self._get_minio_bucket(client, name=s3uri.bucket())
try:
response = client.get_object(bucket, s3uri.item())
return response.read()
except NoSuchKey:
_logger.info('attachment "%s" missing from remote object storage', (fname, ))
return ''
return super(MinioAttachment, self)._store_file_read(fname, bin_size=bin_size)
@api.model
def _store_file_write(self, key, bin_data):
if self._storage() == 's3':
client = self._get_minio_client()
bucket = self._get_minio_bucket(client)
minio_key = self._get_minio_key(key)
with io.BytesIO(bin_data) as bin_data_io:
client.put_object(bucket, minio_key, bin_data_io, len(bin_data),
content_type=self.mimetype)
return self._get_minio_fname(bucket, minio_key)
return super(MinioAttachment, self)._store_file_write(key, bin_data)
@api.model
def _store_file_delete(self, fname):
if fname.startswith('s3://'):
client = self._get_minio_client()
try:
s3uri = S3Uri(fname)
except ValueError:
# Cannot delete unparsable file
return True
bucket_name = s3uri.bucket()
if bucket_name == self._get_minio_bucket(client):
try:
client.remove_object(bucket_name, s3uri.item())
except NoSuchKey:
_logger.info('unable to remove missing attachment "%s" from remote object storage', (fname, ))
else:
_logger.info('skip delete "%s" because of bucket-mismatch', (fname, ))
return
return super(MinioAttachment, self)._store_file_delete(fname)

22
attachment_minio/s3uri.py Normal file
View File

@@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
import re
class S3Uri(object):
_url_re = re.compile("^s3:///*([^/]*)/?(.*)", re.IGNORECASE | re.UNICODE)
def __init__(self, uri):
match = self._url_re.match(uri)
if not match:
raise ValueError("%s: is not a valid S3 URI" % (uri,))
self._bucket, self._item = match.groups()
def bucket(self):
return self._bucket
def item(self):
return self._item