diff --git a/.gitmodules b/.gitmodules index 04d086b5..5309e9a7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,10 @@ [submodule "external/hibou-oca/social"] path = external/hibou-oca/social url = https://github.com/hibou-io/oca-social.git +[submodule "external/camptocamp-cloud-platform"] + path = external/camptocamp-cloud-platform + url = https://github.com/hibou-io/camptocamp-cloud-platform +[submodule "external/hibou-shipbox"] + path = external/hibou-shipbox + url = https://gitlab.com/hibou-io/hibou-odoo/shipbox.git +>>>>>>> .gitmodules diff --git a/attachment_minio/README.md b/attachment_minio/README.md new file mode 100644 index 00000000..3a81269c --- /dev/null +++ b/attachment_minio/README.md @@ -0,0 +1,60 @@ +# Use MinIO (or Amazon S3) for Attachment/filestore + +MinIO provides S3 API compatible storage to scale out without a shared filesystem like NFS. + +This module will store the bucket used in the attachment database object, thus allowing +you to retain read-only access to the filestore by simply overriding the bucket. + +## Setup details + +Before installing this app, you should add several System Parameters (the most important of +which is `ir_attachment.location`), OR set them through the config file as described later. + +**The in database System Parameters will act as overrides to the Config File versions.** + +| Key | Example Value | Default Value | +|-----------------------------------|---------------|---------------| +| ir_attachment.location | s3 | | +| ir_attachment.location.host | minio:9000 | | +| ir_attachment.location.bucket | odoo | | +| ir_attachment.location.region | us-west-1 | us-west-1 | +| ir_attachment.location.access_key | minio | | +| ir_attachment.location.secret_key | minio_secret | | +| ir_attachment.location.secure | 1 | | + +**Config File:** + +``` +attachment_minio_host = minio:9000 +attachment_minio_region = us-west-1 +attachment_minio_access_key = minio +attachment_minio_secret_key = minio_secret +attachment_minio_bucket = odoo +attachment_minio_secure = False +``` + +In general, they should all be specified other than "region" (if you are not using AWS S3) +and "secure" which should be set if the "host" needs to be accessed over SSL/TLS. + +Install `attachment_minio` and during the installation `base_attachment_object_storage` should move +your existing filestore attachment files into the database or object storage. + +For example, you can run a shell command like the following to set the parameter: + +``` +env['ir.config_parameter'].set_param('ir_attachment.location', 's3') +# If already installed... +# env['ir.attachment'].force_storage() +env.cr.commit() +``` + +If `attachment_minio` is not already installed, you can then install it and the migration +should be noted in the logs. **Ensure that the timeouts are long enough that the migration can finish.** + +### Base Setup + +This module utilizes `base_attachment_object_storage` + +The System Parameter `ir_attachment.storage.force.database` can be customized to +force storage of files in the database. See the documentation of the module +`base_attachment_object_storage`. diff --git a/attachment_minio/__init__.py b/attachment_minio/__init__.py new file mode 100755 index 00000000..899bcc97 --- /dev/null +++ b/attachment_minio/__init__.py @@ -0,0 +1,2 @@ +from . import models + diff --git a/attachment_minio/__manifest__.py b/attachment_minio/__manifest__.py new file mode 100755 index 00000000..fe230ae8 --- /dev/null +++ b/attachment_minio/__manifest__.py @@ -0,0 +1,75 @@ +{ + "name": "Attachment MinIO", + "version": "13.0.1.0.0", + "depends": [ + "base_attachment_object_storage", + ], + "author": "Hibou Corp.", + "license": "AGPL-3", + "description": """ +# Use MinIO (or Amazon S3) for Attachment/filestore + +MinIO provides S3 API compatible storage to scale out without a shared filesystem like NFS. + +This module will store the bucket used in the attachment database object, thus allowing +you to retain read-only access to the filestore by simply overriding the bucket. + +## Setup details + +Before installing this app, you should add several System Parameters (the most important of +which is `ir_attachment.location`), OR set them through the config file as described later. + +**The in database System Parameters will act as overrides to the Config File versions.** + +| Key | Example Value | Default Value | +|-----------------------------------|---------------|---------------| +| ir_attachment.location | s3 | | +| ir_attachment.location.host | minio:9000 | | +| ir_attachment.location.bucket | odoo | | +| ir_attachment.location.region | us-west-1 | us-west-1 | +| ir_attachment.location.access_key | minio | | +| ir_attachment.location.secret_key | minio_secret | | +| ir_attachment.location.secure | 1 | | + +**Config File:** + +``` +attachment_minio_host = minio:9000 +attachment_minio_region = us-west-1 +attachment_minio_access_key = minio +attachment_minio_secret_key = minio_secret +attachment_minio_bucket = odoo +attachment_minio_secure = False +``` + +In general, they should all be specified other than "region" (if you are not using AWS S3) +and "secure" which should be set if the "host" needs to be accessed over SSL/TLS. + +Install `attachment_minio` and during the installation `base_attachment_object_storage` should move +your existing filestore attachment files into the database or object storage. + +For example, you can run a shell command like the following to set the parameter: + +``` +env['ir.config_parameter'].set_param('ir_attachment.location', 's3') +# If already installed... +# env['ir.attachment'].force_storage() +env.cr.commit() +``` + +If `attachment_minio` is not already installed, you can then install it and the migration +should be noted in the logs. **Ensure that the timeouts are long enough that the migration can finish.** + + """, + "summary": "", + "website": "", + "category": 'Tools', + "auto_install": False, + "installable": True, + "application": False, + "external_dependencies": { + "python": [ + "minio", + ], + }, +} diff --git a/attachment_minio/migrations/13.0.0.0.1/post-migration.py b/attachment_minio/migrations/13.0.0.0.1/post-migration.py new file mode 100644 index 00000000..b8a62e3a --- /dev/null +++ b/attachment_minio/migrations/13.0.0.0.1/post-migration.py @@ -0,0 +1,67 @@ +# Copyright 2020 Hibou Corp. +# Copyright 2016-2019 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import logging + +from contextlib import closing + +import odoo + +_logger = logging.getLogger(__name__) + + +def migrate(cr, version): + if not version: + return + cr.execute(""" + SELECT value FROM ir_config_parameter + WHERE key = 'ir_attachment.location' + """) + row = cr.fetchone() + + if row[0] == 's3': + uid = odoo.SUPERUSER_ID + registry = odoo.modules.registry.Registry(cr.dbname) + new_cr = registry.cursor() + with closing(new_cr): + with odoo.api.Environment.manage(): + env = odoo.api.Environment(new_cr, uid, {}) + store_local = env['ir.attachment'].search( + [('store_fname', '=like', 's3://%'), + '|', ('res_model', '=', 'ir.ui.view'), + ('res_field', 'in', ['image_small', + 'image_medium', + 'web_icon_data', + # image.mixin sizes + # image_128 is essentially image_medium + 'image_128', + # depending on use case, these may need migrated/moved + # 'image_256', + # 'image_512', + ]) + ], + ) + + _logger.info( + 'Moving %d attachments from S3 to DB for fast access', + len(store_local) + ) + for attachment_id in store_local.ids: + # force re-storing the document, will move + # it from the object storage to the database + + # This is a trick to avoid having the 'datas' function + # fields computed for every attachment on each + # iteration of the loop. The former issue being that + # it reads the content of the file of ALL the + # attachments on each loop. + try: + env.clear() + attachment = env['ir.attachment'].browse(attachment_id) + _logger.info('Moving attachment %s (id: %s)', + attachment.name, attachment.id) + attachment.write({'datas': attachment.datas}) + new_cr.commit() + except: + new_cr.rollback() diff --git a/attachment_minio/models/__init__.py b/attachment_minio/models/__init__.py new file mode 100644 index 00000000..aaf38a16 --- /dev/null +++ b/attachment_minio/models/__init__.py @@ -0,0 +1 @@ +from . import ir_attachment diff --git a/attachment_minio/models/ir_attachment.py b/attachment_minio/models/ir_attachment.py new file mode 100644 index 00000000..70ba2640 --- /dev/null +++ b/attachment_minio/models/ir_attachment.py @@ -0,0 +1,112 @@ +# Copyright 2020 Hibou Corp. +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import io +import base64 +import logging +from minio import Minio +from minio.error import NoSuchKey + +from odoo import api, exceptions, models, tools +from ..s3uri import S3Uri + +_logger = logging.getLogger(__name__) + + +class MinioAttachment(models.Model): + _inherit = 'ir.attachment' + + @api.model + def _get_minio_client(self): + config = tools.config + params = self.env['ir.config_parameter'].sudo() + host = params.get_param('ir_attachment.location.host') or config.get('attachment_minio_host') + region = params.get_param('ir_attachment.location.region') or config.get('attachment_minio_region', 'us-west-1') + access_key = params.get_param('ir_attachment.location.access_key') or config.get('attachment_minio_access_key') + secret_key = params.get_param('ir_attachment.location.secret_key') or config.get('attachment_minio_secret_key') + secure = params.get_param('ir_attachment.location.secure') or config.get('attachment_minio_secure') + if not all((host, access_key, secret_key)): + raise exceptions.UserError('Incorrect configuration of attachment_minio.') + return Minio(host, + access_key=access_key, + secret_key=secret_key, + region=region, + secure=bool(secure)) + + @api.model + def _get_minio_bucket(self, client, name=None): + config = tools.config + params = self.env['ir.config_parameter'].sudo() + bucket = name or params.get_param('ir_attachment.location.bucket') or config.get('attachment_minio_bucket') + if not bucket: + raise exceptions.UserError('Incorrect configuration of attachment_minio -- Missing bucket.') + if not client.bucket_exists(bucket): + region = params.get_param('ir_attachment.location.region', 'us-west-1') + client.make_bucket(bucket, region) + return bucket + + @api.model + def _get_minio_key(self, sha): + # scatter files across 256 dirs + # This mirrors Odoo's own object storage so that it is easier to migrate + # to or from external storage. + fname = sha[:2] + '/' + sha + return fname + + @api.model + def _get_minio_fname(self, bucket, key): + return 's3://%s/%s' % (bucket, key) + + # core API methods from base_attachment_object_storage + + def _get_stores(self): + res = super(MinioAttachment, self)._get_stores() + res.append('s3') + return res + + @api.model + def _store_file_read(self, fname, bin_size=False): + if fname.startswith('s3://'): + client = self._get_minio_client() + s3uri = S3Uri(fname) + bucket = self._get_minio_bucket(client, name=s3uri.bucket()) + try: + response = client.get_object(bucket, s3uri.item()) + return base64.b64encode(response.read()) + except NoSuchKey: + _logger.info('attachment "%s" missing from remote object storage', (fname, )) + return '' + return super(MinioAttachment, self)._store_file_read(fname, bin_size=bin_size) + + @api.model + def _store_file_write(self, key, bin_data): + if self._storage() == 's3': + client = self._get_minio_client() + bucket = self._get_minio_bucket(client) + minio_key = self._get_minio_key(key) + with io.BytesIO(bin_data) as bin_data_io: + client.put_object(bucket, minio_key, bin_data_io, len(bin_data), + content_type=self.mimetype) + return self._get_minio_fname(bucket, minio_key) + return super(MinioAttachment, self)._store_file_write(key, bin_data) + + @api.model + def _store_file_delete(self, fname): + if fname.startswith('s3://'): + client = self._get_minio_client() + try: + s3uri = S3Uri(fname) + except ValueError: + # Cannot delete unparsable file + return True + bucket_name = s3uri.bucket() + if bucket_name == self._get_minio_bucket(client): + try: + client.remove_object(bucket_name, s3uri.item()) + except NoSuchKey: + _logger.info('unable to remove missing attachment "%s" from remote object storage', (fname, )) + else: + _logger.info('skip delete "%s" because of bucket-mismatch', (fname, )) + return + return super(MinioAttachment, self)._store_file_delete(fname) diff --git a/attachment_minio/s3uri.py b/attachment_minio/s3uri.py new file mode 100644 index 00000000..f94df79e --- /dev/null +++ b/attachment_minio/s3uri.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import re + + +class S3Uri(object): + + _url_re = re.compile("^s3:///*([^/]*)/?(.*)", re.IGNORECASE | re.UNICODE) + + def __init__(self, uri): + match = self._url_re.match(uri) + if not match: + raise ValueError("%s: is not a valid S3 URI" % (uri,)) + self._bucket, self._item = match.groups() + + def bucket(self): + return self._bucket + + def item(self): + return self._item diff --git a/auth_admin/__init__.py b/auth_admin/__init__.py new file mode 100755 index 00000000..e4f4917a --- /dev/null +++ b/auth_admin/__init__.py @@ -0,0 +1,3 @@ +from . import controllers +from . import models +from . import wizard diff --git a/auth_admin/__manifest__.py b/auth_admin/__manifest__.py new file mode 100755 index 00000000..1420a7a3 --- /dev/null +++ b/auth_admin/__manifest__.py @@ -0,0 +1,28 @@ +{ + 'name': 'Auth Admin', + 'author': 'Hibou Corp. ', + 'category': 'Hidden', + 'version': '13.0.1.0.0', + 'description': + """ +Login as other user +=================== + +Provides a way for an authenticated user, with certain permissions, to login as a different user. +Can also create a URL that logs in as that user. + +Out of the box, only allows you to generate a login for an 'External User', e.g. portal users. + +*2017-11-15* New button to generate the login on the Portal User Wizard (Action on Contact) + """, + 'depends': [ + 'base', + 'website', + 'portal', + ], + 'auto_install': False, + 'data': [ + 'views/res_users.xml', + 'wizard/portal_wizard_views.xml', + ], +} diff --git a/auth_admin/controllers/__init__.py b/auth_admin/controllers/__init__.py new file mode 100755 index 00000000..757b12a1 --- /dev/null +++ b/auth_admin/controllers/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +from . import main diff --git a/auth_admin/controllers/main.py b/auth_admin/controllers/main.py new file mode 100755 index 00000000..cf0ac9df --- /dev/null +++ b/auth_admin/controllers/main.py @@ -0,0 +1,39 @@ +from odoo import http, exceptions +from ..models.res_users import check_admin_auth_login + +from logging import getLogger +_logger = getLogger(__name__) + + +class AuthAdmin(http.Controller): + + @http.route(['/auth_admin'], type='http', auth='public', website=True) + def index(self, *args, **post): + u = post.get('u') + e = post.get('e') + o = post.get('o') + h = post.get('h') + + if not all([u, e, o, h]): + exceptions.Warning('Invalid Request') + + u = str(u) + e = str(e) + o = str(o) + h = str(h) + + try: + user = check_admin_auth_login(http.request.env, u, e, o, h) + + http.request.session.uid = user.id + http.request.session.login = user.login + http.request.session.password = '' + http.request.session.auth_admin = int(o) + http.request.uid = user.id + uid = http.request.session.authenticate(http.request.session.db, user.login, 'x') + if uid is not False: + http.request.params['login_success'] = True + return http.redirect_with_hash('/my/home') + return http.local_redirect('/my/home') + except (exceptions.Warning, ) as e: + return http.Response(e.message, status=400) diff --git a/auth_admin/models/__init__.py b/auth_admin/models/__init__.py new file mode 100755 index 00000000..88351653 --- /dev/null +++ b/auth_admin/models/__init__.py @@ -0,0 +1 @@ +from . import res_users diff --git a/auth_admin/models/res_users.py b/auth_admin/models/res_users.py new file mode 100755 index 00000000..074abca0 --- /dev/null +++ b/auth_admin/models/res_users.py @@ -0,0 +1,92 @@ +from odoo import models, api, exceptions +from odoo.http import request +from datetime import datetime +from time import mktime +import hmac +from hashlib import sha256 + +from logging import getLogger +_logger = getLogger(__name__) + + +def admin_auth_generate_login(env, user): + """ + Generates a URL to allow the current user to login as the portal user. + + :param env: Odoo environment + :param user: `res.users` in + :return: + """ + if not env['res.partner'].check_access_rights('write'): + return None + u = str(user.id) + now = datetime.utcnow() + fifteen = int(mktime(now.timetuple())) + (15 * 60) + e = str(fifteen) + o = str(env.uid) + + config = env['ir.config_parameter'].sudo() + key = str(config.search([('key', '=', 'database.secret')], limit=1).value) + h = hmac.new(key.encode(), (u + e + o).encode(), sha256) + + base_url = str(config.search([('key', '=', 'web.base.url')], limit=1).value) + + _logger.warn('login url for user id: ' + u + ' original user id: ' + o) + + return base_url + '/auth_admin?u=' + u + '&e=' + e + '&o=' + o + '&h=' + h.hexdigest() + + +def check_admin_auth_login(env, u_user_id, e_expires, o_org_user_id, hash_): + """ + Checks that the parameters are valid and that the user exists. + + :param env: Odoo environment + :param u_user_id: Desired user id to login as. + :param e_expires: Expiration timestamp + :param o_org_user_id: Original user id. + :param hash_: HMAC generated hash + :return: `res.users` + """ + + now = datetime.utcnow() + now = int(mktime(now.timetuple())) + fifteen = now + (15 * 60) + + config = env['ir.config_parameter'].sudo() + key = str(config.search([('key', '=', 'database.secret')], limit=1).value) + + myh = hmac.new(key.encode(), str(str(u_user_id) + str(e_expires) + str(o_org_user_id)).encode(), sha256) + + if not hmac.compare_digest(hash_, myh.hexdigest()): + raise exceptions.AccessDenied('Invalid Request') + + if not (now <= int(e_expires) <= fifteen): + raise exceptions.AccessDenied('Expired') + + user = env['res.users'].sudo().search([('id', '=', int(u_user_id))], limit=1) + if not user.id: + raise exceptions.AccessDenied('Invalid User') + return user + + +class ResUsers(models.Model): + _inherit = 'res.users' + + def admin_auth_generate_login(self): + self.ensure_one() + + login_url = admin_auth_generate_login(self.env, self) + if login_url: + raise exceptions.Warning(login_url) + + return False + + def _check_credentials(self, password): + try: + return super(ResUsers, self)._check_credentials(password) + except exceptions.AccessDenied: + if request and hasattr(request, 'session') and request.session.get('auth_admin'): + _logger.warn('_check_credentials for user id: ' + \ + str(request.session.uid) + ' original user id: ' + str(request.session.auth_admin)) + else: + raise diff --git a/auth_admin/views/res_users.xml b/auth_admin/views/res_users.xml new file mode 100755 index 00000000..b818c870 --- /dev/null +++ b/auth_admin/views/res_users.xml @@ -0,0 +1,14 @@ + + + + auth_admin.res.users.tree + res.users + + + + + +

+ +
+ +

+ + +

+

+ + +

+

+ + +

+

+ + +

+

+

You can use Markdown for formatting.

+