From d72d7520941e5f3869a59315124ee25645fc12ac Mon Sep 17 00:00:00 2001 From: Repumba Date: Mon, 12 Dec 2022 10:11:36 +0100 Subject: [PATCH 01/33] migration and background in progress --- mwdb/app.py | 6 + mwdb/model/__init__.py | 3 +- mwdb/model/file.py | 127 +++++++++++++++ .../3c610b0ddebc_add_related_files_table.py | 36 +++++ mwdb/model/object.py | 5 + mwdb/resources/file.py | 145 +++++++++++++++++- mwdb/web/src/commons/api/index.js | 4 + 7 files changed, 324 insertions(+), 2 deletions(-) create mode 100644 mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py diff --git a/mwdb/app.py b/mwdb/app.py index 384db861a..451d32efa 100755 --- a/mwdb/app.py +++ b/mwdb/app.py @@ -44,6 +44,8 @@ FileDownloadZipResource, FileItemResource, FileResource, + RelatedFileDownloadResource, + RelatedFileItemResource, ) from mwdb.resources.group import GroupListResource, GroupMemberResource, GroupResource from mwdb.resources.karton import KartonAnalysisResource, KartonObjectResource @@ -259,6 +261,10 @@ def require_auth(): api.add_resource(FileDownloadResource, "/file//download") api.add_resource(FileDownloadZipResource, "/file//download/zip") +# RelatedFiles endpoints +api.add_resource(RelatedFileItemResource, "/related_file/") +api.add_resource(RelatedFileDownloadResource, "/related_file//download") + # Config endpoints api.add_resource(ConfigResource, "/config") api.add_resource(ConfigStatsResource, "/config/stats") diff --git a/mwdb/model/__init__.py b/mwdb/model/__init__.py index 2d6323e44..443f33ffc 100644 --- a/mwdb/model/__init__.py +++ b/mwdb/model/__init__.py @@ -47,7 +47,7 @@ def after_cursor_execute(conn, cursor, statement, parameters, context, executema from .blob import TextBlob # noqa: E402 from .comment import Comment # noqa: E402 from .config import Config, StaticConfig # noqa: E402 -from .file import File # noqa: E402 +from .file import File, RelatedFile # noqa: E402 from .group import Group, Member # noqa: E402 from .karton import KartonAnalysis, karton_object # noqa: E402 from .oauth import OpenIDProvider, OpenIDUserIdentity # noqa: E402 @@ -76,6 +76,7 @@ def after_cursor_execute(conn, cursor, statement, parameters, context, executema "OpenIDProvider", "OpenIDUserIdentity", "relation", + "RelatedFile", "QuickQuery", "Tag", "User", diff --git a/mwdb/model/file.py b/mwdb/model/file.py index f8d3b3090..7990cd210 100644 --- a/mwdb/model/file.py +++ b/mwdb/model/file.py @@ -9,6 +9,7 @@ from sqlalchemy.dialects.postgresql.array import ARRAY from sqlalchemy.ext.mutable import MutableList from werkzeug.utils import secure_filename +from flask import g from mwdb.core.auth import AuthScope, generate_token, verify_token from mwdb.core.config import StorageProviderType, app_config @@ -328,3 +329,129 @@ def get_by_download_token(download_token): def _send_to_karton(self): return send_file_to_karton(self) + + +class RelatedFile(db.Model): + __tablename__ = "related_file" + + id = db.Column(db.Integer, primary_key=True) + object_id = db.Column( + db.Integer, + db.ForeignKey("object.id", ondelete="CASCADE"), + nullable=False, + ) + file_name = db.Column(db.String, nullable=False) + file_size = db.Column(db.Integer, nullable=False) + sha256 = db.Column(db.String, nullable=False) + + related_object = db.relationship( + "Object", + back_populates="related_files", + lazy=True, + ) + + def _calculate_path(self): + if app_config.mwdb.storage_provider == StorageProviderType.DISK: + upload_path = ( + "related_files" + if app_config.mwdb.related_files_folder == "" + else app_config.mwdb.related_files_folder + "/related_files" + ) + elif app_config.mwdb.storage_provider == StorageProviderType.S3: + upload_path = "related_files/" + else: + raise RuntimeError( + f"StorageProvider {app_config.mwdb.storage_provider} is not supported" + ) + + sample_sha256 = self.sha256.lower() + + if app_config.mwdb.hash_pathing: + # example: related_files/9/f/8/6/9f86d0818... + upload_path = os.path.join(upload_path, *list(sample_sha256)[0:4]) + + if app_config.mwdb.storage_provider == StorageProviderType.DISK: + upload_path = os.path.abspath(upload_path) + os.makedirs(upload_path, mode=0o755, exist_ok=True) + + return os.path.join(upload_path, sample_sha256) + + @classmethod + def create( + cls, + file_name, + file_stream, + related_object, + ): + file_stream.seek(0, os.SEEK_END) + file_size = file_stream.tell() + if file_size == 0: + raise EmptyFileError + + sha256 = calc_hash(file_stream, hashlib.sha256(), lambda h: h.hexdigest()) + + new_related_file = ( + db.session.query(RelatedFile).filter(RelatedFile.sha256 == sha256).first() + ) + + # If file already exists + if new_related_file is not None: + return + + new_related_file = RelatedFile( + object_id=related_object.id, + file_name=secure_filename(file_name), + file_size=file_size, + sha256=sha256, + ) + + file_stream.seek(0, os.SEEK_SET) + if app_config.mwdb.storage_provider == StorageProviderType.S3: + get_s3_client( + app_config.mwdb.s3_storage_endpoint, + app_config.mwdb.s3_storage_access_key, + app_config.mwdb.s3_storage_secret_key, + app_config.mwdb.s3_storage_region_name, + app_config.mwdb.s3_storage_secure, + app_config.mwdb.s3_storage_iam_auth, + ).put_object( + Bucket=app_config.mwdb.s3_storage_bucket_name, + Key=new_related_file._calculate_path(), + Body=file_stream, + ) + elif app_config.mwdb.storage_provider == StorageProviderType.DISK: + with open(new_related_file._calculate_path(), "wb") as f: + shutil.copyfileobj(file_stream, f) + else: + raise RuntimeError( + f"StorageProvider {app_config.mwdb.storage_provider} " + f"is not supported" + ) + + @classmethod + def access(cls, identifier): + related_file_obj = db.session.query(RelatedFile).filter(RelatedFile.sha256 == identifier).first() + if related_file_obj is None: + return None + if not g.auth_user.has_access_to_object(related_file_obj.object_id): + return None + + return related_file_obj + + def iterate(self, chunk_size=1024 * 256): + """ + Iterates over bytes in the file contents + """ + fh = self.open() + try: + if hasattr(fh, "stream"): + yield from fh.stream(chunk_size) + else: + while True: + chunk = fh.read(chunk_size) + if chunk: + yield chunk + else: + return + finally: + fh.close() \ No newline at end of file diff --git a/mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py b/mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py new file mode 100644 index 000000000..4baefba68 --- /dev/null +++ b/mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py @@ -0,0 +1,36 @@ +"""add related_files table + +Revision ID: 3c610b0ddebc +Revises: c7c72fd7fac5 +Create Date: 2022-12-12 09:02:40.406370 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '3c610b0ddebc' +down_revision = 'c7c72fd7fac5' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "related_file", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("object_id", sa.Integer(), nullable=False), + sa.Column("file_name", sa.String(), nullable=False), + sa.Column("file_size", sa.Integer, nullable=False), + sa.Column("sha256", sa.String(length=64), nullable=False), + sa.ForeignKeyConstraint( + ["object_id"], + ["object.id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade(): + op.drop_table("related_file") diff --git a/mwdb/model/object.py b/mwdb/model/object.py index e60e3bfb3..6a946a15d 100644 --- a/mwdb/model/object.py +++ b/mwdb/model/object.py @@ -257,6 +257,11 @@ class Object(db.Model): lazy="joined", cascade="save-update, merge, delete", ) + related_files = db.relationship( + "RelatedFile", + back_populates="related_object", + lazy=True, + ) followers = db.relationship( "User", secondary=favorites, back_populates="favorites", lazy="joined" diff --git a/mwdb/resources/file.py b/mwdb/resources/file.py index be2fad3cd..c60e4148b 100644 --- a/mwdb/resources/file.py +++ b/mwdb/resources/file.py @@ -5,7 +5,7 @@ from mwdb.core.capabilities import Capabilities from mwdb.core.plugins import hooks from mwdb.core.rate_limit import rate_limited_resource -from mwdb.model import File +from mwdb.model import File, RelatedFile from mwdb.model.file import EmptyFileError from mwdb.model.object import ObjectTypeConflictError from mwdb.schema.file import ( @@ -585,3 +585,146 @@ def post(self, identifier): download_token = file.generate_download_token() schema = FileDownloadTokenResponseSchema() return schema.dump({"token": download_token}) + + +@rate_limited_resource +class RelatedFileDownloadResource(Resource): + def get(self, identifier): + """ + --- + summary: Download related file + description: | + Returns related file contents. + security: + - bearerAuth: [] + tags: + - related_file + parameters: + - in: path + name: identifier + schema: + type: string + description: File identifier (SHA256) + required: true + responses: + 200: + description: File contents + content: + application/octet-stream: + schema: + type: string + format: binary + 404: + description: | + When related file doesn't exist + or user doesn't have access to it. + 503: + description: | + Request canceled due to database statement timeout. + """ + + if not g.auth_user: + raise Unauthorized("Not authenticated.") + + related_file_obj = RelatedFile.access(identifier) + + if related_file_obj is None: + raise NotFound("Object not found") + + return Response( + related_file_obj.iterate(), + content_type="application/octet-stream", + headers={"Content-disposition": f"attachment; filename={related_file_obj.sha256}"}, + ) + + +@rate_limited_resource +class RelatedFileItemResource(Resource): + @requires_authorization + @requires_capabilities(Capabilities.adding_files) + def post(self, identifier): + """ + --- + summary: Upload file + description: | + Uploads a new file. + + Requires `adding_files` capability. + security: + - bearerAuth: [] + tags: + - related_file + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + file: + type: string + format: binary + description: RelatedFile contents to be uploaded + identifier: + type: string + description: | + sha256 of the relating file + required: + - file + - identifier + responses: + 200: + description: Information about uploaded file + 400: + description: RelatedFile is empty + 409: + description: RelatedFile already exists + 503: + description: | + Request canceled due to database statement timeout. + """ + try: + RelatedFile.create( + request.files["file"].filename, + request.files["file"].stream, + identifier + ) + except EmptyFileError: + raise BadRequest("RelatedFile cannot be empty") + + return "OK" #self.create_object(obj["options"]) + + @requires_authorization + @requires_capabilities(Capabilities.removing_objects) + def delete(self, identifier): + """ + --- + summary: Delete file + description: | + Removes a file from the database along with its references. + + Requires `removing_objects` capability. + security: + - bearerAuth: [] + tags: + - file + parameters: + - in: path + name: identifier + schema: + type: string + description: File identifier (SHA256/SHA512/SHA1/MD5) + responses: + 200: + description: When file was deleted + 403: + description: When user doesn't have `removing_objects` capability + 404: + description: | + When file doesn't exist, object is not a file + or user doesn't have access to this object. + 503: + description: | + Request canceled due to database statement timeout. + """ + return super().delete(identifier) \ No newline at end of file diff --git a/mwdb/web/src/commons/api/index.js b/mwdb/web/src/commons/api/index.js index 847c2be97..5a8be281f 100644 --- a/mwdb/web/src/commons/api/index.js +++ b/mwdb/web/src/commons/api/index.js @@ -444,6 +444,10 @@ function uploadFile(file, parent, upload_as, attributes, fileUploadTimeout) { return axios.post(`/file`, formData, { timeout: fileUploadTimeout }); } +//function uploadRelatedFile(file, id) { +// return axios.post(`/related_file/${id}`) +//} + function getRemoteNames() { return axios.get("/remote"); } From c33d5c17ce06af7366e238e9c2acc6f12d5c309c Mon Sep 17 00:00:00 2001 From: Repumba Date: Mon, 12 Dec 2022 11:19:33 +0100 Subject: [PATCH 02/33] working upload (I think) --- mwdb/app.py | 4 ++- mwdb/model/file.py | 32 +++++++++++++------ .../3c610b0ddebc_add_related_files_table.py | 7 ++-- mwdb/resources/file.py | 30 ++++++++++------- 4 files changed, 47 insertions(+), 26 deletions(-) diff --git a/mwdb/app.py b/mwdb/app.py index 451d32efa..088181278 100755 --- a/mwdb/app.py +++ b/mwdb/app.py @@ -263,7 +263,9 @@ def require_auth(): # RelatedFiles endpoints api.add_resource(RelatedFileItemResource, "/related_file/") -api.add_resource(RelatedFileDownloadResource, "/related_file//download") +api.add_resource( + RelatedFileDownloadResource, "/related_file//download" +) # Config endpoints api.add_resource(ConfigResource, "/config") diff --git a/mwdb/model/file.py b/mwdb/model/file.py index 7990cd210..81b311c33 100644 --- a/mwdb/model/file.py +++ b/mwdb/model/file.py @@ -5,11 +5,11 @@ import tempfile import pyzipper +from flask import g from sqlalchemy import or_ from sqlalchemy.dialects.postgresql.array import ARRAY from sqlalchemy.ext.mutable import MutableList from werkzeug.utils import secure_filename -from flask import g from mwdb.core.auth import AuthScope, generate_token, verify_token from mwdb.core.config import StorageProviderType, app_config @@ -352,11 +352,12 @@ class RelatedFile(db.Model): def _calculate_path(self): if app_config.mwdb.storage_provider == StorageProviderType.DISK: - upload_path = ( - "related_files" - if app_config.mwdb.related_files_folder == "" - else app_config.mwdb.related_files_folder + "/related_files" - ) + # upload_path = ( + # "related_files" + # if app_config.mwdb.related_files_folder == "" + # else app_config.mwdb.related_files_folder + "/related_files" + # ) + upload_path = "/app/uploads/related_files" elif app_config.mwdb.storage_provider == StorageProviderType.S3: upload_path = "related_files/" else: @@ -381,7 +382,7 @@ def create( cls, file_name, file_stream, - related_object, + related_object_dhash, ): file_stream.seek(0, os.SEEK_END) file_size = file_stream.tell() @@ -393,11 +394,20 @@ def create( new_related_file = ( db.session.query(RelatedFile).filter(RelatedFile.sha256 == sha256).first() ) + related_object = ( + db.session.query(Object) + .filter(Object.dhash == related_object_dhash) + .first() + ) # If file already exists if new_related_file is not None: return + # If related file doesn't exist + if related_object is None: + raise ValueError("There is no object with this sha256") + new_related_file = RelatedFile( object_id=related_object.id, file_name=secure_filename(file_name), @@ -430,7 +440,11 @@ def create( @classmethod def access(cls, identifier): - related_file_obj = db.session.query(RelatedFile).filter(RelatedFile.sha256 == identifier).first() + related_file_obj = ( + db.session.query(RelatedFile) + .filter(RelatedFile.sha256 == identifier) + .first() + ) if related_file_obj is None: return None if not g.auth_user.has_access_to_object(related_file_obj.object_id): @@ -454,4 +468,4 @@ def iterate(self, chunk_size=1024 * 256): else: return finally: - fh.close() \ No newline at end of file + fh.close() diff --git a/mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py b/mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py index 4baefba68..21d625637 100644 --- a/mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py +++ b/mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py @@ -5,13 +5,12 @@ Create Date: 2022-12-12 09:02:40.406370 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. -revision = '3c610b0ddebc' -down_revision = 'c7c72fd7fac5' +revision = "3c610b0ddebc" +down_revision = "c7c72fd7fac5" branch_labels = None depends_on = None diff --git a/mwdb/resources/file.py b/mwdb/resources/file.py index c60e4148b..04abd3b5a 100644 --- a/mwdb/resources/file.py +++ b/mwdb/resources/file.py @@ -634,7 +634,9 @@ def get(self, identifier): return Response( related_file_obj.iterate(), content_type="application/octet-stream", - headers={"Content-disposition": f"attachment; filename={related_file_obj.sha256}"}, + headers={ + "Content-disposition": f"attachment; filename={related_file_obj.sha256}" + }, ) @@ -654,6 +656,13 @@ def post(self, identifier): - bearerAuth: [] tags: - related_file + parameters: + - in: path + name: identifier + schema: + type: string + description: File identifier (SHA256) + required: true requestBody: required: true content: @@ -665,18 +674,15 @@ def post(self, identifier): type: string format: binary description: RelatedFile contents to be uploaded - identifier: - type: string - description: | - sha256 of the relating file required: - file - - identifier responses: 200: description: Information about uploaded file 400: description: RelatedFile is empty + 404: + description: There is no file with provided sha256 409: description: RelatedFile already exists 503: @@ -685,14 +691,14 @@ def post(self, identifier): """ try: RelatedFile.create( - request.files["file"].filename, - request.files["file"].stream, - identifier + request.files["file"].filename, request.files["file"].stream, identifier ) except EmptyFileError: raise BadRequest("RelatedFile cannot be empty") - - return "OK" #self.create_object(obj["options"]) + except ValueError: + raise NotFound("There is no file with provided sha256") + + return "OK" # self.create_object(obj["options"]) @requires_authorization @requires_capabilities(Capabilities.removing_objects) @@ -727,4 +733,4 @@ def delete(self, identifier): description: | Request canceled due to database statement timeout. """ - return super().delete(identifier) \ No newline at end of file + return super().delete(identifier) From da5f8bc79b19c9d592fed2a34828bd1e15289838 Mon Sep 17 00:00:00 2001 From: Repumba Date: Mon, 12 Dec 2022 11:30:18 +0100 Subject: [PATCH 03/33] adding related files in database --- mwdb/model/file.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mwdb/model/file.py b/mwdb/model/file.py index 81b311c33..09618ca68 100644 --- a/mwdb/model/file.py +++ b/mwdb/model/file.py @@ -437,6 +437,8 @@ def create( f"StorageProvider {app_config.mwdb.storage_provider} " f"is not supported" ) + db.session.add(new_related_file) + db.session.commit() @classmethod def access(cls, identifier): From f9b30065d135d7fdc3f2c97ad192cadbbc2f758e Mon Sep 17 00:00:00 2001 From: Repumba Date: Mon, 12 Dec 2022 14:54:24 +0100 Subject: [PATCH 04/33] (probably) working backend --- mwdb/model/file.py | 71 ++++++++++++++++++++++++++++++++- mwdb/resources/file.py | 89 +++++++++++++++++++++++++++++++++++------- mwdb/schema/file.py | 11 ++++++ 3 files changed, 154 insertions(+), 17 deletions(-) diff --git a/mwdb/model/file.py b/mwdb/model/file.py index 09618ca68..36ece00ae 100644 --- a/mwdb/model/file.py +++ b/mwdb/model/file.py @@ -406,7 +406,9 @@ def create( # If related file doesn't exist if related_object is None: - raise ValueError("There is no object with this sha256") + raise ValueError( + "There is no object with this sha256 or you don't have access" + ) new_related_file = RelatedFile( object_id=related_object.id, @@ -449,11 +451,76 @@ def access(cls, identifier): ) if related_file_obj is None: return None - if not g.auth_user.has_access_to_object(related_file_obj.object_id): + + main_obj = ( + db.session.query(Object) + .filter(Object.id == related_file_obj.object_id) + .first() + ) + if not main_obj.has_explicit_access(g.auth_user): return None return related_file_obj + @classmethod + def delete(cls, identifier): + related_file_obj = ( + db.session.query(RelatedFile) + .filter(RelatedFile.sha256 == identifier) + .first() + ) + + if related_file_obj is None: + raise ValueError( + "There is no object with this sha256 or you don't have access" + ) + main_obj = ( + db.session.query(Object) + .filter(Object.id == related_file_obj.object_id) + .first() + ) + if not main_obj.has_explicit_access(g.auth_user): + raise ValueError( + "There is no object with this sha256 or you don't have access" + ) + + db.session.delete(related_file_obj) + db.session.commit() + return + + def open(self): + """ + Opens the related file stream with contents. + """ + if app_config.mwdb.storage_provider == StorageProviderType.S3: + # Stream coming from Boto3 get_object is not buffered and not seekable. + # We need to download it to the temporary file first. + stream = tempfile.TemporaryFile(mode="w+b") + try: + get_s3_client( + app_config.mwdb.s3_storage_endpoint, + app_config.mwdb.s3_storage_access_key, + app_config.mwdb.s3_storage_secret_key, + app_config.mwdb.s3_storage_region_name, + app_config.mwdb.s3_storage_secure, + app_config.mwdb.s3_storage_iam_auth, + ).download_fileobj( + Bucket=app_config.mwdb.s3_storage_bucket_name, + Key=self._calculate_path(), + Fileobj=stream, + ) + stream.seek(0, io.SEEK_SET) + return stream + except Exception: + stream.close() + raise + elif app_config.mwdb.storage_provider == StorageProviderType.DISK: + return open(self._calculate_path(), "rb") + else: + raise RuntimeError( + f"StorageProvider {app_config.mwdb.storage_provider} is not supported" + ) + def iterate(self, chunk_size=1024 * 256): """ Iterates over bytes in the file contents diff --git a/mwdb/resources/file.py b/mwdb/resources/file.py index 04abd3b5a..773570639 100644 --- a/mwdb/resources/file.py +++ b/mwdb/resources/file.py @@ -5,7 +5,7 @@ from mwdb.core.capabilities import Capabilities from mwdb.core.plugins import hooks from mwdb.core.rate_limit import rate_limited_resource -from mwdb.model import File, RelatedFile +from mwdb.model import File, Object, RelatedFile, db from mwdb.model.file import EmptyFileError from mwdb.model.object import ObjectTypeConflictError from mwdb.schema.file import ( @@ -14,6 +14,7 @@ FileItemResponseSchema, FileLegacyCreateRequestSchema, FileListResponseSchema, + RelatedFileResponseSchema, ) from . import load_schema, requires_authorization, requires_capabilities @@ -604,11 +605,11 @@ def get(self, identifier): name: identifier schema: type: string - description: File identifier (SHA256) + description: RelatedFile identifier (SHA256) required: true responses: 200: - description: File contents + description: RelatedFile contents content: application/octet-stream: schema: @@ -642,14 +643,62 @@ def get(self, identifier): @rate_limited_resource class RelatedFileItemResource(Resource): + def get(self, identifier): + """ + --- + summary: Get list of RelatedFiles + description: | + Returns list of RelatedFiles for a File specified by sha256 + security: + - bearerAuth: [] + tags: + - related_file + parameters: + - in: path + name: identifier + schema: + type: string + description: Master File identifier (SHA256) + required: true + responses: + 200: + description: List of RelatedFiles + content: + application/json: + schema: RelatedFileResponseSchema + 404: + description: | + There is no file with provided sha256 or you don't have access to it + 503: + description: | + Request canceled due to database statement timeout. + """ + master_object = ( + db.session.query(Object) + .filter(Object.dhash == identifier) + .filter(g.auth_user.has_access_to_object(Object.id)) + ).first() + + if master_object is None: + raise NotFound( + "There is no file with provided sha256 or you don't have access to it" + ) + + related_files = db.session.query(RelatedFile).filter( + RelatedFile.object_id == master_object.id + ) + schema = RelatedFileResponseSchema() + + return schema.dump({"related_files": related_files}) + @requires_authorization @requires_capabilities(Capabilities.adding_files) def post(self, identifier): """ --- - summary: Upload file + summary: Upload related file description: | - Uploads a new file. + Uploads a new related file. Requires `adding_files` capability. security: @@ -661,7 +710,7 @@ def post(self, identifier): name: identifier schema: type: string - description: File identifier (SHA256) + description: Master File identifier (SHA256) required: true requestBody: required: true @@ -678,11 +727,12 @@ def post(self, identifier): - file responses: 200: - description: Information about uploaded file + description: OK 400: description: RelatedFile is empty 404: - description: There is no file with provided sha256 + description: | + There is no file with provided sha256 or you don't have access to it 409: description: RelatedFile already exists 503: @@ -696,9 +746,11 @@ def post(self, identifier): except EmptyFileError: raise BadRequest("RelatedFile cannot be empty") except ValueError: - raise NotFound("There is no file with provided sha256") + raise NotFound( + "There is no file with provided sha256 or you don't have access to it" + ) - return "OK" # self.create_object(obj["options"]) + return Response("OK") @requires_authorization @requires_capabilities(Capabilities.removing_objects) @@ -713,24 +765,31 @@ def delete(self, identifier): security: - bearerAuth: [] tags: - - file + - related_file parameters: - in: path name: identifier schema: type: string - description: File identifier (SHA256/SHA512/SHA1/MD5) + description: RelatedFile identifier (SHA256/SHA512/SHA1/MD5) responses: 200: - description: When file was deleted + description: When related file was deleted 403: description: When user doesn't have `removing_objects` capability 404: description: | - When file doesn't exist, object is not a file + When related file doesn't exist or user doesn't have access to this object. 503: description: | Request canceled due to database statement timeout. """ - return super().delete(identifier) + try: + RelatedFile.delete(identifier) + except ValueError: + raise NotFound( + "There is no file with provided sha256 or you don't have access to it" + ) + + return Response("OK") diff --git a/mwdb/schema/file.py b/mwdb/schema/file.py index 8825135ce..0f8e1581a 100644 --- a/mwdb/schema/file.py +++ b/mwdb/schema/file.py @@ -73,3 +73,14 @@ class FileItemResponseSchema(ObjectItemResponseSchema): class FileDownloadTokenResponseSchema(Schema): token = fields.Str(required=True, allow_none=False) + + +class RelatedFileItemResponseSchema(Schema): + file_name = fields.Str(required=True, allow_none=False) + file_size = fields.Int(required=True, allow_none=False) + + +class RelatedFileResponseSchema(Schema): + related_files = fields.Nested( + RelatedFileItemResponseSchema, many=True, required=True, allow_none=False + ) From 434e5e8a27d6aa6150b00cda34c0612bbc13422c Mon Sep 17 00:00:00 2001 From: Repumba Date: Mon, 12 Dec 2022 15:05:03 +0100 Subject: [PATCH 05/33] Raise exception when the same related file is uploaded second time --- mwdb/model/file.py | 2 +- mwdb/resources/file.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mwdb/model/file.py b/mwdb/model/file.py index 36ece00ae..8965c3d19 100644 --- a/mwdb/model/file.py +++ b/mwdb/model/file.py @@ -402,7 +402,7 @@ def create( # If file already exists if new_related_file is not None: - return + raise FileExistsError("Related file with this sha256 already exists") # If related file doesn't exist if related_object is None: diff --git a/mwdb/resources/file.py b/mwdb/resources/file.py index 773570639..29fcc53bc 100644 --- a/mwdb/resources/file.py +++ b/mwdb/resources/file.py @@ -749,6 +749,8 @@ def post(self, identifier): raise NotFound( "There is no file with provided sha256 or you don't have access to it" ) + except FileExistsError: + raise Conflict("Related file with this sha256 already exists") return Response("OK") From 6f57b7e0d028c3a3a8a03616c2ccab750b416757 Mon Sep 17 00:00:00 2001 From: Repumba Date: Mon, 12 Dec 2022 17:07:33 +0100 Subject: [PATCH 06/33] Initial front-end commit --- mwdb/schema/file.py | 1 + mwdb/web/src/commons/api/index.js | 7 +- .../ShowObject/Views/RelatedFilesTab.js | 69 +++++++++++++++++++ mwdb/web/src/components/ShowObject/index.js | 1 + mwdb/web/src/components/ShowSample.js | 2 + 5 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js diff --git a/mwdb/schema/file.py b/mwdb/schema/file.py index 0f8e1581a..b0a9114f3 100644 --- a/mwdb/schema/file.py +++ b/mwdb/schema/file.py @@ -78,6 +78,7 @@ class FileDownloadTokenResponseSchema(Schema): class RelatedFileItemResponseSchema(Schema): file_name = fields.Str(required=True, allow_none=False) file_size = fields.Int(required=True, allow_none=False) + sha256 = fields.Str(required=True, allow_none=False) class RelatedFileResponseSchema(Schema): diff --git a/mwdb/web/src/commons/api/index.js b/mwdb/web/src/commons/api/index.js index 5a8be281f..772cd8c3c 100644 --- a/mwdb/web/src/commons/api/index.js +++ b/mwdb/web/src/commons/api/index.js @@ -444,9 +444,9 @@ function uploadFile(file, parent, upload_as, attributes, fileUploadTimeout) { return axios.post(`/file`, formData, { timeout: fileUploadTimeout }); } -//function uploadRelatedFile(file, id) { -// return axios.post(`/related_file/${id}`) -//} +function getListOfRelatedFiles(id) { + return axios.get(`/related_file/${id}`) +} function getRemoteNames() { return axios.get("/remote"); @@ -630,6 +630,7 @@ const api = { requestFileDownloadLink, requestZipFileDownloadLink, uploadFile, + getListOfRelatedFiles, getRemoteNames, pushObjectRemote, pullObjectRemote, diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js new file mode 100644 index 000000000..9e2bd0111 --- /dev/null +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -0,0 +1,69 @@ +import React, { useContext, useState } from "react"; +import { Link } from "react-router-dom"; + +import { faPlus, faExternalLinkSquare } from "@fortawesome/free-solid-svg-icons"; + +import { ObjectContext } from "@mwdb-web/commons/context"; +import { ObjectAction, ObjectTab } from "@mwdb-web/commons/ui"; +import { APIContext } from "@mwdb-web/commons/api/context"; + +function RelatedFileItem({ file_name, file_size, sha256 }){ + console.log(file_name); + return ( + + + {file_name} + + + {file_size} + + + + Pobierz + + + + ); +} + +function ShowRelatedFiles(){ + const [relatedFiles, setRelatedFiles] = useState([]); + + async function updateRelatedFiles(){ + try{ + let response = await api.getListOfRelatedFiles(context.object.sha256); + setRelatedFiles(response.data.related_files); + } catch (error) { + console.log(error); + } + } + + const api = useContext(APIContext); + const context = useContext(ObjectContext); + + updateRelatedFiles(); + + return ( + + {relatedFiles.map((related_file) => ( + + ))} +
+ ); +} + +export default function RelatedFilesTab(){ + return ( + , + ]} + /> + ) +} \ No newline at end of file diff --git a/mwdb/web/src/components/ShowObject/index.js b/mwdb/web/src/components/ShowObject/index.js index e8be8f093..f8af59d65 100644 --- a/mwdb/web/src/components/ShowObject/index.js +++ b/mwdb/web/src/components/ShowObject/index.js @@ -1,6 +1,7 @@ export { default as ShowObject } from "./ShowObject"; export { default as RelationsTab } from "./Views/RelationsTab"; +export { default as RelatedFilesTab } from "./Views/RelatedFilesTab" export { default as LatestConfigTab } from "./Views/LatestConfigTab"; export { default as DownloadAction } from "./Actions/DownloadAction"; diff --git a/mwdb/web/src/components/ShowSample.js b/mwdb/web/src/components/ShowSample.js index 1c87015d1..251815b87 100644 --- a/mwdb/web/src/components/ShowSample.js +++ b/mwdb/web/src/components/ShowSample.js @@ -8,6 +8,7 @@ import { useTabContext, LatestConfigTab, RelationsTab, + RelatedFilesTab, DownloadAction, ZipAction, FavoriteAction, @@ -335,6 +336,7 @@ export default function ShowSample(props) { , ]} /> + ); From 735fc1fb949361e16f0201e7955fd2517f7cdfce Mon Sep 17 00:00:00 2001 From: Repumba Date: Mon, 12 Dec 2022 17:35:39 +0100 Subject: [PATCH 07/33] Front-end adjustments --- .../ShowObject/Views/RelatedFilesTab.js | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index 9e2bd0111..eeed963f8 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -1,7 +1,8 @@ import React, { useContext, useState } from "react"; import { Link } from "react-router-dom"; -import { faPlus, faExternalLinkSquare } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { faPlus, faExternalLinkSquare, faDownload } from "@fortawesome/free-solid-svg-icons"; import { ObjectContext } from "@mwdb-web/commons/context"; import { ObjectAction, ObjectTab } from "@mwdb-web/commons/ui"; @@ -10,18 +11,23 @@ import { APIContext } from "@mwdb-web/commons/api/context"; function RelatedFileItem({ file_name, file_size, sha256 }){ console.log(file_name); return ( - + {file_name} - {file_size} + {file_size} B { + + }} > - Pobierz + +  Download @@ -46,7 +52,18 @@ function ShowRelatedFiles(){ updateRelatedFiles(); return ( - +
+ + + + + {relatedFiles.map((related_file) => ( ))} @@ -62,7 +79,7 @@ export default function RelatedFilesTab(){ icon = {faExternalLinkSquare} component = {ShowRelatedFiles} actions={[ - , + , ]} /> ) From 1fd981a9a3a896a52c2ef95bb59f4b1e4df48c7e Mon Sep 17 00:00:00 2001 From: Repumba Date: Tue, 13 Dec 2022 10:19:19 +0100 Subject: [PATCH 08/33] Fix problem with too many rerenders --- .../components/ShowObject/Views/RelatedFilesTab.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index eeed963f8..5ef548cc0 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -1,4 +1,4 @@ -import React, { useContext, useState } from "react"; +import React, { useCallback, useContext, useEffect, useState } from "react"; import { Link } from "react-router-dom"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; @@ -49,7 +49,14 @@ function ShowRelatedFiles(){ const api = useContext(APIContext); const context = useContext(ObjectContext); - updateRelatedFiles(); + const getRelatedFiles = useCallback(updateRelatedFiles, [ + api, + context + ]); + + useEffect(() => { + getRelatedFiles(); + }, [getRelatedFiles]); return (
+ File name + + File size + + Download +
From 8dd170e1fdcff252ba76884e6b6628014af3d84d Mon Sep 17 00:00:00 2001 From: Repumba Date: Tue, 13 Dec 2022 12:30:48 +0100 Subject: [PATCH 09/33] progress in front-end --- mwdb/web/src/commons/api/index.js | 19 +- .../ShowObject/Views/RelatedFilesTab.js | 174 ++++++++++++------ mwdb/web/src/components/ShowObject/index.js | 2 +- 3 files changed, 134 insertions(+), 61 deletions(-) diff --git a/mwdb/web/src/commons/api/index.js b/mwdb/web/src/commons/api/index.js index 772cd8c3c..064095908 100644 --- a/mwdb/web/src/commons/api/index.js +++ b/mwdb/web/src/commons/api/index.js @@ -444,8 +444,22 @@ function uploadFile(file, parent, upload_as, attributes, fileUploadTimeout) { return axios.post(`/file`, formData, { timeout: fileUploadTimeout }); } +function uploadRelatedFile(file, masterFileDhash) { + let formData = new FormData(); + formData.append("file", file); + return axios.post(`/related_file/${masterFileDhash}`, formData); +} + +function downloadRelatedFile(id) { + return axios.get(`/related_file/${id}/download`); +} + +function deleteRelatedFile(id) { + return axios.delete(`/related_file/${id}`); +} + function getListOfRelatedFiles(id) { - return axios.get(`/related_file/${id}`) + return axios.get(`/related_file/${id}`); } function getRemoteNames() { @@ -630,6 +644,9 @@ const api = { requestFileDownloadLink, requestZipFileDownloadLink, uploadFile, + uploadRelatedFile, + downloadRelatedFile, + deleteRelatedFile, getListOfRelatedFiles, getRemoteNames, pushObjectRemote, diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index 5ef548cc0..e20726aa9 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -2,57 +2,69 @@ import React, { useCallback, useContext, useEffect, useState } from "react"; import { Link } from "react-router-dom"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { faPlus, faExternalLinkSquare, faDownload } from "@fortawesome/free-solid-svg-icons"; +import { + faPlus, + faExternalLinkSquare, + faDownload, + faTrash, +} from "@fortawesome/free-solid-svg-icons"; import { ObjectContext } from "@mwdb-web/commons/context"; import { ObjectAction, ObjectTab } from "@mwdb-web/commons/ui"; import { APIContext } from "@mwdb-web/commons/api/context"; +import ReactModal from "react-modal"; -function RelatedFileItem({ file_name, file_size, sha256 }){ - console.log(file_name); - return ( - - - - - - ); -} - -function ShowRelatedFiles(){ +function ShowRelatedFiles() { const [relatedFiles, setRelatedFiles] = useState([]); + const api = useContext(APIContext); + const context = useContext(ObjectContext); - async function updateRelatedFiles(){ - try{ - let response = await api.getListOfRelatedFiles(context.object.sha256); + function RelatedFileItem({ file_name, file_size, sha256 }) { + return ( + + + + + + + ); + } + + async function updateRelatedFiles() { + try { + let response = await api.getListOfRelatedFiles( + context.object.sha256 + ); setRelatedFiles(response.data.related_files); } catch (error) { console.log(error); } } - const api = useContext(APIContext); - const context = useContext(ObjectContext); - - const getRelatedFiles = useCallback(updateRelatedFiles, [ - api, - context - ]); + const getRelatedFiles = useCallback(updateRelatedFiles, [api, context]); useEffect(() => { getRelatedFiles(); @@ -61,15 +73,10 @@ function ShowRelatedFiles(){ return (
- {file_name} - - {file_size} B - - { - - }} - > - -  Download - -
{file_name}{file_size} B + { + console.log(sha256); + await api.downloadRelatedFile(sha256); + }} + > + +  Download + + + { + await api.deleteRelatedFile(sha256); + }} + > + +  Remove + +
- - - + + + + {relatedFiles.map((related_file) => ( @@ -78,16 +85,65 @@ function ShowRelatedFiles(){ ); } -export default function RelatedFilesTab(){ +export default function RelatedFilesTab() { + const [showModal, setShowModal] = useState(); + const [file, setFile] = useState(null); + const context = useContext(ObjectContext); + const api = useContext(APIContext); + + async function handleSubmit() { + try { + await api.uploadRelatedFile(file, context.object.sha256); + } catch (error) { + console.log(error); + } + } + return ( - , - ]} - /> - ) -} \ No newline at end of file +
+ { + setShowModal(true); + }} + />, + ]} + /> + { + setShowModal(false); + }} + > +
{ + handleSubmit(); + setShowModal(false); + }} + > + setFile(data)} + />{" "} +
+ + + + +
+ ); +} diff --git a/mwdb/web/src/components/ShowObject/index.js b/mwdb/web/src/components/ShowObject/index.js index f8af59d65..afb75a817 100644 --- a/mwdb/web/src/components/ShowObject/index.js +++ b/mwdb/web/src/components/ShowObject/index.js @@ -1,7 +1,7 @@ export { default as ShowObject } from "./ShowObject"; export { default as RelationsTab } from "./Views/RelationsTab"; -export { default as RelatedFilesTab } from "./Views/RelatedFilesTab" +export { default as RelatedFilesTab } from "./Views/RelatedFilesTab"; export { default as LatestConfigTab } from "./Views/LatestConfigTab"; export { default as DownloadAction } from "./Actions/DownloadAction"; From 55ea824d2907c97ad278b53539a1825dfaec8c37 Mon Sep 17 00:00:00 2001 From: Repumba Date: Tue, 13 Dec 2022 13:25:50 +0100 Subject: [PATCH 10/33] working download --- mwdb/web/src/commons/api/index.js | 5 ++++- .../components/ShowObject/Views/RelatedFilesTab.js | 13 ++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/mwdb/web/src/commons/api/index.js b/mwdb/web/src/commons/api/index.js index 064095908..537e26030 100644 --- a/mwdb/web/src/commons/api/index.js +++ b/mwdb/web/src/commons/api/index.js @@ -451,7 +451,10 @@ function uploadRelatedFile(file, masterFileDhash) { } function downloadRelatedFile(id) { - return axios.get(`/related_file/${id}/download`); + return axios.get(`/related_file/${id}/download`, { + responseType: "arraybuffer", + responseEncoding: "binary", + }); } function deleteRelatedFile(id) { diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index e20726aa9..334d5290b 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -26,11 +26,18 @@ function ShowRelatedFiles() { + - + - - - - + + + + {relatedFiles.map((related_file) => ( From 2c7cdcef78614346a05b830f5242627eee8f015c Mon Sep 17 00:00:00 2001 From: Repumba Date: Wed, 14 Dec 2022 10:18:16 +0100 Subject: [PATCH 16/33] auto rerender tab after change --- .../ShowObject/Views/RelatedFilesTab.js | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index f3d56d905..5f4448c2b 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -15,8 +15,19 @@ import { APIContext } from "@mwdb-web/commons/api/context"; import { humanFileSize } from "@mwdb-web/commons/helpers"; import ReactModal from "react-modal"; +async function updateRelatedFiles(api, context) { + const { setObjectError, updateObjectData } = context; + try { + let response = await api.getListOfRelatedFiles(context.object.sha256); + updateObjectData({ + related_files: response.data.related_files, + }); + } catch (error) { + setObjectError(error); + } +} + function ShowRelatedFiles() { - const [relatedFiles, setRelatedFiles] = useState([]); const api = useContext(APIContext); const context = useContext(ObjectContext); const { setObjectError, updateObjectData } = context; @@ -52,7 +63,7 @@ function ShowRelatedFiles() { className="nav-link" onClick={async () => { await api.deleteRelatedFile(sha256); - updateRelatedFiles(); + updateRelatedFiles(api, context); }} > @@ -63,19 +74,6 @@ function ShowRelatedFiles() { ); } - async function updateRelatedFiles() { - try { - let response = await api.getListOfRelatedFiles( - context.object.sha256 - ); - setRelatedFiles(response.data.related_files); - updateObjectData({ - related_files: response.data.related_files, - }); - } catch (error) { - setObjectError(error); - } - } const getRelatedFiles = useCallback(updateRelatedFiles, [ api, setObjectError, @@ -83,10 +81,18 @@ function ShowRelatedFiles() { context.object.sha256, ]); + // JS throws a warning "Line 92:8: React Hook useEffect has missing dependencies: 'api' and 'context'" + // Those dependencies are skipped on purpose + // To disable this warning I used 'eslint-disable-next-line' useEffect(() => { - getRelatedFiles(); + getRelatedFiles(api, context); + // eslint-disable-next-line }, [getRelatedFiles]); + if (!context.object.related_files) { + return "Loading..."; + } + return (
- File name - - File size - - Download - File nameFile sizeDownloadRemove
{file_size} B { - console.log(sha256); - await api.downloadRelatedFile(sha256); + let content = await api.downloadRelatedFile(sha256); + let blob = new Blob([content.data], { + type: "application/octet-stream", + }); + let tempLink = document.createElement("a"); + tempLink.style.display = "none"; + tempLink.href = window.URL.createObjectURL(blob); + tempLink.download = file_name; + tempLink.click(); }} > From d247ee2f454a00946172990b2c885caf6820049c Mon Sep 17 00:00:00 2001 From: Repumba Date: Tue, 13 Dec 2022 13:27:20 +0100 Subject: [PATCH 11/33] little fix in removing related files --- mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index 334d5290b..9445b1f8a 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -46,7 +46,7 @@ function ShowRelatedFiles() { { await api.deleteRelatedFile(sha256); From d589b0cd4088e84c986c930fe0494856e0477358 Mon Sep 17 00:00:00 2001 From: Repumba Date: Tue, 13 Dec 2022 14:00:40 +0100 Subject: [PATCH 12/33] Working upload --- .../src/components/ShowObject/Views/RelatedFilesTab.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index 9445b1f8a..369ce622d 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -130,15 +130,23 @@ export default function RelatedFilesTab() { }} >
{ handleSubmit(); setShowModal(false); }} > setFile(data)} + onChange={() => + setFile( + document.forms["RelatedFileUploadForm"][ + "RelatedFileUploadField" + ].files[0] + ) + } />{" "}
From eb666673f46383871e852f685f755370db452b7c Mon Sep 17 00:00:00 2001 From: Repumba Date: Tue, 13 Dec 2022 17:21:24 +0100 Subject: [PATCH 13/33] almost working front-end with better error handling --- .../ShowObject/Views/RelatedFilesTab.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index 369ce622d..5ea9c7fd8 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -18,6 +18,7 @@ function ShowRelatedFiles() { const [relatedFiles, setRelatedFiles] = useState([]); const api = useContext(APIContext); const context = useContext(ObjectContext); + const { setObjectError, updateObjectData } = context; function RelatedFileItem({ file_name, file_size, sha256 }) { return ( @@ -50,6 +51,7 @@ function ShowRelatedFiles() { className="nav-link" onClick={async () => { await api.deleteRelatedFile(sha256); + updateRelatedFiles(); }} > @@ -66,12 +68,19 @@ function ShowRelatedFiles() { context.object.sha256 ); setRelatedFiles(response.data.related_files); + updateObjectData({ + related_files: response.data.related_files, + }); } catch (error) { - console.log(error); + setObjectError(error); } } - - const getRelatedFiles = useCallback(updateRelatedFiles, [api, context]); + const getRelatedFiles = useCallback(updateRelatedFiles, [ + api, + setObjectError, + updateObjectData, + context.object.sha256, + ]); useEffect(() => { getRelatedFiles(); @@ -97,12 +106,13 @@ export default function RelatedFilesTab() { const [file, setFile] = useState(null); const context = useContext(ObjectContext); const api = useContext(APIContext); + const { setObjectError } = context; async function handleSubmit() { try { await api.uploadRelatedFile(file, context.object.sha256); } catch (error) { - console.log(error); + setObjectError(error); } } @@ -134,6 +144,7 @@ export default function RelatedFilesTab() { onSubmit={() => { handleSubmit(); setShowModal(false); + ShowRelatedFiles().updateRelatedFiles(); }} > Date: Tue, 13 Dec 2022 17:48:38 +0100 Subject: [PATCH 14/33] visual improvements --- .../ShowObject/Views/RelatedFilesTab.js | 40 ++++++++++++++----- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index 5ea9c7fd8..5c72ea2e0 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -13,6 +13,7 @@ import { ObjectContext } from "@mwdb-web/commons/context"; import { ObjectAction, ObjectTab } from "@mwdb-web/commons/ui"; import { APIContext } from "@mwdb-web/commons/api/context"; import ReactModal from "react-modal"; +import { useDropzone } from "react-dropzone"; function ShowRelatedFiles() { const [relatedFiles, setRelatedFiles] = useState([]); @@ -108,6 +109,17 @@ export default function RelatedFilesTab() { const api = useContext(APIContext); const { setObjectError } = context; + const modalStyle = { + content: { + top: "50%", + left: "50%", + right: "auto", + bottom: "auto", + marginRight: "-50%", + transform: "translate(-50%, -50%)", + }, + }; + async function handleSubmit() { try { await api.uploadRelatedFile(file, context.object.sha256); @@ -138,6 +150,7 @@ export default function RelatedFilesTab() { onRequestClose={() => { setShowModal(false); }} + style={modalStyle} > {" "} -
- + /> +
+ + +
- ); From 3ee25b09c44b6848ccd611d1263aa13710060d05 Mon Sep 17 00:00:00 2001 From: Repumba Date: Tue, 13 Dec 2022 18:01:22 +0100 Subject: [PATCH 15/33] more visual improvements --- .../components/ShowObject/Views/RelatedFilesTab.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index 5c72ea2e0..f3d56d905 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -12,8 +12,8 @@ import { import { ObjectContext } from "@mwdb-web/commons/context"; import { ObjectAction, ObjectTab } from "@mwdb-web/commons/ui"; import { APIContext } from "@mwdb-web/commons/api/context"; +import { humanFileSize } from "@mwdb-web/commons/helpers"; import ReactModal from "react-modal"; -import { useDropzone } from "react-dropzone"; function ShowRelatedFiles() { const [relatedFiles, setRelatedFiles] = useState([]); @@ -23,9 +23,9 @@ function ShowRelatedFiles() { function RelatedFileItem({ file_name, file_size, sha256 }) { return ( -
{file_name}{file_size} B{humanFileSize(file_size)}
File nameFile sizeDownloadRemoveFile nameFile sizeDownloadRemove
@@ -95,7 +101,7 @@ function ShowRelatedFiles() { - {relatedFiles.map((related_file) => ( + {context.object.related_files.map((related_file) => ( ))}
Download Remove
@@ -123,6 +129,7 @@ export default function RelatedFilesTab() { async function handleSubmit() { try { await api.uploadRelatedFile(file, context.object.sha256); + updateRelatedFiles(api, context); } catch (error) { setObjectError(error); } From 252cc2fc910b79253c3367d6971cb1c2aa89350b Mon Sep 17 00:00:00 2001 From: Repumba Date: Wed, 14 Dec 2022 12:38:58 +0100 Subject: [PATCH 17/33] rename variables for clarity --- mwdb/model/file.py | 10 +++++----- .../src/components/ShowObject/Views/RelatedFilesTab.js | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/mwdb/model/file.py b/mwdb/model/file.py index 8965c3d19..e50250a69 100644 --- a/mwdb/model/file.py +++ b/mwdb/model/file.py @@ -382,7 +382,7 @@ def create( cls, file_name, file_stream, - related_object_dhash, + main_obj_dhash, ): file_stream.seek(0, os.SEEK_END) file_size = file_stream.tell() @@ -394,9 +394,9 @@ def create( new_related_file = ( db.session.query(RelatedFile).filter(RelatedFile.sha256 == sha256).first() ) - related_object = ( + main_obj = ( db.session.query(Object) - .filter(Object.dhash == related_object_dhash) + .filter(Object.dhash == main_obj_dhash) .first() ) @@ -405,13 +405,13 @@ def create( raise FileExistsError("Related file with this sha256 already exists") # If related file doesn't exist - if related_object is None: + if main_obj is None: raise ValueError( "There is no object with this sha256 or you don't have access" ) new_related_file = RelatedFile( - object_id=related_object.id, + object_id=main_obj.id, file_name=secure_filename(file_name), file_size=file_size, sha256=sha256, diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index 5f4448c2b..6289539b1 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -81,7 +81,7 @@ function ShowRelatedFiles() { context.object.sha256, ]); - // JS throws a warning "Line 92:8: React Hook useEffect has missing dependencies: 'api' and 'context'" + // JS throws a warning "Line 90:8: React Hook useEffect has missing dependencies: 'api' and 'context'" // Those dependencies are skipped on purpose // To disable this warning I used 'eslint-disable-next-line' useEffect(() => { @@ -164,7 +164,6 @@ export default function RelatedFilesTab() { onSubmit={() => { handleSubmit(); setShowModal(false); - ShowRelatedFiles().updateRelatedFiles(); }} > Date: Wed, 14 Dec 2022 14:40:40 +0100 Subject: [PATCH 18/33] RelatedFiles Capabilities, migration and small visual improvement --- docs/user-guide/9-Sharing-objects.rst | 47 ++++++++++----- mwdb/core/capabilities.py | 6 ++ mwdb/model/file.py | 4 +- ...5_assign_new_capabilities_related_files.py | 60 +++++++++++++++++++ mwdb/resources/file.py | 31 +++++++--- mwdb/web/src/commons/auth/capabilities.js | 6 ++ .../ShowObject/Views/RelatedFilesTab.js | 5 +- 7 files changed, 130 insertions(+), 29 deletions(-) create mode 100644 mwdb/model/migrations/versions/02f584212ea5_assign_new_capabilities_related_files.py diff --git a/docs/user-guide/9-Sharing-objects.rst b/docs/user-guide/9-Sharing-objects.rst index dfb5b1e4b..d8bb111f2 100644 --- a/docs/user-guide/9-Sharing-objects.rst +++ b/docs/user-guide/9-Sharing-objects.rst @@ -155,47 +155,47 @@ By default, ``admin`` private group has enabled all capabilities. All other grou Each capability has its own name and scope: -* +* **manage_users - Managing users and groups (system administration)** Allows to access all users and groups in MWDB. Rules described in *Who is who?* don't apply to users with that permission. Enables user to create new user accounts, new groups and change their capabilities and membership. Allows to manage attribute keys, define new ones, delete and set the group permissions for them. -* +* **share_queried_objects - Query for all objects in system** That one is a bit tricky and will be possibly deprecated. MWDB will automatically share object and all descendants with group if member directly accessed it via identifier (knows the hash e.g. have direct link to the object). It can be used for bot accounts, so they have access only to these objects that are intended to be processed by them. Internally, we abandoned that idea, so that capability may not be stable. -* +* **access_all_objects - Has access to all new uploaded objects into system** Capability used by ``everything`` group, useful when you want to make additional "everything" that is separate from the original one. Keep in mind that it applies only to the **uploads made during the capability was enabled**\ , so if you want the new group to be truly "everything", you may need to share the old objects manually. -* +* **sharing_with_all - Can share objects with all groups in system** Implies the access to the list of all group names, but without access to the membership information and management features. Allows to share object with arbitrary group in MWDB. -* +* **adding_tags - Can add tags** Allows to tag objects. This feature is disabled by default, as you may want to have only tags from automated analyses. -* +* **removing_tags - Can remove tags** Allows to remove tags. Tag doesn't have "owner", so user will be able to remove all tags from the object. -* +* **adding_comments - Can add comments** Allows to add comments to the objects. Keep in mind that comments are public. -* +* **removing_comments - Can remove (all) comments** Allows to remove **all** comments, not only these authored by the user. -* +* **adding_parents - Can add parents** Allows to add new relationships by specifying object parent during upload or adding new relationship between existing objects. @@ -210,22 +210,22 @@ Each capability has its own name and scope: Enables upload of files. Enabled by default for ``registered`` group. -* +* **adding_configs - Can upload configs** Enables upload of configurations. Configurations are intended to be uploaded by automated systems or trusted entities that follow the conventions. -* +* **adding_blobs - Can upload text blobs** Enables upload of blobs. Blobs may have similar meaning as configurations in terms of user roles. -* +* **reading_all_attributes - Has access to all attributes of object (including hidden)** With that capability, you can read all the attributes, even if you don't have ``read`` permission for that attribute key. It allows to list hidden attribute values. -* +* **adding_all_attributes - Can add all attributes to object** Enables group to add all the attributes, even if it doesn't have ``set`` permission for that attribute key. @@ -235,12 +235,12 @@ Each capability has its own name and scope: Allows to remove attribute from object. To remove attribute, you need to have ``set`` permission for key. Combined with ``adding_all_attributes``\ , allows to remove all attributes. -* +* **unlimited_requests - API requests are not rate-limited for this group** Disables rate limiting for users from that group, if rate limiting feature is enabled. -* +* **removing_objects - Can remove objects** Can remove all accessible objects from the MWDB. May be quite destructive, we suggest to keep that capability enabled only for ``admin`` account. @@ -255,7 +255,7 @@ Each capability has its own name and scope: Allows to use personalization features like favorites or quick queries. -* +* **karton_assign - Can assign existing analysis to the object** Allows to assign Karton analysis to the object by setting ``karton`` attribute or using dedicated API. @@ -264,6 +264,21 @@ Each capability has its own name and scope: **karton_reanalyze - Can resubmit any object for analysis** Can manually resubmit object to Karton. +* + **access_related_files - Can view and download RelatedFiles** + + Allows to view list of RelatedFiles and download them. + +* + **adding_related_files - Can upload new RelatedFiles** + + Allows to upload new RelatedFiles. + +* + **removing_related_files - removing_related_files** + + Allows to remove existing RelatedFiles. + User capabilities are the sum of all group capabilities. If you want to enable capability system-wide (e.g. enable all users to add tags), enable that capability for ``registered`` group or ``public`` group if you want to include guests. diff --git a/mwdb/core/capabilities.py b/mwdb/core/capabilities.py index 4c5718975..f21a2e8d4 100644 --- a/mwdb/core/capabilities.py +++ b/mwdb/core/capabilities.py @@ -45,6 +45,12 @@ class Capabilities(object): karton_reanalyze = "karton_reanalyze" # Can remove Karton analysis from the object karton_unassign = "karton_unassign" + # Can view and download RelatedFiles + access_related_files = "access_related_files" + # Can upload new RelatedFiles + adding_related_files = "adding_related_files" + # Can remove existing RelatedFiles + removing_related_files = "removing_related_files" @classmethod def all(cls): diff --git a/mwdb/model/file.py b/mwdb/model/file.py index e50250a69..3a60f6027 100644 --- a/mwdb/model/file.py +++ b/mwdb/model/file.py @@ -395,9 +395,7 @@ def create( db.session.query(RelatedFile).filter(RelatedFile.sha256 == sha256).first() ) main_obj = ( - db.session.query(Object) - .filter(Object.dhash == main_obj_dhash) - .first() + db.session.query(Object).filter(Object.dhash == main_obj_dhash).first() ) # If file already exists diff --git a/mwdb/model/migrations/versions/02f584212ea5_assign_new_capabilities_related_files.py b/mwdb/model/migrations/versions/02f584212ea5_assign_new_capabilities_related_files.py new file mode 100644 index 000000000..5da6d05bf --- /dev/null +++ b/mwdb/model/migrations/versions/02f584212ea5_assign_new_capabilities_related_files.py @@ -0,0 +1,60 @@ +"""assign new capabilities for related files + +Revision ID: 02f584212ea5 +Revises: 3c610b0ddebc +Create Date: 2022-12-14 12:03:22.613573 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "02f584212ea5" +down_revision = "3c610b0ddebc" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute( + """ + UPDATE public.group + SET capabilities = array_append(capabilities, 'access_related_files') + WHERE name='public' OR array_position(capabilities, 'manage_users') IS NOT NULL; + """ + ) + op.execute( + """ + UPDATE public.group + SET capabilities = array_append(capabilities, 'adding_related_files') + WHERE name='registered' OR array_position(capabilities, 'manage_users') IS NOT NULL; + """ + ) + op.execute( + """ + UPDATE public.group + SET capabilities = array_append(capabilities, 'removing_related_files') + WHERE array_position(capabilities, 'manage_users') IS NOT NULL; + """ + ) + + +def downgrade(): + op.execute( + """ + UPDATE public.group + SET capabilities = array_remove(capabilities, 'access_related_files'); + """ + ) + op.execute( + """ + UPDATE public.group + SET capabilities = array_remove(capabilities, 'adding_related_files'); + """ + ) + op.execute( + """ + UPDATE public.group + SET capabilities = array_remove(capabilities, 'removing_related_files'); + """ + ) diff --git a/mwdb/resources/file.py b/mwdb/resources/file.py index 29fcc53bc..b9dd3afc4 100644 --- a/mwdb/resources/file.py +++ b/mwdb/resources/file.py @@ -590,12 +590,16 @@ def post(self, identifier): @rate_limited_resource class RelatedFileDownloadResource(Resource): + @requires_authorization + @requires_capabilities(Capabilities.access_related_files) def get(self, identifier): """ --- summary: Download related file description: | Returns related file contents. + + Requires `access_related_files` capability. security: - bearerAuth: [] tags: @@ -615,6 +619,8 @@ def get(self, identifier): schema: type: string format: binary + 403: + description: When user doesn't have `access_related_files` capability 404: description: | When related file doesn't exist @@ -684,15 +690,20 @@ def get(self, identifier): "There is no file with provided sha256 or you don't have access to it" ) - related_files = db.session.query(RelatedFile).filter( - RelatedFile.object_id == master_object.id - ) - schema = RelatedFileResponseSchema() + if g.auth_user.has_rights(Capabilities.access_related_files): + related_files = ( + db.session.query(RelatedFile) + .filter(RelatedFile.object_id == master_object.id) + .all() + ) + else: + related_files = [] + schema = RelatedFileResponseSchema() return schema.dump({"related_files": related_files}) @requires_authorization - @requires_capabilities(Capabilities.adding_files) + @requires_capabilities(Capabilities.adding_related_files) def post(self, identifier): """ --- @@ -700,7 +711,7 @@ def post(self, identifier): description: | Uploads a new related file. - Requires `adding_files` capability. + Requires `adding_related_files` capability. security: - bearerAuth: [] tags: @@ -730,6 +741,8 @@ def post(self, identifier): description: OK 400: description: RelatedFile is empty + 403: + description: When user doesn't have `adding_related_files` capability 404: description: | There is no file with provided sha256 or you don't have access to it @@ -755,7 +768,7 @@ def post(self, identifier): return Response("OK") @requires_authorization - @requires_capabilities(Capabilities.removing_objects) + @requires_capabilities(Capabilities.removing_related_files) def delete(self, identifier): """ --- @@ -763,7 +776,7 @@ def delete(self, identifier): description: | Removes a file from the database along with its references. - Requires `removing_objects` capability. + Requires `removing_related_files` capability. security: - bearerAuth: [] tags: @@ -778,7 +791,7 @@ def delete(self, identifier): 200: description: When related file was deleted 403: - description: When user doesn't have `removing_objects` capability + description: When user doesn't have `removing_related_files` capability 404: description: | When related file doesn't exist diff --git a/mwdb/web/src/commons/auth/capabilities.js b/mwdb/web/src/commons/auth/capabilities.js index 73119f1a4..2d3f2f706 100644 --- a/mwdb/web/src/commons/auth/capabilities.js +++ b/mwdb/web/src/commons/auth/capabilities.js @@ -24,6 +24,9 @@ export const Capability = { kartonAssign: "karton_assign", kartonReanalyze: "karton_reanalyze", removingKarton: "karton_unassign", + accessRelatedFiles: "access_related_files", + addingRelatedFiles: "adding_related_files", + removingRelatedFiles: "removing_related_files", }; export let capabilitiesList = { @@ -56,6 +59,9 @@ export let capabilitiesList = { "Can assign existing analysis to the object (required by karton-mwdb-reporter)", [Capability.kartonReanalyze]: "Can resubmit any object for analysis", [Capability.removingKarton]: "Can remove analysis from object", + [Capability.accessRelatedFiles]: "Can view and download RelatedFiles", + [Capability.addingRelatedFiles]: "Can upload new RelatedFiles", + [Capability.removingRelatedFiles]: "Can remove existing RelatedFiles", }; for (let extraCapabilities of fromPlugin("capabilities")) { diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index 6289539b1..d96c31a32 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -90,7 +90,10 @@ function ShowRelatedFiles() { }, [getRelatedFiles]); if (!context.object.related_files) { - return "Loading..."; + return
Loading...
; + } + if (context.object.related_files.length === 0) { + return
Nothing to show here
; } return ( From 561418f5a3e2e7e2ad4bbd2c03e8c1462cd120e7 Mon Sep 17 00:00:00 2001 From: Repumba Date: Wed, 14 Dec 2022 16:31:36 +0100 Subject: [PATCH 19/33] allow one RelatedFile to be associated with many Files, delete RelatedFile when not linked with anything --- mwdb/app.py | 5 + mwdb/model/file.py | 142 ++++++++++++------ mwdb/resources/file.py | 15 +- mwdb/web/src/commons/api/index.js | 4 +- .../ShowObject/Views/RelatedFilesTab.js | 5 +- 5 files changed, 116 insertions(+), 55 deletions(-) diff --git a/mwdb/app.py b/mwdb/app.py index 088181278..5ef735754 100755 --- a/mwdb/app.py +++ b/mwdb/app.py @@ -44,6 +44,7 @@ FileDownloadZipResource, FileItemResource, FileResource, + RelatedFileDeleteResource, RelatedFileDownloadResource, RelatedFileItemResource, ) @@ -266,6 +267,10 @@ def require_auth(): api.add_resource( RelatedFileDownloadResource, "/related_file//download" ) +api.add_resource( + RelatedFileDeleteResource, + "/related_file//delete/", +) # Config endpoints api.add_resource(ConfigResource, "/config") diff --git a/mwdb/model/file.py b/mwdb/model/file.py index 3a60f6027..cbce92af3 100644 --- a/mwdb/model/file.py +++ b/mwdb/model/file.py @@ -6,7 +6,7 @@ import pyzipper from flask import g -from sqlalchemy import or_ +from sqlalchemy import not_, or_ from sqlalchemy.dialects.postgresql.array import ARRAY from sqlalchemy.ext.mutable import MutableList from werkzeug.utils import secure_filename @@ -391,23 +391,32 @@ def create( sha256 = calc_hash(file_stream, hashlib.sha256(), lambda h: h.hexdigest()) - new_related_file = ( - db.session.query(RelatedFile).filter(RelatedFile.sha256 == sha256).first() - ) main_obj = ( db.session.query(Object).filter(Object.dhash == main_obj_dhash).first() ) - - # If file already exists - if new_related_file is not None: - raise FileExistsError("Related file with this sha256 already exists") - - # If related file doesn't exist - if main_obj is None: + # If main file doesn't exist or no access + if main_obj is None or not main_obj.has_explicit_access(g.auth_user): raise ValueError( "There is no object with this sha256 or you don't have access" ) + is_new = True + new_related_file = ( + db.session.query(RelatedFile).filter(RelatedFile.sha256 == sha256).first() + ) + # If RelatedFile already exists + if new_related_file is not None: + is_new = False + new_related_file = ( + db.session.query(RelatedFile) + .filter(RelatedFile.sha256 == sha256) + .filter(RelatedFile.object_id == main_obj.id) + .first() + ) + # If RelatedFile related to main_obj already exists + if new_related_file is not None: + raise FileExistsError("Related file with this sha256 already exists") + new_related_file = RelatedFile( object_id=main_obj.id, file_name=secure_filename(file_name), @@ -415,56 +424,69 @@ def create( sha256=sha256, ) - file_stream.seek(0, os.SEEK_SET) - if app_config.mwdb.storage_provider == StorageProviderType.S3: - get_s3_client( - app_config.mwdb.s3_storage_endpoint, - app_config.mwdb.s3_storage_access_key, - app_config.mwdb.s3_storage_secret_key, - app_config.mwdb.s3_storage_region_name, - app_config.mwdb.s3_storage_secure, - app_config.mwdb.s3_storage_iam_auth, - ).put_object( - Bucket=app_config.mwdb.s3_storage_bucket_name, - Key=new_related_file._calculate_path(), - Body=file_stream, - ) - elif app_config.mwdb.storage_provider == StorageProviderType.DISK: - with open(new_related_file._calculate_path(), "wb") as f: - shutil.copyfileobj(file_stream, f) - else: - raise RuntimeError( - f"StorageProvider {app_config.mwdb.storage_provider} " - f"is not supported" - ) + if is_new: + file_stream.seek(0, os.SEEK_SET) + if app_config.mwdb.storage_provider == StorageProviderType.S3: + get_s3_client( + app_config.mwdb.s3_storage_endpoint, + app_config.mwdb.s3_storage_access_key, + app_config.mwdb.s3_storage_secret_key, + app_config.mwdb.s3_storage_region_name, + app_config.mwdb.s3_storage_secure, + app_config.mwdb.s3_storage_iam_auth, + ).put_object( + Bucket=app_config.mwdb.s3_storage_bucket_name, + Key=new_related_file._calculate_path(), + Body=file_stream, + ) + elif app_config.mwdb.storage_provider == StorageProviderType.DISK: + with open(new_related_file._calculate_path(), "wb") as f: + shutil.copyfileobj(file_stream, f) + else: + raise RuntimeError( + f"StorageProvider {app_config.mwdb.storage_provider} " + f"is not supported" + ) db.session.add(new_related_file) db.session.commit() @classmethod def access(cls, identifier): - related_file_obj = ( - db.session.query(RelatedFile) - .filter(RelatedFile.sha256 == identifier) - .first() + related_files = ( + db.session.query(RelatedFile).filter(RelatedFile.sha256 == identifier).all() ) - if related_file_obj is None: + # Empty list - no such RelatedFile + if not related_files: return None + main_obj_ids = [rf.object_id for rf in related_files] main_obj = ( db.session.query(Object) - .filter(Object.id == related_file_obj.object_id) + .filter(Object.id.in_(main_obj_ids)) + .filter(g.auth_user.has_access_to_object(Object.id)) .first() ) - if not main_obj.has_explicit_access(g.auth_user): + if main_obj is None: return None - return related_file_obj + return related_files[0] @classmethod - def delete(cls, identifier): + def delete(cls, identifier, main_file_identifier): + main_obj = ( + db.session.query(Object) + .filter(Object.dhash == main_file_identifier) + .first() + ) + if not main_obj.has_explicit_access(g.auth_user): + raise ValueError( + "There is no object with this sha256 or you don't have access" + ) + related_file_obj = ( db.session.query(RelatedFile) .filter(RelatedFile.sha256 == identifier) + .filter(RelatedFile.object_id == main_obj.id) .first() ) @@ -472,15 +494,37 @@ def delete(cls, identifier): raise ValueError( "There is no object with this sha256 or you don't have access" ) - main_obj = ( - db.session.query(Object) - .filter(Object.id == related_file_obj.object_id) + + is_last = False + other_related_file_obj = ( + db.session.query(RelatedFile) + .filter(RelatedFile.sha256 == identifier) + .filter(not_(RelatedFile.object_id == main_obj.id)) .first() ) - if not main_obj.has_explicit_access(g.auth_user): - raise ValueError( - "There is no object with this sha256 or you don't have access" - ) + if other_related_file_obj is None: + is_last = True + + if is_last: + if app_config.mwdb.storage_provider == StorageProviderType.S3: + get_s3_client( + app_config.mwdb.s3_storage_endpoint, + app_config.mwdb.s3_storage_access_key, + app_config.mwdb.s3_storage_secret_key, + app_config.mwdb.s3_storage_region_name, + app_config.mwdb.s3_storage_secure, + app_config.mwdb.s3_storage_iam_auth, + ).delete_object( + Bucket=app_config.mwdb.s3_storage_bucket_name, + Key=related_file_obj._calculate_path(), + ) + elif app_config.mwdb.storage_provider == StorageProviderType.DISK: + os.remove(related_file_obj._calculate_path()) + else: + raise RuntimeError( + f"StorageProvider {app_config.mwdb.storage_provider} " + f"is not supported" + ) db.session.delete(related_file_obj) db.session.commit() diff --git a/mwdb/resources/file.py b/mwdb/resources/file.py index b9dd3afc4..71eb62d5d 100644 --- a/mwdb/resources/file.py +++ b/mwdb/resources/file.py @@ -767,9 +767,11 @@ def post(self, identifier): return Response("OK") + +class RelatedFileDeleteResource(Resource): @requires_authorization @requires_capabilities(Capabilities.removing_related_files) - def delete(self, identifier): + def delete(self, identifier, main_file_identifier): """ --- summary: Delete file @@ -784,9 +786,16 @@ def delete(self, identifier): parameters: - in: path name: identifier + required: true + schema: + type: string + description: RelatedFile identifier (SHA256) + - in: path + name: main_file_identifier + required: true schema: type: string - description: RelatedFile identifier (SHA256/SHA512/SHA1/MD5) + description: Main file identifier (SHA256) responses: 200: description: When related file was deleted @@ -801,7 +810,7 @@ def delete(self, identifier): Request canceled due to database statement timeout. """ try: - RelatedFile.delete(identifier) + RelatedFile.delete(identifier, main_file_identifier) except ValueError: raise NotFound( "There is no file with provided sha256 or you don't have access to it" diff --git a/mwdb/web/src/commons/api/index.js b/mwdb/web/src/commons/api/index.js index 537e26030..197d53a34 100644 --- a/mwdb/web/src/commons/api/index.js +++ b/mwdb/web/src/commons/api/index.js @@ -457,8 +457,8 @@ function downloadRelatedFile(id) { }); } -function deleteRelatedFile(id) { - return axios.delete(`/related_file/${id}`); +function deleteRelatedFile(id, main_file_id) { + return axios.delete(`/related_file/${id}/delete/${main_file_id}`); } function getListOfRelatedFiles(id) { diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index d96c31a32..f3b40f3c7 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -62,7 +62,10 @@ function ShowRelatedFiles() { to={`/file/${context.object.sha256}/related_files`} className="nav-link" onClick={async () => { - await api.deleteRelatedFile(sha256); + await api.deleteRelatedFile( + sha256, + context.object.sha256 + ); updateRelatedFiles(api, context); }} > From a58c7edf6d145681deb23859381905ed099c10e1 Mon Sep 17 00:00:00 2001 From: Repumba Date: Thu, 15 Dec 2022 14:04:19 +0100 Subject: [PATCH 20/33] implement requested changes --- mwdb/app.py | 12 +- mwdb/core/file_util.py | 102 ++++++++++ mwdb/model/file.py | 187 +++--------------- mwdb/resources/file.py | 177 ++++++++++------- mwdb/web/src/commons/api/index.js | 16 +- mwdb/web/src/commons/auth/capabilities.js | 6 +- .../ShowObject/Views/RelatedFilesTab.js | 123 ++++++------ 7 files changed, 322 insertions(+), 301 deletions(-) create mode 100644 mwdb/core/file_util.py diff --git a/mwdb/app.py b/mwdb/app.py index 5ef735754..a4afb3a4b 100755 --- a/mwdb/app.py +++ b/mwdb/app.py @@ -44,9 +44,8 @@ FileDownloadZipResource, FileItemResource, FileResource, - RelatedFileDeleteResource, - RelatedFileDownloadResource, RelatedFileItemResource, + RelatedFileResource, ) from mwdb.resources.group import GroupListResource, GroupMemberResource, GroupResource from mwdb.resources.karton import KartonAnalysisResource, KartonObjectResource @@ -263,13 +262,14 @@ def require_auth(): api.add_resource(FileDownloadZipResource, "/file//download/zip") # RelatedFiles endpoints -api.add_resource(RelatedFileItemResource, "/related_file/") api.add_resource( - RelatedFileDownloadResource, "/related_file//download" + RelatedFileResource, + "///related_file", ) api.add_resource( - RelatedFileDeleteResource, - "/related_file//delete/", + RelatedFileItemResource, + "//" + "/related_file/", ) # Config endpoints diff --git a/mwdb/core/file_util.py b/mwdb/core/file_util.py new file mode 100644 index 000000000..bd8320d4e --- /dev/null +++ b/mwdb/core/file_util.py @@ -0,0 +1,102 @@ +import io +import os +import shutil +import tempfile + +from mwdb.core.config import StorageProviderType, app_config +from mwdb.core.util import get_s3_client + + +def write_to_storage(file_stream, file_object): + file_stream.seek(0, os.SEEK_SET) + if app_config.mwdb.storage_provider == StorageProviderType.S3: + get_s3_client( + app_config.mwdb.s3_storage_endpoint, + app_config.mwdb.s3_storage_access_key, + app_config.mwdb.s3_storage_secret_key, + app_config.mwdb.s3_storage_region_name, + app_config.mwdb.s3_storage_secure, + app_config.mwdb.s3_storage_iam_auth, + ).put_object( + Bucket=app_config.mwdb.s3_storage_bucket_name, + Key=file_object._calculate_path(), + Body=file_stream, + ) + elif app_config.mwdb.storage_provider == StorageProviderType.DISK: + with open(file_object._calculate_path(), "wb") as f: + shutil.copyfileobj(file_stream, f) + else: + raise RuntimeError( + f"StorageProvider {app_config.mwdb.storage_provider} " f"is not supported" + ) + + +def get_from_storage(file_object): + if app_config.mwdb.storage_provider == StorageProviderType.S3: + # Stream coming from Boto3 get_object is not buffered and not seekable. + # We need to download it to the temporary file first. + stream = tempfile.TemporaryFile(mode="w+b") + try: + get_s3_client( + app_config.mwdb.s3_storage_endpoint, + app_config.mwdb.s3_storage_access_key, + app_config.mwdb.s3_storage_secret_key, + app_config.mwdb.s3_storage_region_name, + app_config.mwdb.s3_storage_secure, + app_config.mwdb.s3_storage_iam_auth, + ).download_fileobj( + Bucket=app_config.mwdb.s3_storage_bucket_name, + Key=file_object._calculate_path(), + Fileobj=stream, + ) + stream.seek(0, io.SEEK_SET) + return stream + except Exception: + stream.close() + raise + elif app_config.mwdb.storage_provider == StorageProviderType.DISK: + return open(file_object._calculate_path(), "rb") + else: + raise RuntimeError( + f"StorageProvider {app_config.mwdb.storage_provider} is not supported" + ) + + +def delete_from_storage(file_object): + if app_config.mwdb.storage_provider == StorageProviderType.S3: + get_s3_client( + app_config.mwdb.s3_storage_endpoint, + app_config.mwdb.s3_storage_access_key, + app_config.mwdb.s3_storage_secret_key, + app_config.mwdb.s3_storage_region_name, + app_config.mwdb.s3_storage_secure, + app_config.mwdb.s3_storage_iam_auth, + ).delete_object( + Bucket=app_config.mwdb.s3_storage_bucket_name, + Key=file_object._calculate_path(), + ) + elif app_config.mwdb.storage_provider == StorageProviderType.DISK: + os.remove(file_object._calculate_path()) + else: + raise RuntimeError( + f"StorageProvider {app_config.mwdb.storage_provider} " f"is not supported" + ) + + +def iterate_buffer(file_object, chunk_size=1024 * 256): + """ + Iterates over bytes in the file contents + """ + fh = file_object.open() + try: + if hasattr(fh, "stream"): + yield from fh.stream(chunk_size) + else: + while True: + chunk = fh.read(chunk_size) + if chunk: + yield chunk + else: + return + finally: + fh.close() diff --git a/mwdb/model/file.py b/mwdb/model/file.py index cbce92af3..e3987551a 100644 --- a/mwdb/model/file.py +++ b/mwdb/model/file.py @@ -13,15 +13,14 @@ from mwdb.core.auth import AuthScope, generate_token, verify_token from mwdb.core.config import StorageProviderType, app_config -from mwdb.core.karton import send_file_to_karton -from mwdb.core.util import ( - calc_crc32, - calc_hash, - calc_magic, - calc_ssdeep, - get_fd_path, - get_s3_client, +from mwdb.core.file_util import ( + delete_from_storage, + get_from_storage, + iterate_buffer, + write_to_storage, ) +from mwdb.core.karton import send_file_to_karton +from mwdb.core.util import calc_crc32, calc_hash, calc_magic, calc_ssdeep, get_fd_path from . import db from .object import Object @@ -126,28 +125,7 @@ def get_or_create( file_obj.alt_names.append(original_filename) if is_new: - file_stream.seek(0, os.SEEK_SET) - if app_config.mwdb.storage_provider == StorageProviderType.S3: - get_s3_client( - app_config.mwdb.s3_storage_endpoint, - app_config.mwdb.s3_storage_access_key, - app_config.mwdb.s3_storage_secret_key, - app_config.mwdb.s3_storage_region_name, - app_config.mwdb.s3_storage_secure, - app_config.mwdb.s3_storage_iam_auth, - ).put_object( - Bucket=app_config.mwdb.s3_storage_bucket_name, - Key=file_obj._calculate_path(), - Body=file_stream, - ) - elif app_config.mwdb.storage_provider == StorageProviderType.DISK: - with open(file_obj._calculate_path(), "wb") as f: - shutil.copyfileobj(file_stream, f) - else: - raise RuntimeError( - f"StorageProvider {app_config.mwdb.storage_provider} " - f"is not supported" - ) + write_to_storage(file_stream, file_obj) file_obj.upload_stream = file_stream return file_obj, is_new @@ -223,34 +201,7 @@ def open(self): stream = os.fdopen(dupfd, "rb") stream.seek(0, os.SEEK_SET) return stream - if app_config.mwdb.storage_provider == StorageProviderType.S3: - # Stream coming from Boto3 get_object is not buffered and not seekable. - # We need to download it to the temporary file first. - stream = tempfile.TemporaryFile(mode="w+b") - try: - get_s3_client( - app_config.mwdb.s3_storage_endpoint, - app_config.mwdb.s3_storage_access_key, - app_config.mwdb.s3_storage_secret_key, - app_config.mwdb.s3_storage_region_name, - app_config.mwdb.s3_storage_secure, - app_config.mwdb.s3_storage_iam_auth, - ).download_fileobj( - Bucket=app_config.mwdb.s3_storage_bucket_name, - Key=self._calculate_path(), - Fileobj=stream, - ) - stream.seek(0, io.SEEK_SET) - return stream - except Exception: - stream.close() - raise - elif app_config.mwdb.storage_provider == StorageProviderType.DISK: - return open(self._calculate_path(), "rb") - else: - raise RuntimeError( - f"StorageProvider {app_config.mwdb.storage_provider} is not supported" - ) + return get_from_storage(self) def read(self): """ @@ -425,54 +376,37 @@ def create( ) if is_new: - file_stream.seek(0, os.SEEK_SET) - if app_config.mwdb.storage_provider == StorageProviderType.S3: - get_s3_client( - app_config.mwdb.s3_storage_endpoint, - app_config.mwdb.s3_storage_access_key, - app_config.mwdb.s3_storage_secret_key, - app_config.mwdb.s3_storage_region_name, - app_config.mwdb.s3_storage_secure, - app_config.mwdb.s3_storage_iam_auth, - ).put_object( - Bucket=app_config.mwdb.s3_storage_bucket_name, - Key=new_related_file._calculate_path(), - Body=file_stream, - ) - elif app_config.mwdb.storage_provider == StorageProviderType.DISK: - with open(new_related_file._calculate_path(), "wb") as f: - shutil.copyfileobj(file_stream, f) - else: - raise RuntimeError( - f"StorageProvider {app_config.mwdb.storage_provider} " - f"is not supported" - ) + write_to_storage(file_stream, new_related_file) + db.session.add(new_related_file) db.session.commit() @classmethod - def access(cls, identifier): - related_files = ( - db.session.query(RelatedFile).filter(RelatedFile.sha256 == identifier).all() + def access(cls, main_obj_identifier, identifier): + main_obj = ( + db.session.query(Object).filter(Object.dhash == main_obj_identifier).first() ) - # Empty list - no such RelatedFile - if not related_files: - return None + if not main_obj.has_explicit_access(g.auth_user): + raise ValueError( + "There is no object with this sha256 or you don't have access" + ) - main_obj_ids = [rf.object_id for rf in related_files] - main_obj = ( - db.session.query(Object) - .filter(Object.id.in_(main_obj_ids)) - .filter(g.auth_user.has_access_to_object(Object.id)) + related_file = ( + db.session.query(RelatedFile) + .filter(RelatedFile.sha256 == identifier) + .filter(RelatedFile.object_id == main_obj.id) .first() ) - if main_obj is None: - return None + # Empty list - no such RelatedFile + if related_file is None: + raise ValueError( + "There is no object with this sha256 or you don't have access" + ) - return related_files[0] + return related_file @classmethod - def delete(cls, identifier, main_file_identifier): + def delete(cls, main_file_identifier, identifier): main_obj = ( db.session.query(Object) .filter(Object.dhash == main_file_identifier) @@ -506,25 +440,7 @@ def delete(cls, identifier, main_file_identifier): is_last = True if is_last: - if app_config.mwdb.storage_provider == StorageProviderType.S3: - get_s3_client( - app_config.mwdb.s3_storage_endpoint, - app_config.mwdb.s3_storage_access_key, - app_config.mwdb.s3_storage_secret_key, - app_config.mwdb.s3_storage_region_name, - app_config.mwdb.s3_storage_secure, - app_config.mwdb.s3_storage_iam_auth, - ).delete_object( - Bucket=app_config.mwdb.s3_storage_bucket_name, - Key=related_file_obj._calculate_path(), - ) - elif app_config.mwdb.storage_provider == StorageProviderType.DISK: - os.remove(related_file_obj._calculate_path()) - else: - raise RuntimeError( - f"StorageProvider {app_config.mwdb.storage_provider} " - f"is not supported" - ) + delete_from_storage(related_file_obj) db.session.delete(related_file_obj) db.session.commit() @@ -534,49 +450,10 @@ def open(self): """ Opens the related file stream with contents. """ - if app_config.mwdb.storage_provider == StorageProviderType.S3: - # Stream coming from Boto3 get_object is not buffered and not seekable. - # We need to download it to the temporary file first. - stream = tempfile.TemporaryFile(mode="w+b") - try: - get_s3_client( - app_config.mwdb.s3_storage_endpoint, - app_config.mwdb.s3_storage_access_key, - app_config.mwdb.s3_storage_secret_key, - app_config.mwdb.s3_storage_region_name, - app_config.mwdb.s3_storage_secure, - app_config.mwdb.s3_storage_iam_auth, - ).download_fileobj( - Bucket=app_config.mwdb.s3_storage_bucket_name, - Key=self._calculate_path(), - Fileobj=stream, - ) - stream.seek(0, io.SEEK_SET) - return stream - except Exception: - stream.close() - raise - elif app_config.mwdb.storage_provider == StorageProviderType.DISK: - return open(self._calculate_path(), "rb") - else: - raise RuntimeError( - f"StorageProvider {app_config.mwdb.storage_provider} is not supported" - ) + return get_from_storage(self) def iterate(self, chunk_size=1024 * 256): """ Iterates over bytes in the file contents """ - fh = self.open() - try: - if hasattr(fh, "stream"): - yield from fh.stream(chunk_size) - else: - while True: - chunk = fh.read(chunk_size) - if chunk: - yield chunk - else: - return - finally: - fh.close() + return iterate_buffer(self) diff --git a/mwdb/resources/file.py b/mwdb/resources/file.py index 71eb62d5d..ec7daeffd 100644 --- a/mwdb/resources/file.py +++ b/mwdb/resources/file.py @@ -589,82 +589,29 @@ def post(self, identifier): @rate_limited_resource -class RelatedFileDownloadResource(Resource): - @requires_authorization - @requires_capabilities(Capabilities.access_related_files) - def get(self, identifier): +class RelatedFileResource(Resource): + def get(self, type, main_obj_identifier): """ --- - summary: Download related file + summary: Get list of RelatedFiles description: | - Returns related file contents. - - Requires `access_related_files` capability. + Returns list of RelatedFiles for a File specified by sha256 security: - bearerAuth: [] tags: - related_file parameters: - in: path - name: identifier + name: type schema: type: string - description: RelatedFile identifier (SHA256) - required: true - responses: - 200: - description: RelatedFile contents - content: - application/octet-stream: - schema: - type: string - format: binary - 403: - description: When user doesn't have `access_related_files` capability - 404: - description: | - When related file doesn't exist - or user doesn't have access to it. - 503: - description: | - Request canceled due to database statement timeout. - """ - - if not g.auth_user: - raise Unauthorized("Not authenticated.") - - related_file_obj = RelatedFile.access(identifier) - - if related_file_obj is None: - raise NotFound("Object not found") - - return Response( - related_file_obj.iterate(), - content_type="application/octet-stream", - headers={ - "Content-disposition": f"attachment; filename={related_file_obj.sha256}" - }, - ) - - -@rate_limited_resource -class RelatedFileItemResource(Resource): - def get(self, identifier): - """ - --- - summary: Get list of RelatedFiles - description: | - Returns list of RelatedFiles for a File specified by sha256 - security: - - bearerAuth: [] - tags: - - related_file - parameters: + enum: [file, config, blob, object] + description: Type of object - in: path - name: identifier + name: main_obj_identifier schema: type: string - description: Master File identifier (SHA256) + description: Main object identifier (SHA256) required: true responses: 200: @@ -681,7 +628,7 @@ def get(self, identifier): """ master_object = ( db.session.query(Object) - .filter(Object.dhash == identifier) + .filter(Object.dhash == main_obj_identifier) .filter(g.auth_user.has_access_to_object(Object.id)) ).first() @@ -704,7 +651,7 @@ def get(self, identifier): @requires_authorization @requires_capabilities(Capabilities.adding_related_files) - def post(self, identifier): + def post(self, type, main_obj_identifier): """ --- summary: Upload related file @@ -718,10 +665,16 @@ def post(self, identifier): - related_file parameters: - in: path - name: identifier + name: type + schema: + type: string + enum: [file, config, blob, object] + description: Type of object + - in: path + name: main_obj_identifier schema: type: string - description: Master File identifier (SHA256) + description: Main object identifier (SHA256) required: true requestBody: required: true @@ -754,7 +707,9 @@ def post(self, identifier): """ try: RelatedFile.create( - request.files["file"].filename, request.files["file"].stream, identifier + request.files["file"].filename, + request.files["file"].stream, + main_obj_identifier, ) except EmptyFileError: raise BadRequest("RelatedFile cannot be empty") @@ -768,10 +723,78 @@ def post(self, identifier): return Response("OK") -class RelatedFileDeleteResource(Resource): +class RelatedFileItemResource(Resource): + @requires_authorization + @requires_capabilities(Capabilities.access_related_files) + def get(self, type, main_obj_identifier, identifier): + """ + --- + summary: Download related file + description: | + Returns related file contents. + + Requires `access_related_files` capability. + security: + - bearerAuth: [] + tags: + - related_file + parameters: + - in: path + name: type + schema: + type: string + enum: [file, config, blob, object] + description: Type of object + - in: path + name: main_obj_identifier + required: true + schema: + type: string + description: Main object identifier (SHA256) + - in: path + name: identifier + required: true + schema: + type: string + description: RelatedFile identifier (SHA256) + responses: + 200: + description: RelatedFile contents + content: + application/octet-stream: + schema: + type: string + format: binary + 403: + description: When user doesn't have `access_related_files` capability + 404: + description: | + When related file doesn't exist + or user doesn't have access to it. + 503: + description: | + Request canceled due to database statement timeout. + """ + + if not g.auth_user: + raise Unauthorized("Not authenticated.") + + try: + related_file_obj = RelatedFile.access(main_obj_identifier, identifier) + except ValueError: + raise NotFound("Object not found") + + return Response( + related_file_obj.iterate(), + content_type="application/octet-stream", + headers={ + "Content-disposition": f"attachment; filename={related_file_obj.sha256}" + }, + ) + @requires_authorization @requires_capabilities(Capabilities.removing_related_files) - def delete(self, identifier, main_file_identifier): + def delete(self, type, main_obj_identifier, identifier): """ --- summary: Delete file @@ -785,17 +808,23 @@ def delete(self, identifier, main_file_identifier): - related_file parameters: - in: path - name: identifier + name: type + schema: + type: string + enum: [file, config, blob, object] + description: Type of object + - in: path + name: main_obj_identifier required: true schema: type: string - description: RelatedFile identifier (SHA256) + description: Main object identifier (SHA256) - in: path - name: main_file_identifier + name: identifier required: true schema: type: string - description: Main file identifier (SHA256) + description: RelatedFile identifier (SHA256) responses: 200: description: When related file was deleted @@ -810,7 +839,7 @@ def delete(self, identifier, main_file_identifier): Request canceled due to database statement timeout. """ try: - RelatedFile.delete(identifier, main_file_identifier) + RelatedFile.delete(main_obj_identifier, identifier) except ValueError: raise NotFound( "There is no file with provided sha256 or you don't have access to it" diff --git a/mwdb/web/src/commons/api/index.js b/mwdb/web/src/commons/api/index.js index 197d53a34..a1c839282 100644 --- a/mwdb/web/src/commons/api/index.js +++ b/mwdb/web/src/commons/api/index.js @@ -444,25 +444,25 @@ function uploadFile(file, parent, upload_as, attributes, fileUploadTimeout) { return axios.post(`/file`, formData, { timeout: fileUploadTimeout }); } -function uploadRelatedFile(file, masterFileDhash) { +function uploadRelatedFile(file, mainFileDhash, type = "object") { let formData = new FormData(); formData.append("file", file); - return axios.post(`/related_file/${masterFileDhash}`, formData); + return axios.post(`/${type}/${mainFileDhash}/related_file`, formData); } -function downloadRelatedFile(id) { - return axios.get(`/related_file/${id}/download`, { +function downloadRelatedFile(mainFileDhash, id, type = "object") { + return axios.get(`/${type}/${mainFileDhash}/related_file/${id}`, { responseType: "arraybuffer", responseEncoding: "binary", }); } -function deleteRelatedFile(id, main_file_id) { - return axios.delete(`/related_file/${id}/delete/${main_file_id}`); +function deleteRelatedFile(mainFileDhash, id, type = "object") { + return axios.delete(`/${type}/${mainFileDhash}/related_file/${id}`); } -function getListOfRelatedFiles(id) { - return axios.get(`/related_file/${id}`); +function getListOfRelatedFiles(mainFileDhash, type = "object") { + return axios.get(`/${type}/${mainFileDhash}/related_file`); } function getRemoteNames() { diff --git a/mwdb/web/src/commons/auth/capabilities.js b/mwdb/web/src/commons/auth/capabilities.js index 2d3f2f706..f3cabba79 100644 --- a/mwdb/web/src/commons/auth/capabilities.js +++ b/mwdb/web/src/commons/auth/capabilities.js @@ -59,9 +59,9 @@ export let capabilitiesList = { "Can assign existing analysis to the object (required by karton-mwdb-reporter)", [Capability.kartonReanalyze]: "Can resubmit any object for analysis", [Capability.removingKarton]: "Can remove analysis from object", - [Capability.accessRelatedFiles]: "Can view and download RelatedFiles", - [Capability.addingRelatedFiles]: "Can upload new RelatedFiles", - [Capability.removingRelatedFiles]: "Can remove existing RelatedFiles", + [Capability.accessRelatedFiles]: "Can view and download related files", + [Capability.addingRelatedFiles]: "Can upload new related files", + [Capability.removingRelatedFiles]: "Can remove existing related files", }; for (let extraCapabilities of fromPlugin("capabilities")) { diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index f3b40f3c7..4a93d0449 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -10,9 +10,13 @@ import { } from "@fortawesome/free-solid-svg-icons"; import { ObjectContext } from "@mwdb-web/commons/context"; -import { ObjectAction, ObjectTab } from "@mwdb-web/commons/ui"; +import { + ObjectAction, + ObjectTab, + ConfirmationModal, +} from "@mwdb-web/commons/ui"; import { APIContext } from "@mwdb-web/commons/api/context"; -import { humanFileSize } from "@mwdb-web/commons/helpers"; +import { humanFileSize, downloadData } from "@mwdb-web/commons/helpers"; import ReactModal from "react-modal"; async function updateRelatedFiles(api, context) { @@ -27,56 +31,71 @@ async function updateRelatedFiles(api, context) { } } +function RelatedFileItem({ file_name, file_size, sha256 }) { + const api = useContext(APIContext); + const context = useContext(ObjectContext); + const [isConfirmationModalOpen, setConfirmationModalOpen] = useState(null); + + return ( + + {file_name} + {humanFileSize(file_size)} + + { + let content = await api.downloadRelatedFile( + context.object.sha256, + sha256 + ); + downloadData( + content.data, + file_name, + "application/octet-stream" + ); + }} + > + +  Download + + + + setConfirmationModalOpen(true)} + > + +  Remove + + { + setConfirmationModalOpen(false); + }} + onConfirm={async () => { + await api.deleteRelatedFile( + context.object.sha256, + sha256 + ); + updateRelatedFiles(api, context); + setConfirmationModalOpen(false); + }} + message={`Are you sure you want to delete this related file?`} + buttonStyle="btn-success" + confirmText="Yes" + /> + + + ); +} + function ShowRelatedFiles() { const api = useContext(APIContext); const context = useContext(ObjectContext); const { setObjectError, updateObjectData } = context; - function RelatedFileItem({ file_name, file_size, sha256 }) { - return ( - - {file_name} - {humanFileSize(file_size)} - - { - let content = await api.downloadRelatedFile(sha256); - let blob = new Blob([content.data], { - type: "application/octet-stream", - }); - let tempLink = document.createElement("a"); - tempLink.style.display = "none"; - tempLink.href = window.URL.createObjectURL(blob); - tempLink.download = file_name; - tempLink.click(); - }} - > - -  Download - - - - { - await api.deleteRelatedFile( - sha256, - context.object.sha256 - ); - updateRelatedFiles(api, context); - }} - > - -  Remove - - - - ); - } - const getRelatedFiles = useCallback(updateRelatedFiles, [ api, setObjectError, @@ -84,7 +103,7 @@ function ShowRelatedFiles() { context.object.sha256, ]); - // JS throws a warning "Line 90:8: React Hook useEffect has missing dependencies: 'api' and 'context'" + // JS throws a warning "Line ***: React Hook useEffect has missing dependencies: 'api' and 'context'" // Those dependencies are skipped on purpose // To disable this warning I used 'eslint-disable-next-line' useEffect(() => { @@ -177,13 +196,7 @@ export default function RelatedFilesTab() { name="RelatedFileUploadField" type="file" required="required" - onChange={() => - setFile( - document.forms["RelatedFileUploadForm"][ - "RelatedFileUploadField" - ].files[0] - ) - } + onChange={(event) => setFile(event.target.files[0])} />
Date: Fri, 16 Dec 2022 11:32:12 +0100 Subject: [PATCH 21/33] visual improvements --- .../ShowObject/Views/RelatedFilesTab.js | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index 4a93d0449..21498534f 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -35,13 +35,27 @@ function RelatedFileItem({ file_name, file_size, sha256 }) { const api = useContext(APIContext); const context = useContext(ObjectContext); const [isConfirmationModalOpen, setConfirmationModalOpen] = useState(null); + const linkStyle = { + display: "inline-block", + padding: "8px", + }; + const tdStyle = { + padding: "10px 20px 10px 20px", + width: "80%", + borderRight: "none", + borderLeft: "none", + }; return ( - {file_name} - {humanFileSize(file_size)} - + + {file_name} +
+ {humanFileSize(file_size)} + + { @@ -56,18 +70,15 @@ function RelatedFileItem({ file_name, file_size, sha256 }) { ); }} > - -  Download + - - setConfirmationModalOpen(true)} > - -  Remove + - - File name - File size - Download - Remove + + + + {context.object.related_files.map((related_file) => ( From e4d0b5ef27bfeebda7a4d6b403694ffe302ba53b Mon Sep 17 00:00:00 2001 From: Repumba Date: Fri, 16 Dec 2022 14:42:28 +0100 Subject: [PATCH 22/33] New action - dowload all --- mwdb/app.py | 6 ++ mwdb/model/file.py | 36 ++++++++++-- mwdb/resources/file.py | 57 +++++++++++++++++++ mwdb/web/src/commons/api/index.js | 6 ++ .../ShowObject/Views/RelatedFilesTab.js | 12 ++++ 5 files changed, 112 insertions(+), 5 deletions(-) diff --git a/mwdb/app.py b/mwdb/app.py index a4afb3a4b..51d67115f 100755 --- a/mwdb/app.py +++ b/mwdb/app.py @@ -46,6 +46,7 @@ FileResource, RelatedFileItemResource, RelatedFileResource, + RelatedFileZipDownloadResource, ) from mwdb.resources.group import GroupListResource, GroupMemberResource, GroupResource from mwdb.resources.karton import KartonAnalysisResource, KartonObjectResource @@ -271,6 +272,11 @@ def require_auth(): "//" "/related_file/", ) +api.add_resource( + RelatedFileZipDownloadResource, + "//" + "/related_file/zip", +) # Config endpoints api.add_resource(ConfigResource, "/config") diff --git a/mwdb/model/file.py b/mwdb/model/file.py index e3987551a..fdda8c2e7 100644 --- a/mwdb/model/file.py +++ b/mwdb/model/file.py @@ -386,6 +386,10 @@ def access(cls, main_obj_identifier, identifier): main_obj = ( db.session.query(Object).filter(Object.dhash == main_obj_identifier).first() ) + if main_obj is None: + raise ValueError( + "There is no object with this sha256 or you don't have access" + ) if not main_obj.has_explicit_access(g.auth_user): raise ValueError( "There is no object with this sha256 or you don't have access" @@ -406,12 +410,14 @@ def access(cls, main_obj_identifier, identifier): return related_file @classmethod - def delete(cls, main_file_identifier, identifier): + def delete(cls, main_obj_identifier, identifier): main_obj = ( - db.session.query(Object) - .filter(Object.dhash == main_file_identifier) - .first() + db.session.query(Object).filter(Object.dhash == main_obj_identifier).first() ) + if main_obj is None: + raise ValueError( + "There is no object with this sha256 or you don't have access" + ) if not main_obj.has_explicit_access(g.auth_user): raise ValueError( "There is no object with this sha256 or you don't have access" @@ -456,4 +462,24 @@ def iterate(self, chunk_size=1024 * 256): """ Iterates over bytes in the file contents """ - return iterate_buffer(self) + return iterate_buffer(self, chunk_size) + + @classmethod + def zip_all(cls, main_obj_identifier): + main_obj = ( + db.session.query(Object).filter(Object.dhash == main_obj_identifier).first() + ) + if main_obj is None: + raise ValueError( + "There is no object with this sha256 or you don't have access" + ) + related_files = ( + db.session.query(RelatedFile).filter(RelatedFile.object_id == main_obj.id) + ).all() + + with tempfile.NamedTemporaryFile() as temp_file: + with open(temp_file.name, "rb") as reader: + with pyzipper.ZipFile(temp_file, "w") as zip_file: + for rf in related_files: + zip_file.write(rf._calculate_path(), arcname=rf.file_name) + return reader.read() diff --git a/mwdb/resources/file.py b/mwdb/resources/file.py index ec7daeffd..7bfaefb9b 100644 --- a/mwdb/resources/file.py +++ b/mwdb/resources/file.py @@ -846,3 +846,60 @@ def delete(self, type, main_obj_identifier, identifier): ) return Response("OK") + + +class RelatedFileZipDownloadResource(Resource): + def get(self, type, main_obj_identifier): + """ + --- + summary: Download every related file for a provided main object + description: | + Returns zipped related file contents. + + Requires `access_related_files` capability. + security: + - bearerAuth: [] + tags: + - related_file + parameters: + - in: path + name: type + schema: + type: string + enum: [file, config, blob, object] + description: Type of object + - in: path + name: main_obj_identifier + required: true + schema: + type: string + description: Main object identifier (SHA256) + responses: + 200: + description: RelatedFile contents + content: + application/octet-stream: + schema: + type: string + format: binary + 403: + description: When user doesn't have `access_related_files` capability + 404: + description: | + When related file doesn't exist + or user doesn't have access to it. + 503: + description: | + Request canceled due to database statement timeout. + """ + try: + zipped_related_files = RelatedFile.zip_all(main_obj_identifier) + except ValueError: + raise NotFound("Object not found") + + zip_file_name = "related_files" + main_obj_identifier + ".zip" + return Response( + zipped_related_files, + content_type="application/octet-stream", + headers={"Content-disposition": f"attachment; filename={zip_file_name}"}, + ) diff --git a/mwdb/web/src/commons/api/index.js b/mwdb/web/src/commons/api/index.js index a1c839282..9f23cff25 100644 --- a/mwdb/web/src/commons/api/index.js +++ b/mwdb/web/src/commons/api/index.js @@ -465,6 +465,11 @@ function getListOfRelatedFiles(mainFileDhash, type = "object") { return axios.get(`/${type}/${mainFileDhash}/related_file`); } +function getZippedRelatedFilesLink(mainFileDhash, type = "object") { + const baseURL = getApiForEnvironment(); + return `${baseURL}/${type}/${mainFileDhash}/related_file/zip`; +} + function getRemoteNames() { return axios.get("/remote"); } @@ -651,6 +656,7 @@ const api = { downloadRelatedFile, deleteRelatedFile, getListOfRelatedFiles, + getZippedRelatedFilesLink, getRemoteNames, pushObjectRemote, pullObjectRemote, diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js index 21498534f..a44689def 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js @@ -7,6 +7,7 @@ import { faExternalLinkSquare, faDownload, faTrash, + faArchive, } from "@fortawesome/free-solid-svg-icons"; import { ObjectContext } from "@mwdb-web/commons/context"; @@ -183,6 +184,12 @@ export default function RelatedFilesTab() { } } + async function handleDownloadAll() { + window.location.href = await api.getZippedRelatedFilesLink( + context.object.sha256 + ); + } + return (
, + , ]} /> Date: Fri, 16 Dec 2022 15:06:16 +0100 Subject: [PATCH 23/33] Rename variables, improve descriptions --- mwdb/resources/file.py | 58 ++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/mwdb/resources/file.py b/mwdb/resources/file.py index 7bfaefb9b..b53705a50 100644 --- a/mwdb/resources/file.py +++ b/mwdb/resources/file.py @@ -593,9 +593,9 @@ class RelatedFileResource(Resource): def get(self, type, main_obj_identifier): """ --- - summary: Get list of RelatedFiles + summary: Get list of related files description: | - Returns list of RelatedFiles for a File specified by sha256 + Returns list of related files for an object specified by sha256 security: - bearerAuth: [] tags: @@ -615,32 +615,33 @@ def get(self, type, main_obj_identifier): required: true responses: 200: - description: List of RelatedFiles + description: List of related files content: application/json: schema: RelatedFileResponseSchema 404: description: | - There is no file with provided sha256 or you don't have access to it + There is no object with provided sha256 + or user doesn't have access to it 503: description: | Request canceled due to database statement timeout. """ - master_object = ( + main_object = ( db.session.query(Object) .filter(Object.dhash == main_obj_identifier) .filter(g.auth_user.has_access_to_object(Object.id)) ).first() - if master_object is None: + if main_object is None: raise NotFound( - "There is no file with provided sha256 or you don't have access to it" + "There is no object with provided sha256 or you don't have access to it" ) if g.auth_user.has_rights(Capabilities.access_related_files): related_files = ( db.session.query(RelatedFile) - .filter(RelatedFile.object_id == master_object.id) + .filter(RelatedFile.object_id == main_object.id) .all() ) else: @@ -686,21 +687,22 @@ def post(self, type, main_obj_identifier): file: type: string format: binary - description: RelatedFile contents to be uploaded + description: Related file contents to be uploaded required: - file responses: 200: description: OK 400: - description: RelatedFile is empty + description: Related file is empty 403: description: When user doesn't have `adding_related_files` capability 404: description: | - There is no file with provided sha256 or you don't have access to it + There is no object with provided sha256 + or user doesn't have access to it 409: - description: RelatedFile already exists + description: Such related file already exists 503: description: | Request canceled due to database statement timeout. @@ -712,13 +714,13 @@ def post(self, type, main_obj_identifier): main_obj_identifier, ) except EmptyFileError: - raise BadRequest("RelatedFile cannot be empty") + raise BadRequest("Related file cannot be empty") except ValueError: raise NotFound( - "There is no file with provided sha256 or you don't have access to it" + "There is no object with provided sha256 or you don't have access to it" ) except FileExistsError: - raise Conflict("Related file with this sha256 already exists") + raise Conflict("Such related file already exists") return Response("OK") @@ -756,10 +758,10 @@ def get(self, type, main_obj_identifier, identifier): required: true schema: type: string - description: RelatedFile identifier (SHA256) + description: Related file identifier (SHA256) responses: 200: - description: RelatedFile contents + description: Related file contents content: application/octet-stream: schema: @@ -782,7 +784,7 @@ def get(self, type, main_obj_identifier, identifier): try: related_file_obj = RelatedFile.access(main_obj_identifier, identifier) except ValueError: - raise NotFound("Object not found") + raise NotFound("Related file not found") return Response( related_file_obj.iterate(), @@ -797,9 +799,9 @@ def get(self, type, main_obj_identifier, identifier): def delete(self, type, main_obj_identifier, identifier): """ --- - summary: Delete file + summary: Delete related file description: | - Removes a file from the database along with its references. + Removes a related file from the database. Requires `removing_related_files` capability. security: @@ -824,7 +826,7 @@ def delete(self, type, main_obj_identifier, identifier): required: true schema: type: string - description: RelatedFile identifier (SHA256) + description: Related file identifier (SHA256) responses: 200: description: When related file was deleted @@ -833,7 +835,7 @@ def delete(self, type, main_obj_identifier, identifier): 404: description: | When related file doesn't exist - or user doesn't have access to this object. + or user doesn't have access to it. 503: description: | Request canceled due to database statement timeout. @@ -876,7 +878,7 @@ def get(self, type, main_obj_identifier): description: Main object identifier (SHA256) responses: 200: - description: RelatedFile contents + description: Contents of related file content: application/octet-stream: schema: @@ -886,8 +888,8 @@ def get(self, type, main_obj_identifier): description: When user doesn't have `access_related_files` capability 404: description: | - When related file doesn't exist - or user doesn't have access to it. + There is no object with provided sha256 + or user doesn't have access to it 503: description: | Request canceled due to database statement timeout. @@ -895,9 +897,11 @@ def get(self, type, main_obj_identifier): try: zipped_related_files = RelatedFile.zip_all(main_obj_identifier) except ValueError: - raise NotFound("Object not found") + raise NotFound( + "There is no object with provided sha256 or you don't have access to it" + ) - zip_file_name = "related_files" + main_obj_identifier + ".zip" + zip_file_name = "related_files_" + main_obj_identifier + ".zip" return Response( zipped_related_files, content_type="application/octet-stream", From 7f6145006059441f676362a6b743095576a80211 Mon Sep 17 00:00:00 2001 From: Repumba Date: Mon, 19 Dec 2022 10:03:43 +0100 Subject: [PATCH 24/33] fix: unable to delete object with related files --- mwdb/model/object.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mwdb/model/object.py b/mwdb/model/object.py index 6a946a15d..c73f0b5ad 100644 --- a/mwdb/model/object.py +++ b/mwdb/model/object.py @@ -261,6 +261,7 @@ class Object(db.Model): "RelatedFile", back_populates="related_object", lazy=True, + cascade="save-update, merge, delete", ) followers = db.relationship( From 0003992f20f2cd440e8dc0b7b3bacf6fa0027eaa Mon Sep 17 00:00:00 2001 From: Repumba Date: Mon, 19 Dec 2022 15:38:53 +0100 Subject: [PATCH 25/33] add tests --- tests/backend/test_permissions.py | 81 +++++++++++++++++++++++++++++++ tests/backend/utils.py | 27 +++++++++++ 2 files changed, 108 insertions(+) diff --git a/tests/backend/test_permissions.py b/tests/backend/test_permissions.py index 5ddfbc4ae..cbe0ac059 100644 --- a/tests/backend/test_permissions.py +++ b/tests/backend/test_permissions.py @@ -239,3 +239,84 @@ def test_removing_object_with_comments(admin_session): with ShouldRaise(status_code=404): admin_session.get_sample(sample.dhash) + + +def test_adding_related_files(admin_session): + testCase = RelationTestCase(admin_session) + + Alice = testCase.new_user("Alice") + Bob = testCase.new_user("Bob", capabilities=["adding_related_files"]) + + SampleA = testCase.new_sample("SampleA") + SampleA.create(Alice, upload_as="public") + SampleB = testCase.new_sample("SampleB") + + # Alice doesn't have capability + with ShouldRaise(status_code=403): + Alice.session.add_related_file(SampleA.dhash, "RelatedFileA") + + # Everything works fine + Bob.session.add_related_file(SampleA.dhash, "RelatedFileA") + assert len( testCase.session.get_related_files(SampleA.dhash)['related_files'] ) == 1 + + # Bob can't add related files to objects not accessible for him + with ShouldRaise(status_code=404): + Bob.session.add_related_file(SampleB.dhash, "RelatedFileB") + + +def test_removing_related_files(admin_session): + testCase = RelationTestCase(admin_session) + + Alice = testCase.new_user("Alice") + Bob = testCase.new_user("Bob", capabilities=["removing_related_files"]) + + SampleA = testCase.new_sample("SampleA") + SampleA.create(Alice, upload_as="public") + SampleB = testCase.new_sample("SampleB") + + related_file_content = rand_string() + related_file_dhash = calc_sha256(related_file_content) + + # Related files added by admin session + testCase.session.add_related_file(SampleA.dhash, "RelatedFile", related_file_content) + testCase.session.add_related_file(SampleB.dhash, "RelatedFile", related_file_content) + + assert len( testCase.session.get_related_files(SampleA.dhash)['related_files'] ) == 1 + assert len( testCase.session.get_related_files(SampleB.dhash)['related_files'] ) == 1 + + # Alice doesn't have capability + with ShouldRaise(status_code=403): + Alice.session.remove_related_file(SampleA.dhash, related_file_dhash) + + # Bob can't remove related files for objects not accessible for him + with ShouldRaise(status_code=404): + Bob.session.remove_related_file(SampleB.dhash, related_file_dhash) + + # Everything works file + Bob.session.remove_related_file(SampleA.dhash, related_file_dhash) + + assert len( testCase.session.get_related_files(SampleA.dhash)['related_files'] ) == 0 + assert len( testCase.session.get_related_files(SampleB.dhash)['related_files'] ) == 1 + + +def test_accessing_related_files(admin_session): + testCase = RelationTestCase(admin_session) + + Alice = testCase.new_user("Alice") + Bob = testCase.new_user("Bob", capabilities=["access_related_files"]) + + SampleA = testCase.new_sample("SampleA") + SampleA.create(Alice, upload_as="public") + SampleB = testCase.new_sample("SampleB") + + testCase.session.add_related_file(SampleA.dhash, "RelatedFile") + + # Alice doesn't have capability + assert len( Alice.session.get_related_files(SampleA.dhash)['related_files'] ) == 0 + + # Everything works fine + assert len( Bob.session.get_related_files(SampleA.dhash)['related_files'] ) == 1 + + # Bob can't access related files for objects not accessible for him + with ShouldRaise(status_code=404): + Bob.session.get_related_files(SampleB.dhash) diff --git a/tests/backend/utils.py b/tests/backend/utils.py index db7cc39c8..f4cd002bb 100644 --- a/tests/backend/utils.py +++ b/tests/backend/utils.py @@ -5,6 +5,7 @@ import string import time import uuid +import hashlib import baseconv import requests @@ -19,6 +20,10 @@ def base62uuid(): return converter.encode(uuid4_as_int) +def calc_sha256(content): + return hashlib.sha256(bytes(str(content), "utf-8")).hexdigest() + + def rand_string(size=20): return "".join(random.choices(string.ascii_lowercase + string.digits, k=size)) @@ -471,3 +476,25 @@ def unassign_analysis_from_object(self, identifier, analysis_id): res = self.session.delete(self.mwdb_url + "/object/" + identifier + "/karton/" + analysis_id) res.raise_for_status() return res.json() + + def get_related_files(self, identifier): + res = self.session.get(self.mwdb_url + "/object/" + identifier + "/related_file") + res.raise_for_status() + return res.json() + + def add_related_file(self, main_obj_identifier, filename=None, content=None): + if filename is None: + filename = str(uuid.uuid4()) + + if content is None: + content = str(uuid.uuid4()) + + res = self.session.post( + self.mwdb_url + "/object/" + main_obj_identifier + "/related_file", + files={"file": (filename, content)}, + ) + res.raise_for_status() + + def remove_related_file(self, main_obj_identifier, identifier): + res = self.session.delete(self.mwdb_url + "/object/" + main_obj_identifier + "/related_file/" + identifier) + res.raise_for_status() From dc4c9178611d264ec23c4591accaf36962d46352 Mon Sep 17 00:00:00 2001 From: Repumba Date: Mon, 19 Dec 2022 16:58:11 +0100 Subject: [PATCH 26/33] Documentation update --- docs/_static/related-files-tab.png | Bin 0 -> 40492 bytes docs/user-guide/2-Storing-malware-samples.rst | 20 ++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 docs/_static/related-files-tab.png diff --git a/docs/_static/related-files-tab.png b/docs/_static/related-files-tab.png new file mode 100644 index 0000000000000000000000000000000000000000..9675ab063356e7e7a607d346997d9cca9184c2ce GIT binary patch literal 40492 zcmdqIbx>SO*Eb5m-2x0Q!CiuDf_nlaxVyU#5InfMg#>pA?hG&xG{K!=a39?HIL|rf zed|_zU)6nY-9K*Kp4v4vYkK$Y?$vwgZ$*AkmBT_OMTdif!%~o!R)>Q_OirU%L$n~Hrzy^&BLtb;2N z_2Y8^D&hO8$?}llBBRMd8U+`Z*ZLUP0c@Ws^m9BsoCy(oad71r8D)r>VSx$-GBGIN zM{lnamuu`(feK|lzd`s<4Kd0r%$sKw?@tM7q;Ut|;^XsP3Vl=txdB%|;JJ3ei8Jx* z9l!^41_axpqHs=1Felui)gqy^J`VprbCG2}2%jqG@?Z_Yf z@5`dMI4mus=<{gfn9F5vpK?afci6{m8^;%~H#sp0f2LYgPm#-_M{5V8!t?BxenZD_ z(dAQDHn*18gHX9X(6*Ype(D*j`$DQIA4m0o6X0b>#K^c@_yK>{zwRg=kGcipCu!58 zzx`4b0re@Sz70iS71}5;X?54S?hSPZmcG8%TOXX3Wq9O#_u&L`X2~fdwpS-s7b-r~ zt{4|Jt(}e6E_ZO|Wfd1#&d#VYX-s}A&=voc&PTuEl{K6rUDtUogJkI!wji4-$w|XK z|MmISS(5xhL35VZaf5^7c=Oie<`}JJ!6=ZGn(6;Ck z3LmbqFZcqxm^8*^S;-W<5(mvX7Vt?1?u|KZye(f&f+Xb1sCs|LjVRg)aT5 zH7XDJQtwYJbw>q8ZkohR9MSmzVo>bGq_Y8W%^Yn_ZA@R1BHSX~Kyq&4dnR)MY&91w z;)=m3fEpT-$ zHyZ&aF(xtCBN8@^LLzjlo#xY@tC01yRIl8$J&i(2i4%9gUk$ERK8ykdh5K6n%Lp-5 zu~BQ+IecFG$*XO5;F|}*twbSiXE-n%ifIRt&)CI~H7aKuwg9QG^(J8&KD04TJKZKr zaEreSFJN&sCypep0|oR}GAR7-0f$Hqra_o*@Htwz6M$C2d1wLeX~-B?d*VFm%nbU( zQPgr|4BwIH#qB4WWHQ7-Kd7tAx+st|L5+2`@o2zlIlwdYU#I;#q5i5+-(2sI4?}~z zy=rH_vWto2%n#wk>~8A;;$klBOL= z=DI7$klf!-cW7aPd2niL03IONDKd9JLb)b{ocp;rpnyHNRg(9!bk&oRTQ-s~xVgMf zezd+$Pr;))0lSc6f_ipf^%h3rN3+GrGWd|iAyM+!rCNi^emZ>Fo~~OYfH>cV%->uc z*DRs5Y~k?6;=vwBd?*MzlHTt6D!R3CDeG%azN{5`;KIzzLC?&LH&Ty^9O8cPCvu1C z83h650>v0q%np2P@|&+?jhNS_iX-!hl~|qV6_P2mZ`jSqd^z@1brd6SBP*iK@|y!P zhIRgk{38)I?cMfG!cD$FjmfY*bD|yE6C?H+@0azof6WF1Ph0318C=%{oFIF|hDGIV z;aU5nj(2x_nBZ9>ZWPWpB&xH*wzfCtadVCyQgN{ywjXDAZ{h6Jjpj zn)-JUa1f&+e>!Gl7Gh46tc@GJ_-V$`;1!sMzR!ekcWJqOQ8xOtZpLrFSpMoSVU#$< z^&$w~!s#O{4^86d&w-wThYXZN(`1IQLQGB%p0A}u(*`&LF5xWg*?NsBdfS9MR)_S; zDk?y5qTJlCsaUq7l&sF$jUFz!V)x#Cwr6i;wK`wtq}&{EX!Nd^b=i6Ja(yfea92ciGkyd= zK+^-)VIHZ{#Z--e2MH$_#2`}C@9`FB^{zGmgXppQS~wrr&Uok>x~D;9?$PqZe1b}; zeIeK!nz!8-iBtRi$>mKM=bZnw-ZHlvs@NY<&Eu6GKeeY!qZ;8}M_3{E@zwoJz8xc1 zso5)47za`3C0+N^frx$CohgHYnDCG_?!mds(M&q*VWeTY))Ah!M9oC9Tb!};AzVc- z3+U!n^;IJ!BS2<(55QPQrrCq#oi4bsr-T~Dty%^XC=u(%jtV(OMh9`Cu6#CKll zRogeD#~b-)rW2(n2we$N`d;nm|8YXb>3qWV^5nFDk=2|%F#6D8w&tMZ1$;TECySc4 zJWu-cdMdom%#Ttvc2s#V?l>69dz_@VB`#v;`t$-YJdYXjhj2t^4cECJXG3m6Vr-}1yw$%|HzC}4JPV^>q+g44xH_s$VXp2yCrm>Jxt)7QLN*ShZc9m(=l z7He5=jxGGZ-X~!+(sb-jb7vbYy>=tZunxk!`f8TJvj53_I;`P@ z{HitXV>y_X_7A#ht5rA#z|njaeJ%{3=AYm)ZJ&eO5(_^9Aa79e?cz@*5k#&ZWRYw>L>pVhBKjC#?H~ zd2pa`r4-M-Ie1j3BhC0R%=LLGaL|9}ZvJ+8W%L2|Dk^74V|2XT`-*DAQj`AM*M*k{($7eIC{RXPy*0R3VUe_IC_e-0KjNifJq-$bMOoE|G z01Cz__VPpzc%%HSEhpIjAXCJOHG+AB&w?Zqq5abxNlPC{(;$%ndJ&NO2yw!mhs*v+RfTK} zE6&V1c6nO^jh`Y$K))>a*^LUu>Px&Wq}Z)iRa)59+i|S^4CT4olEs6|ew07p`jDl= zR`wSj$8p z!da|VCEjG`=?f{jL_0>wKk>9&&*xuz2HvpMIF;YNkH;BHjsN;p`dvm4Mm1#Pm0``m zY6yKk8qbS@bkR38^CxFxGC!PF%HY2heR_y6!Y3osllWMQ(kz~EdTR<0vS+$}TrcUk zJdU{Z5{Lvh<-VC+n3O^w7H+f)QQGVTl;m0Bu0HZ~`+lbxf-AM$T5=oiy#3sUxDC7S z7CLw=Wss3E}-Qjez+Q9@e+?NCT0onxXYJ35`p`+3<1q^ z+*e9p)u?WHYmS{(4dH^gK8oxcC!A)-t$vyER`*4L-JBP}KhLpK;H{StIw>yfkNG?Cs!Lw2u zL{maMrh03=Q|E3$Dc8)G`!K^c;sIDH!g+kCt)mrYDNjsLP3z7%vhz~cK7uo0i9Vm# zUv5Ek$qnZ=v*)bWc=V3NweZo?=CVQ(VWsH#4(dN>8ST!KDL9TfPL;K<{Th2NVQL~{ zRjf+#KE6r(6^g(a#7}8nXUp{F-gTzxk=ZCw?R`m8|Lw!gt5&u@heFxRt__(tGi8{z(B2$xcCVONd(g!ILB2$OU`Jh;E^+0iqjC;^ zIDDpfUuu}WuAcOj4~f0+2K%fg37scN?@t&1-HbzcM&+@B*pB-}#Vfhq8nZa)x*&(D z1DXi`w)%od?oaKI zz3p@FDPC)WQ zH8O#Yet!texcWYkFLx!y6WNytDnD{pQ5B7#%OB^-xY`p|82QIH7APafQw>a3lhFB6 z1QUi;-dXdHVC<94q2*?J-#Q~(GMZ@a)cKBs*+w0LjJjgY7mNIu_1lAT2NG9bL4HXc zE=;>q(noJrgEDxE1#;C*X|#VG^I_cXPK6+9jBh0|8^CT2INIyTOaywGgrZSnNu=cB zu>ku z)k@5O#nnYcHWqC1!rAfFvi+A$}{bXsD=gW2lp2U#F1 zRiLTxWsy@KJW;1n|2m03NkUUln2ekuJjMNu_jhQ8OgxEzo8dsVXk81FtAT|j7vxS! z-ta;2H*3qaVL8p&sAXeAx27I=vweq&APuvpB0 zHE!L8rRM)Fae={gWG=0R*UsQuX7y?uflwq;Fye}ts8xmv18vsM|$ZacLqteqzITa)4|t{H>8ebm~V}*9|;krD26g zNps2&Rf+~Os*`D&`$BB21%rC->oGGsH-keES*(`h z=BAfQz@N?4;wcBx_rG*H9^R9fI3e#CxS?s>9YKpcd`A5s6z}H@6b7SqG8@@30fpG@ zIoTq!PGI|yubIhv`lsCS6N^%*j3FkxmVd^XObEWr~v%ixcQ=}z# z0L6`JFM8Oi0WBzR8^`#<*y`nshGNEjI?nt24lA#c#2#K%w|L_>PWbnv>B1$CKZ?L$ zmFYgEfW@o)z8PZGqW#up9X6b-i{NQ#@a(Jg5W2OGUTJ?}27mh;}cYhF&^m&uRt zD`*E5c;B5t`ky;AlvANU9uv-6H~Q*cOYlWdy!(1295*6&dwUUEqLWC)PzFY<&Y@H) z@S2>ZpitQTM&F#y^*GGPMAI9|N%WE*<-l}T&=fh{-vV)K zf^$o9zNhS$1tj}i=mTZH^6{_2=o07O>r$c(^@ZXI3MF!P!g+o^@LT-Gblf**nK3S( z&$r#28oEh=Wja6W$eL_jRiEAFh3@$%g%L?Wmd<0hpdlYuct5p9ltjJ3t`6OB?A#uo zD@)HE;*Xl@3f~kE1d2gS)mf`~D^#N!xnijai^(?WqsFphN~4P%c>+7;&M!8pB_D;Q zdQ0e4^EKmjDg#IvXfrmQYetb>e(FI6E&>57y`d!H?5K(J{-Ro=*7I}73jfHO1>QjZ zk^1Ta=v{g?CJ7AJE60EB&@QI%O&1mTPFdCFe9PWL{0xvPWXh0!$qr3!&NnkN!F|(( zbwm){Qstk+Z@RbI&6>9yX1{76tqhYKKU#|`%HQN8$@0j7nuYm1KEO#JSZ zkt!{qmXa<+;wpQp64r7*?nZ}Hdo=Y>6gIjFu0z>3{N#dCxz#N7ro6hh%aFO?bEB{oO%a4XV zeqTh2G14^l)k#5|AiYltMF_I$_`_qNyq4-b3ShQ4Y4xP6B@(nkhRhaKKEh8Qr<2fj z8ct^fw4EpAt`n16x9{|BW{|K(Ycyv0d{ZGZEZu<;e++ zUDeg0H4;$Sc-+|?GJd!j30(dSUsHu zjq2(sqC-TZJVN+gqdrc{P`^jDo`m0rL0~n2?AlNwq%N0$3%)@`4cUvyWUC{Vyt1*k z+G6e3r^ByDvL~1rs2a6qtbQY}LJiW}>Q~rSK1gS)9$~AM9;7w+T&j`^r)N6&@s#?L zjOx1Lx zfb5%I5oto3&R}5q!{MHBj&}Z?4`%80Hg4ju&b!I2uZ|U9A@RIv*yS`IA^@F7T3S~% z8k4uqAzM|)!8-Z%#7h)jd`bk=}5V*(4e%(XLeG z4xXC5#2N!`I`6shmre(x zkKh{lA~+Y1oB?bE6|nuH@bWAW{cgBz8QN{+HCa9yFFroBF6rKN4{Ig+A;}i+;59xp ztb8p0)bbEDT&*o7av0g--mBjAHxEB0dZYa9>(>;f9oSl@<~ z-SpP4;&`&0bulAC|8%2m#5&T4e-j(k{_)LepwQ0Ms*4*jRkK+ z_At9sQv;c23y&k~{SsGp!|tq~tld5Qc>*%0waMqxN%pOyqnchquvbb%`{6e>5rU;X znQu(iuP|bj;_b3#9jbi!;LO1x)*6dizZ(Ix8k1U923JGtDSyEotfTksbX4I7U*_UJ z)t7MVhr-RM_&x)k5I2rof5i!L7b|&;K@?u%I2l|FX%^GZm@eh-Jm_2Ph!fk1xJ2oi zPQbUss++DQIdVInl5%jR?3WalmkWkJ4n@)EQ%dE)m$+Yu&eoL`RBwI~SeSdq!b%os zYrO@%g&p)}S^A;pjCM&p@vH6ek_04eyh=$A6W3$ay}U6?nk~Uu2r}r%A|N^WROgS; zYBphJ>?I*W#`sEX90m&o&!yeRvG(KQ361q(HV4#?wI!%{%Az+Ny%m9H3hy0dtzmo# zR*2`vXLsNG-(Bp#V|v{Og-x$v(VsDrV^4M35g*^KdiiWDq_-T;`t2^4AsFLk8bX~D zOqQ_yg{=}_@@^k(2&dJ$O#medXzg2U2KAYfDe%hya_nfI&UI=Fp8BXeVv$BhTI{w} zmMsQwSDSL^=~Xxo`m((}X0f|&bJ2h&e$dSOWRMpx%%eRke`WrBUS#r==a6g-5_R1* z!HNatuVUkP%GU!>5SWi0G%VncX+W16U)$?U%od34B_@8U6G_A)j$yO|FNsRDf7!2% z&J+*M?#c>?O73MyN-}?+X|PtTOea(v9>RBQ#d%X>Jv0=GSYdrb zTp|XZFy6kMA@{c7qi3J|hKU?JwyN@@_H+_2K3{VJvu|^Yyho^pu@m33_Gwp$91{VA zrT*z4AtB|fg3hDNcNz7hd}j|V5BAA%7dzY1$YHwQR+9di*^!ZSZ##E6mwzocH}lM$ zCd>y=^K#}5hHjK(bhdl2c>Bm~x%jo@Jk&Vy_hyln+bzpmH@Q%VL>S1faVP+NOAkwf zON&@=m0V4et%sGVg?<;c6%(Zk_E2*(!zUX*W*j3X;TKp2VmY#=P31PPNV@(>Ti>$VKvGp4935 zEnpShNgW^7t807sW89gsvW93Urt{11rl~VgSRfQ?$#k5z(;Hbhk;|X736p-yP7K$VLr&ZtiofZ13PsBoN zYN&ole^9!!-^q{o&nnSvQML!gTiiw-BJWi12wuAX)9*urxnk>1D=4@yw%fnaj z4wXH><|R4w$%3QAnPT#T7CsbbPU1~?zclAtAm7ofSRPdk<)m_k3EibgnSbDe)So&D zje*Exf}PJAAIX$l>xrRX{58M*jsm}kv7IZsXu|aVfo9LDwd$a`Gt;XRg?*$@rhl;j zzqT7My?`R)u-}(?>o+eii*c3;zWN(vqfsl{99=?MTBBMGss@uU5K@37lle>TS@DEJ zXCA9ru?puU=4rf>r*FTsJw{BqQy}q}7Tou~!;rJd?Pk zqm4u4dXs-V^*6v=5WBP8{N;60WHL9y_2Dd;yX`l5u1`Iz>ntpy$JcGh*@_jT4X)Jf zNRtHW{n;SDV%K{Y=WlLi1p;+%)*OVmKi^5QWn9Vycs=_+9C|&QuVzbp1Kn+HQrewW zo>Fc*9gP{S`Gl1OK2ij@KgauQRB8F#{9$9!dQ3i%fNd?OU3OPn!VCbsPx@_E40tDv zwH;_b4@45gnc>gQS)DOCxw-j2(OXQ18gN1P_)8gWN_2^11n*+Vl|6IN@}zy^D5dX{ zDC{4Vbs_4VZf`^$c6fZWQIa~40YqK>m~CDPU@HKf=dNn=g?9&opa-8OQlH#IAF!Ns zdsI+#l+Kk<>YKH<{ng|VZ_)pBbbQ| z76KQfXIqIM%MxBKcB|GfCP@^Ndsz?WuY9*5nC!>x8lVlZo%0;cF*LH2I zRzA_^?C@%a57B|RIc&*X1@x`+{_J$%F0S=(XRp8{i=o)Y)MsO8cQu^@=8=*%%YQ$z zz3sr~PRX5g97z;TB6l^JcyYa1Z=?1N_Da0!*?C+gx|gft=MiskFEh$QM+JH_;mF=D zHG=Urk_K~4srk!SJX7<_L$VB>8fI#uDD-e}m#pZ1OehRmWqLo@Tu_K)#LbAzC(Ui{ zZtV>Xeaw}w?TI=W%1F#bJDgt~?GI)wuVLWkCh|z7PeGzhwkf`%l$^L>s9DPjshMka zSNobHZ1(4#w?$**67j;5d^E*m#X6rfe!>hAtM+5AcGWx#Lr8mL{k4Po zHwFTNnmzu)ib&!%JcvvRvoVE8i@^0wvlzvq1ja$jS{+5DNJ8IG#Rz!JBYY*F02)5WuVM#HkP8x!&x-nK=9fKsnkoXZ z#Q`%oM`}rG{z%RI-*g0MxTxUdtcSHHLjqbaJM2|)5J?UM)I;04gc@#{9ZM`;CAwkeMbt8GqW8&9vZM3E0$h9oAfACMzn- zN*M^uz5B!1l^|0iI$<`^H3a?p>iOa1@QtSd=Wz=Aq`{1mO3fSSA*lJ`JrEpM=wt0_ zZ>_$gsww`GLt|0(&p=xTOh;gTAjY1ec|-W6$`AW`l1nAon0jcgx-GOUnUU-9PfeZY zqbyOt4YF9f7syOUFgyrcKQz2DGJnolpjJZ(e7G9X4}f9j`9Sy{TwDY%X(Aouf@82r z$`GYYIHC0;h z1SUZxlgZll*5L&ADV?YBW(o{Q{~NCcgY`L~{mG1I^QV1H zjEs_2?TbAVWSjv1c=vYI_rjW0tH$msj&iJCnER2;i-&@EuC9!o)BJklI-|X!m3!ah ziqj|kIu@qfar~L%N_d*+$ONVry`X+HOLal+%7666>}+N^`50U&-jn6EQ(N7KbV975 zip4WE6{jQ1csuX|2#V-1jfHe5XmQaB5%KD)z`Bk)?gtfJv&flN&M53LU(!l?msfDd)>?rZCYzcZ_6@YY!Tod5p*N*4d5BzeI}96!rFQ%=&+K8WC6KAXSdn zY=3P{cAJIS1W<+JSWndZ@PQ@7^LB60HSKkW9uh+y_E?tLp~T#WNVFCd8u+F zbF}2N+#vGszMv|c0u!E61Y=wpFp*L)P+rtp5k92AaezTXc61I2MM&tBjhOb;FBSU@}@>_DUH zwnm)x{D9m~TP4rRktBOI{dYh{eGXjAytFO<`bunQ>mw}{mBbyl2&hDjL_nZ_$KLt; z(&$mYSLWSsnmq6A0iuPblKZ#EBznEk6KrqR-WBqxp<=c@9OSfLTjUw&gqLUk6&atH zqP4TxWwmZeG+``$H;J8`n~9EWKgQmJ9cGiBsH)XVfx`{N5faAmy8C8sm z&oyQ<`yMP`IVInsyawC6TOZk#zjuajrGJMRC?Xqk9NB2aac`dr+xu1&jFvbRl__1q z;gYgRq;BD7J>~Wm)S##>QS<)Y3#e>}gurroqRFO};d2$ds<|o#1_Gn;6tfT^>y+_% zR|q)^yV>++jqwscEI^{*10bI>`~^l;;A}n~-QSO>KS%!>8+r|{Gdce(tDF^i(AK|R zq-II9Fw7iUZr>5Ua6`n#dq%l$g(?)hgB;NP%n=hlg= zOBdOc&D6ai|iNx)(e~_y}X1Tr@CV-Z>Xb7LPD93=>Ydrmr zA({3NK~-}MOS;oij0JP6!awawL`_q`Z!Xda5W(1F|Jks^H?v!U*p~G&&N5gpH$l8< zPkKGx8vP3{%&e3dGzVM;h1+Brzk@+#5YuOj>2+F~5USy;gKPlPVIcP*d55M5SvN5r zVe>+DVr(-~S*Wl6Qm^P9`eTUKp4x-`~?O&SH4yp{10?DvEhC6c-n$J zPVCmpypA?rwIpr@7jP}?m0gH4MmziCV9j};En#;~4AY8;r5ZDlqcQ0_+wd&xxLywZ zyIFcZy1rp5e%6A5DWadHUG0TGo6@G41aFzPAg7UKUVqnU=hsdfp3MQm-Fexmy*T&< z39)naJ#cf*b(jdOziC@KlFq7RIqyyPV9Fo2abw?{!aAC%2kGra{q|eAYc9&C<{EK{Le^VrNqR|y^1Ii%=`!0JJ zEGJ;B4kaW`uth|JV`99~L?S?OZ`PP`qn7ImO>)6Y#s8kHOwp>=v*LmfNCgg0))>hP z40fqOciW{HR%!%>y^1b|kNR&Ma(qdnz$AdpILEn_z*TMe{5wPl>A$8>Ow^h^92p#v zhDp>l4CJ7)lZr7UnHV`CkX&V_|V| zxU-?Q@XA+E$g~_1EuYr$@v-)*BAM z?qZSIy;I1cK4m*ZSiEyv?8(K2c5t|3P~Kz{Ypp;->v@=+^Y1aw=tU0tVjPy5+@4~h zNuMR`cX9IqHhw327v*C--*9wi;!Y*JGWPB8uBz528ACFqoAlc!yqNWR_Ybc^k1bu# zssb=;aPn&)YM6#2LarIZR-r@L&z#5YmdHP3c{9&rnv!uqf(6^;$F21AI`3RYl`Xew1*X$4Q(SI^v?%-Ndg!>W<()KiukaWcY9s zKd!5stvqzvbw- zHNkrKPk{tyYG2G}&4xRgN$|;NS9MAIqe~H(`h{Y6d+{H+I@Qrs;oKZxw;SBp3SR^ zxfwt2kho`Gz3o;S;xQPF;a%69ww6h|f2o=u<%!qXiCb^B(|XXl&u zCf%~-Ka)P+ULh`4%>E+|)8)thB*!z{>;{(bT0RsGC)V=V zD!#8N42(h%EyL{GB4S7#gFLc>_c`#C*3P^8zN$v{0~nrElUty_3=a)`Iw)x`F*yy@ z68a35<_)w%9O(bJ7<2M`viqCyn@XD>@p38!$M1W2AH3}!b4X+{6%>%$;~HBudV7OO ze|bHJk7&JR-u}Q>M^VA#MZNV^N6J7EJ5{nB`==9d`MZC_RcvIkhIc~Wq$lP5U~0#t zpk9ee6_l&)OnX>5VOc2tY_4Rd4kAe(X91Z_!QU56Bqd&?Z1%+5JeKf}2oyVeyVicj zm_m1D-nZp-+Le2&8F z=f~V;St~p%;nyABjJ#_eYWuv5SFIWMR2>=jD{x45M(cqm_eM{D{Q0|caVuE<(PBpE zSO9CjfI&kKyj8EKUoVchx&?wKyNpO0x7TBV<@P9-LcTLR$aQU_RgKjJzIj6WZ+DV4=6aRX7n&*lT|L5;?}^)YKko^jMvcFX8+qeV~bkz5oFV#-_K_0llJV*~h^# zd3dA=aJ`fwuk?V?H>$ z)lRs`59w#;=X=Yo=fq8&yu$xgY&Cl= z6G4*o_+Qmo|A&IB|Bq;HZ`BfkB6?o36x?U%n7E1xnvJbe4U zxy0lJpZTBmf61pIo04wgmocNSNX&|>D;a~Ptj>}&LRL;9# z4;t*(5J?yU&Zbkl=GPO#`TKpD_X_JzITLY1OA-l?||G!|y{qObk%L?O;hG$N< ziM-8}HlIW4jX#LoEST@+l%L&xA6-I_ee7m*w+YCshpQ9;+#@3>+I*fl=f<+^5^X|-&NoR0N3 zco}&;NigB$J)AMueYi_zdiW;SoUdy_K-G=ZoK<^HEB?&a_wj^t`q!nAiEK*djkW)q zTa-jNX#Ea#37cy|myLX3B5U#j^E%)GFxrlucQ~g= zkOAN-rNR^rx%xe^({y;(<;gC(Lbh)UX}pFsrGHy%HrXeJX_O=JM&ygF)pWB7k()nQ zY(v1DM`W$2q!%B|wdWQ?o(E(Sk)EI6mq~H9BGyYddq17@G}(PS`pDRufMBR`e`M~y z2gLp%hb^*ES2Utk$H++hCAjZk~{_*YHGnd$K0Yf0ggv5RZu_?Bu8Yh_#EAqSbp=b=uGjX?)}gL@s2R z#C_QhV$0>XbbpM;nZqItOrG*UV!YccM0>5UK}+wmsELJLo*pw<(vK88kXeiZ6z8XV zXFn%^QxMsg2#GH!0-zu+bFWY%eZ;xK-iVW4gj?JjIA7qaRr{M$+aV+5H9h!j7V6C) zJcRWEjBm{m1fI#imax47;4ntu2zYoPuxzyx4#ne|_=o2XF|NJ3H&pkcZdriK`vC$R zBMM}RCAb3?6I-&LAfM>l=a=$j_t!I^LDVcr+DUW;DJ6h6F>;okz1*h-egVvqkgWT$2vR$46H)7jo}M zJG}s{L`}tCjJPoOQ75!Uq3bSvp|_{Us`J^%#{rX7LLF;xuav!gcq)g28=(+hvJ^Yi z=zXc{q#rQM~Vtz~eI><`7K66vyK#Ak%b1;W0UU z?`T!__Vwa`R(+G*s*bDLSesa zm*Df`!z@1H;-QeKzKiwb65sCwFLtdJ3?gg0NobBS3xc_=#gW@2P~-3O*2d^OQ81s; z(kM_m*0&i>O?a6s`&jeexw~cPOzrP`fc0uSSPqY{#Tk?{UggX6{31$UP&_cwZ1DYb zyypRypsX1Dx?|YZCqy#uz<=l|@>I0{r`3j;=$Tw8r-2#XlS|{O*89)g=&e5vBhL!6 zZk^EevKKMCQeHkwmfRd{W~V`?@c@;Px2MKkrR{>OhhheS3mUNAI5C?D{4;1LHMik- z^LwK=eb;Dg^Ym^Z=H3!1dqcganeMXv!s4_lwTf#%*#!K(=Ei&$d4v2ZrFt%NTa^HK z?#pRYNqNqeUoHHO)~qEkSx!|kZM5#9Er$G=I`#}ziWj@m^v_Le<-AMG+iZy~(v@H9 zK@O2;{M2G;1JFTUvz34(a%4wwnzx6Kf*@T&sU0U;_4z7(?|cR1Rf_;H1jW5`^^ z3ahEhhZ?7gUR<6yId8uy`gs2sS(wc_nAP80CK*!vQnJ%`gRoAB>Q5R1#Xp#X(up3U zHQ-1A(Wh-Njrc+&e;DU7KAh*=`-W3ljYEUKxtUo5;P^^F8F#13F4Sa}Y+)bCOq_^P zP{Y6?pmlXge0LyKn>Zp4%DX|wuUV*#jv*)e!HGx+t3e5?r?Pmv7ucHiDEH0p?9u7& zdv}i~p|3o?Lj54W>1$891Vsxz8Qgqhm!9vAFrcMTHF#ds6oYnOXPlwZ#fEbJ>H)3woGeTh5QvhXI|%oU zgN^TVaUH*i!5OxpV2fe6T)(>&n{>JykZ`YBBKGSc@_QZeN)7~#fGmNjVLJ0707|CRz&1dxW1e@x zx9^_2mpXB8uq>ajC}j$i$FrX;P%d`i14S_j$_*d1F=~*jRD0%H+k_leFqirYv=YzL z)A2D3EZ>=n%s;aolEFD1tUvPyO(8qbnv6Nhqx`uBId$Q-5YLKyVY}Ib`^lXC9E|zJ zXN|r1{Mi>a9${8(M4gF0vcD_1+^49%jGQ5N3+bH83@FD_0}96JB(H5lSh#^#gUX)f zTYM;-v?9wBOZoo2mCPqF_Xf@FE|JVN4HbJ63V=`xe89jG&lwcw#&xvfqQ;WZf2q0S z6vCjmeEBN5i*lm>V{YB!DJ4~Dk-;(GjCr*hxtDe#wUh|(F%5Uqf(f` zAol`jSnjfo0r19=Xbf`0#%SC*PO3O9@P@VUr>dfXt$}b?40-q3_I;N|SCIzm_UIBg z!D9%$W>}&f6r-Z!JFCw`3@yDlYLXkyyZKlvB_Iwf)uLO|opDufCt)Bl;(=Vx>O0(1 z#)s{Wn>&g%tOXV08HCe0IC*_|Dstxe9kh~(PPb=$wE-4M6dY|LJBOVaiQY~T$jp6} zTXW)0uTz%W+xnVpVvofNm>tyt86>>I1M6!+?6F|^#%Db=ZJuT8c z0k)EPDkVJI0vnnTr`6(5h(zdhl-H)~ozE=CO{@viR8lyF2e^AzbRb*whE_bW@n~^R zw3>%!X>nP_K?!(G_r{$ts0D(<(ksGI2wRr+@Xmy=${5lS6SfiDkf!6bg3;WSh?s)C zF6V>d6?77K&H#v1UiqU#Ju$B&$9GlqR9aXf7#|g%9U4$AKFQIg-s5TpH_o5I@0O2w;#zJ=8Jio^Y|po)F%J6PW5jB0xCWlsh~oCRDV z+hw;Wmr?$mpuwC_HW2o`vZrMh38~St0=w`6w;WAa$D=i2av@TgB7`>NV^Di=$m-6J zRfkW*R@)EX5Tw~E*w52`$H%7^^1ZxoXpcr~K0QB4na$*qgNTVbj^k&G(j>0gyZM;o z8(|-b`R)Y<2*lKhT3eypVK|XX^~Tb2>fLo*X<%mCi9TK}SZ2wepFB3~JZAO0DG*%n zL&Z3&@Z$6A*K8c0tnrrqiOoGR`0oAzGn@TLWaYl=j2WRxl5Hb3NqDZx23-QcJw`jN zZ#m|@Kzpyt!u!U2K1#i8!4sc?68I9Kw0wkf*;0DT!j_;Sj#sxKLF&S@CuFY=}ID2-JoIKeOc>;HA(@e zYB-E))rY}dtkWA&TB{0Z!896_ZQT?Fb_@7g+;X~|%-M12RpqxtwB}Qk{YtgnJB{(C z1uK{oCi&;cw#R$C9U;QXo}t@?f5wViGzGr{m4 zhVPZfqAJ7Y=5)9QulAv%;DU$Qd$!zdzAXo87THf^1sAtb->~CfMPU-bxTuj#V^)G} zA=iksPcMDm-Bd0>zw10Pb8+p{AM%u`)}7V(I!&Z@Jh{Hodo?l?fZsU`gN); z2s7<{xQ-gV172Q7g}zR(FIGyrFmn@uJ~WktVTLaM6YDUc!4Icna)8v`{yxB+g%g*C*$2p%pFNvO~#xq094Rw2`8 z#GXckvL`I%-9fLGmg{D1r3z|tLUx_fJ2t|gsyl&HnZ&MS2JQ7CC&~<^r1MsOK4TT- zn=15(NByrXMUvkv%v*ds?1ztp@KEiXFIW%`8t3l9=;m2V$}H7O88nGoZAXAX55m`4 zv=|vO2ONXyyUO~Bw>Oy9~ZeeaYJ2%3`)zuZqh61`4~DE@Og)um`<$* zxCCh1UM)nAP@`_L71>Z>@LV!V8d8K{{cx8V3htX$3e^Y65}VZ;uxUzA#U~sM=%8Z1 zG$i3!qI?BN2%8~(&Y6-oh&**%?Y<%0RT@+a9?_27I_orb#p8?XyI`(VGL2oIVo_k-(6_h;|MjT6sMv1n zd_j432e~9zRk}xLNvHSKtu^ij+#F}xe?7q;ZM8*Jt`YlDmBpQ1K!{r(fyED_l;ne8;TsFsY!@rCG~Pc%CWgp@rLm+`+*DvSo_Rq1^?&> z4M-9zYK1(Mt*w^e(#l(chh&3+u}BDZq^VY20!ZsKzsNNcILO6=TlLm=PQ5J^3Is~m zMLWEQ1Q>m6<_;&e{DFomf1Uh}l$<=~r?xa17x`7c5uk1ZA4l9u2c`W8=KR|u%a2<( z66uvgnNL=z-}z`puclD#22FdHF=D#TLSOBPU@LJ5Y99J8WW+?M)wd9ZRknJ3i%!I|oRYH?WZ93eIWp%4&tHmvfg_5;n^mIa8$iFVd^v&axfJGrZO=fRgS4y4FGzEeuJAJDF5 zBf?DklS`KTM6Uo7!#WJ`!gRib9_sa!V#vobaK9n%0nIKM2{0l$QtiX=$Z9y<9hpL^ zuZDc>WnNpAaWdq&EQXG|s6p;*%XAXA|DHq{efd)@U^7p;!7ZGvR6?&|iq3v=L^&LF z*O<`qXW%GcvD$I1;aj|Qfj(jz;{Bv&SJB#1Yr2uSDav${of+71(rSI{YV^dtO$ZOd z#mx>Y0#h-qs_9Hy?w8S7!pDB0A;{A+l5f}%0+uOmo4fKYsq7b0;oyvp?Rv4#P3w3& zhcb5-OU8D3aM~}40jSH#!!i2t>=x8KXqAjieV_R|u2*Wx1r8n+Iq2!%mg(iZxu}Fx z?R+|iG_|p}VS9W$ljI-Fz_=Jx)msXsw^rwmkvLcvw}%KtkU3oHumn=-t&1GR+DoX{ z`>dc94|wk}36xveZrjsHc)t%>nBY8f?H<01%G#rdpUWm%0L=^NNia!=KHcFT=%M#n zeP817%zRFrMM!E$2oP`IhHEZr`@Y4v^TvRaWR0q+oZv7fUB9&3c}X;uKfp`8 z!JfSEIT%{=F@sI!gp>5}AGXa9hje<^2_Ji03McRgu|ocR&u5BO*M{P zMvnHy&C0UY0xU9fzdMQPm)DKbTxzf)Zds0pN^icQDAD~)wVH<1tWI^yERJJM8F231 z!OvaTviD@Fko=}?0RMLa`SLKLh(6$kHebjgpHE~MJ$zgRyxbF3k07hxuZ09nUo0hp zCY|Nh9Z%o*+qkIT189Ss7l4I^FVc#v&eTjpmXxsb!#8x+&#ig9M_TF!Kr z<1IvGhMvOLi!iUx-n8vaNT<7IPCRoBNfINar=oWQ>Q6ruPPZZIcE~spspQf4_hB&4 z<;`$64MR*Tn2rx;aZ%fi)VZpOjC}FUyW1-Sr!P75c(gCS4#+pMM{C0s39n|2W2dHQ zlhL&hiqiSpu)G_9(Iv4Gquotpk&n@76)?cwATWXsS@ae_lU@F%NxfpiCzdRgh z2y4;VEe8vNc0$wmiN+@M%>B@%+(Pd-{Sa^_40B$vCcDtHvgM`bcrgyTz!7j~g?K#C0Pi{D8#0LXzYw=3nV@^~KDQ+^{YCqZrwADZlo!sX{k z#mM7n>FOw0-bN~%X6xf_iAhM4-7MMIi}!;S+`>C=TfM%k8AuE0y}IXrUZJ-#p>AR% zI{plfto=<56RWzm)Fn+hn`UweZVIcz8f4;oFF(agbI0yfjGgTPYray|J(;ga1i18h zJz?5J+ZkrZSw;^X_I6P(nFdqp7k)Cm<^7Pq12>~EO_I;lZ=$hxfVd?oRQrLkczcd# zn$;9Gjc2V-HbVu=$EV|>IU{am?x{eDOFX!`d!!tk-!LNseOmP6Q4shWt-y+l9@jwl&B`)gK!fJa=5!3vKwA{OE24R>NR15^o*o5IYws-r!bpqlA9_4l`0piddGqpuH7oGL0T z|MUsIzP?!r#&?f^{yMO715Qioh!GmsW`lZg)W1qH=uOLgV_bl#o2rFk@Q)WoMFSuB zUk(?;JYtQB?OGxhP=7DRA}!fY5UgonQ14(7>HoP6{O}*IsXB~*Z_w7i{OdIpAo(A! zssH)PwcjrRde=S1!9n5~SQx8$LMTV*uNQw`Rx$G#Ntt=2)D=H{B8yx72*mvUI#?K- z`+(P9f0zH-XSCbY&?kMZ*BZY{9nprimR1Ws3n?AhS68p!|J!_PErzZrhFbq)V{;|f z;vX*mm-F*KFDWpCmv&P^e7R}>I+0pDgh9n+A3(84H4TuZ9F>EH_S^j=;$ZNnEy2>k zbdE{0mlXG|qL+r(cVYmi9?n7$iIV9w-$|bG3iY8QgdcHEp<)&wq%az>!nEv<|4>9_ z#mnZBI{sJtJ~in*jw=)uG(OH6kB>Xt_62G{#uXyA5nc-?>^!}z{mEh?5hhv8ix?|f zUdyRAB1ON#wP&2X6*omDUhu=1-J55q`;-KB1=n8}!LJ1@yv2im<>=M5KOXnfhz^-UrLE;A6G~bjJ2^CjwJsKIvXvP4`!CSSWmA}Q2@h2gyf+R>bN3sOj? znSn?pnctYP1h_s;Guq!Fh2oQc8@-?wMK=G;gr9w)I_PRCz4|cZ%9RQLQI_N6WU4W5 zUiH)!#2+}0Qz_~LvP<03_?sWbiyXCGu87)56&%;<24}eS^3G1v!%6#(;{Q^m?8Pw5aUo7*amL>-fNGj%- zFYu?+Y}S?Qu^ISnIZz!ahZY?s91mq;rL+b|YntI^tP1}QZ$Omr%7~G4P~cE2b&8r| z=6v9OQ>!tg7%+LHUpM62$%)RuU_mk8R;N}W?1`*4Yfur7lF_ zn%U0HinCeBCUxa5OwF`o#I1St-Q4ZO@!rU7xhbQln!4!HJ>$fN3e#32PQwzmM~#s7 zh6le#EwbUPpSiw3%dGA9BOqb>SX;=XB5dxOul7&Ienl$TPyRf(-0$BYXX%(F2(bbD zv}YVu6I3{&ByFrVPvw(6?`AlUEm)3|jm%+htb4uwCgqLz-iZqO?wr)iXb#1G@KFKC z(_(I!c?~;O%y&my3FK~NxZS#eor<)~hC{>d;Epl0d3*nZTO@TKYCvz(Apv z3DXF`8zmQ|Q>#1ajoju7n^|hDOdNJ3y&@ZIO9WHF)r6-Bvf%fxnLGZtlVeb+0zc?H zugt}NTwm`ouHw9@EO=zUaYeE_kT@GfF*~Gm-4SJ{bj%D)W9!Ma{Q*1CxyS0& zlq5DAfHBB$xd+ z;-3$WXB&BCOdcSc?%_qLu)KptH=J~uTSF{`kOfzH$Kl4}m7n3(wue=N`0>RrbXt zXM!Rl;?12gZ)Yb>L|(_TRevYgn$_6LV>Kk6xi~HR3$lyU-4Q!6K9iWYIr%DRjbVm5 zcradq;21us6!0t`qM8>q-FSz?Wcic2$ER3s?Wk{|FthBoTo&hw=qU3 zGwt~6b0Toi+<3$=5am17EXOtY$=C7?UfwPZrq#y0xIW786A<_A=;xk4-6PT^huCvO zb+^xgsRzpNGVZD%$*PW|Vph%>()yOT1Jxd$S%%)EZ%w3~>F!eg&^*O2+8OD3&5zSH z?Y{f4>c05poVkack@rp67o?5Gm_hX@@h^UXju92Cd(POP2W2XhD-{sykQ-C9v6&fN z?ow2FxDFxP@5LF(&`BsQM<}w%9EuK#c-=^Jw$?}?s#5=1x$pn0IsR|lFZ_S9ueFag zLa%1OO6iZcp*;p_{k!Ua+`3TXB2Ux+pZxrdi~bm~fVzPAC+nAc<|*cXa%*e+$JNW4 z`rky{ojL!ydZEVsClUAm{>m8lj38d-5ZC?1o~n&)wh>y>p05ovg(6{P*BecTDqLKQ z*6*HL_}!q(`p9iDN0zVQO)IKc{^TPUUrUe0osoQv#dmXN$_tt;yYxL=Y)LvS(kLc9 zwtVK^Q@s0IF+-lJY z%vwFH$-28p4zWxc1Ar+rgX7A7WW9E(wc>h)0T;&@`n}fF$wrAF62GehO=IPF0MP|R zr6khM(DIfQlcBm(_@VmGjKk2efPN;N|2%3tC~H@F zw*iTw9Cn|x3~<$8pX_sfc(r7QW6^pJ@3Jv#t4}x2su%i>=fj62JvI%3clJ=XLcpOo z2)b(p@^q1VjlCkOqt41cS0y4h#NS zuP+79l{z8_oxIN{^tQ7?w>m@hVN*!HyBWS^k8eHV1=+2OD9@&SFgwY-llX!`t|^oK ze1RoYUBRg`Z_J>e?jxH7aM3B@dSus$vEqx;7th{0jNI`Ar=V-^?A*&TW%yx-yF96^ zV13?kUAV9=lek4n5)Rf8PWzja{pDxi*!0O3(zNJfhuQas z>K3m{b2t8b*#0F)v1sEvmKBN(t+VsN_4o#pLcNOsiNP z3r`J`qexv;)t%&Vt0N|1p7eH$4W{$zLJY4Jz_?(Lcsxkrijv7)$Eg%tqC)jWhJi?P z!A~}qUwmj$-v^J#$u*H+xiQV|^J+>=XP=bicekZBDZtUg2qW#Ib*{j8Y*i$(&5c5u zg5|jPWG`+q==8js)S2gJ4*@HjcU4pYj@Pr(9uC&;1!HZD?_Y=jZyvA9un+=k@e-)= zU}q%1w;pZI7T$ATy7AKy_ROKL4}l>mWTKgW2r}3D#GI7NjxGg_Fm2~4)CkbHx5@y{|z%>)PtM$H9Nyt8L&wSVBn*Buq~oBQtlm_YAel zyD`$lisIb_a{qXs+Q8v&6!XMY;W~Pxj{#%t2PWfv-ue@&(27I8-L!8Ku-beauhhLdz5S~FT%zGB<;3UNQbT`vB z@9p{bv!$t{P0@d$d89w`bn?sWIgYqgrO=N~)`J)D$_tMtQfrRuniAiHzq?e_2wFxG zylsnGHOOvpr`de9AbW!8hKzZ4*THayim4cQ%aUj zC8epx3YPy5fe?IUk8J3fBYp%kq#6Nl8Xm9Y27W+v^+oJ<4XRf9Klz8+2AM4K>?4_M zx$Gn)wHS8JEcEtY(cGBu;VhL=I3Kgj@x(ulD^BMod3xzO_LP0&v|n(@RbN8k{ZKE1 zq=)4(I2$d~v{;#$*z>2)O<{ch)RmA;8)#?EG%J$#x98KO?|Tz#%Be7}zP%ph6mk@G zN>d-)x>+wz%yy_F=+^f$iru* zNVrR6g43=OoWwQZ?i%4s1HV-3e%=D6VY&Uhzf|Q)f7VtlqqX__g4gtdcpFF-@{cDO z>iK(XtDV=FgU=+1DLvzDJU%o-7P(>r6}PzI?sji&cj)v%?FJP;*KpwmGV3O zv3ITG=Ni~Zwur3}O+FTvR+ z)n^=SQgiK=3w-%J?}FYNX6-tjg6S)8pGlQkP$V!(-r1?qDV-3Auk;JX_tY@DKeD!* z0X+rn`DP!V93L^qz4J~=JX`6LHnXgg&zY*u{%o+a<>ceRXF>bcGkinTgzLe4SBy!q z^ni}pCI5yt!^x!Bt`cRmoc$?svnd#Mvgu4EMly@Q z?UPkCgtxWAsU^b9cL%lK#!Z(6XK32%Ablp&ZKJB2TCB0D?F-ko0i=9PsgOONzqHgb zcyxizCbs{GBtl437eLFDIsfFL)zL7_vxf|N=f|WPlW#H+u*Z`I# zIJ=I1Fzt>l%I#!h*~JS-#AKAr2AY4$ zh}h9!Rh~74b;27@(>3CB<1<#MwuGA>G3pflc(sD?`$_JSza#KYdEsqWBP*eS zkx|qUV>_$0fl*Y5T!ZT=5$%u0UUv9G6LJGrjJ^Z(nF;a&lMZs9xVCN*eUpNajV4caqDIH$TMp z*laZKk#K|i&;r=nx*<5P^3B88o|7AYk$Sx zku;fA^>6I$fe@tUGKx&-Gk0JuFaT60qz=Q5FkXlCx4)Udjvx3PqT<2NO=Cjj{}`qx z)1|+^zyI**A0rd+-+3haUm(`v`2u{`Vc5i9bBY@Aw%<`#^wq%D{0%`52)<>&BhOFz z8-n}4i?09KLuhIrT|e)Vla_fx7mnr6ZqYwH{7sw;dC9V@Ys?A5vZGA0e#~Byr z>R-9nJ_wcR`xsyxwW|zhXZt?fjTnlC>Fy>q?s5ZA7548?YB$O6e>nNt1ow=Rc8+#q z{^>nVkxC7aZI&`PT5A4nehs+j@zO{qus(QI?CmYkBC;Z5ZaiPeCOv<%(?t$jKCfa> z>3mG!Fip@lu|$i0cqIF2LCo3qvbl;%ceBc)?(;CDo4txYkw)}=7$t$3OC#-&_G3zR ziGE%4)RWXw+{8TTlPv%4m>+-CjfUs}e3aBp|Ki!<#Ntf)8w+-Z4oeeR_XRBnLK~A2 z?7#FWoq`a0h$vcbnXEPWH+G1(88>3)4~CtaphGLD$i;BoL(u6Zao@K=md)*{sOLaT zo+(7;@qdTgK4eLRcxhjiE&@&2+`t$mzE($9A5Rsa7i4_-qhvenF>fs>0OJ(X$3N1a zlD5tu-WU&=(+8g_xC}#im^<4V>nHPj%z9>C9VqadGx^JKx3h#qvz}gGZQncgm!qq; zu=>{=QPLh@Sgvrz*Kein^dOVY)iZ>{Zw2eDiYlI{;Ixzq$nyRSog|!cvNpl)DCG=h zlvqB;MCNM@|g8v_IoM60l5ZUAD4NnT#?CH%-7bhWrQ}(o{>gLw0 z4+$;VVR7)&HD|{+O9(%PXOHm6Xd83Q&*uiDtTYfeMwv{h|8b#B&} ztAeU_r2a=lw`%MC=HeFNUMb1?8r)UlURJH+0#U|N4b~s%?B7nRB7e4L-Ly!K(8%xj zM|P!so!@>rdVV8x){m8c(4WNso*A=ud!HT7y?8ejdy@AS>BIrOPAqk9@7Bc7@*-s% z>H4;Be|EC0ed-^vJUdnCvvYTazW_+iM+9coXE&8pwM0J&ioU$@ z@+*zUOCEa?bGveFSZGj#x74snUx| zw+5mlfjc(!qSKyipE(||=o^rJAd)qxG`^N%Pvh=Bc3oiF8{8mb1?qwm+nvoU|A{q^ z9N9V+vf+wda@}ZX-Yl|VlAahurhR=|B-r?d?DfLX_3vrk=1ISuOs`cI$eR|C1ugaI zrf1;w{`S8>zp=k3m;cw$PkR*hnOFKBd)rHH_M`GxV;rz}=Oy~0I!&*jkM@DUL8GJt zO-6(IUxOLPctcba)7kArJIZ-9<~wb!2Fa&;_s$pj1?xX`P&N&HG4o!*9eZ~gBD2X$ zkMKt=<2VeJ+<#v5+hpEEVwGte6YP#b_@EZA=38g%i19}`a;=p6z8;k<_L%squKkKn zNYZ&~W@mhGSFB#AR->{B%tH4Tm~dN?q05` zW>okW*FsZ#0xcdX7X{CD+NMc5O`D7{G2P$O(e^%^=RKA69Nu#L2bKy2?m4gcU5)uN zT0UH<563`Z_!dY8!+;H86C7%t5{(Ql`U?M!=zRY^k^#y^>&}$oQyL;ysaPEfs@>f| zMl3Eg9n~z;gC_qSJ>12hzC&gMUkB-sH)F5CQ|C;=tH*Tn~CO4S#Oq5u>A-ezZN0jx)>_~jA_Zp9(jL>B~r~O5-F;x#1@BdCR>Y~;e3z~K_D}6 zSt_Sg5Xxz;kGDPyvl4LDwf>&>rq!kn68EP76JTB2$Dt{v=RUfPk|*#sYkLY7*$?*B z$3k$W&?0`WfFMY#7n{Xs6`ws`M%P5=Pi(bKvk25qnCH9JVlj^|PThm$bJn)Dpi~tZ z9Y@g9r2qi!fa9>ESyZ<0K_eZ`cTIMW>V@USWg+-WXOHN4;3sA&s~s{^poS#Cn}2bs zN@}>tor>5l;lL!f;$de`D(E^-vQ^zUzH$QZ|BP4vnLF7i)MUDT66f2mFW4BdBaP_v zE1QewIj~XfIeUCvS#r`Dlpt)j8jxp`d&! z2twSIqX;cnDJEHl^Yf=n^$Yoes5Ztp`8H`OE?}EJ|G?gpzVQnq17qutS$nz_>`xO( zCMoy^U^qmI&08L#yxcyonL5r5UPNa-o)RU1$xW+|e`NFt9w6O{!hFbX7SA~%0WKB! zqA@}tCXzUL3aSgb;|?X>sDpFD$Wvs(*I3hNDs(Uu7en^AGr=1`2Bh9JHP$kKKYKty zK`9h|QAiXs{AoU6@g@2-uKQOJC;9Ta7u%7qDP8IFLhs8q%#CugqP@WS6!oQsM(N4c z7jawS`PQ97i|s&u>R}P3Luz4wAsDE80hh-%08+oZ3*#gBEKbfx>YLR#!@QBl=k!TL zz8hvXXd+$SIJE4=S0EIi$2~FI3LqcDHOyCSzj=@951W&8wcd_&T#WO=wsImZkT>OP z%x%q`!bzBryhuDW&!-B7LHj)}gHZMF90~g_0Psr?eB90%)No3UGDbS&7ok$GM)c-k z%l=wQ)q_hsl=#%n+E9Hs$CH#C>kxq0TAF8<1DK?Pz)|fi)9$RegCjGHcY^UjQ7w5Z z%stuGFJ|V$TJP;TfyL~2sO}bJvZ|iRTk_zL>x{Y)nUwwMi3q_yZ6pt!GQ!%?(jpM- zFE|DB)9v{WIQ89)W&I1>bmDUp7`UlBySw$~`2rD;|AkZ5s7wzVj0Cu^oIIKiOqMYI z9n$~bTrmBYd#?4^6=f;_F!^8Pry*{T0mL( z=mMcYfa3(aA-a$!7zy|%A<9cFSrjM?=n;so<-tdEjx`r_PW#rWzG=H&l<=t-fu5`} z@AK^>cHhxYg2BPhg?*ae! ze#G#@`I`<@mVVEIV#3y4 z&l_goVzaus14gfb7{ssN-W5c4{`j84adSDzTZtw?wV;~^q*JlCXB=PfZqdS{6x zZhNIhnPyfKK%$fcpd0?F1W3$xCT(I}%vf3V7z4h4KdV7_{TZWhjOvKf$rU|E%VFyfhQ#p z1un_cW!Z^nk*u8i(#713TJs}pNdQo)gn*##7pb28MeAu;3xx;U$XLeV^j~Epg3LwB`#sL2C?}I#krD;v-c~>U{$O69N<;rc%xNbQ#7YnzgDq-Lcf`3VqX>#Jh4)RMIPM zj0U1r_Ij8x2==(m#qOx626x~s(!xsdT>R%lh;y7nw8mG|x$jiKfA!d3FN+@Z^6*4d zAhxsuAf)g4e1NR&l-dc3JQM0$5q=7Wl*|WqEx%xT#0MrYD=_e1RVR~nn}|NLcF8&m z3!A20)ikU24{|%BBbHTE*kByON020SbZGWVNviavN4w=v94XseV>XcICFEe9JW9f) z5Ra}-p1If$qdMO?{wBeSD=$9Rj<;;Rbp#%0=*PjMxtIFJ-c(VE|D0+oZ|?p%KxOR? zn%VxnlCy(J`%DtsV6a;D)T6S_?p1{O%RCy5SZY9Z34`gY?DvlP}TCSgD{9Hm=pod zT+njwUQr`cTeOB*09T>im(7y>dXjIyApoIovyE3;=$!S1w}yt~Qb9R7+8z?WI7ojl zNzld5nFdeHayaIFxf&A8>T)ZcsM462*|<_75b`xkpxHgFdonqjGHdDlzDhnWya3M` zWXm0$Shnx}Qs{W!U?2|v?WV4q?<`iUrM30Btc`YJ?jxi&P{DDE=OPssAm|rLAfYhK zbpu^pa9vHx>NNZf(Sk>iUnVXS=1E#1_2_-8H_O%IE@8uQ%E%;1Z#dpk3CFQo1}ZQ$ zy-t=`P5~&vUt)j2CwBn= zbg@cT@|uZ%`4TDBn6@t}VpH{7t@PjYT?Y1^s`q$nG@7O4BvHYW+Ah?EU7hf<@SP159U`DFzkqUjmOQ%RP}rN+ZI5N>?1q)SDUj8NdB`Vu>+Z1GXUlVYNE&*A()vCCBRS={_wX zUJi|`^pi#aT=V78YX6U*&PoOCAF0_}bKAI4h^|#I0`9dg%1pyg5zTKe=j*mUUwvjq zm?Hp(FI$r1gSk|gp#tJ!tLNRsV|g&jG;;G7JgC8HxOx2Q;A4Fh=l-HDSscyeOMjqNe~>eZcDO7N@E{8PP-ezA8{3=8t9ir*(}gxUSG40xrVWlHA0rq6l}VK_uc zQXnKrvnP=xu?P({W;?fw%TpER;}qSoH>ZVX3Dtqz@kpHU z=4Wa~!&yj0`eSZB&Q%kx`YOm@6nZMq%(21bGINwniMo<5&2>8OfdrbI=Nri-5;E*D zF%5dL#=j7n($5rjvf3XKeMrjg(1cTqTagpS^Us$bSh(_c->53ufFuV=4Q{BNbs4xH z>=rbp5oCOOaXV7uRU(#E%cKd@%HpOjVs)2j;pV-}Zcw?1QH{^=7HhUaVdnoqaR&y5 zKPmA^Q1KT%V4T8RLAcATC92h8?beP387Cs!S7gT93ooMt63ZRDTU;jl$e>1=?hg_l zs!h~YE?Tt9gf(3YY9h+$O>6t79|3IVahvppQ`GpE8^>?1TnbAvArU3K3RfzN5=#^F ze7u_#Dce3Ya;S?T6DhffODml@QWvtr{+VS)MT$%yTSh@zWF-~<3^v2o*}-cnP1hBa zYB*Mdh6_8pg`0c5acp?EQ<*ZRDWhT!W_$Jxczod$r#sTHRg}x(52-;MemH{LwO_rwZuJT_8LYvRtOm{b`nrDiM zS+KFNYcIDqatkkVaDpgZ4+@sH6E&w48A&1UUDo84lI66dBkKWqy{(&j(|x_>Mq1a3ZZFE60bi}JX7 zO-?E)7Jj)xJW7Q?zPV;d>1uPC6#`?IH99qs;--&Kv)Q+ylVIGja+->IfLn9Y6!g}s9rlOk>OC477J#eY(l`;4yNTzjiknH z2bygr_e%1_YE<$T{043T8P5;LLOGb?EyHqCg)7Q4J=1YP<5g=5^@-MG>LL|=3{^Gr z2+cC)5Cwm9Q1US2&;+dTqetZH6DRXeiX1fKLXBl|Nx98M#@FpGYq>?617}dbXwEmd zK9r4Jqn4ABl1eC<=(cEeOP#b=Yao{+pni$2ja@)MalkY_ASkU ztqbUvNozX${vCplJFy$}ch8<(lf}v+9v&WsH7z#iEZOVn5fKp?i;EhezqtnM=gOO) zjvG+QB@w1E`b*=g-ak6J9OJJo`)~LVbDocXCDqWtfQo}7PWHFW^n86!%dUURrf+1V zOlrn7KG!(bpHNOtF8wtFs;`ev@1C8_r$3S%-rC-#VPR2t<;D1WJf1d! zZ9c^NRVBZZ86M%Oe8!=hSYkfTdi%1uCxo_Uc#)nz2{%dfrAKQvo<|;8r#u!?d4jVv`ap)X@7A=0~VWnUryA1 ziThcaEyX4HnCm=0ws~2TKa#90^zpRiIE`BtKNN#1yYC&})i`NGZcW&x%q0HIi|QT2 z;MfXxYW%}0kZnD8t3h|!b^kRioev!-p0KRKW|h*qxu7-T_T1#c6KrR-2i1JnZdS4! zM-JVSKX4?w=Zi{@X(I0H+Bz0L!$00c-=k;1=OBeBwVC|PYPI;{E685p& zE%E~#e*HDBrRZJOpP4i(OC{c%@=0)z=HH(Vo|4oONZcIFw>(r!q*f|gn~JhWOnGK= z9kXuKNsJuGw(JVS-L|Gr_$ls8_+0G8rx&9FECjg4J%zm}C#?IVHLkACzQ491yMLh4xWCyYT7S^J-;V8} z95+L;8&b-aG<0a-i+$nwGg4~d}$WRCigE?Plpg<8}cG(N7+AiwC%e2EhSnj6#1h6@!sU%E-26wqQVQaeaXpY4wk5p~^) zhk}(69jq&bz@IleLbo!5vX*QC)a_I18{9CGktWxK>DT%_-@4cJ_wC(p%hZ!m3kx%_ z?>5b+UUP5tS+EHe7WrMofBDipt`uhvMSCY+%Ipv}9a1FwP#0vgNgPvxP^Q3_Mv%1` z#y{a2=9alsuQZi;Gvso4mNfOf=r;mLU&NUmEfV}0$KGkMAlj&W>x!vec6(xv=Q_FJ zN-Vv2Jne~<`=tUA_k)7H%mY!QOuP(Vaz~OrRo9H7)W^O9{tpWBqFtP_#j{LVR#||- z<&4ZKZZFM8`oSeWI~`A7tbZB^lf&q)r(XKVRLglD^(sD=Q-v9(1S z69S7U1ns-~x|ppG2}^^+UqYR|uI9fhH5azN$7{|^kgZiL!1nd&F`l)x#-j+cK`tGy z%S#(e>){wrrc6?ayonj%GB^HBlSLYUejg@$06m%92pZ+)gvm`JZGm%>FT`#31gtsr3{ zZ@*|y;cDuVfo-Q*pzwZsF0md&3Sv;qXrZ2O6lf)*ie^3<*B33^20qra8@ObDH=uuB zwI%PoR9Y!IilqM>f*=3+ky>6gmrS-WRyv_9B{ed7tq?71VUNj+XwUcmR6Tpw1PAfB{kIQD@15icDm&?3j;WLucuQ}&wZ*cU}R4rD8 z;Rn@d;sbp3eP^0a2GP{#8q;%m>IyHhl(0EA3+*#f^&z(;e(|b;-A-l#R3gLzy@}gu z_AT+qnbG7ahoi!GOj+w5)|R=Z5|3z@_U_8HY$2n#S;E{)xeJNI)7iK74L5T&cn+4u zsqQ~p=P~Co_{@@Xd7X}{7(QGfQHOTpO+M9ihwhdkid}4(@XD37*!$}pMPd}25NdmAO=eaZU)BWu{=RD7Q&i6g<_dTEYbKd3R zEZLkDnhO;|rcw9r)~`-=s6KP0FZtro8OIeueidH6E_|Sdvf%?K603B2Ab4)y06bol z*TFrC$U*}!_xt+7)VG(8-6^25t<=Q9R=dV*!eZKDuJG!xvAWWGLMl>Ymec|X3Z zyBllMz>$gc%RX_`utmiLvum$@-DxaXSaw+(%nY4)@^Y8Gy}>6wQ(G{`32Ps5pRHCt zd!G_hAaJ0Yl2^{H{PHj<&T5?4R);3Lr4p4{6w)XseSQZ&=8z5rEyNL(w8W`59O4s^ zvMPjtXuQJL{5e|~%~?D#)Uz&kua#o7onSf^J~w$c1qH{gHps*YC~XoBBEW}6#DCdk zI<;v&H_!~P&_3R=r@YMV^Bo;ofGGz5enSRxGp8P(Me=#^isu{Bh6E+Xan1-ty$xh? z=ey0!kqK+JLo>S7MU~Q?lC+ooq42(OyUcy38oY^*0C$*+)qLUgUd#)tWq z$0&J(yAk2*NOOB3)Jq}I_Wd>K*F_pX2hZx6aF(u0Tz2o^kF}s^4bBZ8(*jKlyajUJ z4LSwS%7$HvKk{hfITgJFE@X%5H5GG8l0H);l}FySx>U9~eSginO&tI+Ga9LZEVHiM z*N5kxI+3rJQ_4*C?%@Sp4yNmM?ZRURL_5=MmA0b#d&IqIx}%=inZMNGij5r6_7p+l z9#lTvtA}b#LD`d;vqF1{*JQM*g<-~S)b)J@eK3thgd~BP&T5j~3KAUQ(&^GcnxieQ zCTXnL6F($V7KRN9+soB3^~4gX$H||6izxRqX2MyH%E7TItJy*ig~j~*BU!~YO^|ImV{OAqF67gbuW_WD-OccR+SmxTs#PBkQ}I0v?;nZKbi0>| zi>YDUe+@_Pk_xxj@K3=5$N;?;SHiao>ePJ(9mVm$Jd#P)dY%Zthh}SLFO2IA(Yt4S zc9bb=!_0{W1BO-OR)OvBUMY3Mp&#eTNK#mrsbcNUcLF#WQEPdyIBGx5E=d4fRK`aOwq>pZke{{c zRjwLb;`HmwT^%zX&-GF8hNl2H%ial(1ky5VEc!Gu@v@QP{pti1c^q?d_>7zmBhl;m zJ&f%Tu$_L3@_n}Z7S@l_-KXzw$qd@O6Hqa5?7c2C1^UDu}RYfO@ z%9>I3Rd%A^HAItLn1VD6ZaZ8nS=F$f+2T8!F$xl|x|5`{)!zBx1w->0UbacBcM03= zKY;L8d>0rjU_L6jzk)?r?KYPx%)}V%;oSew#!xJqeA4f;BV=l*iS{Y_#|I^54Mt7j z;Hcz7Pe+R}0;M#X8^n*zik9p8qgCusE?s@5Zt|5SgR+eT#j?B7M~C^hpa{IyXhA^0 z=&=PU#QivFZIcnZLO_HDh(AXATmU$!*pK8ATvL~^e6rtT#PGFNC11Dj5V1&=x%AX& z6Q_q6n*~r|o=WZxQ|g+$WQ1$v3WlR~1Q69ecw^#hkMW)wv`02pEzJesr{YD|ME`Q! z(jRg;CK9iUb4&JZ?_g2Ai#yNM2P{}MmY)-hYFY0mC#a2wz#B8nz=Gxe&5jidNJTpQgxJq9PHoC{ zPqu>|^^e={q`DYISLCZIO7}86pF*w~e%-T4OD=eeI2St-eBk057Mm;LI^#b#o`tEZH_C@x=pMc3ezfo6= z9Tm?#i0`HCSomvhS;X)~p6Vmz99#1&`e|AsyAiWR?_8lae045fEV)6cS>V@zTJ|DtB9_yN-OukO4iA^+I&YMKBPp+=TaP_TKQx%;$E<9hQR1v4=SbIf zQ0?$DiK{hQ7qoh(0tzvf&n?cY`1Q%BivL1L^hDP zRI@(OMtQb6qNh->z`~qVBa{dYMp`9T+VW}SyrfG0QIyPVrl1)TrL{_ft z>O6lSrLJzP(Wtee8JY*!=M^lMQj^lhYR6xSuzpoCuMJ$gg`BP%fq_$Sk!u02nU=3E z7&;Udy0)D;h3Zz9?_48qI}Zmj#gyzA$`81P8i;DqsZTctCDjK>k%chN(=cF2+2q-y zr1RKg8oQsrZKn)6yMh0bf$v}t^nJQG=_2#iX?eSZtYWPJMpo@b>uLvZRI8HbdT&_V zBTu|)VRm&bAMd2#58#9wnnRaQUoP!g5!EeHjT)bWJU7E8ib;u>TMU!#PFt;HEs|FZ zB8)4{^#`N}l5m0jHL-W2z2ZvjO{{B!B!}$J3s?$6J@-5^Qh(~ceRKi~l;BJ#I4Z(f zmg|8mkM49A`i9ZU8OhZp8N`PSiM~#&UMyiawC=Xp(8L@+G&#>C+9@?eQQM;<32|DA zz;WOUcJFYh3Ea()*mjH0vmwXzREsviUi-qN!uKQ*o!7a&*nYcDMt_*mFdM1fTt{vY zd$ZXdlAwC}D8+77_!b-h@ptLHx@%&Ngq(Y~t!|V56bT{+BpTf1fu7A6S3K z%mGX%aZ8Kmey&+}y2yD#H$?ziB>dU=&G1?u+i)46=~fY`osG9F%>_g) z4zznJ&XaA6&E^BTLgoTZGQfx!b$q_S_LqE2_`4F_ycr+Ii8hxFuz^nMrw}#_oJk*| z=s7ZSaV}d}{?LbTg?XHlx@VdV&Bz{IFc;uRI~td#0gW%ewc^XPES~cjWD6LbQ*~|| zs-o#(yLw1T8t>kGyuB5rQ{2dmumOj3alJn>K4Vb9)MA9z|HT?de;tMQP11yOhtqK@ zaPVa2NAKebdrhxxW_Z(S;Z*=8L85hEp2A?(kpVK#h+P(gIK=ltU57}D^^{v+U@fL} z83;6#laoVT+!RNW=H@om@**QbT-&T*UwL^M=q}Wbf8qYPMZ#l$KiXk3!=@->W-Csl z>RKsCiTI9@i=7?*<3HaI<>R$HRXhwq6Qq9@dL0UG#ca>P9fN*6c$vFnKe>IQ=&lEv zGHx8k8UIXz#Q&IsF?q#JCCAj&)pa@cU>!X7E?8Sz`>E!*)^u|TU^|(NL{T-pcZZi6 zE^_C51kAsetF|M8>`M>y>ky=^r4<1Jfl@J;j+2Le;r7kx0M$vq>WS%T7 literal 0 HcmV?d00001 diff --git a/docs/user-guide/2-Storing-malware-samples.rst b/docs/user-guide/2-Storing-malware-samples.rst index 897093452..99151acd4 100644 --- a/docs/user-guide/2-Storing-malware-samples.rst +++ b/docs/user-guide/2-Storing-malware-samples.rst @@ -98,3 +98,23 @@ Using that feature we can chain together various files extracted during the anal - two memory dumps with unpacked code Finally, from both of these dumps we got a malware configuration. + + +Related files +------------- + +Sometimes we want to add the file that is related to the object, but not containing any malware i.e. pdf report. Every object of type **file** has designated tab for such purpose. + +.. image:: ../_static/related-files-tab.png + :target: ../_static/related_files_tab + :alt: related files tab + +In this tab you can: + +- view list of all related files associated with this object +- upload new related file +- delete related file +- download single related file +- download .zip file containing every related file + +Keep in mind that those actions might require special capabilities. For more details, please read :ref:`9. Sharing objects with other collaborators` From 64d653ae3a4bf66a8a8ee96e2645fb0567742385 Mon Sep 17 00:00:00 2001 From: Repumba Date: Wed, 21 Dec 2022 17:15:30 +0100 Subject: [PATCH 27/33] Querying by related files --- mwdb/core/search/fields.py | 83 ++++++++++++++++++++++++++++++++++++ mwdb/core/search/mappings.py | 2 + 2 files changed, 85 insertions(+) diff --git a/mwdb/core/search/fields.py b/mwdb/core/search/fields.py index f7ee471ff..974c6c1c9 100644 --- a/mwdb/core/search/fields.py +++ b/mwdb/core/search/fields.py @@ -19,6 +19,7 @@ Member, Object, ObjectPermission, + RelatedFile, TextBlob, User, db, @@ -732,3 +733,85 @@ def _get_condition( else: condition = or_((self.column == value), File.alt_names.any(value)) return condition + + +class RelatedField(BaseField): + accepts_range = True + accepts_subfields = True + accepts_wildcards = True + + def count_condition(self, expression): + def parse_count_value(value): + try: + value = int(value) + if value < 0: + raise ValueError + except ValueError: + raise UnsupportedGrammarException( + "Field related.count accepts statements with " + "only correct integer values" + ) + return value + + if isinstance(expression, Range): + low_value = expression.low.value + high_value = expression.high.value + + if low_value != "*": + low_value = parse_count_value(low_value) + if high_value != "*": + high_value = parse_count_value(high_value) + + low_condition = ( + func.count() >= low_value + if expression.include_low + else func.count() > low_value + ) + high_condition = ( + func.count() <= high_value + if expression.include_high + else func.count() < high_value + ) + + if high_value == "*" and low_value == "*": + condition = True + elif high_value == "*": + condition = low_condition + elif low_value == "*": + condition = high_condition + else: + condition = and_(low_condition, high_condition) + else: + upload_value = parse_count_value(expression.value) + condition = func.count() == upload_value + + related = ( + db.session.query(RelatedFile.object_id) + .group_by(RelatedFile.object_id) + .having(condition) + ).all() + found_ids = [r[0] for r in related] + return RelatedFile.object_id.in_(found_ids) + + def _get_condition( + self, expression: Expression, subfields: List[Tuple[str, int]] + ) -> Any: + if len(subfields) <= 1: # [('related', 0)] + raise FieldNotQueryableException("One subfield is required") + if len(subfields) > 2: + raise FieldNotQueryableException(f"Too many subfields: {len(subfields)}") + + key = subfields[1][0] + + if key == "name": + field = StringField(RelatedFile.file_name) + elif key == "size": + field = SizeField(RelatedFile.file_size) + elif key == "sha256": + field = StringField(RelatedFile.sha256) + elif key == "count": + return self.column.any(self.count_condition(expression)) + else: + raise FieldNotQueryableException(f"No such subfield: {key}") + + return self.column.any(field._get_condition(expression, subfields)) diff --git a/mwdb/core/search/mappings.py b/mwdb/core/search/mappings.py index 8e746f97d..1547e9ea4 100644 --- a/mwdb/core/search/mappings.py +++ b/mwdb/core/search/mappings.py @@ -23,6 +23,7 @@ JSONField, ListField, MultiField, + RelatedField, RelationField, ShareField, SizeField, @@ -56,6 +57,7 @@ "karton": UUIDField(Object.analyses, KartonAnalysis.id), "comment_author": CommentAuthorField(Object.comment_authors, User.login), "upload_count": UploadCountField(Object.upload_count), + "related": RelatedField(Object.related_files), }, File.__name__: { "name": FileNameField(File.file_name), From 06251c5d7b3293e93a6ebf543fad7b793aa16f3a Mon Sep 17 00:00:00 2001 From: Repumba Date: Wed, 21 Dec 2022 17:24:59 +0100 Subject: [PATCH 28/33] Documentation update --- docs/user-guide/7-Lucene-search.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/user-guide/7-Lucene-search.rst b/docs/user-guide/7-Lucene-search.rst index a7aacb576..b6ac61141 100644 --- a/docs/user-guide/7-Lucene-search.rst +++ b/docs/user-guide/7-Lucene-search.rst @@ -548,3 +548,13 @@ Afterwards, you can see your newly added query as another black-coloured badge. .. image:: ../_static/7dXJkSH.png :target: ../_static/7dXJkSH.png :alt: + +Related file field (\ ``related.:``\ ) +------------------------------------------------------------ + +You can query objects by their related files. There are 4 ways to do it: + +* ``related.name:`` - query by related file's name +* ``related.size:`` - query by related file's size +* ``related.sha256:`` - query by related file's sha256 +* ``related.count:`` - query by number of files related to the object \ No newline at end of file From 52a3497c963c822f56f86f34e1552402c9e63322 Mon Sep 17 00:00:00 2001 From: Repumba Date: Thu, 22 Dec 2022 12:19:43 +0100 Subject: [PATCH 29/33] Documentation for querying and automated tests --- docs/user-guide/7-Lucene-search.rst | 32 ++++++++++- tests/backend/test_search.py | 84 ++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 2 deletions(-) diff --git a/docs/user-guide/7-Lucene-search.rst b/docs/user-guide/7-Lucene-search.rst index b6ac61141..19b5761e1 100644 --- a/docs/user-guide/7-Lucene-search.rst +++ b/docs/user-guide/7-Lucene-search.rst @@ -555,6 +555,36 @@ Related file field (\ ``related.:``\ ) You can query objects by their related files. There are 4 ways to do it: * ``related.name:`` - query by related file's name + +.. code-block:: python + + related.name:"name.txt" + + +This field accepts wildcards. + * ``related.size:`` - query by related file's size + +.. code-block:: python + + related.size:"<5kb" + + * ``related.sha256:`` - query by related file's sha256 -* ``related.count:`` - query by number of files related to the object \ No newline at end of file + +.. code-block:: python + related.sha256:"2ed91d820157c0530ffbae54122d998e0de6d958f266b682f7c528942f770470" + + +* ``related.count:`` - query by number of files related to the object + +.. code-block:: python + + # get files, which have at least 0 related files and no more than 2 + related.count:[0 TO 2] + # get files, which have at least 2 related files + related.count:">=2" + # get files, which have more than 2 related files + related.count:">2" + # get files, which have exactly 1 related file + related.count:"1" diff --git a/tests/backend/test_search.py b/tests/backend/test_search.py index 6ae8ac0b3..4d7138c6f 100644 --- a/tests/backend/test_search.py +++ b/tests/backend/test_search.py @@ -2,7 +2,7 @@ import datetime from .relations import * -from .utils import base62uuid +from .utils import base62uuid, calc_sha256 from .utils import ShouldRaise from .utils import rand_string import random @@ -668,3 +668,85 @@ def test_search_multi(admin_session): wildcard_hash = samples[0].get("sha512")[:-100] + "*" with ShouldRaise(status_code=400): found_objs = test.search(f'file.multi:"{wildcard_hash}"') + + +def test_related_files_search(admin_session): + test = admin_session + + filename = base62uuid() + + # File A + file_content = base62uuid() + FileA = test.add_sample(filename, file_content) + + test.add_related_file(FileA["sha256"], "related.txt", "b"*1024) + test.add_related_file(FileA["sha256"], "related.mp3") + test.add_related_file(FileA["sha256"], "related.png") + + # File 2 + file_content = base62uuid() + FileB = test.add_sample(filename, file_content) + + test.add_related_file(FileB["sha256"], "related.txt", "b"*2048) + test.add_related_file(FileB["sha256"], "related.mp3") + test.add_related_file(FileB["sha256"], "related.jpg") + test.add_related_file(FileB["sha256"], "related.csv") + + # Related file name + results = [ + result["id"] for result in + test.search(f'related.name:"*.png" AND file.name:"{filename}"') + ] + assert sorted(results) == sorted([FileA["sha256"]]) + results = [ + result["id"] for result in + test.search(f'related.name:"related.mp3" AND file.name:"{filename}"') + ] + assert sorted(results) == sorted([FileA["sha256"], FileB["sha256"]]) + + # Related file size + results = [ + result["id"] for result in + test.search(f'related.size:">2000" AND file.name:"{filename}"') + ] + assert sorted(results) == sorted([FileB["sha256"]]) + results = [ + result["id"] for result in + test.search(f'related.size:"<=2048" AND file.name:"{filename}"') + ] + assert sorted(results) == sorted([FileA["sha256"], FileB["sha256"]]) + results = [ + result["id"] for result in + test.search(f'related.size:[1000 TO 1100] AND file.name:"{filename}"') + ] + assert sorted(results) == sorted([FileA["sha256"]]) + + # Related file sha256 + sha256 = calc_sha256("b"*1024) + results = [ + result["id"] for result in + test.search(f'related.sha256:"{sha256}" AND file.name:"{filename}"') + ] + assert sorted(results) == sorted([FileA["sha256"]]) + + # Related file count + results = [ + result["id"] for result in + test.search(f'related.count:"3" AND file.name:"{filename}"') + ] + assert sorted(results) == sorted([FileA["sha256"]]) + results = [ + result["id"] for result in + test.search(f'related.count:">3" AND file.name:"{filename}"') + ] + assert sorted(results) == sorted([FileB["sha256"]]) + results = [ + result["id"] for result in + test.search(f'related.count:"<=4" AND file.name:"{filename}"') + ] + assert sorted(results) == sorted([FileA["sha256"], FileB["sha256"]]) + results = [ + result["id"] for result in + test.search(f'related.count:[1 TO 3] AND file.name:"{filename}"') + ] + assert sorted(results) == sorted([FileA["sha256"]]) From 60e97d31d049c454f2c238b97ca5d88f9cd30eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Srokosz?= Date: Mon, 23 Jan 2023 16:19:21 +0100 Subject: [PATCH 30/33] js => jsx --- .../Views/{RelatedFilesTab.js => RelatedFilesTab.jsx} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename mwdb/web/src/components/ShowObject/Views/{RelatedFilesTab.js => RelatedFilesTab.jsx} (99%) diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.jsx similarity index 99% rename from mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js rename to mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.jsx index a44689def..d92981a84 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.js +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.jsx @@ -10,13 +10,13 @@ import { faArchive, } from "@fortawesome/free-solid-svg-icons"; +import { APIContext } from "@mwdb-web/commons/api"; import { ObjectContext } from "@mwdb-web/commons/context"; import { ObjectAction, ObjectTab, ConfirmationModal, } from "@mwdb-web/commons/ui"; -import { APIContext } from "@mwdb-web/commons/api/context"; import { humanFileSize, downloadData } from "@mwdb-web/commons/helpers"; import ReactModal from "react-modal"; From 6c9bbce26f5e4b5d2169af0b40af34e64996f015 Mon Sep 17 00:00:00 2001 From: Repumba Date: Wed, 15 Mar 2023 14:09:28 +0100 Subject: [PATCH 31/33] fix migrations --- .../versions/3c610b0ddebc_add_related_files_table.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py b/mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py index 21d625637..02c539458 100644 --- a/mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py +++ b/mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py @@ -1,7 +1,7 @@ """add related_files table Revision ID: 3c610b0ddebc -Revises: c7c72fd7fac5 +Revises: 717b5da712b8 Create Date: 2022-12-12 09:02:40.406370 """ @@ -10,7 +10,7 @@ # revision identifiers, used by Alembic. revision = "3c610b0ddebc" -down_revision = "c7c72fd7fac5" +down_revision = "717b5da712b8" branch_labels = None depends_on = None From 05d729db98e610c27e41188d561f819508ff1a77 Mon Sep 17 00:00:00 2001 From: Repumba Date: Wed, 15 Mar 2023 14:23:04 +0100 Subject: [PATCH 32/33] fix migrations 2 --- .../versions/3c610b0ddebc_add_related_files_table.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py b/mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py index 02c539458..3286fb65b 100644 --- a/mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py +++ b/mwdb/model/migrations/versions/3c610b0ddebc_add_related_files_table.py @@ -1,7 +1,7 @@ """add related_files table Revision ID: 3c610b0ddebc -Revises: 717b5da712b8 +Revises: bd93d1497694 Create Date: 2022-12-12 09:02:40.406370 """ @@ -10,7 +10,7 @@ # revision identifiers, used by Alembic. revision = "3c610b0ddebc" -down_revision = "717b5da712b8" +down_revision = "bd93d1497694" branch_labels = None depends_on = None From 441612d7c8ebde705be79210fdb473ddd789b4a1 Mon Sep 17 00:00:00 2001 From: Repumba Date: Fri, 31 Mar 2023 15:04:09 +0200 Subject: [PATCH 33/33] add toastify for alerts --- .../ShowObject/Views/RelatedFilesTab.jsx | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.jsx b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.jsx index d92981a84..89041d668 100644 --- a/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.jsx +++ b/mwdb/web/src/components/ShowObject/Views/RelatedFilesTab.jsx @@ -1,5 +1,6 @@ import React, { useCallback, useContext, useEffect, useState } from "react"; import { Link } from "react-router-dom"; +import { toast } from "react-toastify"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { @@ -16,19 +17,22 @@ import { ObjectAction, ObjectTab, ConfirmationModal, + getErrorMessage, } from "@mwdb-web/commons/ui"; import { humanFileSize, downloadData } from "@mwdb-web/commons/helpers"; import ReactModal from "react-modal"; async function updateRelatedFiles(api, context) { - const { setObjectError, updateObjectData } = context; + const { updateObjectData } = context; try { let response = await api.getListOfRelatedFiles(context.object.sha256); updateObjectData({ related_files: response.data.related_files, }); } catch (error) { - setObjectError(error); + toast(getErrorMessage(error), { + type: "error", + }); } } @@ -87,10 +91,19 @@ function RelatedFileItem({ file_name, file_size, sha256 }) { setConfirmationModalOpen(false); }} onConfirm={async () => { - await api.deleteRelatedFile( - context.object.sha256, - sha256 - ); + try { + await api.deleteRelatedFile( + context.object.sha256, + sha256 + ); + toast("Related file deleted successfully", { + type: "success", + }); + } catch (error) { + toast(getErrorMessage(error), { + type: "error", + }); + } updateRelatedFiles(api, context); setConfirmationModalOpen(false); }} @@ -106,11 +119,10 @@ function RelatedFileItem({ file_name, file_size, sha256 }) { function ShowRelatedFiles() { const api = useContext(APIContext); const context = useContext(ObjectContext); - const { setObjectError, updateObjectData } = context; + const { updateObjectData } = context; const getRelatedFiles = useCallback(updateRelatedFiles, [ api, - setObjectError, updateObjectData, context.object.sha256, ]); @@ -162,7 +174,6 @@ export default function RelatedFilesTab() { const [file, setFile] = useState(null); const context = useContext(ObjectContext); const api = useContext(APIContext); - const { setObjectError } = context; const modalStyle = { content: { @@ -179,8 +190,13 @@ export default function RelatedFilesTab() { try { await api.uploadRelatedFile(file, context.object.sha256); updateRelatedFiles(api, context); + toast("Related file uploaded successfully", { + type: "success", + }); } catch (error) { - setObjectError(error); + toast(getErrorMessage(error), { + type: "error", + }); } }
+ File + + Actions +