From e30a22d4283bb71ce53aeb2267e34f83c20a2e1e Mon Sep 17 00:00:00 2001 From: Greg Kostin Date: Thu, 23 Jan 2025 11:09:58 -0500 Subject: [PATCH 01/23] Scratch workflow placeholder to create new workflows from a branch other then mainrefactor scratch directory --- .github/workflows/scratch.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/scratch.yaml diff --git a/.github/workflows/scratch.yaml b/.github/workflows/scratch.yaml new file mode 100644 index 0000000..beaa90b --- /dev/null +++ b/.github/workflows/scratch.yaml @@ -0,0 +1,19 @@ +#Scratch workflow placeholder to create new workflows from a branch other then main +name: Scratch + +on: + workflow_dispatch: + inputs: + tag: + description: tag + required: true + +jobs: + get_short_tag: + name: get-short-tag + runs-on: ubuntu-latest + steps: + - name: save short tag to environment + run: echo "short_tag=$(echo ${{ github.event.inputs.tag }} | head -c 8 )" >> $GITHUB_ENV + - name: echo env var + run: echo "${{ github.env.short_tag }}" From b0fa09930d93b6761d0ce69c9313c943c0d607ee Mon Sep 17 00:00:00 2001 From: Greg Kostin Date: Thu, 23 Jan 2025 11:45:51 -0500 Subject: [PATCH 02/23] fixes --- .github/workflows/scratch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scratch.yaml b/.github/workflows/scratch.yaml index beaa90b..5929ddb 100644 --- a/.github/workflows/scratch.yaml +++ b/.github/workflows/scratch.yaml @@ -16,4 +16,4 @@ jobs: - name: save short tag to environment run: echo "short_tag=$(echo ${{ github.event.inputs.tag }} | head -c 8 )" >> $GITHUB_ENV - name: echo env var - run: echo "${{ github.env.short_tag }}" + run: echo "short_tag ${short_tag}" From 3bde69431f0c919e57c601d5d1eedee6bb04e649 Mon Sep 17 00:00:00 2001 From: Greg Kostin Date: Thu, 16 Jan 2025 11:03:23 -0500 Subject: [PATCH 03/23] comit for roger --- features/steps/store_resource.py | 195 ++---------------- ...resource.py => test_store_resource.py.bak} | 0 2 files changed, 18 insertions(+), 177 deletions(-) rename features/steps/{test_store_resource.py => test_store_resource.py.bak} (100%) diff --git a/features/steps/store_resource.py b/features/steps/store_resource.py index 9b750e2..dd30980 100644 --- a/features/steps/store_resource.py +++ b/features/steps/store_resource.py @@ -7,6 +7,13 @@ from behave import given, when, then +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + + +from dor.adapters.catalog import Base, _custom_json_serializer +from dor.config import config + from dor.adapters.bag_adapter import BagAdapter from dor.domain.events import ( Event, @@ -21,183 +28,11 @@ from dor.providers.package_resource_provider import PackageResourceProvider from dor.service_layer.handlers.store_files import store_files from dor.service_layer.message_bus.memory_message_bus import MemoryMessageBus -from dor.service_layer.unit_of_work import UnitOfWork +from dor.service_layer.unit_of_work import UnitOfWork, SqlalchemyUnitOfWork from gateway.ocfl_repository_gateway import OcflRepositoryGateway from dor.service_layer.handlers.receive_package import receive_package from dor.service_layer.handlers.verify_package import verify_package from dor.service_layer.handlers.unpack_package import unpack_package -from dor.providers.models import ( - Agent, - FileMetadata, - FileReference, - PackageResource, - PreservationEvent, - AlternateIdentifier, - StructMap, - StructMapItem, - StructMapType, -) - - -class FakePackageResourceProvider: - - def __init__(self, path: Path): - self.path = path - - def get_resources(self): - return [ - PackageResource( - id=uuid.UUID("00000000-0000-0000-0000-000000000001"), - type="Monograph", - alternate_identifier=AlternateIdentifier( - id="xyzzy:00000001", type="DLXS" - ), - events=[ - PreservationEvent( - identifier="e01727d0-b4d9-47a5-925a-4018f9cac6b8", - type="ingest", - datetime=datetime(1983, 5, 17, 11, 9, 45, tzinfo=UTC), - detail="Girl voice lot another blue nearly.", - agent=Agent( - address="matthew24@example.net", role="collection manager" - ), - ) - ], - metadata_files=[ - FileMetadata( - id="_0193972b-e591-7e28-b8cb-1babed52f606", - use="DESCRIPTIVE/COMMON", - ref=FileReference( - locref="../metadata/00000000-0000-0000-0000-000000000001.common.json", - mdtype="DOR:SCHEMA", - mimetype="application/json", - ), - ), - FileMetadata( - id="_0193972b-e592-7647-8e51-10db514433f7", - use="DESCRIPTIVE", - ref=FileReference( - locref="../metadata/00000000-0000-0000-0000-000000000001.metadata.json", - mdtype="DOR:SCHEMA", - mimetype="application/json", - ), - ), - FileMetadata( - id="RIGHTS1", - use="RIGHTS", - ref=FileReference( - locref="https://creativecommons.org/publicdomain/zero/1.0/", - mdtype="OTHER", - ), - ), - ], - struct_maps=[ - StructMap( - id="SM1", - type=StructMapType.PHYSICAL, - items=[ - StructMapItem( - order=1, - type="page", - label="Page 1", - asset_id="urn:dor:00000000-0000-0000-0000-000000001001", - ), - StructMapItem( - order=2, - type="page", - label="Page 2", - asset_id="urn:dor:00000000-0000-0000-0000-000000001002", - ), - ], - ) - ], - ), - PackageResource( - id=uuid.UUID("00000000-0000-0000-0000-000000001001"), - type="Asset", - alternate_identifier=AlternateIdentifier( - id="xyzzy:00000001:00000001", type="DLXS" - ), - events=[ - PreservationEvent( - identifier="81388465-aefd-4a3d-ba99-a868d062b92e", - type="generate access derivative", - datetime=datetime(2005, 8, 22, 22, 54, 45, tzinfo=UTC), - detail="Method south agree until.", - agent=Agent( - address="rguzman@example.net", role="image processing" - ), - ), - PreservationEvent( - identifier="d53540b9-cd23-4e92-9dff-4b28bf050b26", - type="extract text", - datetime=datetime(2006, 8, 23, 16, 21, 57, tzinfo=UTC), - detail="Hear thus part probably that.", - agent=Agent( - address="kurt16@example.org", role="ocr processing" - ), - ), - ], - metadata_files=[ - FileMetadata( - id="_0193972b-e4a4-7985-abe2-f3f1259b78ec", - use="TECHNICAL", - ref=FileReference( - locref="../metadata/00000001.source.jpg.mix.xml", - mdtype="NISOIMG", - ), - ), - FileMetadata( - id="_0193972b-e4ae-73eb-848d-5f8893b68253", - use="TECHNICAL", - ref=FileReference( - locref="../metadata/00000001.access.jpg.mix.xml", - mdtype="NISOIMG", - ), - ), - FileMetadata( - id="_0193972b-e572-7107-b69c-e2f4c660a9aa", - use="TECHNICAL", - ref=FileReference( - locref="../metadata/00000001.plaintext.txt.textmd.xml", - mdtype="TEXTMD", - ), - ), - ], - data_files=[ - FileMetadata( - id="_be653ff450ae7f3520312a53e56c00bc", - mdid="_0193972b-e4a4-7985-abe2-f3f1259b78ec", - use="SOURCE", - ref=FileReference( - locref="../data/00000001.source.jpg", - mimetype="image/jpeg", - ), - ), - FileMetadata( - id="_7e923d9c33b3859e1327fa53a8e609a1", - groupid="_be653ff450ae7f3520312a53e56c00bc", - mdid="_0193972b-e4ae-73eb-848d-5f8893b68253", - use="ACCESS", - ref=FileReference( - locref="../data/00000001.access.jpg", - mimetype="image/jpeg", - ), - ), - FileMetadata( - id="_764ba9761fbc6cbf0462d28d19356148", - groupid="_be653ff450ae7f3520312a53e56c00bc", - mdid="_0193972b-e572-7107-b69c-e2f4c660a9aa", - use="SOURCE", - ref=FileReference( - locref="../data/00000001.plaintext.txt", - mimetype="text/plain", - ), - ), - ], - ), - ] - # Test @@ -216,11 +51,17 @@ def step_impl(context) -> None: gateway = OcflRepositoryGateway(storage_path=storage) gateway.create_repository() - context.uow = UnitOfWork(gateway=gateway) - - context.translocator = Translocator( - inbox_path=inbox, workspaces_path=workspaces, minter=lambda: value + + engine = create_engine( + config.get_test_database_engine_url(), json_serializer=_custom_json_serializer ) + session_factory = sessionmaker(bind=engine) + Base.metadata.drop_all(engine) + Base.metadata.create_all(engine) + + context.uow = SqlalchemyUnitOfWork(gateway=gateway, session_factory=session_factory) + + context.translocator = Translocator(inbox_path = inbox, workspaces_path = workspaces, minter = lambda: value) def stored_callback(event: PackageStored, uow: UnitOfWork) -> None: context.stored_event = event diff --git a/features/steps/test_store_resource.py b/features/steps/test_store_resource.py.bak similarity index 100% rename from features/steps/test_store_resource.py rename to features/steps/test_store_resource.py.bak From a98bc116dad4d1f22358aac9a61fe6125a951169 Mon Sep 17 00:00:00 2001 From: Greg Kostin Date: Thu, 16 Jan 2025 11:39:11 -0500 Subject: [PATCH 04/23] green features --- dor/domain/events.py | 7 +++++++ dor/service_layer/handlers/catalog_bin.py | 19 +++++++++++++++++++ dor/service_layer/handlers/store_files.py | 8 +++++--- features/steps/store_resource.py | 17 +++++++++++++---- 4 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 dor/service_layer/handlers/catalog_bin.py diff --git a/dor/domain/events.py b/dor/domain/events.py index 8730047..23b763d 100644 --- a/dor/domain/events.py +++ b/dor/domain/events.py @@ -49,3 +49,10 @@ class PackageUnpacked(Event): class PackageStored(Event): identifier: str tracking_identifier: str + resources: list[Any] + +@dataclass +class BinCataloged(Event): + identifier: str + tracking_identifier: str + diff --git a/dor/service_layer/handlers/catalog_bin.py b/dor/service_layer/handlers/catalog_bin.py new file mode 100644 index 0000000..61270a3 --- /dev/null +++ b/dor/service_layer/handlers/catalog_bin.py @@ -0,0 +1,19 @@ +from dor.domain.events import PackageStored, BinCataloged +from dor.service_layer.unit_of_work import AbstractUnitOfWork +from dor.domain.models import Bin + +def catalog_bin(event: PackageStored, uow: AbstractUnitOfWork) -> None: + root_resource = [resource for resource in event.resources if resource.type == 'Monograph'][0] + + bin = Bin( + identifier=event.identifier, + alternate_identifiers=[root_resource.alternate_identifier.id], + common_metadata={}, + package_resources=event.resources + ) + with uow: + uow.catalog.add(bin) + uow.commit() + + uow.add_event(BinCataloged(identifier=event.identifier, tracking_identifier=event.tracking_identifier)) + \ No newline at end of file diff --git a/dor/service_layer/handlers/store_files.py b/dor/service_layer/handlers/store_files.py index 0184921..4619399 100644 --- a/dor/service_layer/handlers/store_files.py +++ b/dor/service_layer/handlers/store_files.py @@ -1,9 +1,9 @@ from pathlib import Path from dor.domain.events import PackageStored, PackageUnpacked -from dor.service_layer.unit_of_work import UnitOfWork +from dor.service_layer.unit_of_work import AbstractUnitOfWork -def store_files(event: PackageUnpacked, uow: UnitOfWork, workspace_class: type) -> None: +def store_files(event: PackageUnpacked, uow: AbstractUnitOfWork, workspace_class: type) -> None: workspace = workspace_class(event.workspace_identifier, event.identifier) entries: list[Path] = [] @@ -24,6 +24,8 @@ def store_files(event: PackageUnpacked, uow: UnitOfWork, workspace_class: type) ) stored_event = PackageStored( - identifier=event.identifier, tracking_identifier=event.tracking_identifier + identifier=event.identifier, + tracking_identifier=event.tracking_identifier, + resources=event.resources ) uow.add_event(stored_event) diff --git a/features/steps/store_resource.py b/features/steps/store_resource.py index dd30980..4c5598e 100644 --- a/features/steps/store_resource.py +++ b/features/steps/store_resource.py @@ -22,6 +22,7 @@ PackageSubmitted, PackageVerified, PackageUnpacked, + BinCataloged, ) from dor.providers.file_system_file_provider import FilesystemFileProvider from dor.providers.translocator import Translocator, Workspace @@ -33,6 +34,7 @@ from dor.service_layer.handlers.receive_package import receive_package from dor.service_layer.handlers.verify_package import verify_package from dor.service_layer.handlers.unpack_package import unpack_package +from dor.service_layer.handlers.catalog_bin import catalog_bin # Test @@ -63,8 +65,8 @@ def step_impl(context) -> None: context.translocator = Translocator(inbox_path = inbox, workspaces_path = workspaces, minter = lambda: value) - def stored_callback(event: PackageStored, uow: UnitOfWork) -> None: - context.stored_event = event + def cataloged_callback(event: BinCataloged, uow: UnitOfWork) -> None: + context.cataloged_event = event handlers: dict[Type[Event], list[Callable]] = { PackageSubmitted: [ @@ -84,7 +86,8 @@ def stored_callback(event: PackageStored, uow: UnitOfWork) -> None: ) ], PackageUnpacked: [lambda event: store_files(event, context.uow, Workspace)], - PackageStored: [lambda event: stored_callback(event, context.uow)], + PackageStored: [lambda event: catalog_bin(event, context.uow)], + BinCataloged: [lambda event: cataloged_callback(event, context.uow)] } context.message_bus = MemoryMessageBus(handlers) @@ -99,6 +102,12 @@ def step_impl(context): @then("the Collection Manager can see that it was preserved.") def step_impl(context): - event = context.stored_event + event = context.cataloged_event assert event.identifier == "00000000-0000-0000-0000-000000000001" assert context.uow.gateway.has_object(event.identifier) + + with context.uow: + bin = context.uow.catalog.get(event.identifier) + assert bin is not None + + From 0b182e42f8f3bb6fc73073aa0019cd0741828148 Mon Sep 17 00:00:00 2001 From: Greg Kostin Date: Thu, 16 Jan 2025 11:46:50 -0500 Subject: [PATCH 05/23] catalog_bin comments for common metadata --- dor/service_layer/handlers/catalog_bin.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dor/service_layer/handlers/catalog_bin.py b/dor/service_layer/handlers/catalog_bin.py index 61270a3..15d0fb4 100644 --- a/dor/service_layer/handlers/catalog_bin.py +++ b/dor/service_layer/handlers/catalog_bin.py @@ -5,6 +5,21 @@ def catalog_bin(event: PackageStored, uow: AbstractUnitOfWork) -> None: root_resource = [resource for resource in event.resources if resource.type == 'Monograph'][0] + # Need to ask gatway for path to file + # Gatway has get object file then search for logical path then get the literal path + # Then we can add the literal path to the resource + # root_resource.path = gatway.get_path(root_resource.identifier) + # Going to have to parse the file name from the path and deserialize + + # I want common metadata + # We should already have it + # Goes back to the root resource problem + # We can refactor next MVP, still working with a baby sample. + # There is a lot of work to do here + + # Roger is really good at this stuff + + bin = Bin( identifier=event.identifier, alternate_identifiers=[root_resource.alternate_identifier.id], From d2474008c8b065409bc7a07a1f8bbb1475a96eeb Mon Sep 17 00:00:00 2001 From: Samuel Sciolla Date: Thu, 16 Jan 2025 15:19:01 -0500 Subject: [PATCH 06/23] Get common metadata from OCFL storage; introduce ABC for Catalog; improve typing in handlers, unit_of_work, events --- dor/adapters/catalog.py | 21 ++++++++++-- dor/domain/events.py | 6 ++-- dor/service_layer/handlers/catalog_bin.py | 33 +++++++++---------- dor/service_layer/handlers/receive_package.py | 4 +-- dor/service_layer/handlers/unpack_package.py | 4 +-- dor/service_layer/handlers/verify_package.py | 4 +-- dor/service_layer/unit_of_work.py | 4 ++- 7 files changed, 47 insertions(+), 29 deletions(-) diff --git a/dor/adapters/catalog.py b/dor/adapters/catalog.py index a509b91..b779939 100644 --- a/dor/adapters/catalog.py +++ b/dor/adapters/catalog.py @@ -1,4 +1,5 @@ # from dor.domain.models import Bin +from abc import ABC, abstractmethod from dor.domain import models import uuid @@ -46,7 +47,23 @@ class Bin(Base): # DateTime(timezone=True), server_default=func.now() # ) -class MemoryCatalog: + +class Catalog(ABC): + + @abstractmethod + def add(self, bin: models.Bin): + raise NotImplementedError + + @abstractmethod + def get(self, identifier: str): + raise NotImplementedError + + @abstractmethod + def get_by_alternate_identifier(self, identifier: str): + raise NotImplementedError + + +class MemoryCatalog(Catalog): def __init__(self): self.bins = [] @@ -66,7 +83,7 @@ def get_by_alternate_identifier(self, identifier): return None -class SqlalchemyCatalog: +class SqlalchemyCatalog(Catalog): def __init__(self, session): self.session = session diff --git a/dor/domain/events.py b/dor/domain/events.py index 23b763d..7977afb 100644 --- a/dor/domain/events.py +++ b/dor/domain/events.py @@ -2,6 +2,7 @@ from typing import Any from dor.domain.models import VersionInfo +from dor.providers.models import PackageResource @dataclass @@ -38,7 +39,7 @@ class PackageNotVerified(Event): @dataclass class PackageUnpacked(Event): identifier: str - resources: list[Any] + resources: list[PackageResource] tracking_identifier: str version_info: VersionInfo workspace_identifier: str @@ -49,10 +50,9 @@ class PackageUnpacked(Event): class PackageStored(Event): identifier: str tracking_identifier: str - resources: list[Any] + resources: list[PackageResource] @dataclass class BinCataloged(Event): identifier: str tracking_identifier: str - diff --git a/dor/service_layer/handlers/catalog_bin.py b/dor/service_layer/handlers/catalog_bin.py index 15d0fb4..7a35a4c 100644 --- a/dor/service_layer/handlers/catalog_bin.py +++ b/dor/service_layer/handlers/catalog_bin.py @@ -1,29 +1,28 @@ +import json +from pathlib import Path + from dor.domain.events import PackageStored, BinCataloged -from dor.service_layer.unit_of_work import AbstractUnitOfWork from dor.domain.models import Bin +from dor.service_layer.unit_of_work import AbstractUnitOfWork def catalog_bin(event: PackageStored, uow: AbstractUnitOfWork) -> None: root_resource = [resource for resource in event.resources if resource.type == 'Monograph'][0] - - # Need to ask gatway for path to file - # Gatway has get object file then search for logical path then get the literal path - # Then we can add the literal path to the resource - # root_resource.path = gatway.get_path(root_resource.identifier) - # Going to have to parse the file name from the path and deserialize - - # I want common metadata - # We should already have it - # Goes back to the root resource problem - # We can refactor next MVP, still working with a baby sample. - # There is a lot of work to do here - - # Roger is really good at this stuff - + common_metadata_file = [ + metadata_file for metadata_file in root_resource.metadata_files if "common" in metadata_file.ref.locref + ][0] + # Path is still relative, but won't be once after Jaya's change + common_metadata_file_path = Path(common_metadata_file.ref.locref.replace("../", "")) + object_files = uow.gateway.get_object_files(event.identifier) + matching_object_file = [ + object_file for object_file in object_files if common_metadata_file_path == object_file.logical_path + ][0] + literal_common_metadata_path = matching_object_file.literal_path + common_metadata = json.loads(literal_common_metadata_path.read_text()) bin = Bin( identifier=event.identifier, alternate_identifiers=[root_resource.alternate_identifier.id], - common_metadata={}, + common_metadata=common_metadata, package_resources=event.resources ) with uow: diff --git a/dor/service_layer/handlers/receive_package.py b/dor/service_layer/handlers/receive_package.py index 370bb7d..0ed9316 100644 --- a/dor/service_layer/handlers/receive_package.py +++ b/dor/service_layer/handlers/receive_package.py @@ -1,9 +1,9 @@ from typing import Any from dor.domain.events import PackageSubmitted, PackageReceived -from dor.service_layer.unit_of_work import UnitOfWork +from dor.service_layer.unit_of_work import AbstractUnitOfWork -def receive_package(event: PackageSubmitted, uow: UnitOfWork, translocator: Any) -> None: +def receive_package(event: PackageSubmitted, uow: AbstractUnitOfWork, translocator: Any) -> None: workspace = translocator.create_workspace_for_package(event.package_identifier) received_event = PackageReceived( diff --git a/dor/service_layer/handlers/unpack_package.py b/dor/service_layer/handlers/unpack_package.py index 5e9b724..edcebd0 100644 --- a/dor/service_layer/handlers/unpack_package.py +++ b/dor/service_layer/handlers/unpack_package.py @@ -1,13 +1,13 @@ from dor.domain.events import PackageUnpacked, PackageVerified from dor.domain.models import VersionInfo from dor.providers.file_provider import FileProvider -from dor.service_layer.unit_of_work import UnitOfWork +from dor.service_layer.unit_of_work import AbstractUnitOfWork from gateway.coordinator import Coordinator def unpack_package( event: PackageVerified, - uow: UnitOfWork, + uow: AbstractUnitOfWork, bag_adapter_class: type, package_resource_provider_class: type, workspace_class: type, diff --git a/dor/service_layer/handlers/verify_package.py b/dor/service_layer/handlers/verify_package.py index 52502cd..4454f25 100644 --- a/dor/service_layer/handlers/verify_package.py +++ b/dor/service_layer/handlers/verify_package.py @@ -1,10 +1,10 @@ from dor.adapters.bag_adapter import ValidationError from dor.domain.events import PackageNotVerified, PackageReceived, PackageVerified -from dor.service_layer.unit_of_work import UnitOfWork +from dor.service_layer.unit_of_work import AbstractUnitOfWork def verify_package( - event: PackageReceived, uow: UnitOfWork, bag_adapter_class: type, workspace_class: type + event: PackageReceived, uow: AbstractUnitOfWork, bag_adapter_class: type, workspace_class: type ) -> None: workspace = workspace_class(event.workspace_identifier) diff --git a/dor/service_layer/unit_of_work.py b/dor/service_layer/unit_of_work.py index e35fc68..5cc2314 100644 --- a/dor/service_layer/unit_of_work.py +++ b/dor/service_layer/unit_of_work.py @@ -3,13 +3,15 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from dor.adapters.catalog import MemoryCatalog, SqlalchemyCatalog, _custom_json_serializer +from dor.adapters.catalog import Catalog, MemoryCatalog, SqlalchemyCatalog, _custom_json_serializer from dor.config import config from dor.domain.events import Event from gateway.repository_gateway import RepositoryGateway class AbstractUnitOfWork(ABC): + catalog: Catalog + gateway: RepositoryGateway @abstractmethod def __enter__(self): From 170900d70eacea64f1f37f23dae1afd00b1132e3 Mon Sep 17 00:00:00 2001 From: Samuel Sciolla Date: Thu, 16 Jan 2025 17:03:41 -0500 Subject: [PATCH 07/23] Add rough draft of repo CLI with initialize, store; create bootstrap function; add paths to config; modify uow type in MessageBus --- dor/cli/main.py | 2 + dor/cli/repo.py | 81 +++++++++++++++++++ dor/config.py | 7 ++ .../message_bus/memory_message_bus.py | 6 +- env.example | 4 + 5 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 dor/cli/repo.py diff --git a/dor/cli/main.py b/dor/cli/main.py index 61714d9..4057383 100644 --- a/dor/cli/main.py +++ b/dor/cli/main.py @@ -1,8 +1,10 @@ import typer import dor.cli.samples as samples +import dor.cli.repo as repo app = typer.Typer() app.add_typer(samples.app, name="samples") +app.add_typer(repo.app, name="repo") if __name__ == "__main__": # pragma: no cover diff --git a/dor/cli/repo.py b/dor/cli/repo.py new file mode 100644 index 0000000..b421c79 --- /dev/null +++ b/dor/cli/repo.py @@ -0,0 +1,81 @@ +import uuid +from typing import Callable, Type, Tuple + +import typer +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from dor.adapters.bag_adapter import BagAdapter +from dor.adapters.catalog import Base, _custom_json_serializer +from dor.config import config +from dor.domain.events import ( + Event, + BinCataloged, + PackageReceived, + PackageStored, + PackageSubmitted, + PackageUnpacked, + PackageVerified, +) +from dor.providers.package_resource_provider import PackageResourceProvider +from dor.providers.translocator import Translocator, Workspace +from dor.service_layer.handlers.catalog_bin import catalog_bin +from dor.service_layer.handlers.receive_package import receive_package +from dor.service_layer.handlers.store_files import store_files +from dor.service_layer.handlers.unpack_package import unpack_package +from dor.service_layer.handlers.verify_package import verify_package +from dor.service_layer.message_bus.memory_message_bus import MemoryMessageBus +from dor.service_layer.unit_of_work import SqlalchemyUnitOfWork +from gateway.ocfl_repository_gateway import OcflRepositoryGateway + + +app = typer.Typer() + + +def bootstrap() -> Tuple[MemoryMessageBus, SqlalchemyUnitOfWork]: + gateway = OcflRepositoryGateway(storage_path=config.storage_path) + + engine = create_engine( + config.get_database_engine_url(), json_serializer=_custom_json_serializer + ) + session_factory = sessionmaker(bind=engine) + uow = SqlalchemyUnitOfWork(gateway=gateway, session_factory=session_factory) + + translocator = Translocator( + inbox_path=config.inbox_path, + workspaces_path=config.workspaces_path, + minter = lambda: str(uuid.uuid4()) + ) + + handlers: dict[Type[Event], list[Callable]] = { + PackageSubmitted: [lambda event: receive_package(event, uow, translocator)], + PackageReceived: [lambda event: verify_package(event, uow, BagAdapter, Workspace)], + PackageVerified: [lambda event: unpack_package( + event, uow, BagAdapter, PackageResourceProvider, Workspace + )], + PackageUnpacked: [lambda event: store_files(event, uow, Workspace)], + PackageStored: [lambda event: catalog_bin(event, uow)], + BinCataloged: [] + } + message_bus = MemoryMessageBus(handlers) + return (message_bus, uow) + + +@app.command() +def initialize(): + gateway = OcflRepositoryGateway(storage_path=config.storage_path) + gateway.create_repository() + + engine = create_engine( + config.get_database_engine_url(), json_serializer=_custom_json_serializer + ) + Base.metadata.create_all(engine) + + +@app.command() +def store( + package_identifier: str = typer.Option(help="Name of the package directory"), +): + message_bus, uow = bootstrap() + event = PackageSubmitted(package_identifier=package_identifier) + message_bus.handle(event, uow) diff --git a/dor/config.py b/dor/config.py index 3add4cd..18972cd 100644 --- a/dor/config.py +++ b/dor/config.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import sqlalchemy from pydantic.dataclasses import dataclass @@ -15,11 +16,17 @@ class DatabaseConfig: @dataclass class Config: + storage_path: Path + inbox_path: Path + workspaces_path: Path database: DatabaseConfig @classmethod def from_env(cls): return cls( + storage_path=Path(os.getenv("STORAGE_PATH", "")), + inbox_path=Path(os.getenv("INBOX_PATH", "")), + workspaces_path=Path(os.getenv("WORKSPACES_PATH", "")), database=DatabaseConfig( user=os.getenv("POSTGRES_USER", "postgres"), password=os.getenv("POSTGRES_PASSWORD", "postgres"), diff --git a/dor/service_layer/message_bus/memory_message_bus.py b/dor/service_layer/message_bus/memory_message_bus.py index 1aa7b82..3bd35f5 100644 --- a/dor/service_layer/message_bus/memory_message_bus.py +++ b/dor/service_layer/message_bus/memory_message_bus.py @@ -1,6 +1,6 @@ from typing import Callable, Type from dor.domain.events import Event -from dor.service_layer.unit_of_work import UnitOfWork +from dor.service_layer.unit_of_work import AbstractUnitOfWork class MemoryMessageBus: @@ -13,14 +13,14 @@ def register_event_handler(self, event_type: Type[Event], handler: Callable): self.event_handlers[event_type] = [] self.event_handlers[event_type].append(handler) - def handle(self, message, uow: UnitOfWork): + def handle(self, message, uow: AbstractUnitOfWork): # Handles a message, which must be an event. if isinstance(message, Event): self._handle_event(message, uow) else: raise ValueError(f"Message of type {type(message)} is not a valid Event") - def _handle_event(self, event: Event, uow: UnitOfWork): + def _handle_event(self, event: Event, uow: AbstractUnitOfWork): # Handles an event by executing its registered handlers. if event.__class__ not in self.event_handlers: raise NoHandlerForEventError(f"No handler found for event type {type(event)}") diff --git a/env.example b/env.example index 5b3b9aa..dc4db15 100644 --- a/env.example +++ b/env.example @@ -9,3 +9,7 @@ POSTGRES_USER=postgres POSTGRES_PASSWORD=postgres POSTGRES_DATABASE=dor_local POSTGRES_HOST=db + +STORAGE_PATH= +INBOX_PATH= +WORKSPACES_PATH= From fbb4fb94853eb95f1eb5c99d565255d311403da7 Mon Sep 17 00:00:00 2001 From: Greg Kostin Date: Thu, 16 Jan 2025 12:10:44 -0500 Subject: [PATCH 08/23] refactor scratch directory --- .gitignore | 5 +---- features/scratch/workspaces/.keep | 0 features/steps/store_resource.py | 14 +++++++++----- 3 files changed, 10 insertions(+), 9 deletions(-) delete mode 100644 features/scratch/workspaces/.keep diff --git a/.gitignore b/.gitignore index 629147c..a0ba3e8 100644 --- a/.gitignore +++ b/.gitignore @@ -19,8 +19,5 @@ output/ tests/test_storage tests/test_workspaces +features/scratch/* -features/scratch/storage/* -!features/scratch/storage/.keep -features/scratch/workspaces/* -!features/scratch/workspaces/.keep diff --git a/features/scratch/workspaces/.keep b/features/scratch/workspaces/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/features/steps/store_resource.py b/features/steps/store_resource.py index 4c5598e..408f1cb 100644 --- a/features/steps/store_resource.py +++ b/features/steps/store_resource.py @@ -42,16 +42,20 @@ @given("a package containing the scanned pages, OCR, and metadata") def step_impl(context) -> None: inbox = Path("./features/fixtures/inbox") - storage = Path("./features/scratch/storage") - workspaces = Path("./features/scratch/workspaces") value = "55ce2f63-c11a-4fac-b3a9-160305b1a0c4" - shutil.rmtree(path=f"./features/scratch/workspaces/{value}", ignore_errors=True) - shutil.rmtree(path=storage, ignore_errors=True) + scratch = Path("./features/scratch") + shutil.rmtree(path = scratch, ignore_errors = True) + os.mkdir(scratch) + + storage = Path("./features/scratch/storage") os.mkdir(storage) - gateway = OcflRepositoryGateway(storage_path=storage) + workspaces = Path("./features/scratch/workspaces") + os.mkdir(workspaces) + + gateway = OcflRepositoryGateway(storage_path = storage) gateway.create_repository() engine = create_engine( From 4e55dd3a51b251c7786c131332c4fa86e71a5f27 Mon Sep 17 00:00:00 2001 From: Greg Kostin Date: Fri, 17 Jan 2025 12:15:19 -0500 Subject: [PATCH 09/23] one step back --- features/steps/test_store_resource.py.bak | 102 ---------------------- 1 file changed, 102 deletions(-) delete mode 100644 features/steps/test_store_resource.py.bak diff --git a/features/steps/test_store_resource.py.bak b/features/steps/test_store_resource.py.bak deleted file mode 100644 index cce25d4..0000000 --- a/features/steps/test_store_resource.py.bak +++ /dev/null @@ -1,102 +0,0 @@ -import os -import shutil -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from typing import Callable, Type - -from functools import partial -from dor.providers.file_system_file_provider import FilesystemFileProvider -from pytest_bdd import scenario, given, when, then - -from dor.adapters.bag_adapter import BagAdapter -from dor.domain.events import ( - Event, - PackageReceived, - PackageStored, - PackageSubmitted, - PackageVerified, - PackageUnpacked -) -from dor.providers.translocator import Translocator, Workspace -from dor.providers.package_resource_provider import PackageResourceProvider -from dor.service_layer.handlers.store_files import store_files -from dor.service_layer.message_bus.memory_message_bus import MemoryMessageBus -from dor.service_layer.unit_of_work import UnitOfWork -from gateway.ocfl_repository_gateway import OcflRepositoryGateway -from dor.service_layer.handlers.receive_package import receive_package -from dor.service_layer.handlers.verify_package import verify_package -from dor.service_layer.handlers.unpack_package import unpack_package - -@dataclass -class Context: - uow: UnitOfWork = None - translocator: Translocator = None - message_bus: MemoryMessageBus = None - stored_event: PackageStored = None - - -scenario = partial(scenario, '../store_resource.feature') - -@scenario('Storing a new resource for immediate release') -def test_store_resource(): - pass - -@given(u'a package containing the scanned pages, OCR, and metadata', target_fixture="context") -def _(): - context = Context(uow=None, translocator=None, message_bus=None) - - inbox = Path("./features/fixtures/inbox") - storage = Path("./features/scratch/storage") - workspaces = Path("./features/scratch/workspaces") - - value = '55ce2f63-c11a-4fac-b3a9-160305b1a0c4' - - shutil.rmtree(path = f"./features/scratch/workspaces/{value}", ignore_errors = True) - shutil.rmtree(path = storage, ignore_errors = True) - os.mkdir(storage) - - gateway = OcflRepositoryGateway(storage_path = storage) - gateway.create_repository() - context.uow = UnitOfWork(gateway=gateway) - - context.translocator = Translocator(inbox_path = inbox, workspaces_path = workspaces, minter = lambda: value) - - def stored_callback(event: PackageStored, uow: UnitOfWork) -> None: - context.stored_event = event - - handlers: dict[Type[Event], list[Callable]] = { - PackageSubmitted: [ - lambda event: receive_package(event, context.uow, context.translocator) - ], - PackageReceived: [ - lambda event: verify_package(event, context.uow, BagAdapter, Workspace) - ], - PackageVerified: [ - lambda event: unpack_package( - event, - context.uow, - BagAdapter, - PackageResourceProvider, - Workspace, - FilesystemFileProvider(), - ) - ], - PackageUnpacked: [lambda event: store_files(event, context.uow, Workspace)], - PackageStored: [lambda event: stored_callback(event, context.uow)], - } - context.message_bus = MemoryMessageBus(handlers) - return context - -@when(u'the Collection Manager places the packaged resource in the incoming location') -def _(context): - submission_id = "xyzzy-0001-v1" - - event = PackageSubmitted(package_identifier=submission_id) - context.message_bus.handle(event, context.uow) - -@then(u'the Collection Manager can see that it was preserved.') -def _(context): - event = context.stored_event - assert event.identifier == "00000000-0000-0000-0000-000000000001" - assert context.uow.gateway.has_object(event.identifier) From 73eb29dbb6bc515dd89c26ef1c756fb3da583d06 Mon Sep 17 00:00:00 2001 From: Greg Kostin Date: Fri, 17 Jan 2025 12:16:41 -0500 Subject: [PATCH 10/23] one step forward - rename steps --- features/steps/{inspect_bin.py => test_inspect_bin.py} | 0 features/steps/{store_resource.py => test_store_resource.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename features/steps/{inspect_bin.py => test_inspect_bin.py} (100%) rename features/steps/{store_resource.py => test_store_resource.py} (100%) diff --git a/features/steps/inspect_bin.py b/features/steps/test_inspect_bin.py similarity index 100% rename from features/steps/inspect_bin.py rename to features/steps/test_inspect_bin.py diff --git a/features/steps/store_resource.py b/features/steps/test_store_resource.py similarity index 100% rename from features/steps/store_resource.py rename to features/steps/test_store_resource.py From 879514d53f6e17b21de9e51f05f9fe735d647616 Mon Sep 17 00:00:00 2001 From: Greg Kostin Date: Wed, 22 Jan 2025 13:23:51 -0500 Subject: [PATCH 11/23] remove behave --- poetry.lock | 22 +--------------------- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/poetry.lock b/poetry.lock index b2795fe..62e0ecb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -42,26 +42,6 @@ files = [ {file = "bagit-1.9b2.tar.gz", hash = "sha256:4450cbe591fd3669471fdf5aab30186e47f71ae596b58bf3d6a416182ea5e1bf"}, ] -[[package]] -name = "behave" -version = "1.2.6" -description = "behave is behaviour-driven development, Python style" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "behave-1.2.6-py2.py3-none-any.whl", hash = "sha256:ebda1a6c9e5bfe95c5f9f0a2794e01c7098b3dde86c10a95d8621c5907ff6f1c"}, - {file = "behave-1.2.6.tar.gz", hash = "sha256:b9662327aa53294c1351b0a9c369093ccec1d21026f050c3bd9b3e5cccf81a86"}, -] - -[package.dependencies] -parse = ">=1.8.2" -parse-type = ">=0.4.2" -six = ">=1.11" - -[package.extras] -develop = ["coverage", "invoke (>=0.21.0)", "modernize (>=0.5)", "path.py (>=8.1.2)", "pathlib", "pycmd", "pylint", "pytest (>=3.0)", "pytest-cov", "tox"] -docs = ["sphinx (>=1.6)", "sphinx-bootstrap-theme (>=0.6)"] - [[package]] name = "certifi" version = "2024.12.14" @@ -1566,4 +1546,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "54c21a9779a8b8549941c4946450b9efc5a88c55e7d4ec15e0129e4640f2f73a" +content-hash = "f466b41b966b0b0ce6f52545f2545166ca0ca66bdef7af5a58a6eff70248923c" diff --git a/pyproject.toml b/pyproject.toml index 10f9966..e8aa7e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,6 @@ fastapi = {extras = ["standard"], version = "^0.115.6"} [tool.poetry.group.dev.dependencies] pytest = "^8.0.2" ruff = "^0.2.2" -behave = "^1.2.6" pytest-bdd = "^8.1.0" [build-system] From e05e1682a161e2e5d3c8bddcc67d8bcdb629469b Mon Sep 17 00:00:00 2001 From: Greg Kostin Date: Wed, 22 Jan 2025 14:31:42 -0500 Subject: [PATCH 12/23] refactor steps - no alt_id argument --- features/inspect_bin.feature | 4 +- features/steps/test_inspect_bin.py | 43 ++++++++++--- features/steps/test_store_resource.py | 87 +++++++++++---------------- 3 files changed, 72 insertions(+), 62 deletions(-) diff --git a/features/inspect_bin.feature b/features/inspect_bin.feature index 70531f1..881f102 100644 --- a/features/inspect_bin.feature +++ b/features/inspect_bin.feature @@ -11,12 +11,12 @@ Feature: Inspect Bin As a Collection Manager I want to review the contents of its bin. - Scenario: + Scenario: Revision summary Given a preserved monograph with an alternate identifier of "xyzzy:00000001" When the Collection Manager looks up the bin by "xyzzy:00000001" Then the Collection Manager sees the summary of the bin - Scenario: + Scenario: Revision file sets Given a preserved monograph with an alternate identifier of "xyzzy:00000001" When the Collection Manager lists the contents of the bin for "xyzzy:00000001" Then the Collection Manager sees the file sets. diff --git a/features/steps/test_inspect_bin.py b/features/steps/test_inspect_bin.py index 5e8586e..71d1fd7 100644 --- a/features/steps/test_inspect_bin.py +++ b/features/steps/test_inspect_bin.py @@ -1,25 +1,46 @@ -from behave import given, then, when import uuid +from dataclasses import dataclass from datetime import datetime, UTC from pydantic_core import to_jsonable_python from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from functools import partial +from pytest_bdd import scenario, given, when, then, parsers from dor.adapters.catalog import Base, _custom_json_serializer from dor.config import config from dor.domain.models import Bin from dor.service_layer import catalog_service -from dor.service_layer.unit_of_work import SqlalchemyUnitOfWork, UnitOfWork +from dor.service_layer.unit_of_work import SqlalchemyUnitOfWork from dor.providers.models import ( Agent, AlternateIdentifier, FileMetadata, FileReference, PackageResource, PreservationEvent, StructMap, StructMapItem, StructMapType ) from gateway.fake_repository_gateway import FakeRepositoryGateway +@dataclass +class Context: + uow: SqlalchemyUnitOfWork = None + bin: Bin = None + alt_id: str = None + summary: dict = None + file_sets: list = None + +scenario = partial(scenario, '../inspect_bin.feature') + +@scenario('Revision summary') +def test_revision_summary(): + pass + +@scenario('Revision file sets') +def test_revision_file_sets(): + pass + +@given(u'a preserved monograph with an alternate identifier of "xyzzy:00000001"', target_fixture="context") +def _(): + context = Context() -@given(u'a preserved monograph with an alternate identifier of "{alt_id}"') -def step_impl(context, alt_id): bin = Bin( identifier=uuid.UUID("00000000-0000-0000-0000-000000000001"), alternate_identifiers=["xyzzy:00000001"], @@ -192,9 +213,12 @@ def step_impl(context, alt_id): with context.uow: context.uow.catalog.add(bin) context.uow.commit() + + return context -@when(u'the Collection Manager looks up the bin by "{alt_id}"') -def step_impl(context, alt_id): +@when(u'the Collection Manager looks up the bin by "xyzzy:00000001"') +def step_impl(context): + alt_id = "xyzzy:00000001" context.alt_id = alt_id with context.uow: context.bin = context.uow.catalog.get_by_alternate_identifier(alt_id) @@ -218,14 +242,15 @@ def step_impl(context): ) assert context.summary == expected_summary -@when(u'the Collection Manager lists the contents of the bin for "{alt_id}"') -def step_impl(context, alt_id): +@when(u'the Collection Manager lists the contents of the bin for "xyzzy:00000001"') +def _(context): + alt_id = "xyzzy:00000001" with context.uow: context.bin = context.uow.catalog.get_by_alternate_identifier(alt_id) context.file_sets = catalog_service.get_file_sets(context.bin) @then(u'the Collection Manager sees the file sets.') -def step_impl(context): +def _(context): expected_file_sets = [ to_jsonable_python(resource) for resource in context.bin.package_resources if resource.type == 'Asset' diff --git a/features/steps/test_store_resource.py b/features/steps/test_store_resource.py index 408f1cb..634699d 100644 --- a/features/steps/test_store_resource.py +++ b/features/steps/test_store_resource.py @@ -1,18 +1,12 @@ import os import shutil -import uuid +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Callable, Type -from behave import given, when, then - -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - - -from dor.adapters.catalog import Base, _custom_json_serializer -from dor.config import config +from functools import partial +from pytest_bdd import scenario, given, when, then from dor.adapters.bag_adapter import BagAdapter from dor.domain.events import ( @@ -21,56 +15,55 @@ PackageStored, PackageSubmitted, PackageVerified, - PackageUnpacked, - BinCataloged, + PackageUnpacked ) from dor.providers.file_system_file_provider import FilesystemFileProvider from dor.providers.translocator import Translocator, Workspace from dor.providers.package_resource_provider import PackageResourceProvider from dor.service_layer.handlers.store_files import store_files from dor.service_layer.message_bus.memory_message_bus import MemoryMessageBus -from dor.service_layer.unit_of_work import UnitOfWork, SqlalchemyUnitOfWork +from dor.service_layer.unit_of_work import UnitOfWork from gateway.ocfl_repository_gateway import OcflRepositoryGateway from dor.service_layer.handlers.receive_package import receive_package from dor.service_layer.handlers.verify_package import verify_package from dor.service_layer.handlers.unpack_package import unpack_package -from dor.service_layer.handlers.catalog_bin import catalog_bin -# Test +@dataclass +class Context: + uow: UnitOfWork = None + translocator: Translocator = None + message_bus: MemoryMessageBus = None + stored_event: PackageStored = None -@given("a package containing the scanned pages, OCR, and metadata") -def step_impl(context) -> None: - inbox = Path("./features/fixtures/inbox") +scenario = partial(scenario, '../store_resource.feature') - value = "55ce2f63-c11a-4fac-b3a9-160305b1a0c4" +@scenario('Storing a new resource for immediate release') +def test_store_resource(): + pass - scratch = Path("./features/scratch") - shutil.rmtree(path = scratch, ignore_errors = True) - os.mkdir(scratch) +@given(u'a package containing the scanned pages, OCR, and metadata', target_fixture="context") +def _(): + context = Context() + inbox = Path("./features/fixtures/inbox") storage = Path("./features/scratch/storage") - os.mkdir(storage) - workspaces = Path("./features/scratch/workspaces") - os.mkdir(workspaces) + + value = "55ce2f63-c11a-4fac-b3a9-160305b1a0c4" + + shutil.rmtree(path = f"./features/scratch/workspaces/{value}", ignore_errors = True) + shutil.rmtree(path = storage, ignore_errors = True) + os.mkdir(storage) gateway = OcflRepositoryGateway(storage_path = storage) gateway.create_repository() - - engine = create_engine( - config.get_test_database_engine_url(), json_serializer=_custom_json_serializer - ) - session_factory = sessionmaker(bind=engine) - Base.metadata.drop_all(engine) - Base.metadata.create_all(engine) - - context.uow = SqlalchemyUnitOfWork(gateway=gateway, session_factory=session_factory) + context.uow = UnitOfWork(gateway=gateway) context.translocator = Translocator(inbox_path = inbox, workspaces_path = workspaces, minter = lambda: value) - def cataloged_callback(event: BinCataloged, uow: UnitOfWork) -> None: - context.cataloged_event = event + def stored_callback(event: PackageStored, uow: UnitOfWork) -> None: + context.stored_event = event handlers: dict[Type[Event], list[Callable]] = { PackageSubmitted: [ @@ -90,28 +83,20 @@ def cataloged_callback(event: BinCataloged, uow: UnitOfWork) -> None: ) ], PackageUnpacked: [lambda event: store_files(event, context.uow, Workspace)], - PackageStored: [lambda event: catalog_bin(event, context.uow)], - BinCataloged: [lambda event: cataloged_callback(event, context.uow)] + PackageStored: [lambda event: stored_callback(event, context.uow)] } context.message_bus = MemoryMessageBus(handlers) + return context - -@when("the Collection Manager places the packaged resource in the incoming location") -def step_impl(context): +@when(u'the Collection Manager places the packaged resource in the incoming location') +def _(context): submission_id = "xyzzy-0001-v1" event = PackageSubmitted(package_identifier=submission_id) context.message_bus.handle(event, context.uow) - -@then("the Collection Manager can see that it was preserved.") -def step_impl(context): - event = context.cataloged_event +@then(u'the Collection Manager can see that it was preserved.') +def _(context): + event = context.stored_event assert event.identifier == "00000000-0000-0000-0000-000000000001" - assert context.uow.gateway.has_object(event.identifier) - - with context.uow: - bin = context.uow.catalog.get(event.identifier) - assert bin is not None - - + assert context.uow.gateway.has_object(event.identifier) \ No newline at end of file From 5c723f7d31de4379a751cd6bd2afff631786b936 Mon Sep 17 00:00:00 2001 From: Greg Kostin Date: Wed, 22 Jan 2025 14:54:59 -0500 Subject: [PATCH 13/23] with alt_id argument and asserts --- features/steps/test_inspect_bin.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/features/steps/test_inspect_bin.py b/features/steps/test_inspect_bin.py index 71d1fd7..b3264bb 100644 --- a/features/steps/test_inspect_bin.py +++ b/features/steps/test_inspect_bin.py @@ -37,8 +37,10 @@ def test_revision_summary(): def test_revision_file_sets(): pass -@given(u'a preserved monograph with an alternate identifier of "xyzzy:00000001"', target_fixture="context") -def _(): +@given(parsers.parse(u'a preserved monograph with an alternate identifier of "{alt_id}"'), target_fixture="context") +def _(alt_id): + assert alt_id == "xyzzy:00000001" + context = Context() bin = Bin( @@ -216,9 +218,10 @@ def _(): return context -@when(u'the Collection Manager looks up the bin by "xyzzy:00000001"') -def step_impl(context): - alt_id = "xyzzy:00000001" +@when(parsers.parse(u'the Collection Manager looks up the bin by "{alt_id}"')) +def step_impl(context, alt_id): + assert alt_id == "xyzzy:00000001" + context.alt_id = alt_id with context.uow: context.bin = context.uow.catalog.get_by_alternate_identifier(alt_id) @@ -242,9 +245,10 @@ def step_impl(context): ) assert context.summary == expected_summary -@when(u'the Collection Manager lists the contents of the bin for "xyzzy:00000001"') -def _(context): - alt_id = "xyzzy:00000001" +@when(parsers.parse(u'the Collection Manager lists the contents of the bin for "{alt_id}"')) +def _(context, alt_id): + assert alt_id == "xyzzy:00000001" + with context.uow: context.bin = context.uow.catalog.get_by_alternate_identifier(alt_id) context.file_sets = catalog_service.get_file_sets(context.bin) From aee58a8893154e8c83fa9651a4aaa60efd2d5860 Mon Sep 17 00:00:00 2001 From: Greg Kostin Date: Wed, 22 Jan 2025 14:57:12 -0500 Subject: [PATCH 14/23] removal of alt_id asserts --- features/steps/test_inspect_bin.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/features/steps/test_inspect_bin.py b/features/steps/test_inspect_bin.py index b3264bb..a731e70 100644 --- a/features/steps/test_inspect_bin.py +++ b/features/steps/test_inspect_bin.py @@ -39,8 +39,6 @@ def test_revision_file_sets(): @given(parsers.parse(u'a preserved monograph with an alternate identifier of "{alt_id}"'), target_fixture="context") def _(alt_id): - assert alt_id == "xyzzy:00000001" - context = Context() bin = Bin( @@ -220,8 +218,6 @@ def _(alt_id): @when(parsers.parse(u'the Collection Manager looks up the bin by "{alt_id}"')) def step_impl(context, alt_id): - assert alt_id == "xyzzy:00000001" - context.alt_id = alt_id with context.uow: context.bin = context.uow.catalog.get_by_alternate_identifier(alt_id) @@ -247,8 +243,6 @@ def step_impl(context): @when(parsers.parse(u'the Collection Manager lists the contents of the bin for "{alt_id}"')) def _(context, alt_id): - assert alt_id == "xyzzy:00000001" - with context.uow: context.bin = context.uow.catalog.get_by_alternate_identifier(alt_id) context.file_sets = catalog_service.get_file_sets(context.bin) From a38252523e1d8056883080d0eca7731d7284c838 Mon Sep 17 00:00:00 2001 From: Greg Kostin Date: Wed, 22 Jan 2025 15:13:44 -0500 Subject: [PATCH 15/23] refactor scratch directory --- features/steps/test_store_resource.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/features/steps/test_store_resource.py b/features/steps/test_store_resource.py index 634699d..8cebe68 100644 --- a/features/steps/test_store_resource.py +++ b/features/steps/test_store_resource.py @@ -47,15 +47,19 @@ def _(): context = Context() inbox = Path("./features/fixtures/inbox") + + scratch = Path("./features/scratch") + shutil.rmtree(path = scratch, ignore_errors = True) + os.mkdir(scratch) + storage = Path("./features/scratch/storage") + os.mkdir(storage) + workspaces = Path("./features/scratch/workspaces") + os.mkdir(workspaces) value = "55ce2f63-c11a-4fac-b3a9-160305b1a0c4" - shutil.rmtree(path = f"./features/scratch/workspaces/{value}", ignore_errors = True) - shutil.rmtree(path = storage, ignore_errors = True) - os.mkdir(storage) - gateway = OcflRepositoryGateway(storage_path = storage) gateway.create_repository() context.uow = UnitOfWork(gateway=gateway) From 8751a69f64b5a0e54f4d2816124902c12f2e7a2b Mon Sep 17 00:00:00 2001 From: Samuel Sciolla Date: Thu, 23 Jan 2025 11:59:52 -0500 Subject: [PATCH 16/23] Update test_store_resource to use catalog_bin; extract unit_of_work as fixture --- features/steps/test_store_resource.py | 81 +++++++++++++++++---------- 1 file changed, 52 insertions(+), 29 deletions(-) diff --git a/features/steps/test_store_resource.py b/features/steps/test_store_resource.py index 8cebe68..d5b79d8 100644 --- a/features/steps/test_store_resource.py +++ b/features/steps/test_store_resource.py @@ -1,16 +1,21 @@ import os import shutil from dataclasses import dataclass -from datetime import UTC, datetime +from functools import partial from pathlib import Path from typing import Callable, Type -from functools import partial +import pytest from pytest_bdd import scenario, given, when, then +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker from dor.adapters.bag_adapter import BagAdapter +from dor.adapters.catalog import Base, _custom_json_serializer +from dor.config import config from dor.domain.events import ( Event, + BinCataloged, PackageReceived, PackageStored, PackageSubmitted, @@ -22,18 +27,34 @@ from dor.providers.package_resource_provider import PackageResourceProvider from dor.service_layer.handlers.store_files import store_files from dor.service_layer.message_bus.memory_message_bus import MemoryMessageBus -from dor.service_layer.unit_of_work import UnitOfWork +from dor.service_layer.unit_of_work import AbstractUnitOfWork, SqlalchemyUnitOfWork from gateway.ocfl_repository_gateway import OcflRepositoryGateway +from dor.service_layer.handlers.catalog_bin import catalog_bin from dor.service_layer.handlers.receive_package import receive_package from dor.service_layer.handlers.verify_package import verify_package from dor.service_layer.handlers.unpack_package import unpack_package @dataclass class Context: - uow: UnitOfWork = None - translocator: Translocator = None - message_bus: MemoryMessageBus = None - stored_event: PackageStored = None + message_bus: MemoryMessageBus | None = None + cataloged_event: BinCataloged | None = None + +@pytest.fixture +def storage_path() -> Path: + return Path("./features/scratch/storage") + +@pytest.fixture +def unit_of_work(storage_path: Path) -> AbstractUnitOfWork: + engine = create_engine( + config.get_test_database_engine_url(), json_serializer=_custom_json_serializer + ) + session_factory = sessionmaker(bind=engine) + Base.metadata.drop_all(engine) + Base.metadata.create_all(engine) + + gateway = OcflRepositoryGateway(storage_path=storage_path) + + return SqlalchemyUnitOfWork(gateway=gateway, session_factory=session_factory) scenario = partial(scenario, '../store_resource.feature') @@ -43,64 +64,66 @@ def test_store_resource(): pass @given(u'a package containing the scanned pages, OCR, and metadata', target_fixture="context") -def _(): +def _(storage_path: Path, unit_of_work: AbstractUnitOfWork): context = Context() - inbox = Path("./features/fixtures/inbox") - scratch = Path("./features/scratch") shutil.rmtree(path = scratch, ignore_errors = True) os.mkdir(scratch) - storage = Path("./features/scratch/storage") - os.mkdir(storage) + os.mkdir(storage_path) + + unit_of_work.gateway.create_repository() + + inbox = Path("./features/fixtures/inbox") workspaces = Path("./features/scratch/workspaces") os.mkdir(workspaces) value = "55ce2f63-c11a-4fac-b3a9-160305b1a0c4" - gateway = OcflRepositoryGateway(storage_path = storage) - gateway.create_repository() - context.uow = UnitOfWork(gateway=gateway) - - context.translocator = Translocator(inbox_path = inbox, workspaces_path = workspaces, minter = lambda: value) + translocator = Translocator(inbox_path = inbox, workspaces_path = workspaces, minter = lambda: value) - def stored_callback(event: PackageStored, uow: UnitOfWork) -> None: - context.stored_event = event + def cataloged_callback(event: BinCataloged, uow: AbstractUnitOfWork) -> None: + context.cataloged_event = event handlers: dict[Type[Event], list[Callable]] = { PackageSubmitted: [ - lambda event: receive_package(event, context.uow, context.translocator) + lambda event: receive_package(event, unit_of_work, translocator) ], PackageReceived: [ - lambda event: verify_package(event, context.uow, BagAdapter, Workspace) + lambda event: verify_package(event, unit_of_work, BagAdapter, Workspace) ], PackageVerified: [ lambda event: unpack_package( event, - context.uow, + unit_of_work, BagAdapter, PackageResourceProvider, Workspace, FilesystemFileProvider(), ) ], - PackageUnpacked: [lambda event: store_files(event, context.uow, Workspace)], - PackageStored: [lambda event: stored_callback(event, context.uow)] + PackageUnpacked: [lambda event: store_files(event, unit_of_work, Workspace)], + PackageStored: [lambda event: catalog_bin(event, unit_of_work)], + BinCataloged: [lambda event: cataloged_callback(event, unit_of_work)] } context.message_bus = MemoryMessageBus(handlers) return context @when(u'the Collection Manager places the packaged resource in the incoming location') -def _(context): +def _(context, unit_of_work: AbstractUnitOfWork): submission_id = "xyzzy-0001-v1" event = PackageSubmitted(package_identifier=submission_id) - context.message_bus.handle(event, context.uow) + context.message_bus.handle(event, unit_of_work) @then(u'the Collection Manager can see that it was preserved.') -def _(context): - event = context.stored_event +def _(context, unit_of_work: AbstractUnitOfWork): + event = context.cataloged_event assert event.identifier == "00000000-0000-0000-0000-000000000001" - assert context.uow.gateway.has_object(event.identifier) \ No newline at end of file + assert unit_of_work.gateway.has_object(event.identifier) + + with unit_of_work: + bin = unit_of_work.catalog.get(event.identifier) + assert bin is not None From aeb5e0b652e569df42cc39e19f45e0fdb2395cb9 Mon Sep 17 00:00:00 2001 From: Samuel Sciolla Date: Thu, 23 Jan 2025 13:32:18 -0500 Subject: [PATCH 17/23] Remove cataloged_callback/cataloged_event; extract path_data, message_bus fixtures --- features/steps/test_store_resource.py | 95 ++++++++++++++------------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/features/steps/test_store_resource.py b/features/steps/test_store_resource.py index d5b79d8..bebbad6 100644 --- a/features/steps/test_store_resource.py +++ b/features/steps/test_store_resource.py @@ -35,16 +35,25 @@ from dor.service_layer.handlers.unpack_package import unpack_package @dataclass -class Context: - message_bus: MemoryMessageBus | None = None - cataloged_event: BinCataloged | None = None +class PathData: + scratch: Path + storage: Path + workspaces: Path + inbox: Path @pytest.fixture -def storage_path() -> Path: - return Path("./features/scratch/storage") +def path_data() -> PathData: + scratch = Path("./features/scratch") + + return PathData( + scratch=scratch, + inbox=Path("./features/fixtures/inbox"), + workspaces=scratch / "workspaces", + storage=scratch / "storage" + ) @pytest.fixture -def unit_of_work(storage_path: Path) -> AbstractUnitOfWork: +def unit_of_work(path_data: PathData) -> AbstractUnitOfWork: engine = create_engine( config.get_test_database_engine_url(), json_serializer=_custom_json_serializer ) @@ -52,40 +61,18 @@ def unit_of_work(storage_path: Path) -> AbstractUnitOfWork: Base.metadata.drop_all(engine) Base.metadata.create_all(engine) - gateway = OcflRepositoryGateway(storage_path=storage_path) + gateway = OcflRepositoryGateway(storage_path=path_data.storage) return SqlalchemyUnitOfWork(gateway=gateway, session_factory=session_factory) - -scenario = partial(scenario, '../store_resource.feature') - -@scenario('Storing a new resource for immediate release') -def test_store_resource(): - pass - -@given(u'a package containing the scanned pages, OCR, and metadata', target_fixture="context") -def _(storage_path: Path, unit_of_work: AbstractUnitOfWork): - context = Context() - - scratch = Path("./features/scratch") - shutil.rmtree(path = scratch, ignore_errors = True) - os.mkdir(scratch) - - os.mkdir(storage_path) - - unit_of_work.gateway.create_repository() - - inbox = Path("./features/fixtures/inbox") - - workspaces = Path("./features/scratch/workspaces") - os.mkdir(workspaces) - - value = "55ce2f63-c11a-4fac-b3a9-160305b1a0c4" - - translocator = Translocator(inbox_path = inbox, workspaces_path = workspaces, minter = lambda: value) - - def cataloged_callback(event: BinCataloged, uow: AbstractUnitOfWork) -> None: - context.cataloged_event = event +@pytest.fixture +def message_bus(path_data: PathData, unit_of_work: AbstractUnitOfWork) -> MemoryMessageBus: + value = '55ce2f63-c11a-4fac-b3a9-160305b1a0c4' + translocator = Translocator( + inbox_path=path_data.inbox, + workspaces_path=path_data.workspaces, + minter = lambda: value + ) handlers: dict[Type[Event], list[Callable]] = { PackageSubmitted: [ @@ -106,24 +93,38 @@ def cataloged_callback(event: BinCataloged, uow: AbstractUnitOfWork) -> None: ], PackageUnpacked: [lambda event: store_files(event, unit_of_work, Workspace)], PackageStored: [lambda event: catalog_bin(event, unit_of_work)], - BinCataloged: [lambda event: cataloged_callback(event, unit_of_work)] + BinCataloged: [] } - context.message_bus = MemoryMessageBus(handlers) - return context + message_bus = MemoryMessageBus(handlers) + return message_bus + +scenario = partial(scenario, '../store_resource.feature') + +@scenario('Storing a new resource for immediate release') +def test_store_resource(): + pass + +@given(u'a package containing the scanned pages, OCR, and metadata') +def _(path_data: PathData, unit_of_work: AbstractUnitOfWork): + shutil.rmtree(path=path_data.scratch, ignore_errors = True) + os.mkdir(path_data.scratch) + os.mkdir(path_data.storage) + os.mkdir(path_data.workspaces) + + unit_of_work.gateway.create_repository() @when(u'the Collection Manager places the packaged resource in the incoming location') -def _(context, unit_of_work: AbstractUnitOfWork): +def _(message_bus: MemoryMessageBus, unit_of_work: AbstractUnitOfWork): submission_id = "xyzzy-0001-v1" event = PackageSubmitted(package_identifier=submission_id) - context.message_bus.handle(event, unit_of_work) + message_bus.handle(event, unit_of_work) @then(u'the Collection Manager can see that it was preserved.') -def _(context, unit_of_work: AbstractUnitOfWork): - event = context.cataloged_event - assert event.identifier == "00000000-0000-0000-0000-000000000001" - assert unit_of_work.gateway.has_object(event.identifier) +def _(unit_of_work: AbstractUnitOfWork): + expected_identifier = "00000000-0000-0000-0000-000000000001" + assert unit_of_work.gateway.has_object(expected_identifier) with unit_of_work: - bin = unit_of_work.catalog.get(event.identifier) + bin = unit_of_work.catalog.get(expected_identifier) assert bin is not None From bd1ae9ac5d73fe7858d2f8ddd7d96b9f5bf449bd Mon Sep 17 00:00:00 2001 From: Samuel Sciolla Date: Thu, 23 Jan 2025 14:30:04 -0500 Subject: [PATCH 18/23] Refactor test_inspect_bin to use fixture instead of context --- features/steps/test_inspect_bin.py | 93 ++++++++++++++++-------------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/features/steps/test_inspect_bin.py b/features/steps/test_inspect_bin.py index a731e70..abae3bc 100644 --- a/features/steps/test_inspect_bin.py +++ b/features/steps/test_inspect_bin.py @@ -1,32 +1,25 @@ import uuid from dataclasses import dataclass from datetime import datetime, UTC +from functools import partial +import pytest from pydantic_core import to_jsonable_python +from pytest_bdd import scenario, given, when, then, parsers from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from functools import partial -from pytest_bdd import scenario, given, when, then, parsers from dor.adapters.catalog import Base, _custom_json_serializer from dor.config import config from dor.domain.models import Bin from dor.service_layer import catalog_service -from dor.service_layer.unit_of_work import SqlalchemyUnitOfWork +from dor.service_layer.unit_of_work import AbstractUnitOfWork, SqlalchemyUnitOfWork from dor.providers.models import ( Agent, AlternateIdentifier, FileMetadata, FileReference, PackageResource, PreservationEvent, StructMap, StructMapItem, StructMapType ) from gateway.fake_repository_gateway import FakeRepositoryGateway -@dataclass -class Context: - uow: SqlalchemyUnitOfWork = None - bin: Bin = None - alt_id: str = None - summary: dict = None - file_sets: list = None - scenario = partial(scenario, '../inspect_bin.feature') @scenario('Revision summary') @@ -37,13 +30,26 @@ def test_revision_summary(): def test_revision_file_sets(): pass -@given(parsers.parse(u'a preserved monograph with an alternate identifier of "{alt_id}"'), target_fixture="context") -def _(alt_id): - context = Context() +@pytest.fixture +def unit_of_work() -> AbstractUnitOfWork: + engine = create_engine( + config.get_test_database_engine_url(), json_serializer=_custom_json_serializer + ) + session_factory = sessionmaker(bind=engine) + Base.metadata.drop_all(engine) + Base.metadata.create_all(engine) + uow = SqlalchemyUnitOfWork(gateway=FakeRepositoryGateway(), session_factory=session_factory) + return uow + +@given( + parsers.parse(u'a preserved monograph with an alternate identifier of "{alt_id}"'), + target_fixture="bin" +) +def _(alt_id, unit_of_work: AbstractUnitOfWork): bin = Bin( identifier=uuid.UUID("00000000-0000-0000-0000-000000000001"), - alternate_identifiers=["xyzzy:00000001"], + alternate_identifiers=[alt_id], common_metadata={ "@schema": "urn:umich.edu:dor:schema:common", "title": "Discussion also Republican owner hot already itself.", @@ -202,32 +208,27 @@ def _(alt_id): ] ) - engine = create_engine( - config.get_test_database_engine_url(), json_serializer=_custom_json_serializer - ) - session_factory = sessionmaker(bind=engine) - Base.metadata.drop_all(engine) - Base.metadata.create_all(engine) + with unit_of_work: + unit_of_work.catalog.add(bin) + unit_of_work.commit() - context.uow = SqlalchemyUnitOfWork(gateway=FakeRepositoryGateway(), session_factory=session_factory) - with context.uow: - context.uow.catalog.add(bin) - context.uow.commit() - - return context + return bin -@when(parsers.parse(u'the Collection Manager looks up the bin by "{alt_id}"')) -def step_impl(context, alt_id): - context.alt_id = alt_id - with context.uow: - context.bin = context.uow.catalog.get_by_alternate_identifier(alt_id) - context.summary = catalog_service.summarize(context.bin) +@when( + parsers.parse(u'the Collection Manager looks up the bin by "{alt_id}"'), + target_fixture="summary" +) +def _(alt_id, unit_of_work: AbstractUnitOfWork): + with unit_of_work: + bin = unit_of_work.catalog.get_by_alternate_identifier(alt_id) + summary = catalog_service.summarize(bin) + return summary @then(u'the Collection Manager sees the summary of the bin') -def step_impl(context): +def _(bin: Bin, summary): expected_summary = dict( identifier="00000000-0000-0000-0000-000000000001", - alternate_identifiers=[context.alt_id], + alternate_identifiers=bin.alternate_identifiers, common_metadata={ "@schema": "urn:umich.edu:dor:schema:common", "title": "Discussion also Republican owner hot already itself.", @@ -239,19 +240,23 @@ def step_impl(context): ] } ) - assert context.summary == expected_summary + assert summary == expected_summary -@when(parsers.parse(u'the Collection Manager lists the contents of the bin for "{alt_id}"')) -def _(context, alt_id): - with context.uow: - context.bin = context.uow.catalog.get_by_alternate_identifier(alt_id) - context.file_sets = catalog_service.get_file_sets(context.bin) +@when( + parsers.parse(u'the Collection Manager lists the contents of the bin for "{alt_id}"'), + target_fixture="file_sets" +) +def _(alt_id, unit_of_work): + with unit_of_work: + bin = unit_of_work.catalog.get_by_alternate_identifier(alt_id) + file_sets = catalog_service.get_file_sets(bin) + return file_sets @then(u'the Collection Manager sees the file sets.') -def _(context): +def _(bin: Bin, file_sets): expected_file_sets = [ to_jsonable_python(resource) - for resource in context.bin.package_resources if resource.type == 'Asset' + for resource in bin.package_resources if resource.type == 'Asset' ] - assert context.file_sets == expected_file_sets + assert file_sets == expected_file_sets From c89fe6567b934ff989be8247d4937b1369748d66 Mon Sep 17 00:00:00 2001 From: Samuel Sciolla Date: Thu, 23 Jan 2025 14:48:07 -0500 Subject: [PATCH 19/23] Tidy up some imports --- features/steps/test_inspect_bin.py | 5 ++--- features/steps/test_store_resource.py | 10 +++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/features/steps/test_inspect_bin.py b/features/steps/test_inspect_bin.py index abae3bc..6c4ad50 100644 --- a/features/steps/test_inspect_bin.py +++ b/features/steps/test_inspect_bin.py @@ -1,5 +1,4 @@ import uuid -from dataclasses import dataclass from datetime import datetime, UTC from functools import partial @@ -12,12 +11,12 @@ from dor.adapters.catalog import Base, _custom_json_serializer from dor.config import config from dor.domain.models import Bin -from dor.service_layer import catalog_service -from dor.service_layer.unit_of_work import AbstractUnitOfWork, SqlalchemyUnitOfWork from dor.providers.models import ( Agent, AlternateIdentifier, FileMetadata, FileReference, PackageResource, PreservationEvent, StructMap, StructMapItem, StructMapType ) +from dor.service_layer import catalog_service +from dor.service_layer.unit_of_work import AbstractUnitOfWork, SqlalchemyUnitOfWork from gateway.fake_repository_gateway import FakeRepositoryGateway scenario = partial(scenario, '../inspect_bin.feature') diff --git a/features/steps/test_store_resource.py b/features/steps/test_store_resource.py index bebbad6..0055fd5 100644 --- a/features/steps/test_store_resource.py +++ b/features/steps/test_store_resource.py @@ -23,16 +23,16 @@ PackageUnpacked ) from dor.providers.file_system_file_provider import FilesystemFileProvider -from dor.providers.translocator import Translocator, Workspace from dor.providers.package_resource_provider import PackageResourceProvider +from dor.providers.translocator import Translocator, Workspace +from dor.service_layer.handlers.catalog_bin import catalog_bin +from dor.service_layer.handlers.receive_package import receive_package from dor.service_layer.handlers.store_files import store_files +from dor.service_layer.handlers.unpack_package import unpack_package +from dor.service_layer.handlers.verify_package import verify_package from dor.service_layer.message_bus.memory_message_bus import MemoryMessageBus from dor.service_layer.unit_of_work import AbstractUnitOfWork, SqlalchemyUnitOfWork from gateway.ocfl_repository_gateway import OcflRepositoryGateway -from dor.service_layer.handlers.catalog_bin import catalog_bin -from dor.service_layer.handlers.receive_package import receive_package -from dor.service_layer.handlers.verify_package import verify_package -from dor.service_layer.handlers.unpack_package import unpack_package @dataclass class PathData: From ee58f1e06aa69ad85e71b2ca38c4ea37b9a71786 Mon Sep 17 00:00:00 2001 From: Samuel Sciolla Date: Thu, 23 Jan 2025 16:08:46 -0500 Subject: [PATCH 20/23] Fix CLI by passing file provider to unpack_package handler --- dor/cli/repo.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dor/cli/repo.py b/dor/cli/repo.py index b421c79..2a92df6 100644 --- a/dor/cli/repo.py +++ b/dor/cli/repo.py @@ -17,6 +17,7 @@ PackageUnpacked, PackageVerified, ) +from dor.providers.file_system_file_provider import FilesystemFileProvider from dor.providers.package_resource_provider import PackageResourceProvider from dor.providers.translocator import Translocator, Workspace from dor.service_layer.handlers.catalog_bin import catalog_bin @@ -46,13 +47,16 @@ def bootstrap() -> Tuple[MemoryMessageBus, SqlalchemyUnitOfWork]: workspaces_path=config.workspaces_path, minter = lambda: str(uuid.uuid4()) ) + file_provider = FilesystemFileProvider() handlers: dict[Type[Event], list[Callable]] = { PackageSubmitted: [lambda event: receive_package(event, uow, translocator)], PackageReceived: [lambda event: verify_package(event, uow, BagAdapter, Workspace)], - PackageVerified: [lambda event: unpack_package( - event, uow, BagAdapter, PackageResourceProvider, Workspace - )], + PackageVerified: [ + lambda event: unpack_package( + event, uow, BagAdapter, PackageResourceProvider, Workspace, file_provider + ) + ], PackageUnpacked: [lambda event: store_files(event, uow, Workspace)], PackageStored: [lambda event: catalog_bin(event, uow)], BinCataloged: [] From ca1e2fb0fadbaa5378cb100cb70290ef709fb14d Mon Sep 17 00:00:00 2001 From: Samuel Sciolla Date: Thu, 23 Jan 2025 16:14:08 -0500 Subject: [PATCH 21/23] Remove unnecessary replace since paths are now relative to package root --- dor/service_layer/handlers/catalog_bin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dor/service_layer/handlers/catalog_bin.py b/dor/service_layer/handlers/catalog_bin.py index 7a35a4c..93e77ca 100644 --- a/dor/service_layer/handlers/catalog_bin.py +++ b/dor/service_layer/handlers/catalog_bin.py @@ -10,8 +10,7 @@ def catalog_bin(event: PackageStored, uow: AbstractUnitOfWork) -> None: common_metadata_file = [ metadata_file for metadata_file in root_resource.metadata_files if "common" in metadata_file.ref.locref ][0] - # Path is still relative, but won't be once after Jaya's change - common_metadata_file_path = Path(common_metadata_file.ref.locref.replace("../", "")) + common_metadata_file_path = Path(common_metadata_file.ref.locref) object_files = uow.gateway.get_object_files(event.identifier) matching_object_file = [ object_file for object_file in object_files if common_metadata_file_path == object_file.logical_path From 84448fc2595da7d7ba7859e8f0e6d1cd922e9332 Mon Sep 17 00:00:00 2001 From: Greg Kostin Date: Thu, 23 Jan 2025 15:22:33 -0500 Subject: [PATCH 22/23] GitHub Action Workflow Continuous Integration --- .github/workflows/ci.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..a6a559e --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,22 @@ +--- +name: Continuous Integration + +on: + workflow_dispatch: + push: + pull_request: + branches: + - 'main' + +jobs: + ci: + if: ${{ github.event_name == 'push' || github.event.pull_request.merged == true }} + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run tests + run: | + ./init.sh + docker compose up -d api + docker compose run --rm app poetry run pytest From 0e79853cba152e86cc3146ad7c640087bb859adb Mon Sep 17 00:00:00 2001 From: Greg Kostin Date: Thu, 23 Jan 2025 17:12:42 -0500 Subject: [PATCH 23/23] don't spin up the api for test --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a6a559e..0b75b14 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,5 +18,4 @@ jobs: - name: Run tests run: | ./init.sh - docker compose up -d api docker compose run --rm app poetry run pytest