From 49dc4649c2db7b90b7528a72083f8690a86cb4ef Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:34:50 +1000
Subject: [PATCH 01/36] Add prune bookkeeping and a PRUNING state to SyncRun
Schema only: nothing writes these yet. A run will need somewhere to record
how many images it removed, how many it found missing, which prune the
caller asked for, and why a prune removed nothing.
prune_mode is persisted rather than passed around because the caller that
finalises a run may be a different process entirely, so command-line flags
cannot reach it any other way.
PRUNING joins ACTIVE_STATES so that, once pruning exists, the
unique-active-run constraint keeps a second sync from starting while a
prune walks the tree and re-importing files it is about to remove. The
constraint is rebuilt because its condition embeds the state tuple.
failure_reason lets a caller tell "folder is missing on disk" apart from
any other failure, which today is guessed from the state alone.
Co-Authored-By: Claude Opus 5
---
.../migrations/0035_syncrun_prune_fields.py | 46 ++++++++++++++++
src/data/models/_sync_run.py | 55 +++++++++++++++++--
2 files changed, 96 insertions(+), 5 deletions(-)
create mode 100644 src/data/migrations/0035_syncrun_prune_fields.py
diff --git a/src/data/migrations/0035_syncrun_prune_fields.py b/src/data/migrations/0035_syncrun_prune_fields.py
new file mode 100644
index 0000000..40900cc
--- /dev/null
+++ b/src/data/migrations/0035_syncrun_prune_fields.py
@@ -0,0 +1,46 @@
+# Generated by Django 6.0.3 on 2026-08-08 04:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('data', '0034_syncrun'),
+ ]
+
+ operations = [
+ migrations.RemoveConstraint(
+ model_name='syncrun',
+ name='unique_active_sync_run_per_folder',
+ ),
+ migrations.AddField(
+ model_name='syncrun',
+ name='failure_reason',
+ field=models.CharField(blank=True, default='', max_length=32),
+ ),
+ migrations.AddField(
+ model_name='syncrun',
+ name='missing_found',
+ field=models.IntegerField(default=0),
+ ),
+ migrations.AddField(
+ model_name='syncrun',
+ name='prune_mode',
+ field=models.CharField(default='PRUNE_AUTO', max_length=16),
+ ),
+ migrations.AddField(
+ model_name='syncrun',
+ name='prune_skipped',
+ field=models.CharField(blank=True, default='', max_length=32),
+ ),
+ migrations.AddField(
+ model_name='syncrun',
+ name='removed',
+ field=models.IntegerField(default=0),
+ ),
+ migrations.AddConstraint(
+ model_name='syncrun',
+ constraint=models.UniqueConstraint(condition=models.Q(('state__in', ('SCANNING', 'PROCESSING', 'PRUNING'))), fields=('folder',), name='unique_active_sync_run_per_folder'),
+ ),
+ ]
diff --git a/src/data/models/_sync_run.py b/src/data/models/_sync_run.py
index ad18021..873b307 100644
--- a/src/data/models/_sync_run.py
+++ b/src/data/models/_sync_run.py
@@ -5,25 +5,59 @@
_STATE_SCANNING = "SCANNING"
_STATE_PROCESSING = "PROCESSING"
+_STATE_PRUNING = "PRUNING"
_STATE_COMPLETED = "COMPLETED"
_STATE_FAILED = "FAILED"
_STATE_INTERRUPTED = "INTERRUPTED"
-# A run is "active" while it is scanning or processing; at most one active run is
-# allowed per folder (enforced by a conditional UniqueConstraint below).
-_ACTIVE_STATES = (_STATE_SCANNING, _STATE_PROCESSING)
+# A run is "active" while it is scanning, processing or pruning; at most one active run is
+# allowed per folder (enforced by a conditional UniqueConstraint below). Pruning counts as
+# active so a second sync cannot start while the prune is still walking the tree.
+_ACTIVE_STATES = (_STATE_SCANNING, _STATE_PROCESSING, _STATE_PRUNING)
+
+# What the caller asked the run's prune phase to do.
+_PRUNE_MODE_AUTO = "PRUNE_AUTO"
+_PRUNE_MODE_FORCE = "PRUNE_FORCE"
+_PRUNE_MODE_DRY_RUN = "PRUNE_DRY_RUN"
+_PRUNE_MODE_OFF = "PRUNE_OFF"
+
+# Why a run's prune phase removed nothing.
+_SKIPPED_GUARD = "SKIPPED_GUARD"
+_SKIPPED_DRY_RUN = "SKIPPED_DRY_RUN"
+_SKIPPED_OFF = "SKIPPED_OFF"
+_SKIPPED_FOLDER_MISSING = "SKIPPED_FOLDER_MISSING"
+_SKIPPED_DEFERRED = "SKIPPED_DEFERRED"
+
+# Why a run failed. error_message keeps the free-text detail.
+_FAILED_FOLDER_MISSING = "FAILED_FOLDER_MISSING"
_STATE_MAX_LEN = 16
+_PRUNE_MODE_MAX_LEN = 16
+_CODE_MAX_LEN = 32
class SyncRun(models.Model):
STATE_SCANNING = _STATE_SCANNING
STATE_PROCESSING = _STATE_PROCESSING
+ STATE_PRUNING = _STATE_PRUNING
STATE_COMPLETED = _STATE_COMPLETED
STATE_FAILED = _STATE_FAILED
STATE_INTERRUPTED = _STATE_INTERRUPTED
ACTIVE_STATES = _ACTIVE_STATES
+ PRUNE_MODE_AUTO = _PRUNE_MODE_AUTO
+ PRUNE_MODE_FORCE = _PRUNE_MODE_FORCE
+ PRUNE_MODE_DRY_RUN = _PRUNE_MODE_DRY_RUN
+ PRUNE_MODE_OFF = _PRUNE_MODE_OFF
+
+ SKIPPED_GUARD = _SKIPPED_GUARD
+ SKIPPED_DRY_RUN = _SKIPPED_DRY_RUN
+ SKIPPED_OFF = _SKIPPED_OFF
+ SKIPPED_FOLDER_MISSING = _SKIPPED_FOLDER_MISSING
+ SKIPPED_DEFERRED = _SKIPPED_DEFERRED
+
+ FAILED_FOLDER_MISSING = _FAILED_FOLDER_MISSING
+
folder = models.ForeignKey(
LibraryFolder,
on_delete=models.CASCADE,
@@ -34,7 +68,12 @@ class SyncRun(models.Model):
processed = models.IntegerField(default=0)
skipped = models.IntegerField(default=0)
errors = models.IntegerField(default=0)
+ removed = models.IntegerField(default=0)
+ missing_found = models.IntegerField(default=0)
+ prune_mode = models.CharField(max_length=_PRUNE_MODE_MAX_LEN, default=_PRUNE_MODE_AUTO)
+ prune_skipped = models.CharField(max_length=_CODE_MAX_LEN, blank=True, default="")
error_message = models.TextField(null=True)
+ failure_reason = models.CharField(max_length=_CODE_MAX_LEN, blank=True, default="")
started_at = models.DateTimeField(default=timezone.now)
finished_at = models.DateTimeField(null=True)
created_at = models.DateTimeField(default=timezone.now)
@@ -55,8 +94,8 @@ class Meta:
# Factories
@classmethod
- def create(cls, *, folder: LibraryFolder) -> "SyncRun":
- return cls.objects.create(folder=folder, state=_STATE_SCANNING)
+ def create(cls, *, folder: LibraryFolder, prune_mode: str = _PRUNE_MODE_AUTO) -> "SyncRun":
+ return cls.objects.create(folder=folder, state=_STATE_SCANNING, prune_mode=prune_mode)
# Mutators
@@ -100,6 +139,12 @@ def mark_failed(self, *, message: str) -> None:
self.finished_at = timezone.now()
self.save(update_fields=["state", "error_message", "finished_at", "updated_at"])
+ def record_prune_result(self, *, missing_found: int, removed: int, skipped_reason: str) -> None:
+ self.missing_found = missing_found
+ self.removed = removed
+ self.prune_skipped = skipped_reason
+ self.save(update_fields=["missing_found", "removed", "prune_skipped", "updated_at"])
+
# Queries
def all_images_accounted_for(self) -> bool:
From ada8c6b4b305557536c56b99ce5f76a30cbdd07b Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:35:57 +1000
Subject: [PATCH 02/36] Replace mark_completed with a general transition_state
mutator
mark_completed baked one specific transition, and its legality, into the
model. Pruning needs a second guarded transition, and copying the pattern
would put more policy there.
transition_state does the guarded write and nothing else: it reports how
many rows it touched and leaves the caller to decide which transitions are
legal and whether the target state is terminal. The WHERE clause has to
stay in SQL, since moving it into Python would turn the exactly-one-winner
election into a read-then-write race.
complete_sync_run moves onto it in the same commit, because the mutator it
called no longer exists. It also now accepts a run that is pruning, which
is the transition the next commits need.
Co-Authored-By: Claude Opus 5
---
src/data/models/_sync_run.py | 29 ++++++++++++++-----
src/domain/library/operations.py | 8 +++--
.../domain/library/test_operations.py | 8 +++++
3 files changed, 35 insertions(+), 10 deletions(-)
diff --git a/src/data/models/_sync_run.py b/src/data/models/_sync_run.py
index 873b307..b1625e5 100644
--- a/src/data/models/_sync_run.py
+++ b/src/data/models/_sync_run.py
@@ -1,3 +1,6 @@
+from collections.abc import Sequence
+from datetime import datetime
+
from django.db import models
from django.utils import timezone
@@ -122,14 +125,24 @@ def _increment(self, field: str) -> None:
**{field: models.F(field) + 1},
)
- def mark_completed(self) -> bool:
- # Conditional so exactly one caller wins the finalize under concurrency.
- # Returns True if this call transitioned the run to COMPLETED.
- now = timezone.now()
- rows = type(self).objects.filter(pk=self.pk, state=_STATE_PROCESSING).update(
- state=_STATE_COMPLETED,
- finished_at=now,
- updated_at=now,
+ def transition_state(
+ self,
+ *,
+ from_states: Sequence[str],
+ to_state: str,
+ finished_at: datetime | None,
+ ) -> bool:
+ """
+ Move this run to *to_state* only if it is currently in one of *from_states*.
+
+ A single conditional UPDATE, so under concurrent workers exactly one caller
+ gets True. The caller decides which transitions are legal and whether the
+ target state is terminal, passing finished_at=None when it is not.
+ """
+ rows = type(self).objects.filter(pk=self.pk, state__in=from_states).update(
+ state=to_state,
+ finished_at=finished_at,
+ updated_at=timezone.now(),
)
return rows > 0
diff --git a/src/domain/library/operations.py b/src/domain/library/operations.py
index 80dd424..fd3ff5e 100644
--- a/src/domain/library/operations.py
+++ b/src/domain/library/operations.py
@@ -137,13 +137,17 @@ def start_sync_run(*, folder: models.LibraryFolder) -> models.SyncRun:
def complete_sync_run(*, run: models.SyncRun) -> bool:
"""
- Mark *run* as completed if it is still processing.
+ Mark *run* as completed if it is still processing or pruning.
Uses a conditional update so that, under concurrent workers, exactly one
caller transitions the run and publishes the completion event. Returns True
if this call completed the run.
"""
- completed = run.mark_completed()
+ completed = run.transition_state(
+ from_states=(models.SyncRun.STATE_PROCESSING, models.SyncRun.STATE_PRUNING),
+ to_state=models.SyncRun.STATE_COMPLETED,
+ finished_at=timezone.now(),
+ )
if completed:
events.publish_event(
event_type=events.LIBRARY_SYNC_RUN_COMPLETED,
diff --git a/tests/integration/domain/library/test_operations.py b/tests/integration/domain/library/test_operations.py
index 9f9a6e1..07fe97c 100644
--- a/tests/integration/domain/library/test_operations.py
+++ b/tests/integration/domain/library/test_operations.py
@@ -216,6 +216,14 @@ def test_returns_false_and_publishes_nothing_when_already_completed(self, captur
matching = [e for e in captured_logs if e.get("event_type") == events.LIBRARY_SYNC_RUN_COMPLETED]
assert matching == []
+ def test_transitions_a_pruning_run_to_completed(self):
+ run = SyncRunFactory(state=models.SyncRun.STATE_PRUNING, total=1)
+
+ assert complete_sync_run(run=run) is True
+
+ run.refresh_from_db()
+ assert run.state == models.SyncRun.STATE_COMPLETED
+
@pytest.mark.django_db
class TestFailSyncRun:
From f591821c93848c4f06583e9efe7e7589bbd6acc7 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 18:26:41 +1000
Subject: [PATCH 03/36] Add library prune guard settings
Bound how much an automatic prune may remove in one pass, so an unmounted
drive or an unreadable directory is reported rather than applied. Both
thresholds must be exceeded for the guard to engage, which keeps ordinary
small cleanups from tripping it.
Co-Authored-By: Claude Opus 5
---
src/config/settings.py | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/src/config/settings.py b/src/config/settings.py
index cf20512..c902a4e 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -81,6 +81,16 @@
THUMBNAIL_CACHE_DIR = BASE_DIR / "thumbnail_cache" # filesystem directory where generated thumbnails are cached
RECIPE_CARDS_DIR: Path = Path(env.str("RECIPE_CARDS_DIR", default=str(BASE_DIR / "recipe_cards"))) # filesystem directory where generated recipe card images are stored
+# Library sync removes catalog entries whose files have disappeared. A sync that finds most of a
+# folder's images missing is far more likely to be an unmounted drive or an unreadable directory
+# than a real deletion, so an automatic prune above this share of the folder's catalogued images is
+# reported instead of applied. Override a single run with `sync_library --force-prune`.
+LIBRARY_PRUNE_GUARD_FRACTION: float = env.float("LIBRARY_PRUNE_GUARD_FRACTION", default=0.5)
+
+# The guard above only engages once at least this many images would be removed, so that ordinary
+# small cleanups (emptying a folder of a handful of photos) are applied without a warning.
+LIBRARY_PRUNE_GUARD_MIN_IMAGES: int = env.int("LIBRARY_PRUNE_GUARD_MIN_IMAGES", default=20)
+
TEMPLATES = [
{
From 3e8a7ee58ec7fba9482c61e146fd698d046bd07e Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 18:29:38 +1000
Subject: [PATCH 04/36] Make thumbnail widths a setting instead of a command
constant
The width lived as a constant inside the generate_thumbnails command, so
nothing else could know which widths exist. Removing an image has to clear
its cached thumbnail at every width, which needs a definitive list.
Overridable from the env file as a comma-separated list
(THUMBNAIL_WIDTHS=600,1200); the command now generates each configured
width rather than a single hardcoded one.
Co-Authored-By: Claude Opus 5
---
src/config/settings.py | 6 +++++
.../commands/generate_thumbnails.py | 24 +++++++++----------
2 files changed, 18 insertions(+), 12 deletions(-)
diff --git a/src/config/settings.py b/src/config/settings.py
index c902a4e..fc01766 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -81,6 +81,12 @@
THUMBNAIL_CACHE_DIR = BASE_DIR / "thumbnail_cache" # filesystem directory where generated thumbnails are cached
RECIPE_CARDS_DIR: Path = Path(env.str("RECIPE_CARDS_DIR", default=str(BASE_DIR / "recipe_cards"))) # filesystem directory where generated recipe card images are stored
+# Thumbnail widths that are generated and cached, as a comma-separated list in the env file
+# (THUMBNAIL_WIDTHS=600,1200). A cache key is derived from the image path, so removing an image has
+# to clear every width: a later file reusing that path (Fujifilm filenames wrap around from
+# DSCF9999 to DSCF0001) would otherwise be served the previous image's thumbnail.
+THUMBNAIL_WIDTHS: tuple[int, ...] = tuple(env.list("THUMBNAIL_WIDTHS", subcast=int, default=[600]))
+
# Library sync removes catalog entries whose files have disappeared. A sync that finds most of a
# folder's images missing is far more likely to be an unmounted drive or an unreadable directory
# than a real deletion, so an automatic prune above this share of the folder's catalogued images is
diff --git a/src/interfaces/management/commands/generate_thumbnails.py b/src/interfaces/management/commands/generate_thumbnails.py
index 035b103..ab959f9 100644
--- a/src/interfaces/management/commands/generate_thumbnails.py
+++ b/src/interfaces/management/commands/generate_thumbnails.py
@@ -1,27 +1,27 @@
from typing import Any
+from django.conf import settings
from django.core.management.base import BaseCommand
from src.application.usecases.images import generate_thumbnails
-THUMBNAIL_WIDTH = 600
-
class Command(BaseCommand):
help = "Pre-generate thumbnail cache for all images."
def handle(self, *args: object, **options: Any) -> None:
- self.stdout.write(f"Generating thumbnails at width={THUMBNAIL_WIDTH}px…")
+ for width in settings.THUMBNAIL_WIDTHS:
+ self.stdout.write(f"Generating thumbnails at width={width}px…")
- result = generate_thumbnails.generate_thumbnails_for_all_images(width=THUMBNAIL_WIDTH)
+ result = generate_thumbnails.generate_thumbnails_for_all_images(width=width)
- for path in result.missing_paths:
- self.stderr.write(f" Missing file: {path}")
+ for path in result.missing_paths:
+ self.stderr.write(f" Missing file: {path}")
- self.stdout.write(
- self.style.SUCCESS(
- f"Done. enqueued={result.enqueued}"
- f" already_cached={result.already_cached}"
- f" missing={len(result.missing_paths)}"
+ self.stdout.write(
+ self.style.SUCCESS(
+ f"Done. enqueued={result.enqueued}"
+ f" already_cached={result.already_cached}"
+ f" missing={len(result.missing_paths)}"
+ )
)
- )
From 7f5e90525dbb64ff195cadeab061eafaa60c32a9 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 14:36:25 +1000
Subject: [PATCH 05/36] Add delete_cached_thumbnails domain operation
Thumbnail cache keys are derived from the image path, so removing an image
without clearing its cache leaves files that a later image reusing that
path would be served instead of its own. Fujifilm filenames wrap around
from DSCF9999 to DSCF0001, which makes that a real possibility rather than
a theoretical one.
Co-Authored-By: Claude Opus 5
---
src/domain/images/thumbnails/operations.py | 23 ++++++++++
.../test_delete_cached_thumbnails.py | 42 +++++++++++++++++++
2 files changed, 65 insertions(+)
create mode 100644 tests/integration/domain/images/thumbnails/test_delete_cached_thumbnails.py
diff --git a/src/domain/images/thumbnails/operations.py b/src/domain/images/thumbnails/operations.py
index 128e4f1..3d88b13 100644
--- a/src/domain/images/thumbnails/operations.py
+++ b/src/domain/images/thumbnails/operations.py
@@ -1,5 +1,6 @@
from pathlib import Path
+from django import conf
from PIL import Image as PILImage
from src.domain.images.thumbnails import queries as thumbnail_queries
@@ -18,6 +19,28 @@
}
+def delete_cached_thumbnails(*, original_path: Path) -> int:
+ """
+ Remove every cached thumbnail generated for *original_path*.
+
+ Cache keys are derived from the image path, so a path that is later reused by
+ a different file would otherwise be served the previous image's thumbnail.
+ Fujifilm filenames wrap around, which makes that a real possibility.
+
+ Returns how many cache files were removed. A cache file that was never
+ generated, or that another process removed first, is not an error.
+ """
+ removed = 0
+ for width in conf.settings.THUMBNAIL_WIDTHS:
+ cache_path = thumbnail_queries.thumbnail_cache_path(original_path=original_path, width=width)
+ try:
+ cache_path.unlink()
+ except OSError:
+ continue
+ removed += 1
+ return removed
+
+
def generate_thumbnail(*, original_path: Path, width: int) -> Path:
"""
Resize *original_path* to *width* px wide, applying EXIF orientation, and
diff --git a/tests/integration/domain/images/thumbnails/test_delete_cached_thumbnails.py b/tests/integration/domain/images/thumbnails/test_delete_cached_thumbnails.py
new file mode 100644
index 0000000..f87d2c2
--- /dev/null
+++ b/tests/integration/domain/images/thumbnails/test_delete_cached_thumbnails.py
@@ -0,0 +1,42 @@
+from pathlib import Path
+
+from django.test import override_settings
+
+from src.domain.images.thumbnails.operations import delete_cached_thumbnails
+from src.domain.images.thumbnails.queries import thumbnail_cache_path
+
+ORIGINAL = Path("/photos/2024/DSCF0001.JPG")
+
+
+class TestDeleteCachedThumbnails:
+ def test_removes_the_cache_file_for_every_configured_width(self, tmp_path):
+ with override_settings(THUMBNAIL_CACHE_DIR=tmp_path, THUMBNAIL_WIDTHS=(600, 1200)):
+ cached = [thumbnail_cache_path(original_path=ORIGINAL, width=w) for w in (600, 1200)]
+ for path in cached:
+ path.write_bytes(b"\xff\xd8")
+
+ removed = delete_cached_thumbnails(original_path=ORIGINAL)
+
+ assert removed == 2
+ assert not any(path.exists() for path in cached)
+
+ def test_leaves_thumbnails_of_other_images_alone(self, tmp_path):
+ with override_settings(THUMBNAIL_CACHE_DIR=tmp_path, THUMBNAIL_WIDTHS=(600,)):
+ other = Path("/photos/2024/DSCF0002.JPG")
+ other_cache = thumbnail_cache_path(original_path=other, width=600)
+ other_cache.write_bytes(b"\xff\xd8")
+ thumbnail_cache_path(original_path=ORIGINAL, width=600).write_bytes(b"\xff\xd8")
+
+ delete_cached_thumbnails(original_path=ORIGINAL)
+
+ assert other_cache.exists()
+
+ def test_reports_nothing_removed_when_no_thumbnail_was_cached(self, tmp_path):
+ with override_settings(THUMBNAIL_CACHE_DIR=tmp_path, THUMBNAIL_WIDTHS=(600,)):
+ assert delete_cached_thumbnails(original_path=ORIGINAL) == 0
+
+ def test_does_not_raise_when_the_cache_directory_is_absent(self, tmp_path):
+ missing = tmp_path / "no_such_cache"
+
+ with override_settings(THUMBNAIL_CACHE_DIR=missing, THUMBNAIL_WIDTHS=(600,)):
+ assert delete_cached_thumbnails(original_path=ORIGINAL) == 0
From 7aeef3713dbb944255e4e5245d3a8202c80e09e7 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 14:35:48 +1000
Subject: [PATCH 06/36] Add exclusive-ownership queries for library folders
Removing a folder from the Library can take its images with it, but
library folders may nest: an image below /photos/2024 is also below
/photos. These queries scope removal to images no other registered folder
covers, so removing the inner folder never takes images the outer one
still monitors.
Co-Authored-By: Claude Opus 5
---
src/domain/library/queries.py | 34 ++++++++
.../test_exclusive_ownership_queries.py | 79 +++++++++++++++++++
2 files changed, 113 insertions(+)
create mode 100644 tests/integration/domain/library/test_exclusive_ownership_queries.py
diff --git a/src/domain/library/queries.py b/src/domain/library/queries.py
index 7d81f84..e46ff4d 100644
--- a/src/domain/library/queries.py
+++ b/src/domain/library/queries.py
@@ -95,6 +95,40 @@ def get_sync_run(*, run_id: int) -> models.SyncRun:
raise SyncRunNotFound(run_id=run_id)
+def _folder_prefix(*, path: str) -> str:
+ return path.rstrip(os.sep) + os.sep
+
+
+def get_exclusively_owned_image_ids(*, folder_id: int) -> list[int]:
+ """
+ Return the ids of images under this folder's path that no other registered
+ library folder also covers.
+
+ Library folders may nest, so an image below ``/photos/2024`` is also below
+ ``/photos``. Removing the inner folder must not take images the outer one
+ still monitors, so anything covered by another registered folder is excluded.
+
+ :raises LibraryFolderNotFound: If no folder with *folder_id* exists.
+ """
+ folder = get_library_folder(folder_id=folder_id)
+
+ owned = models.Image.objects.filter(filepath__startswith=_folder_prefix(path=folder.path))
+ for other in models.LibraryFolder.objects.exclude(pk=folder_id):
+ owned = owned.exclude(filepath__startswith=_folder_prefix(path=other.path))
+
+ return list(owned.order_by("id").values_list("id", flat=True))
+
+
+def count_exclusively_owned_images(*, folder_id: int) -> int:
+ """
+ Return how many images would leave the gallery if this folder were removed
+ together with its images.
+
+ :raises LibraryFolderNotFound: If no folder with *folder_id* exists.
+ """
+ return len(get_exclusively_owned_image_ids(folder_id=folder_id))
+
+
def get_active_sync_run(*, folder_id: int) -> models.SyncRun | None:
"""
Return the in-progress (scanning or processing) sync run for *folder_id*, or
diff --git a/tests/integration/domain/library/test_exclusive_ownership_queries.py b/tests/integration/domain/library/test_exclusive_ownership_queries.py
new file mode 100644
index 0000000..e1c1856
--- /dev/null
+++ b/tests/integration/domain/library/test_exclusive_ownership_queries.py
@@ -0,0 +1,79 @@
+import pytest
+
+from src.domain.library.queries import (
+ LibraryFolderNotFound,
+ count_exclusively_owned_images,
+ get_exclusively_owned_image_ids,
+)
+from tests.factories import ImageFactory, LibraryFolderFactory
+
+
+@pytest.mark.django_db
+class TestGetExclusivelyOwnedImageIds:
+ def test_returns_images_under_the_folder(self):
+ folder = LibraryFolderFactory(path="/photos")
+ image = ImageFactory(filepath="/photos/2024/DSCF0001.JPG")
+
+ result = get_exclusively_owned_image_ids(folder_id=folder.pk)
+
+ assert result == [image.pk]
+
+ def test_excludes_images_outside_the_folder(self):
+ folder = LibraryFolderFactory(path="/photos")
+ ImageFactory(filepath="/elsewhere/DSCF0001.JPG")
+
+ result = get_exclusively_owned_image_ids(folder_id=folder.pk)
+
+ assert result == []
+
+ def test_excludes_images_a_nested_registered_folder_also_covers(self):
+ outer = LibraryFolderFactory(path="/photos")
+ LibraryFolderFactory(path="/photos/2024")
+ ImageFactory(filepath="/photos/2024/DSCF0001.JPG")
+ only_in_outer = ImageFactory(filepath="/photos/2023/DSCF0002.JPG")
+
+ result = get_exclusively_owned_image_ids(folder_id=outer.pk)
+
+ assert result == [only_in_outer.pk]
+
+ def test_excludes_images_an_enclosing_registered_folder_also_covers(self):
+ LibraryFolderFactory(path="/photos")
+ inner = LibraryFolderFactory(path="/photos/2024")
+ ImageFactory(filepath="/photos/2024/DSCF0001.JPG")
+
+ result = get_exclusively_owned_image_ids(folder_id=inner.pk)
+
+ assert result == []
+
+ def test_returns_images_when_the_only_other_folder_is_unrelated(self):
+ folder = LibraryFolderFactory(path="/photos")
+ LibraryFolderFactory(path="/scans")
+ image = ImageFactory(filepath="/photos/DSCF0001.JPG")
+
+ result = get_exclusively_owned_image_ids(folder_id=folder.pk)
+
+ assert result == [image.pk]
+
+ def test_raises_library_folder_not_found_for_unknown_id(self):
+ with pytest.raises(LibraryFolderNotFound) as exc_info:
+ get_exclusively_owned_image_ids(folder_id=9999)
+
+ assert exc_info.value.folder_id == 9999
+
+
+@pytest.mark.django_db
+class TestCountExclusivelyOwnedImages:
+ def test_counts_only_images_no_other_folder_covers(self):
+ outer = LibraryFolderFactory(path="/photos")
+ LibraryFolderFactory(path="/photos/2024")
+ ImageFactory(filepath="/photos/2024/DSCF0001.JPG")
+ ImageFactory(filepath="/photos/2023/DSCF0002.JPG")
+
+ assert count_exclusively_owned_images(folder_id=outer.pk) == 1
+
+ def test_counts_zero_for_a_folder_fully_covered_by_another(self):
+ LibraryFolderFactory(path="/photos")
+ inner = LibraryFolderFactory(path="/photos/2024")
+ ImageFactory(filepath="/photos/2024/DSCF0001.JPG")
+
+ assert count_exclusively_owned_images(folder_id=inner.pk) == 0
From eaf978eabf9c39fd4a4a0cc7cab5cc7573104fc8 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:37:01 +1000
Subject: [PATCH 07/36] Add a set_location mutator to Image
Nothing calls it yet. A catalog record will need to follow its file when
the file is renamed or moved, so that the record survives instead of being
stranded on a path that no longer exists.
Co-Authored-By: Claude Opus 5
---
src/data/models/_images.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/src/data/models/_images.py b/src/data/models/_images.py
index 1fd1f6d..2e00a6c 100644
--- a/src/data/models/_images.py
+++ b/src/data/models/_images.py
@@ -196,6 +196,11 @@ def set_content_hash(self, *, content_hash: str) -> None:
self.content_hash = content_hash
self.save(update_fields=["content_hash"])
+ def set_location(self, *, filepath: str, filename: str) -> None:
+ self.filepath = filepath
+ self.filename = filename
+ self.save(update_fields=["filepath", "filename"])
+
def set_as_favorite(self) -> None:
self.is_favorite = True
self.save(update_fields=["is_favorite"])
From a31c809a113ea6d3e565d61d4ff5352035c092cf Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:37:58 +1000
Subject: [PATCH 08/36] Let a catalog record follow its file when the file
moves
An import that matches an existing record by content hash is either a move
or a copy, and the old file tells them apart: gone means the bytes were
renamed or moved, still there means this is a second copy. Only a move
repoints the record.
Until now the filepath was never updated, so a renamed or moved photo left
its record pointing at a path that no longer exists: the gallery kept
showing it and every thumbnail request 404'd. Repointing also means a
rename keeps the photo's rating, favourite mark and album membership,
which a delete-and-re-import would throw away.
Wired into process_image here, since a record that never followed its file
is exactly the bug being fixed. One dedup test asserted the old behaviour
and now asserts relocation.
Never touches either file on disk.
Co-Authored-By: Claude Opus 5
---
src/domain/images/events.py | 1 +
src/domain/images/operations.py | 52 ++++++++-
.../domain/images/test_process_image_dedup.py | 8 +-
.../images/test_process_image_relocation.py | 108 ++++++++++++++++++
.../domain/images/test_relocate_image.py | 99 ++++++++++++++++
5 files changed, 259 insertions(+), 9 deletions(-)
create mode 100644 tests/integration/domain/images/test_process_image_relocation.py
create mode 100644 tests/integration/domain/images/test_relocate_image.py
diff --git a/src/domain/images/events.py b/src/domain/images/events.py
index 2b4d730..b28ab15 100644
--- a/src/domain/images/events.py
+++ b/src/domain/images/events.py
@@ -23,6 +23,7 @@
IMAGE_RATING_FAILED = "image.rating.failed"
IMAGE_DEDUP_FILE_MISSING = "image.dedup.file.missing"
IMAGE_IMPORT_SKIPPED = "image.import.skipped"
+IMAGE_FILE_RELOCATED = "image.file.relocated"
TASK_IMAGE_ENQUEUED = "task.image.enqueued"
TASK_IMAGE_STARTED = "task.image.started"
TASK_IMAGE_COMPLETED = "task.image.completed"
diff --git a/src/domain/images/operations.py b/src/domain/images/operations.py
index b5d5aa0..1474151 100644
--- a/src/domain/images/operations.py
+++ b/src/domain/images/operations.py
@@ -110,6 +110,41 @@ def toggle_image_favorite(*, image_id: int) -> bool:
return image.is_favorite
+def relocate_image(*, image: models.Image, new_path: str) -> bool:
+ """
+ Point *image* at *new_path* when the file it recorded has disappeared.
+
+ An import that matches an existing record by content hash is either a move or
+ a copy, and the two are told apart by the old file: if it is gone, the same
+ bytes have simply been renamed or moved and the record must follow so that
+ its rating, favourite flag and album membership survive. If the old file is
+ still there, this is a second copy of the same photo and the record is left
+ where it is.
+
+ Never touches either file on disk. Returns True if the record was moved.
+ """
+ if new_path == image.filepath:
+ return False
+ if os.path.lexists(image.filepath):
+ return False
+
+ # filepath is unique. A legacy record already sitting at the new path would
+ # collide, and raising inside the caller's transaction would poison it, so
+ # check first rather than catching IntegrityError.
+ if models.Image.objects.filter(filepath=new_path).exclude(pk=image.pk).exists():
+ return False
+
+ old_path = image.filepath
+ image.set_location(filepath=new_path, filename=os.path.basename(new_path))
+ events.publish_event(
+ event_type=events.IMAGE_FILE_RELOCATED,
+ image_id=image.pk,
+ old_filepath=old_path,
+ new_filepath=new_path,
+ )
+ return True
+
+
@transaction.atomic()
def process_image(*, image_path: str) -> models.Image:
"""
@@ -118,8 +153,11 @@ def process_image(*, image_path: str) -> models.Image:
Images are deduplicated by a SHA-256 hash of their file bytes: the same
photo stored under several paths resolves to a single record. A legacy
record imported before hashing existed is matched by its filepath (or, if
- the file has moved, by its EXIF identity) and has its hash backfilled. An
- already-hashed record is left untouched.
+ the file has moved, by its EXIF identity) and has its hash backfilled.
+
+ A matched record keeps its EXIF, recipe and user data, but follows its file:
+ if the path it recorded no longer exists, the file has been renamed or moved
+ and the record is repointed at *image_path*. See relocate_image.
A FujifilmExif record is looked up or created for the image's EXIF field
combination and linked via the recipe FK.
@@ -169,11 +207,15 @@ def process_image(*, image_path: str) -> models.Image:
else:
image = existing
created = False
- # A row found by its hash is already complete; never override it. An
- # un-hashed legacy row only gets its content hash backfilled — the data
- # we did not store before. Its filepath is left untouched.
+ # A row found by its hash is already complete; never override its EXIF or
+ # recipe. An un-hashed legacy row only gets its content hash backfilled,
+ # the data we did not store before.
if existing.content_hash == "":
existing.set_content_hash(content_hash=content_hash)
+ # The path is the exception: a record whose file has moved follows it, so
+ # a rename or a move keeps the same row rather than stranding it on a
+ # path that no longer exists.
+ relocate_image(image=existing, new_path=image_path)
events.publish_event(
event_type=events.RECIPE_IMAGE_CREATED if created else events.RECIPE_IMAGE_UPDATED,
diff --git a/tests/integration/domain/images/test_process_image_dedup.py b/tests/integration/domain/images/test_process_image_dedup.py
index ceb12c7..9b27179 100644
--- a/tests/integration/domain/images/test_process_image_dedup.py
+++ b/tests/integration/domain/images/test_process_image_dedup.py
@@ -100,7 +100,7 @@ def test_legacy_unhashed_row_at_same_path_has_its_hash_backfilled(self):
legacy.refresh_from_db()
assert legacy.content_hash == queries.compute_content_hash(image_path=FIXTURE_IMAGE)
- def test_exif_bridge_matches_a_moved_legacy_image_without_changing_its_path(self, tmp_path):
+ def test_exif_bridge_matches_a_moved_legacy_image_and_relocates_it(self, tmp_path):
exif = FujifilmExifFactory(
internal_serial_number=FIXTURE_SERIAL,
image_count=FIXTURE_IMAGE_COUNT,
@@ -119,9 +119,9 @@ def test_exif_bridge_matches_a_moved_legacy_image_without_changing_its_path(self
assert result.pk == legacy.pk
assert models.Image.objects.count() == 1
legacy.refresh_from_db()
- # The existing record's filepath is left untouched — the moved copy is
- # not created as a duplicate, but it also does not relocate the record.
- assert legacy.filepath == "/old/location/XS107114.JPG"
+ # The record follows its file: its old path no longer exists, so leaving
+ # it behind would strand the record where nothing can be served from.
+ assert legacy.filepath == new_path
assert legacy.content_hash == queries.compute_content_hash(image_path=new_path)
def test_exif_bridge_does_not_merge_an_original_with_its_edited_copy(self, tmp_path):
diff --git a/tests/integration/domain/images/test_process_image_relocation.py b/tests/integration/domain/images/test_process_image_relocation.py
new file mode 100644
index 0000000..a676bf3
--- /dev/null
+++ b/tests/integration/domain/images/test_process_image_relocation.py
@@ -0,0 +1,108 @@
+import shutil
+from pathlib import Path
+
+import pytest
+
+from src.data import models
+from src.domain.images import events
+from src.domain.images.operations import process_image
+from tests.factories import ImageFactory
+
+FIXTURES_DIR = Path(__file__).resolve().parent.parent.parent.parent / "fixtures" / "images"
+FIXTURE_IMAGE = str(FIXTURES_DIR / "XS107114.JPG")
+
+
+def _copy_fixture(*, destination: Path) -> str:
+ """Copy XS107114.JPG to *destination*, creating parent folders, and return its path."""
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy(FIXTURE_IMAGE, destination)
+ return str(destination)
+
+
+@pytest.mark.django_db
+class TestProcessImageRelocatesMovedFiles:
+ def test_a_moved_file_keeps_its_record(self, tmp_path):
+ old_path = _copy_fixture(destination=tmp_path / "inbox" / "XS107114.JPG")
+ original = process_image(image_path=old_path)
+
+ new_path = str(tmp_path / "2025" / "XS107114.JPG")
+ Path(new_path).parent.mkdir()
+ shutil.move(old_path, new_path)
+
+ moved = process_image(image_path=new_path)
+
+ assert moved.pk == original.pk
+ assert models.Image.objects.count() == 1
+ assert moved.filepath == new_path
+
+ def test_a_renamed_file_keeps_its_record_and_gains_the_new_filename(self, tmp_path):
+ old_path = _copy_fixture(destination=tmp_path / "XS107114.JPG")
+ original = process_image(image_path=old_path)
+
+ new_path = str(tmp_path / "spain-2025.JPG")
+ shutil.move(old_path, new_path)
+
+ renamed = process_image(image_path=new_path)
+
+ assert renamed.pk == original.pk
+ assert renamed.filepath == new_path
+ assert renamed.filename == "spain-2025.JPG"
+
+ def test_a_moved_file_keeps_its_rating_and_favourite(self, tmp_path):
+ old_path = _copy_fixture(destination=tmp_path / "inbox" / "XS107114.JPG")
+ original = process_image(image_path=old_path)
+ original.set_rating(5)
+ original.set_as_favorite()
+ original.set_as_in_album()
+
+ new_path = str(tmp_path / "keepers" / "XS107114.JPG")
+ Path(new_path).parent.mkdir()
+ shutil.move(old_path, new_path)
+
+ moved = process_image(image_path=new_path)
+
+ assert moved.pk == original.pk
+ assert moved.rating == 5
+ assert moved.is_favorite is True
+ assert moved.in_album is True
+
+ def test_a_copy_alongside_the_original_leaves_the_record_where_it_is(self, tmp_path):
+ original_path = _copy_fixture(destination=tmp_path / "XS107114.JPG")
+ original = process_image(image_path=original_path)
+
+ copy_path = _copy_fixture(destination=tmp_path / "backup" / "XS107114.JPG")
+ result = process_image(image_path=copy_path)
+
+ assert result.pk == original.pk
+ assert models.Image.objects.count() == 1
+ assert result.filepath == original_path
+
+ def test_publishes_image_file_relocated_for_a_move(self, tmp_path, captured_logs):
+ old_path = _copy_fixture(destination=tmp_path / "XS107114.JPG")
+ process_image(image_path=old_path)
+
+ new_path = str(tmp_path / "moved.JPG")
+ shutil.move(old_path, new_path)
+ process_image(image_path=new_path)
+
+ matching = [e for e in captured_logs if e.get("event_type") == events.IMAGE_FILE_RELOCATED]
+ assert len(matching) == 1
+ assert matching[0]["old_filepath"] == old_path
+ assert matching[0]["new_filepath"] == new_path
+
+ def test_does_not_relocate_onto_a_path_another_record_already_holds(self, tmp_path):
+ old_path = _copy_fixture(destination=tmp_path / "XS107114.JPG")
+ original = process_image(image_path=old_path)
+
+ # A legacy record already sits at the destination. Relocating onto it
+ # would violate the unique filepath constraint.
+ new_path = str(tmp_path / "occupied.JPG")
+ squatter = ImageFactory(filepath=new_path, content_hash="")
+ shutil.move(old_path, new_path)
+
+ result = process_image(image_path=new_path)
+
+ assert result.pk == original.pk
+ assert result.filepath == old_path
+ squatter.refresh_from_db()
+ assert squatter.filepath == new_path
diff --git a/tests/integration/domain/images/test_relocate_image.py b/tests/integration/domain/images/test_relocate_image.py
new file mode 100644
index 0000000..e068716
--- /dev/null
+++ b/tests/integration/domain/images/test_relocate_image.py
@@ -0,0 +1,99 @@
+import pytest
+
+from src.domain.images import events
+from src.domain.images.operations import relocate_image
+from tests.factories import ImageFactory
+
+
+@pytest.mark.django_db
+class TestRelocateImage:
+ def test_repoints_a_record_whose_file_has_moved(self, tmp_path):
+ new_file = tmp_path / "2024" / "DSCF0002.JPG"
+ new_file.parent.mkdir()
+ new_file.write_bytes(b"\xff\xd8")
+ image = ImageFactory(filepath=str(tmp_path / "DSCF0001.JPG"), filename="DSCF0001.JPG")
+
+ assert relocate_image(image=image, new_path=str(new_file)) is True
+
+ image.refresh_from_db()
+ assert image.filepath == str(new_file)
+ assert image.filename == "DSCF0002.JPG"
+
+ def test_keeps_user_data_when_the_record_moves(self, tmp_path):
+ new_file = tmp_path / "DSCF0002.JPG"
+ new_file.write_bytes(b"\xff\xd8")
+ image = ImageFactory(
+ filepath=str(tmp_path / "DSCF0001.JPG"),
+ rating=4,
+ is_favorite=True,
+ in_album=True,
+ )
+ original_pk = image.pk
+
+ relocate_image(image=image, new_path=str(new_file))
+
+ image.refresh_from_db()
+ assert image.pk == original_pk
+ assert image.rating == 4
+ assert image.is_favorite is True
+ assert image.in_album is True
+
+ def test_leaves_the_record_alone_when_the_old_file_still_exists(self, tmp_path):
+ original = tmp_path / "DSCF0001.JPG"
+ original.write_bytes(b"\xff\xd8")
+ copy = tmp_path / "DSCF0001_copy.JPG"
+ copy.write_bytes(b"\xff\xd8")
+ image = ImageFactory(filepath=str(original))
+
+ assert relocate_image(image=image, new_path=str(copy)) is False
+
+ image.refresh_from_db()
+ assert image.filepath == str(original)
+
+ def test_leaves_the_record_alone_when_the_path_is_unchanged(self, tmp_path):
+ path = tmp_path / "DSCF0001.JPG"
+ image = ImageFactory(filepath=str(path))
+
+ assert relocate_image(image=image, new_path=str(path)) is False
+
+ def test_leaves_the_record_alone_when_another_record_holds_the_new_path(self, tmp_path):
+ new_path = str(tmp_path / "DSCF0002.JPG")
+ ImageFactory(filepath=new_path)
+ image = ImageFactory(filepath=str(tmp_path / "DSCF0001.JPG"))
+
+ assert relocate_image(image=image, new_path=new_path) is False
+
+ image.refresh_from_db()
+ assert image.filepath == str(tmp_path / "DSCF0001.JPG")
+
+ def test_never_touches_either_file_on_disk(self, tmp_path):
+ new_file = tmp_path / "DSCF0002.JPG"
+ new_file.write_bytes(b"\xff\xd8")
+ image = ImageFactory(filepath=str(tmp_path / "DSCF0001.JPG"))
+
+ relocate_image(image=image, new_path=str(new_file))
+
+ assert new_file.read_bytes() == b"\xff\xd8"
+
+ def test_publishes_image_file_relocated(self, tmp_path, captured_logs):
+ old_path = str(tmp_path / "DSCF0001.JPG")
+ new_file = tmp_path / "DSCF0002.JPG"
+ new_file.write_bytes(b"\xff\xd8")
+ image = ImageFactory(filepath=old_path)
+
+ relocate_image(image=image, new_path=str(new_file))
+
+ matching = [e for e in captured_logs if e.get("event_type") == events.IMAGE_FILE_RELOCATED]
+ assert len(matching) == 1
+ assert matching[0]["image_id"] == image.pk
+ assert matching[0]["old_filepath"] == old_path
+ assert matching[0]["new_filepath"] == str(new_file)
+
+ def test_publishes_nothing_when_it_does_not_relocate(self, tmp_path, captured_logs):
+ original = tmp_path / "DSCF0001.JPG"
+ original.write_bytes(b"\xff\xd8")
+ image = ImageFactory(filepath=str(original))
+
+ relocate_image(image=image, new_path=str(tmp_path / "DSCF0002.JPG"))
+
+ assert [e for e in captured_logs if e.get("event_type") == events.IMAGE_FILE_RELOCATED] == []
From 24baf6b6a0f0932b81bc0967ec34096af711271d Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:38:47 +1000
Subject: [PATCH 09/36] Add a remove_image domain operation
Nothing calls it yet. Removes a catalog entry and garbage-collects the
FujifilmExif row it orphans, mirroring the tail of merge_image_into.
The image file on disk is never touched; only the derived thumbnail cache
is cleared, and that is for correctness rather than tidiness: cache keys
come from the path, and Fujifilm filenames wrap around from DSCF9999 to
DSCF0001, so a later file reusing a path would be served the previous
image's thumbnail.
The recipe is never deleted, since recipes are shared and outlive their
images. Cover-image and recipe-card references are left to the schema's
SET_NULL: a recipe with no explicit cover already falls back to its
most-used image, and a card is a rendered JPEG that stands on its own.
Co-Authored-By: Claude Opus 5
---
src/domain/images/events.py | 6 +
src/domain/images/operations.py | 38 +++++++
.../domain/images/test_remove_image.py | 103 ++++++++++++++++++
3 files changed, 147 insertions(+)
create mode 100644 tests/integration/domain/images/test_remove_image.py
diff --git a/src/domain/images/events.py b/src/domain/images/events.py
index b28ab15..8784722 100644
--- a/src/domain/images/events.py
+++ b/src/domain/images/events.py
@@ -24,6 +24,7 @@
IMAGE_DEDUP_FILE_MISSING = "image.dedup.file.missing"
IMAGE_IMPORT_SKIPPED = "image.import.skipped"
IMAGE_FILE_RELOCATED = "image.file.relocated"
+IMAGE_REMOVED = "image.removed"
TASK_IMAGE_ENQUEUED = "task.image.enqueued"
TASK_IMAGE_STARTED = "task.image.started"
TASK_IMAGE_COMPLETED = "task.image.completed"
@@ -32,6 +33,11 @@
SKIP_REASON_NO_FILM_SIMULATION = "no_film_simulation"
SKIP_REASON_INVALID_RECIPE_DATA = "invalid_recipe_data"
+# Values carried on the `reason` field of IMAGE_REMOVED. Both mean the catalog
+# entry was removed; neither involves deleting the file from disk.
+REMOVE_REASON_FILE_MISSING = "file_missing"
+REMOVE_REASON_FOLDER_REMOVED = "folder_removed"
+
def publish_event(*, event_type: str, **kwargs: object) -> None:
"""
diff --git a/src/domain/images/operations.py b/src/domain/images/operations.py
index 1474151..b0e1c33 100644
--- a/src/domain/images/operations.py
+++ b/src/domain/images/operations.py
@@ -1,5 +1,6 @@
import os
from collections.abc import Sequence
+from pathlib import Path
import attrs
from django import conf
@@ -8,6 +9,7 @@
from src.data import models
from src.domain.images import events, queries
from src.domain.images.queries import NoFilmSimulationError as NoFilmSimulationError
+from src.domain.images.thumbnails import operations as thumbnail_operations
from src.domain.recipes import operations as recipe_operations
@@ -145,6 +147,42 @@ def relocate_image(*, image: models.Image, new_path: str) -> bool:
return True
+@transaction.atomic(durable=True)
+def remove_image(*, image: models.Image, reason: str) -> None:
+ """
+ Remove *image* from the catalog because its file is gone or its library
+ folder was removed.
+
+ Removes the catalog entry only: the image file on disk is never touched.
+
+ Recipe-card and cover-image references are set to null by the schema and are
+ deliberately not repointed. A card is a self-contained rendered JPEG that
+ outlives its source image, and a recipe with no explicit cover already falls
+ back to its most-used image. The image's FujifilmExif row is deleted if this
+ leaves it orphaned. The FujifilmRecipe is never touched, because recipes are
+ shared and worth keeping even with no images left.
+ """
+ image_id = image.pk
+ filepath = image.filepath
+ exif_id = image.fujifilm_exif_id
+
+ image.delete()
+
+ if exif_id is not None and not models.Image.objects.filter(fujifilm_exif_id=exif_id).exists():
+ models.FujifilmExif.objects.filter(pk=exif_id).delete()
+
+ # Durable, so the row is committed before the cache files go: a rollback must
+ # never leave a record pointing at thumbnails that have been unlinked.
+ thumbnail_operations.delete_cached_thumbnails(original_path=Path(filepath))
+
+ events.publish_event(
+ event_type=events.IMAGE_REMOVED,
+ image_id=image_id,
+ filepath=filepath,
+ reason=reason,
+ )
+
+
@transaction.atomic()
def process_image(*, image_path: str) -> models.Image:
"""
diff --git a/tests/integration/domain/images/test_remove_image.py b/tests/integration/domain/images/test_remove_image.py
new file mode 100644
index 0000000..6def56c
--- /dev/null
+++ b/tests/integration/domain/images/test_remove_image.py
@@ -0,0 +1,103 @@
+from pathlib import Path
+
+import pytest
+from django.test import override_settings
+
+from src.data import models
+from src.domain.images import events
+from src.domain.images.operations import remove_image
+from src.domain.images.thumbnails.queries import thumbnail_cache_path
+from tests.factories import (
+ FujifilmExifFactory,
+ FujifilmRecipeFactory,
+ ImageFactory,
+ RecipeCardFactory,
+)
+
+
+@pytest.mark.django_db
+class TestRemoveImage:
+ def test_removes_the_catalog_entry(self):
+ image = ImageFactory()
+
+ remove_image(image=image, reason=events.REMOVE_REASON_FILE_MISSING)
+
+ assert not models.Image.objects.filter(pk=image.pk).exists()
+
+ def test_never_deletes_the_file_on_disk(self, tmp_path):
+ photo = tmp_path / "DSCF0001.JPG"
+ photo.write_bytes(b"\xff\xd8")
+ image = ImageFactory(filepath=str(photo))
+
+ remove_image(image=image, reason=events.REMOVE_REASON_FOLDER_REMOVED)
+
+ assert photo.exists()
+ assert photo.read_bytes() == b"\xff\xd8"
+
+ def test_deletes_the_exif_row_it_leaves_orphaned(self):
+ exif = FujifilmExifFactory()
+ image = ImageFactory(fujifilm_exif=exif)
+
+ remove_image(image=image, reason=events.REMOVE_REASON_FILE_MISSING)
+
+ assert not models.FujifilmExif.objects.filter(pk=exif.pk).exists()
+
+ def test_keeps_an_exif_row_another_image_still_uses(self):
+ exif = FujifilmExifFactory()
+ image = ImageFactory(fujifilm_exif=exif)
+ ImageFactory(fujifilm_exif=exif)
+
+ remove_image(image=image, reason=events.REMOVE_REASON_FILE_MISSING)
+
+ assert models.FujifilmExif.objects.filter(pk=exif.pk).exists()
+
+ def test_keeps_the_recipe_even_when_no_image_is_left(self):
+ recipe = FujifilmRecipeFactory()
+ image = ImageFactory(fujifilm_recipe=recipe)
+
+ remove_image(image=image, reason=events.REMOVE_REASON_FILE_MISSING)
+
+ assert models.FujifilmRecipe.objects.filter(pk=recipe.pk).exists()
+
+ def test_clears_the_recipe_cover_it_pointed_at(self):
+ recipe = FujifilmRecipeFactory()
+ image = ImageFactory(fujifilm_recipe=recipe)
+ recipe.cover_image = image
+ recipe.save(update_fields=["cover_image"])
+
+ remove_image(image=image, reason=events.REMOVE_REASON_FILE_MISSING)
+
+ recipe.refresh_from_db()
+ assert recipe.cover_image_id is None
+
+ def test_keeps_recipe_cards_that_referenced_it(self):
+ image = ImageFactory()
+ card = RecipeCardFactory(image=image)
+
+ remove_image(image=image, reason=events.REMOVE_REASON_FILE_MISSING)
+
+ card.refresh_from_db()
+ assert card.image_id is None
+
+ def test_deletes_the_cached_thumbnails(self, tmp_path):
+ image = ImageFactory(filepath="/photos/DSCF0001.JPG")
+
+ with override_settings(THUMBNAIL_CACHE_DIR=tmp_path, THUMBNAIL_WIDTHS=(600,)):
+ cache_path = thumbnail_cache_path(original_path=Path(image.filepath), width=600)
+ cache_path.write_bytes(b"\xff\xd8")
+
+ remove_image(image=image, reason=events.REMOVE_REASON_FILE_MISSING)
+
+ assert not cache_path.exists()
+
+ def test_publishes_image_removed_with_the_reason(self, captured_logs):
+ image = ImageFactory()
+ image_id, filepath = image.pk, image.filepath
+
+ remove_image(image=image, reason=events.REMOVE_REASON_FOLDER_REMOVED)
+
+ matching = [e for e in captured_logs if e.get("event_type") == events.IMAGE_REMOVED]
+ assert len(matching) == 1
+ assert matching[0]["image_id"] == image_id
+ assert matching[0]["filepath"] == filepath
+ assert matching[0]["reason"] == events.REMOVE_REASON_FOLDER_REMOVED
From b0cd593eb9f26e45341ba6b21e257494b3fa2ce9 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:41:04 +1000
Subject: [PATCH 10/36] Retire mtime-based directory gating from the library
scan
The scan skipped directories whose mtime predated the folder's
last_checked_at. That cannot survive alongside removal: renaming a
directory updates its parent's mtime and never its own, so a renamed
subtree keeps an old timestamp, last_checked_at only moves forward, and a
gated walk never looks at it again. Every image under a renamed folder
would be treated as deleted and never found.
The gate bought little anyway. os.walk has already listed each directory
by the time the check runs, so it added a getmtime() per directory and
saved only a filename suffix check; the costly work is already avoided by
diffing against the known catalog paths.
last_checked_at stays, since the Library page shows it and it is the only
evidence a sync ran when nothing changed. Only its gating role goes, which
also makes clearing it on a path update pointless.
Co-Authored-By: Claude Opus 5
---
.../usecases/library/sync_folder.py | 8 +-
src/data/models/_library.py | 4 -
src/domain/images/queries.py | 30 ++----
src/domain/library/operations.py | 5 -
.../domain/images/test_collect_image_paths.py | 97 +++++++++++++++++++
.../images/test_get_image_paths_in_folder.py | 62 ------------
.../domain/library/test_operations.py | 13 ---
tests/unit/domain/images/test_queries.py | 57 +----------
8 files changed, 109 insertions(+), 167 deletions(-)
create mode 100644 tests/integration/domain/images/test_collect_image_paths.py
delete mode 100644 tests/integration/domain/images/test_get_image_paths_in_folder.py
diff --git a/src/application/usecases/library/sync_folder.py b/src/application/usecases/library/sync_folder.py
index 2a48277..69a5b42 100644
--- a/src/application/usecases/library/sync_folder.py
+++ b/src/application/usecases/library/sync_folder.py
@@ -16,8 +16,7 @@ def sync_folder(*, folder_id: int) -> None:
Scan a single library folder and import new images, tracking progress in a
SyncRun.
- Creates a run, walks the folder (mtime-gated by its last_checked_at), and
- dispatches each new image: in async mode by enqueuing a Celery task, in sync
+ Creates a run, walks the whole folder, and dispatches each new image: in async mode by enqueuing a Celery task, in sync
mode by processing inline. The run is completed here only when nothing new is
found; otherwise the last processed image completes it.
@@ -39,10 +38,7 @@ def sync_folder(*, folder_id: int) -> None:
now = datetime.now(tz=timezone.utc)
try:
- found_paths = image_queries.get_image_paths_in_folder(
- folder_path=folder.path,
- last_checked_at=folder.last_checked_at,
- )
+ found_paths = image_queries.collect_image_paths(folder=folder.path)
except FileNotFoundError:
folder.set_last_checked_at(value=now)
library_operations.fail_sync_run(run=run, message="Folder does not exist")
diff --git a/src/data/models/_library.py b/src/data/models/_library.py
index 64ce591..9004285 100644
--- a/src/data/models/_library.py
+++ b/src/data/models/_library.py
@@ -41,9 +41,5 @@ def set_last_checked_at(self, *, value: datetime) -> None:
self.last_checked_at = value
self.save(update_fields=["last_checked_at", "updated_at"])
- def clear_last_checked_at(self) -> None:
- self.last_checked_at = None
- self.save(update_fields=["last_checked_at", "updated_at"])
-
def __str__(self) -> str:
return f"#{self.id} {self.path}"
diff --git a/src/domain/images/queries.py b/src/domain/images/queries.py
index d8705a7..5ec9479 100644
--- a/src/domain/images/queries.py
+++ b/src/domain/images/queries.py
@@ -433,14 +433,18 @@ def find_image_by_content_hash(*, content_hash: str) -> models.Image | None:
)
-def collect_image_paths(*, folder: str, last_checked_at: datetime | None = None) -> list[str]:
+def collect_image_paths(*, folder: str) -> list[str]:
"""
Return absolute paths of all JPG files inside *folder* (recursively).
- When *last_checked_at* is provided, directories whose mtime is older than
- that timestamp are skipped. Their subdirectories are still walked because
- only the immediate parent's mtime updates when a file is added deeper in
- the tree.
+ The whole tree is walked every time. Directory mtimes are deliberately not
+ used to skip anything: renaming a directory updates its parent's mtime and
+ not its own, so a gated walk never revisits a renamed subtree, which would
+ leave every image under it pointing at a path that no longer exists. The
+ expensive part of an import (reading EXIF and hashing) is already avoided by
+ diffing against the known catalog paths, so the walk costs little.
+
+ :raises FileNotFoundError: If *folder* does not exist or is not a directory.
"""
root = Path(folder)
if not root.is_dir():
@@ -449,10 +453,6 @@ def collect_image_paths(*, folder: str, last_checked_at: datetime | None = None)
extensions = {".jpg", ".jpeg"}
paths: list[str] = []
for dirpath, _dirnames, filenames in os.walk(root):
- if last_checked_at is not None:
- dir_mtime = datetime.fromtimestamp(os.path.getmtime(dirpath), tz=timezone.utc)
- if dir_mtime <= last_checked_at:
- continue
for fname in filenames:
if Path(fname).suffix.lower() in extensions:
paths.append(os.path.join(dirpath, fname))
@@ -468,18 +468,6 @@ def get_all_known_image_paths() -> frozenset[str]:
return frozenset(models.Image.objects.values_list("filepath", flat=True))
-def get_image_paths_in_folder(*, folder_path: str, last_checked_at: datetime | None = None) -> list[str]:
- """
- Return absolute paths of all JPG/JPEG files inside *folder_path* (recursively).
-
- When *last_checked_at* is provided, directories unchanged since that
- timestamp are skipped (mtime gating).
-
- :raises FileNotFoundError: If *folder_path* does not exist or is not a directory.
- """
- return collect_image_paths(folder=folder_path, last_checked_at=last_checked_at)
-
-
@attrs.frozen
class ImageDetailContext:
image: models.Image
diff --git a/src/domain/library/operations.py b/src/domain/library/operations.py
index fd3ff5e..c202584 100644
--- a/src/domain/library/operations.py
+++ b/src/domain/library/operations.py
@@ -101,11 +101,6 @@ def update_library_folder_path(*, folder_id: int, path: str) -> models.LibraryFo
except IntegrityError:
raise FolderAlreadyInLibrary(path=normalized)
- # The new path is a different tree, so the previous scan timestamp no longer
- # applies. Clearing it forces a full rescan (mtime gating would otherwise skip
- # directories older than the old last_checked_at).
- folder.clear_last_checked_at()
-
events.publish_event(
event_type=events.LIBRARY_FOLDER_PATH_UPDATED,
folder_id=folder.pk,
diff --git a/tests/integration/domain/images/test_collect_image_paths.py b/tests/integration/domain/images/test_collect_image_paths.py
new file mode 100644
index 0000000..97c2625
--- /dev/null
+++ b/tests/integration/domain/images/test_collect_image_paths.py
@@ -0,0 +1,97 @@
+import os
+import time
+
+import pytest
+
+from src.domain.images.queries import collect_image_paths
+
+
+class TestCollectImagePaths:
+ def test_returns_jpg_files(self, tmp_path):
+ (tmp_path / "photo1.jpg").write_bytes(b"\xff\xd8")
+ (tmp_path / "photo2.JPG").write_bytes(b"\xff\xd8")
+ (tmp_path / "photo3.jpeg").write_bytes(b"\xff\xd8")
+ (tmp_path / "document.pdf").write_bytes(b"%PDF")
+
+ paths = collect_image_paths(folder=str(tmp_path))
+
+ filenames = [os.path.basename(p) for p in paths]
+ assert "photo1.jpg" in filenames
+ assert "photo2.JPG" in filenames
+ assert "photo3.jpeg" in filenames
+ assert "document.pdf" not in filenames
+
+ def test_excludes_non_jpeg_files(self, tmp_path):
+ (tmp_path / "photo.jpg").touch()
+ (tmp_path / "photo.png").touch()
+ (tmp_path / "document.txt").touch()
+
+ paths = collect_image_paths(folder=str(tmp_path))
+
+ assert paths == [str(tmp_path / "photo.jpg")]
+
+ def test_returns_sorted_paths(self, tmp_path):
+ (tmp_path / "c.jpg").write_bytes(b"\xff\xd8")
+ (tmp_path / "a.jpg").write_bytes(b"\xff\xd8")
+ (tmp_path / "b.jpg").write_bytes(b"\xff\xd8")
+
+ paths = collect_image_paths(folder=str(tmp_path))
+
+ filenames = [os.path.basename(p) for p in paths]
+ assert filenames == sorted(filenames)
+
+ def test_finds_files_recursively(self, tmp_path):
+ sub = tmp_path / "sub"
+ sub.mkdir()
+ (tmp_path / "top.jpg").write_bytes(b"\xff\xd8")
+ (sub / "nested.jpg").write_bytes(b"\xff\xd8")
+
+ paths = collect_image_paths(folder=str(tmp_path))
+
+ filenames = [os.path.basename(p) for p in paths]
+ assert "top.jpg" in filenames
+ assert "nested.jpg" in filenames
+
+ def test_returns_absolute_paths(self, tmp_path):
+ (tmp_path / "photo.jpg").write_bytes(b"\xff\xd8")
+
+ paths = collect_image_paths(folder=str(tmp_path))
+
+ for p in paths:
+ assert os.path.isabs(p)
+
+ def test_empty_folder_returns_empty_list(self, tmp_path):
+ paths = collect_image_paths(folder=str(tmp_path))
+
+ assert paths == []
+
+ def test_nonexistent_folder_raises(self):
+ with pytest.raises(FileNotFoundError):
+ collect_image_paths(folder="/nonexistent/folder")
+
+ def test_returns_files_from_a_directory_untouched_for_hours(self, tmp_path):
+ # The walk used to skip directories whose mtime predated the folder's
+ # last_checked_at. Nothing may depend on directory mtimes any more.
+ (tmp_path / "photo.jpg").write_bytes(b"\xff\xd8")
+ long_ago = time.time() - 86400
+ os.utime(tmp_path, (long_ago, long_ago))
+
+ paths = collect_image_paths(folder=str(tmp_path))
+
+ assert paths == [str(tmp_path / "photo.jpg")]
+
+ def test_returns_files_from_a_renamed_subdirectory(self, tmp_path):
+ # Renaming a directory updates its parent's mtime, never its own, so a
+ # walk that skipped unchanged directories would miss the whole subtree
+ # for good. Every image under it would then look deleted.
+ original = tmp_path / "2024"
+ original.mkdir()
+ (original / "photo.jpg").write_bytes(b"\xff\xd8")
+ long_ago = time.time() - 86400
+ os.utime(original, (long_ago, long_ago))
+ renamed = tmp_path / "2024-trip"
+ original.rename(renamed)
+
+ paths = collect_image_paths(folder=str(tmp_path))
+
+ assert paths == [str(renamed / "photo.jpg")]
diff --git a/tests/integration/domain/images/test_get_image_paths_in_folder.py b/tests/integration/domain/images/test_get_image_paths_in_folder.py
deleted file mode 100644
index 33e542b..0000000
--- a/tests/integration/domain/images/test_get_image_paths_in_folder.py
+++ /dev/null
@@ -1,62 +0,0 @@
-from datetime import datetime, timedelta, timezone
-
-import pytest
-
-from src.domain.images.queries import get_image_paths_in_folder
-
-
-class TestGetImagePathsInFolder:
- def test_returns_jpeg_files_in_folder(self, tmp_path):
- (tmp_path / "photo.jpg").touch()
- result = get_image_paths_in_folder(folder_path=str(tmp_path))
- assert result == [str(tmp_path / "photo.jpg")]
-
- def test_returns_jpeg_files_recursively_from_subdirectories(self, tmp_path):
- subdir = tmp_path / "2024"
- subdir.mkdir()
- (subdir / "photo.jpg").touch()
- result = get_image_paths_in_folder(folder_path=str(tmp_path))
- assert result == [str(subdir / "photo.jpg")]
-
- def test_excludes_non_jpeg_files(self, tmp_path):
- (tmp_path / "photo.jpg").touch()
- (tmp_path / "photo.png").touch()
- (tmp_path / "document.txt").touch()
- result = get_image_paths_in_folder(folder_path=str(tmp_path))
- assert result == [str(tmp_path / "photo.jpg")]
-
- def test_raises_file_not_found_for_nonexistent_path(self, tmp_path):
- missing = str(tmp_path / "does_not_exist")
- with pytest.raises(FileNotFoundError):
- get_image_paths_in_folder(folder_path=missing)
-
- def test_skips_directory_unchanged_since_last_checked_at(self, tmp_path):
- (tmp_path / "photo.jpg").touch()
- # A last_checked_at in the future means the directory's mtime is older than it
- future = datetime.now(tz=timezone.utc) + timedelta(hours=1)
- result = get_image_paths_in_folder(folder_path=str(tmp_path), last_checked_at=future)
- assert result == []
-
- def test_includes_directory_changed_after_last_checked_at(self, tmp_path):
- (tmp_path / "photo.jpg").touch()
- # A last_checked_at in the past means the directory's mtime is newer than it
- past = datetime.now(tz=timezone.utc) - timedelta(hours=1)
- result = get_image_paths_in_folder(folder_path=str(tmp_path), last_checked_at=past)
- assert result == [str(tmp_path / "photo.jpg")]
-
- def test_unchanged_parent_directory_does_not_block_changed_subdirectory(self, tmp_path):
- subdir = tmp_path / "2024"
- subdir.mkdir()
- (subdir / "photo.jpg").touch()
- # Set last_checked_at to between the parent dir creation and now.
- # Parent dir mtime predates last_checked_at (created before subdir touch),
- # but subdir mtime is after last_checked_at.
- import os, time
- past_mtime = time.time() - 3600
- os.utime(tmp_path, (past_mtime, past_mtime))
- last_checked = datetime.fromtimestamp(past_mtime + 1, tz=timezone.utc)
-
- result = get_image_paths_in_folder(folder_path=str(tmp_path), last_checked_at=last_checked)
-
- # Parent dir is skipped but subdir is NOT skipped (its mtime > last_checked_at)
- assert result == [str(subdir / "photo.jpg")]
diff --git a/tests/integration/domain/library/test_operations.py b/tests/integration/domain/library/test_operations.py
index 07fe97c..7dd5866 100644
--- a/tests/integration/domain/library/test_operations.py
+++ b/tests/integration/domain/library/test_operations.py
@@ -1,5 +1,4 @@
import pytest
-from django.utils import timezone
from src.data import models
from src.domain.library import events
@@ -95,18 +94,6 @@ def test_updates_path_on_folder(self, tmp_path):
folder.refresh_from_db()
assert folder.path == str(new_dir)
- def test_resets_last_checked_at_so_new_tree_is_fully_rescanned(self, tmp_path):
- old_dir = tmp_path / "old"
- new_dir = tmp_path / "new"
- old_dir.mkdir()
- new_dir.mkdir()
-
- folder = LibraryFolderFactory(path=str(old_dir), last_checked_at=timezone.now())
- update_library_folder_path(folder_id=folder.pk, path=str(new_dir))
-
- folder.refresh_from_db()
- assert folder.last_checked_at is None
-
def test_publishes_folder_path_updated_event(self, tmp_path, captured_logs):
old_dir = tmp_path / "old"
new_dir = tmp_path / "new"
diff --git a/tests/unit/domain/images/test_queries.py b/tests/unit/domain/images/test_queries.py
index e1bc6d9..ce9ad5d 100644
--- a/tests/unit/domain/images/test_queries.py
+++ b/tests/unit/domain/images/test_queries.py
@@ -1,10 +1,9 @@
-import os
import subprocess
from unittest.mock import patch
import pytest
-from src.domain.images.queries import _normalize_wb_fine_tune, collect_image_paths, read_image_exif
+from src.domain.images.queries import _normalize_wb_fine_tune, read_image_exif
SAMPLE_EXIFTOOL_OUTPUT = """\
[ExifTool] ExifTool Version Number : 12.76
@@ -265,57 +264,3 @@ def test_handles_zero(self):
def test_handles_negative_red(self):
assert _normalize_wb_fine_tune(raw="Red -40, Blue +60") == "Red -2, Blue +3"
-
-
-class TestCollectImagePaths:
- def test_returns_jpg_files(self, tmp_path):
- (tmp_path / "photo1.jpg").write_bytes(b"\xff\xd8")
- (tmp_path / "photo2.JPG").write_bytes(b"\xff\xd8")
- (tmp_path / "photo3.jpeg").write_bytes(b"\xff\xd8")
- (tmp_path / "document.pdf").write_bytes(b"%PDF")
-
- paths = collect_image_paths(folder=str(tmp_path))
-
- filenames = [os.path.basename(p) for p in paths]
- assert "photo1.jpg" in filenames
- assert "photo2.JPG" in filenames
- assert "photo3.jpeg" in filenames
- assert "document.pdf" not in filenames
-
- def test_returns_sorted_paths(self, tmp_path):
- (tmp_path / "c.jpg").write_bytes(b"\xff\xd8")
- (tmp_path / "a.jpg").write_bytes(b"\xff\xd8")
- (tmp_path / "b.jpg").write_bytes(b"\xff\xd8")
-
- paths = collect_image_paths(folder=str(tmp_path))
-
- filenames = [os.path.basename(p) for p in paths]
- assert filenames == sorted(filenames)
-
- def test_finds_files_recursively(self, tmp_path):
- sub = tmp_path / "sub"
- sub.mkdir()
- (tmp_path / "top.jpg").write_bytes(b"\xff\xd8")
- (sub / "nested.jpg").write_bytes(b"\xff\xd8")
-
- paths = collect_image_paths(folder=str(tmp_path))
-
- filenames = [os.path.basename(p) for p in paths]
- assert "top.jpg" in filenames
- assert "nested.jpg" in filenames
-
- def test_returns_absolute_paths(self, tmp_path):
- (tmp_path / "photo.jpg").write_bytes(b"\xff\xd8")
-
- paths = collect_image_paths(folder=str(tmp_path))
-
- for p in paths:
- assert os.path.isabs(p)
-
- def test_empty_folder_returns_empty_list(self, tmp_path):
- paths = collect_image_paths(folder=str(tmp_path))
- assert paths == []
-
- def test_nonexistent_folder_raises(self):
- with pytest.raises(FileNotFoundError):
- collect_image_paths(folder="/nonexistent/folder")
From 976f2198bb4efaaf9c5e16bae641f4aa3df629eb Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:41:52 +1000
Subject: [PATCH 11/36] Add a get_image_paths_under_folder query
Nothing calls it yet. Removal has to be scoped to one library folder, and
Image has no foreign key to LibraryFolder, so membership can only be
decided by path prefix. The separator is part of the prefix so that a
folder named /photos does not also claim /photos-old.
Scoping this way is what will keep images imported from outside the
library, by the process_images command, out of any removal.
Co-Authored-By: Claude Opus 5
---
src/domain/images/queries.py | 19 +++++++
.../test_get_image_paths_under_folder.py | 50 +++++++++++++++++++
2 files changed, 69 insertions(+)
create mode 100644 tests/integration/domain/images/test_get_image_paths_under_folder.py
diff --git a/src/domain/images/queries.py b/src/domain/images/queries.py
index 5ec9479..745c61e 100644
--- a/src/domain/images/queries.py
+++ b/src/domain/images/queries.py
@@ -468,6 +468,25 @@ def get_all_known_image_paths() -> frozenset[str]:
return frozenset(models.Image.objects.values_list("filepath", flat=True))
+def get_image_paths_under_folder(*, folder_path: str) -> frozenset[str]:
+ """
+ Return the filepath of every catalogued image stored under *folder_path*.
+
+ Membership is decided by path prefix because Image has no foreign key to
+ LibraryFolder. The separator is part of the prefix so that a folder named
+ ``/Photos`` does not also claim ``/Photos-old``.
+
+ Images imported from outside every registered library folder are never
+ returned, which is what keeps them out of any prune.
+ """
+ prefix = folder_path.rstrip(os.sep) + os.sep
+ return frozenset(
+ models.Image.objects
+ .filter(filepath__startswith=prefix)
+ .values_list("filepath", flat=True)
+ )
+
+
@attrs.frozen
class ImageDetailContext:
image: models.Image
diff --git a/tests/integration/domain/images/test_get_image_paths_under_folder.py b/tests/integration/domain/images/test_get_image_paths_under_folder.py
new file mode 100644
index 0000000..4520cb8
--- /dev/null
+++ b/tests/integration/domain/images/test_get_image_paths_under_folder.py
@@ -0,0 +1,50 @@
+import pytest
+
+from src.domain.images.queries import get_image_paths_under_folder
+from tests.factories import ImageFactory
+
+
+@pytest.mark.django_db
+class TestGetImagePathsUnderFolder:
+ def test_returns_images_stored_directly_in_the_folder(self):
+ image = ImageFactory(filepath="/photos/2024/DSCF0001.JPG")
+
+ result = get_image_paths_under_folder(folder_path="/photos")
+
+ assert result == frozenset({image.filepath})
+
+ def test_returns_images_stored_in_nested_subdirectories(self):
+ image = ImageFactory(filepath="/photos/2024/spain/DSCF0001.JPG")
+
+ result = get_image_paths_under_folder(folder_path="/photos")
+
+ assert result == frozenset({image.filepath})
+
+ def test_excludes_images_outside_the_folder(self):
+ ImageFactory(filepath="/elsewhere/DSCF0001.JPG")
+
+ result = get_image_paths_under_folder(folder_path="/photos")
+
+ assert result == frozenset()
+
+ def test_excludes_a_sibling_folder_sharing_the_name_as_a_prefix(self):
+ # Without the separator in the prefix, "/photos" would also claim these.
+ ImageFactory(filepath="/photos-old/DSCF0001.JPG")
+
+ result = get_image_paths_under_folder(folder_path="/photos")
+
+ assert result == frozenset()
+
+ def test_tolerates_a_folder_path_with_a_trailing_separator(self):
+ image = ImageFactory(filepath="/photos/DSCF0001.JPG")
+
+ result = get_image_paths_under_folder(folder_path="/photos/")
+
+ assert result == frozenset({image.filepath})
+
+ def test_returns_an_empty_set_for_a_folder_with_no_images(self):
+ ImageFactory(filepath="/elsewhere/DSCF0001.JPG")
+
+ result = get_image_paths_under_folder(folder_path="/photos")
+
+ assert result == frozenset()
From 780d258f151b9a035b7ca6507468abc48a724ac3 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:43:56 +1000
Subject: [PATCH 12/36] Add a prune_missing_images domain operation
Nothing calls it yet. Removes catalog entries for images under one library
folder whose files are gone, and nothing else: no image file is ever
deleted from disk.
Candidates come from the difference between the catalogued paths and a
fresh walk, then each is confirmed with a stat. The walk alone would be
wrong, because it does not follow symlinked directories, silently yields
nothing for a directory it cannot read, and matches only JPEG extensions,
so anything it cannot see would look deleted. Removal is permanent, so the
filesystem gets the final say. lexists rather than exists, because a
broken symlink still occupies the path and the conservative reading has to
win.
prune_guard_trips stops a pass that would take most of a folder at once,
which is far more often an unmounted drive or an unreadable directory than
a real deletion. Both thresholds must be exceeded, so ordinary cleanups
are applied without a warning.
Nothing is pruned when the folder itself is absent, since an unplugged
drive must not empty the gallery.
Co-Authored-By: Claude Opus 5
---
src/domain/library/events.py | 4 +
src/domain/library/operations.py | 143 ++++++++++++++-
.../library/test_prune_missing_images.py | 172 ++++++++++++++++++
tests/unit/domain/library/test_prune_guard.py | 53 ++++++
4 files changed, 371 insertions(+), 1 deletion(-)
create mode 100644 tests/integration/domain/library/test_prune_missing_images.py
create mode 100644 tests/unit/domain/library/test_prune_guard.py
diff --git a/src/domain/library/events.py b/src/domain/library/events.py
index 7888150..f95e355 100644
--- a/src/domain/library/events.py
+++ b/src/domain/library/events.py
@@ -7,10 +7,14 @@
LIBRARY_FOLDER_ADDED = "library.folder.added"
LIBRARY_FOLDER_REMOVED = "library.folder.removed"
LIBRARY_FOLDER_PATH_UPDATED = "library.folder.path.updated"
+LIBRARY_FOLDER_IMAGES_REMOVED = "library.folder.images.removed"
LIBRARY_SYNC_RUN_STARTED = "library.sync.run.started"
LIBRARY_SYNC_RUN_COMPLETED = "library.sync.run.completed"
LIBRARY_SYNC_RUN_FAILED = "library.sync.run.failed"
LIBRARY_SYNC_RUN_INTERRUPTED = "library.sync.run.interrupted"
+LIBRARY_SYNC_PRUNE_STARTED = "library.sync.prune.started"
+LIBRARY_SYNC_PRUNE_COMPLETED = "library.sync.prune.completed"
+LIBRARY_SYNC_PRUNE_SKIPPED = "library.sync.prune.skipped"
def publish_event(*, event_type: str, **kwargs: object) -> None:
diff --git a/src/domain/library/operations.py b/src/domain/library/operations.py
index c202584..0e02e4a 100644
--- a/src/domain/library/operations.py
+++ b/src/domain/library/operations.py
@@ -1,13 +1,22 @@
-import attrs
+import os
from pathlib import Path
+import attrs
+from django import conf
from django.db import IntegrityError, transaction
from django.utils import timezone
from src.data import models
+from src.domain.images import events as image_events
+from src.domain.images import operations as image_operations
+from src.domain.images import queries as image_queries
from src.domain.library import events
from src.domain.library.queries import FolderNotFound, LibraryFolderNotFound
+# How many missing paths a prune reports back for a dry run, so the caller can
+# show what would go without echoing an unbounded list.
+_PRUNE_SAMPLE_LIMIT = 20
+
@attrs.frozen
class FolderAlreadyInLibrary(Exception):
@@ -27,6 +36,22 @@ class SyncAlreadyInProgress(Exception):
folder_id: int
+@attrs.frozen
+class PruneResult:
+ """
+ Outcome of a prune pass over one library folder.
+
+ ``skipped_reason`` is empty when the prune ran; otherwise it carries the
+ SyncRun.SKIPPED_* code explaining why nothing was removed.
+ """
+
+ missing_found: int
+ removed: int
+ total: int
+ skipped_reason: str
+ sample_paths: tuple[str, ...]
+
+
def _normalize_path(path: str) -> str:
return str(Path(path).expanduser().resolve())
@@ -109,6 +134,122 @@ def update_library_folder_path(*, folder_id: int, path: str) -> models.LibraryFo
return folder
+def prune_guard_trips(*, missing: int, total: int) -> bool:
+ """
+ Return True when removing *missing* of *total* images looks like a mass wipe
+ rather than a deliberate cleanup.
+
+ Both thresholds must be exceeded, so a small folder losing all its photos and
+ a large folder losing a handful are both applied without complaint. What the
+ guard is there to catch is a drive that is mounted but empty, or a directory
+ that has become unreadable, where nearly everything looks gone at once.
+ """
+ if total <= 0:
+ return False
+ if missing <= conf.settings.LIBRARY_PRUNE_GUARD_MIN_IMAGES:
+ return False
+ return missing / total > conf.settings.LIBRARY_PRUNE_GUARD_FRACTION
+
+
+def prune_missing_images(*, folder: models.LibraryFolder, mode: str) -> PruneResult:
+ """
+ Remove catalog entries for images under *folder* whose files are gone.
+
+ Removes catalog entries only: no image file is ever deleted from disk.
+
+ Only paths under this folder are considered, so images imported from outside
+ the library can never be pruned. Nothing is pruned when the folder itself is
+ not on disk, because an unplugged drive must not empty the gallery.
+
+ Candidates come from the difference between the catalogued paths and the
+ files found by a fresh walk, then each candidate is confirmed with a stat.
+ The walk alone is not enough: it does not follow symlinked directories, it
+ silently yields nothing for a directory it cannot read, and it only matches
+ JPEG extensions, so anything it misses would otherwise look deleted.
+ """
+ if not Path(folder.path).is_dir():
+ return PruneResult(
+ missing_found=0,
+ removed=0,
+ total=0,
+ skipped_reason=models.SyncRun.SKIPPED_FOLDER_MISSING,
+ sample_paths=(),
+ )
+
+ if mode == models.SyncRun.PRUNE_MODE_OFF:
+ return PruneResult(
+ missing_found=0,
+ removed=0,
+ total=0,
+ skipped_reason=models.SyncRun.SKIPPED_OFF,
+ sample_paths=(),
+ )
+
+ known = image_queries.get_image_paths_under_folder(folder_path=folder.path)
+ found = set(image_queries.collect_image_paths(folder=folder.path))
+
+ # os.path.lexists, not exists: a broken symlink still occupies the path, and
+ # for a destructive step "something is there" has to mean "keep the record".
+ missing = sorted(path for path in known - found if not os.path.lexists(path))
+ sample = tuple(missing[:_PRUNE_SAMPLE_LIMIT])
+
+ if mode == models.SyncRun.PRUNE_MODE_AUTO and prune_guard_trips(
+ missing=len(missing), total=len(known)
+ ):
+ events.publish_event(
+ event_type=events.LIBRARY_SYNC_PRUNE_SKIPPED,
+ folder_id=folder.pk,
+ missing_found=len(missing),
+ total=len(known),
+ reason=models.SyncRun.SKIPPED_GUARD,
+ )
+ return PruneResult(
+ missing_found=len(missing),
+ removed=0,
+ total=len(known),
+ skipped_reason=models.SyncRun.SKIPPED_GUARD,
+ sample_paths=sample,
+ )
+
+ if mode == models.SyncRun.PRUNE_MODE_DRY_RUN:
+ events.publish_event(
+ event_type=events.LIBRARY_SYNC_PRUNE_SKIPPED,
+ folder_id=folder.pk,
+ missing_found=len(missing),
+ total=len(known),
+ reason=models.SyncRun.SKIPPED_DRY_RUN,
+ )
+ return PruneResult(
+ missing_found=len(missing),
+ removed=0,
+ total=len(known),
+ skipped_reason=models.SyncRun.SKIPPED_DRY_RUN,
+ sample_paths=sample,
+ )
+
+ removed = 0
+ for image in models.Image.objects.filter(filepath__in=missing):
+ image_operations.remove_image(
+ image=image,
+ reason=image_events.REMOVE_REASON_FILE_MISSING,
+ )
+ removed += 1
+
+ events.publish_event(
+ event_type=events.LIBRARY_SYNC_PRUNE_COMPLETED,
+ folder_id=folder.pk,
+ missing_found=len(missing),
+ removed=removed,
+ )
+ return PruneResult(
+ missing_found=len(missing),
+ removed=removed,
+ total=len(known),
+ skipped_reason="",
+ sample_paths=sample,
+ )
+
+
def start_sync_run(*, folder: models.LibraryFolder) -> models.SyncRun:
"""
Create a new sync run for *folder* in the scanning state.
diff --git a/tests/integration/domain/library/test_prune_missing_images.py b/tests/integration/domain/library/test_prune_missing_images.py
new file mode 100644
index 0000000..33c7450
--- /dev/null
+++ b/tests/integration/domain/library/test_prune_missing_images.py
@@ -0,0 +1,172 @@
+import pytest
+
+from src.data import models
+from src.domain.library import events
+from src.domain.library.operations import prune_missing_images
+from tests.factories import ImageFactory, LibraryFolderFactory
+
+
+def _photo(*, folder, name="DSCF0001.JPG"):
+ """Create a real JPEG under *folder* and a catalog record pointing at it."""
+ path = folder / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(b"\xff\xd8")
+ return ImageFactory(filepath=str(path), filename=name), path
+
+
+@pytest.fixture(autouse=True)
+def _generous_guard(settings):
+ # Most cases here are deliberately small; the guard has its own tests.
+ settings.LIBRARY_PRUNE_GUARD_MIN_IMAGES = 1000
+
+
+@pytest.mark.django_db
+class TestPruneMissingImages:
+ def test_removes_the_record_of_a_deleted_file(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ image, path = _photo(folder=tmp_path)
+ path.unlink()
+
+ result = prune_missing_images(folder=folder, mode=models.SyncRun.PRUNE_MODE_AUTO)
+
+ assert result.removed == 1
+ assert result.missing_found == 1
+ assert result.skipped_reason == ""
+ assert not models.Image.objects.filter(pk=image.pk).exists()
+
+ def test_keeps_records_whose_files_are_present(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ image, _ = _photo(folder=tmp_path)
+
+ result = prune_missing_images(folder=folder, mode=models.SyncRun.PRUNE_MODE_AUTO)
+
+ assert result.removed == 0
+ assert models.Image.objects.filter(pk=image.pk).exists()
+
+ def test_never_touches_images_outside_the_folder(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path / "library"))
+ (tmp_path / "library").mkdir()
+ outsider = ImageFactory(filepath=str(tmp_path / "elsewhere" / "DSCF9999.JPG"))
+
+ result = prune_missing_images(folder=folder, mode=models.SyncRun.PRUNE_MODE_AUTO)
+
+ assert result.missing_found == 0
+ assert models.Image.objects.filter(pk=outsider.pk).exists()
+
+ def test_removes_nothing_when_the_folder_is_not_on_disk(self, tmp_path):
+ missing_root = tmp_path / "unplugged"
+ folder = LibraryFolderFactory(path=str(missing_root))
+ image = ImageFactory(filepath=str(missing_root / "DSCF0001.JPG"))
+
+ result = prune_missing_images(folder=folder, mode=models.SyncRun.PRUNE_MODE_AUTO)
+
+ assert result.skipped_reason == models.SyncRun.SKIPPED_FOLDER_MISSING
+ assert result.removed == 0
+ assert models.Image.objects.filter(pk=image.pk).exists()
+
+ def test_removes_nothing_when_pruning_is_off(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ image, path = _photo(folder=tmp_path)
+ path.unlink()
+
+ result = prune_missing_images(folder=folder, mode=models.SyncRun.PRUNE_MODE_OFF)
+
+ assert result.skipped_reason == models.SyncRun.SKIPPED_OFF
+ assert models.Image.objects.filter(pk=image.pk).exists()
+
+ def test_reports_without_removing_on_a_dry_run(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ image, path = _photo(folder=tmp_path)
+ path.unlink()
+
+ result = prune_missing_images(folder=folder, mode=models.SyncRun.PRUNE_MODE_DRY_RUN)
+
+ assert result.skipped_reason == models.SyncRun.SKIPPED_DRY_RUN
+ assert result.missing_found == 1
+ assert result.removed == 0
+ assert result.sample_paths == (str(path),)
+ assert models.Image.objects.filter(pk=image.pk).exists()
+
+ def test_keeps_a_record_whose_extension_the_walk_ignores(self, tmp_path):
+ # The walk only matches JPEGs, so a PNG record is absent from its result.
+ # Only the stat confirms whether the file is really gone.
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ png = tmp_path / "screenshot.png"
+ png.write_bytes(b"\x89PNG")
+ image = ImageFactory(filepath=str(png), filename="screenshot.png")
+
+ result = prune_missing_images(folder=folder, mode=models.SyncRun.PRUNE_MODE_AUTO)
+
+ assert result.missing_found == 0
+ assert models.Image.objects.filter(pk=image.pk).exists()
+
+ def test_keeps_a_record_pointing_at_a_broken_symlink(self, tmp_path):
+ # Something still occupies the path, so the conservative reading wins.
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ link = tmp_path / "DSCF0001.JPG"
+ link.symlink_to(tmp_path / "nowhere.JPG")
+ image = ImageFactory(filepath=str(link), filename="DSCF0001.JPG")
+
+ result = prune_missing_images(folder=folder, mode=models.SyncRun.PRUNE_MODE_AUTO)
+
+ assert result.missing_found == 0
+ assert models.Image.objects.filter(pk=image.pk).exists()
+
+ def test_publishes_prune_completed(self, tmp_path, captured_logs):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ _, path = _photo(folder=tmp_path)
+ path.unlink()
+
+ prune_missing_images(folder=folder, mode=models.SyncRun.PRUNE_MODE_AUTO)
+
+ matching = [e for e in captured_logs if e.get("event_type") == events.LIBRARY_SYNC_PRUNE_COMPLETED]
+ assert len(matching) == 1
+ assert matching[0]["folder_id"] == folder.pk
+ assert matching[0]["removed"] == 1
+
+
+@pytest.mark.django_db
+class TestPruneMissingImagesSafetyGuard:
+ @pytest.fixture(autouse=True)
+ def _strict_guard(self, settings):
+ settings.LIBRARY_PRUNE_GUARD_FRACTION = 0.5
+ settings.LIBRARY_PRUNE_GUARD_MIN_IMAGES = 2
+
+ def test_removes_nothing_when_the_guard_trips(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ for index in range(4):
+ _, path = _photo(folder=tmp_path, name=f"DSCF000{index}.JPG")
+ path.unlink()
+
+ result = prune_missing_images(folder=folder, mode=models.SyncRun.PRUNE_MODE_AUTO)
+
+ assert result.skipped_reason == models.SyncRun.SKIPPED_GUARD
+ assert result.missing_found == 4
+ assert result.removed == 0
+ assert models.Image.objects.count() == 4
+
+ def test_forcing_the_prune_overrides_the_guard(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ for index in range(4):
+ _, path = _photo(folder=tmp_path, name=f"DSCF000{index}.JPG")
+ path.unlink()
+
+ result = prune_missing_images(folder=folder, mode=models.SyncRun.PRUNE_MODE_FORCE)
+
+ assert result.skipped_reason == ""
+ assert result.removed == 4
+ assert models.Image.objects.count() == 0
+
+ def test_publishes_prune_skipped_when_the_guard_trips(self, tmp_path, captured_logs):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ for index in range(4):
+ _, path = _photo(folder=tmp_path, name=f"DSCF000{index}.JPG")
+ path.unlink()
+
+ prune_missing_images(folder=folder, mode=models.SyncRun.PRUNE_MODE_AUTO)
+
+ matching = [e for e in captured_logs if e.get("event_type") == events.LIBRARY_SYNC_PRUNE_SKIPPED]
+ assert len(matching) == 1
+ assert matching[0]["reason"] == models.SyncRun.SKIPPED_GUARD
+ assert matching[0]["missing_found"] == 4
+ assert matching[0]["total"] == 4
diff --git a/tests/unit/domain/library/test_prune_guard.py b/tests/unit/domain/library/test_prune_guard.py
new file mode 100644
index 0000000..c71e7bd
--- /dev/null
+++ b/tests/unit/domain/library/test_prune_guard.py
@@ -0,0 +1,53 @@
+import pytest
+
+from src.domain.library.operations import prune_guard_trips
+
+
+@pytest.fixture(autouse=True)
+def _default_thresholds(settings):
+ settings.LIBRARY_PRUNE_GUARD_FRACTION = 0.5
+ settings.LIBRARY_PRUNE_GUARD_MIN_IMAGES = 20
+
+
+class TestPruneGuardTrips:
+ @pytest.mark.parametrize(
+ ("missing", "total"),
+ [
+ (21, 40), # over both thresholds
+ (25, 25), # the whole of a folder large enough to matter
+ (5000, 5000),
+ ],
+ )
+ def test_trips_when_both_thresholds_are_exceeded(self, missing, total):
+ assert prune_guard_trips(missing=missing, total=total) is True
+
+ @pytest.mark.parametrize(
+ ("missing", "total"),
+ [
+ (20, 40), # exactly at the minimum, so not over it
+ (15, 15), # a small folder emptied entirely is an ordinary cleanup
+ (0, 100),
+ ],
+ )
+ def test_stays_clear_when_too_few_images_would_go(self, missing, total):
+ assert prune_guard_trips(missing=missing, total=total) is False
+
+ @pytest.mark.parametrize(
+ ("missing", "total"),
+ [
+ (21, 100), # well under the fraction
+ (50, 100), # exactly at the fraction, so not over it
+ ],
+ )
+ def test_stays_clear_when_the_share_is_not_exceeded(self, missing, total):
+ assert prune_guard_trips(missing=missing, total=total) is False
+
+ def test_stays_clear_for_a_folder_with_no_catalogued_images(self):
+ assert prune_guard_trips(missing=0, total=0) is False
+
+ def test_honours_a_reconfigured_fraction(self, settings):
+ settings.LIBRARY_PRUNE_GUARD_FRACTION = 0.9
+ settings.LIBRARY_PRUNE_GUARD_MIN_IMAGES = 1
+
+ assert prune_guard_trips(missing=60, total=100) is False
+ assert prune_guard_trips(missing=95, total=100) is True
From 25afe145e3fddbd8e8021e24c18836e6610eda81 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:44:56 +1000
Subject: [PATCH 13/36] Add a begin_pruning domain operation
Nothing calls it yet. Moves a run from processing into a prune phase,
electing a single winner under concurrent workers so the prune runs once
however many of them arrive together.
Electing here rather than after completion is deliberate. Pruning after a
run is already complete would stop the progress poller before the counts
are written, let a second sync start while the tree is still being walked,
and leave a run claiming COMPLETED if the process dies mid-prune. A
distinct active state avoids all three, and the existing startup recovery
already handles it.
Co-Authored-By: Claude Opus 5
---
src/domain/library/operations.py | 25 +++++++++++
.../domain/library/test_operations.py | 44 +++++++++++++++++++
2 files changed, 69 insertions(+)
diff --git a/src/domain/library/operations.py b/src/domain/library/operations.py
index 0e02e4a..50542ad 100644
--- a/src/domain/library/operations.py
+++ b/src/domain/library/operations.py
@@ -271,6 +271,31 @@ def start_sync_run(*, folder: models.LibraryFolder) -> models.SyncRun:
return run
+def begin_pruning(*, run: models.SyncRun) -> bool:
+ """
+ Move *run* from processing into its prune phase.
+
+ Uses a conditional update so that, under concurrent workers, exactly one
+ caller wins and the prune runs once. Returns True if this call won.
+
+ Pruning is an active state, so the folder stays locked against a second sync
+ while its tree is walked: a concurrent import could otherwise re-add files
+ the prune is about to remove.
+ """
+ started = run.transition_state(
+ from_states=(models.SyncRun.STATE_PROCESSING,),
+ to_state=models.SyncRun.STATE_PRUNING,
+ finished_at=None,
+ )
+ if started:
+ events.publish_event(
+ event_type=events.LIBRARY_SYNC_PRUNE_STARTED,
+ run_id=run.pk,
+ folder_id=run.folder_id,
+ )
+ return started
+
+
def complete_sync_run(*, run: models.SyncRun) -> bool:
"""
Mark *run* as completed if it is still processing or pruning.
diff --git a/tests/integration/domain/library/test_operations.py b/tests/integration/domain/library/test_operations.py
index 7dd5866..64101f4 100644
--- a/tests/integration/domain/library/test_operations.py
+++ b/tests/integration/domain/library/test_operations.py
@@ -6,6 +6,7 @@
FolderAlreadyInLibrary,
SyncAlreadyInProgress,
add_library_folder,
+ begin_pruning,
complete_sync_run,
fail_sync_run,
interrupt_active_sync_runs,
@@ -212,6 +213,49 @@ def test_transitions_a_pruning_run_to_completed(self):
assert run.state == models.SyncRun.STATE_COMPLETED
+@pytest.mark.django_db
+class TestBeginPruning:
+ def test_transitions_a_processing_run_to_pruning(self):
+ run = SyncRunFactory(state=models.SyncRun.STATE_PROCESSING, total=1)
+
+ assert begin_pruning(run=run) is True
+
+ run.refresh_from_db()
+ assert run.state == models.SyncRun.STATE_PRUNING
+ assert run.finished_at is None
+
+ def test_leaves_the_folder_locked_against_a_second_sync_while_pruning(self):
+ run = SyncRunFactory(state=models.SyncRun.STATE_PROCESSING, total=1)
+
+ begin_pruning(run=run)
+
+ run.refresh_from_db()
+ assert run.state in models.SyncRun.ACTIVE_STATES
+
+ def test_only_the_first_caller_wins(self):
+ run = SyncRunFactory(state=models.SyncRun.STATE_PROCESSING, total=1)
+ contender = models.SyncRun.objects.get(pk=run.pk)
+
+ assert begin_pruning(run=run) is True
+ assert begin_pruning(run=contender) is False
+
+ def test_does_not_start_from_a_failed_run(self):
+ run = SyncRunFactory(state=models.SyncRun.STATE_FAILED, total=1)
+
+ assert begin_pruning(run=run) is False
+
+ def test_publishes_prune_started_only_for_the_winner(self, captured_logs):
+ run = SyncRunFactory(state=models.SyncRun.STATE_PROCESSING, total=1)
+ contender = models.SyncRun.objects.get(pk=run.pk)
+
+ begin_pruning(run=run)
+ begin_pruning(run=contender)
+
+ matching = [e for e in captured_logs if e.get("event_type") == events.LIBRARY_SYNC_PRUNE_STARTED]
+ assert len(matching) == 1
+ assert matching[0]["run_id"] == run.pk
+
+
@pytest.mark.django_db
class TestFailSyncRun:
def test_marks_run_failed_with_message(self):
From 71f08967da4d3c6032c0cef56874e1bd82ac6ac2 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:46:00 +1000
Subject: [PATCH 14/36] Record a structured failure reason on a failed sync run
sync_library inferred "this folder is missing from disk" from the run
state alone, so any failure whatsoever was reported to the user as a
missing folder. The free-text message that held the real reason was never
surfaced anywhere.
fail_sync_run now takes a code alongside the message, and its one caller
passes it. The distinction matters more once removal exists: a missing
folder is usually an unplugged drive, and it is the moment a user most
needs telling that nothing was taken out of their gallery.
Co-Authored-By: Claude Opus 5
---
src/application/usecases/library/sync_folder.py | 7 ++++++-
src/application/usecases/library/sync_library.py | 3 ++-
src/data/models/_sync_run.py | 5 +++--
src/domain/library/operations.py | 13 ++++++++++---
tests/integration/domain/library/test_operations.py | 9 +++++++--
5 files changed, 28 insertions(+), 9 deletions(-)
diff --git a/src/application/usecases/library/sync_folder.py b/src/application/usecases/library/sync_folder.py
index 69a5b42..1826378 100644
--- a/src/application/usecases/library/sync_folder.py
+++ b/src/application/usecases/library/sync_folder.py
@@ -3,6 +3,7 @@
from django.conf import settings
from src.application.usecases.library.process_synced_image import process_synced_image
+from src.data import models
from src.domain.images import queries as image_queries
from src.domain.library import operations as library_operations
from src.domain.library import queries as library_queries
@@ -41,7 +42,11 @@ def sync_folder(*, folder_id: int) -> None:
found_paths = image_queries.collect_image_paths(folder=folder.path)
except FileNotFoundError:
folder.set_last_checked_at(value=now)
- library_operations.fail_sync_run(run=run, message="Folder does not exist")
+ library_operations.fail_sync_run(
+ run=run,
+ reason=models.SyncRun.FAILED_FOLDER_MISSING,
+ message="Folder does not exist",
+ )
return
known_paths = image_queries.get_all_known_image_paths()
diff --git a/src/application/usecases/library/sync_library.py b/src/application/usecases/library/sync_library.py
index 956a29d..7ae8512 100644
--- a/src/application/usecases/library/sync_library.py
+++ b/src/application/usecases/library/sync_library.py
@@ -3,6 +3,7 @@
from django.conf import settings
from src.application.usecases.library.sync_folder import sync_folder
+from src.data import models
from src.domain.library import operations as library_operations
from src.domain.library import queries as library_queries
from src.services import workertasks
@@ -54,7 +55,7 @@ def sync_library() -> SyncLibraryResult:
run = library_queries.get_latest_sync_run(folder_id=folder.pk)
if run is None:
continue
- if run.state == run.STATE_FAILED:
+ if run.failure_reason == models.SyncRun.FAILED_FOLDER_MISSING:
missing_folders.append(folder.path)
elif settings.USE_ASYNC_TASKS:
new_files_found += run.total or 0
diff --git a/src/data/models/_sync_run.py b/src/data/models/_sync_run.py
index b1625e5..9810d12 100644
--- a/src/data/models/_sync_run.py
+++ b/src/data/models/_sync_run.py
@@ -146,11 +146,12 @@ def transition_state(
)
return rows > 0
- def mark_failed(self, *, message: str) -> None:
+ def mark_failed(self, *, reason: str, message: str) -> None:
self.state = _STATE_FAILED
+ self.failure_reason = reason
self.error_message = message
self.finished_at = timezone.now()
- self.save(update_fields=["state", "error_message", "finished_at", "updated_at"])
+ self.save(update_fields=["state", "failure_reason", "error_message", "finished_at", "updated_at"])
def record_prune_result(self, *, missing_found: int, removed: int, skipped_reason: str) -> None:
self.missing_found = missing_found
diff --git a/src/domain/library/operations.py b/src/domain/library/operations.py
index 50542ad..59dbc34 100644
--- a/src/domain/library/operations.py
+++ b/src/domain/library/operations.py
@@ -318,15 +318,22 @@ def complete_sync_run(*, run: models.SyncRun) -> bool:
return completed
-def fail_sync_run(*, run: models.SyncRun, message: str) -> None:
+def fail_sync_run(*, run: models.SyncRun, reason: str, message: str) -> None:
"""
- Mark *run* as failed, recording *message* as the failure reason.
+ Mark *run* as failed, recording *reason* as the failure code and *message* as
+ the human-readable detail.
+
+ The code lets callers tell a folder that is missing from disk apart from any
+ other failure, which matters because the two need very different responses:
+ a missing folder is usually an unplugged drive, and nothing should be removed
+ from the gallery on its account.
"""
- run.mark_failed(message=message)
+ run.mark_failed(reason=reason, message=message)
events.publish_event(
event_type=events.LIBRARY_SYNC_RUN_FAILED,
run_id=run.pk,
folder_id=run.folder_id,
+ failure_reason=reason,
reason=message,
)
diff --git a/tests/integration/domain/library/test_operations.py b/tests/integration/domain/library/test_operations.py
index 64101f4..f324cd4 100644
--- a/tests/integration/domain/library/test_operations.py
+++ b/tests/integration/domain/library/test_operations.py
@@ -261,17 +261,22 @@ class TestFailSyncRun:
def test_marks_run_failed_with_message(self):
run = SyncRunFactory(state=models.SyncRun.STATE_SCANNING)
- fail_sync_run(run=run, message="folder no longer exists")
+ fail_sync_run(
+ run=run,
+ reason=models.SyncRun.FAILED_FOLDER_MISSING,
+ message="folder no longer exists",
+ )
run.refresh_from_db()
assert run.state == models.SyncRun.STATE_FAILED
+ assert run.failure_reason == models.SyncRun.FAILED_FOLDER_MISSING
assert run.error_message == "folder no longer exists"
assert run.finished_at is not None
def test_publishes_sync_run_failed_event(self, captured_logs):
run = SyncRunFactory(state=models.SyncRun.STATE_SCANNING)
- fail_sync_run(run=run, message="boom")
+ fail_sync_run(run=run, reason=models.SyncRun.FAILED_FOLDER_MISSING, message="boom")
matching = [e for e in captured_logs if e.get("event_type") == events.LIBRARY_SYNC_RUN_FAILED]
assert len(matching) == 1
From 876f1cae1c94cf287be025e6bb4c972f2ad49d8a Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:46:54 +1000
Subject: [PATCH 15/36] Add a finalize_sync_run use case
Nothing calls it yet. Elects a single finaliser for a run, prunes the
folder, then completes the run whatever the prune did.
Pruning belongs at the end of a run rather than inside the scan, because
imports have to land first: a file that moved has only repointed its
record once it has been re-imported, and until then it looks deleted.
Putting the orchestration in a use case keeps Celery out of the domain,
and gives every possible last caller, the scan that found nothing and each
per-image task, one function to reach.
It also defers when another folder is still syncing, which is what stops a
file moved between two library folders being removed by the source before
the destination has picked it up. Delaying a removal is always safe;
removing early is not.
Co-Authored-By: Claude Opus 5
---
.../usecases/library/finalize_sync_run.py | 75 +++++++++
.../library/test_finalize_sync_run.py | 145 ++++++++++++++++++
2 files changed, 220 insertions(+)
create mode 100644 src/application/usecases/library/finalize_sync_run.py
create mode 100644 tests/unit/application/library/test_finalize_sync_run.py
diff --git a/src/application/usecases/library/finalize_sync_run.py b/src/application/usecases/library/finalize_sync_run.py
new file mode 100644
index 0000000..3ae4d4e
--- /dev/null
+++ b/src/application/usecases/library/finalize_sync_run.py
@@ -0,0 +1,75 @@
+import structlog
+
+from src.data import models
+from src.domain.library import operations as library_operations
+from src.domain.library import queries as library_queries
+
+logger = structlog.get_logger("application.library.finalize_sync_run")
+
+
+def finalize_sync_run(*, run: models.SyncRun) -> None:
+ """
+ Finish *run*: elect a single finaliser, prune the folder, then complete it.
+
+ Called from every place that could be the last one standing, which is the
+ scan when it found nothing new and each per-image task or thread as it
+ finishes. ``begin_pruning`` is a conditional update, so exactly one caller
+ gets past it and the prune runs once no matter how many workers arrive here
+ together.
+
+ Pruning last is what makes a move survive: by the time this runs, every file
+ that moved has been re-imported and its record repointed, so it no longer
+ looks missing. Failed and interrupted runs never reach this function, so a
+ folder that vanished from disk never loses images.
+ """
+ if not library_operations.begin_pruning(run=run):
+ return
+
+ try:
+ result = _prune_for_run(run=run)
+ run.record_prune_result(
+ missing_found=result.missing_found,
+ removed=result.removed,
+ skipped_reason=result.skipped_reason,
+ )
+ except Exception:
+ logger.exception("Failed to prune missing images for sync run")
+ finally:
+ library_operations.complete_sync_run(run=run)
+
+
+def _prune_for_run(*, run: models.SyncRun) -> library_operations.PruneResult:
+ if _another_folder_is_still_importing(folder_id=run.folder_id):
+ # A file moved from this folder into one still importing has not been
+ # re-imported yet, so it would look deleted. Deferring only delays the
+ # removal to the next sync, which is always safe; removing early is not.
+ return library_operations.PruneResult(
+ missing_found=0,
+ removed=0,
+ total=0,
+ skipped_reason=models.SyncRun.SKIPPED_DEFERRED,
+ sample_paths=(),
+ )
+
+ folder = library_queries.get_library_folder(folder_id=run.folder_id)
+ return library_operations.prune_missing_images(folder=folder, mode=run.prune_mode)
+
+
+def _another_folder_is_still_importing(*, folder_id: int) -> bool:
+ """
+ Return True while another folder still has images left to import.
+
+ What a prune has to wait for is an import that might re-point a record it
+ would otherwise treat as missing, so the question is whether another folder
+ still has images outstanding, not merely whether its run is open. A run that
+ has accounted for every image cannot re-point anything, and treating it as a
+ reason to wait would make a folder with nothing to import block its
+ neighbours for no gain.
+ """
+ for folder in library_queries.get_all_library_folders():
+ if folder.pk == folder_id:
+ continue
+ run = library_queries.get_active_sync_run(folder_id=folder.pk)
+ if run is not None and not run.all_images_accounted_for():
+ return True
+ return False
diff --git a/tests/unit/application/library/test_finalize_sync_run.py b/tests/unit/application/library/test_finalize_sync_run.py
new file mode 100644
index 0000000..c774cdf
--- /dev/null
+++ b/tests/unit/application/library/test_finalize_sync_run.py
@@ -0,0 +1,145 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from src.application.usecases.library.finalize_sync_run import finalize_sync_run
+from src.data import models
+from src.domain.library.operations import PruneResult
+
+_BEGIN = "src.application.usecases.library.finalize_sync_run.library_operations.begin_pruning"
+_COMPLETE = "src.application.usecases.library.finalize_sync_run.library_operations.complete_sync_run"
+_PRUNE = "src.application.usecases.library.finalize_sync_run.library_operations.prune_missing_images"
+_GET_FOLDER = "src.application.usecases.library.finalize_sync_run.library_queries.get_library_folder"
+_ALL_FOLDERS = "src.application.usecases.library.finalize_sync_run.library_queries.get_all_library_folders"
+_ACTIVE_RUN = "src.application.usecases.library.finalize_sync_run.library_queries.get_active_sync_run"
+
+
+def _pruned(*, removed: int = 1) -> PruneResult:
+ return PruneResult(
+ missing_found=removed,
+ removed=removed,
+ total=10,
+ skipped_reason="",
+ sample_paths=(),
+ )
+
+
+def _run(*, prune_mode: str = models.SyncRun.PRUNE_MODE_AUTO) -> MagicMock:
+ run = MagicMock(spec=models.SyncRun)
+ run.folder_id = 1
+ run.prune_mode = prune_mode
+ return run
+
+
+@pytest.fixture
+def _no_other_folders():
+ with patch(_ALL_FOLDERS, return_value=[]):
+ yield
+
+
+class TestFinalizeSyncRun:
+ def test_prunes_and_completes_when_it_wins_the_election(self, _no_other_folders):
+ run = _run()
+ with (
+ patch(_BEGIN, return_value=True),
+ patch(_GET_FOLDER),
+ patch(_PRUNE, return_value=_pruned()) as prune,
+ patch(_COMPLETE) as complete,
+ ):
+ finalize_sync_run(run=run)
+
+ prune.assert_called_once()
+ complete.assert_called_once_with(run=run)
+ run.record_prune_result.assert_called_once_with(
+ missing_found=1, removed=1, skipped_reason=""
+ )
+
+ def test_does_nothing_when_another_caller_won_the_election(self):
+ run = _run()
+ with (
+ patch(_BEGIN, return_value=False),
+ patch(_PRUNE) as prune,
+ patch(_COMPLETE) as complete,
+ ):
+ finalize_sync_run(run=run)
+
+ prune.assert_not_called()
+ complete.assert_not_called()
+
+ def test_completes_the_run_even_when_the_prune_raises(self, _no_other_folders):
+ run = _run()
+ with (
+ patch(_BEGIN, return_value=True),
+ patch(_GET_FOLDER),
+ patch(_PRUNE, side_effect=OSError("disk went away")),
+ patch(_COMPLETE) as complete,
+ ):
+ finalize_sync_run(run=run)
+
+ complete.assert_called_once_with(run=run)
+
+ def test_passes_the_runs_prune_mode_through(self, _no_other_folders):
+ run = _run(prune_mode=models.SyncRun.PRUNE_MODE_FORCE)
+ with (
+ patch(_BEGIN, return_value=True),
+ patch(_GET_FOLDER, return_value="folder"),
+ patch(_PRUNE, return_value=_pruned()) as prune,
+ patch(_COMPLETE),
+ ):
+ finalize_sync_run(run=run)
+
+ assert prune.call_args.kwargs["mode"] == models.SyncRun.PRUNE_MODE_FORCE
+
+ def test_defers_the_prune_while_another_folder_is_still_importing(self):
+ run = _run()
+ other = MagicMock(pk=2)
+ still_importing = MagicMock()
+ still_importing.all_images_accounted_for.return_value = False
+ with (
+ patch(_BEGIN, return_value=True),
+ patch(_ALL_FOLDERS, return_value=[other]),
+ patch(_ACTIVE_RUN, return_value=still_importing),
+ patch(_PRUNE) as prune,
+ patch(_COMPLETE) as complete,
+ ):
+ finalize_sync_run(run=run)
+
+ prune.assert_not_called()
+ complete.assert_called_once_with(run=run)
+ run.record_prune_result.assert_called_once_with(
+ missing_found=0, removed=0, skipped_reason=models.SyncRun.SKIPPED_DEFERRED
+ )
+
+ def test_prunes_when_another_folder_is_open_but_has_nothing_left_to_import(self):
+ # A folder waiting only to finalise cannot re-point anything, so making
+ # it block its neighbours would defer their removals for no gain.
+ run = _run()
+ other = MagicMock(pk=2)
+ nothing_outstanding = MagicMock()
+ nothing_outstanding.all_images_accounted_for.return_value = True
+ with (
+ patch(_BEGIN, return_value=True),
+ patch(_ALL_FOLDERS, return_value=[other]),
+ patch(_ACTIVE_RUN, return_value=nothing_outstanding),
+ patch(_GET_FOLDER),
+ patch(_PRUNE, return_value=_pruned()) as prune,
+ patch(_COMPLETE),
+ ):
+ finalize_sync_run(run=run)
+
+ prune.assert_called_once()
+
+ def test_prunes_when_the_only_other_folder_is_idle(self):
+ run = _run()
+ other = MagicMock(pk=2)
+ with (
+ patch(_BEGIN, return_value=True),
+ patch(_ALL_FOLDERS, return_value=[other]),
+ patch(_ACTIVE_RUN, return_value=None),
+ patch(_GET_FOLDER),
+ patch(_PRUNE, return_value=_pruned()) as prune,
+ patch(_COMPLETE),
+ ):
+ finalize_sync_run(run=run)
+
+ prune.assert_called_once()
From 76ef660c7c7cca6daab405740896d0d9621f1719 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:47:36 +1000
Subject: [PATCH 16/36] Add a prune_folder use case
Nothing calls it yet. Removes catalog entries for one folder's missing
files without scanning for new ones, and records the outcome on that
folder's latest run so the Library page reflects it.
Lite mode needs this: it scans folders one after another, so pruning
inside each sync would remove a file moved into a folder that has not been
looked at yet. Importing everything first and then pruning needs a way to
prune on its own.
Co-Authored-By: Claude Opus 5
---
.../usecases/library/prune_folder.py | 59 +++++++++++++++++++
1 file changed, 59 insertions(+)
create mode 100644 src/application/usecases/library/prune_folder.py
diff --git a/src/application/usecases/library/prune_folder.py b/src/application/usecases/library/prune_folder.py
new file mode 100644
index 0000000..8711974
--- /dev/null
+++ b/src/application/usecases/library/prune_folder.py
@@ -0,0 +1,59 @@
+import attrs
+
+from src.domain.library import operations as library_operations
+from src.domain.library import queries as library_queries
+
+
+@attrs.frozen
+class LibraryFolderNotFound(Exception):
+ """
+ Raised when no library folder matches the given folder_id.
+ """
+
+ folder_id: int
+
+
+@attrs.frozen
+class PruneFolderResult:
+ folder_path: str
+ missing_found: int
+ removed: int
+ total: int
+ skipped_reason: str
+ sample_paths: tuple[str, ...]
+
+
+def prune_folder(*, folder_id: int, mode: str) -> PruneFolderResult:
+ """
+ Remove catalog entries for images under one library folder whose files are
+ gone, without scanning for new ones.
+
+ Removes catalog entries only: no image file is ever deleted from disk.
+
+ :raises LibraryFolderNotFound: If no folder with *folder_id* exists.
+ """
+ try:
+ folder = library_queries.get_library_folder(folder_id=folder_id)
+ except library_queries.LibraryFolderNotFound:
+ raise LibraryFolderNotFound(folder_id=folder_id)
+
+ result = library_operations.prune_missing_images(folder=folder, mode=mode)
+
+ # The folder's latest run is what the Library page reads, so record the
+ # outcome there even though the run itself has already finished.
+ run = library_queries.get_latest_sync_run(folder_id=folder_id)
+ if run is not None:
+ run.record_prune_result(
+ missing_found=result.missing_found,
+ removed=result.removed,
+ skipped_reason=result.skipped_reason,
+ )
+
+ return PruneFolderResult(
+ folder_path=folder.path,
+ missing_found=result.missing_found,
+ removed=result.removed,
+ total=result.total,
+ skipped_reason=result.skipped_reason,
+ sample_paths=result.sample_paths,
+ )
From fc9677d0785900c6824d8098a4967515ce6b3a0e Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:49:46 +1000
Subject: [PATCH 17/36] Let removing a library folder take its images out of
the gallery too
Removing a folder used to leave every image it had imported behind, with
nothing recording where they came from.
The choice is a required argument rather than a defaulted one, because
which of the two happens is the whole point and no caller should get it by
accident. That makes this one commit across three layers: the domain
signature change forces the use case and the view with it.
Only images no other registered folder covers are taken, so removing a
folder nested inside another never touches images the outer one still
monitors. The photo files themselves stay on disk.
The view reads the choice from the form; the confirmation that offers it
comes next.
Co-Authored-By: Claude Opus 5
---
.../usecases/library/remove_library_folder.py | 18 +++-
src/domain/library/operations.py | 100 +++++++++++-------
src/interfaces/library/views.py | 8 +-
tests/functional/test_library_views.py | 18 +++-
.../library/test_remove_library_folder.py | 27 ++++-
.../domain/library/test_operations.py | 70 +++++++++++-
6 files changed, 192 insertions(+), 49 deletions(-)
diff --git a/src/application/usecases/library/remove_library_folder.py b/src/application/usecases/library/remove_library_folder.py
index 1bf7df6..1ae90cf 100644
--- a/src/application/usecases/library/remove_library_folder.py
+++ b/src/application/usecases/library/remove_library_folder.py
@@ -13,15 +13,27 @@ class LibraryFolderNotFound(Exception):
folder_id: int
-def remove_library_folder(*, folder_id: int) -> None:
+@attrs.frozen
+class RemoveLibraryFolderResult:
+ images_removed: int
+
+
+def remove_library_folder(*, folder_id: int, delete_images: bool) -> RemoveLibraryFolderResult:
"""
Remove the library folder with *folder_id* from the monitored list.
- Does not delete any images from the catalog.
+ When *delete_images* is true the folder's images also leave the gallery,
+ except any a second registered folder still covers. No image file is ever
+ deleted from disk.
:raises LibraryFolderNotFound: If no folder with *folder_id* exists.
"""
try:
- domain_operations.remove_library_folder(folder_id=folder_id)
+ removed = domain_operations.remove_library_folder(
+ folder_id=folder_id,
+ delete_images=delete_images,
+ )
except DomainLibraryFolderNotFound as exc:
raise LibraryFolderNotFound(folder_id=exc.folder_id)
+
+ return RemoveLibraryFolderResult(images_removed=removed)
diff --git a/src/domain/library/operations.py b/src/domain/library/operations.py
index 59dbc34..59993f8 100644
--- a/src/domain/library/operations.py
+++ b/src/domain/library/operations.py
@@ -10,7 +10,7 @@
from src.domain.images import events as image_events
from src.domain.images import operations as image_operations
from src.domain.images import queries as image_queries
-from src.domain.library import events
+from src.domain.library import events, queries
from src.domain.library.queries import FolderNotFound, LibraryFolderNotFound
# How many missing paths a prune reports back for a dry run, so the caller can
@@ -81,11 +81,16 @@ def add_library_folder(*, path: str) -> models.LibraryFolder:
return folder
-def remove_library_folder(*, folder_id: int) -> None:
+def remove_library_folder(*, folder_id: int, delete_images: bool) -> int:
"""
Remove the library folder with *folder_id* from the monitored list.
- Does not delete any images from the catalog.
+ When *delete_images* is true the folder's images also leave the gallery.
+ Only images no other registered folder covers are removed, so removing a
+ folder nested inside another one never takes images the outer folder still
+ monitors. No image file is ever deleted from disk.
+
+ Returns the number of images removed from the gallery.
:raises LibraryFolderNotFound: If no folder with *folder_id* exists.
"""
@@ -95,43 +100,31 @@ def remove_library_folder(*, folder_id: int) -> None:
raise LibraryFolderNotFound(folder_id=folder_id)
path = folder.path
- folder.delete()
- events.publish_event(event_type=events.LIBRARY_FOLDER_REMOVED, folder_id=folder_id, path=path)
-
-
-def update_library_folder_path(*, folder_id: int, path: str) -> models.LibraryFolder:
- """
- Update the path of the library folder with *folder_id*.
+ removed = 0
- Normalizes the new path before storing it.
+ if delete_images:
+ # Resolved before the folder row goes, because ownership is worked out
+ # by comparing this folder's path against the other registered ones.
+ image_ids = queries.get_exclusively_owned_image_ids(folder_id=folder_id)
+ for image in models.Image.objects.filter(pk__in=image_ids):
+ image_operations.remove_image(
+ image=image,
+ reason=image_events.REMOVE_REASON_FOLDER_REMOVED,
+ )
+ removed += 1
- :raises LibraryFolderNotFound: If no folder with *folder_id* exists.
- :raises FolderNotFound: If the normalized path does not exist on disk
- or is not a directory.
- :raises FolderAlreadyInLibrary: If the normalized path is already
- registered under a different folder_id.
- """
- try:
- folder = models.LibraryFolder.objects.get(pk=folder_id)
- except models.LibraryFolder.DoesNotExist:
- raise LibraryFolderNotFound(folder_id=folder_id)
-
- normalized = _normalize_path(path)
- if not Path(normalized).is_dir():
- raise FolderNotFound(path=normalized)
+ folder.delete()
+ events.publish_event(event_type=events.LIBRARY_FOLDER_REMOVED, folder_id=folder_id, path=path)
- try:
- with transaction.atomic():
- folder.set_path(path=normalized)
- except IntegrityError:
- raise FolderAlreadyInLibrary(path=normalized)
+ if removed:
+ events.publish_event(
+ event_type=events.LIBRARY_FOLDER_IMAGES_REMOVED,
+ folder_id=folder_id,
+ path=path,
+ removed=removed,
+ )
- events.publish_event(
- event_type=events.LIBRARY_FOLDER_PATH_UPDATED,
- folder_id=folder.pk,
- path=folder.path,
- )
- return folder
+ return removed
def prune_guard_trips(*, missing: int, total: int) -> bool:
@@ -250,6 +243,41 @@ def prune_missing_images(*, folder: models.LibraryFolder, mode: str) -> PruneRes
)
+def update_library_folder_path(*, folder_id: int, path: str) -> models.LibraryFolder:
+ """
+ Update the path of the library folder with *folder_id*.
+
+ Normalizes the new path before storing it.
+
+ :raises LibraryFolderNotFound: If no folder with *folder_id* exists.
+ :raises FolderNotFound: If the normalized path does not exist on disk
+ or is not a directory.
+ :raises FolderAlreadyInLibrary: If the normalized path is already
+ registered under a different folder_id.
+ """
+ try:
+ folder = models.LibraryFolder.objects.get(pk=folder_id)
+ except models.LibraryFolder.DoesNotExist:
+ raise LibraryFolderNotFound(folder_id=folder_id)
+
+ normalized = _normalize_path(path)
+ if not Path(normalized).is_dir():
+ raise FolderNotFound(path=normalized)
+
+ try:
+ with transaction.atomic():
+ folder.set_path(path=normalized)
+ except IntegrityError:
+ raise FolderAlreadyInLibrary(path=normalized)
+
+ events.publish_event(
+ event_type=events.LIBRARY_FOLDER_PATH_UPDATED,
+ folder_id=folder.pk,
+ path=folder.path,
+ )
+ return folder
+
+
def start_sync_run(*, folder: models.LibraryFolder) -> models.SyncRun:
"""
Create a new sync run for *folder* in the scanning state.
diff --git a/src/interfaces/library/views.py b/src/interfaces/library/views.py
index b4f9ebb..7925445 100644
--- a/src/interfaces/library/views.py
+++ b/src/interfaces/library/views.py
@@ -84,14 +84,18 @@ def post(self, request: http.HttpRequest) -> http.HttpResponse:
class LibraryFolderRemove(generic.View):
- """Remove a folder from the image library.
+ """Remove a folder from the image library, optionally with its images.
:raises Http404: if no folder with the given ID exists.
"""
def post(self, request: http.HttpRequest, folder_id: int) -> http.HttpResponse:
+ delete_images = request.POST.get("delete_images") == "on"
try:
- remove_library_folder_uc.remove_library_folder(folder_id=folder_id)
+ remove_library_folder_uc.remove_library_folder(
+ folder_id=folder_id,
+ delete_images=delete_images,
+ )
except remove_library_folder_uc.LibraryFolderNotFound:
raise http.Http404
return shortcuts.redirect(urls.reverse("library-list"))
diff --git a/tests/functional/test_library_views.py b/tests/functional/test_library_views.py
index d58b1ba..4c2e2a2 100644
--- a/tests/functional/test_library_views.py
+++ b/tests/functional/test_library_views.py
@@ -5,7 +5,7 @@
from src.application.usecases.library.trigger_folder_sync import CeleryWorkerUnavailable
from src.data import models
-from tests.factories import LibraryFolderFactory
+from tests.factories import ImageFactory, LibraryFolderFactory
TRIGGER = "src.interfaces.library.views.trigger_folder_sync_uc.trigger_folder_sync"
@@ -128,6 +128,22 @@ def test_returns_404_for_unknown_folder_id(self, client):
response = client.post("/library/99999/delete/")
assert response.status_code == 404
+ def test_keeps_the_images_by_default(self, client):
+ folder = LibraryFolderFactory(path="/photos")
+ image = ImageFactory(filepath="/photos/DSCF0001.JPG")
+
+ client.post(f"/library/{folder.pk}/delete/")
+
+ assert models.Image.objects.filter(pk=image.pk).exists()
+
+ def test_removes_the_images_when_asked_to(self, client):
+ folder = LibraryFolderFactory(path="/photos")
+ image = ImageFactory(filepath="/photos/DSCF0001.JPG")
+
+ client.post(f"/library/{folder.pk}/delete/", {"delete_images": "on"})
+
+ assert not models.Image.objects.filter(pk=image.pk).exists()
+
@pytest.mark.django_db
class TestLibraryFolderPathUpdate:
diff --git a/tests/integration/application/library/test_remove_library_folder.py b/tests/integration/application/library/test_remove_library_folder.py
index b8d1c22..d8128a3 100644
--- a/tests/integration/application/library/test_remove_library_folder.py
+++ b/tests/integration/application/library/test_remove_library_folder.py
@@ -5,17 +5,38 @@
remove_library_folder,
)
from src.data import models
-from tests.factories import LibraryFolderFactory
+from tests.factories import ImageFactory, LibraryFolderFactory
@pytest.mark.django_db
class TestRemoveLibraryFolder:
def test_deletes_folder_from_db(self):
folder = LibraryFolderFactory()
- remove_library_folder(folder_id=folder.pk)
+
+ remove_library_folder(folder_id=folder.pk, delete_images=False)
+
assert not models.LibraryFolder.objects.filter(pk=folder.pk).exists()
def test_raises_library_folder_not_found_for_unknown_id(self):
with pytest.raises(LibraryFolderNotFound) as exc_info:
- remove_library_folder(folder_id=99999)
+ remove_library_folder(folder_id=99999, delete_images=False)
assert exc_info.value.folder_id == 99999
+
+ def test_reports_no_images_removed_when_only_the_folder_goes(self):
+ folder = LibraryFolderFactory(path="/photos")
+ image = ImageFactory(filepath="/photos/DSCF0001.JPG")
+
+ result = remove_library_folder(folder_id=folder.pk, delete_images=False)
+
+ assert result.images_removed == 0
+ assert models.Image.objects.filter(pk=image.pk).exists()
+
+ def test_reports_how_many_images_left_the_gallery(self):
+ folder = LibraryFolderFactory(path="/photos")
+ ImageFactory(filepath="/photos/DSCF0001.JPG")
+ ImageFactory(filepath="/photos/2024/DSCF0002.JPG")
+
+ result = remove_library_folder(folder_id=folder.pk, delete_images=True)
+
+ assert result.images_removed == 2
+ assert models.Image.objects.count() == 0
diff --git a/tests/integration/domain/library/test_operations.py b/tests/integration/domain/library/test_operations.py
index f324cd4..b9e5a84 100644
--- a/tests/integration/domain/library/test_operations.py
+++ b/tests/integration/domain/library/test_operations.py
@@ -15,7 +15,7 @@
update_library_folder_path,
)
from src.domain.library.queries import FolderNotFound, LibraryFolderNotFound
-from tests.factories import LibraryFolderFactory, SyncRunFactory
+from tests.factories import ImageFactory, LibraryFolderFactory, SyncRunFactory
@pytest.mark.django_db
@@ -62,12 +62,12 @@ def test_raises_folder_already_in_library_for_duplicate_path(self, tmp_path):
class TestRemoveLibraryFolder:
def test_deletes_library_folder_row(self, tmp_path):
folder = LibraryFolderFactory(path=str(tmp_path))
- remove_library_folder(folder_id=folder.pk)
+ remove_library_folder(folder_id=folder.pk, delete_images=False)
assert not models.LibraryFolder.objects.filter(pk=folder.pk).exists()
def test_publishes_folder_removed_event(self, tmp_path, captured_logs):
folder = LibraryFolderFactory(path=str(tmp_path))
- remove_library_folder(folder_id=folder.pk)
+ remove_library_folder(folder_id=folder.pk, delete_images=False)
matching = [e for e in captured_logs if e.get("event_type") == events.LIBRARY_FOLDER_REMOVED]
assert len(matching) == 1
@@ -76,9 +76,71 @@ def test_publishes_folder_removed_event(self, tmp_path, captured_logs):
def test_raises_library_folder_not_found_for_unknown_id(self):
with pytest.raises(LibraryFolderNotFound) as exc_info:
- remove_library_folder(folder_id=99999)
+ remove_library_folder(folder_id=99999, delete_images=False)
assert exc_info.value.folder_id == 99999
+ def test_keeps_the_images_when_only_the_folder_is_removed(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ image = ImageFactory(filepath=str(tmp_path / "DSCF0001.JPG"))
+
+ removed = remove_library_folder(folder_id=folder.pk, delete_images=False)
+
+ assert removed == 0
+ assert models.Image.objects.filter(pk=image.pk).exists()
+
+ def test_removes_the_images_when_asked_to(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ image = ImageFactory(filepath=str(tmp_path / "DSCF0001.JPG"))
+
+ removed = remove_library_folder(folder_id=folder.pk, delete_images=True)
+
+ assert removed == 1
+ assert not models.Image.objects.filter(pk=image.pk).exists()
+
+ def test_never_deletes_the_image_files(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ photo = tmp_path / "DSCF0001.JPG"
+ photo.write_bytes(b"\xff\xd8")
+ ImageFactory(filepath=str(photo))
+
+ remove_library_folder(folder_id=folder.pk, delete_images=True)
+
+ assert photo.exists()
+
+ def test_keeps_images_a_second_registered_folder_still_covers(self, tmp_path):
+ outer = LibraryFolderFactory(path=str(tmp_path))
+ inner_dir = tmp_path / "2024"
+ LibraryFolderFactory(path=str(inner_dir))
+ shared = ImageFactory(filepath=str(inner_dir / "DSCF0001.JPG"))
+ only_outer = ImageFactory(filepath=str(tmp_path / "DSCF0002.JPG"))
+
+ removed = remove_library_folder(folder_id=outer.pk, delete_images=True)
+
+ assert removed == 1
+ assert models.Image.objects.filter(pk=shared.pk).exists()
+ assert not models.Image.objects.filter(pk=only_outer.pk).exists()
+
+ def test_publishes_folder_images_removed_when_images_go(self, tmp_path, captured_logs):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ ImageFactory(filepath=str(tmp_path / "DSCF0001.JPG"))
+
+ remove_library_folder(folder_id=folder.pk, delete_images=True)
+
+ matching = [
+ e for e in captured_logs if e.get("event_type") == events.LIBRARY_FOLDER_IMAGES_REMOVED
+ ]
+ assert len(matching) == 1
+ assert matching[0]["removed"] == 1
+
+ def test_publishes_no_images_removed_event_when_none_go(self, tmp_path, captured_logs):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+
+ remove_library_folder(folder_id=folder.pk, delete_images=True)
+
+ assert [
+ e for e in captured_logs if e.get("event_type") == events.LIBRARY_FOLDER_IMAGES_REMOVED
+ ] == []
+
@pytest.mark.django_db
class TestUpdateLibraryFolderPath:
From 7fbb69f3244318c5dc4ca1136f48e7368fbf5e5f Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:52:26 +1000
Subject: [PATCH 18/36] Remove images from the gallery when a sync finishes
This is where removal starts happening. The pieces existed but nothing
called them.
Both places that could be the last one standing now finalise through the
same use case: the scan when it found nothing new, which is exactly the
"the user deleted photos" case, and each per-image task as it finishes.
Imports therefore always land before anything is removed, so a file that
moved has already repointed its record and no longer looks missing.
Lite mode scans folders one after another, so a prune inside each sync
would run before later folders had been looked at, and a photo moved from
the first folder to the last would be removed moments before being
re-imported, losing its rating. It imports everything first, then prunes.
Full mode starts every run back to back, so the deferral rule already
covers the same case there.
The scenario tests are the acceptance suite for all of this: one test per
row of the matrix, building real trees of JPEGs, mutating them exactly as
each scenario describes and syncing again. Moves assert the record kept
its id, rating and favourite rather than merely still existing, and every
scenario asserts the photo files are still on disk.
Co-Authored-By: Claude Opus 5
---
.../usecases/library/process_synced_image.py | 4 +-
.../usecases/library/sync_folder.py | 24 +-
.../usecases/library/sync_library.py | 106 +++++-
src/domain/library/operations.py | 15 +-
.../application/library/test_sync_library.py | 2 +
.../domain/library/test_prune_scenarios.py | 339 ++++++++++++++++++
6 files changed, 469 insertions(+), 21 deletions(-)
create mode 100644 tests/integration/domain/library/test_prune_scenarios.py
diff --git a/src/application/usecases/library/process_synced_image.py b/src/application/usecases/library/process_synced_image.py
index 397c39d..1e2e219 100644
--- a/src/application/usecases/library/process_synced_image.py
+++ b/src/application/usecases/library/process_synced_image.py
@@ -1,9 +1,9 @@
import structlog
+from src.application.usecases.library.finalize_sync_run import finalize_sync_run
from src.domain.images import events as image_events
from src.domain.images import operations as image_operations
from src.domain.images.queries import NoFilmSimulationError
-from src.domain.library import operations as library_operations
from src.domain.library import queries as library_queries
from src.domain.recipes import validation as recipe_validation
@@ -54,4 +54,4 @@ def process_synced_image(*, image_path: str, sync_run_id: int) -> None:
run.refresh_from_db()
if run.all_images_accounted_for():
- library_operations.complete_sync_run(run=run)
+ finalize_sync_run(run=run)
diff --git a/src/application/usecases/library/sync_folder.py b/src/application/usecases/library/sync_folder.py
index 1826378..90b3c58 100644
--- a/src/application/usecases/library/sync_folder.py
+++ b/src/application/usecases/library/sync_folder.py
@@ -2,6 +2,7 @@
from django.conf import settings
+from src.application.usecases.library.finalize_sync_run import finalize_sync_run
from src.application.usecases.library.process_synced_image import process_synced_image
from src.data import models
from src.domain.images import queries as image_queries
@@ -12,17 +13,20 @@
_SYNC_PROCESS_IMAGE_TASK = "src.interfaces.tasks.sync_process_image_task"
-def sync_folder(*, folder_id: int) -> None:
+def sync_folder(*, folder_id: int, prune_mode: str = models.SyncRun.PRUNE_MODE_AUTO) -> None:
"""
- Scan a single library folder and import new images, tracking progress in a
- SyncRun.
+ Scan a single library folder, import new images and remove catalog entries
+ whose files have disappeared, tracking progress in a SyncRun.
- Creates a run, walks the whole folder, and dispatches each new image: in async mode by enqueuing a Celery task, in sync
- mode by processing inline. The run is completed here only when nothing new is
- found; otherwise the last processed image completes it.
+ Creates a run, walks the whole folder, and dispatches each new image: in
+ async mode by enqueuing a Celery task, in sync mode by processing inline.
+ Whoever handles the last image finalises the run, which is also where the
+ prune happens, so imports always land before anything is removed.
Returns without doing anything if the folder no longer exists or already has
- an active run (the concurrency guard).
+ an active run (the concurrency guard). A folder that is missing from disk
+ fails its run and removes nothing, because an unplugged drive is far more
+ likely than a deletion of everything at once.
"""
try:
folder = library_queries.get_library_folder(folder_id=folder_id)
@@ -30,7 +34,7 @@ def sync_folder(*, folder_id: int) -> None:
return
try:
- run = library_operations.start_sync_run(folder=folder)
+ run = library_operations.start_sync_run(folder=folder, prune_mode=prune_mode)
except library_operations.SyncAlreadyInProgress:
return
@@ -58,7 +62,9 @@ def sync_folder(*, folder_id: int) -> None:
folder.set_last_processed_at(value=now)
if not new_paths:
- library_operations.complete_sync_run(run=run)
+ # Nothing new is exactly the case where photos were deleted, so this
+ # branch still has to finalise (and therefore prune).
+ finalize_sync_run(run=run)
return
for path in new_paths:
diff --git a/src/application/usecases/library/sync_library.py b/src/application/usecases/library/sync_library.py
index 7ae8512..84e5e37 100644
--- a/src/application/usecases/library/sync_library.py
+++ b/src/application/usecases/library/sync_library.py
@@ -2,6 +2,7 @@
from django.conf import settings
+from src.application.usecases.library.prune_folder import prune_folder
from src.application.usecases.library.sync_folder import sync_folder
from src.data import models
from src.domain.library import operations as library_operations
@@ -9,6 +10,14 @@
from src.services import workertasks
+# Skip reasons the user needs to hear about. A dry run and an explicit --no-prune
+# are what they asked for, so neither is a warning.
+_REPORTED_SKIP_REASONS = (
+ models.SyncRun.SKIPPED_GUARD,
+ models.SyncRun.SKIPPED_DRY_RUN,
+)
+
+
@attrs.frozen
class CeleryWorkerUnavailable(Exception):
"""
@@ -16,17 +25,28 @@ class CeleryWorkerUnavailable(Exception):
"""
+@attrs.frozen
+class PruneWarning:
+ folder_path: str
+ missing_found: int
+ total: int
+ reason: str
+
+
@attrs.frozen
class SyncLibraryResult:
folders_scanned: int
new_files_found: int
skipped_non_fujifilm: int
missing_folders: tuple[str, ...]
+ images_removed: int
+ prune_warnings: tuple[PruneWarning, ...]
-def sync_library() -> SyncLibraryResult:
+def sync_library(*, prune_mode: str = models.SyncRun.PRUNE_MODE_AUTO) -> SyncLibraryResult:
"""
- Scan every registered library folder and import new images into the catalog.
+ Scan every registered library folder, import new images into the catalog and
+ remove entries whose files have disappeared.
Recovers any runs abandoned by a previous process (marking them interrupted),
then syncs each folder in turn via the single-folder use case. In async mode,
@@ -35,7 +55,11 @@ def sync_library() -> SyncLibraryResult:
The result aggregates per-folder outcomes from each folder's sync run: in async
mode ``new_files_found`` counts images enqueued for processing, in sync mode it
counts images actually imported. Folders that no longer exist on disk are
- reported in ``missing_folders``.
+ reported in ``missing_folders`` and lose no images.
+
+ Removal removes catalog entries only; no image file is deleted from disk. In
+ async mode it happens in the worker after this returns, so ``images_removed``
+ is zero there, the same limitation the other counts already have.
:raises CeleryWorkerUnavailable: If USE_ASYNC_TASKS is True and no Celery
worker responds within the ping timeout.
@@ -46,26 +70,96 @@ def sync_library() -> SyncLibraryResult:
library_operations.interrupt_active_sync_runs()
folders = library_queries.get_all_library_folders()
+
+ if settings.USE_ASYNC_TASKS:
+ return _sync_with_worker(folders=folders, prune_mode=prune_mode)
+ return _sync_inline(folders=folders, prune_mode=prune_mode)
+
+
+def _sync_with_worker(
+ *,
+ folders: list[models.LibraryFolder],
+ prune_mode: str,
+) -> SyncLibraryResult:
+ # Every run is started back to back and stays active while its images are
+ # processed, so a folder finalising early sees the others still running and
+ # defers its prune. That is what protects a file moved between two folders.
+ for folder in folders:
+ sync_folder(folder_id=folder.pk, prune_mode=prune_mode)
+
new_files_found = 0
- skipped_non_fujifilm = 0
missing_folders: list[str] = []
for folder in folders:
- sync_folder(folder_id=folder.pk)
run = library_queries.get_latest_sync_run(folder_id=folder.pk)
if run is None:
continue
if run.failure_reason == models.SyncRun.FAILED_FOLDER_MISSING:
missing_folders.append(folder.path)
- elif settings.USE_ASYNC_TASKS:
+ else:
new_files_found += run.total or 0
+
+ return SyncLibraryResult(
+ folders_scanned=len(folders),
+ new_files_found=new_files_found,
+ skipped_non_fujifilm=0,
+ missing_folders=tuple(missing_folders),
+ # Necessarily zero: in async mode every run is finished by the worker,
+ # so nothing has been removed by the time this returns. The Library page
+ # reports what each run removed once it has.
+ images_removed=0,
+ prune_warnings=(),
+ )
+
+
+def _sync_inline(
+ *,
+ folders: list[models.LibraryFolder],
+ prune_mode: str,
+) -> SyncLibraryResult:
+ # Folders are scanned one after another here, so a prune run per folder would
+ # fire before the later folders had been looked at, and a file moved from the
+ # first folder to the last would be removed just before being re-imported.
+ # Importing everything first, then pruning, keeps such a move a move.
+ new_files_found = 0
+ skipped_non_fujifilm = 0
+ missing_folders: list[str] = []
+
+ for folder in folders:
+ sync_folder(folder_id=folder.pk, prune_mode=models.SyncRun.PRUNE_MODE_OFF)
+ run = library_queries.get_latest_sync_run(folder_id=folder.pk)
+ if run is None:
+ continue
+ if run.failure_reason == models.SyncRun.FAILED_FOLDER_MISSING:
+ missing_folders.append(folder.path)
else:
new_files_found += run.processed
skipped_non_fujifilm += run.skipped
+ images_removed = 0
+ prune_warnings: list[PruneWarning] = []
+
+ if prune_mode != models.SyncRun.PRUNE_MODE_OFF:
+ for folder in folders:
+ if folder.path in missing_folders:
+ continue
+ result = prune_folder(folder_id=folder.pk, mode=prune_mode)
+ images_removed += result.removed
+ if result.skipped_reason in _REPORTED_SKIP_REASONS:
+ prune_warnings.append(
+ PruneWarning(
+ folder_path=result.folder_path,
+ missing_found=result.missing_found,
+ total=result.total,
+ reason=result.skipped_reason,
+ )
+ )
+
return SyncLibraryResult(
folders_scanned=len(folders),
new_files_found=new_files_found,
skipped_non_fujifilm=skipped_non_fujifilm,
missing_folders=tuple(missing_folders),
+ images_removed=images_removed,
+ prune_warnings=tuple(prune_warnings),
)
diff --git a/src/domain/library/operations.py b/src/domain/library/operations.py
index 59993f8..4b426fa 100644
--- a/src/domain/library/operations.py
+++ b/src/domain/library/operations.py
@@ -278,16 +278,23 @@ def update_library_folder_path(*, folder_id: int, path: str) -> models.LibraryFo
return folder
-def start_sync_run(*, folder: models.LibraryFolder) -> models.SyncRun:
+def start_sync_run(
+ *,
+ folder: models.LibraryFolder,
+ prune_mode: str = models.SyncRun.PRUNE_MODE_AUTO,
+) -> models.SyncRun:
"""
Create a new sync run for *folder* in the scanning state.
- :raises SyncAlreadyInProgress: If *folder* already has an active (scanning or
- processing) run.
+ *prune_mode* is stored on the run because the caller that finalises it may be
+ a different process entirely, and cannot be told any other way.
+
+ :raises SyncAlreadyInProgress: If *folder* already has an active (scanning,
+ processing or pruning) run.
"""
try:
with transaction.atomic():
- run = models.SyncRun.create(folder=folder)
+ run = models.SyncRun.create(folder=folder, prune_mode=prune_mode)
except IntegrityError:
raise SyncAlreadyInProgress(folder_id=folder.pk)
diff --git a/tests/integration/application/library/test_sync_library.py b/tests/integration/application/library/test_sync_library.py
index feccbbd..3a1933b 100644
--- a/tests/integration/application/library/test_sync_library.py
+++ b/tests/integration/application/library/test_sync_library.py
@@ -32,6 +32,8 @@ def test_returns_zero_result_when_no_folders_are_registered(self):
new_files_found=0,
skipped_non_fujifilm=0,
missing_folders=(),
+ images_removed=0,
+ prune_warnings=(),
)
diff --git a/tests/integration/domain/library/test_prune_scenarios.py b/tests/integration/domain/library/test_prune_scenarios.py
new file mode 100644
index 0000000..c00df72
--- /dev/null
+++ b/tests/integration/domain/library/test_prune_scenarios.py
@@ -0,0 +1,339 @@
+"""
+One test per row of the scenario matrix in ADR 013.
+
+Each test builds a real tree of JPEGs, imports it through the ordinary sync, then
+mutates the filesystem exactly as the scenario describes and syncs again. Test
+names carry the scenario number so the matrix and this suite stay tied together.
+
+Two invariants run through all of them: user data (rating, favourite, album)
+survives a move, and no image file is ever deleted from disk.
+"""
+
+import shutil
+from pathlib import Path
+
+import pytest
+
+from src.application.usecases.library.sync_folder import sync_folder
+from src.application.usecases.library.sync_library import sync_library
+from src.data import models
+from src.domain.library.operations import remove_library_folder, update_library_folder_path
+from src.domain.images.operations import process_image
+from tests.factories import LibraryFolderFactory
+
+FIXTURES_DIR = Path(__file__).resolve().parent.parent.parent.parent / "fixtures" / "images"
+FIXTURE_A = FIXTURES_DIR / "XS107114.JPG"
+FIXTURE_B = FIXTURES_DIR / "XS107209.jpg"
+FIXTURE_C = FIXTURES_DIR / "XS107336.jpg"
+
+pytestmark = pytest.mark.django_db
+
+
+def _place(*, fixture: Path, destination: Path) -> Path:
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy(fixture, destination)
+ return destination
+
+
+def _catalogued() -> set[str]:
+ return set(models.Image.objects.values_list("filepath", flat=True))
+
+
+@pytest.fixture(autouse=True)
+def _lite_mode_with_a_generous_guard(settings):
+ # Run everything inline so a sync is finished, prune included, when it
+ # returns. These trees are tiny, so the guard is held off; it has its own tests.
+ settings.USE_ASYNC_TASKS = False
+ settings.LIBRARY_PRUNE_GUARD_MIN_IMAGES = 1000
+
+
+class TestFileLevelScenarios:
+ def test_01_a_deleted_file_leaves_the_gallery(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ photo = _place(fixture=FIXTURE_A, destination=tmp_path / "DSCF0001.JPG")
+ sync_folder(folder_id=folder.pk)
+ assert models.Image.objects.count() == 1
+
+ photo.unlink()
+ sync_folder(folder_id=folder.pk)
+
+ assert models.Image.objects.count() == 0
+
+ def test_02_a_file_renamed_in_place_is_relocated(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ photo = _place(fixture=FIXTURE_A, destination=tmp_path / "DSCF0001.JPG")
+ sync_folder(folder_id=folder.pk)
+ image = models.Image.objects.get()
+ image.set_rating(4)
+ image.set_as_favorite()
+
+ renamed = tmp_path / "spain.JPG"
+ photo.rename(renamed)
+ sync_folder(folder_id=folder.pk)
+
+ image.refresh_from_db()
+ assert models.Image.objects.count() == 1
+ assert image.filepath == str(renamed)
+ assert image.filename == "spain.JPG"
+ assert image.rating == 4
+ assert image.is_favorite is True
+ assert renamed.exists()
+
+ def test_03_a_file_moved_into_a_subfolder_is_relocated(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ photo = _place(fixture=FIXTURE_A, destination=tmp_path / "DSCF0001.JPG")
+ sync_folder(folder_id=folder.pk)
+ image = models.Image.objects.get()
+ image.set_rating(3)
+
+ moved = tmp_path / "2025" / "DSCF0001.JPG"
+ moved.parent.mkdir()
+ photo.rename(moved)
+ sync_folder(folder_id=folder.pk)
+
+ image.refresh_from_db()
+ assert models.Image.objects.count() == 1
+ assert image.filepath == str(moved)
+ assert image.rating == 3
+
+ def test_04_a_file_moved_between_tracked_folders_is_relocated(self, tmp_path):
+ source_dir = tmp_path / "inbox"
+ target_dir = tmp_path / "keepers"
+ source_dir.mkdir()
+ target_dir.mkdir()
+ LibraryFolderFactory(path=str(source_dir))
+ LibraryFolderFactory(path=str(target_dir))
+ photo = _place(fixture=FIXTURE_A, destination=source_dir / "DSCF0001.JPG")
+ sync_library()
+ image = models.Image.objects.get()
+ image.set_rating(5)
+
+ moved = target_dir / "DSCF0001.JPG"
+ photo.rename(moved)
+ sync_library()
+
+ image.refresh_from_db()
+ assert models.Image.objects.count() == 1
+ assert image.filepath == str(moved)
+ assert image.rating == 5
+
+ def test_05_a_file_moved_outside_every_tracked_folder_leaves_the_gallery(self, tmp_path):
+ library_dir = tmp_path / "library"
+ outside_dir = tmp_path / "outside"
+ library_dir.mkdir()
+ outside_dir.mkdir()
+ folder = LibraryFolderFactory(path=str(library_dir))
+ photo = _place(fixture=FIXTURE_A, destination=library_dir / "DSCF0001.JPG")
+ sync_folder(folder_id=folder.pk)
+
+ moved = outside_dir / "DSCF0001.JPG"
+ photo.rename(moved)
+ sync_folder(folder_id=folder.pk)
+
+ assert models.Image.objects.count() == 0
+ assert moved.exists()
+
+ def test_06_a_copy_alongside_the_original_adds_nothing(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ original = _place(fixture=FIXTURE_A, destination=tmp_path / "DSCF0001.JPG")
+ sync_folder(folder_id=folder.pk)
+
+ _place(fixture=FIXTURE_A, destination=tmp_path / "backup" / "DSCF0001.JPG")
+ sync_folder(folder_id=folder.pk)
+
+ assert models.Image.objects.count() == 1
+ assert models.Image.objects.get().filepath == str(original)
+
+ def test_07_a_copy_kept_after_the_original_goes_is_re_imported(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ original = _place(fixture=FIXTURE_A, destination=tmp_path / "DSCF0001.JPG")
+ sync_folder(folder_id=folder.pk)
+ copy = _place(fixture=FIXTURE_A, destination=tmp_path / "backup" / "DSCF0001.JPG")
+ sync_folder(folder_id=folder.pk)
+
+ original.unlink()
+ sync_folder(folder_id=folder.pk)
+
+ # Self-healing but lossy: the record went with the original and the copy
+ # comes back as a fresh import on the next pass. See ADR 013, risk 3.
+ assert _catalogued() == {str(copy)}
+
+ def test_08_a_file_edited_in_place_keeps_its_record(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ photo = _place(fixture=FIXTURE_A, destination=tmp_path / "DSCF0001.JPG")
+ sync_folder(folder_id=folder.pk)
+ image = models.Image.objects.get()
+
+ shutil.copy(FIXTURE_B, photo)
+ sync_folder(folder_id=folder.pk)
+
+ image.refresh_from_db()
+ assert models.Image.objects.count() == 1
+ assert image.filepath == str(photo)
+
+
+class TestDirectoryLevelScenarios:
+ def test_09_a_deleted_subfolder_takes_its_images_out_of_the_gallery(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ subdir = tmp_path / "2024"
+ _place(fixture=FIXTURE_A, destination=subdir / "DSCF0001.JPG")
+ _place(fixture=FIXTURE_B, destination=subdir / "DSCF0002.jpg")
+ kept = _place(fixture=FIXTURE_C, destination=tmp_path / "2025" / "DSCF0003.jpg")
+ sync_folder(folder_id=folder.pk)
+ assert models.Image.objects.count() == 3
+
+ shutil.rmtree(subdir)
+ sync_folder(folder_id=folder.pk)
+
+ assert _catalogued() == {str(kept)}
+
+ def test_10_a_subfolder_moved_out_of_the_tree_takes_its_images(self, tmp_path):
+ library_dir = tmp_path / "library"
+ library_dir.mkdir()
+ folder = LibraryFolderFactory(path=str(library_dir))
+ subdir = library_dir / "2024"
+ _place(fixture=FIXTURE_A, destination=subdir / "DSCF0001.JPG")
+ sync_folder(folder_id=folder.pk)
+
+ subdir.rename(tmp_path / "2024")
+ sync_folder(folder_id=folder.pk)
+
+ assert models.Image.objects.count() == 0
+
+ def test_11_a_renamed_subfolder_relocates_every_image_under_it(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ subdir = tmp_path / "2024"
+ _place(fixture=FIXTURE_A, destination=subdir / "DSCF0001.JPG")
+ _place(fixture=FIXTURE_B, destination=subdir / "DSCF0002.jpg")
+ sync_folder(folder_id=folder.pk)
+ original_ids = set(models.Image.objects.values_list("pk", flat=True))
+ for image in models.Image.objects.all():
+ image.set_rating(2)
+
+ renamed = tmp_path / "2024-spain"
+ subdir.rename(renamed)
+ sync_folder(folder_id=folder.pk)
+
+ assert set(models.Image.objects.values_list("pk", flat=True)) == original_ids
+ assert _catalogued() == {
+ str(renamed / "DSCF0001.JPG"),
+ str(renamed / "DSCF0002.jpg"),
+ }
+ assert all(image.rating == 2 for image in models.Image.objects.all())
+
+ def test_12_a_subfolder_moved_in_from_outside_is_imported(self, tmp_path):
+ library_dir = tmp_path / "library"
+ library_dir.mkdir()
+ folder = LibraryFolderFactory(path=str(library_dir))
+ sync_folder(folder_id=folder.pk)
+
+ incoming = tmp_path / "incoming"
+ _place(fixture=FIXTURE_A, destination=incoming / "DSCF0001.JPG")
+ incoming.rename(library_dir / "incoming")
+ sync_folder(folder_id=folder.pk)
+
+ assert _catalogued() == {str(library_dir / "incoming" / "DSCF0001.JPG")}
+
+
+class TestFolderLevelScenarios:
+ def test_13_removing_a_folder_only_keeps_its_images(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ _place(fixture=FIXTURE_A, destination=tmp_path / "DSCF0001.JPG")
+ sync_folder(folder_id=folder.pk)
+
+ removed = remove_library_folder(folder_id=folder.pk, delete_images=False)
+
+ assert removed == 0
+ assert models.Image.objects.count() == 1
+
+ def test_14_removing_a_folder_with_its_images_empties_the_gallery(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ photo = _place(fixture=FIXTURE_A, destination=tmp_path / "DSCF0001.JPG")
+ sync_folder(folder_id=folder.pk)
+
+ removed = remove_library_folder(folder_id=folder.pk, delete_images=True)
+
+ assert removed == 1
+ assert models.Image.objects.count() == 0
+ assert photo.exists()
+
+ def test_15_a_folder_moved_on_disk_relocates_its_images(self, tmp_path):
+ old_dir = tmp_path / "photos"
+ old_dir.mkdir()
+ folder = LibraryFolderFactory(path=str(old_dir))
+ _place(fixture=FIXTURE_A, destination=old_dir / "2024" / "DSCF0001.JPG")
+ sync_folder(folder_id=folder.pk)
+ image = models.Image.objects.get()
+ image.set_rating(5)
+
+ new_dir = tmp_path / "pictures"
+ old_dir.rename(new_dir)
+ update_library_folder_path(folder_id=folder.pk, path=str(new_dir))
+ sync_folder(folder_id=folder.pk)
+
+ image.refresh_from_db()
+ assert models.Image.objects.count() == 1
+ assert image.filepath == str(new_dir / "2024" / "DSCF0001.JPG")
+ assert image.rating == 5
+
+ def test_16_a_folder_missing_from_disk_removes_nothing(self, tmp_path):
+ library_dir = tmp_path / "external-drive"
+ library_dir.mkdir()
+ folder = LibraryFolderFactory(path=str(library_dir))
+ _place(fixture=FIXTURE_A, destination=library_dir / "DSCF0001.JPG")
+ _place(fixture=FIXTURE_B, destination=library_dir / "DSCF0002.jpg")
+ sync_folder(folder_id=folder.pk)
+ assert models.Image.objects.count() == 2
+
+ shutil.rmtree(library_dir)
+ sync_folder(folder_id=folder.pk)
+
+ assert models.Image.objects.count() == 2
+ run = models.SyncRun.objects.order_by("-id").first()
+ assert run.state == models.SyncRun.STATE_FAILED
+ assert run.failure_reason == models.SyncRun.FAILED_FOLDER_MISSING
+ assert run.removed == 0
+
+ def test_17_removing_a_nested_folder_keeps_images_the_outer_one_covers(self, tmp_path):
+ outer = LibraryFolderFactory(path=str(tmp_path))
+ inner_dir = tmp_path / "2024"
+ _place(fixture=FIXTURE_A, destination=inner_dir / "DSCF0001.JPG")
+ inner = LibraryFolderFactory(path=str(inner_dir))
+ sync_folder(folder_id=outer.pk)
+ assert models.Image.objects.count() == 1
+
+ removed = remove_library_folder(folder_id=inner.pk, delete_images=True)
+
+ assert removed == 0
+ assert models.Image.objects.count() == 1
+
+ def test_18_an_image_outside_every_tracked_folder_is_never_removed(self, tmp_path):
+ library_dir = tmp_path / "library"
+ library_dir.mkdir()
+ folder = LibraryFolderFactory(path=str(library_dir))
+ # Imported by hand from a folder that was never registered.
+ outside = _place(fixture=FIXTURE_A, destination=tmp_path / "elsewhere" / "DSCF0001.JPG")
+ process_image(image_path=str(outside))
+ outside.unlink()
+
+ sync_folder(folder_id=folder.pk)
+
+ assert _catalogued() == {str(outside)}
+
+ def test_19_a_symlinked_subfolder_is_never_pruned(self, tmp_path):
+ library_dir = tmp_path / "library"
+ library_dir.mkdir()
+ real_dir = tmp_path / "real"
+ photo = _place(fixture=FIXTURE_A, destination=real_dir / "DSCF0001.JPG")
+ folder = LibraryFolderFactory(path=str(library_dir))
+ (library_dir / "linked").symlink_to(real_dir)
+
+ # os.walk does not follow the symlink, so the sync never imports through
+ # it. Import the path by hand to put the record where a prune would see it.
+ linked_path = str(library_dir / "linked" / "DSCF0001.JPG")
+ process_image(image_path=linked_path)
+
+ sync_folder(folder_id=folder.pk)
+
+ assert linked_path in _catalogued()
+ assert photo.exists()
From 16610cc8164d4243558e5e5446d32ca880efb95c Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:53:32 +1000
Subject: [PATCH 19/36] Ask what should happen to a folder's images before
removing it
Removing a folder can now take its images out of the gallery, so the user
has to be able to choose, and to see what the choice costs before making
it. A browser confirm() cannot show a count.
The dialog names how many images come only from this folder, which is the
number that would actually go, and says plainly that the photo files stay
on disk. When nothing would leave the gallery it offers only the plain
removal. It reuses the modal shell the folder browser already has rather
than adding a second one.
Co-Authored-By: Claude Opus 5
---
.../library/get_folder_removal_preview.py | 43 ++++++++++++++++++
src/interfaces/library/urls.py | 1 +
src/interfaces/library/views.py | 17 +++++++
.../library/includes/folder_row.html | 12 ++---
src/interfaces/templates/library/library.html | 7 +++
.../partials/remove_folder_confirm.html | 35 +++++++++++++++
tests/functional/test_library_views.py | 44 +++++++++++++++++++
7 files changed, 151 insertions(+), 8 deletions(-)
create mode 100644 src/application/usecases/library/get_folder_removal_preview.py
create mode 100644 src/interfaces/templates/library/partials/remove_folder_confirm.html
diff --git a/src/application/usecases/library/get_folder_removal_preview.py b/src/application/usecases/library/get_folder_removal_preview.py
new file mode 100644
index 0000000..c04503b
--- /dev/null
+++ b/src/application/usecases/library/get_folder_removal_preview.py
@@ -0,0 +1,43 @@
+import attrs
+
+from src.domain.library import queries as library_queries
+
+
+@attrs.frozen
+class LibraryFolderNotFound(Exception):
+ """
+ Raised when no library folder with the given id exists.
+ """
+
+ folder_id: int
+
+
+@attrs.frozen
+class FolderRemovalPreview:
+ folder_id: int
+ path: str
+ removable_images: int
+
+
+def get_folder_removal_preview(*, folder_id: int) -> FolderRemovalPreview:
+ """
+ Describe what removing a library folder would cost, so the user can choose
+ before anything happens.
+
+ ``removable_images`` counts only images no other registered folder covers,
+ which is exactly what would leave the gallery. The image files themselves are
+ never deleted.
+
+ :raises LibraryFolderNotFound: If no folder with *folder_id* exists.
+ """
+ try:
+ folder = library_queries.get_library_folder(folder_id=folder_id)
+ removable = library_queries.count_exclusively_owned_images(folder_id=folder_id)
+ except library_queries.LibraryFolderNotFound:
+ raise LibraryFolderNotFound(folder_id=folder_id)
+
+ return FolderRemovalPreview(
+ folder_id=folder.pk,
+ path=folder.path,
+ removable_images=removable,
+ )
diff --git a/src/interfaces/library/urls.py b/src/interfaces/library/urls.py
index dbd4ee7..3fca856 100644
--- a/src/interfaces/library/urls.py
+++ b/src/interfaces/library/urls.py
@@ -6,6 +6,7 @@
path("library/", views.LibraryFolderList.as_view(), name="library-list"),
path("library/new/", views.LibraryFolderAdd.as_view(), name="library-folder-new"),
path("library/browse/partial/", views.FilesystemBrowser.as_view(), name="library-browse"),
+ path("library//confirm-delete/", views.LibraryFolderRemoveConfirm.as_view(), name="library-folder-confirm-delete"),
path("library//delete/", views.LibraryFolderRemove.as_view(), name="library-folder-delete"),
path("library//edit/", views.LibraryFolderPathUpdate.as_view(), name="library-folder-edit"),
path("library//sync-status/", views.LibraryFolderSyncStatus.as_view(), name="library-folder-sync-status"),
diff --git a/src/interfaces/library/views.py b/src/interfaces/library/views.py
index 7925445..577d665 100644
--- a/src/interfaces/library/views.py
+++ b/src/interfaces/library/views.py
@@ -4,6 +4,7 @@
from src.application.usecases.library import add_library_folder as add_library_folder_uc
from src.application.usecases.library import browse_filesystem as browse_filesystem_uc
from src.application.usecases.library import dataclasses as library_dataclasses
+from src.application.usecases.library import get_folder_removal_preview as get_folder_removal_preview_uc
from src.application.usecases.library import remove_library_folder as remove_library_folder_uc
from src.application.usecases.library import trigger_folder_sync as trigger_folder_sync_uc
from src.application.usecases.library import update_library_folder_path as update_library_folder_path_uc
@@ -83,6 +84,22 @@ def post(self, request: http.HttpRequest) -> http.HttpResponse:
return shortcuts.redirect(urls.reverse("library-list"))
+class LibraryFolderRemoveConfirm(generic.View):
+ """Show what removing a folder would cost before anything happens.
+
+ :raises Http404: if no folder with the given ID exists.
+ """
+
+ def get(self, request: http.HttpRequest, folder_id: int) -> http.HttpResponse:
+ try:
+ preview = get_folder_removal_preview_uc.get_folder_removal_preview(folder_id=folder_id)
+ except get_folder_removal_preview_uc.LibraryFolderNotFound:
+ raise http.Http404
+ return shortcuts.render(request, "library/partials/remove_folder_confirm.html", {
+ "preview": preview,
+ })
+
+
class LibraryFolderRemove(generic.View):
"""Remove a folder from the image library, optionally with its images.
diff --git a/src/interfaces/templates/library/includes/folder_row.html b/src/interfaces/templates/library/includes/folder_row.html
index d71417a..409b858 100644
--- a/src/interfaces/templates/library/includes/folder_row.html
+++ b/src/interfaces/templates/library/includes/folder_row.html
@@ -26,13 +26,9 @@
class="btn-update"
onclick="openBrowserModal({{ folder.folder_id }}, '{{ folder.path|escapejs }}')"
>Update Path
-
+
diff --git a/src/interfaces/templates/library/library.html b/src/interfaces/templates/library/library.html
index da5d01f..303f229 100644
--- a/src/interfaces/templates/library/library.html
+++ b/src/interfaces/templates/library/library.html
@@ -250,6 +250,13 @@
Image Library
document.getElementById('browser-modal').hidden = true;
}
+ // Reuses the same modal shell as the folder browser: the confirmation is just
+ // another partial rendered into it.
+ function openRemoveFolderModal(url) {
+ htmx.ajax('GET', url, {target: '#browser-modal-content'});
+ document.getElementById('browser-modal').hidden = false;
+ }
+
document.getElementById('browser-modal').addEventListener('click', function (e) {
if (e.target === this) closeBrowserModal();
});
diff --git a/src/interfaces/templates/library/partials/remove_folder_confirm.html b/src/interfaces/templates/library/partials/remove_folder_confirm.html
new file mode 100644
index 0000000..2e665cc
--- /dev/null
+++ b/src/interfaces/templates/library/partials/remove_folder_confirm.html
@@ -0,0 +1,35 @@
+
Remove folder from the library
+
+
{{ preview.path }}
+
+{% if preview.removable_images %}
+
+ {{ preview.removable_images }} image{{ preview.removable_images|pluralize }}
+ in the gallery come{{ preview.removable_images|pluralize:"s," }} only from this folder.
+
+{% else %}
+
No image in the gallery comes only from this folder.
+{% endif %}
+
+
+ Your photo files are never deleted. Removing images here only takes them out of the gallery;
+ everything stays on disk exactly where it is.
+
+
+
+
+
diff --git a/tests/functional/test_library_views.py b/tests/functional/test_library_views.py
index 4c2e2a2..66046c6 100644
--- a/tests/functional/test_library_views.py
+++ b/tests/functional/test_library_views.py
@@ -145,6 +145,50 @@ def test_removes_the_images_when_asked_to(self, client):
assert not models.Image.objects.filter(pk=image.pk).exists()
+@pytest.mark.django_db
+class TestLibraryFolderRemoveConfirm:
+ def test_shows_how_many_images_would_leave_the_gallery(self, client):
+ folder = LibraryFolderFactory(path="/photos")
+ ImageFactory(filepath="/photos/DSCF0001.JPG")
+ ImageFactory(filepath="/photos/2024/DSCF0002.JPG")
+
+ response = client.get(f"/library/{folder.pk}/confirm-delete/")
+
+ content = response.content.decode()
+ assert response.status_code == 200
+ assert "2" in content
+ assert "Remove folder and its 2 images from the gallery" in content
+
+ def test_says_the_files_stay_on_disk(self, client):
+ folder = LibraryFolderFactory(path="/photos")
+ ImageFactory(filepath="/photos/DSCF0001.JPG")
+
+ response = client.get(f"/library/{folder.pk}/confirm-delete/")
+
+ assert "Your photo files are never deleted" in response.content.decode()
+
+ def test_offers_only_the_folder_when_nothing_would_leave_the_gallery(self, client):
+ folder = LibraryFolderFactory(path="/photos")
+
+ response = client.get(f"/library/{folder.pk}/confirm-delete/")
+
+ content = response.content.decode()
+ assert "No image in the gallery comes only from this folder" in content
+ assert "delete_images" not in content
+
+ def test_reports_nothing_removable_for_a_folder_nested_in_another(self, client):
+ LibraryFolderFactory(path="/photos")
+ inner = LibraryFolderFactory(path="/photos/2024")
+ ImageFactory(filepath="/photos/2024/DSCF0001.JPG")
+
+ response = client.get(f"/library/{inner.pk}/confirm-delete/")
+
+ assert "No image in the gallery comes only from this folder" in response.content.decode()
+
+ def test_returns_404_for_unknown_folder_id(self, client):
+ assert client.get("/library/99999/confirm-delete/").status_code == 404
+
+
@pytest.mark.django_db
class TestLibraryFolderPathUpdate:
def test_updates_path_and_redirects_to_list(self, client, tmp_path):
From 6329abfe1fb3d1dbac165cf376a2e83182db3479 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:54:28 +1000
Subject: [PATCH 20/36] Add prune flags to the sync_library command
Removal has been happening automatically with no way to look before
leaping, no way to get past the safety guard, and no way to turn it off.
--dry-run-prune lists what would go and removes nothing, which is the flag
to reach for the first time the guard fires and you want to know what it
caught. --force-prune overrides the guard once you have looked.
--no-prune imports only.
The guard warning now names the folder and the counts and points at the
remedy, rather than leaving the user to guess why images they deleted are
still in the gallery. A missing folder says outright that nothing was
removed. PruneWarning carries the sampled paths the dry run prints.
Co-Authored-By: Claude Opus 5
---
.../usecases/library/sync_library.py | 2 +
.../management/commands/sync_library.py | 75 ++++++++++++++--
tests/functional/test_sync_library_command.py | 86 +++++++++++++++++++
3 files changed, 158 insertions(+), 5 deletions(-)
diff --git a/src/application/usecases/library/sync_library.py b/src/application/usecases/library/sync_library.py
index 84e5e37..f226cc0 100644
--- a/src/application/usecases/library/sync_library.py
+++ b/src/application/usecases/library/sync_library.py
@@ -31,6 +31,7 @@ class PruneWarning:
missing_found: int
total: int
reason: str
+ sample_paths: tuple[str, ...]
@attrs.frozen
@@ -152,6 +153,7 @@ def _sync_inline(
missing_found=result.missing_found,
total=result.total,
reason=result.skipped_reason,
+ sample_paths=result.sample_paths,
)
)
diff --git a/src/interfaces/management/commands/sync_library.py b/src/interfaces/management/commands/sync_library.py
index 6e7e5e1..b91f461 100644
--- a/src/interfaces/management/commands/sync_library.py
+++ b/src/interfaces/management/commands/sync_library.py
@@ -1,18 +1,42 @@
from typing import Any
from django.conf import settings
-from django.core.management.base import BaseCommand
+from django.core.management.base import BaseCommand, CommandParser
from src.application.usecases.library import sync_library as sync_library_usecase
from src.application.usecases.library.sync_library import CeleryWorkerUnavailable
+from src.data import models
class Command(BaseCommand):
- help = "Scan all library folders and import new images into the catalog."
+ help = (
+ "Scan all library folders, import new images into the catalog and remove entries whose"
+ " files are gone. Only catalog entries are removed; image files are never deleted."
+ )
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument(
+ "--force-prune",
+ action="store_true",
+ help="Remove missing images even when the mass-removal safety guard would stop it.",
+ )
+ group.add_argument(
+ "--dry-run-prune",
+ action="store_true",
+ help="Report which catalog entries would be removed without removing any.",
+ )
+ group.add_argument(
+ "--no-prune",
+ action="store_true",
+ help="Import only; never remove catalog entries for missing files.",
+ )
def handle(self, *args: object, **options: Any) -> None:
+ prune_mode = _prune_mode_from(options=options)
+
try:
- result = sync_library_usecase.sync_library()
+ result = sync_library_usecase.sync_library(prune_mode=prune_mode)
except CeleryWorkerUnavailable:
self.stdout.write(
self.style.WARNING(
@@ -29,14 +53,55 @@ def handle(self, *args: object, **options: Any) -> None:
f"{result.new_files_found} task(s) enqueued."
)
)
+ self.stdout.write(
+ " Images whose files are gone are removed by the worker once it finishes."
+ )
else:
self.stdout.write(
self.style.SUCCESS(
f"Library sync complete: {result.folders_scanned} folder(s) scanned, "
f"{result.new_files_found} new file(s) imported, "
- f"{result.skipped_non_fujifilm} skipped (non-Fujifilm)."
+ f"{result.skipped_non_fujifilm} skipped (non-Fujifilm), "
+ f"{result.images_removed} image(s) removed from the gallery."
)
)
for path in result.missing_folders:
- self.stdout.write(self.style.WARNING(f" Missing folder (no longer on disk): {path}"))
+ self.stdout.write(
+ self.style.WARNING(
+ f" Missing folder (no longer on disk): {path}."
+ " Nothing was removed from the gallery."
+ )
+ )
+
+ for warning in result.prune_warnings:
+ self._report(warning=warning)
+
+ def _report(self, *, warning: sync_library_usecase.PruneWarning) -> None:
+ if warning.reason == models.SyncRun.SKIPPED_DRY_RUN:
+ self.stdout.write(
+ f" Would remove {warning.missing_found} of {warning.total} image(s)"
+ f" from the gallery for {warning.folder_path}:"
+ )
+ for path in warning.sample_paths:
+ self.stdout.write(f" {path}")
+ return
+
+ self.stdout.write(
+ self.style.WARNING(
+ f" Skipped removing {warning.missing_found} of {warning.total} image(s)"
+ f" in {warning.folder_path} (safety guard)."
+ " That usually means a drive is not mounted rather than that the photos were"
+ " deleted. Re-run with --force-prune to remove them anyway."
+ )
+ )
+
+
+def _prune_mode_from(*, options: dict[str, Any]) -> str:
+ if options["force_prune"]:
+ return models.SyncRun.PRUNE_MODE_FORCE
+ if options["dry_run_prune"]:
+ return models.SyncRun.PRUNE_MODE_DRY_RUN
+ if options["no_prune"]:
+ return models.SyncRun.PRUNE_MODE_OFF
+ return models.SyncRun.PRUNE_MODE_AUTO
diff --git a/tests/functional/test_sync_library_command.py b/tests/functional/test_sync_library_command.py
index db3812b..3979b57 100644
--- a/tests/functional/test_sync_library_command.py
+++ b/tests/functional/test_sync_library_command.py
@@ -4,6 +4,7 @@
import pytest
from django.core.management import call_command
+from django.core.management.base import CommandError
from django.test import override_settings
from src.application.usecases.library.sync_library import CeleryWorkerUnavailable
@@ -59,3 +60,88 @@ def test_prints_warning_and_exits_when_celery_worker_unavailable(self, capsys):
captured = capsys.readouterr()
assert "No Celery worker is reachable" in captured.out
+
+
+@pytest.mark.django_db
+class TestSyncLibraryCommandRemovesMissingImages:
+ @pytest.fixture(autouse=True)
+ def _lite_mode(self, settings):
+ settings.USE_ASYNC_TASKS = False
+
+ def _library_with_a_deleted_photo(self, tmp_path):
+ photo = tmp_path / FUJIFILM_FIXTURE.name
+ shutil.copy(FUJIFILM_FIXTURE, photo)
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ call_command("sync_library")
+ photo.unlink()
+ return folder
+
+ def test_reports_how_many_images_left_the_gallery(self, tmp_path, capsys):
+ self._library_with_a_deleted_photo(tmp_path)
+
+ call_command("sync_library")
+
+ assert "1 image(s) removed from the gallery" in capsys.readouterr().out
+ assert models.Image.objects.count() == 0
+
+ def test_dry_run_reports_without_removing_anything(self, tmp_path, capsys):
+ self._library_with_a_deleted_photo(tmp_path)
+
+ call_command("sync_library", "--dry-run-prune")
+
+ captured = capsys.readouterr()
+ assert "Would remove 1 of 1 image(s)" in captured.out
+ assert FUJIFILM_FIXTURE.name in captured.out
+ assert models.Image.objects.count() == 1
+
+ def test_no_prune_removes_nothing(self, tmp_path, capsys):
+ self._library_with_a_deleted_photo(tmp_path)
+
+ call_command("sync_library", "--no-prune")
+
+ assert models.Image.objects.count() == 1
+
+ def test_rejects_conflicting_prune_flags(self, tmp_path):
+ with pytest.raises(CommandError):
+ call_command("sync_library", "--no-prune", "--force-prune")
+
+ def test_says_nothing_was_removed_when_a_folder_is_missing(self, tmp_path, capsys):
+ LibraryFolderFactory(path=str(tmp_path / "does_not_exist"))
+
+ call_command("sync_library")
+
+ assert "Nothing was removed from the gallery" in capsys.readouterr().out
+
+
+@pytest.mark.django_db
+class TestSyncLibraryCommandSafetyGuard:
+ @pytest.fixture(autouse=True)
+ def _strict_guard(self, settings):
+ settings.USE_ASYNC_TASKS = False
+ settings.LIBRARY_PRUNE_GUARD_FRACTION = 0.5
+ settings.LIBRARY_PRUNE_GUARD_MIN_IMAGES = 1
+
+ def _library_emptied_on_disk(self, tmp_path):
+ for fixture in (FUJIFILM_FIXTURE, FIXTURES_DIR / "XS107209.jpg", FIXTURES_DIR / "XS107336.jpg"):
+ shutil.copy(fixture, tmp_path / fixture.name)
+ LibraryFolderFactory(path=str(tmp_path))
+ call_command("sync_library")
+ for photo in tmp_path.iterdir():
+ photo.unlink()
+
+ def test_warns_and_removes_nothing_when_the_guard_trips(self, tmp_path, capsys):
+ self._library_emptied_on_disk(tmp_path)
+
+ call_command("sync_library")
+
+ captured = capsys.readouterr()
+ assert "Skipped removing 3 of 3 image(s)" in captured.out
+ assert "--force-prune" in captured.out
+ assert models.Image.objects.count() == 3
+
+ def test_force_prune_overrides_the_guard(self, tmp_path, capsys):
+ self._library_emptied_on_disk(tmp_path)
+
+ call_command("sync_library", "--force-prune")
+
+ assert models.Image.objects.count() == 0
From 0163f68eb2a32f8a1d3e5a43ca73f2b981ef4d23 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 21:55:30 +1000
Subject: [PATCH 21/36] Report removals and their warnings on the Library page
The Sync column knew nothing about removal: a prune the guard stopped was
invisible, and a folder missing from disk rendered a bare "Sync failed"
with no reason, which is the single most alarming thing the page can show
and the least informative.
It now reports how many images a run removed, shows the prune phase while
it runs, and carries both warnings. The missing-folder one says "Folder
not found on disk. Nothing was removed from the gallery", because an
unplugged drive looks exactly like a mass deletion and that is the moment
the reassurance is worth most.
The view resolves the stored codes into booleans so templates never
compare against database values. Warnings are styled as warnings rather
than errors: nothing is broken and nothing was lost.
Co-Authored-By: Claude Opus 5
---
.../usecases/library/dataclasses.py | 7 +++
src/interfaces/library/views.py | 5 ++
src/interfaces/templates/library/library.html | 16 +++++
.../library/partials/sync_status.html | 15 ++++-
.../test_library_sync_status_view.py | 59 +++++++++++++++++++
5 files changed, 101 insertions(+), 1 deletion(-)
diff --git a/src/application/usecases/library/dataclasses.py b/src/application/usecases/library/dataclasses.py
index 4f208f1..b13b70f 100644
--- a/src/application/usecases/library/dataclasses.py
+++ b/src/application/usecases/library/dataclasses.py
@@ -21,12 +21,19 @@ class SyncRunData:
errors: int
handled: int
percent: int
+ removed: int
+ missing_found: int
is_active: bool
is_scanning: bool
is_processing: bool
+ is_pruning: bool
is_completed: bool
is_failed: bool
is_interrupted: bool
+ # Resolved from the stored codes here so templates never compare against
+ # database values.
+ folder_is_missing: bool
+ prune_skipped_by_guard: bool
@attrs.frozen
diff --git a/src/interfaces/library/views.py b/src/interfaces/library/views.py
index 577d665..84f31d1 100644
--- a/src/interfaces/library/views.py
+++ b/src/interfaces/library/views.py
@@ -38,12 +38,17 @@ def _sync_status(run: models.SyncRun) -> library_dataclasses.SyncRunData:
errors=run.errors,
handled=handled,
percent=percent,
+ removed=run.removed,
+ missing_found=run.missing_found,
is_active=run.state in models.SyncRun.ACTIVE_STATES,
is_scanning=run.state == models.SyncRun.STATE_SCANNING,
is_processing=run.state == models.SyncRun.STATE_PROCESSING,
+ is_pruning=run.state == models.SyncRun.STATE_PRUNING,
is_completed=run.state == models.SyncRun.STATE_COMPLETED,
is_failed=run.state == models.SyncRun.STATE_FAILED,
is_interrupted=run.state == models.SyncRun.STATE_INTERRUPTED,
+ folder_is_missing=run.failure_reason == models.SyncRun.FAILED_FOLDER_MISSING,
+ prune_skipped_by_guard=run.prune_skipped == models.SyncRun.SKIPPED_GUARD,
)
diff --git a/src/interfaces/templates/library/library.html b/src/interfaces/templates/library/library.html
index 303f229..b2429f4 100644
--- a/src/interfaces/templates/library/library.html
+++ b/src/interfaces/templates/library/library.html
@@ -134,6 +134,22 @@
.btn-remove:hover { background: #fef2f2; }
+ /* Sync status */
+ .sync-status__label--done { color: #16a34a; }
+ .sync-status__label--error { color: #dc2626; }
+
+ /* A folder missing from disk, or a prune the guard stopped, needs to stand
+ out without reading as a failure: nothing is broken and nothing was lost. */
+ .sync-status__label--warning {
+ display: inline-block;
+ color: #92400e;
+ background: #fef3c7;
+ border: 1px solid #fcd34d;
+ border-radius: 4px;
+ padding: 0.1rem 0.4rem;
+ cursor: help;
+ }
+
/* Browser modal */
.browser-overlay {
position: fixed;
diff --git a/src/interfaces/templates/library/partials/sync_status.html b/src/interfaces/templates/library/partials/sync_status.html
index c8aed04..ca2a1ca 100644
--- a/src/interfaces/templates/library/partials/sync_status.html
+++ b/src/interfaces/templates/library/partials/sync_status.html
@@ -12,8 +12,21 @@
{% elif status.is_processing %}
Processing {{ status.handled }}/{{ status.total }}
+ {% elif status.is_pruning %}
+ Removing missing images…
{% elif status.is_completed %}
- Imported {{ status.processed }}{% if status.skipped %}, skipped {{ status.skipped }}{% endif %}{% if status.errors %}, {{ status.errors }} error{{ status.errors|pluralize }}{% endif %}
+ Imported {{ status.processed }}{% if status.skipped %}, skipped {{ status.skipped }}{% endif %}{% if status.removed %}, removed {{ status.removed }}{% endif %}{% if status.errors %}, {{ status.errors }} error{{ status.errors|pluralize }}{% endif %}
+ {% if status.prune_skipped_by_guard %}
+ Skipped removing {{ status.missing_found }} missing image{{ status.missing_found|pluralize }}
+ {% endif %}
+ {% elif status.folder_is_missing %}
+ Folder not found on disk. Nothing was removed from the gallery.
{% elif status.is_failed %}
Sync failed
{% elif status.is_interrupted %}
diff --git a/tests/functional/test_library_sync_status_view.py b/tests/functional/test_library_sync_status_view.py
index 4f33fee..8e7cb88 100644
--- a/tests/functional/test_library_sync_status_view.py
+++ b/tests/functional/test_library_sync_status_view.py
@@ -78,3 +78,62 @@ def test_shows_failed_state(self, client):
content = response.content.decode()
assert "Sync failed" in content
assert "hx-trigger" not in content
+
+
+@pytest.mark.django_db
+class TestLibraryFolderSyncStatusRemovals:
+ def test_shows_removed_count_on_a_completed_run(self, client):
+ folder = LibraryFolderFactory()
+ run = SyncRunFactory(folder=folder, state=models.SyncRun.STATE_COMPLETED, total=2)
+ run.record_prune_result(missing_found=3, removed=3, skipped_reason="")
+
+ response = client.get(f"/library/{folder.pk}/sync-status/")
+
+ assert "removed 3" in response.content.decode()
+
+ def test_polls_while_pruning(self, client):
+ folder = LibraryFolderFactory()
+ SyncRunFactory(folder=folder, state=models.SyncRun.STATE_PRUNING, total=2)
+
+ response = client.get(f"/library/{folder.pk}/sync-status/")
+
+ content = response.content.decode()
+ assert "Removing missing images" in content
+ assert 'hx-trigger="every 2s"' in content
+
+ def test_warns_when_the_guard_skipped_a_removal(self, client):
+ folder = LibraryFolderFactory()
+ run = SyncRunFactory(folder=folder, state=models.SyncRun.STATE_COMPLETED, total=0)
+ run.record_prune_result(
+ missing_found=30, removed=0, skipped_reason=models.SyncRun.SKIPPED_GUARD
+ )
+
+ response = client.get(f"/library/{folder.pk}/sync-status/")
+
+ content = response.content.decode()
+ assert "Skipped removing 30 missing image" in content
+ assert "--force-prune" in content
+
+ def test_explains_a_folder_that_is_missing_from_disk(self, client):
+ folder = LibraryFolderFactory()
+ run = SyncRunFactory(folder=folder, state=models.SyncRun.STATE_SCANNING)
+ run.mark_failed(
+ reason=models.SyncRun.FAILED_FOLDER_MISSING,
+ message="Folder does not exist",
+ )
+
+ response = client.get(f"/library/{folder.pk}/sync-status/")
+
+ content = response.content.decode()
+ assert "Folder not found on disk" in content
+ assert "Nothing was removed from the gallery" in content
+ assert "Sync failed" not in content
+
+ def test_still_reports_a_plain_failure_without_a_reason_code(self, client):
+ folder = LibraryFolderFactory()
+ run = SyncRunFactory(folder=folder, state=models.SyncRun.STATE_SCANNING)
+ run.mark_failed(reason="", message="something else went wrong")
+
+ response = client.get(f"/library/{folder.pk}/sync-status/")
+
+ assert "Sync failed" in response.content.decode()
From d131fb7fe8e78a83dc8bf22be8791f81e0dcbe29 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sat, 8 Aug 2026 15:28:30 +1000
Subject: [PATCH 22/36] Document image removal and add ADR 013
ADR 013 records the decisions and carries the scenario matrix as the
specification of what "consistent with the folders" means,
cross-referenced to the test module that covers each numbered row.
ADRs 010 and 011 are left untouched: they record decisions that were taken
at the time, and a later decision does not get to rewrite them. The
supersession is stated in ADR 013 and in the docs index, following the
same direction ADR 012 used for ADR 005.
The user-facing docs lead with the thing that matters most: Filmcase never
deletes a photo file, and "removing" an image only removes its catalog
entry. The gating section is gone, since gating is.
Co-Authored-By: Claude Opus 5
---
...013-library-sync-removes-missing-images.md | 214 ++++++++++++++++++
docs/index.md | 1 +
docs/library_sync.md | 97 ++++++--
docs/management_commands.md | 28 ++-
docs/web_interface.md | 21 +-
5 files changed, 329 insertions(+), 32 deletions(-)
create mode 100644 docs/ADRs/013-library-sync-removes-missing-images.md
diff --git a/docs/ADRs/013-library-sync-removes-missing-images.md b/docs/ADRs/013-library-sync-removes-missing-images.md
new file mode 100644
index 0000000..919a730
--- /dev/null
+++ b/docs/ADRs/013-library-sync-removes-missing-images.md
@@ -0,0 +1,214 @@
+# ADR 013 — Library sync removes images that are gone from disk
+
+**Status**: Accepted
+**Date**: 2026-08-08
+**Supersedes**: the *add-only sync* and *mtime gating* decisions of [ADR 010](010-image-library.md), and the *remove is pure deregistration* and *reset `last_checked_at` on a path update* decisions of [ADR 011](011-library-sync-on-folder-change.md) (everything else in both still stands)
+
+---
+
+## Context
+
+ADR 010 introduced the Library and made its sync deliberately add-only: it walks each registered folder, diffs what it finds against every known `Image.filepath`, and imports what is new. ADR 011 kept that stance, stating that removing a folder "just deregisters it".
+
+That was the right first step, but it leaves the catalog permanently wrong the moment anything on disk changes:
+
+- a deleted photo stays in the gallery with a dead `filepath`, so its thumbnail and its full-size view both 404, forever;
+- a renamed or moved photo is re-imported under its new path. The hash match in `find_existing_image_for_import` stops a duplicate row being created, but the record kept pointing at the old, nonexistent path;
+- removing a folder from the Library left every image it had imported behind, with no way to tell where they came from.
+
+The Library is presented to the user as the thing that keeps the gallery in step with their folders. In one direction it did not.
+
+## Problem
+
+"Keep the gallery consistent with the folders" sounds simple and is not, because the filesystem does not tell us what happened. It only shows us a before and an after. A move is indistinguishable from a delete plus an unrelated add unless something ties the two together. Several forces pull against each other:
+
+- **User data lives on the record, not the file.** Rating, favourite and album membership are in the database. Treating a move as a delete plus an add silently throws them away.
+- **Removal is destructive and, if we hard-delete, irreversible.** Every false positive is permanent, so the cost of over-removing is much higher than the cost of under-removing.
+- **"Missing" is ambiguous.** An unplugged external drive, an unmounted network share and a directory that has become unreadable all look exactly like "the user deleted everything".
+- **Two install modes** (ADR 003). In lite mode a sync finishes inline; in full mode the Celery tasks outlive the command that started them, so "the import is done" is not a moment the caller can observe.
+- **Not every catalogued image belongs to the Library.** `manage.py process_images` imports from any folder, and those images must never be touched by a library prune.
+
+So: how do we remove what is genuinely gone, without ever removing what is merely out of sight, and without losing user data to a rename?
+
+## Scenario matrix
+
+This matrix is the specification. Every numbered row has a matching test in `tests/integration/domain/library/test_prune_scenarios.py`, with the scenario number in the test name.
+
+"Relocate" means the same `Image` record follows the file, keeping its rating, favourite flag and album membership. "Removed" always means removed from the gallery; **the image file itself is never touched**.
+
+### File level, inside a tracked folder
+
+| # | Scenario | Expected |
+|---|---|---|
+| 1 | File deleted | Image removed |
+| 2 | File renamed in place | Relocate |
+| 3 | File moved to another subfolder of the same tracked folder | Relocate |
+| 4 | File moved to a different tracked folder | Relocate (see consequence 2) |
+| 5 | File moved outside every tracked folder | Image removed |
+| 6 | File copied, original kept | Nothing; hash dedup resolves the copy to the existing record |
+| 7 | File copied, then original deleted | Record removed, copy re-imported next sync (see consequence 3) |
+| 8 | File edited in place, same path, different bytes | Record keeps its path; content is not re-read |
+
+### Directory level, inside a tracked folder
+
+| # | Scenario | Expected |
+|---|---|---|
+| 9 | Subfolder deleted | Every image under it removed |
+| 10 | Subfolder moved out of the tree | Every image under it removed |
+| 11 | Subfolder renamed | Every image under it relocated, none removed |
+| 12 | Subfolder moved in from outside | Imported as new |
+
+### Folder level
+
+| # | Scenario | Expected |
+|---|---|---|
+| 13 | Folder removed from the Library, "folder only" | Images kept |
+| 14 | Folder removed from the Library, "and its images" | Images removed, except any another registered folder also covers |
+| 15 | Folder's path edited because the folder itself moved | Every image relocated, none removed |
+| 16 | Tracked folder missing on disk (unplugged drive) | Run fails, **nothing removed** |
+| 17 | Nested folders both registered (`/Photos` and `/Photos/2024`) | Removing the inner one takes nothing the outer one still covers |
+| 18 | Image imported by `process_images` from an untracked folder | **Never** removed |
+| 19 | Symlinked subfolder, broken symlink, unreadable directory | Never removed |
+
+---
+
+## Decisions
+
+### The app never touches your files
+
+**"Remove" always means "remove from the gallery", never "delete from disk."** Nothing in this feature writes to, moves or deletes an image file. The only filesystem writes are to Filmcase's own derived thumbnail cache. Every piece of user-facing copy says so: the folder-removal dialog, the command output and the docs all say "remove from the gallery", never a bare "delete".
+
+This is not decoration. The one case where a photo disappears from the gallery is the case where the user already deleted the file themselves, and a photo manager that is vague about which of the two it is doing is a photo manager nobody should trust with their library.
+
+### Hard delete of the catalog row
+
+The `Image` row is deleted outright, and its `FujifilmExif` row with it if that leaves it orphaned (the same garbage collection `merge_image_into` already does). The `FujifilmRecipe` is **never** deleted: recipes are shared, are the point of the app, and are worth keeping with no images left.
+
+Two schema-level side effects are deliberately left alone rather than compensated for:
+
+- `FujifilmRecipe.cover_image` is `SET_NULL`. Nulling it restores the automatic fallback to the recipe's most-used image, which `get_recipe_data` already resolves. Picking a replacement would fabricate an explicit user choice that was never made.
+- `RecipeCard.image` is `SET_NULL`, and cards survive. A card is a rendered JPEG that stands on its own and outlives its source photo.
+
+**Alternative considered: quarantine.** Mark the record instead of deleting it, hide it from the gallery, and offer a restore view. It neutralises the whole false-positive class and would let both the deferral rule and the two-phase lite sync below be dropped. It was rejected for now because it touches every gallery and filter query and adds a UI surface, for a single-user local app where the safety guard already covers the dangerous case. A cheaper version of the same idea, a `missing_since` timestamp requiring a path to be missing on two consecutive syncs, remains the obvious next step if the consequences below ever bite.
+
+### A move is a relocation, not a delete plus an add
+
+`relocate_image` repoints a record at its new path. An import that matches an existing record by content hash is either a move or a copy, and **the old file decides which**: if it is gone the bytes were renamed or moved and the record follows; if it is still there this is a second copy and the record stays put. That single test separates scenario 2 from scenario 6.
+
+This also settles scenario 15 with no special case at all. After a folder's path is edited every file below it looks new, each hash-matches a record whose old path has gone, and every record relocates.
+
+Ordering is what makes it work: **imports always run before the prune**. By the time anything is removed, a moved file has already repointed its record, so it no longer looks missing.
+
+### Detecting what is missing: walk to narrow, stat to confirm
+
+Two strategies were compared.
+
+**Set difference from the walk** is cheap but produces false positives that hard delete makes permanent: `os.walk` does not follow symlinked directories, silently yields nothing for a directory it cannot read (its `onerror` default swallows `EACCES`), and matches only JPEG extensions, so any catalogued record it cannot see looks deleted.
+
+**A stat per catalogued path** is exact but costs a syscall per image in the library.
+
+**Chosen: the hybrid.** The set difference produces candidates; `os.path.lexists` on each candidate has the final say. Correctness is the stat approach's, cost is the walk's, because the candidate set is normally empty. `lexists`, not `exists`: a broken symlink still occupies the path, and for a destructive step "something is there" has to mean "keep the record".
+
+### A mass-removal safety guard
+
+A pass that would remove more than `LIBRARY_PRUNE_GUARD_FRACTION` of a folder's catalogued images, **and** more than `LIBRARY_PRUNE_GUARD_MIN_IMAGES` of them, is reported instead of applied. Both thresholds must be exceeded, so a small folder emptied on purpose and a large folder losing a handful are both applied without complaint. What the guard exists to catch is the shape of an unmounted drive: nearly everything gone at once.
+
+`sync_library --force-prune` overrides it. `--dry-run-prune` reports what would go without removing anything. `--no-prune` imports only.
+
+### A `PRUNING` run state, not a post-completion callback
+
+`SyncRun.mark_completed()` was already an exactly-one-winner conditional update, so it looked like the natural hook. Pruning *after* it breaks three things: the HTMX poller stops before `removed` is written; a second sync of the folder can start while the prune is still walking, because the unique-active-run constraint no longer applies, and could re-import files the prune is about to remove; and a crash mid-prune leaves a run claiming `COMPLETED`.
+
+Moving the election one step earlier into a `PRUNING` state fixes all three at once. `PRUNING` joins `ACTIVE_STATES`, so the constraint holds, the poller keeps polling, and `interrupt_active_sync_runs()` already recovers a crashed prune with no new code.
+
+`mark_completed()` was replaced by a general `transition_state(from_states, to_state, finished_at)`. The guarded `UPDATE ... WHERE` stays on the model because it is a persistence concern (moving it into Python would make it a read-then-write race), but it carries no policy: which transitions are legal, and what each one means, belong to the domain operation that calls it.
+
+### Where the prune runs
+
+In the `finalize_sync_run` use case, which every possible last-caller reaches: each per-image task or thread as it finishes, and a dedicated task for a run that had no images at all. The domain never learns about Celery; the use case owns the orchestration.
+
+**In full mode the prune always happens in the worker, never in whoever started the sync.** The tempting shortcut is to finalise inline when the scan found nothing new, since there is no work to hand off. That is exactly the pure-deletion case, so it would put a second full walk of the folder and every removal that follows it inside the startup command, before the server is reachable, or inside a web request. It would also mean the same work ran in the worker or in the caller depending only on whether anything happened to be new. Lite mode still finalises inline, because there is nothing to hand off to.
+
+Failed and interrupted runs never reach it, which is scenario 16: a folder missing from disk fails its run and removes nothing.
+
+Two protections against cross-folder moves (scenario 4):
+
+- **Deferral.** Before pruning, the use case checks whether any *other* library folder still has images left to import. If one does, it defers; the next sync prunes. Delaying a removal is always safe, removing early is not. The test is images outstanding rather than merely an open run: a run that has accounted for every image cannot re-point anything, and treating it as a reason to wait would make a folder with nothing to import block its neighbours for no gain.
+- **Two-phase lite sync.** Lite mode scans folders one after another, so deferral has nothing to see: folder A's prune would run before folder B had even been looked at. `sync_library` therefore imports every folder first with pruning off, then prunes each one.
+
+### mtime gating is retired
+
+`collect_image_paths` used to skip directories whose mtime predated the folder's `last_checked_at`. That has to go, because **renaming a directory updates its parent's mtime and never its own**. A renamed subtree keeps an old mtime, and `last_checked_at` only moves forward, so a gated walk never revisits it. Scenario 11 would have removed every image under a renamed folder and never found them again: permanent, silent loss.
+
+The gate also bought almost nothing. `os.walk` has already listed each directory by the time the check runs, so the gate *added* a `getmtime()` per directory and saved only a filename suffix check. The expensive work, reading EXIF and hashing, was already avoided by the known-paths diff.
+
+`last_checked_at` itself stays: it is shown in the Library page's "Last Checked" column and is the only evidence a sync ran when nothing changed. Only its gating role is gone, which also makes `clear_last_checked_at` (ADR 011's path-update reset) unnecessary.
+
+### Removing a folder offers a choice
+
+The Remove button opens a confirmation showing how many images come **only** from this folder, and offers "remove folder only" or "remove folder and its N images from the gallery", stating plainly that the files stay on disk. Ownership excludes anything another registered folder also covers, which is scenario 17 in both directions.
+
+### Failures carry a code
+
+`SyncRun.failure_reason` distinguishes `FAILED_FOLDER_MISSING` from any other failure. Before this, `sync_library` inferred "folder is missing" from `state == FAILED`, so every failure was reported as a missing folder, and the Library page showed a bare "Sync failed" with no reason at all. It now says:
+
+> Folder not found on disk. Nothing was removed from the gallery.
+
+That reassurance is the point. An unplugged drive looks exactly like a mass deletion, and it is the moment a user most needs to be told that nothing was lost.
+
+---
+
+## Consequences
+
+1. **Hard delete makes every false positive permanent.** The `lexists` confirmation and the guard are the only nets. The `missing_since` two-strike scheme described above remains the cheapest way to remove that risk entirely.
+2. **A cross-folder move is protected by deferral only while the other folder's run is active.** If the destination folder is registered much later, the source folder's prune has already removed the record and the destination re-imports it as new, losing rating and favourite.
+3. **Copy-then-delete inside one sync window** (scenario 7) is self-healing but lossy for the same reason: the record goes with the original and the copy returns as a fresh import.
+4. **The guard has a blind spot by design.** Needing *both* thresholds means wiping a 15-photo folder never trips it. That is intentional (it must not nag on ordinary cleanups) but it is the common case for small folders.
+5. **The guard is per folder, not per subtree.** Deleting one 500-image subfolder from a 5,000-image library is 10%, so it is applied silently.
+6. **`--force-prune` is global**, bypassing the guard for every folder in the pass. A per-folder "remove them" button on the Library warning is the natural follow-up; `prune_folder` already exists as a use case to back it.
+7. **Two full walks per sync**, one to scan and one to prune, on top of retiring the gate. In full mode the second walk happens in the worker, so it does not hold up `make start`; in lite mode both run in the same process and could share their result if it ever bites.
+8. **`PRUNING` is an active state**, so a process killed mid-prune blocks that folder from syncing until the next startup `interrupt_active_sync_runs()`. Same recovery model as `PROCESSING`, but a new window.
+9. **A relocated image's cached thumbnail is orphaned**, not moved, and regenerates on the next view. Renaming folders is rare and regeneration is cheap. Removal *does* clear the cache, because cache keys are derived from the path and Fujifilm filenames wrap around from `DSCF9999` to `DSCF0001`: a later file reusing a path would otherwise be served the previous image's thumbnail.
+
+---
+
+## Diagrams
+
+### Where the prune sits in a run
+
+```mermaid
+stateDiagram-v2
+ [*] --> SCANNING: start_sync_run
+ SCANNING --> PROCESSING: begin_processing(total)
+ PROCESSING --> PRUNING: begin_pruning (exactly one caller wins)
+ PRUNING --> COMPLETED: complete_sync_run
+ PROCESSING --> COMPLETED: complete_sync_run (prune deferred or off)
+ SCANNING --> FAILED: folder missing on disk, nothing removed
+ PROCESSING --> INTERRUPTED: process died
+ PRUNING --> INTERRUPTED: process died
+ COMPLETED --> [*]
+ FAILED --> [*]
+ INTERRUPTED --> [*]
+```
+
+### Deciding a single file's fate
+
+```mermaid
+flowchart TD
+ A[Path found by the walk] --> B{Already in the catalog?}
+ B -->|yes| Z[Nothing to do]
+ B -->|no| C[process_image: read EXIF, hash]
+ C --> D{Hash matches an existing record?}
+ D -->|no| E[Create a new record]
+ D -->|yes| F{Does that record's stored file still exist?}
+ F -->|yes| G[A copy: leave the record where it is]
+ F -->|no| H[A move: relocate the record, keeping rating and favourite]
+
+ I[Catalogued path under the folder] --> J{Found by the walk?}
+ J -->|yes| Z2[Keep]
+ J -->|no| K{os.path.lexists?}
+ K -->|yes| Z2
+ K -->|no| L{Guard tripped?}
+ L -->|yes| M[Report, remove nothing]
+ L -->|no| N[Remove from the gallery, file untouched]
+```
diff --git a/docs/index.md b/docs/index.md
index 5278b50..b5807ed 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -36,4 +36,5 @@
- [ADR 010 — Image Library: Folder Monitoring and Catalog Sync](ADRs/010-image-library.md) — persisting monitored folders and detecting and importing new images automatically at startup
- [ADR 011 — Library Sync on Folder Add/Update](ADRs/011-library-sync-on-folder-change.md) — triggering a single-folder sync from the Library page with server-side, DB-backed progress in both install modes
- [ADR 012 — Pluggable Card Designs](ADRs/012-pluggable-card-designs.md) — a CardDesign abstraction replacing the flat CardTemplate, enabling fundamentally different card layouts (supersedes ADR 005's composition model)
+- [ADR 013 — Library Sync Removes Missing Images](ADRs/013-library-sync-removes-missing-images.md) — removing catalog entries whose files are gone, telling a move apart from a deletion, and the guard against a mass wipe (supersedes ADR 010's add-only sync and mtime gating, and ADR 011's removal and rescan decisions)
diff --git a/docs/library_sync.md b/docs/library_sync.md
index bc2e330..aeb4bf0 100644
--- a/docs/library_sync.md
+++ b/docs/library_sync.md
@@ -31,7 +31,9 @@ run and excludes it from subsequent folders.
**Missing folders.** If a registered folder is no longer present on disk, the command records
it as missing, updates its last-checked timestamp, and moves on to the next folder. The
-missing path is reported in the command output and does not abort the sync.
+missing path is reported in the command output and does not abort the sync. **Nothing is
+removed from the gallery in this case**, because an unplugged drive looks exactly like a folder
+whose photos were all deleted, and the safe reading is that the drive will come back.
**Processing new files.** New images are handled according to your install mode:
@@ -47,8 +49,8 @@ missing path is reported in the command output and does not abort the sync.
You no longer have to restart the app to pick up a newly registered folder. Adding a folder,
or changing an existing folder's path, triggers a sync of that one folder straight away.
-Removing a folder does not trigger anything: it only stops the folder being monitored and
-never deletes images that were already imported.
+Removing a folder asks what you want to happen to its images: you can keep them in the gallery
+or take them out along with the folder. Either way the photo files themselves stay on disk.
The triggered sync reuses the same per-folder scan described above and behaves according to
your install mode:
@@ -60,25 +62,9 @@ your install mode:
once. If no worker is reachable, the folder is still added but a message explains that it
could not be synced (start a worker with `make worker`, then re-add or re-save the folder).
-Changing a folder's path also clears its last-checked timestamp, so the whole new location is
-rescanned from scratch. Progress appears live in the folder's **Sync** column (see below).
-
-## Timestamp-based directory gating
-
-For large collections, walking every subdirectory on every startup would be slow. To avoid
-that, the sync uses filesystem modification times to skip directories that cannot have changed.
-
-Each library folder in the database records a `last_checked_at` timestamp, set at the end of
-every sync pass. Before listing the files inside a directory during a walk, the sync compares
-that directory's modification time against `last_checked_at`. If the directory's modification
-time is at or before the last check time, the directory is skipped entirely. Adding a file to
-a directory updates that directory's modification time, so any directory that has received new
-files since the last check is always included.
-
-This gating applies independently to each directory in the tree. If a parent directory has
-not changed but one of its subdirectories has, the subdirectory is still scanned. The result
-is that the second `make start` after an initial import typically does very little work, even
-if the library spans thousands of files across many folders.
+Changing a folder's path rescans the whole new location. If you moved the folder rather than
+pointing it somewhere new, the photos inside it are recognised and simply follow the move; you
+do not lose ratings or favourites. Progress appears live in the folder's **Sync** column.
## Timestamps shown in the Library page
@@ -94,3 +80,70 @@ while the folder is being walked, a progress bar while images are imported, and
summary such as `Imported 36, skipped 3` when it finishes. While a sync is active, the column
refreshes on its own every couple of seconds, so you can watch it progress without reloading
the page.
+
+## Removing images that disappeared
+
+The sync keeps the gallery in step with your folders in both directions. When a photo is no
+longer where the catalog expects it, its entry is taken out of the gallery.
+
+**Filmcase never deletes your photo files.** "Removing" an image only removes Filmcase's record
+of it. Every file stays on disk exactly where it is. The only case where a photo leaves the
+gallery is the case where you already deleted or moved the file yourself.
+
+**Moves and renames are not removals.** If you rename a photo, move it into another subfolder,
+rename a whole subfolder, or move a photo from one library folder to another, Filmcase
+recognises the file by its contents and simply updates where it is. The photo keeps its
+rating, its favourite mark and its album membership. Nothing is lost and nothing is
+re-imported.
+
+The distinction is made by looking at the old location: if the file is no longer there, the
+photo moved. If it is still there, you made a copy, and the copy does not become a second
+entry.
+
+**Ordering.** Everything new is imported before anything is removed, so a photo that moved has
+already been re-linked by the time removal is considered.
+
+### The safety guard
+
+Removal is permanent, and "the file is missing" is ambiguous: an external drive that is not
+plugged in, a network share that has not mounted, or a folder that has become unreadable all
+look identical to "every photo in here was deleted".
+
+So if a single sync would remove more than half of a folder's images, **and** more than twenty
+of them, it removes nothing and tells you instead:
+
+```
+Skipped removing 340 of 512 image(s) in /Volumes/Photos (safety guard). That usually means a
+drive is not mounted rather than that the photos were deleted. Re-run with --force-prune to
+remove them anyway.
+```
+
+The same warning appears on the Library page against that folder. Both thresholds have to be
+crossed, so ordinary cleanups (emptying a folder of a handful of photos) are applied without
+any fuss. You can change where the line sits with `LIBRARY_PRUNE_GUARD_FRACTION` and
+`LIBRARY_PRUNE_GUARD_MIN_IMAGES`.
+
+### Controlling removal from the command line
+
+```sh
+python manage.py sync_library --dry-run-prune # list what would go, remove nothing
+python manage.py sync_library --force-prune # remove even if the guard would stop it
+python manage.py sync_library --no-prune # import only, never remove
+```
+
+In full install mode the removal happens in the Celery worker after the command has exited, so
+the command's own count is always zero; watch the Library page for the result.
+
+## Removing a folder from the Library
+
+Pressing **Remove** on a folder asks what should happen to its images. It tells you how many
+images in the gallery come only from that folder, and offers two choices:
+
+- **Remove folder only** stops monitoring the folder and leaves its images in the gallery.
+- **Remove folder and its images** also takes those images out of the gallery.
+
+Again, no photo file is deleted either way.
+
+If folders are nested (say both `/Photos` and `/Photos/2024` are registered), removing the
+inner one never takes images the outer one still monitors. Only images that come *exclusively*
+from the folder you are removing are counted, and only those can go.
diff --git a/docs/management_commands.md b/docs/management_commands.md
index a521846..fb0d848 100644
--- a/docs/management_commands.md
+++ b/docs/management_commands.md
@@ -27,13 +27,29 @@ python manage.py [args]
python manage.py sync_library
```
-Scans all folders registered in the Library, finds JPEG files not yet in the catalog, and
-imports them. This command is run automatically by `make start` before the web server starts,
-so in normal use you do not need to call it directly.
+Scans all folders registered in the Library, finds JPEG files not yet in the catalog, imports
+them, and takes out of the gallery any entry whose file has disappeared. This command is run
+automatically by `make start` before the web server starts, so in normal use you do not need to
+call it directly.
-The command skips directories whose modification time predates the folder's last check
-timestamp, so repeated runs are fast even over large collections. If a registered folder is
-no longer present on disk, a warning is printed and the remaining folders are still scanned.
+Removing an entry never deletes the photo file: it only removes Filmcase's record of it. A
+photo you renamed or moved is recognised by its contents and keeps its rating and favourite
+mark rather than being removed and re-imported.
+
+If a registered folder is no longer present on disk, a warning is printed, **nothing is removed
+from the gallery**, and the remaining folders are still scanned.
+
+Three flags control removal (they are mutually exclusive):
+
+| Flag | Effect |
+|---|---|
+| `--dry-run-prune` | List which entries would be removed, and remove none |
+| `--force-prune` | Remove even when the mass-removal safety guard would stop it |
+| `--no-prune` | Import only; never remove anything |
+
+The safety guard stops a pass that would remove more than half of a folder's images and more
+than twenty of them, since that usually means a drive is not mounted rather than that the
+photos were deleted. See `docs/library_sync.md` for the full picture.
Behaviour depends on your install mode:
diff --git a/docs/web_interface.md b/docs/web_interface.md
index bdb170f..afde5b2 100644
--- a/docs/web_interface.md
+++ b/docs/web_interface.md
@@ -36,17 +36,30 @@ folder was checked and the last time new images were found in it.
directory you want to register. Subfolders are included automatically; you do not need to
register them separately.
- **Update path**: if you move a folder on disk, click _Update Path_ on its row and pick the
- new location. The folder's sync history is preserved.
-- **Remove**: click _Remove_ to unregister a folder. This removes it from the monitored list
- but does not delete any images from the catalog or from disk.
+ new location. The folder's sync history is preserved, and the photos inside it keep their
+ ratings and favourites: they are recognised at the new location rather than re-imported.
+- **Remove**: click _Remove_ to unregister a folder. A confirmation appears showing how many
+ images in the gallery come only from that folder, and offers two choices: remove the folder
+ only, leaving its images in the gallery, or remove the folder and take those images out of
+ the gallery too. **Neither option deletes a photo file**; your files stay on disk. If folders
+ are nested, removing the inner one never takes images the outer one still monitors.
### 1.2 Automatic sync on startup
Every time you start the app with `make start`, Filmcase runs a sync pass across all
registered library folders before the web server comes up. New images are imported
-automatically; images already in the catalog are skipped. See
+automatically; images already in the catalog are skipped, and entries whose files have
+disappeared are taken out of the gallery. See
[Library Sync](library_sync.md) for a full explanation of how the sync works.
+Two warnings can appear in a folder's **Sync** column:
+
+- **Folder not found on disk.** The folder is registered but is not there, usually because an
+ external drive is not plugged in. Nothing is removed from the gallery when this happens.
+- **Skipped removing N missing images.** Most of the folder's images looked missing at once,
+ which is far more often an unmounted drive than a real deletion, so the removal was reported
+ rather than applied. Hover for the explanation and the command that overrides it.
+
---
## 2 Images
From 287c47a1141b99ee57537548e882895fe253e349 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sun, 9 Aug 2026 12:07:34 +1000
Subject: [PATCH 23/36] Add an IgnoredImage model
Nothing writes it yet. A file the sync cannot import leaves no trace at
all today: process_image raises before any write, the invalid-recipe path
is rolled back by its own atomic block, and errors persist nothing. The
outcome survives only as a counter on the run and a log line.
That is why such a file is rediscovered on every single sync. On a 40k
library with 14.5k non-Fujifilm JPEGs, each startup re-runs exiftool on
every one of them to reach the same conclusion.
file_size and file_modified_at are the change fingerprint. A file whose
fingerprint still matches cannot have become importable, so a later sync
can pass over it for the cost of one stat rather than one process. A file
the user fixes in place changes its fingerprint and is examined again on
its own.
The foreign key scopes records to a folder for display, and takes them
with the folder when it is removed.
Co-Authored-By: Claude Opus 5
---
src/data/migrations/0036_ignoredimage.py | 33 +++++++++
src/data/models/__init__.py | 2 +
src/data/models/_ignored_image.py | 90 ++++++++++++++++++++++++
3 files changed, 125 insertions(+)
create mode 100644 src/data/migrations/0036_ignoredimage.py
create mode 100644 src/data/models/_ignored_image.py
diff --git a/src/data/migrations/0036_ignoredimage.py b/src/data/migrations/0036_ignoredimage.py
new file mode 100644
index 0000000..e96bd70
--- /dev/null
+++ b/src/data/migrations/0036_ignoredimage.py
@@ -0,0 +1,33 @@
+# Generated by Django 6.0.3 on 2026-08-09 02:06
+
+import django.db.models.deletion
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('data', '0035_syncrun_prune_fields'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='IgnoredImage',
+ fields=[
+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('filepath', models.CharField(max_length=1024)),
+ ('reason', models.CharField(max_length=32)),
+ ('detail', models.TextField(blank=True, default='')),
+ ('file_size', models.BigIntegerField()),
+ ('file_modified_at', models.DateTimeField()),
+ ('created_at', models.DateTimeField(default=django.utils.timezone.now)),
+ ('updated_at', models.DateTimeField(auto_now=True)),
+ ('folder', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='ignored_images', to='data.libraryfolder')),
+ ],
+ options={
+ 'indexes': [models.Index(fields=['folder', 'reason'], name='idx_ignored_image_folder')],
+ 'constraints': [models.UniqueConstraint(fields=('filepath',), name='unique_ignored_image_filepath')],
+ },
+ ),
+ ]
diff --git a/src/data/models/__init__.py b/src/data/models/__init__.py
index 1a240b6..c210d4b 100644
--- a/src/data/models/__init__.py
+++ b/src/data/models/__init__.py
@@ -1,3 +1,4 @@
+from ._ignored_image import IgnoredImage
from ._images import FujifilmExif, Image, ImageQuerySet
from ._library import LibraryFolder
from ._recipes import (
@@ -14,6 +15,7 @@
"RECIPE_FIELDS",
"FujifilmExif",
"FujifilmRecipe",
+ "IgnoredImage",
"Image",
"ImageQuerySet",
"LibraryFolder",
diff --git a/src/data/models/_ignored_image.py b/src/data/models/_ignored_image.py
new file mode 100644
index 0000000..4d067b0
--- /dev/null
+++ b/src/data/models/_ignored_image.py
@@ -0,0 +1,90 @@
+from datetime import datetime
+
+from django.db import models
+from django.utils import timezone
+
+from ._library import LibraryFolder
+
+# Why the sync will not import a file. A file carrying one of these is left alone
+# until it changes on disk or the record is removed; none of them means the file
+# was deleted or altered in any way.
+_IGNORED_NO_FILM_SIMULATION = "IGNORED_NO_FILM_SIMULATION"
+_IGNORED_INVALID_RECIPE_DATA = "IGNORED_INVALID_RECIPE_DATA"
+_IGNORED_ERROR = "IGNORED_ERROR"
+
+_PATH_MAX_LEN = 1024
+_CODE_MAX_LEN = 32
+
+
+class IgnoredImage(models.Model):
+ REASON_NO_FILM_SIMULATION = _IGNORED_NO_FILM_SIMULATION
+ REASON_INVALID_RECIPE_DATA = _IGNORED_INVALID_RECIPE_DATA
+ REASON_ERROR = _IGNORED_ERROR
+
+ folder = models.ForeignKey(
+ LibraryFolder,
+ on_delete=models.CASCADE,
+ related_name="ignored_images",
+ )
+ filepath = models.CharField(max_length=_PATH_MAX_LEN)
+ reason = models.CharField(max_length=_CODE_MAX_LEN)
+ detail = models.TextField(blank=True, default="")
+ # Size and modification time as they were when the file was last examined. A
+ # file whose fingerprint still matches cannot have become importable, so the
+ # sync can skip it for the cost of one stat instead of one exiftool process.
+ file_size = models.BigIntegerField()
+ file_modified_at = models.DateTimeField()
+ created_at = models.DateTimeField(default=timezone.now)
+ updated_at = models.DateTimeField(auto_now=True)
+
+ class Meta:
+ indexes = [
+ models.Index(fields=["folder", "reason"], name="idx_ignored_image_folder"),
+ ]
+ constraints = [
+ models.UniqueConstraint(
+ fields=["filepath"],
+ name="unique_ignored_image_filepath",
+ ),
+ ]
+
+ # Factories
+
+ @classmethod
+ def create(
+ cls,
+ *,
+ folder: LibraryFolder,
+ filepath: str,
+ reason: str,
+ detail: str,
+ file_size: int,
+ file_modified_at: datetime,
+ ) -> "IgnoredImage":
+ return cls.objects.create(
+ folder=folder,
+ filepath=filepath,
+ reason=reason,
+ detail=detail,
+ file_size=file_size,
+ file_modified_at=file_modified_at,
+ )
+
+ # Mutators
+
+ def set_outcome(
+ self,
+ *,
+ reason: str,
+ detail: str,
+ file_size: int,
+ file_modified_at: datetime,
+ ) -> None:
+ self.reason = reason
+ self.detail = detail
+ self.file_size = file_size
+ self.file_modified_at = file_modified_at
+ self.save(update_fields=["reason", "detail", "file_size", "file_modified_at", "updated_at"])
+
+ def __str__(self) -> str:
+ return f"#{self.id} {self.reason} {self.filepath}"
From f66769e20b037f82a6417537f8baf4095a4afdd0 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sun, 9 Aug 2026 12:08:33 +1000
Subject: [PATCH 24/36] Add a sync image batch size setting
Nothing reads it yet. Sync currently publishes one broker message per new
file, synchronously, before the command returns, which is what makes a
large import block startup for tens of seconds.
Co-Authored-By: Claude Opus 5
---
src/config/settings.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/src/config/settings.py b/src/config/settings.py
index fc01766..5f6a56f 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -123,6 +123,11 @@
CELERY_TASK_SERIALIZER = "json"
CELERY_RESULT_SERIALIZER = "json"
PROCESS_IMAGE_QUEUE: str = env.str("PROCESS_IMAGE_QUEUE", default="process-image") # Celery queue name for image-processing tasks
+
+# Library sync hands image processing to the worker in batches, so a large import costs one broker
+# message per batch rather than one per file. Larger batches make dispatch faster; smaller ones
+# spread the work more evenly across worker processes and lose less if a single batch dies.
+SYNC_IMAGE_BATCH_SIZE: int = env.int("SYNC_IMAGE_BATCH_SIZE", default=100)
USE_ASYNC_TASKS: bool = env.bool("USE_ASYNC_TASKS", default=True) # True: enqueue Celery tasks (full stack); False: run sequentially (SQLite / lite install)
CELERY_TASK_QUEUES: tuple[Queue, ...] = (Queue(PROCESS_IMAGE_QUEUE),)
From 5dd835f2ebe955a50c6eaa2b161b24becd2c5940 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sun, 9 Aug 2026 12:11:02 +1000
Subject: [PATCH 25/36] Add ignored-image domain queries
Nothing calls them yet. Four reads the feature needs, each shaped by how
many rows it has to cope with: a 40k library can carry tens of thousands of
ignored files.
get_ignored_fingerprints returns one mapping in one query, because the sync
compares it against every candidate path it found on disk.
get_ignored_images returns a queryset rather than a list so the caller can
paginate rather than materialise the lot. get_ignored_counts_by_folder is a
single aggregate, so showing a count on every Library row does not become a
query per folder.
Co-Authored-By: Claude Opus 5
---
src/domain/library/queries.py | 90 ++++++++++-
tests/factories.py | 15 ++
.../library/test_ignored_image_queries.py | 140 ++++++++++++++++++
3 files changed, 244 insertions(+), 1 deletion(-)
create mode 100644 tests/integration/domain/library/test_ignored_image_queries.py
diff --git a/src/domain/library/queries.py b/src/domain/library/queries.py
index e46ff4d..8a12eb4 100644
--- a/src/domain/library/queries.py
+++ b/src/domain/library/queries.py
@@ -1,7 +1,10 @@
-import attrs
import os
+from datetime import datetime
from pathlib import Path
+import attrs
+from django.db.models import Count, QuerySet
+
from src.data import models
@@ -32,6 +35,91 @@ class SyncRunNotFound(Exception):
run_id: int
+@attrs.frozen
+class IgnoredImageNotFound(Exception):
+ """
+ Raised when no IgnoredImage row matches the given ignored_id.
+ """
+
+ ignored_id: int
+
+
+@attrs.frozen
+class IgnoredFingerprint:
+ """
+ Size and modification time of a file as it was when last examined.
+ """
+
+ file_size: int
+ file_modified_at: datetime
+
+
+def get_ignored_fingerprints(*, folder_id: int) -> dict[str, IgnoredFingerprint]:
+ """
+ Return the recorded fingerprint of every ignored file under *folder_id*,
+ keyed by path.
+
+ One query, because the sync compares this against every candidate path it
+ found on disk.
+ """
+ rows = models.IgnoredImage.objects.filter(folder_id=folder_id).values_list(
+ "filepath", "file_size", "file_modified_at"
+ )
+ return {
+ filepath: IgnoredFingerprint(file_size=file_size, file_modified_at=file_modified_at)
+ for filepath, file_size, file_modified_at in rows
+ }
+
+
+def get_ignored_image(*, ignored_id: int) -> models.IgnoredImage:
+ """
+ Return the IgnoredImage with the given id.
+
+ :raises IgnoredImageNotFound: If no row with *ignored_id* exists.
+ """
+ try:
+ return models.IgnoredImage.objects.get(pk=ignored_id)
+ except models.IgnoredImage.DoesNotExist:
+ raise IgnoredImageNotFound(ignored_id=ignored_id)
+
+
+def get_ignored_images(*, folder_id: int, reason: str | None = None) -> QuerySet[models.IgnoredImage]:
+ """
+ Return the ignored files under *folder_id*, oldest path first, optionally
+ limited to one reason.
+
+ Returns a queryset rather than a list because the caller paginates it: these
+ lists run to tens of thousands of rows.
+ """
+ ignored = models.IgnoredImage.objects.filter(folder_id=folder_id)
+ if reason is not None:
+ ignored = ignored.filter(reason=reason)
+ return ignored.order_by("filepath", "id")
+
+
+def count_ignored_images_by_reason(*, folder_id: int) -> dict[str, int]:
+ """
+ Return how many files are ignored under *folder_id*, keyed by reason.
+ """
+ rows = (
+ models.IgnoredImage.objects.filter(folder_id=folder_id)
+ .values("reason")
+ .annotate(total=Count("id"))
+ )
+ return {row["reason"]: row["total"] for row in rows}
+
+
+def get_ignored_counts_by_folder() -> dict[int, int]:
+ """
+ Return how many files are ignored, keyed by library folder id.
+
+ One aggregate for the whole Library page, so listing a count per row does
+ not cost a query per folder.
+ """
+ rows = models.IgnoredImage.objects.values("folder_id").annotate(total=Count("id"))
+ return {row["folder_id"]: row["total"] for row in rows}
+
+
def get_all_library_folders() -> list[models.LibraryFolder]:
"""
Return all registered library folders ordered by path.
diff --git a/tests/factories.py b/tests/factories.py
index ad8b79f..993453c 100644
--- a/tests/factories.py
+++ b/tests/factories.py
@@ -16,6 +16,8 @@
image = ImageFactory(is_favorite=True, camera_model="X-T5")
"""
+from datetime import datetime, timezone as dt_timezone
+
import factory
from django.utils import timezone
@@ -102,6 +104,19 @@ class Meta:
path = factory.Sequence(lambda n: f"/photos/library_{n:04d}")
+class IgnoredImageFactory(factory.django.DjangoModelFactory):
+ class Meta:
+ model = models.IgnoredImage
+
+ folder = factory.SubFactory(LibraryFolderFactory)
+ # filepath has a unique constraint, so use a sequence to avoid collisions.
+ filepath = factory.Sequence(lambda n: f"/photos/ignored_{n:04d}.jpg")
+ reason = models.IgnoredImage.REASON_NO_FILM_SIMULATION
+ detail = ""
+ file_size = 1024
+ file_modified_at = datetime(2026, 1, 1, tzinfo=dt_timezone.utc)
+
+
class SyncRunFactory(factory.django.DjangoModelFactory):
class Meta:
model = models.SyncRun
diff --git a/tests/integration/domain/library/test_ignored_image_queries.py b/tests/integration/domain/library/test_ignored_image_queries.py
new file mode 100644
index 0000000..81c54b2
--- /dev/null
+++ b/tests/integration/domain/library/test_ignored_image_queries.py
@@ -0,0 +1,140 @@
+from datetime import datetime, timezone
+
+import pytest
+
+from src.data import models
+from src.domain.library.queries import (
+ IgnoredImageNotFound,
+ count_ignored_images_by_reason,
+ get_ignored_counts_by_folder,
+ get_ignored_fingerprints,
+ get_ignored_image,
+ get_ignored_images,
+)
+from tests.factories import IgnoredImageFactory, LibraryFolderFactory
+
+MODIFIED_AT = datetime(2026, 1, 15, 12, 0, 0, tzinfo=timezone.utc)
+
+
+@pytest.mark.django_db
+class TestGetIgnoredFingerprints:
+ def test_returns_size_and_modification_time_keyed_by_path(self):
+ folder = LibraryFolderFactory()
+ IgnoredImageFactory(
+ folder=folder,
+ filepath="/photos/other.jpg",
+ file_size=1234,
+ file_modified_at=MODIFIED_AT,
+ )
+
+ result = get_ignored_fingerprints(folder_id=folder.pk)
+
+ assert set(result) == {"/photos/other.jpg"}
+ assert result["/photos/other.jpg"].file_size == 1234
+ assert result["/photos/other.jpg"].file_modified_at == MODIFIED_AT
+
+ def test_excludes_records_belonging_to_another_folder(self):
+ folder = LibraryFolderFactory()
+ IgnoredImageFactory(folder=LibraryFolderFactory(), filepath="/elsewhere/other.jpg")
+
+ assert get_ignored_fingerprints(folder_id=folder.pk) == {}
+
+ def test_returns_an_empty_mapping_for_a_folder_with_no_records(self):
+ folder = LibraryFolderFactory()
+
+ assert get_ignored_fingerprints(folder_id=folder.pk) == {}
+
+
+@pytest.mark.django_db
+class TestGetIgnoredImage:
+ def test_returns_the_matching_record(self):
+ ignored = IgnoredImageFactory()
+
+ assert get_ignored_image(ignored_id=ignored.pk).pk == ignored.pk
+
+ def test_raises_ignored_image_not_found_for_unknown_id(self):
+ with pytest.raises(IgnoredImageNotFound) as exc_info:
+ get_ignored_image(ignored_id=9999)
+
+ assert exc_info.value.ignored_id == 9999
+
+
+@pytest.mark.django_db
+class TestGetIgnoredImages:
+ def test_returns_the_folders_records_ordered_by_path(self):
+ folder = LibraryFolderFactory()
+ IgnoredImageFactory(folder=folder, filepath="/photos/b.jpg")
+ IgnoredImageFactory(folder=folder, filepath="/photos/a.jpg")
+
+ result = get_ignored_images(folder_id=folder.pk)
+
+ assert [i.filepath for i in result] == ["/photos/a.jpg", "/photos/b.jpg"]
+
+ def test_limits_to_one_reason_when_asked(self):
+ folder = LibraryFolderFactory()
+ IgnoredImageFactory(
+ folder=folder,
+ filepath="/photos/a.jpg",
+ reason=models.IgnoredImage.REASON_NO_FILM_SIMULATION,
+ )
+ errored = IgnoredImageFactory(
+ folder=folder,
+ filepath="/photos/b.jpg",
+ reason=models.IgnoredImage.REASON_ERROR,
+ )
+
+ result = get_ignored_images(folder_id=folder.pk, reason=models.IgnoredImage.REASON_ERROR)
+
+ assert [i.pk for i in result] == [errored.pk]
+
+ def test_excludes_another_folders_records(self):
+ folder = LibraryFolderFactory()
+ IgnoredImageFactory(folder=LibraryFolderFactory())
+
+ assert list(get_ignored_images(folder_id=folder.pk)) == []
+
+
+@pytest.mark.django_db
+class TestCountIgnoredImagesByReason:
+ def test_counts_each_reason_separately(self):
+ folder = LibraryFolderFactory()
+ IgnoredImageFactory(
+ folder=folder,
+ filepath="/photos/a.jpg",
+ reason=models.IgnoredImage.REASON_NO_FILM_SIMULATION,
+ )
+ IgnoredImageFactory(
+ folder=folder,
+ filepath="/photos/b.jpg",
+ reason=models.IgnoredImage.REASON_NO_FILM_SIMULATION,
+ )
+ IgnoredImageFactory(
+ folder=folder,
+ filepath="/photos/c.jpg",
+ reason=models.IgnoredImage.REASON_ERROR,
+ )
+
+ assert count_ignored_images_by_reason(folder_id=folder.pk) == {
+ models.IgnoredImage.REASON_NO_FILM_SIMULATION: 2,
+ models.IgnoredImage.REASON_ERROR: 1,
+ }
+
+ def test_returns_an_empty_mapping_when_nothing_is_ignored(self):
+ assert count_ignored_images_by_reason(folder_id=LibraryFolderFactory().pk) == {}
+
+
+@pytest.mark.django_db
+class TestGetIgnoredCountsByFolder:
+ def test_counts_records_per_folder(self):
+ first = LibraryFolderFactory()
+ second = LibraryFolderFactory()
+ IgnoredImageFactory(folder=first, filepath="/a/1.jpg")
+ IgnoredImageFactory(folder=first, filepath="/a/2.jpg")
+ IgnoredImageFactory(folder=second, filepath="/b/1.jpg")
+
+ assert get_ignored_counts_by_folder() == {first.pk: 2, second.pk: 1}
+
+ def test_omits_folders_with_no_records(self):
+ LibraryFolderFactory()
+
+ assert get_ignored_counts_by_folder() == {}
From 348a62affbd0b13c4e27a0594d0b7f0da8321951 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sun, 9 Aug 2026 12:13:28 +1000
Subject: [PATCH 26/36] Add domain operations for remembering and forgetting
ignored images
Nothing calls them yet.
record_ignored_image captures the file's size and modification time at the
moment it was examined. Re-recording an existing entry replaces that
fingerprint, which is the part that is easy to get wrong: a file that
changed, was examined again and failed again would otherwise keep its stale
fingerprint and be re-examined on every sync from then on.
forget_ignored_image and forget_ignored_images undo it, singly or in bulk
by reason, so a batch that failed for an environmental reason can be
reconsidered without touching the permanently unimportable ones.
None of this touches a file or moves an image in or out of the gallery. An
ignored file was never imported; forgetting its record only means the next
sync looks at it again.
Co-Authored-By: Claude Opus 5
---
src/domain/library/events.py | 3 +
src/domain/library/operations.py | 129 +++++++++
.../library/test_ignored_image_operations.py | 258 ++++++++++++++++++
3 files changed, 390 insertions(+)
create mode 100644 tests/integration/domain/library/test_ignored_image_operations.py
diff --git a/src/domain/library/events.py b/src/domain/library/events.py
index f95e355..abb81d0 100644
--- a/src/domain/library/events.py
+++ b/src/domain/library/events.py
@@ -8,6 +8,9 @@
LIBRARY_FOLDER_REMOVED = "library.folder.removed"
LIBRARY_FOLDER_PATH_UPDATED = "library.folder.path.updated"
LIBRARY_FOLDER_IMAGES_REMOVED = "library.folder.images.removed"
+LIBRARY_IMAGE_IGNORED = "library.image.ignored"
+LIBRARY_IMAGE_IGNORE_REMOVED = "library.image.ignore.removed"
+LIBRARY_IMAGE_IGNORES_CLEARED = "library.image.ignores.cleared"
LIBRARY_SYNC_RUN_STARTED = "library.sync.run.started"
LIBRARY_SYNC_RUN_COMPLETED = "library.sync.run.completed"
LIBRARY_SYNC_RUN_FAILED = "library.sync.run.failed"
diff --git a/src/domain/library/operations.py b/src/domain/library/operations.py
index 4b426fa..ab7d4a7 100644
--- a/src/domain/library/operations.py
+++ b/src/domain/library/operations.py
@@ -1,3 +1,4 @@
+import datetime
import os
from pathlib import Path
@@ -127,6 +128,134 @@ def remove_library_folder(*, folder_id: int, delete_images: bool) -> int:
return removed
+def record_ignored_image(
+ *,
+ folder: models.LibraryFolder,
+ filepath: str,
+ reason: str,
+ detail: str,
+) -> models.IgnoredImage:
+ """
+ Remember that *filepath* could not be imported, so later syncs leave it alone.
+
+ Records the file's current size and modification time. A file whose
+ fingerprint still matches cannot have become importable, so the next sync can
+ pass over it for the cost of one stat rather than one exiftool process. A
+ file the user later fixes in place changes its fingerprint and is examined
+ again on its own.
+
+ Re-recording an existing entry replaces its fingerprint. That matters: a file
+ that changed, was examined again and failed again would otherwise keep its
+ stale fingerprint and be re-examined on every sync from then on.
+
+ The file itself is never touched, and no image leaves the gallery.
+
+ :raises OSError: If *filepath* cannot be stat'ed.
+ """
+ stat_result = os.stat(filepath)
+ file_size = stat_result.st_size
+ file_modified_at = datetime.datetime.fromtimestamp(
+ stat_result.st_mtime, tz=datetime.timezone.utc
+ )
+
+ existing = models.IgnoredImage.objects.filter(filepath=filepath).first()
+ if existing is not None:
+ existing.set_outcome(
+ reason=reason,
+ detail=detail,
+ file_size=file_size,
+ file_modified_at=file_modified_at,
+ )
+ ignored = existing
+ else:
+ ignored = models.IgnoredImage.create(
+ folder=folder,
+ filepath=filepath,
+ reason=reason,
+ detail=detail,
+ file_size=file_size,
+ file_modified_at=file_modified_at,
+ )
+
+ events.publish_event(
+ event_type=events.LIBRARY_IMAGE_IGNORED,
+ folder_id=folder.pk,
+ filepath=filepath,
+ reason=reason,
+ )
+ return ignored
+
+
+def forget_ignored_image(*, ignored_id: int) -> str:
+ """
+ Forget one ignored file, so the next sync examines it again.
+
+ Returns the path that was forgotten. Removes only the record: the file is
+ untouched and nothing enters the gallery until a sync imports it.
+
+ :raises IgnoredImageNotFound: If no record with *ignored_id* exists.
+ """
+ ignored = queries.get_ignored_image(ignored_id=ignored_id)
+ filepath = ignored.filepath
+ folder_id = ignored.folder_id
+ ignored.delete()
+
+ events.publish_event(
+ event_type=events.LIBRARY_IMAGE_IGNORE_REMOVED,
+ folder_id=folder_id,
+ filepath=filepath,
+ )
+ return filepath
+
+
+def forget_ignored_path(*, filepath: str) -> bool:
+ """
+ Forget *filepath* if it is currently ignored, and report whether it was.
+
+ Called when a file that could not be imported before succeeds, so that a
+ record which no longer describes reality does not linger and show an
+ imported photo as ignored.
+
+ Silent when the path was not ignored, because that is the ordinary case.
+ """
+ ignored = models.IgnoredImage.objects.filter(filepath=filepath).first()
+ if ignored is None:
+ return False
+
+ folder_id = ignored.folder_id
+ ignored.delete()
+ events.publish_event(
+ event_type=events.LIBRARY_IMAGE_IGNORE_REMOVED,
+ folder_id=folder_id,
+ filepath=filepath,
+ )
+ return True
+
+
+def forget_ignored_images(*, folder_id: int, reason: str | None = None) -> int:
+ """
+ Forget every ignored file under *folder_id*, or every one with *reason*.
+
+ Returns how many records were forgotten. The next sync examines all of them
+ again, which for a large set of permanently unimportable files means one slow
+ sync before they are recorded afresh.
+ """
+ ignored = models.IgnoredImage.objects.filter(folder_id=folder_id)
+ if reason is not None:
+ ignored = ignored.filter(reason=reason)
+
+ count, _ = ignored.delete()
+
+ if count:
+ events.publish_event(
+ event_type=events.LIBRARY_IMAGE_IGNORES_CLEARED,
+ folder_id=folder_id,
+ reason=reason or "",
+ count=count,
+ )
+ return count
+
+
def prune_guard_trips(*, missing: int, total: int) -> bool:
"""
Return True when removing *missing* of *total* images looks like a mass wipe
diff --git a/tests/integration/domain/library/test_ignored_image_operations.py b/tests/integration/domain/library/test_ignored_image_operations.py
new file mode 100644
index 0000000..303c90d
--- /dev/null
+++ b/tests/integration/domain/library/test_ignored_image_operations.py
@@ -0,0 +1,258 @@
+import datetime
+import os
+
+import pytest
+
+from src.data import models
+from src.domain.library import events
+from src.domain.library.operations import (
+ forget_ignored_image,
+ forget_ignored_images,
+ forget_ignored_path,
+ record_ignored_image,
+)
+from src.domain.library.queries import IgnoredImageNotFound
+from tests.factories import IgnoredImageFactory, LibraryFolderFactory
+
+
+def _photo(*, folder_path, name="other_brand.jpg", content=b"\xff\xd8abc"):
+ path = folder_path / name
+ path.write_bytes(content)
+ return path
+
+
+@pytest.mark.django_db
+class TestRecordIgnoredImage:
+ def test_records_the_file_with_its_current_fingerprint(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ photo = _photo(folder_path=tmp_path)
+
+ ignored = record_ignored_image(
+ folder=folder,
+ filepath=str(photo),
+ reason=models.IgnoredImage.REASON_NO_FILM_SIMULATION,
+ detail="",
+ )
+
+ assert ignored.filepath == str(photo)
+ assert ignored.file_size == photo.stat().st_size
+ assert ignored.file_modified_at == datetime.datetime.fromtimestamp(
+ photo.stat().st_mtime, tz=datetime.timezone.utc
+ )
+
+ def test_keeps_the_file_on_disk(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ photo = _photo(folder_path=tmp_path)
+
+ record_ignored_image(
+ folder=folder,
+ filepath=str(photo),
+ reason=models.IgnoredImage.REASON_ERROR,
+ detail="boom",
+ )
+
+ assert photo.exists()
+
+ def test_stores_the_detail_for_an_error(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ photo = _photo(folder_path=tmp_path)
+
+ ignored = record_ignored_image(
+ folder=folder,
+ filepath=str(photo),
+ reason=models.IgnoredImage.REASON_ERROR,
+ detail="OSError: disk went away",
+ )
+
+ assert ignored.detail == "OSError: disk went away"
+
+ def test_re_recording_a_changed_file_updates_its_fingerprint(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ photo = _photo(folder_path=tmp_path)
+ first = record_ignored_image(
+ folder=folder,
+ filepath=str(photo),
+ reason=models.IgnoredImage.REASON_NO_FILM_SIMULATION,
+ detail="",
+ )
+ original_size = first.file_size
+
+ photo.write_bytes(b"\xff\xd8" + b"much longer content")
+ second = record_ignored_image(
+ folder=folder,
+ filepath=str(photo),
+ reason=models.IgnoredImage.REASON_ERROR,
+ detail="failed again",
+ )
+
+ assert second.pk == first.pk
+ assert models.IgnoredImage.objects.count() == 1
+ assert second.file_size != original_size
+ assert second.reason == models.IgnoredImage.REASON_ERROR
+ assert second.detail == "failed again"
+
+ def test_publishes_image_ignored(self, tmp_path, captured_logs):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ photo = _photo(folder_path=tmp_path)
+
+ record_ignored_image(
+ folder=folder,
+ filepath=str(photo),
+ reason=models.IgnoredImage.REASON_NO_FILM_SIMULATION,
+ detail="",
+ )
+
+ matching = [e for e in captured_logs if e.get("event_type") == events.LIBRARY_IMAGE_IGNORED]
+ assert len(matching) == 1
+ assert matching[0]["filepath"] == str(photo)
+ assert matching[0]["reason"] == models.IgnoredImage.REASON_NO_FILM_SIMULATION
+
+ def test_raises_when_the_file_cannot_be_stated(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+
+ with pytest.raises(OSError):
+ record_ignored_image(
+ folder=folder,
+ filepath=str(tmp_path / "gone.jpg"),
+ reason=models.IgnoredImage.REASON_ERROR,
+ detail="",
+ )
+
+
+@pytest.mark.django_db
+class TestForgetIgnoredImage:
+ def test_removes_the_record_and_returns_its_path(self):
+ ignored = IgnoredImageFactory(filepath="/photos/other.jpg")
+
+ assert forget_ignored_image(ignored_id=ignored.pk) == "/photos/other.jpg"
+ assert not models.IgnoredImage.objects.filter(pk=ignored.pk).exists()
+
+ def test_keeps_the_file_on_disk(self, tmp_path):
+ photo = _photo(folder_path=tmp_path)
+ ignored = IgnoredImageFactory(filepath=str(photo))
+
+ forget_ignored_image(ignored_id=ignored.pk)
+
+ assert photo.exists()
+
+ def test_publishes_ignore_removed(self, captured_logs):
+ ignored = IgnoredImageFactory()
+
+ forget_ignored_image(ignored_id=ignored.pk)
+
+ matching = [
+ e for e in captured_logs if e.get("event_type") == events.LIBRARY_IMAGE_IGNORE_REMOVED
+ ]
+ assert len(matching) == 1
+
+ def test_raises_ignored_image_not_found_for_unknown_id(self):
+ with pytest.raises(IgnoredImageNotFound) as exc_info:
+ forget_ignored_image(ignored_id=9999)
+
+ assert exc_info.value.ignored_id == 9999
+
+
+@pytest.mark.django_db
+class TestForgetIgnoredPath:
+ def test_forgets_a_path_that_was_ignored(self):
+ ignored = IgnoredImageFactory(filepath="/photos/other.jpg")
+
+ assert forget_ignored_path(filepath="/photos/other.jpg") is True
+ assert not models.IgnoredImage.objects.filter(pk=ignored.pk).exists()
+
+ def test_reports_false_for_a_path_that_was_not_ignored(self):
+ assert forget_ignored_path(filepath="/photos/never-seen.jpg") is False
+
+ def test_leaves_other_records_alone(self):
+ kept = IgnoredImageFactory(filepath="/photos/a.jpg")
+ IgnoredImageFactory(filepath="/photos/b.jpg")
+
+ forget_ignored_path(filepath="/photos/b.jpg")
+
+ assert list(models.IgnoredImage.objects.values_list("pk", flat=True)) == [kept.pk]
+
+ def test_publishes_ignore_removed(self, captured_logs):
+ IgnoredImageFactory(filepath="/photos/other.jpg")
+
+ forget_ignored_path(filepath="/photos/other.jpg")
+
+ matching = [
+ e for e in captured_logs if e.get("event_type") == events.LIBRARY_IMAGE_IGNORE_REMOVED
+ ]
+ assert len(matching) == 1
+
+ def test_publishes_nothing_when_there_was_no_record(self, captured_logs):
+ forget_ignored_path(filepath="/photos/never-seen.jpg")
+
+ assert [
+ e for e in captured_logs if e.get("event_type") == events.LIBRARY_IMAGE_IGNORE_REMOVED
+ ] == []
+
+
+@pytest.mark.django_db
+class TestForgetIgnoredImages:
+ def test_forgets_every_record_in_the_folder(self):
+ folder = LibraryFolderFactory()
+ IgnoredImageFactory(folder=folder, filepath="/a/1.jpg")
+ IgnoredImageFactory(folder=folder, filepath="/a/2.jpg")
+
+ assert forget_ignored_images(folder_id=folder.pk) == 2
+ assert models.IgnoredImage.objects.count() == 0
+
+ def test_forgets_only_the_requested_reason(self):
+ folder = LibraryFolderFactory()
+ kept = IgnoredImageFactory(
+ folder=folder,
+ filepath="/a/1.jpg",
+ reason=models.IgnoredImage.REASON_NO_FILM_SIMULATION,
+ )
+ IgnoredImageFactory(
+ folder=folder,
+ filepath="/a/2.jpg",
+ reason=models.IgnoredImage.REASON_ERROR,
+ )
+
+ count = forget_ignored_images(folder_id=folder.pk, reason=models.IgnoredImage.REASON_ERROR)
+
+ assert count == 1
+ assert list(models.IgnoredImage.objects.values_list("pk", flat=True)) == [kept.pk]
+
+ def test_leaves_another_folders_records_alone(self):
+ folder = LibraryFolderFactory()
+ other = IgnoredImageFactory(folder=LibraryFolderFactory())
+
+ forget_ignored_images(folder_id=folder.pk)
+
+ assert models.IgnoredImage.objects.filter(pk=other.pk).exists()
+
+ def test_reports_zero_and_publishes_nothing_when_there_is_nothing_to_forget(self, captured_logs):
+ folder = LibraryFolderFactory()
+
+ assert forget_ignored_images(folder_id=folder.pk) == 0
+ assert [
+ e for e in captured_logs if e.get("event_type") == events.LIBRARY_IMAGE_IGNORES_CLEARED
+ ] == []
+
+ def test_publishes_ignores_cleared_with_the_count(self, captured_logs):
+ folder = LibraryFolderFactory()
+ IgnoredImageFactory(folder=folder, filepath="/a/1.jpg")
+ IgnoredImageFactory(folder=folder, filepath="/a/2.jpg")
+
+ forget_ignored_images(folder_id=folder.pk)
+
+ matching = [
+ e for e in captured_logs if e.get("event_type") == events.LIBRARY_IMAGE_IGNORES_CLEARED
+ ]
+ assert len(matching) == 1
+ assert matching[0]["count"] == 2
+
+
+@pytest.mark.django_db
+class TestRemovingAFolderForgetsItsIgnoredImages:
+ def test_records_go_with_the_folder(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ IgnoredImageFactory(folder=folder, filepath=str(tmp_path / "other.jpg"))
+
+ folder.delete()
+
+ assert models.IgnoredImage.objects.count() == 0
From c969ac1212df4236207c34aa09893ed4d9f207d7 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sun, 9 Aug 2026 12:18:24 +1000
Subject: [PATCH 27/36] Remember the images a sync skips or fails on
Nothing reads these records yet, so behaviour is unchanged.
Every non-success branch now remembers the file alongside the counter it
already bumped. The error branch keeps the exception type and message,
truncated, which is what turns a run reporting "93 errors" into 93 files
you can actually look at: until now that detail existed only in a log line.
Success is the other half. A file that failed before and imports now must
stop being ignored, or the record outlives the truth and shows a photo
that is in the gallery as though it had been rejected.
A file that disappeared between the scan and processing cannot be stat'ed.
That is an ordinary race rather than a failure worth aborting the run for,
so it is logged and skipped: the next sync will not find it either.
Co-Authored-By: Claude Opus 5
---
.../usecases/library/process_synced_image.py | 55 ++++++++++++++-
.../library/test_process_synced_image.py | 67 ++++++++++++++++++-
2 files changed, 120 insertions(+), 2 deletions(-)
diff --git a/src/application/usecases/library/process_synced_image.py b/src/application/usecases/library/process_synced_image.py
index 1e2e219..e9b1bdf 100644
--- a/src/application/usecases/library/process_synced_image.py
+++ b/src/application/usecases/library/process_synced_image.py
@@ -1,14 +1,20 @@
import structlog
from src.application.usecases.library.finalize_sync_run import finalize_sync_run
+from src.data import models
from src.domain.images import events as image_events
from src.domain.images import operations as image_operations
from src.domain.images.queries import NoFilmSimulationError
+from src.domain.library import operations as library_operations
from src.domain.library import queries as library_queries
from src.domain.recipes import validation as recipe_validation
logger = structlog.get_logger("application.library.process_synced_image")
+# Enough of an error to recognise it on the ignored-images page without letting a
+# pathological message fill the column.
+_DETAIL_MAX_LEN = 500
+
def process_synced_image(*, image_path: str, sync_run_id: int) -> None:
"""
@@ -21,6 +27,11 @@ def process_synced_image(*, image_path: str, sync_run_id: int) -> None:
as errors (the run continues); and successful imports count as processed. When
every image in the run has a terminal outcome, the run is completed.
+ Every outcome other than success is also remembered against the file itself,
+ so later syncs leave it alone until it changes on disk. Without that the file
+ has no trace in the catalog at all and is rediscovered, and re-read, on every
+ single sync.
+
If the run no longer exists (its folder was removed while this work was
queued), the call returns without processing.
"""
@@ -33,6 +44,12 @@ def process_synced_image(*, image_path: str, sync_run_id: int) -> None:
image_operations.process_image(image_path=image_path)
except NoFilmSimulationError:
run.record_skipped()
+ _ignore(
+ run=run,
+ image_path=image_path,
+ reason=models.IgnoredImage.REASON_NO_FILM_SIMULATION,
+ detail="",
+ )
image_events.publish_event(
event_type=image_events.IMAGE_IMPORT_SKIPPED,
image_path=image_path,
@@ -40,18 +57,54 @@ def process_synced_image(*, image_path: str, sync_run_id: int) -> None:
)
except recipe_validation.InvalidFujifilmRecipeData as exc:
run.record_skipped()
+ _ignore(
+ run=run,
+ image_path=image_path,
+ reason=models.IgnoredImage.REASON_INVALID_RECIPE_DATA,
+ detail=exc.field,
+ )
image_events.publish_event(
event_type=image_events.IMAGE_IMPORT_SKIPPED,
image_path=image_path,
reason=image_events.SKIP_REASON_INVALID_RECIPE_DATA,
recipe_field=exc.field,
)
- except Exception:
+ except Exception as exc:
logger.exception("Failed to process image during sync")
run.record_error()
+ _ignore(
+ run=run,
+ image_path=image_path,
+ reason=models.IgnoredImage.REASON_ERROR,
+ detail=f"{type(exc).__name__}: {exc}"[:_DETAIL_MAX_LEN],
+ )
else:
run.record_processed()
+ # A file that failed before and imports now must stop being ignored, or
+ # the record outlives the truth and shows a photo that is in the gallery
+ # as though it had been rejected.
+ library_operations.forget_ignored_path(filepath=image_path)
run.refresh_from_db()
if run.all_images_accounted_for():
finalize_sync_run(run=run)
+
+
+def _ignore(*, run: models.SyncRun, image_path: str, reason: str, detail: str) -> None:
+ """
+ Remember that this file could not be imported, without letting that
+ bookkeeping break the run.
+
+ A file that disappeared between the scan and now cannot be stat'ed, and that
+ is an ordinary race rather than a failure worth aborting for: the next sync
+ will not find it either.
+ """
+ try:
+ library_operations.record_ignored_image(
+ folder=run.folder,
+ filepath=image_path,
+ reason=reason,
+ detail=detail,
+ )
+ except OSError:
+ logger.warning("Could not remember an ignored image", image_path=image_path)
diff --git a/tests/integration/application/library/test_process_synced_image.py b/tests/integration/application/library/test_process_synced_image.py
index 62d1ef0..8c23d39 100644
--- a/tests/integration/application/library/test_process_synced_image.py
+++ b/tests/integration/application/library/test_process_synced_image.py
@@ -7,7 +7,7 @@
from src.application.usecases.library.process_synced_image import process_synced_image
from src.data import models
from src.domain.images import events
-from tests.factories import SyncRunFactory
+from tests.factories import IgnoredImageFactory, LibraryFolderFactory, SyncRunFactory
FIXTURES_DIR = Path(__file__).resolve().parent.parent.parent.parent / "fixtures" / "images"
FUJIFILM_FIXTURE = FIXTURES_DIR / "XS107114.JPG"
@@ -106,3 +106,68 @@ def test_leaves_run_processing_while_images_remain(self, tmp_path):
def test_returns_silently_when_run_missing(self):
# No exception should escape when the run (and its folder) is already gone.
process_synced_image(image_path="/whatever.jpg", sync_run_id=99999)
+
+
+@pytest.mark.django_db
+class TestProcessSyncedImageRemembersFailures:
+ def test_remembers_a_non_fujifilm_file(self, tmp_path):
+ photo = tmp_path / NON_FUJIFILM_FIXTURE.name
+ shutil.copy(NON_FUJIFILM_FIXTURE, photo)
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ run = SyncRunFactory(folder=folder, state=models.SyncRun.STATE_PROCESSING, total=1)
+
+ process_synced_image(image_path=str(photo), sync_run_id=run.pk)
+
+ ignored = models.IgnoredImage.objects.get()
+ assert ignored.filepath == str(photo)
+ assert ignored.reason == models.IgnoredImage.REASON_NO_FILM_SIMULATION
+ assert ignored.folder_id == folder.pk
+ assert ignored.file_size == photo.stat().st_size
+
+ def test_remembers_an_unexpected_failure_with_its_message(self, tmp_path):
+ photo = tmp_path / FUJIFILM_FIXTURE.name
+ shutil.copy(FUJIFILM_FIXTURE, photo)
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ run = SyncRunFactory(folder=folder, state=models.SyncRun.STATE_PROCESSING, total=1)
+
+ with patch(
+ "src.domain.images.operations.process_image",
+ side_effect=OSError("disk went away"),
+ ):
+ process_synced_image(image_path=str(photo), sync_run_id=run.pk)
+
+ ignored = models.IgnoredImage.objects.get()
+ assert ignored.reason == models.IgnoredImage.REASON_ERROR
+ assert ignored.detail == "OSError: disk went away"
+
+ def test_remembers_nothing_for_a_successful_import(self, tmp_path):
+ photo = tmp_path / FUJIFILM_FIXTURE.name
+ shutil.copy(FUJIFILM_FIXTURE, photo)
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ run = SyncRunFactory(folder=folder, state=models.SyncRun.STATE_PROCESSING, total=1)
+
+ process_synced_image(image_path=str(photo), sync_run_id=run.pk)
+
+ assert models.IgnoredImage.objects.count() == 0
+
+ def test_a_file_that_vanished_does_not_break_the_run(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ run = SyncRunFactory(folder=folder, state=models.SyncRun.STATE_PROCESSING, total=1)
+
+ process_synced_image(image_path=str(tmp_path / "gone.jpg"), sync_run_id=run.pk)
+
+ run.refresh_from_db()
+ assert run.errors == 1
+ assert models.IgnoredImage.objects.count() == 0
+
+ def test_a_previously_ignored_file_that_imports_stops_being_ignored(self, tmp_path):
+ photo = tmp_path / FUJIFILM_FIXTURE.name
+ shutil.copy(FUJIFILM_FIXTURE, photo)
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ run = SyncRunFactory(folder=folder, state=models.SyncRun.STATE_PROCESSING, total=1)
+ IgnoredImageFactory(folder=folder, filepath=str(photo))
+
+ process_synced_image(image_path=str(photo), sync_run_id=run.pk)
+
+ assert models.Image.objects.count() == 1
+ assert models.IgnoredImage.objects.count() == 0
From 4e93c17b24af060efc5d9673e3a55ca21545112f Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sun, 9 Aug 2026 12:25:14 +1000
Subject: [PATCH 28/36] Stop re-examining ignored images on every sync
This is the fix. A file the sync cannot import has no catalog entry, so
the path diff counted it as new every single time and handed it to the
worker to be read from scratch again. On a 40k library with 14.5k
non-Fujifilm JPEGs that is 14.5k exiftool processes per startup, forever,
every one of them reaching the conclusion already reached.
Measured before this commit, on a folder holding one such file:
run1: total=1 skipped=1
run2: total=1 skipped=1
run3: total=1 skipped=1
Candidates that already carry a record are now checked against the size
and modification time captured when they were last examined, and dropped
if neither has moved. The stat costs one syscall and is only paid for
files that are already ignored, so the extra work is bounded by how many
have failed rather than by the size of the tree.
Anything whose fingerprint has moved falls through and is examined again,
which is how a file the user fixes in place comes back on its own without
needing to be told.
This is a regression introduced earlier on this branch. Retiring mtime
gating was right, since it made renamed folders undetectable, but it
turned "retry a failed file when its directory changes" into "retry it
always". The tests here pin the inverse of the measurements above.
Co-Authored-By: Claude Opus 5
---
.../usecases/library/sync_folder.py | 45 +++++++++-
.../application/library/test_sync_folder.py | 89 +++++++++++++++++++
2 files changed, 133 insertions(+), 1 deletion(-)
diff --git a/src/application/usecases/library/sync_folder.py b/src/application/usecases/library/sync_folder.py
index 90b3c58..c2a48b3 100644
--- a/src/application/usecases/library/sync_folder.py
+++ b/src/application/usecases/library/sync_folder.py
@@ -1,3 +1,4 @@
+import os
from datetime import datetime, timezone
from django.conf import settings
@@ -54,7 +55,8 @@ def sync_folder(*, folder_id: int, prune_mode: str = models.SyncRun.PRUNE_MODE_A
return
known_paths = image_queries.get_all_known_image_paths()
- new_paths = sorted(set(found_paths) - known_paths)
+ candidates = sorted(set(found_paths) - known_paths)
+ new_paths = _drop_unchanged_ignored(folder_id=folder.pk, candidates=candidates)
run.begin_processing(total=len(new_paths))
folder.set_last_checked_at(value=now)
@@ -67,6 +69,47 @@ def sync_folder(*, folder_id: int, prune_mode: str = models.SyncRun.PRUNE_MODE_A
finalize_sync_run(run=run)
return
+ _dispatch(new_paths=new_paths, run=run)
+
+
+def _drop_unchanged_ignored(*, folder_id: int, candidates: list[str]) -> list[str]:
+ """
+ Return the candidates worth examining, dropping files already known to be
+ unimportable that have not changed since they were last looked at.
+
+ A file the sync cannot import has no catalog entry, so it would otherwise be
+ rediscovered on every single sync and re-read from scratch each time. On a
+ large library that is thousands of pointless exiftool processes per startup.
+
+ The fingerprint check costs one stat, and only for candidates that already
+ have a record, so the extra syscalls are bounded by how many files are
+ ignored rather than by the size of the tree. Anything whose size or
+ modification time has moved falls through to be examined again, which is how
+ a file the user fixes in place comes back on its own.
+ """
+ fingerprints = library_queries.get_ignored_fingerprints(folder_id=folder_id)
+ if not fingerprints:
+ return candidates
+
+ worth_examining = []
+ for path in candidates:
+ fingerprint = fingerprints.get(path)
+ if fingerprint is None:
+ worth_examining.append(path)
+ continue
+ try:
+ stat_result = os.stat(path)
+ except OSError:
+ # Gone between the walk and now. The next sync will not find it either.
+ continue
+ modified_at = datetime.fromtimestamp(stat_result.st_mtime, tz=timezone.utc)
+ if stat_result.st_size != fingerprint.file_size or modified_at != fingerprint.file_modified_at:
+ worth_examining.append(path)
+
+ return worth_examining
+
+
+def _dispatch(*, new_paths: list[str], run: models.SyncRun) -> None:
for path in new_paths:
if settings.USE_ASYNC_TASKS:
workertasks.enqueue_task(
diff --git a/tests/integration/application/library/test_sync_folder.py b/tests/integration/application/library/test_sync_folder.py
index f0ff101..b567330 100644
--- a/tests/integration/application/library/test_sync_folder.py
+++ b/tests/integration/application/library/test_sync_folder.py
@@ -1,3 +1,4 @@
+import os
import shutil
from pathlib import Path
from unittest.mock import patch
@@ -11,6 +12,7 @@
FIXTURES_DIR = Path(__file__).resolve().parent.parent.parent.parent / "fixtures" / "images"
FUJIFILM_FIXTURE = FIXTURES_DIR / "XS107114.JPG"
+NON_FUJIFILM_FIXTURE = FIXTURES_DIR / "sub-folder" / "img_4968_dng_embedded.jpg"
@pytest.mark.django_db
@@ -113,3 +115,90 @@ def test_returns_without_creating_a_run_when_folder_missing(self):
sync_folder(folder_id=99999)
assert models.SyncRun.objects.count() == 0
+
+
+@pytest.mark.django_db
+class TestSyncFolderDoesNotReExamineIgnoredImages:
+ """
+ The regression this guards: a file the sync cannot import leaves no catalog
+ entry, so before this it was rediscovered and re-read on every single sync.
+ """
+
+ @pytest.fixture(autouse=True)
+ def _lite_mode(self, settings):
+ settings.USE_ASYNC_TASKS = False
+
+ def _folder_with_a_non_fujifilm_file(self, tmp_path):
+ photo = tmp_path / NON_FUJIFILM_FIXTURE.name
+ shutil.copy(NON_FUJIFILM_FIXTURE, photo)
+ return LibraryFolderFactory(path=str(tmp_path)), photo
+
+ def test_the_second_sync_finds_nothing_to_do(self, tmp_path):
+ folder, _ = self._folder_with_a_non_fujifilm_file(tmp_path)
+
+ sync_folder(folder_id=folder.pk)
+ sync_folder(folder_id=folder.pk)
+
+ runs = list(models.SyncRun.objects.order_by("id"))
+ assert runs[0].total == 1
+ assert runs[0].skipped == 1
+ assert runs[1].total == 0
+ assert runs[1].skipped == 0
+
+ def test_it_stays_ignored_however_often_the_sync_runs(self, tmp_path):
+ folder, _ = self._folder_with_a_non_fujifilm_file(tmp_path)
+
+ for _ in range(4):
+ sync_folder(folder_id=folder.pk)
+
+ assert [r.total for r in models.SyncRun.objects.order_by("id")] == [1, 0, 0, 0]
+ assert models.IgnoredImage.objects.count() == 1
+
+ def test_a_file_that_changes_is_examined_again(self, tmp_path):
+ folder, photo = self._folder_with_a_non_fujifilm_file(tmp_path)
+ sync_folder(folder_id=folder.pk)
+
+ # A different file now occupies that path.
+ shutil.copy(FUJIFILM_FIXTURE, photo)
+ sync_folder(folder_id=folder.pk)
+
+ runs = list(models.SyncRun.objects.order_by("id"))
+ assert runs[1].total == 1
+ assert runs[1].processed == 1
+ assert models.Image.objects.count() == 1
+ assert models.IgnoredImage.objects.count() == 0
+
+ def test_a_file_whose_timestamp_alone_moves_is_examined_again(self, tmp_path):
+ folder, photo = self._folder_with_a_non_fujifilm_file(tmp_path)
+ sync_folder(folder_id=folder.pk)
+
+ later = photo.stat().st_mtime + 120
+ os.utime(photo, (later, later))
+ sync_folder(folder_id=folder.pk)
+
+ runs = list(models.SyncRun.objects.order_by("id"))
+ assert runs[1].total == 1
+ assert runs[1].skipped == 1
+ # Re-recorded with the new fingerprint, so it settles again rather than
+ # being re-examined for ever.
+ assert models.IgnoredImage.objects.count() == 1
+ sync_folder(folder_id=folder.pk)
+ assert models.SyncRun.objects.order_by("id").last().total == 0
+
+ def test_forgetting_the_record_brings_the_file_back(self, tmp_path):
+ folder, _ = self._folder_with_a_non_fujifilm_file(tmp_path)
+ sync_folder(folder_id=folder.pk)
+
+ models.IgnoredImage.objects.all().delete()
+ sync_folder(folder_id=folder.pk)
+
+ assert models.SyncRun.objects.order_by("id").last().total == 1
+
+ def test_an_importable_file_is_unaffected(self, tmp_path):
+ shutil.copy(FUJIFILM_FIXTURE, tmp_path / FUJIFILM_FIXTURE.name)
+ folder = LibraryFolderFactory(path=str(tmp_path))
+
+ sync_folder(folder_id=folder.pk)
+
+ assert models.Image.objects.count() == 1
+ assert models.IgnoredImage.objects.count() == 0
From e676254c602c1f2aabfc0087c42ded5f796c439e Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sun, 9 Aug 2026 13:07:04 +1000
Subject: [PATCH 29/36] Add enqueue_tasks to the worker-task service
Nothing calls it yet. Sync dispatches one message per file, synchronously,
before the command can return, so the per-message bookkeeping sits
directly on the critical path of startup: 14.5k files means resolving the
same dotted path 14.5k times and running the structlog processor chain
14.5k times for a log record that is dropped at the handler anyway.
Resolving once and publishing one event carrying the count removes both.
Deliberately stopping there: sharing a broker producer and skipping the
result-backend call would shave more off each publish, but the next commit
cuts the message count by two orders of magnitude, after which the
per-message cost stops being worth machinery.
Co-Authored-By: Claude Opus 5
---
src/services/workertasks.py | 52 ++++++++++++++++---
tests/unit/services/test_workertasks.py | 67 ++++++++++++++++++++++++-
2 files changed, 110 insertions(+), 9 deletions(-)
diff --git a/src/services/workertasks.py b/src/services/workertasks.py
index 41f86f2..eb8be2f 100644
--- a/src/services/workertasks.py
+++ b/src/services/workertasks.py
@@ -1,5 +1,5 @@
import pkgutil
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
import attrs
from celery import Task
@@ -35,13 +35,7 @@ def enqueue_task(*, task_name: str, kwargs: Mapping[str, object], queue: str) ->
:raises TaskNotFoundError: If *task_name* does not resolve to any Python object.
:raises NotACeleryTaskError: If *task_name* resolves to something that is not a Celery task.
"""
- try:
- task = pkgutil.resolve_name(task_name)
- except (AttributeError, ModuleNotFoundError, ValueError) as e:
- raise TaskNotFoundError(task_name=task_name) from e
-
- if not isinstance(task, Task):
- raise NotACeleryTaskError(task_name=task_name)
+ task = _resolve_task(task_name=task_name)
task.apply_async(kwargs=dict(kwargs), queue=queue)
@@ -52,6 +46,48 @@ def enqueue_task(*, task_name: str, kwargs: Mapping[str, object], queue: str) ->
)
+def enqueue_tasks(*, task_name: str, kwargs_list: Sequence[Mapping[str, object]], queue: str) -> int:
+ """
+ Dispatch the same Celery task once per entry in *kwargs_list*, and report how
+ many were sent.
+
+ Resolves the task once rather than per message, and publishes one event
+ carrying the count rather than one per message. Dispatching happens before
+ the caller can return, so a large import pays this cost up front: doing the
+ per-message bookkeeping once keeps it off the critical path.
+
+ :raises TaskNotFoundError: If *task_name* does not resolve to any Python object.
+ :raises NotACeleryTaskError: If *task_name* resolves to something that is not a Celery task.
+ """
+ if not kwargs_list:
+ return 0
+
+ task = _resolve_task(task_name=task_name)
+
+ for kwargs in kwargs_list:
+ task.apply_async(kwargs=dict(kwargs), queue=queue)
+
+ events.publish_event(
+ event_type=events.TASK_ENQUEUED,
+ task_name=task_name,
+ queue=queue,
+ count=len(kwargs_list),
+ )
+ return len(kwargs_list)
+
+
+def _resolve_task(*, task_name: str) -> "Task[..., object]":
+ try:
+ task = pkgutil.resolve_name(task_name)
+ except (AttributeError, ModuleNotFoundError, ValueError) as e:
+ raise TaskNotFoundError(task_name=task_name) from e
+
+ if not isinstance(task, Task):
+ raise NotACeleryTaskError(task_name=task_name)
+
+ return task
+
+
def is_celery_worker_available(*, timeout: float = 2.0) -> bool:
"""
Return True if at least one Celery worker responds within *timeout* seconds.
diff --git a/tests/unit/services/test_workertasks.py b/tests/unit/services/test_workertasks.py
index 69d6af6..6f54c30 100644
--- a/tests/unit/services/test_workertasks.py
+++ b/tests/unit/services/test_workertasks.py
@@ -4,7 +4,13 @@
from celery import Task
from src.services import events
-from src.services.workertasks import NotACeleryTaskError, TaskNotFoundError, enqueue_task, is_celery_worker_available
+from src.services.workertasks import (
+ NotACeleryTaskError,
+ TaskNotFoundError,
+ enqueue_task,
+ enqueue_tasks,
+ is_celery_worker_available,
+)
class TestEnqueueTask:
@@ -71,3 +77,62 @@ def test_returns_false_when_ping_returns_none(self):
def test_returns_false_when_ping_returns_empty_dict(self):
with self._patch_inspect({}):
assert is_celery_worker_available() is False
+
+
+class TestEnqueueTasks:
+ def _task(self):
+ task = MagicMock(spec=Task)
+ return task
+
+ def test_dispatches_one_message_per_entry(self):
+ task = self._task()
+ with patch("pkgutil.resolve_name", return_value=task):
+ sent = enqueue_tasks(
+ task_name="src.tasks.some_task",
+ kwargs_list=[{"n": 1}, {"n": 2}, {"n": 3}],
+ queue="default",
+ )
+
+ assert sent == 3
+ assert task.apply_async.call_count == 3
+
+ def test_resolves_the_task_once_however_many_messages(self):
+ task = self._task()
+ with patch("pkgutil.resolve_name", return_value=task) as resolve:
+ enqueue_tasks(
+ task_name="src.tasks.some_task",
+ kwargs_list=[{"n": i} for i in range(50)],
+ queue="default",
+ )
+
+ assert resolve.call_count == 1
+
+ def test_publishes_one_event_carrying_the_count(self, captured_logs):
+ task = self._task()
+ with patch("pkgutil.resolve_name", return_value=task):
+ enqueue_tasks(
+ task_name="src.tasks.some_task",
+ kwargs_list=[{"n": 1}, {"n": 2}],
+ queue="default",
+ )
+
+ matching = [e for e in captured_logs if e.get("event_type") == events.TASK_ENQUEUED]
+ assert len(matching) == 1
+ assert matching[0]["count"] == 2
+
+ def test_does_nothing_for_an_empty_list(self, captured_logs):
+ with patch("pkgutil.resolve_name") as resolve:
+ assert enqueue_tasks(task_name="src.tasks.some_task", kwargs_list=[], queue="default") == 0
+
+ resolve.assert_not_called()
+ assert [e for e in captured_logs if e.get("event_type") == events.TASK_ENQUEUED] == []
+
+ def test_task_not_found_raises_task_not_found_error(self):
+ with patch("pkgutil.resolve_name", side_effect=ModuleNotFoundError("no module")):
+ with pytest.raises(TaskNotFoundError):
+ enqueue_tasks(task_name="nope.some_task", kwargs_list=[{"n": 1}], queue="default")
+
+ def test_object_is_not_celery_task_raises_not_a_celery_task_error(self):
+ with patch("pkgutil.resolve_name", return_value=lambda: None):
+ with pytest.raises(NotACeleryTaskError):
+ enqueue_tasks(task_name="src.plain_function", kwargs_list=[{"n": 1}], queue="default")
From edc90f31fcac70052d0ed3f5070860d4dbfc6a28 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sun, 9 Aug 2026 13:09:20 +1000
Subject: [PATCH 30/36] Dispatch sync images to the worker in batches
Dispatch is synchronous: the command that starts a sync cannot return
until the last message is published, and make start runs it to completion
before the server binds. One message per file therefore puts the whole
publish loop on the critical path of startup, which on a large import is
tens of seconds of an unreachable server.
Batching cuts the message count by the batch size, so 14.5k files become
~146 messages. Each image is still handled individually inside the batch,
so progress, ignore records and run completion are unchanged; only the
number of broker round trips falls.
The task is renamed rather than reused, because its arguments changed.
That means restarting the worker once after deploying, the same
consequence ADR 011 recorded for adding a task. Messages already queued
under the old name would be rejected as unknown; their run is recovered as
interrupted on the next start and re-imported.
Lite mode still processes inline, unchanged.
Co-Authored-By: Claude Opus 5
---
.../usecases/library/finalize_sync_run.py | 16 ++++
.../usecases/library/sync_folder.py | 60 ++++++++++---
src/interfaces/tasks.py | 44 +++++++--
.../test_sync_process_image_batch_task.py | 90 +++++++++++++++++++
.../test_sync_process_image_task.py | 28 ------
.../application/library/test_sync_folder.py | 73 ++++++++++++++-
6 files changed, 263 insertions(+), 48 deletions(-)
create mode 100644 tests/functional/test_sync_process_image_batch_task.py
delete mode 100644 tests/functional/test_sync_process_image_task.py
diff --git a/src/application/usecases/library/finalize_sync_run.py b/src/application/usecases/library/finalize_sync_run.py
index 3ae4d4e..1c7a2f9 100644
--- a/src/application/usecases/library/finalize_sync_run.py
+++ b/src/application/usecases/library/finalize_sync_run.py
@@ -38,6 +38,22 @@ def finalize_sync_run(*, run: models.SyncRun) -> None:
library_operations.complete_sync_run(run=run)
+def finalize_sync_run_by_id(*, sync_run_id: int) -> None:
+ """
+ Finish the run with *sync_run_id*, if it still exists.
+
+ The entry point for finalising from a worker, which can only be handed an
+ id. A run whose folder was removed while the work was queued is gone, and
+ there is nothing left to finish.
+ """
+ try:
+ run = library_queries.get_sync_run(run_id=sync_run_id)
+ except library_queries.SyncRunNotFound:
+ return
+
+ finalize_sync_run(run=run)
+
+
def _prune_for_run(*, run: models.SyncRun) -> library_operations.PruneResult:
if _another_folder_is_still_importing(folder_id=run.folder_id):
# A file moved from this folder into one still importing has not been
diff --git a/src/application/usecases/library/sync_folder.py b/src/application/usecases/library/sync_folder.py
index c2a48b3..944203d 100644
--- a/src/application/usecases/library/sync_folder.py
+++ b/src/application/usecases/library/sync_folder.py
@@ -11,7 +11,8 @@
from src.domain.library import queries as library_queries
from src.services import workertasks
-_SYNC_PROCESS_IMAGE_TASK = "src.interfaces.tasks.sync_process_image_task"
+_SYNC_PROCESS_IMAGE_BATCH_TASK = "src.interfaces.tasks.sync_process_image_batch_task"
+_FINALIZE_SYNC_RUN_TASK = "src.interfaces.tasks.finalize_sync_run_task"
def sync_folder(*, folder_id: int, prune_mode: str = models.SyncRun.PRUNE_MODE_AUTO) -> None:
@@ -66,7 +67,7 @@ def sync_folder(*, folder_id: int, prune_mode: str = models.SyncRun.PRUNE_MODE_A
if not new_paths:
# Nothing new is exactly the case where photos were deleted, so this
# branch still has to finalise (and therefore prune).
- finalize_sync_run(run=run)
+ _finalize(run=run)
return
_dispatch(new_paths=new_paths, run=run)
@@ -110,12 +111,51 @@ def _drop_unchanged_ignored(*, folder_id: int, candidates: list[str]) -> list[st
def _dispatch(*, new_paths: list[str], run: models.SyncRun) -> None:
- for path in new_paths:
- if settings.USE_ASYNC_TASKS:
- workertasks.enqueue_task(
- task_name=_SYNC_PROCESS_IMAGE_TASK,
- kwargs={"image_path": path, "sync_run_id": run.pk},
- queue=settings.PROCESS_IMAGE_QUEUE,
- )
- else:
+ """
+ Hand the new images to whoever will process them.
+
+ In async mode they go to the worker in batches. Dispatching is synchronous,
+ so the command that started the sync cannot return until the last message is
+ published: one message per file makes a large import block startup for as
+ long as publishing takes, which on tens of thousands of files is tens of
+ seconds before the server is even reachable.
+ """
+ if not settings.USE_ASYNC_TASKS:
+ for path in new_paths:
process_synced_image(image_path=path, sync_run_id=run.pk)
+ return
+
+ workertasks.enqueue_tasks(
+ task_name=_SYNC_PROCESS_IMAGE_BATCH_TASK,
+ kwargs_list=[
+ {"image_paths": batch, "sync_run_id": run.pk}
+ for batch in _batched(new_paths, settings.SYNC_IMAGE_BATCH_SIZE)
+ ],
+ queue=settings.PROCESS_IMAGE_QUEUE,
+ )
+
+
+def _finalize(*, run: models.SyncRun) -> None:
+ """
+ Finish a run that had no images to process.
+
+ In async mode this goes to the worker, like every other part of a sync.
+ Doing it here instead would put a second full walk of the folder, and the
+ removals that follow it, inside whoever started the sync: the startup
+ command before the server is reachable, or a web request. It would also mean
+ the same work happened in the worker or in the caller depending only on
+ whether anything happened to be new.
+ """
+ if not settings.USE_ASYNC_TASKS:
+ finalize_sync_run(run=run)
+ return
+
+ workertasks.enqueue_task(
+ task_name=_FINALIZE_SYNC_RUN_TASK,
+ kwargs={"sync_run_id": run.pk},
+ queue=settings.PROCESS_IMAGE_QUEUE,
+ )
+
+
+def _batched(paths: list[str], size: int) -> list[list[str]]:
+ return [paths[start : start + size] for start in range(0, len(paths), size)]
diff --git a/src/interfaces/tasks.py b/src/interfaces/tasks.py
index 3666de6..ded96aa 100644
--- a/src/interfaces/tasks.py
+++ b/src/interfaces/tasks.py
@@ -4,6 +4,7 @@
from celery import shared_task
from django.conf import settings
+from src.application.usecases.library.finalize_sync_run import finalize_sync_run_by_id
from src.application.usecases.library.process_synced_image import process_synced_image
from src.domain.images import events, operations
from src.domain.images.thumbnails import operations as thumbnail_operations
@@ -46,14 +47,45 @@ def process_image_task(self: Any, /, *, image_path: str, **kwargs: object) -> st
return f"Processed {recipe.filename}"
-@shared_task(name="library.sync_process_image", bind=True, queue=settings.PROCESS_IMAGE_QUEUE)
-def sync_process_image_task(self: Any, /, *, image_path: str, sync_run_id: int, **kwargs: object) -> str:
+@shared_task(name="library.sync_process_image_batch", bind=True, queue=settings.PROCESS_IMAGE_QUEUE)
+def sync_process_image_batch_task(
+ self: Any,
+ /,
+ *,
+ image_paths: list[str],
+ sync_run_id: int,
+ **kwargs: object,
+) -> str:
"""
- Celery task that processes a single image for a library sync run and reports
- progress against the run.
+ Celery task that processes a batch of images for a library sync run and
+ reports progress against the run.
+
+ A batch rather than a single image because dispatch is synchronous: the
+ command that starts a sync cannot return until every message is published, so
+ one message per file makes a large import block startup for as long as it
+ takes to publish them all.
+
+ Each image is still handled one at a time and accounted for individually, so
+ progress, ignore records and run completion behave exactly as they would have
+ per message.
+ """
+ for image_path in image_paths:
+ process_synced_image(image_path=image_path, sync_run_id=sync_run_id)
+ return f"Processed {len(image_paths)} image(s) for sync run {sync_run_id}"
+
+
+@shared_task(name="library.finalize_sync_run", bind=True, queue=settings.PROCESS_IMAGE_QUEUE)
+def finalize_sync_run_task(self: Any, /, *, sync_run_id: int, **kwargs: object) -> str:
+ """
+ Celery task that finishes a sync run that had no images to process.
+
+ A run with images is finished by whichever of them is handled last, so this
+ exists for the case with none: photos were only deleted. Without it that work
+ would run wherever the sync was started, which for a startup sync means
+ walking the whole tree again before the server is reachable.
"""
- process_synced_image(image_path=image_path, sync_run_id=sync_run_id)
- return f"Processed {image_path} for sync run {sync_run_id}"
+ finalize_sync_run_by_id(sync_run_id=sync_run_id)
+ return f"Finalized sync run {sync_run_id}"
@shared_task(name="domain.generate_thumbnail", bind=True, queue=settings.PROCESS_IMAGE_QUEUE)
diff --git a/tests/functional/test_sync_process_image_batch_task.py b/tests/functional/test_sync_process_image_batch_task.py
new file mode 100644
index 0000000..f359dc6
--- /dev/null
+++ b/tests/functional/test_sync_process_image_batch_task.py
@@ -0,0 +1,90 @@
+import shutil
+from pathlib import Path
+
+import pytest
+
+from src.data import models
+from src.interfaces.tasks import finalize_sync_run_task, sync_process_image_batch_task
+from tests.factories import ImageFactory, LibraryFolderFactory, SyncRunFactory
+
+FIXTURES_DIR = Path(__file__).resolve().parent.parent / "fixtures" / "images"
+FUJIFILM_FIXTURE = FIXTURES_DIR / "XS107114.JPG"
+SECOND_FIXTURE = FIXTURES_DIR / "XS107209.jpg"
+NON_FUJIFILM_FIXTURE = FIXTURES_DIR / "sub-folder" / "img_4968_dng_embedded.jpg"
+
+
+@pytest.mark.django_db
+class TestSyncProcessImageBatchTask:
+ def test_processes_a_single_image_and_records_progress_against_run(self, tmp_path):
+ image_path = tmp_path / FUJIFILM_FIXTURE.name
+ shutil.copy(FUJIFILM_FIXTURE, image_path)
+ run = SyncRunFactory(state=models.SyncRun.STATE_PROCESSING, total=1)
+
+ sync_process_image_batch_task.apply(
+ kwargs={"image_paths": [str(image_path)], "sync_run_id": run.pk}
+ ).get()
+
+ run.refresh_from_db()
+ assert run.processed == 1
+ assert run.state == models.SyncRun.STATE_COMPLETED
+ assert models.Image.objects.filter(filepath=str(image_path)).exists()
+
+ def test_processes_every_image_in_the_batch(self, tmp_path):
+ first = tmp_path / FUJIFILM_FIXTURE.name
+ second = tmp_path / SECOND_FIXTURE.name
+ shutil.copy(FUJIFILM_FIXTURE, first)
+ shutil.copy(SECOND_FIXTURE, second)
+ run = SyncRunFactory(state=models.SyncRun.STATE_PROCESSING, total=2)
+
+ sync_process_image_batch_task.apply(
+ kwargs={"image_paths": [str(first), str(second)], "sync_run_id": run.pk}
+ ).get()
+
+ run.refresh_from_db()
+ assert run.processed == 2
+ assert run.state == models.SyncRun.STATE_COMPLETED
+ assert models.Image.objects.count() == 2
+
+ def test_accounts_for_each_image_separately_within_a_batch(self, tmp_path):
+ importable = tmp_path / FUJIFILM_FIXTURE.name
+ rejected = tmp_path / NON_FUJIFILM_FIXTURE.name
+ shutil.copy(FUJIFILM_FIXTURE, importable)
+ shutil.copy(NON_FUJIFILM_FIXTURE, rejected)
+ run = SyncRunFactory(state=models.SyncRun.STATE_PROCESSING, total=2)
+
+ sync_process_image_batch_task.apply(
+ kwargs={"image_paths": [str(importable), str(rejected)], "sync_run_id": run.pk}
+ ).get()
+
+ run.refresh_from_db()
+ assert run.processed == 1
+ assert run.skipped == 1
+ assert models.IgnoredImage.objects.count() == 1
+
+ def test_an_empty_batch_does_nothing(self):
+ run = SyncRunFactory(state=models.SyncRun.STATE_PROCESSING, total=0)
+
+ sync_process_image_batch_task.apply(
+ kwargs={"image_paths": [], "sync_run_id": run.pk}
+ ).get()
+
+ run.refresh_from_db()
+ assert run.processed == 0
+
+
+@pytest.mark.django_db
+class TestFinalizeSyncRunTask:
+ def test_finishes_the_run_and_removes_what_is_gone(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ run = SyncRunFactory(folder=folder, state=models.SyncRun.STATE_PROCESSING, total=0)
+ image = ImageFactory(filepath=str(tmp_path / "gone.jpg"))
+
+ finalize_sync_run_task.apply(kwargs={"sync_run_id": run.pk}).get()
+
+ run.refresh_from_db()
+ assert run.state == models.SyncRun.STATE_COMPLETED
+ assert run.removed == 1
+ assert not models.Image.objects.filter(pk=image.pk).exists()
+
+ def test_does_nothing_for_a_run_that_no_longer_exists(self):
+ finalize_sync_run_task.apply(kwargs={"sync_run_id": 9999}).get()
diff --git a/tests/functional/test_sync_process_image_task.py b/tests/functional/test_sync_process_image_task.py
deleted file mode 100644
index 4c8d5ee..0000000
--- a/tests/functional/test_sync_process_image_task.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import shutil
-from pathlib import Path
-
-import pytest
-
-from src.data import models
-from src.interfaces.tasks import sync_process_image_task
-from tests.factories import SyncRunFactory
-
-FIXTURES_DIR = Path(__file__).resolve().parent.parent / "fixtures" / "images"
-FUJIFILM_FIXTURE = FIXTURES_DIR / "XS107114.JPG"
-
-
-@pytest.mark.django_db
-class TestSyncProcessImageTask:
- def test_processes_image_and_records_progress_against_run(self, tmp_path):
- image_path = tmp_path / FUJIFILM_FIXTURE.name
- shutil.copy(FUJIFILM_FIXTURE, image_path)
- run = SyncRunFactory(state=models.SyncRun.STATE_PROCESSING, total=1)
-
- sync_process_image_task.apply(
- kwargs={"image_path": str(image_path), "sync_run_id": run.pk}
- ).get()
-
- run.refresh_from_db()
- assert run.processed == 1
- assert run.state == models.SyncRun.STATE_COMPLETED
- assert models.Image.objects.filter(filepath=str(image_path)).exists()
diff --git a/tests/integration/application/library/test_sync_folder.py b/tests/integration/application/library/test_sync_folder.py
index b567330..27b50b0 100644
--- a/tests/integration/application/library/test_sync_folder.py
+++ b/tests/integration/application/library/test_sync_folder.py
@@ -13,6 +13,7 @@
FIXTURES_DIR = Path(__file__).resolve().parent.parent.parent.parent / "fixtures" / "images"
FUJIFILM_FIXTURE = FIXTURES_DIR / "XS107114.JPG"
NON_FUJIFILM_FIXTURE = FIXTURES_DIR / "sub-folder" / "img_4968_dng_embedded.jpg"
+SECOND_FUJIFILM_FIXTURE = FIXTURES_DIR / "XS107209.jpg"
@pytest.mark.django_db
@@ -82,12 +83,12 @@ def test_fails_run_and_stamps_check_time_when_folder_missing(self, tmp_path):
@pytest.mark.django_db
class TestSyncFolderAsyncMode:
@override_settings(USE_ASYNC_TASKS=True)
- def test_enqueues_a_task_per_new_image_and_leaves_run_processing(self, tmp_path):
+ def test_enqueues_a_batch_of_new_images_and_leaves_run_processing(self, tmp_path):
image_path = tmp_path / FUJIFILM_FIXTURE.name
shutil.copy(FUJIFILM_FIXTURE, image_path)
folder = LibraryFolderFactory(path=str(tmp_path))
- with patch("src.application.usecases.library.sync_folder.workertasks.enqueue_task") as mock_enqueue:
+ with patch("src.application.usecases.library.sync_folder.workertasks.enqueue_tasks") as mock_enqueue:
sync_folder(folder_id=folder.pk)
run = models.SyncRun.objects.get(folder=folder)
@@ -95,8 +96,24 @@ def test_enqueues_a_task_per_new_image_and_leaves_run_processing(self, tmp_path)
assert run.total == 1
mock_enqueue.assert_called_once()
kwargs = mock_enqueue.call_args.kwargs
- assert kwargs["task_name"] == "src.interfaces.tasks.sync_process_image_task"
- assert kwargs["kwargs"] == {"image_path": str(image_path), "sync_run_id": run.pk}
+ assert kwargs["task_name"] == "src.interfaces.tasks.sync_process_image_batch_task"
+ assert kwargs["kwargs_list"] == [
+ {"image_paths": [str(image_path)], "sync_run_id": run.pk}
+ ]
+
+ def test_splits_new_images_into_batches(self, tmp_path, settings):
+ settings.SYNC_IMAGE_BATCH_SIZE = 2
+ for index, fixture in enumerate(
+ [FUJIFILM_FIXTURE, SECOND_FUJIFILM_FIXTURE, NON_FUJIFILM_FIXTURE]
+ ):
+ shutil.copy(fixture, tmp_path / f"{index}_{fixture.name}")
+ folder = LibraryFolderFactory(path=str(tmp_path))
+
+ with patch("src.application.usecases.library.sync_folder.workertasks.enqueue_tasks") as mock_enqueue:
+ sync_folder(folder_id=folder.pk)
+
+ batches = mock_enqueue.call_args.kwargs["kwargs_list"]
+ assert [len(b["image_paths"]) for b in batches] == [2, 1]
@pytest.mark.django_db
@@ -202,3 +219,51 @@ def test_an_importable_file_is_unaffected(self, tmp_path):
assert models.Image.objects.count() == 1
assert models.IgnoredImage.objects.count() == 0
+
+
+@pytest.mark.django_db
+class TestSyncFolderFinalisesInTheWorker:
+ """
+ A run with images is finished by whichever is handled last, in the worker.
+ A run with none must not be finished somewhere else, or the same work lands
+ in the worker or in the caller depending only on whether anything was new.
+ """
+
+ @pytest.fixture(autouse=True)
+ def _async_mode(self, settings):
+ settings.USE_ASYNC_TASKS = True
+
+ def test_hands_an_empty_run_to_the_worker(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+
+ with patch("src.application.usecases.library.sync_folder.workertasks.enqueue_task") as enqueue:
+ sync_folder(folder_id=folder.pk)
+
+ run = models.SyncRun.objects.get(folder=folder)
+ enqueue.assert_called_once()
+ kwargs = enqueue.call_args.kwargs
+ assert kwargs["task_name"] == "src.interfaces.tasks.finalize_sync_run_task"
+ assert kwargs["kwargs"] == {"sync_run_id": run.pk}
+
+ def test_does_not_prune_in_the_caller(self, tmp_path):
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ image = ImageFactory(filepath=str(tmp_path / "gone.jpg"))
+
+ with patch("src.application.usecases.library.sync_folder.workertasks.enqueue_task"):
+ sync_folder(folder_id=folder.pk)
+
+ run = models.SyncRun.objects.get(folder=folder)
+ assert run.state == models.SyncRun.STATE_PROCESSING
+ assert run.removed == 0
+ assert models.Image.objects.filter(pk=image.pk).exists()
+
+ def test_lite_mode_still_finalises_inline(self, tmp_path, settings):
+ settings.USE_ASYNC_TASKS = False
+ folder = LibraryFolderFactory(path=str(tmp_path))
+ image = ImageFactory(filepath=str(tmp_path / "gone.jpg"))
+
+ sync_folder(folder_id=folder.pk)
+
+ run = models.SyncRun.objects.get(folder=folder)
+ assert run.state == models.SyncRun.STATE_COMPLETED
+ assert not models.Image.objects.filter(pk=image.pk).exists()
From e2c55459627dc50363d7f655e31ac9940fa13fb1 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sun, 9 Aug 2026 13:26:42 +1000
Subject: [PATCH 31/36] Add retry_ignored_image(s) use cases
Nothing calls them yet. Forgetting a record is how a file gets another
chance, so it needs to be reachable from both a button and a flag.
Limiting a bulk retry to one reason is the case that matters: a batch that
failed for an environmental reason deserves another look, while files that
are simply not Fujifilm would only be re-read to reach the same verdict.
Co-Authored-By: Claude Opus 5
---
.../usecases/library/retry_ignored_images.py | 68 ++++++++++++++++++
.../library/test_retry_ignored_images.py | 69 +++++++++++++++++++
2 files changed, 137 insertions(+)
create mode 100644 src/application/usecases/library/retry_ignored_images.py
create mode 100644 tests/integration/application/library/test_retry_ignored_images.py
diff --git a/src/application/usecases/library/retry_ignored_images.py b/src/application/usecases/library/retry_ignored_images.py
new file mode 100644
index 0000000..7776fed
--- /dev/null
+++ b/src/application/usecases/library/retry_ignored_images.py
@@ -0,0 +1,68 @@
+import attrs
+
+from src.domain.library import operations as library_operations
+from src.domain.library import queries as library_queries
+
+
+@attrs.frozen
+class LibraryFolderNotFound(Exception):
+ """
+ Raised when no library folder with the given id exists.
+ """
+
+ folder_id: int
+
+
+@attrs.frozen
+class IgnoredImageNotFound(Exception):
+ """
+ Raised when no ignored-image record with the given id exists.
+ """
+
+ ignored_id: int
+
+
+@attrs.frozen
+class RetryIgnoredImagesResult:
+ forgotten: int
+
+
+@attrs.frozen
+class RetryIgnoredImageResult:
+ filepath: str
+
+
+def retry_ignored_images(*, folder_id: int, reason: str | None = None) -> RetryIgnoredImagesResult:
+ """
+ Forget what a folder has ignored, so the next sync examines those files again.
+
+ Limiting to one reason is the common case: a batch that failed for an
+ environmental reason deserves another look, while files that are simply not
+ Fujifilm do not.
+
+ Forgets records only. No file is touched, and nothing enters the gallery
+ until a sync actually imports it.
+
+ :raises LibraryFolderNotFound: If no folder with *folder_id* exists.
+ """
+ try:
+ library_queries.get_library_folder(folder_id=folder_id)
+ except library_queries.LibraryFolderNotFound:
+ raise LibraryFolderNotFound(folder_id=folder_id)
+
+ forgotten = library_operations.forget_ignored_images(folder_id=folder_id, reason=reason)
+ return RetryIgnoredImagesResult(forgotten=forgotten)
+
+
+def retry_ignored_image(*, ignored_id: int) -> RetryIgnoredImageResult:
+ """
+ Forget one ignored file, so the next sync examines it again.
+
+ :raises IgnoredImageNotFound: If no record with *ignored_id* exists.
+ """
+ try:
+ filepath = library_operations.forget_ignored_image(ignored_id=ignored_id)
+ except library_queries.IgnoredImageNotFound:
+ raise IgnoredImageNotFound(ignored_id=ignored_id)
+
+ return RetryIgnoredImageResult(filepath=filepath)
diff --git a/tests/integration/application/library/test_retry_ignored_images.py b/tests/integration/application/library/test_retry_ignored_images.py
new file mode 100644
index 0000000..17c0757
--- /dev/null
+++ b/tests/integration/application/library/test_retry_ignored_images.py
@@ -0,0 +1,69 @@
+import pytest
+
+from src.application.usecases.library.retry_ignored_images import (
+ IgnoredImageNotFound,
+ LibraryFolderNotFound,
+ retry_ignored_image,
+ retry_ignored_images,
+)
+from src.data import models
+from tests.factories import IgnoredImageFactory, LibraryFolderFactory
+
+
+@pytest.mark.django_db
+class TestRetryIgnoredImages:
+ def test_forgets_every_record_in_the_folder(self):
+ folder = LibraryFolderFactory()
+ IgnoredImageFactory(folder=folder, filepath="/a/1.jpg")
+ IgnoredImageFactory(folder=folder, filepath="/a/2.jpg")
+
+ result = retry_ignored_images(folder_id=folder.pk)
+
+ assert result.forgotten == 2
+ assert models.IgnoredImage.objects.count() == 0
+
+ def test_forgets_only_the_requested_reason(self):
+ folder = LibraryFolderFactory()
+ kept = IgnoredImageFactory(
+ folder=folder,
+ filepath="/a/1.jpg",
+ reason=models.IgnoredImage.REASON_NO_FILM_SIMULATION,
+ )
+ IgnoredImageFactory(
+ folder=folder,
+ filepath="/a/2.jpg",
+ reason=models.IgnoredImage.REASON_ERROR,
+ )
+
+ result = retry_ignored_images(folder_id=folder.pk, reason=models.IgnoredImage.REASON_ERROR)
+
+ assert result.forgotten == 1
+ assert list(models.IgnoredImage.objects.values_list("pk", flat=True)) == [kept.pk]
+
+ def test_reports_zero_when_there_is_nothing_to_forget(self):
+ folder = LibraryFolderFactory()
+
+ assert retry_ignored_images(folder_id=folder.pk).forgotten == 0
+
+ def test_raises_library_folder_not_found_for_unknown_id(self):
+ with pytest.raises(LibraryFolderNotFound) as exc_info:
+ retry_ignored_images(folder_id=9999)
+
+ assert exc_info.value.folder_id == 9999
+
+
+@pytest.mark.django_db
+class TestRetryIgnoredImage:
+ def test_forgets_the_record_and_reports_its_path(self):
+ ignored = IgnoredImageFactory(filepath="/photos/other.jpg")
+
+ result = retry_ignored_image(ignored_id=ignored.pk)
+
+ assert result.filepath == "/photos/other.jpg"
+ assert models.IgnoredImage.objects.count() == 0
+
+ def test_raises_ignored_image_not_found_for_unknown_id(self):
+ with pytest.raises(IgnoredImageNotFound) as exc_info:
+ retry_ignored_image(ignored_id=9999)
+
+ assert exc_info.value.ignored_id == 9999
From 243b342d65eb396955ba5ee5bc42c2cd53d6f5b3 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sun, 9 Aug 2026 13:34:11 +1000
Subject: [PATCH 32/36] Add --retry-failed to the sync_library command
Ignored files are now left alone until they change, which is right for the
common case and wrong when the reason they failed was environmental and
has since been put right. This is the escape hatch: forget everything and
look again.
It warns in its own help text that this is slow on a library carrying many
ignored files, because it is: examining them again is exactly the cost the
records exist to avoid.
Co-Authored-By: Claude Opus 5
---
.../management/commands/sync_library.py | 25 ++++++++++++
tests/functional/test_sync_library_command.py | 38 +++++++++++++++++++
2 files changed, 63 insertions(+)
diff --git a/src/interfaces/management/commands/sync_library.py b/src/interfaces/management/commands/sync_library.py
index b91f461..ba774f1 100644
--- a/src/interfaces/management/commands/sync_library.py
+++ b/src/interfaces/management/commands/sync_library.py
@@ -3,9 +3,11 @@
from django.conf import settings
from django.core.management.base import BaseCommand, CommandParser
+from src.application.usecases.library import retry_ignored_images as retry_ignored_images_usecase
from src.application.usecases.library import sync_library as sync_library_usecase
from src.application.usecases.library.sync_library import CeleryWorkerUnavailable
from src.data import models
+from src.domain.library import queries as library_queries
class Command(BaseCommand):
@@ -31,10 +33,22 @@ def add_arguments(self, parser: CommandParser) -> None:
action="store_true",
help="Import only; never remove catalog entries for missing files.",
)
+ parser.add_argument(
+ "--retry-failed",
+ action="store_true",
+ help=(
+ "Examine every previously skipped or failed file again, instead of leaving them"
+ " alone until they change. Slow on a library with many of them."
+ ),
+ )
def handle(self, *args: object, **options: Any) -> None:
prune_mode = _prune_mode_from(options=options)
+ if options["retry_failed"]:
+ forgotten = _forget_every_ignored_image()
+ self.stdout.write(f"Forgot {forgotten} previously skipped or failed file(s).")
+
try:
result = sync_library_usecase.sync_library(prune_mode=prune_mode)
except CeleryWorkerUnavailable:
@@ -97,6 +111,17 @@ def _report(self, *, warning: sync_library_usecase.PruneWarning) -> None:
)
+def _forget_every_ignored_image() -> int:
+ """
+ Forget what every registered folder has ignored, so the next scan looks at
+ all of it again.
+ """
+ return sum(
+ retry_ignored_images_usecase.retry_ignored_images(folder_id=folder.pk).forgotten
+ for folder in library_queries.get_all_library_folders()
+ )
+
+
def _prune_mode_from(*, options: dict[str, Any]) -> str:
if options["force_prune"]:
return models.SyncRun.PRUNE_MODE_FORCE
diff --git a/tests/functional/test_sync_library_command.py b/tests/functional/test_sync_library_command.py
index 3979b57..19d5fa2 100644
--- a/tests/functional/test_sync_library_command.py
+++ b/tests/functional/test_sync_library_command.py
@@ -13,6 +13,7 @@
FIXTURES_DIR = Path(__file__).resolve().parent.parent / "fixtures" / "images"
FUJIFILM_FIXTURE = FIXTURES_DIR / "XS107114.JPG"
+NON_FUJIFILM_FIXTURE = FIXTURES_DIR / "sub-folder" / "img_4968_dng_embedded.jpg"
@pytest.mark.django_db
@@ -145,3 +146,40 @@ def test_force_prune_overrides_the_guard(self, tmp_path, capsys):
call_command("sync_library", "--force-prune")
assert models.Image.objects.count() == 0
+
+
+@pytest.mark.django_db
+class TestSyncLibraryCommandRetryFailed:
+ @pytest.fixture(autouse=True)
+ def _lite_mode(self, settings):
+ settings.USE_ASYNC_TASKS = False
+
+ def _library_with_an_ignored_file(self, tmp_path):
+ photo = tmp_path / NON_FUJIFILM_FIXTURE.name
+ shutil.copy(NON_FUJIFILM_FIXTURE, photo)
+ LibraryFolderFactory(path=str(tmp_path))
+ call_command("sync_library")
+ return photo
+
+ def test_leaves_ignored_files_alone_without_the_flag(self, tmp_path, capsys):
+ self._library_with_an_ignored_file(tmp_path)
+
+ call_command("sync_library")
+
+ assert "0 new file(s) imported, 0 skipped" in capsys.readouterr().out
+
+ def test_examines_them_again_with_the_flag(self, tmp_path, capsys):
+ self._library_with_an_ignored_file(tmp_path)
+
+ call_command("sync_library", "--retry-failed")
+
+ captured = capsys.readouterr()
+ assert "Forgot 1 previously skipped or failed file(s)." in captured.out
+ assert "1 skipped (non-Fujifilm)" in captured.out
+
+ def test_re_records_the_file_it_examined_again(self, tmp_path):
+ self._library_with_an_ignored_file(tmp_path)
+
+ call_command("sync_library", "--retry-failed")
+
+ assert models.IgnoredImage.objects.count() == 1
From fc4b4b22ab9fa2c873d9ba0d24ddd0a2460bb1a7 Mon Sep 17 00:00:00 2001
From: Gosku
Date: Sun, 9 Aug 2026 16:02:06 +1000
Subject: [PATCH 33/36] Add a page listing the files a folder could not import
Records were being kept but there was nowhere to see them, so 14.5k
skipped files and 93 errors existed only as counters that stopped being
reported once the files stopped being re-examined.
Paginated and filterable by reason, because the two groups want different
attention: the non-Fujifilm ones are noise, and the handful of errors are
the ones worth reading. Without the filter the errors are buried under
thousands of rows and effectively unfindable. Each error shows the message
that caused it, which until now lived only in a log line.
The page leads by saying no file was deleted or changed, since a list of
thousands of "ignored" photos invites exactly that fear.
Co-Authored-By: Claude Opus 5
---
.../usecases/library/dataclasses.py | 22 +++
src/interfaces/library/urls.py | 1 +
src/interfaces/library/views.py | 68 +++++++++
.../templates/library/ignored_images.html | 130 ++++++++++++++++++
.../test_library_ignored_images_view.py | 97 +++++++++++++
5 files changed, 318 insertions(+)
create mode 100644 src/interfaces/templates/library/ignored_images.html
create mode 100644 tests/functional/test_library_ignored_images_view.py
diff --git a/src/application/usecases/library/dataclasses.py b/src/application/usecases/library/dataclasses.py
index b13b70f..a9e4db0 100644
--- a/src/application/usecases/library/dataclasses.py
+++ b/src/application/usecases/library/dataclasses.py
@@ -47,3 +47,25 @@ class FilesystemBrowseResult:
current_path: str
parent_path: str | None
entries: tuple[FilesystemEntry, ...]
+
+
+@attrs.frozen
+class IgnoredImageData:
+ ignored_id: int
+ filepath: str
+ filename: str
+ reason_label: str
+ detail: str
+ created_at: datetime
+ # An unchanged file that is simply not Fujifilm will be rejected again the
+ # moment it is examined, so retrying it does nothing until the file changes.
+ # Resolved here so the template can say so rather than imply otherwise.
+ retry_is_a_no_op_until_the_file_changes: bool
+
+
+@attrs.frozen
+class IgnoredReasonFilter:
+ code: str
+ label: str
+ count: int
+ is_active: bool
diff --git a/src/interfaces/library/urls.py b/src/interfaces/library/urls.py
index 3fca856..bdde2d1 100644
--- a/src/interfaces/library/urls.py
+++ b/src/interfaces/library/urls.py
@@ -9,5 +9,6 @@
path("library//confirm-delete/", views.LibraryFolderRemoveConfirm.as_view(), name="library-folder-confirm-delete"),
path("library//delete/", views.LibraryFolderRemove.as_view(), name="library-folder-delete"),
path("library//edit/", views.LibraryFolderPathUpdate.as_view(), name="library-folder-edit"),
+ path("library//ignored/", views.LibraryFolderIgnoredImages.as_view(), name="library-folder-ignored"),
path("library//sync-status/", views.LibraryFolderSyncStatus.as_view(), name="library-folder-sync-status"),
]
diff --git a/src/interfaces/library/views.py b/src/interfaces/library/views.py
index 84f31d1..9b67a2d 100644
--- a/src/interfaces/library/views.py
+++ b/src/interfaces/library/views.py
@@ -1,4 +1,8 @@
+import os
+
from django import http, shortcuts, urls
+from django.conf import settings
+from django.core import paginator as django_paginator
from django.views import generic
from src.application.usecases.library import add_library_folder as add_library_folder_uc
@@ -203,3 +207,67 @@ def get(self, request: http.HttpRequest) -> http.HttpResponse:
"action_url": action_url,
"folder_id": folder_id,
})
+
+
+_IGNORED_REASON_LABELS = {
+ models.IgnoredImage.REASON_NO_FILM_SIMULATION: "Not a Fujifilm photo",
+ models.IgnoredImage.REASON_INVALID_RECIPE_DATA: "Recipe could not be read",
+ models.IgnoredImage.REASON_ERROR: "Failed with an error",
+}
+# Only an error is worth retrying by hand. The other two are verdicts on the
+# file's own contents, so an unchanged file gets the same verdict again.
+_RETRY_CHANGES_SOMETHING = {models.IgnoredImage.REASON_ERROR}
+
+
+def _ignored_image_data(ignored: models.IgnoredImage) -> library_dataclasses.IgnoredImageData:
+ return library_dataclasses.IgnoredImageData(
+ ignored_id=ignored.pk,
+ filepath=ignored.filepath,
+ filename=os.path.basename(ignored.filepath),
+ reason_label=_IGNORED_REASON_LABELS.get(ignored.reason, ignored.reason),
+ detail=ignored.detail,
+ created_at=ignored.created_at,
+ retry_is_a_no_op_until_the_file_changes=ignored.reason not in _RETRY_CHANGES_SOMETHING,
+ )
+
+
+def _reason_filters(*, counts: dict[str, int], active: str | None) -> list[library_dataclasses.IgnoredReasonFilter]:
+ return [
+ library_dataclasses.IgnoredReasonFilter(
+ code=code,
+ label=label,
+ count=counts.get(code, 0),
+ is_active=active == code,
+ )
+ for code, label in _IGNORED_REASON_LABELS.items()
+ if counts.get(code, 0)
+ ]
+
+
+class LibraryFolderIgnoredImages(generic.View):
+ """List the files this folder's syncs could not import.
+
+ :raises Http404: if no folder with the given ID exists.
+ """
+
+ def get(self, request: http.HttpRequest, folder_id: int) -> http.HttpResponse:
+ try:
+ folder = domain_queries.get_library_folder(folder_id=folder_id)
+ except domain_queries.LibraryFolderNotFound:
+ raise http.Http404
+
+ reason = request.GET.get("reason") or None
+ ignored = domain_queries.get_ignored_images(folder_id=folder_id, reason=reason)
+ counts = domain_queries.count_ignored_images_by_reason(folder_id=folder_id)
+ page_obj = django_paginator.Paginator(ignored, settings.GALLERY_PAGE_SIZE).get_page(
+ request.GET.get("page", 1)
+ )
+
+ return shortcuts.render(request, "library/ignored_images.html", {
+ "folder": _folder_data(folder),
+ "ignored_images": [_ignored_image_data(i) for i in page_obj],
+ "page_obj": page_obj,
+ "reason_filters": _reason_filters(counts=counts, active=reason),
+ "active_reason": reason,
+ "total": sum(counts.values()),
+ })
diff --git a/src/interfaces/templates/library/ignored_images.html b/src/interfaces/templates/library/ignored_images.html
new file mode 100644
index 0000000..2728b7e
--- /dev/null
+++ b/src/interfaces/templates/library/ignored_images.html
@@ -0,0 +1,130 @@
+
+
+
+
+
+ Ignored files
+
+
+
+
+
+{% include "_top_nav.html" with active_section="library" %}
+
+
+ These {{ total }} file{{ total|pluralize }} could not be imported, so the sync leaves them alone
+ instead of reading them again on every run. None of them has been deleted or changed;
+ they simply are not in the gallery. Any file you edit or replace is examined again on its own,
+ without you having to do anything here.
+