upload: report orphan images with find_usages and find_orphans

Makefile

@@ -16,6 +16,10 @@ refresh-karmas:
 refresh-karmas-apply:
 	python scripts/refresh_karmas.py --apply
 
+# Report uploaded images that no body references any more. Nothing is deleted.
+upload-report:
+	python scripts/upload_report.py
+
 dev-run:
 	uvicorn --host $(LOCAL_DEV_HOST) --port $(LOCAL_DEV_PORT) chafan_core.app.main:app --reload
 

chafan_core/app/services/uploads.py

@@ -4,12 +4,15 @@
 
 import hashlib
 import logging
-from typing import Any
+from typing import Any, List
+
+from sqlalchemy.orm import Session
 
 from chafan_core.app import (
     coins,
     crud,
     image_sanitize,
+    models,
     object_storage,
     rules,
     schemas,
@@ -84,3 +87,50 @@ def upload_image(ctx, *, file, file_size: int, purpose: str) -> schemas.Uploaded
         storage_bucket=settings.UPLOADS_S3_BUCKET,
     )
     return schemas.UploadedImage(url=object_storage.public_url(sha, content_type))
+
+
+# ---------------------------------------------------------------------------
+# Usage accounting: computed, not stored.
+#
+# delete_forever overwrites body, body_draft and every archive row with
+# "[DELETED]", so the references leave the database and a scan of bodies gives
+# the correct answer with no join table to maintain. Nothing here deletes.
+# ---------------------------------------------------------------------------
+
+_BODY_COLUMNS = [
+    (models.Article, "body", "article"),
+    (models.Article, "body_draft", "article"),
+    (models.ArticleArchive, "body", "article_archive"),
+    (models.Answer, "body", "answer"),
+    (models.Answer, "body_draft", "answer"),
+    (models.Archive, "body", "answer_archive"),
+    (models.Comment, "body", "comment"),
+    (models.Submission, "description", "submission"),
+    (models.SubmissionArchive, "description", "submission_archive"),
+]
+
+
+def find_usages(db: Session, *, sha: str) -> List[str]:
+    """Locations (``table:id``) whose body text embeds ``sha``.
+
+    The public URL embeds the sha, so a ``LIKE`` hit is exact.
+    """
+    usages: List[str] = []
+    for model, column, label in _BODY_COLUMNS:
+        col = getattr(model, column)
+        for (id_,) in db.query(model.id).filter(col.like(f"%{sha}%")).all():
+            usages.append(f"{label}:{id_}")
+    return usages
+
+
+def find_orphans(db: Session) -> List[Any]:
+    """Uploads whose sha appears in no body or archive text. Report only.
+
+    Returns nothing until the Upload table lands (``crud.upload`` is stubbed);
+    typed ``Any`` so the real model can arrive later without touching this.
+    """
+    orphans: List[Any] = []
+    for sha in crud.upload.all_shas(db):
+        if not find_usages(db, sha=sha):
+            orphans.extend(crud.upload.get_multi_by_sha(db, sha=sha))
+    return orphans

scripts/upload_report.py

@@ -0,0 +1,58 @@
+"""Report which uploaded images are orphans -- referenced by no body any more.
+
+    make upload-report              # list orphans
+    make upload-report-sha --sha=<sha>  # list usages of one sha (not wired to make)
+
+Storage in the upload bucket is treated as losable; the upload table is the
+recovery manifest. Orphans here are objects that no live or archived body
+references any more, and are the candidates for a future garbage-collection
+step. Nothing is deleted from the bucket or the table by this script.
+"""
+
+import os.path
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+
+import argparse
+
+from chafan_core.app import crud
+from chafan_core.app.services import uploads as uploads_service
+from chafan_core.db.session import SessionLocal
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--sha",
+        help="print usages of a single sha instead of scanning for orphans",
+    )
+    args = parser.parse_args()
+
+    db = SessionLocal()
+    try:
+        if args.sha:
+            usages = uploads_service.find_usages(db, sha=args.sha)
+            for usage in usages:
+                print(usage)
+            print(f"{len(usages)} usage(s) for {args.sha}")
+            return 0
+
+        orphans = uploads_service.find_orphans(db)
+        for upload in orphans:
+            print(
+                f"upload_id={upload.id} sha={upload.sha256} purpose={upload.purpose} "
+                f"uploader_id={upload.uploader_id} "
+                f"created_at={upload.created_at.isoformat()}"
+            )
+        print(
+            f"{len(orphans)} orphan upload(s) across {len(crud.upload.all_shas(db))} "
+            "distinct sha(s)"
+        )
+        return 0
+    finally:
+        db.close()
+
+
+if __name__ == "__main__":
+    sys.exit(main())