diff --git a/Makefile b/Makefile index c0b9b7e..b7224fb 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,10 @@ refresh-karmas: refresh-karmas-apply: python scripts/refresh_karmas.py --apply +# Report uploaded images that no body references any more. Nothing is deleted. +upload-report: + python scripts/upload_report.py + dev-run: uvicorn --host $(LOCAL_DEV_HOST) --port $(LOCAL_DEV_PORT) chafan_core.app.main:app --reload diff --git a/chafan_core/app/services/uploads.py b/chafan_core/app/services/uploads.py index 0c09f93..66fd41c 100644 --- a/chafan_core/app/services/uploads.py +++ b/chafan_core/app/services/uploads.py @@ -4,12 +4,15 @@ import hashlib import logging -from typing import Any +from typing import Any, List + +from sqlalchemy.orm import Session from chafan_core.app import ( coins, crud, image_sanitize, + models, object_storage, rules, schemas, @@ -84,3 +87,50 @@ def upload_image(ctx, *, file, file_size: int, purpose: str) -> schemas.Uploaded storage_bucket=settings.UPLOADS_S3_BUCKET, ) return schemas.UploadedImage(url=object_storage.public_url(sha, content_type)) + + +# --------------------------------------------------------------------------- +# Usage accounting: computed, not stored. +# +# delete_forever overwrites body, body_draft and every archive row with +# "[DELETED]", so the references leave the database and a scan of bodies gives +# the correct answer with no join table to maintain. Nothing here deletes. +# --------------------------------------------------------------------------- + +_BODY_COLUMNS = [ + (models.Article, "body", "article"), + (models.Article, "body_draft", "article"), + (models.ArticleArchive, "body", "article_archive"), + (models.Answer, "body", "answer"), + (models.Answer, "body_draft", "answer"), + (models.Archive, "body", "answer_archive"), + (models.Comment, "body", "comment"), + (models.Submission, "description", "submission"), + (models.SubmissionArchive, "description", "submission_archive"), +] + + +def find_usages(db: Session, *, sha: str) -> List[str]: + """Locations (``table:id``) whose body text embeds ``sha``. + + The public URL embeds the sha, so a ``LIKE`` hit is exact. + """ + usages: List[str] = [] + for model, column, label in _BODY_COLUMNS: + col = getattr(model, column) + for (id_,) in db.query(model.id).filter(col.like(f"%{sha}%")).all(): + usages.append(f"{label}:{id_}") + return usages + + +def find_orphans(db: Session) -> List[Any]: + """Uploads whose sha appears in no body or archive text. Report only. + + Returns nothing until the Upload table lands (``crud.upload`` is stubbed); + typed ``Any`` so the real model can arrive later without touching this. + """ + orphans: List[Any] = [] + for sha in crud.upload.all_shas(db): + if not find_usages(db, sha=sha): + orphans.extend(crud.upload.get_multi_by_sha(db, sha=sha)) + return orphans diff --git a/scripts/upload_report.py b/scripts/upload_report.py new file mode 100644 index 0000000..8cfee7d --- /dev/null +++ b/scripts/upload_report.py @@ -0,0 +1,58 @@ +"""Report which uploaded images are orphans -- referenced by no body any more. + + make upload-report # list orphans + make upload-report-sha --sha= # list usages of one sha (not wired to make) + +Storage in the upload bucket is treated as losable; the upload table is the +recovery manifest. Orphans here are objects that no live or archived body +references any more, and are the candidates for a future garbage-collection +step. Nothing is deleted from the bucket or the table by this script. +""" + +import os.path +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) + +import argparse + +from chafan_core.app import crud +from chafan_core.app.services import uploads as uploads_service +from chafan_core.db.session import SessionLocal + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--sha", + help="print usages of a single sha instead of scanning for orphans", + ) + args = parser.parse_args() + + db = SessionLocal() + try: + if args.sha: + usages = uploads_service.find_usages(db, sha=args.sha) + for usage in usages: + print(usage) + print(f"{len(usages)} usage(s) for {args.sha}") + return 0 + + orphans = uploads_service.find_orphans(db) + for upload in orphans: + print( + f"upload_id={upload.id} sha={upload.sha256} purpose={upload.purpose} " + f"uploader_id={upload.uploader_id} " + f"created_at={upload.created_at.isoformat()}" + ) + print( + f"{len(orphans)} orphan upload(s) across {len(crud.upload.all_shas(db))} " + "distinct sha(s)" + ) + return 0 + finally: + db.close() + + +if __name__ == "__main__": + sys.exit(main())