feat(files): a lastTouchedBefore cutoff for cleaning up recent files

CHANGELOG.md

@@ -14,6 +14,8 @@
   `output-error` (#162, #441)
 - Store a step's reasoning once, in content, so long reasoning fits the document
   size limit (#418, #423)
+- `getFilesToDelete` and `deleteFiles` take a `lastTouchedBefore` cutoff to
+  clean up recent unreferenced files (#361, #437)
 
 ## 0.7.9
 

docs/files.mdx

@@ -133,7 +133,13 @@ Saving to the files has 3 components:
 2. Saving a reference (the storageId) to the file in the component. This will
    automatically keep track of how many messages are referencing the file, so
    you can vacuum files that are no longer used (see
-   [files/vacuum.ts](../example/convex/files/vacuum.ts)).
+   [files/vacuum.ts](../example/convex/files/vacuum.ts)). Files are kept for 24
+   hours after they were last used. To clean up sooner, e.g. in a development
+   reset, pass the same past `lastTouchedBefore` timestamp to `getFilesToDelete`
+   and `deleteFiles`, without an age filter of your own like the one in
+   `vacuum.ts`. Files used after it are kept, but unreferenced files from before
+   it are deleted, including uploads and generated files not yet saved in a
+   message, so stop those first.
 3. Inserting a URL in place of the data in the message sent to the LLM, along
    with the media type and other metadata provided. It will be inferred if not
    provided in

src/component/files.test.ts

@@ -204,4 +204,79 @@ describe("files", () => {
       }),
     ).resolves.toMatchObject({ page: [] });
   });
+
+  test("a cutoff lists recent files, and keeps those used at or after it", async () => {
+    vi.useFakeTimers();
+    vi.setSystemTime(new Date("2026-01-01T00:00:00.000Z"));
+    const t = convexTest(schema, modules);
+    const error = vi.spyOn(console, "error").mockImplementation(() => {});
+    const add = (name: string) =>
+      t.mutation(api.files.addFile, {
+        storageId: `storage-${name}`,
+        hash: `hash-${name}`,
+        filename: `${name}.txt`,
+        mediaType: "text/plain",
+      });
+    const { fileId: orphan } = await add("orphan");
+    const { fileId: referenced } = await add("referenced");
+    const { fileId: reused } = await add("reused");
+
+    vi.advanceTimersByTime(1000);
+    const lastTouchedBefore = Date.now();
+    await add("fresh");
+    const { page } = await t.query(api.files.getFilesToDelete, {
+      paginationOpts: { numItems: 10, cursor: null },
+      lastTouchedBefore,
+    });
+    expect(page.map((f) => f._id).sort()).toEqual(
+      [orphan, referenced, reused].sort(),
+    );
+
+    await t.mutation(api.files.copyFile, { fileId: referenced });
+    // Reuse doesn't add a reference: the message saving it does, later.
+    await t.mutation(api.files.useExistingFile, {
+      hash: "hash-reused",
+      filename: "reused.txt",
+      mediaType: "text/plain",
+    });
+    await expect(
+      t.mutation(api.files.deleteFiles, {
+        fileIds: page.map((f) => f._id),
+        lastTouchedBefore,
+      }),
+    ).resolves.toEqual([orphan]);
+    await expect(
+      t.query(api.files.get, { fileId: reused }),
+    ).resolves.not.toBeNull();
+    await expect(
+      t.query(api.files.get, { fileId: referenced }),
+    ).resolves.toMatchObject({ refcount: 1 });
+    error.mockRestore();
+  });
+
+  test.each([NaN, Date.now() + 60_000])(
+    "rejects a %s cutoff without deleting",
+    async (lastTouchedBefore) => {
+      const t = convexTest(schema, modules);
+      const { fileId } = await t.mutation(api.files.addFile, {
+        storageId: "storage-bad-cutoff",
+        hash: "hash-bad-cutoff",
+        filename: "bad.txt",
+        mediaType: "text/plain",
+      });
+      await expect(
+        t.query(api.files.getFilesToDelete, {
+          paginationOpts: { numItems: 10, cursor: null },
+          lastTouchedBefore,
+        }),
+      ).rejects.toThrow("lastTouchedBefore");
+      await expect(
+        t.mutation(api.files.deleteFiles, {
+          fileIds: [fileId],
+          lastTouchedBefore,
+        }),
+      ).rejects.toThrow("lastTouchedBefore");
+      await expect(t.query(api.files.get, { fileId })).resolves.not.toBeNull();
+    },
+  );
 });

src/component/files.ts

@@ -9,6 +9,21 @@ import type { Infer } from "convex/values";
 
 const FILE_CLEANUP_GRACE_MS = 24 * 60 * 60 * 1000;
 
+// Files touched at the cutoff are kept, so one reused in the same millisecond
+// the caller took its cutoff survives.
+function cleanupCutoff(lastTouchedBefore: number | undefined) {
+  if (lastTouchedBefore === undefined) {
+    return Date.now() - FILE_CLEANUP_GRACE_MS;
+  }
+  // A future cutoff would let deleteFiles delete a file reused after listing.
+  if (!(lastTouchedBefore <= Date.now())) {
+    throw new Error(
+      `lastTouchedBefore must be a timestamp no later than now, got ${lastTouchedBefore}`,
+    );
+  }
+  return lastTouchedBefore;
+}
+
 const addFileArgs = v.object({
   storageId: v.string(),
   hash: v.string(),
@@ -188,18 +203,22 @@ export async function copyFileHandler(
  * Files remain protected for 24 hours after registration or their last
  * ownership change, so a file cannot be removed between registration and the
  * transaction that saves its message or stream reference.
+ * `lastTouchedBefore` (ms since epoch, not in the future) moves that cutoff. A later one also
+ * lists uploads and generated files not yet in a saved message, so stop those
+ * first, e.g. in a dev reset.
  */
 export const getFilesToDelete = query({
   args: {
     paginationOpts: paginationOptsValidator,
+    lastTouchedBefore: v.optional(v.number()),
   },
   handler: async (ctx, args) => {
     const files = await ctx.db
       .query("files")
       .withIndex("refcount_lastTouchedAt", (q) =>
         q
           .eq("refcount", 0)
-          .lte("lastTouchedAt", Date.now() - FILE_CLEANUP_GRACE_MS),
+          .lt("lastTouchedAt", cleanupCutoff(args.lastTouchedBefore)),
       )
       .paginate(args.paginationOpts);
     return files;
@@ -211,9 +230,12 @@ export const deleteFiles = mutation({
   args: {
     fileIds: v.array(v.id("files")),
     force: v.optional(v.boolean()),
+    /** Pass the same cutoff as `getFilesToDelete` to keep files used since. */
+    lastTouchedBefore: v.optional(v.number()),
   },
   returns: v.array(v.id("files")),
   handler: async (ctx, args) => {
+    const cutoff = cleanupCutoff(args.lastTouchedBefore);
     const deletedFileIds = await Promise.all(
       args.fileIds.map(async (fileId) => {
         const file = await ctx.db.get("files", fileId);
@@ -229,11 +251,10 @@ export const deleteFiles = mutation({
             return null;
           }
         }
-        if (
-          !args.force &&
-          file.lastTouchedAt > Date.now() - FILE_CLEANUP_GRACE_MS
-        ) {
-          console.error(`File ${fileId} is still within the cleanup grace period, skipping...`);
+        if (!args.force && file.lastTouchedAt >= cutoff) {
+          console.error(
+            `File ${fileId} was touched after the cleanup cutoff, skipping...`,
+          );
           return null;
         }
         await ctx.db.delete("files", fileId);