Merge pull request #1 from mojisdev/upload-files

feat: add stow route for file uploads with tar.gz parsing

Author: luxass

package.json

@@ -16,7 +16,8 @@
   "dependencies": {
     "@hono/arktype-validator": "^2.0.0",
     "arktype": "^2.1.9",
-    "hono": "^4.7.4"
+    "hono": "^4.7.4",
+    "nanotar": "^0.2.0"
   },
   "devDependencies": {
     "@cloudflare/vitest-pool-workers": "^0.8.0",

pnpm-lock.yaml

@@ -2,7 +2,6 @@ import type { HonoEnv } from "./types";
 import { arktypeValidator } from "@hono/arktype-validator";
 import { type } from "arktype";
 import { Hono } from "hono";
-import { HTTPException } from "hono/http-exception";
 import { authMiddleware } from "./middlewares/auth";
 
 export const HASHES_ROUTER = new Hono<HonoEnv>().basePath("/hashes");
@@ -11,9 +10,22 @@ function getKVPrefix(env: string): string {
   return env === "production" ? "prod" : env === "preview" ? "preview" : "dev";
 }
 
-HASHES_ROUTER.get("/", async (c) => {
+HASHES_ROUTER.get("/delete", authMiddleware, async (c) => {
   const result = await c.env.MOJIS_HASHES.list({
-    prefix: getKVPrefix(c.env.ENVIRONMENT),
+    prefix: `${getKVPrefix(c.env.ENVIRONMENT)}`,
+  });
+
+  const keys = result.keys.map((h) => h.name);
+  await Promise.all(keys.map((key) => c.env.MOJIS_HASHES.delete(key)));
+
+  return c.json({
+    done: true,
+  });
+});
+
+HASHES_ROUTER.get("/:version", async (c) => {
+  const result = await c.env.MOJIS_HASHES.list({
+    prefix: `${getKVPrefix(c.env.ENVIRONMENT)}:${c.req.param("version")}`,
   });
 
   const keys = result.keys.map((h) => h.name);
@@ -35,39 +47,6 @@ HASHES_ROUTER.get("/", async (c) => {
   return c.json(hashes.filter((h) => h != null));
 });
 
-HASHES_ROUTER.get("/:version", async (c) => {
-  const version = c.req.param("version");
-
-  const hash = await c.env.MOJIS_HASHES.get(`${getKVPrefix(c.env.ENVIRONMENT)}:${version}`);
-
-  if (hash == null) {
-    throw new HTTPException(404, {
-      message: "Hash not found",
-    });
-  }
-
-  return c.json({
-    hash,
-  });
-});
-
-HASHES_ROUTER.get("/:version/:item", async (c) => {
-  const version = c.req.param("version");
-  const item = c.req.param("item");
-
-  const hash = await c.env.MOJIS_HASHES.get(`${getKVPrefix(c.env.ENVIRONMENT)}:${version}:${item}`);
-
-  if (hash == null) {
-    throw new HTTPException(404, {
-      message: "Hash not found",
-    });
-  }
-
-  return c.json({
-    hash,
-  });
-});
-
 HASHES_ROUTER.post(
   "/",
   authMiddleware,

src/hashes.ts

@@ -3,6 +3,7 @@ import { Hono } from "hono";
 import { cache } from "hono/cache";
 import { HTTPException } from "hono/http-exception";
 import { HASHES_ROUTER } from "./hashes";
+import { STOW_ROUTER } from "./stow";
 
 const app = new Hono<HonoEnv>();
 
@@ -15,6 +16,7 @@ app.get(
 );
 
 app.route("/", HASHES_ROUTER);
+app.route("/", STOW_ROUTER);
 
 app.onError(async (err, c) => {
   console.error(err);
@@ -46,4 +48,10 @@ app.notFound(async (c) => {
   } satisfies ApiError, 404);
 });
 
-export default app;
+export default {
+  fetch: app.fetch,
+  scheduled: async (_ctrl, _env, _ctx) => {
+    // eslint-disable-next-line no-console
+    console.log("Scheduled task");
+  },
+} satisfies ExportedHandler<HonoEnv>;

src/index.ts

@@ -0,0 +1,60 @@
+import { Hono } from "hono";
+import { HTTPException } from "hono/http-exception";
+import { parseTarGzip } from "nanotar";
+import { authMiddleware } from "./middlewares/auth";
+
+export const STOW_ROUTER = new Hono().basePath("/stow");
+
+STOW_ROUTER.post(
+  "/:version",
+  authMiddleware,
+  async (c) => {
+    const { version } = c.req.param();
+    const body = await c.req.parseBody();
+
+    const file = Array.isArray(body.file) ? body.file[0] : body.file;
+
+    if (file == null) {
+      throw new HTTPException(400, {
+        message: "No file uploaded",
+      });
+    }
+
+    if (typeof file === "string") {
+      throw new HTTPException(400, {
+        message: "invalid file uploaded",
+      });
+    }
+
+    const tar = await parseTarGzip(await file.arrayBuffer(), {
+      filter(file) {
+        // if file is pax header, return false
+        if (Number.isNaN(file.type)) return false;
+        if (file.name.includes("PaxHeader")) return false;
+        // some files is prefixed with ._, ignore them
+        if (file.name.includes("._")) return false;
+        return true;
+      },
+    });
+
+    const promises = [];
+
+    for (const entry of tar) {
+      if (entry.type !== "file") continue;
+      const normalizedEntryName = entry.name.replace("./", "");
+      // eslint-disable-next-line no-console
+      console.log(
+        `Uploading ${entry.name} (${entry.size} bytes) to ${version}/${normalizedEntryName}`,
+      );
+      promises.push(c.env.EMOJI_DATA.put(`${version}/${normalizedEntryName}`, entry.text));
+    }
+
+    c.executionCtx.waitUntil(
+      Promise.all(promises),
+    );
+
+    return c.json({
+      message: "Files uploaded",
+    });
+  },
+);

src/stow.ts

@@ -26,6 +26,11 @@
   "vars": {
     "MOJIS_TOKEN": ""
   },
+  "triggers": {
+    "crons": [
+      "0 */12 * * *"
+    ]
+  },
   "placement": { "mode": "smart" },
   "env": {
     "preview": {