RD: legacy migration experiment

.env.docker-compose.dev

@@ -17,6 +17,7 @@ POSTGRES_DB=postgres
 
 VALKEY_HOST='cache'
 
+SITE_BUILDER_ENDPOINT=http://localhost:4000
 # annoying duplication because jobs uses this version
 PGHOST=db
 PGPORT=5432
@@ -41,4 +42,4 @@ INBUCKET_URL=http://localhost:54324
 MAILGUN_SMTP_USERNAME=omitted
 OTEL_SERVICE_NAME=core.core
 PUBPUB_URL=http://localhost:3000
-API_KEY=xxx
+API_KEY=xxx

.github/workflows/e2e.yml

@@ -92,6 +92,7 @@ jobs:
                   echo "core_label=$ECR_REGISTRY/${ECR_REPOSITORY_NAME_OVERRIDE:-$ECR_REPOSITORY_PREFIX-core}:$IMAGE_TAG" >> $GITHUB_OUTPUT
                   echo "jobs_label=$ECR_REGISTRY/${ECR_REPOSITORY_NAME_OVERRIDE:-$ECR_REPOSITORY_PREFIX-jobs}:$IMAGE_TAG" >> $GITHUB_OUTPUT
                   echo "base_label=$ECR_REGISTRY/$ECR_REPOSITORY_PREFIX:$IMAGE_TAG" >> $GITHUB_OUTPUT
+                  echo "site_builder_label=$ECR_REGISTRY/${ECR_REPOSITORY_NAME_OVERRIDE:-$ECR_REPOSITORY_PREFIX-site-builder}:$IMAGE_TAG" >> $GITHUB_OUTPUT
 
             - name: Install dependencies
               run: pnpm install --frozen-lockfile --prefer-offline
@@ -115,6 +116,7 @@ jobs:
               run: pnpm integration:setup
               env:
                   INTEGRATION_TESTS_IMAGE: ${{steps.label.outputs.core_label}}
+                  SITE_BUILDER_IMAGE: ${{steps.label.outputs.site_builder_label}}
                   JOBS_IMAGE: ${{steps.label.outputs.jobs_label}}
 
             - name: Log out Container ID for health check
@@ -136,6 +138,7 @@ jobs:
               run: docker compose -f docker-compose.test.yml --profile integration logs -t
               env:
                   INTEGRATION_TESTS_IMAGE: ${{steps.label.outputs.core_label}}
+                  SITE_BUILDER_IMAGE: ${{steps.label.outputs.site_builder_label}}
                   JOBS_IMAGE: ${{steps.label.outputs.jobs_label}}
 
             - name: Upload core playwright snapshots artifact

.github/workflows/ecrbuild-all.yml

@@ -23,6 +23,9 @@ on:
             jobs-image:
                 description: "Jobs image SHA"
                 value: ${{ jobs.build-jobs.outputs.image-sha }}
+            site-builder-image:
+                description: "Site builder image SHA"
+                value: ${{ jobs.build-site-builder.outputs.image-sha }}
 
 jobs:
     emit-sha-tag:
@@ -71,3 +74,14 @@ jobs:
         secrets:
             AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
             AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+
+    build-site-builder:
+        uses: ./.github/workflows/ecrbuild-template.yml
+        with:
+            package: site-builder
+            target: jobs
+            # publish_to_ghcr: ${{ inputs.publish_to_ghcr }}
+            ghcr_image_name: platform-site-builder
+        secrets:
+            AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+            AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

.github/workflows/on_main.yml

@@ -63,6 +63,7 @@ jobs:
             PLATFORM_IMAGE: ${{ needs.build-all.outputs.core-image }}
             JOBS_IMAGE: ${{ needs.build-all.outputs.jobs-image }}
             MIGRATIONS_IMAGE: ${{ needs.build-all.outputs.base-image }}
+            SITE_BUILDER_IMAGE: ${{ needs.build-all.outputs.site-builder-image }}
             AWS_REGION: "us-east-1"
             ALWAYS_ON: "main"
             COMPOSE_FILES: docker-compose.preview.sandbox.yml

.github/workflows/on_pr.yml

@@ -33,7 +33,7 @@ jobs:
         uses: ./.github/workflows/ci.yml
 
     build-all:
-        if: github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'synchronize' || (github.event.action == 'labeled' && github.event.label.name == 'preview')
+        if: (github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'synchronize' || (github.event.action == 'labeled' && github.event.label.name == 'preview'))
         uses: ./.github/workflows/ecrbuild-all.yml
         secrets:
             AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -61,9 +61,14 @@ jobs:
             pull-requests: write
             statuses: write
         with:
+            # PLATFORM_IMAGE: 246372085946.dkr.ecr.us-east-1.amazonaws.com/pubpub-v7-core:2b9a81a279c4e405bbedcdbb697c897ded52fbc0
+            # JOBS_IMAGE: 246372085946.dkr.ecr.us-east-1.amazonaws.com/pubpub-v7-jobs:c786662f4899de16a621e366a485eca5adda4d6a
+            # MIGRATIONS_IMAGE: 246372085946.dkr.ecr.us-east-1.amazonaws.com/pubpub-v7:c786662f4899de16a621e366a485eca5adda4d6a
+            # SITE_BUILDER_IMAGE: 246372085946.dkr.ecr.us-east-1.amazonaws.com/pubpub-v7-site-builder:c786662f4899de16a621e366a485eca5adda4d6a
             PLATFORM_IMAGE: ${{ needs.build-all.outputs.core-image }}
             JOBS_IMAGE: ${{ needs.build-all.outputs.jobs-image }}
             MIGRATIONS_IMAGE: ${{ needs.build-all.outputs.base-image }}
+            SITE_BUILDER_IMAGE: ${{ needs.build-all.outputs.site-builder-image }}
             AWS_REGION: "us-east-1"
             COMPOSE_FILES: docker-compose.preview.pr.yml
         secrets:
@@ -85,6 +90,7 @@ jobs:
             PLATFORM_IMAGE: "x" # not used
             JOBS_IMAGE: "x" # not used
             MIGRATIONS_IMAGE: "x" # not used
+            SITE_BUILDER_IMAGE: "x" # not used
             AWS_REGION: "us-east-1"
         secrets:
             AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

.github/workflows/pull-preview.yml

@@ -10,6 +10,9 @@ on:
             MIGRATIONS_IMAGE:
                 required: true
                 type: string
+            SITE_BUILDER_IMAGE:
+                required: true
+                type: string
             AWS_REGION:
                 required: true
                 type: string
@@ -52,10 +55,12 @@ jobs:
                   PLATFORM_IMAGE: ${{ inputs.PLATFORM_IMAGE }}
                   JOBS_IMAGE: ${{ inputs.JOBS_IMAGE }}
                   MIGRATIONS_IMAGE: ${{ inputs.MIGRATIONS_IMAGE }}
+                  SITE_BUILDER_IMAGE: ${{ inputs.SITE_BUILDER_IMAGE }}
               run: |
                   sed -i "s|image: PLATFORM_IMAGE|image: $PLATFORM_IMAGE|" docker-compose.preview.yml
                   sed -i "s|image: JOBS_IMAGE|image: $JOBS_IMAGE|" docker-compose.preview.yml
                   sed -i "s|image: MIGRATIONS_IMAGE|image: $MIGRATIONS_IMAGE|" docker-compose.preview.yml
+                  sed -i "s|image: SITE_BUILDER_IMAGE|image: $SITE_BUILDER_IMAGE|" docker-compose.preview.yml
                   sed -i "s|DATACITE_REPOSITORY_ID: DATACITE_REPOSITORY_ID|DATACITE_REPOSITORY_ID: ${{ secrets.PREVIEW_DATACITE_REPOSITORY_ID }}|" docker-compose.preview.yml
                   sed -i "s|DATACITE_PASSWORD: DATACITE_PASSWORD|DATACITE_PASSWORD: ${{ secrets.PREVIEW_DATACITE_PASSWORD }}|" docker-compose.preview.yml
                   sed -i "s|email someone@example.com|email dev@pubpub.org|" self-host/caddy/Caddyfile
@@ -77,7 +82,7 @@ jobs:
                   default_port: 443
                   instance_type: small
                   always_on: ${{ inputs.ALWAYS_ON }}
-                  ports: 80,443,9001
+                  ports: 80,443
                   registries: docker://AWS:${{steps.ecrtoken.outputs.value}}@246372085946.dkr.ecr.us-east-1.amazonaws.com
                   github_token: ${{ secrets.GH_PAT_PR_PREVIEW_CLEANUP }}
                   pre_script: "./.github/workflows/pull-preview-script.sh"

.vscode/settings.json

@@ -9,5 +9,9 @@
 		"NODE_OPTIONS": "--import=\"#register-loader\"",
 		"SKIP_VALIDATION": "true",
 		"DATABASE_URL": "postgresql://postgres:postgres@localhost:54322/postgres"
+	},
+	"prettier.documentSelectors": ["**/*.astro"],
+	"[astro]": {
+		"editor.defaultFormatter": "esbenp.prettier-vscode"
 	}
 }

Dockerfile

@@ -90,10 +90,20 @@ RUN pnpm --filter $PACKAGE --prod deploy /tmp/app
 
 FROM base AS jobs
 
+ARG PACKAGE
+
 WORKDIR /usr/src/app
 
 COPY --from=prepare-jobs --chown=node:node /tmp/app .
 
+# If the package is site-builder, create necessary directories and set permissions
+RUN if [ "$PACKAGE" = "site-builder" ]; then \
+  mkdir -p /usr/src/app/builds /usr/src/app/.astro /usr/src/app/dist && \
+  chown -R node:node /usr/src/app/builds /usr/src/app/.astro /usr/src/app/dist; \
+  fi
+
+
+
 USER node
 
 CMD ["pnpm", "start"]

config/prettier/astro.js

@@ -0,0 +1,23 @@
+// @ts-check
+
+import baseConfig from "./index.js";
+
+/** @typedef {import("prettier").Config} PrettierConfig */
+/** @typedef {import("prettier-plugin-tailwindcss").PluginOptions} TailwindConfig */
+/** @typedef {import("@ianvs/prettier-plugin-sort-imports").PluginConfig} SortImportsConfig */
+
+/** @type {PrettierConfig & SortImportsConfig & TailwindConfig} */
+const config = {
+	...baseConfig,
+	plugins: [...(baseConfig.plugins ?? []), "prettier-plugin-astro"],
+	overrides: [
+		{
+			files: "*.astro",
+			options: {
+				parser: "astro",
+			},
+		},
+	],
+};
+
+export default config;

config/prettier/index.js

@@ -4,7 +4,7 @@ import { fileURLToPath } from "url";
 /** @typedef {import("prettier-plugin-tailwindcss").PluginOptions} TailwindConfig */
 /** @typedef {import("@ianvs/prettier-plugin-sort-imports").PluginConfig} SortImportsConfig */
 
-/** @type {PrettierConfig | SortImportsConfig | TailwindConfig} */
+/** @type {PrettierConfig & SortImportsConfig & TailwindConfig} */
 const config = {
 	arrowParens: "always",
 	printWidth: 100,

config/prettier/package.json

@@ -4,7 +4,8 @@
 	"private": true,
 	"version": "0.1.0",
 	"exports": {
-		".": "./index.js"
+		".": "./index.js",
+		"./astro": "./astro.js"
 	},
 	"scripts": {
 		"clean": "rm -rf .turbo node_modules",
@@ -15,6 +16,7 @@
 	"dependencies": {
 		"@ianvs/prettier-plugin-sort-imports": "^4.6.2",
 		"prettier": "catalog:",
+		"prettier-plugin-astro": "^0.14.1",
 		"prettier-plugin-jsdoc": "^1.3.3",
 		"prettier-plugin-tailwindcss": "^0.6.14"
 	},

core/.env.development

@@ -1,28 +1,31 @@
 MAILGUN_SMTP_HOST=localhost
 MAILGUN_SMTP_PORT=54325
-API_KEY="super_secret_key"
-DATABASE_URL="postgresql://postgres:postgres@localhost:54322/postgres"
-PUBPUB_URL="http://localhost:3000"
-ASSETS_BUCKET_NAME="assets.v7.pubpub.org"
-ASSETS_REGION="us-east-1"
+API_KEY=super_secret_key
+DATABASE_URL=postgresql://postgres:postgres@localhost:54322/postgres
+PUBPUB_URL=http://localhost:3000
+ASSETS_BUCKET_NAME=assets.v7.pubpub.org
+ASSETS_REGION=us-east-1
 
 # mninio defaults
-ASSETS_UPLOAD_KEY="pubpubuser"
-ASSETS_UPLOAD_SECRET_KEY="pubpubpass"
-ASSETS_STORAGE_ENDPOINT="http://localhost:9000"
+ASSETS_UPLOAD_KEY=pubpubuser
+ASSETS_UPLOAD_SECRET_KEY=pubpubpass
+ASSETS_STORAGE_ENDPOINT=http://localhost:9000
 
-MAILGUN_SMTP_PASSWORD="xxx"
-MAILGUN_SMTP_USERNAME="xxx"
+MAILGUN_SMTP_PASSWORD=xxx
+MAILGUN_SMTP_USERNAME=xxx
 
-OTEL_SERVICE_NAME="pubpub-v7-dev" # should be shared across components but not environments
-HONEYCOMB_API_KEY="xxx"
-ARTILLERY_CLOUD_API_KEY="xxx"
+OTEL_SERVICE_NAME=pubpub-v7-dev
+HONEYCOMB_API_KEY=xxx
+ARTILLERY_CLOUD_API_KEY=xxx
 
-KYSELY_DEBUG="true"
+KYSELY_DEBUG=true
 
-DATACITE_REPOSITORY_ID=""
-DATACITE_PASSWORD=""
-DATACITE_API_URL="https://api.test.datacite.org"
+DATACITE_REPOSITORY_ID=
+DATACITE_PASSWORD=
+DATACITE_API_URL=https://api.test.datacite.org
 
-GCLOUD_KEY_FILE='xxx'
-VALKEY_HOST='localhost'
+GCLOUD_KEY_FILE=xxx
+
+SITE_BUILDER_ENDPOINT=http://localhost:4000
+
+VALKEY_HOST='localhost'

core/actions/_lib/zodTypes.ts

@@ -53,6 +53,16 @@ class ActionInstance extends z.ZodObject<typeof actionInstanceShape, "strip", z.
 		});
 }
 
+class FieldName extends z.ZodString {
+	static create = () =>
+		new FieldName({
+			typeName: "FieldName" as z.ZodFirstPartyTypeKind.ZodString,
+			checks: [],
+			coerce: false,
+		});
+}
+
 export const markdown = Markdown.create;
 export const stringWithTokens = StringWithTokens.create;
 export const actionInstance = ActionInstance.create;
+export const fieldName = FieldName.create;

core/actions/api/index.ts

@@ -12,6 +12,7 @@ import {
 	pubInStageForDuration,
 	pubLeftStage,
 } from "../_lib/rules";
+import * as buildJournalSite from "../buildJournalSite/action";
 import * as datacite from "../datacite/action";
 import * as email from "../email/action";
 import * as googleDriveImport from "../googleDriveImport/action";
@@ -31,6 +32,7 @@ export const actions = {
 	[move.action.name]: move.action,
 	[googleDriveImport.action.name]: googleDriveImport.action,
 	[datacite.action.name]: datacite.action,
+	[buildJournalSite.action.name]: buildJournalSite.action,
 } as const;
 
 export const getActionByName = <N extends keyof typeof actions>(name: N) => {

core/actions/buildJournalSite/action.ts

@@ -0,0 +1,28 @@
+import { Book } from "lucide-react";
+import * as z from "zod";
+
+import { Action } from "db/public";
+import { FileText } from "ui/icon";
+
+import { fieldName } from "../_lib/zodTypes";
+import { defineAction } from "../types";
+
+const schema = z.object({
+	token: z.string().describe("The token for the site builder"),
+	articles: fieldName(),
+	collections: fieldName(),
+	mainColor: fieldName(),
+	accentColor: fieldName(),
+});
+
+export const action = defineAction({
+	name: Action.buildJournalSite,
+	config: {
+		schema,
+	},
+	description: "Build a journal site and receive a zip file",
+	params: {
+		schema: schema.partial(),
+	},
+	icon: Book,
+});