feat(router): split q-data.json into separate files with proper caching

.github/workflows/ci.yml

@@ -285,7 +285,7 @@ jobs:
 
       - run: pnpm install
       - if: matrix.settings.wasm
-        run: pnpm install wasm-pack
+        run: pnpm install -w wasm-pack
 
       - name: Lint check
         if: matrix.settings.wasm

PR-TODO-DO-NOT-MERGE.md

@@ -0,0 +1,11 @@
+# TODO
+
+- fix failing e2e tests
+- dev mode support for q-loader-data and other things that read the manifest for qwik-router-config
+- remove every q-data.json load
+- implement eTag option for loaders
+- implement expires option for loaders
+  - changes the cache-control max-age
+  - use on client side to determine if the data is stale before fetching
+  - during SSR, store page create time and loader expires deltas
+- cache q-loader-data responses in the browser

packages/docs/src/routes/api/qwik-router/api.json

@@ -446,7 +446,7 @@
         }
       ],
       "kind": "TypeAlias",
-      "content": "```typescript\nexport type LoaderSignal<TYPE> = TYPE extends () => ValueOrPromise<infer VALIDATOR> ? ReadonlySignal<ValueOrPromise<VALIDATOR>> : ReadonlySignal<TYPE>;\n```",
+      "content": "```typescript\nexport type LoaderSignal<TYPE> = TYPE extends () => ValueOrPromise<infer VALIDATOR> ? AsyncComputedReadonlySignal<ValueOrPromise<VALIDATOR>> : AsyncComputedReadonlySignal<TYPE>;\n```",
       "editUrl": "https://github.com/QwikDev/qwik/tree/main/packages/qwik-router/src/runtime/src/types.ts",
       "mdFile": "router.loadersignal.md"
     },
@@ -656,7 +656,7 @@
         }
       ],
       "kind": "Interface",
-      "content": "```typescript\nexport interface QwikRouterConfig \n```\n\n\n<table><thead><tr><th>\n\nProperty\n\n\n</th><th>\n\nModifiers\n\n\n</th><th>\n\nType\n\n\n</th><th>\n\nDescription\n\n\n</th></tr></thead>\n<tbody><tr><td>\n\n[basePathname?](./router.qwikrouterconfig.basepathname.md)\n\n\n</td><td>\n\n`readonly`\n\n\n</td><td>\n\nstring\n\n\n</td><td>\n\n_(Optional)_\n\n\n</td></tr>\n<tr><td>\n\n[cacheModules?](./router.qwikrouterconfig.cachemodules.md)\n\n\n</td><td>\n\n`readonly`\n\n\n</td><td>\n\nboolean\n\n\n</td><td>\n\n_(Optional)_\n\n\n</td></tr>\n<tr><td>\n\n[menus?](./router.qwikrouterconfig.menus.md)\n\n\n</td><td>\n\n`readonly`\n\n\n</td><td>\n\n[MenuData](#menudata)<!-- -->\\[\\]\n\n\n</td><td>\n\n_(Optional)_\n\n\n</td></tr>\n<tr><td>\n\n[routes](./router.qwikrouterconfig.routes.md)\n\n\n</td><td>\n\n`readonly`\n\n\n</td><td>\n\n[RouteData](#routedata)<!-- -->\\[\\]\n\n\n</td><td>\n\n\n</td></tr>\n<tr><td>\n\n[serverPlugins?](./router.qwikrouterconfig.serverplugins.md)\n\n\n</td><td>\n\n`readonly`\n\n\n</td><td>\n\nRouteModule\\[\\]\n\n\n</td><td>\n\n_(Optional)_\n\n\n</td></tr>\n<tr><td>\n\n[trailingSlash?](./router.qwikrouterconfig.trailingslash.md)\n\n\n</td><td>\n\n`readonly`\n\n\n</td><td>\n\nboolean\n\n\n</td><td>\n\n_(Optional)_\n\n\n</td></tr>\n</tbody></table>",
+      "content": "```typescript\nexport interface QwikRouterConfig \n```\n\n\n<table><thead><tr><th>\n\nProperty\n\n\n</th><th>\n\nModifiers\n\n\n</th><th>\n\nType\n\n\n</th><th>\n\nDescription\n\n\n</th></tr></thead>\n<tbody><tr><td>\n\n[basePathname?](./router.qwikrouterconfig.basepathname.md)\n\n\n</td><td>\n\n`readonly`\n\n\n</td><td>\n\nstring\n\n\n</td><td>\n\n_(Optional)_\n\n\n</td></tr>\n<tr><td>\n\n[cacheModules?](./router.qwikrouterconfig.cachemodules.md)\n\n\n</td><td>\n\n`readonly`\n\n\n</td><td>\n\nboolean\n\n\n</td><td>\n\n_(Optional)_\n\n\n</td></tr>\n<tr><td>\n\n[loaderIdToRoute?](./router.qwikrouterconfig.loaderidtoroute.md)\n\n\n</td><td>\n\n`readonly`\n\n\n</td><td>\n\nRecord&lt;string, string&gt;\n\n\n</td><td>\n\n_(Optional)_\n\n\n</td></tr>\n<tr><td>\n\n[menus?](./router.qwikrouterconfig.menus.md)\n\n\n</td><td>\n\n`readonly`\n\n\n</td><td>\n\n[MenuData](#menudata)<!-- -->\\[\\]\n\n\n</td><td>\n\n_(Optional)_\n\n\n</td></tr>\n<tr><td>\n\n[routes](./router.qwikrouterconfig.routes.md)\n\n\n</td><td>\n\n`readonly`\n\n\n</td><td>\n\n[RouteData](#routedata)<!-- -->\\[\\]\n\n\n</td><td>\n\n\n</td></tr>\n<tr><td>\n\n[serverPlugins?](./router.qwikrouterconfig.serverplugins.md)\n\n\n</td><td>\n\n`readonly`\n\n\n</td><td>\n\nRouteModule\\[\\]\n\n\n</td><td>\n\n_(Optional)_\n\n\n</td></tr>\n<tr><td>\n\n[trailingSlash?](./router.qwikrouterconfig.trailingslash.md)\n\n\n</td><td>\n\n`readonly`\n\n\n</td><td>\n\nboolean\n\n\n</td><td>\n\n_(Optional)_\n\n\n</td></tr>\n</tbody></table>",
       "editUrl": "https://github.com/QwikDev/qwik/tree/main/packages/qwik-router/src/runtime/src/types.ts",
       "mdFile": "router.qwikrouterconfig.md"
     },
@@ -796,7 +796,7 @@
         }
       ],
       "kind": "TypeAlias",
-      "content": "```typescript\nexport type RouteData = [routeName: string, loaders: ModuleLoader[]] | [\n    routeName: string,\n    loaders: ModuleLoader[],\n    originalPathname: string,\n    routeBundleNames: string[]\n];\n```",
+      "content": "```typescript\nexport type RouteData = [\n    routeName: string,\n    moduleLoaders: ModuleLoader[]\n] | [\n    routeName: string,\n    moduleLoaders: ModuleLoader[],\n    originalPathname: string,\n    routeBundleNames: string[]\n];\n```",
       "editUrl": "https://github.com/QwikDev/qwik/tree/main/packages/qwik-router/src/runtime/src/types.ts",
       "mdFile": "router.routedata.md"
     },

packages/docs/src/routes/api/qwik-router/index.mdx

@@ -1166,8 +1166,8 @@ export type Loader<RETURN> = {
 export type LoaderSignal<TYPE> = TYPE extends () => ValueOrPromise<
   infer VALIDATOR
 >
-  ? ReadonlySignal<ValueOrPromise<VALIDATOR>>
-  : ReadonlySignal<TYPE>;
+  ? AsyncComputedReadonlySignal<ValueOrPromise<VALIDATOR>>
+  : AsyncComputedReadonlySignal<TYPE>;
 ```
 
 [Edit this section](https://github.com/QwikDev/qwik/tree/main/packages/qwik-router/src/runtime/src/types.ts)
@@ -1510,6 +1510,23 @@ _(Optional)_
 </td></tr>
 <tr><td>
 
+[loaderIdToRoute?](./router.qwikrouterconfig.loaderidtoroute.md)
+
+</td><td>
+
+`readonly`
+
+</td><td>
+
+Record&lt;string, string&gt;
+
+</td><td>
+
+_(Optional)_
+
+</td></tr>
+<tr><td>
+
 [menus?](./router.qwikrouterconfig.menus.md)
 
 </td><td>
@@ -1854,10 +1871,10 @@ routeAction$: ActionConstructor;
 
 ```typescript
 export type RouteData =
-  | [routeName: string, loaders: ModuleLoader[]]
+  | [routeName: string, moduleLoaders: ModuleLoader[]]
   | [
       routeName: string,
-      loaders: ModuleLoader[],
+      moduleLoaders: ModuleLoader[],
       originalPathname: string,
       routeBundleNames: string[],
     ];

packages/qwik-router/global.d.ts

@@ -1,8 +1,6 @@
 /* eslint-disable no-var */
 // Globals used by qwik-router, for internal use only
 
-type RequestEventInternal =
-  import('./middleware/request-handler/request-event').RequestEventInternal;
 type AsyncStore = import('node:async_hooks').AsyncLocalStorage<RequestEventInternal>;
 type SerializationStrategy = import('@qwik.dev/core/internal').SerializationStrategy;
 

packages/qwik-router/modules.d.ts

@@ -4,12 +4,14 @@ declare module '@qwik-router-config' {
   export const trailingSlash: boolean;
   export const basePathname: string;
   export const cacheModules: boolean;
+  export const loaderIdToRoute: Record<string, string>;
   const defaultExport: {
     routes: any[];
     menus: any[];
     trailingSlash: boolean;
     basePathname: string;
     cacheModules: boolean;
+    loaderIdToRoute: Record<string, string>;
   };
   export default defaultExport;
 }

packages/qwik-router/src/adapters/shared/vite/index.ts

@@ -3,7 +3,7 @@ import type { StaticGenerateOptions, SsgRenderOptions } from 'packages/qwik-rout
 import type { QwikRouterPlugin } from '@qwik.dev/router/vite';
 import { basename, dirname, join, resolve } from 'node:path';
 import type { Plugin, UserConfig } from 'vite';
-import type { BuildRoute } from '../../../buildtime/types';
+import type { BuiltRoute } from '../../../buildtime/types';
 import { postBuild } from './post-build';
 
 /**
@@ -262,7 +262,7 @@ interface ViteAdapterPluginOptions {
     clientPublicOutDir: string;
     serverOutDir: string;
     basePathname: string;
-    routes: BuildRoute[];
+    routes: BuiltRoute[];
     assetsDir?: string;
     warn: (message: string) => void;
     error: (message: string) => void;

packages/qwik-router/src/buildtime/build-layout.unit.ts

@@ -4,7 +4,7 @@ const test = testAppSuite('Build Layout');
 
 test('total layouts', ({ ctx: { layouts } }) => {
   // $ find starters/apps/qwikrouter-test/src/routes -name layout*tsx | wc -l
-  assert.equal(layouts.length, 13, JSON.stringify(layouts, null, 2));
+  assert.equal(layouts.length, 14, JSON.stringify(layouts, null, 2));
 });
 
 test('nested named layout', ({ assertLayout }) => {

packages/qwik-router/src/buildtime/build.ts

@@ -1,12 +1,13 @@
 import { addError, addWarning } from '../utils/format';
 import { resolveSourceFiles } from './routing/resolve-source-file';
+import { routeSortCompare } from './routing/sort-routes';
 import { walkRoutes } from './routing/walk-routes-dir';
 import { walkServerPlugins } from './routing/walk-server-plugins';
-import type { BuildContext, BuildRoute, RewriteRouteOption } from './types';
+import type { RoutingContext, BuiltRoute, RewriteRouteOption } from './types';
 
-export async function build(ctx: BuildContext) {
+export async function parseRoutesDir(ctx: RoutingContext) {
   try {
-    await updateBuildContext(ctx);
+    await updateRoutingContext(ctx);
     validateBuild(ctx);
   } catch (e) {
     addError(ctx, e);
@@ -21,46 +22,40 @@ export async function build(ctx: BuildContext) {
   }
 }
 
-export async function updateBuildContext(ctx: BuildContext) {
-  if (!ctx.activeBuild) {
-    ctx.activeBuild = new Promise<void>((resolve, reject) => {
-      walkServerPlugins(ctx.opts)
-        .then((serverPlugins) => {
-          ctx.serverPlugins = serverPlugins;
-          return walkRoutes(ctx.opts.routesDir);
-        })
-        .then((sourceFiles) => {
-          const resolved = resolveSourceFiles(ctx.opts, sourceFiles);
-          rewriteRoutes(ctx, resolved);
-          ctx.layouts = resolved.layouts;
-          ctx.routes = resolved.routes;
-          ctx.entries = resolved.entries;
-          ctx.serviceWorkers = resolved.serviceWorkers;
-          ctx.menus = resolved.menus;
-          resolve();
-        }, reject)
-        .finally(() => {
-          ctx.activeBuild = null;
-        });
-    });
-  }
+export function updateRoutingContext(ctx: RoutingContext) {
+  ctx.activeBuild ||= _updateRoutingContext(ctx).finally(() => {
+    ctx.activeBuild = null;
+  });
   return ctx.activeBuild;
 }
 
-function rewriteRoutes(ctx: BuildContext, resolvedFiles: ReturnType<typeof resolveSourceFiles>) {
-  if (!ctx.opts.rewriteRoutes || !resolvedFiles.routes) {
-    return;
+async function _updateRoutingContext(ctx: RoutingContext) {
+  const serverPlugins = await walkServerPlugins(ctx.opts);
+  const sourceFiles = await walkRoutes(ctx.opts.routesDir);
+  const resolved = resolveSourceFiles(ctx.opts, sourceFiles);
+  resolved.routes = rewriteRoutes(ctx, resolved.routes);
+  ctx.serverPlugins = serverPlugins;
+  ctx.layouts = resolved.layouts;
+  ctx.routes = resolved.routes;
+  ctx.entries = resolved.entries;
+  ctx.serviceWorkers = resolved.serviceWorkers;
+  ctx.menus = resolved.menus;
+}
+
+function rewriteRoutes(ctx: RoutingContext, routes: BuiltRoute[]) {
+  if (!ctx.opts.rewriteRoutes) {
+    return routes;
   }
 
-  const translatedRoutes: BuildRoute[] = [];
+  const translatedRoutes: BuiltRoute[] = [];
 
   let segmentsToTranslate = ctx.opts.rewriteRoutes.flatMap((rewriteConfig) => {
     return Object.keys(rewriteConfig.paths || {});
   });
 
   segmentsToTranslate = Array.from(new Set(segmentsToTranslate));
 
-  resolvedFiles.routes.forEach((route) => {
+  routes.forEach((route) => {
     // always push the original route
     translatedRoutes.push(route);
 
@@ -91,14 +86,14 @@ function rewriteRoutes(ctx: BuildContext, resolvedFiles: ReturnType<typeof resol
     }
   });
 
-  resolvedFiles.routes = translatedRoutes;
+  return translatedRoutes.sort(routeSortCompare);
 }
 
 function translateRoute(
-  route: BuildRoute,
+  route: BuiltRoute,
   config: RewriteRouteOption,
   configIndex: number
-): BuildRoute {
+): BuiltRoute {
   const replacePath = (part: string) => (config.paths || {})[part] ?? part;
 
   const pathnamePrefix = config.prefix ? '/' + config.prefix : '';
@@ -156,7 +151,7 @@ function translateRoute(
   return routeToPush;
 }
 
-function validateBuild(ctx: BuildContext) {
+function validateBuild(ctx: RoutingContext) {
   const pathnames = Array.from(new Set(ctx.routes.map((r) => r.pathname))).sort();
 
   for (const pathname of pathnames) {

packages/qwik-router/src/buildtime/context.ts

@@ -1,6 +1,6 @@
 import { isAbsolute, resolve } from 'node:path';
 import { normalizePath } from '../utils/fs';
-import type { BuildContext, NormalizedPluginOptions, PluginOptions } from './types';
+import type { RoutingContext, NormalizedPluginOptions, PluginOptions } from './types';
 
 export function createBuildContext(
   rootDir: string,
@@ -9,7 +9,7 @@ export function createBuildContext(
   target?: 'ssr' | 'client',
   dynamicImports?: boolean
 ) {
-  const ctx: BuildContext = {
+  const ctx: RoutingContext = {
     rootDir: normalizePath(rootDir),
     opts: normalizeOptions(rootDir, viteBasePath, userOpts),
     routes: [],
@@ -28,7 +28,7 @@ export function createBuildContext(
   return ctx;
 }
 
-export function resetBuildContext(ctx: BuildContext | null) {
+export function resetBuildContext(ctx: RoutingContext | null) {
   if (ctx) {
     ctx.routes.length = 0;
     ctx.layouts.length = 0;

packages/qwik-router/src/buildtime/markdown/frontmatter.ts

@@ -1,12 +1,12 @@
 import type { Transformer } from 'unified';
-import type { BuildContext, FrontmatterAttrs } from '../types';
+import type { RoutingContext, FrontmatterAttrs } from '../types';
 import { normalizePath } from '../../utils/fs';
 import { visit } from 'unist-util-visit';
 import { parse as parseYaml } from 'yaml';
 import type { ResolvedDocumentHead } from '../../runtime/src';
 import type { DocumentMeta, Editable } from '../../runtime/src/types';
 
-export function parseFrontmatter(ctx: BuildContext): Transformer {
+export function parseFrontmatter(ctx: RoutingContext): Transformer {
   return (mdast, vfile) => {
     const attrs: FrontmatterAttrs = {};
 

packages/qwik-router/src/buildtime/markdown/mdx.ts

@@ -1,12 +1,12 @@
 import type { CompileOptions } from '@mdx-js/mdx';
 import { SourceMapGenerator } from 'source-map';
 import { getExtension } from '../../utils/fs';
-import type { BuildContext } from '../types';
+import type { RoutingContext } from '../types';
 import { parseFrontmatter } from './frontmatter';
 import { rehypePage, rehypeSlug, renameClassname, wrapTableWithDiv } from './rehype';
 import { rehypeSyntaxHighlight } from './syntax-highlight';
 
-export async function createMdxTransformer(ctx: BuildContext): Promise<MdxTransform> {
+export async function createMdxTransformer(ctx: RoutingContext): Promise<MdxTransform> {
   const { compile } = await import('@mdx-js/mdx');
   const { default: remarkFrontmatter } = await import('remark-frontmatter');
   const { default: remarkGfm } = await import('remark-gfm');

packages/qwik-router/src/buildtime/markdown/menu.ts

@@ -1,10 +1,10 @@
-import type { NormalizedPluginOptions, BuildMenu, ParsedMenuItem, RouteSourceFile } from '../types';
+import type { NormalizedPluginOptions, BuiltMenu, ParsedMenuItem, RouteSourceFile } from '../types';
 import { marked } from 'marked';
 import { createFileId, getMenuPathname } from '../../utils/fs';
 import { getMarkdownRelativeUrl } from './markdown-url';
 
 export function createMenu(opts: NormalizedPluginOptions, filePath: string) {
-  const menu: BuildMenu = {
+  const menu: BuiltMenu = {
     pathname: getMenuPathname(opts, filePath),
     filePath,
   };

packages/qwik-router/src/buildtime/markdown/rehype.ts

@@ -7,7 +7,7 @@ import { headingRank } from 'hast-util-heading-rank';
 import { toString } from 'hast-util-to-string';
 import { visit } from 'unist-util-visit';
 import type { ContentHeading } from '../../runtime/src';
-import type { BuildContext, NormalizedPluginOptions } from '../types';
+import type { RoutingContext, NormalizedPluginOptions } from '../types';
 import { getExtension, isMarkdownExt, normalizePath } from '../../utils/fs';
 import { frontmatterAttrsToDocumentHead } from './frontmatter';
 import { isSameOriginUrl } from '../../utils/pathname';
@@ -31,7 +31,7 @@ export function rehypeSlug(): Transformer {
   };
 }
 
-export function rehypePage(ctx: BuildContext): Transformer {
+export function rehypePage(ctx: RoutingContext): Transformer {
   return (ast, vfile) => {
     const mdast = ast as Root;
     const sourcePath = normalizePath(vfile.path);
@@ -96,12 +96,12 @@ function updateContentLinks(mdast: Root, opts: NormalizedPluginOptions, sourcePa
   });
 }
 
-function exportFrontmatter(ctx: BuildContext, mdast: Root, sourcePath: string) {
+function exportFrontmatter(ctx: RoutingContext, mdast: Root, sourcePath: string) {
   const attrs = ctx.frontmatter.get(sourcePath);
   createExport(mdast, 'frontmatter', attrs);
 }
 
-function exportContentHead(ctx: BuildContext, mdast: Root, sourcePath: string) {
+function exportContentHead(ctx: RoutingContext, mdast: Root, sourcePath: string) {
   const attrs = ctx.frontmatter.get(sourcePath);
   const head = frontmatterAttrsToDocumentHead(attrs);
   if (head) {