From 5a6be6961f5d36d4206f2a8c4c17ffe4c3c701ec Mon Sep 17 00:00:00 2001 From: Michael Herzner Date: Sat, 7 Dec 2024 15:39:38 +0100 Subject: [PATCH 01/25] feat: first draft for deps graph --- frontend/deno.json | 4 +- frontend/deno.lock | 5 + .../package/(_islands)/DependencyGraph.tsx | 127 +++ .../routes/package/dependencies/graph.tsx | 873 ++++++++++++++++++ .../index.tsx} | 18 +- 5 files changed, 1017 insertions(+), 10 deletions(-) create mode 100644 frontend/routes/package/(_islands)/DependencyGraph.tsx create mode 100644 frontend/routes/package/dependencies/graph.tsx rename frontend/routes/package/{dependencies.tsx => dependencies/index.tsx} (90%) diff --git a/frontend/deno.json b/frontend/deno.json index 2c4a5cbc..39f125b2 100644 --- a/frontend/deno.json +++ b/frontend/deno.json @@ -38,7 +38,9 @@ "@oramacloud/client": "npm:@oramacloud/client@^1", "tailwindcss": "npm:tailwindcss@3.4", - "postcss": "npm:postcss@8.4" + "postcss": "npm:postcss@8.4", + + "@viz-js/viz": "npm:@viz-js/viz@^3.11.0" }, "compilerOptions": { "lib": [ diff --git a/frontend/deno.lock b/frontend/deno.lock index 42b4f9e9..0addb41b 100644 --- a/frontend/deno.lock +++ b/frontend/deno.lock @@ -31,6 +31,7 @@ "npm:@oramacloud/client@1": "1.3.20", "npm:@preact/signals@1.2.1": "1.2.1_preact@10.24.3", "npm:@preact/signals@^1.2.3": "1.3.0_preact@10.24.3", + "npm:@viz-js/viz@^3.11.0": "3.11.0", "npm:autoprefixer@10.4.17": "10.4.17_postcss@8.4.35", "npm:cssnano@6.0.3": "6.0.3_postcss@8.4.35", "npm:esbuild-wasm@0.23.1": "0.23.1", @@ -390,6 +391,9 @@ "undici-types@6.19.8" ] }, + "@viz-js/viz@3.11.0": { + "integrity": "sha512-3zoKLQUqShIhTPvBAIIgJUf5wO9aY0q+Ftzw1u26KkJX1OJjT7Z5VUqgML2GIzXJYFgjqS6a2VREMwrgChuubA==" + }, "@vue/compiler-core@3.5.12": { "integrity": "sha512-ISyBTRMmMYagUxhcpyEH0hpXRd/KqDU4ymofPgl2XAkY9ZhQ+h0ovEZJIiPop13UmR/54oA2cgMDjgroRelaEw==", "dependencies": [ @@ -1813,6 +1817,7 @@ "npm:@orama/orama@2", "npm:@oramacloud/client@1", "npm:@preact/signals@1.2.1", + "npm:@viz-js/viz@^3.11.0", "npm:marked-smartypants@1.1.6", "npm:postcss@8.4", "npm:preact-render-to-string@6.3.1", diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx new file mode 100644 index 00000000..a9fc4b03 --- /dev/null +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -0,0 +1,127 @@ +// Copyright 2024 the JSR authors. All rights reserved. MIT license. +import { useEffect, useRef } from "preact/hooks"; +import { useSignal } from "@preact/signals"; +import { instance, type Viz } from "@viz-js/viz"; + +interface DependencyGraphKindJsr { + type: "jsr"; + scope: string; + package: string; + version: string; + path: string; +} +interface DependencyGraphKindNpm { + type: "npm"; + package: string; + version: string; +} +interface DependencyGraphKindRoot { + type: "root"; + path: string; +} +interface DependencyGraphKindError { + type: "error"; + error: string; +} + +type DependencyGraphKind = + | DependencyGraphKindJsr + | DependencyGraphKindNpm + | DependencyGraphKindRoot + | DependencyGraphKindError; + +interface DependencyGraphItem { + dependency: DependencyGraphKind; + children: number[]; + size: number | undefined; + mediaType: string | undefined; +} + +export interface DependencyGraphProps { + dependencies: DependencyGraphItem[]; +} + +function createDigraph(dependencies: DependencyGraphProps["dependencies"]) { + return ` +digraph "dependencies" { + node [fontname="Courier", shape="box"] + + ${ + dependencies.map(({ children, dependency }, index) => { + return [ + [index, renderDependency(dependency)].join(" "), + ...children.map((child) => `${index} -> ${child}`), + ].filter(Boolean).join("\n"); + }).join("\n") + } +}`; +} + +function renderDependency(dependency: DependencyGraphKind) { + switch (dependency.type) { + case "jsr": + return renderJsrDependency(dependency); + case "npm": + return renderNpmDependency(dependency); + case "root": + return renderRootDependency(dependency); + case "error": + default: + return renderErrorDependency(dependency); + } +} + +function renderJsrDependency( + dependency: DependencyGraphKindJsr, +) { + const label = + `@${dependency.scope}/${dependency.package}@${dependency.version}`; + const href = `/${label}`; + + return `[href="${href}", label="${label}", tooltip="${label}"]\n`; +} + +function renderNpmDependency(dependency: DependencyGraphKindNpm) { + const label = `${dependency.package}@${dependency.version}`; + const href = `https://www.npmjs.com/package/${dependency.package}`; + + return `[href="${href}", label="${label}", tooltip="${label}"]\n`; +} + +function renderRootDependency(dependency: DependencyGraphKindRoot) { + const label = dependency.path; + + return `[label="${label}", tooltip="${label}"]\n`; +} + +function renderErrorDependency(dependency: DependencyGraphKindError) { + return ``; +} + +export function DependencyGraph(props: DependencyGraphProps) { + const anchor = useRef(null); + const viz = useSignal(undefined); + + useEffect(() => { + (async () => { + viz.value = await instance(); + + if (anchor.current && viz.value) { + const digraph = createDigraph(props.dependencies); + + console.log(digraph); + + anchor.current.appendChild( + viz.value.renderSVGElement(digraph), + ); + } + })(); + }, []); + + return ( +
+ ); +} diff --git a/frontend/routes/package/dependencies/graph.tsx b/frontend/routes/package/dependencies/graph.tsx new file mode 100644 index 00000000..f2d7058e --- /dev/null +++ b/frontend/routes/package/dependencies/graph.tsx @@ -0,0 +1,873 @@ +// Copyright 2024 the JSR authors. All rights reserved. MIT license. +import { HttpError, type RouteConfig } from "fresh"; +import type { Dependency } from "../../../utils/api_types.ts"; +import { path } from "../../../utils/api.ts"; +import { scopeIAM } from "../../../utils/iam.ts"; +import { define } from "../../../util.ts"; +import { + DependencyGraph, + DependencyGraphProps, +} from "../(_islands)/DependencyGraph.tsx"; +import { packageDataWithVersion } from "../../../utils/data.ts"; +import { PackageHeader } from "../(_components)/PackageHeader.tsx"; +import { PackageNav, type Params } from "../(_components)/PackageNav.tsx"; + +const dependencies = [ + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "assert", + "version": "0.225.3", + "path": "/assertion_error.ts", + }, + "children": [], + "size": 484, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "assert", + "version": "0.225.3", + "path": "/assert.ts", + }, + "children": [ + 0, + ], + "size": 562, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "root", + "path": "/cookie.ts", + }, + "children": [ + 1, + ], + "size": 11310, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "encoding", + "version": "0.224.3", + "path": "/_validate_binary_like.ts", + }, + "children": [], + "size": 798, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "encoding", + "version": "0.224.3", + "path": "/base64.ts", + }, + "children": [ + 3, + ], + "size": 3336, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "root", + "path": "/etag.ts", + }, + "children": [ + 4, + ], + "size": 6579, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "root", + "path": "/status.ts", + }, + "children": [], + "size": 13575, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "root", + "path": "/_negotiation/common.ts", + }, + "children": [], + "size": 1801, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "root", + "path": "/_negotiation/encoding.ts", + }, + "children": [ + 7, + ], + "size": 4301, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "root", + "path": "/_negotiation/language.ts", + }, + "children": [ + 7, + ], + "size": 4150, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "root", + "path": "/_negotiation/media_type.ts", + }, + "children": [ + 7, + ], + "size": 4970, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "root", + "path": "/negotiation.ts", + }, + "children": [ + 8, + 9, + 10, + ], + "size": 6414, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "async", + "version": "0.224.1", + "path": "/delay.ts", + }, + "children": [], + "size": 1895, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "root", + "path": "/server.ts", + }, + "children": [ + 12, + ], + "size": 25885, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "encoding", + "version": "0.224.3", + "path": "/hex.ts", + }, + "children": [ + 3, + ], + "size": 3097, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "root", + "path": "/unstable_signed_cookie.ts", + }, + "children": [ + 14, + ], + "size": 3687, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "root", + "path": "/server_sent_event_stream.ts", + }, + "children": [], + "size": 2761, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "root", + "path": "/user_agent.ts", + }, + "children": [ + 1, + ], + "size": 36299, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/_common/assert_path.ts", + }, + "children": [], + "size": 307, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/_common/normalize.ts", + }, + "children": [ + 18, + ], + "size": 263, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/_common/constants.ts", + }, + "children": [], + "size": 2020, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/_common/normalize_string.ts", + }, + "children": [ + 20, + ], + "size": 2301, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/posix/_util.ts", + }, + "children": [ + 20, + ], + "size": 391, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/posix/normalize.ts", + }, + "children": [ + 19, + 21, + 22, + ], + "size": 1056, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/posix/join.ts", + }, + "children": [ + 18, + 23, + ], + "size": 721, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/_os.ts", + }, + "children": [], + "size": 705, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/posix/extname.ts", + }, + "children": [ + 20, + 18, + 22, + ], + "size": 2186, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/windows/_util.ts", + }, + "children": [ + 20, + ], + "size": 828, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/windows/extname.ts", + }, + "children": [ + 20, + 18, + 27, + ], + "size": 2342, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/extname.ts", + }, + "children": [ + 25, + 26, + 28, + ], + "size": 547, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/windows/normalize.ts", + }, + "children": [ + 19, + 20, + 21, + 27, + ], + "size": 3786, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/windows/join.ts", + }, + "children": [ + 1, + 18, + 27, + 30, + ], + "size": 2483, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/join.ts", + }, + "children": [ + 25, + 24, + 31, + ], + "size": 510, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/posix/resolve.ts", + }, + "children": [ + 21, + 18, + 22, + ], + "size": 1586, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/_common/relative.ts", + }, + "children": [ + 18, + ], + "size": 287, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/posix/relative.ts", + }, + "children": [ + 22, + 33, + 34, + ], + "size": 3000, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/windows/resolve.ts", + }, + "children": [ + 20, + 21, + 18, + 27, + ], + "size": 4848, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/windows/relative.ts", + }, + "children": [ + 20, + 36, + 34, + ], + "size": 3978, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/relative.ts", + }, + "children": [ + 25, + 35, + 37, + ], + "size": 788, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/resolve.ts", + }, + "children": [ + 25, + 33, + 36, + ], + "size": 528, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "path", + "version": "0.225.1", + "path": "/constants.ts", + }, + "children": [ + 25, + ], + "size": 348, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "media-types", + "version": "1.0.0-rc.1", + "path": "/_util.ts", + }, + "children": [], + "size": 3253, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "media-types", + "version": "1.0.0-rc.1", + "path": "/parse_media_type.ts", + }, + "children": [ + 41, + ], + "size": 3636, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "media-types", + "version": "1.0.0-rc.1", + "path": "/vendor/mime-db.v1.52.0.ts", + }, + "children": [], + "size": 186498, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "media-types", + "version": "1.0.0-rc.1", + "path": "/_db.ts", + }, + "children": [ + 43, + 41, + ], + "size": 1347, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "media-types", + "version": "1.0.0-rc.1", + "path": "/get_charset.ts", + }, + "children": [ + 42, + 41, + 44, + ], + "size": 1497, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "media-types", + "version": "1.0.0-rc.1", + "path": "/format_media_type.ts", + }, + "children": [ + 41, + ], + "size": 2539, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "media-types", + "version": "1.0.0-rc.1", + "path": "/type_by_extension.ts", + }, + "children": [ + 44, + ], + "size": 1203, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "media-types", + "version": "1.0.0-rc.1", + "path": "/content_type.ts", + }, + "children": [ + 42, + 45, + 46, + 44, + 47, + ], + "size": 3552, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "streams", + "version": "0.224.2", + "path": "/byte_slice_stream.ts", + }, + "children": [ + 1, + ], + "size": 2657, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "cli", + "version": "0.224.4", + "path": "/parse_args.ts", + }, + "children": [ + 1, + ], + "size": 22373, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "fmt", + "version": "0.225.2", + "path": "/colors.ts", + }, + "children": [], + "size": 21644, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "root", + "path": "/deno.json", + }, + "children": [], + "size": 461, + "mediaType": "Json", + }, + { + "dependency": { + "type": "jsr", + "scope": "std", + "package": "fmt", + "version": "0.225.2", + "path": "/bytes.ts", + }, + "children": [], + "size": 4665, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "root", + "path": "/file_server.ts", + }, + "children": [ + 24, + 23, + 29, + 32, + 38, + 39, + 40, + 48, + 5, + 6, + 49, + 50, + 51, + 52, + 53, + ], + "size": 25534, + "mediaType": "TypeScript", + }, + { + "dependency": { + "type": "root", + "path": "/mod.ts", + }, + "children": [ + 2, + 5, + 6, + 11, + 13, + 15, + 16, + 17, + 54, + ], + "size": 2380, + "mediaType": "TypeScript", + }, +] as const satisfies DependencyGraphProps["dependencies"]; + +export default define.page( + function DepsGraph({ data, params, state }) { + const iam = scopeIAM(state, data.member); + + return ( +
+ + + + +
+ +
+
+ ); + }, +); + +export const handler = define.handlers({ + async GET(ctx) { + const res = await packageDataWithVersion( + ctx.state, + ctx.params.scope, + ctx.params.package, + ctx.params.version, + ); + if (res === null) { + throw new HttpError( + 404, + "This package or this package version was not found.", + ); + } + + const { + pkg, + scopeMember, + selectedVersion, + } = res; + + if (selectedVersion === null) { + return new Response(null, { + status: 302, + headers: { + Location: `/@${ctx.params.scope}/${ctx.params.package}`, + }, + }); + } + + const depsResp = await ctx.state.api.get( + path`/scopes/${pkg.scope}/packages/${pkg.name}/versions/${selectedVersion.version}/dependencies`, + ); + if (!depsResp.ok) throw depsResp; + + ctx.state.meta = { + title: `Dependencies - @${pkg.scope}/${pkg.name} - JSR`, + description: `@${pkg.scope}/${pkg.name} on JSR${ + pkg.description ? `: ${pkg.description}` : "" + }`, + }; + + return { + data: { + package: pkg, + deps: depsResp.data, + selectedVersion, + member: scopeMember, + }, + headers: { "X-Robots-Tag": "noindex" }, + }; + }, +}); + +export const config: RouteConfig = { + routeOverride: "/@:scope/:package{@:version}?/dependencies/graph", +}; diff --git a/frontend/routes/package/dependencies.tsx b/frontend/routes/package/dependencies/index.tsx similarity index 90% rename from frontend/routes/package/dependencies.tsx rename to frontend/routes/package/dependencies/index.tsx index a8f7ea8d..7e6f97d8 100644 --- a/frontend/routes/package/dependencies.tsx +++ b/frontend/routes/package/dependencies/index.tsx @@ -1,13 +1,13 @@ // Copyright 2024 the JSR authors. All rights reserved. MIT license. -import { HttpError, RouteConfig } from "fresh"; -import type { Dependency } from "../../utils/api_types.ts"; -import { path } from "../../utils/api.ts"; -import { define } from "../../util.ts"; -import { packageDataWithVersion } from "../../utils/data.ts"; -import { PackageHeader } from "./(_components)/PackageHeader.tsx"; -import { PackageNav, Params } from "./(_components)/PackageNav.tsx"; -import { Table, TableData, TableRow } from "../../components/Table.tsx"; -import { scopeIAM } from "../../utils/iam.ts"; +import { HttpError, type RouteConfig } from "fresh"; +import type { Dependency } from "../../../utils/api_types.ts"; +import { path } from "../../../utils/api.ts"; +import { define } from "../../../util.ts"; +import { packageDataWithVersion } from "../../../utils/data.ts"; +import { PackageHeader } from "../(_components)/PackageHeader.tsx"; +import { PackageNav, type Params } from "../(_components)/PackageNav.tsx"; +import { Table, TableData, TableRow } from "../../../components/Table.tsx"; +import { scopeIAM } from "../../../utils/iam.ts"; function getDependencyLink(dep: Dependency) { if (dep.kind === "jsr") { From ab50303bc345c4e89a90f364d94d92fbc2b54925 Mon Sep 17 00:00:00 2001 From: Michael Herzner Date: Sat, 7 Dec 2024 23:47:00 +0100 Subject: [PATCH 02/25] feat: add custom hook and controls to graph --- frontend/components/icons/ChevronUp.tsx | 19 ++ frontend/components/icons/Minus.tsx | 18 ++ frontend/components/icons/Reset.tsx | 18 ++ .../package/(_islands)/DependencyGraph.tsx | 165 +++++++++++++++--- 4 files changed, 193 insertions(+), 27 deletions(-) create mode 100644 frontend/components/icons/ChevronUp.tsx create mode 100644 frontend/components/icons/Minus.tsx create mode 100644 frontend/components/icons/Reset.tsx diff --git a/frontend/components/icons/ChevronUp.tsx b/frontend/components/icons/ChevronUp.tsx new file mode 100644 index 00000000..88358586 --- /dev/null +++ b/frontend/components/icons/ChevronUp.tsx @@ -0,0 +1,19 @@ +// Copyright 2024 the JSR authors. All rights reserved. MIT license. +export function ChevronUp(props: { class?: string }) { + return ( + + + + ); +} diff --git a/frontend/components/icons/Minus.tsx b/frontend/components/icons/Minus.tsx new file mode 100644 index 00000000..6ef9f650 --- /dev/null +++ b/frontend/components/icons/Minus.tsx @@ -0,0 +1,18 @@ +// Copyright 2024 the JSR authors. All rights reserved. MIT license. +export function Minus(props: { class?: string }) { + return ( + + ); +} diff --git a/frontend/components/icons/Reset.tsx b/frontend/components/icons/Reset.tsx new file mode 100644 index 00000000..3f035cfd --- /dev/null +++ b/frontend/components/icons/Reset.tsx @@ -0,0 +1,18 @@ +// Copyright 2024 the JSR authors. All rights reserved. MIT license. +export function Reset(props: { class?: string }) { + return ( + + ); +} diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index a9fc4b03..5ac5a288 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -1,7 +1,15 @@ // Copyright 2024 the JSR authors. All rights reserved. MIT license. -import { useEffect, useRef } from "preact/hooks"; +import type { ComponentChildren } from "preact"; +import { useCallback, useEffect, useRef } from "preact/hooks"; import { useSignal } from "@preact/signals"; import { instance, type Viz } from "@viz-js/viz"; +import { ChevronDown } from "../../../components/icons/ChevronDown.tsx"; +import { ChevronLeft } from "../../../components/icons/ChevronLeft.tsx"; +import { ChevronRight } from "../../../components/icons/ChevronRight.tsx"; +import { ChevronUp } from "../../../components/icons/ChevronUp.tsx"; +import { Minus } from "../../../components/icons/Minus.tsx"; +import { Plus } from "../../../components/icons/Plus.tsx"; +import { Reset } from "../../../components/icons/Reset.tsx"; interface DependencyGraphKindJsr { type: "jsr"; @@ -42,15 +50,14 @@ export interface DependencyGraphProps { } function createDigraph(dependencies: DependencyGraphProps["dependencies"]) { - return ` -digraph "dependencies" { + return `digraph "dependencies" { node [fontname="Courier", shape="box"] - ${ +${ dependencies.map(({ children, dependency }, index) => { return [ - [index, renderDependency(dependency)].join(" "), - ...children.map((child) => `${index} -> ${child}`), + ` ${index} ${renderDependency(dependency)}`, + ...children.map((child) => ` ${index} -> ${child}`), ].filter(Boolean).join("\n"); }).join("\n") } @@ -71,57 +78,161 @@ function renderDependency(dependency: DependencyGraphKind) { } } -function renderJsrDependency( - dependency: DependencyGraphKindJsr, -) { +function renderJsrDependency(dependency: DependencyGraphKindJsr) { const label = `@${dependency.scope}/${dependency.package}@${dependency.version}`; const href = `/${label}`; - return `[href="${href}", label="${label}", tooltip="${label}"]\n`; + return `[href="${href}", label="${label}", tooltip="${label}"]`; } function renderNpmDependency(dependency: DependencyGraphKindNpm) { const label = `${dependency.package}@${dependency.version}`; const href = `https://www.npmjs.com/package/${dependency.package}`; - return `[href="${href}", label="${label}", tooltip="${label}"]\n`; + return `[href="${href}", label="${label}", tooltip="${label}"]`; } function renderRootDependency(dependency: DependencyGraphKindRoot) { const label = dependency.path; - return `[label="${label}", tooltip="${label}"]\n`; + return `[label="${label}", tooltip="${label}"]`; } function renderErrorDependency(dependency: DependencyGraphKindError) { - return ``; + const label = dependency.error; + + return `[label="${label}", tooltip="${label}"]`; } -export function DependencyGraph(props: DependencyGraphProps) { - const anchor = useRef(null); +function useDigraph(dependencies: DependencyGraphProps["dependencies"]) { + const controls = useSignal({ pan: { x: 0, y: 0 }, zoom: 1 }); + const ref = useRef(null); + const svg = useRef(null); const viz = useSignal(undefined); + const pan = useCallback((x: number, y: number) => { + controls.value.pan.x += x; + controls.value.pan.y += y; + if (svg.current) { + svg.current.style.transform = + `translate(${controls.value.pan.x}px, ${controls.value.pan.y}px) scale(${controls.value.zoom})`; + } + }, [controls]); + + const zoom = useCallback((zoom: number) => { + controls.value.zoom += zoom; + if (svg.current) { + svg.current.style.transform = + `translate(${controls.value.pan.x}px, ${controls.value.pan.y}px) scale(${controls.value.zoom})`; + } + }, [controls]); + + const reset = useCallback(() => { + controls.value = { pan: { x: 0, y: 0 }, zoom: 1 }; + if (svg.current) { + svg.current.style.transform = ""; + } + }, []); + useEffect(() => { (async () => { viz.value = await instance(); - if (anchor.current && viz.value) { - const digraph = createDigraph(props.dependencies); + if (ref.current && viz.value) { + const digraph = createDigraph(dependencies); - console.log(digraph); - - anchor.current.appendChild( - viz.value.renderSVGElement(digraph), - ); + svg.current = viz.value.renderSVGElement(digraph); + ref.current.appendChild(svg.current); } })(); - }, []); + }, [dependencies]); + + return { pan, zoom, reset, ref }; +} + +interface GraphControlButtonProps { + children: ComponentChildren; + class: string; + onClick: () => void; + title: string; +} + +function GraphControlButton(props: GraphControlButtonProps) { + return ( + + ); +} + +export function DependencyGraph(props: DependencyGraphProps) { + const { pan, zoom, reset, ref } = useDigraph(props.dependencies); return ( -
+
+
+
+ {/* zoom */} + zoom(0.1)} + title="Zoom in" + > + + + zoom(-0.1)} + title="Zoom out" + > + + + + {/* pan */} + pan(0, 100)} + title="Pan up" + > + + + pan(100, 0)} + title="Pan left" + > + + + pan(-100, 0)} + title="Pan right" + > + + + pan(0, -100)} + title="Pan down" + > + + + + {/* reset */} + + + +
+
); } From e2cfb6d72158fec2daddc29bca9ebd9f9b42ee63 Mon Sep 17 00:00:00 2001 From: crowlkats Date: Tue, 26 Nov 2024 18:14:59 +0100 Subject: [PATCH 03/25] feat: dependency graph --- api/src/api/package.rs | 585 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 583 insertions(+), 2 deletions(-) diff --git a/api/src/api/package.rs b/api/src/api/package.rs index 8e857fb0..b21b0c74 100644 --- a/api/src/api/package.rs +++ b/api/src/api/package.rs @@ -1,14 +1,24 @@ // Copyright 2024 the JSR authors. All rights reserved. MIT license. use std::borrow::Cow; -use std::io; +use std::collections::HashSet; +use std::fmt::Write; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::Mutex; +use std::{fmt, io}; -use anyhow::Context; +use anyhow::{bail, Context, Error as AnyError}; use chrono::Utc; use comrak::adapters::SyntaxHighlighterAdapter; +use deno_ast::{MediaType, ModuleSpecifier, ParseDiagnostic}; +use deno_graph::source::{ + load_data_url, JsrUrlProvider, LoadOptions, NullFileSystem, +}; +use deno_graph::{ + BuildOptions, CapturingModuleAnalyzer, GraphKind, Module, ModuleError, + ModuleInfo, Resolution, WorkspaceMember, +}; use futures::future::Either; use futures::StreamExt; use hyper::body::HttpBody; @@ -16,6 +26,7 @@ use hyper::Body; use hyper::Request; use hyper::Response; use hyper::StatusCode; +use indexmap::IndexMap; use routerify::prelude::RequestExt; use routerify::Router; use routerify_query::RequestQueryExt; @@ -27,6 +38,7 @@ use tracing::Instrument; use tracing::Span; use url::Url; +use crate::analysis::{JsrResolver, ModuleAnalyzer, ModuleParser}; use crate::auth::access_token; use crate::auth::GithubOauth2Client; use crate::buckets::Buckets; @@ -150,6 +162,10 @@ pub fn package_router() -> Router { "/:package/versions/:version/dependencies", util::json(list_dependencies_handler), ) + .get( + "/:package/versions/:version/dependencies/graph", + util::json(get_dependencies_graph_handler), + ) .get( "/:package/publishing_tasks", util::json(list_publishing_tasks_handler), @@ -1520,6 +1536,571 @@ pub async fn list_dependencies_handler( Ok(deps) } +struct DepTreeLoader { + scope: ScopeName, + package: PackageName, + version: crate::ids::Version, + bucket: crate::buckets::BucketWithQueue, +} + +impl DepTreeLoader { + fn load_inner( + &self, + specifier: &ModuleSpecifier, + ) -> deno_graph::source::LoadFuture { + use futures::FutureExt; + let specifier = specifier.clone(); + + match specifier.scheme() { + "file" => { + let Ok(path) = PackagePath::new(specifier.path().to_string()) else { + return async move { Ok(None) }.boxed(); + }; + + let scope = self.scope.clone(); + let package = self.package.clone(); + let version = self.version.clone(); + let bucket = self.bucket.clone(); + + async move { + let Some(bytes) = bucket + .download( + crate::gcs_paths::file_path(&scope, &package, &version, &path) + .into(), + ) + .await + .expect("hello") + else { + return Ok(None); + }; + + Ok(Some(deno_graph::source::LoadResponse::Module { + content: bytes.to_vec().into(), + specifier: specifier.clone(), + maybe_headers: None, + })) + } + .boxed() + } + "http" | "https" => async move { + // TODO: dont use reqwest, call to bucket directly. + let s = reqwest::Client::builder() + .build() + .unwrap() + .get(specifier.clone()) + .send() + .await + .unwrap(); + let bytes = s.bytes().await.unwrap(); + Ok(Some(deno_graph::source::LoadResponse::Module { + content: bytes.to_vec().into(), + specifier: specifier.clone(), + maybe_headers: None, + })) + } + .boxed(), + "jsr" => unreachable!("{specifier}"), + // TODO: handle npm specifiers + "npm" | "node" | "bun" => async move { + Ok(Some(deno_graph::source::LoadResponse::External { + specifier: specifier.clone(), + })) + } + .boxed(), + _ => async move { Ok(None) }.boxed(), + } + } +} + +impl deno_graph::source::Loader for DepTreeLoader { + fn load( + &self, + specifier: &ModuleSpecifier, + _options: LoadOptions, + ) -> deno_graph::source::LoadFuture { + self.load_inner(specifier) + } +} + +struct DepTreeJsrUrlProvider(Url); + +impl JsrUrlProvider for DepTreeJsrUrlProvider { + fn url(&self) -> &Url { + &self.0 + } +} + +struct DepTreeAnalyzer { + pub analyzer: CapturingModuleAnalyzer, + pub module_info: + std::cell::RefCell>>, +} + +impl Default for DepTreeAnalyzer { + fn default() -> Self { + Self { + analyzer: CapturingModuleAnalyzer::new( + Some(Box::new(ModuleParser::default())), + None, + ), + module_info: Default::default(), + } + } +} + +#[async_trait::async_trait(?Send)] +impl deno_graph::ModuleAnalyzer for DepTreeAnalyzer { + async fn analyze( + &self, + specifier: &ModuleSpecifier, + source: Arc, + media_type: MediaType, + ) -> Result { + let module_info = + self.analyzer.analyze(specifier, source, media_type).await?; + + let deps = module_info + .dependencies + .iter() + .filter_map(|dep| { + dep.as_static().and_then(|dep| { + if dep.specifier.starts_with("jsr:") { + Some(dep.specifier.clone()) + } else { + None + } + }) + }) + .collect::>(); + + if !deps.is_empty() { + self + .module_info + .borrow_mut() + .insert(specifier.clone(), deps.clone()); + } + + Ok(module_info) + } +} + +// We have to spawn another tokio runtime, because +// `deno_graph::ModuleGraph::build` is not thread-safe. +#[tokio::main(flavor = "current_thread")] +async fn analyze_deps_tree( + registry_url: Url, + scope: ScopeName, + package: PackageName, + version: crate::ids::Version, + bucket: crate::buckets::BucketWithQueue, + exports: IndexMap, +) -> Result<(), deno_graph::ModuleGraphError> { + let roots = exports + .values() + .map(|path| Url::parse(&format!("file://{}", path)).unwrap()) + .collect::>(); + + let member = WorkspaceMember { + base: Url::parse("file:///").unwrap(), + name: format!("@{}/{}", scope, package), + version: Some(version.0.clone()), + exports, + }; + + let module_analyzer = DepTreeAnalyzer::default(); + let mut graph = deno_graph::ModuleGraph::new(GraphKind::All); + let loader = DepTreeLoader { + scope, + package, + version, + bucket, + }; + graph + .build( + roots.clone(), + &loader, + BuildOptions { + is_dynamic: false, + module_analyzer: &module_analyzer, + imports: Default::default(), + // todo: use the data in the package for the file system + file_system: &NullFileSystem, + jsr_url_provider: &DepTreeJsrUrlProvider(registry_url), + passthrough_jsr_specifiers: false, + resolver: Some(&JsrResolver { member }), + npm_resolver: None, + reporter: None, + executor: Default::default(), + locker: None, + }, + ) + .await; + graph.valid()?; + + for root_specifier in roots { + let mut output = String::new(); + GraphDisplayContext::write(&graph, &mut output, &root_specifier).unwrap(); + println!("{output}"); + } + + Ok(()) +} + +struct GraphDisplayContext<'a> { + graph: &'a deno_graph::ModuleGraph, + seen: HashSet, + root: &'a ModuleSpecifier, +} + +impl<'a> GraphDisplayContext<'a> { + pub fn write( + graph: &'a deno_graph::ModuleGraph, + writer: &mut TWrite, + root: &'a ModuleSpecifier, + ) -> Result<(), AnyError> { + Self { + graph, + seen: Default::default(), + root, + } + .into_writer(writer) + } + + fn into_writer( + mut self, + writer: &mut TWrite, + ) -> Result<(), AnyError> { + let root_specifier = self.graph.resolve(&self.root); + match self.graph.try_get(root_specifier) { + Ok(Some(root)) => { + let maybe_cache_info = match root { + Module::Js(module) => module.maybe_cache_info.as_ref(), + Module::Json(module) => module.maybe_cache_info.as_ref(), + Module::Node(_) | Module::Npm(_) | Module::External(_) => None, + }; + if let Some(cache_info) = maybe_cache_info { + if let Some(local) = &cache_info.local { + writeln!(writer, "{} {}", "local:", local.to_string_lossy())?; + } + } + if let Some(module) = root.js() { + writeln!(writer, "{} {}", "type:", module.media_type)?; + } + let total_modules_size = self + .graph + .modules() + .map(|m| { + let size = match m { + Module::Js(module) => module.size(), + Module::Json(module) => module.size(), + Module::Node(_) | Module::Npm(_) | Module::External(_) => 0, + }; + size as f64 + }) + .sum::(); + let dep_count = self.graph.modules().count() - 1 // -1 for the root module + ; + writeln!(writer, "{} {} unique", "dependencies:", dep_count,)?; + writeln!(writer, "{} {}", "size:", total_modules_size,)?; + writeln!(writer)?; + let root_node = self.build_module_info(root, false); + print_tree_node(&root_node, writer)?; + Ok(()) + } + Err(err) => { + if let ModuleError::Missing(_, _) = *err { + bail!("module could not be found"); + } else { + bail!("{:#}", err); + } + } + Ok(None) => { + bail!("an internal error occurred"); + } + } + } + + fn build_dep_info(&mut self, dep: &deno_graph::Dependency) -> Vec { + let mut children = Vec::with_capacity(2); + if !dep.maybe_code.is_none() { + if let Some(child) = self.build_resolved_info(&dep.maybe_code, false) { + children.push(child); + } + } + if !dep.maybe_type.is_none() { + if let Some(child) = self.build_resolved_info(&dep.maybe_type, true) { + children.push(child); + } + } + children + } + + fn build_module_info(&mut self, module: &Module, type_dep: bool) -> TreeNode { + let specifier = module.specifier(); + let was_seen = !self.seen.insert(specifier.to_string()); + let header_text = if was_seen { + let specifier_str = if type_dep { + module.specifier().to_string() + } else { + module.specifier().to_string() + }; + format!("{} {}", specifier_str, "*") + } else { + let header_text = if type_dep { + module.specifier().to_string() + } else { + module.specifier().to_string() + }; + let maybe_size = match module { + Module::Js(module) => Some(module.size() as u64), + Module::Json(module) => Some(module.size() as u64), + Module::Node(_) | Module::Npm(_) | Module::External(_) => None, + }; + format!("{} {}", header_text, maybe_size_to_text(maybe_size)) + }; + + let mut tree_node = TreeNode::from_text(header_text); + + if !was_seen { + match module { + Module::Js(module) => { + if let Some(types_dep) = &module.maybe_types_dependency { + if let Some(child) = + self.build_resolved_info(&types_dep.dependency, true) + { + tree_node.children.push(child); + } + } + for dep in module.dependencies.values() { + tree_node.children.extend(self.build_dep_info(dep)); + } + } + Module::Json(_) + | Module::Npm(_) + | Module::Node(_) + | Module::External(_) => {} + } + } + tree_node + } + + fn build_error_info( + &mut self, + err: &ModuleError, + specifier: &ModuleSpecifier, + ) -> TreeNode { + self.seen.insert(specifier.to_string()); + match err { + ModuleError::InvalidTypeAssertion { .. } => { + self.build_error_msg(specifier, "(invalid import attribute)") + } + ModuleError::LoadingErr(_, _, err) => { + use deno_graph::ModuleLoadError::*; + let message = match err { + HttpsChecksumIntegrity(_) => "(checksum integrity error)", + Decode(_) => "(loading decode error)", + Loader(err) => "(loading error)", + Jsr(_) => "(loading error)", + NodeUnknownBuiltinModule(_) => "(unknown node built-in error)", + Npm(_) => "(npm loading error)", + TooManyRedirects => "(too many redirects error)", + }; + self.build_error_msg(specifier, message.as_ref()) + } + ModuleError::ParseErr(_, _) => { + self.build_error_msg(specifier, "(parsing error)") + } + ModuleError::UnsupportedImportAttributeType { .. } => { + self.build_error_msg(specifier, "(unsupported import attribute)") + } + ModuleError::UnsupportedMediaType { .. } => { + self.build_error_msg(specifier, "(unsupported)") + } + ModuleError::Missing(_, _) | ModuleError::MissingDynamic(_, _) => { + self.build_error_msg(specifier, "(missing)") + } + } + } + + fn build_error_msg( + &self, + specifier: &ModuleSpecifier, + error_msg: &str, + ) -> TreeNode { + TreeNode::from_text(format!("{specifier} {error_msg}",)) + } + + fn build_resolved_info( + &mut self, + resolution: &Resolution, + type_dep: bool, + ) -> Option { + match resolution { + Resolution::Ok(resolved) => { + let specifier = &resolved.specifier; + let resolved_specifier = self.graph.resolve(specifier); + Some(match self.graph.try_get(resolved_specifier) { + Ok(Some(module)) => self.build_module_info(module, type_dep), + Err(err) => self.build_error_info(err, resolved_specifier), + Ok(None) => { + TreeNode::from_text(format!("{} {}", specifier, "(missing)")) + } + }) + } + Resolution::Err(err) => Some(TreeNode::from_text(format!( + "{} {}", + err.to_string(), + "(resolve error)" + ))), + _ => None, + } + } +} + +struct TreeNode { + text: String, + children: Vec, +} + +impl TreeNode { + pub fn from_text(text: String) -> Self { + Self { + text, + children: Default::default(), + } + } +} + +fn maybe_size_to_text(maybe_size: Option) -> String { + format!( + "({})", + match maybe_size { + Some(size) => human_size(size as f64), + None => "unknown".to_string(), + } + ) +} + +fn print_tree_node( + tree_node: &TreeNode, + writer: &mut TWrite, +) -> fmt::Result { + fn print_children( + writer: &mut TWrite, + prefix: &str, + children: &[TreeNode], + ) -> fmt::Result { + const SIBLING_CONNECTOR: char = '├'; + const LAST_SIBLING_CONNECTOR: char = '└'; + const CHILD_DEPS_CONNECTOR: char = '┬'; + const CHILD_NO_DEPS_CONNECTOR: char = '─'; + const VERTICAL_CONNECTOR: char = '│'; + const EMPTY_CONNECTOR: char = ' '; + + let child_len = children.len(); + for (index, child) in children.iter().enumerate() { + let is_last = index + 1 == child_len; + let sibling_connector = if is_last { + LAST_SIBLING_CONNECTOR + } else { + SIBLING_CONNECTOR + }; + let child_connector = if child.children.is_empty() { + CHILD_NO_DEPS_CONNECTOR + } else { + CHILD_DEPS_CONNECTOR + }; + writeln!( + writer, + "{} {}", + format!("{prefix}{sibling_connector}─{child_connector}"), + child.text + )?; + let child_prefix = format!( + "{}{}{}", + prefix, + if is_last { + EMPTY_CONNECTOR + } else { + VERTICAL_CONNECTOR + }, + EMPTY_CONNECTOR + ); + print_children(writer, &child_prefix, &child.children)?; + } + + Ok(()) + } + + writeln!(writer, "{}", tree_node.text)?; + print_children(writer, "", &tree_node.children)?; + Ok(()) +} + +/// A function that converts a float to a string the represents a human +/// readable version of that number. +pub fn human_size(size: f64) -> String { + let negative = if size.is_sign_positive() { "" } else { "-" }; + let size = size.abs(); + let units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]; + if size < 1_f64 { + return format!("{}{}{}", negative, size, "B"); + } + let delimiter = 1024_f64; + let exponent = std::cmp::min( + (size.ln() / delimiter.ln()).floor() as i32, + (units.len() - 1) as i32, + ); + let pretty_bytes = format!("{:.2}", size / delimiter.powi(exponent)) + .parse::() + .unwrap() + * 1_f64; + let unit = units[exponent as usize]; + format!("{negative}{pretty_bytes}{unit}") +} + +#[instrument( + name = "GET /api/scopes/:scope/packages/:package/versions/:version/dependencies/graph", + skip(req), + err, + fields(scope, package, version) +)] +pub async fn get_dependencies_graph_handler( + req: Request, +) -> ApiResult> { + let scope = req.param_scope()?; + let package = req.param_package()?; + let version = req.param_version()?; + Span::current().record("scope", &field::display(&scope)); + Span::current().record("package", &field::display(&package)); + Span::current().record("version", &field::display(&version)); + + let buckets = req.data::().unwrap().clone(); + let gcs_path = + crate::gcs_paths::version_metadata(&scope, &package, &version).into(); + let version_meta = buckets.modules_bucket.download(gcs_path).await?.unwrap(); + let version_meta = + serde_json::from_slice::(&version_meta)?; + + let registry_url = req.data::().unwrap().0.clone(); + + tokio::task::spawn_blocking(|| { + analyze_deps_tree( + registry_url, + scope, + package, + version, + buckets.modules_bucket, + version_meta.exports, + ) + }) + .await + .unwrap() + .unwrap(); + + Ok(vec![]) +} + #[instrument( name = "GET /api/scopes/:scope/packages/:package/publishing_tasks", skip(req), From 679bdca6a1a738cbc1e81aa194922ba91fdf991c Mon Sep 17 00:00:00 2001 From: crowlkats Date: Sun, 1 Dec 2024 09:39:23 +0100 Subject: [PATCH 04/25] clean up --- Cargo.lock | 1 + api/Cargo.toml | 1 + api/src/api/package.rs | 493 +++++++++++++++++------------------------ api/src/api/types.rs | 30 +++ 4 files changed, 231 insertions(+), 294 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f7e740d..ffc20896 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2973,6 +2973,7 @@ dependencies = [ "jsonc-parser", "jsonwebkey", "jsonwebtoken", + "lazy_static", "oauth2", "once_cell", "opentelemetry", diff --git a/api/Cargo.toml b/api/Cargo.toml index 9dd0107c..34e7991f 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -109,6 +109,7 @@ tree-sitter-rust = "0.21.2" tree-sitter-html = "0.20.3" tree-sitter-bash = "0.21.0" tree-sitter-xml = "0.6.4" +lazy_static = "1.5.0" [dev-dependencies] flate2 = "1" diff --git a/api/src/api/package.rs b/api/src/api/package.rs index b21b0c74..a7c316d7 100644 --- a/api/src/api/package.rs +++ b/api/src/api/package.rs @@ -1,14 +1,5 @@ // Copyright 2024 the JSR authors. All rights reserved. MIT license. -use std::borrow::Cow; -use std::collections::HashSet; -use std::fmt::Write; -use std::sync::atomic::AtomicU64; -use std::sync::atomic::Ordering; -use std::sync::Arc; -use std::sync::Mutex; -use std::{fmt, io}; - -use anyhow::{bail, Context, Error as AnyError}; +use anyhow::Context; use chrono::Utc; use comrak::adapters::SyntaxHighlighterAdapter; use deno_ast::{MediaType, ModuleSpecifier, ParseDiagnostic}; @@ -16,8 +7,8 @@ use deno_graph::source::{ load_data_url, JsrUrlProvider, LoadOptions, NullFileSystem, }; use deno_graph::{ - BuildOptions, CapturingModuleAnalyzer, GraphKind, Module, ModuleError, - ModuleInfo, Resolution, WorkspaceMember, + BuildOptions, CapturingModuleAnalyzer, GraphKind, Module, ModuleInfo, + Resolution, WorkspaceMember, }; use futures::future::Either; use futures::StreamExt; @@ -27,10 +18,18 @@ use hyper::Request; use hyper::Response; use hyper::StatusCode; use indexmap::IndexMap; +use regex::Regex; use routerify::prelude::RequestExt; use routerify::Router; use routerify_query::RequestQueryExt; +use serde::{Deserialize, Serialize}; use sha2::Digest; +use std::borrow::Cow; +use std::io; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::sync::Mutex; use tracing::error; use tracing::field; use tracing::instrument; @@ -38,7 +37,8 @@ use tracing::Instrument; use tracing::Span; use url::Url; -use crate::analysis::{JsrResolver, ModuleAnalyzer, ModuleParser}; +use crate::analysis::JsrResolver; +use crate::analysis::ModuleParser; use crate::auth::access_token; use crate::auth::GithubOauth2Client; use crate::buckets::Buckets; @@ -78,7 +78,6 @@ use crate::util::VersionOrLatest; use crate::NpmUrl; use crate::RegistryUrl; -use super::ApiCreatePackageRequest; use super::ApiDependency; use super::ApiDependent; use super::ApiDownloadDataPoint; @@ -102,6 +101,7 @@ use super::ApiStats; use super::ApiUpdatePackageGithubRepositoryRequest; use super::ApiUpdatePackageRequest; use super::ApiUpdatePackageVersionRequest; +use super::{ApiCreatePackageRequest, ApiDependencyGraphItem}; const MAX_PUBLISH_TARBALL_SIZE: u64 = 20 * 1024 * 1024; // 20mb @@ -1582,23 +1582,40 @@ impl DepTreeLoader { } .boxed() } - "http" | "https" => async move { - // TODO: dont use reqwest, call to bucket directly. - let s = reqwest::Client::builder() - .build() - .unwrap() - .get(specifier.clone()) - .send() - .await - .unwrap(); - let bytes = s.bytes().await.unwrap(); - Ok(Some(deno_graph::source::LoadResponse::Module { - content: bytes.to_vec().into(), - specifier: specifier.clone(), - maybe_headers: None, - })) + "http" | "https" => { + let bucket = self.bucket.clone(); + async move { + let jsr_matches = JSR_DEP_PATH_RE.captures(specifier.path()).unwrap(); + + let scope = jsr_matches.name("scope").unwrap(); + let package = jsr_matches.name("package").unwrap(); + let version = jsr_matches.name("version"); + let path = jsr_matches.name("path").unwrap(); + + let Some(bytes) = bucket + .download( + format!( + "@{}/{}/{}{}", + scope.as_str(), + package.as_str(), + version.map(|version| version.as_str()).unwrap_or_default(), + path.as_str() + ) + .into(), + ) + .await? + else { + return Ok(None); + }; + + Ok(Some(deno_graph::source::LoadResponse::Module { + content: bytes.to_vec().into(), + specifier: specifier.clone(), + maybe_headers: None, + })) + } + .boxed() } - .boxed(), "jsr" => unreachable!("{specifier}"), // TODO: handle npm specifiers "npm" | "node" | "bun" => async move { @@ -1684,6 +1701,10 @@ impl deno_graph::ModuleAnalyzer for DepTreeAnalyzer { } } +lazy_static::lazy_static! { + static ref JSR_DEP_PATH_RE: Regex = Regex::new(r"/@(?.+?)/(?.+?)(?:/(?.+?))?(?/.+)").unwrap(); +} + // We have to spawn another tokio runtime, because // `deno_graph::ModuleGraph::build` is not thread-safe. #[tokio::main(flavor = "current_thread")] @@ -1694,7 +1715,10 @@ async fn analyze_deps_tree( version: crate::ids::Version, bucket: crate::buckets::BucketWithQueue, exports: IndexMap, -) -> Result<(), deno_graph::ModuleGraphError> { +) -> Result< + IndexMap, + deno_graph::ModuleGraphError, +> { let roots = exports .values() .map(|path| Url::parse(&format!("file://{}", path)).unwrap()) @@ -1737,142 +1761,112 @@ async fn analyze_deps_tree( .await; graph.valid()?; - for root_specifier in roots { - let mut output = String::new(); - GraphDisplayContext::write(&graph, &mut output, &root_specifier).unwrap(); - println!("{output}"); + let mut index = 0; + let mut dependencies = Default::default(); + + for root in roots { + GraphDependencyCollector::collect( + &graph, + &root, + &mut index, + &mut dependencies, + ); } - Ok(()) + Ok(dependencies) } -struct GraphDisplayContext<'a> { +struct GraphDependencyCollector<'a> { graph: &'a deno_graph::ModuleGraph, - seen: HashSet, - root: &'a ModuleSpecifier, + dependencies: &'a mut IndexMap, + id_index: &'a mut usize, } -impl<'a> GraphDisplayContext<'a> { - pub fn write( +impl<'a> GraphDependencyCollector<'a> { + pub fn collect( graph: &'a deno_graph::ModuleGraph, - writer: &mut TWrite, root: &'a ModuleSpecifier, - ) -> Result<(), AnyError> { + id_index: &'a mut usize, + dependencies: &'a mut IndexMap, + ) { + let root_module = graph.try_get(root).unwrap().unwrap(); + Self { graph, - seen: Default::default(), - root, + dependencies, + id_index, } - .into_writer(writer) + .build_module_info(root_module) + .unwrap(); } - fn into_writer( - mut self, - writer: &mut TWrite, - ) -> Result<(), AnyError> { - let root_specifier = self.graph.resolve(&self.root); - match self.graph.try_get(root_specifier) { - Ok(Some(root)) => { - let maybe_cache_info = match root { - Module::Js(module) => module.maybe_cache_info.as_ref(), - Module::Json(module) => module.maybe_cache_info.as_ref(), - Module::Node(_) | Module::Npm(_) | Module::External(_) => None, - }; - if let Some(cache_info) = maybe_cache_info { - if let Some(local) = &cache_info.local { - writeln!(writer, "{} {}", "local:", local.to_string_lossy())?; + fn build_module_info(&mut self, module: &Module) -> Option { + let specifier = module.specifier(); + + let dependency = match module { + Module::Js(_) | Module::Json(_) => { + if let Some(jsr_matches) = JSR_DEP_PATH_RE.captures(specifier.as_str()) + { + let scope = jsr_matches.name("scope").unwrap(); + let package = jsr_matches.name("package").unwrap(); + let version = jsr_matches.name("version").unwrap(); + let path = jsr_matches.name("path").unwrap(); + + DependencyKind::Jsr { + scope: scope.as_str().to_string(), + package: package.as_str().to_string(), + version: version.as_str().to_string(), + path: path.as_str().to_string(), } - } - if let Some(module) = root.js() { - writeln!(writer, "{} {}", "type:", module.media_type)?; - } - let total_modules_size = self - .graph - .modules() - .map(|m| { - let size = match m { - Module::Js(module) => module.size(), - Module::Json(module) => module.size(), - Module::Node(_) | Module::Npm(_) | Module::External(_) => 0, - }; - size as f64 - }) - .sum::(); - let dep_count = self.graph.modules().count() - 1 // -1 for the root module - ; - writeln!(writer, "{} {} unique", "dependencies:", dep_count,)?; - writeln!(writer, "{} {}", "size:", total_modules_size,)?; - writeln!(writer)?; - let root_node = self.build_module_info(root, false); - print_tree_node(&root_node, writer)?; - Ok(()) - } - Err(err) => { - if let ModuleError::Missing(_, _) = *err { - bail!("module could not be found"); } else { - bail!("{:#}", err); + DependencyKind::Root { + path: specifier.path().to_string(), + } } } - Ok(None) => { - bail!("an internal error occurred"); + Module::Npm(_) => { + return None; } - } - } - - fn build_dep_info(&mut self, dep: &deno_graph::Dependency) -> Vec { - let mut children = Vec::with_capacity(2); - if !dep.maybe_code.is_none() { - if let Some(child) = self.build_resolved_info(&dep.maybe_code, false) { - children.push(child); - } - } - if !dep.maybe_type.is_none() { - if let Some(child) = self.build_resolved_info(&dep.maybe_type, true) { - children.push(child); + Module::Node(_) | Module::External(_) => { + return None; } - } - children - } + }; - fn build_module_info(&mut self, module: &Module, type_dep: bool) -> TreeNode { - let specifier = module.specifier(); - let was_seen = !self.seen.insert(specifier.to_string()); - let header_text = if was_seen { - let specifier_str = if type_dep { - module.specifier().to_string() - } else { - module.specifier().to_string() - }; - format!("{} {}", specifier_str, "*") + if let Some(info) = self.dependencies.get(&dependency) { + Some(info.id) } else { - let header_text = if type_dep { - module.specifier().to_string() - } else { - module.specifier().to_string() - }; let maybe_size = match module { Module::Js(module) => Some(module.size() as u64), Module::Json(module) => Some(module.size() as u64), Module::Node(_) | Module::Npm(_) | Module::External(_) => None, }; - format!("{} {}", header_text, maybe_size_to_text(maybe_size)) - }; - let mut tree_node = TreeNode::from_text(header_text); + let media_type = match module { + Module::Js(js) => Some(js.media_type), + Module::Json(json) => Some(json.media_type), + Module::Npm(_) | Module::Node(_) | Module::External(_) => None, + }; - if !was_seen { + let mut children = vec![]; match module { Module::Js(module) => { if let Some(types_dep) = &module.maybe_types_dependency { - if let Some(child) = - self.build_resolved_info(&types_dep.dependency, true) + if let Some(child) = self.build_resolved_info(&types_dep.dependency) { - tree_node.children.push(child); + children.push(child); } } for dep in module.dependencies.values() { - tree_node.children.extend(self.build_dep_info(dep)); + if !dep.maybe_code.is_none() { + if let Some(child) = self.build_resolved_info(&dep.maybe_code) { + children.push(child); + } + } + if !dep.maybe_type.is_none() { + if let Some(child) = self.build_resolved_info(&dep.maybe_type) { + children.push(child); + } + } } } Module::Json(_) @@ -1880,183 +1874,85 @@ impl<'a> GraphDisplayContext<'a> { | Module::Node(_) | Module::External(_) => {} } - } - tree_node - } - fn build_error_info( - &mut self, - err: &ModuleError, - specifier: &ModuleSpecifier, - ) -> TreeNode { - self.seen.insert(specifier.to_string()); - match err { - ModuleError::InvalidTypeAssertion { .. } => { - self.build_error_msg(specifier, "(invalid import attribute)") - } - ModuleError::LoadingErr(_, _, err) => { - use deno_graph::ModuleLoadError::*; - let message = match err { - HttpsChecksumIntegrity(_) => "(checksum integrity error)", - Decode(_) => "(loading decode error)", - Loader(err) => "(loading error)", - Jsr(_) => "(loading error)", - NodeUnknownBuiltinModule(_) => "(unknown node built-in error)", - Npm(_) => "(npm loading error)", - TooManyRedirects => "(too many redirects error)", - }; - self.build_error_msg(specifier, message.as_ref()) - } - ModuleError::ParseErr(_, _) => { - self.build_error_msg(specifier, "(parsing error)") - } - ModuleError::UnsupportedImportAttributeType { .. } => { - self.build_error_msg(specifier, "(unsupported import attribute)") - } - ModuleError::UnsupportedMediaType { .. } => { - self.build_error_msg(specifier, "(unsupported)") - } - ModuleError::Missing(_, _) | ModuleError::MissingDynamic(_, _) => { - self.build_error_msg(specifier, "(missing)") - } - } - } + let id = *self.id_index; - fn build_error_msg( - &self, - specifier: &ModuleSpecifier, - error_msg: &str, - ) -> TreeNode { - TreeNode::from_text(format!("{specifier} {error_msg}",)) + self.dependencies.insert( + dependency, + DependencyInfo { + id, + children, + size: maybe_size, + media_type, + }, + ); + + *self.id_index += 1; + + Some(id) + } } - fn build_resolved_info( - &mut self, - resolution: &Resolution, - type_dep: bool, - ) -> Option { + fn build_resolved_info(&mut self, resolution: &Resolution) -> Option { match resolution { Resolution::Ok(resolved) => { let specifier = &resolved.specifier; let resolved_specifier = self.graph.resolve(specifier); - Some(match self.graph.try_get(resolved_specifier) { - Ok(Some(module)) => self.build_module_info(module, type_dep), - Err(err) => self.build_error_info(err, resolved_specifier), - Ok(None) => { - TreeNode::from_text(format!("{} {}", specifier, "(missing)")) + match self.graph.try_get(resolved_specifier) { + Ok(Some(module)) => self.build_module_info(module), + Err(err) => { + let id = *self.id_index; + + self.dependencies.insert( + DependencyKind::Error { + error: err.to_string(), + }, + DependencyInfo { + id, + children: vec![], + size: None, + media_type: None, + }, + ); + + *self.id_index += 1; + + Some(id) } - }) + Ok(None) => None, + } } - Resolution::Err(err) => Some(TreeNode::from_text(format!( - "{} {}", - err.to_string(), - "(resolve error)" - ))), _ => None, } } } -struct TreeNode { - text: String, - children: Vec, +#[derive(Serialize, Deserialize, Hash, Debug, Clone, Eq, PartialEq)] +#[serde(rename_all = "camelCase", tag = "type")] +pub enum DependencyKind { + Jsr { + scope: String, + package: String, + version: String, + path: String, + }, + Npm { + package: String, + }, + Root { + path: String, + }, + Error { + error: String, + }, } -impl TreeNode { - pub fn from_text(text: String) -> Self { - Self { - text, - children: Default::default(), - } - } -} - -fn maybe_size_to_text(maybe_size: Option) -> String { - format!( - "({})", - match maybe_size { - Some(size) => human_size(size as f64), - None => "unknown".to_string(), - } - ) -} - -fn print_tree_node( - tree_node: &TreeNode, - writer: &mut TWrite, -) -> fmt::Result { - fn print_children( - writer: &mut TWrite, - prefix: &str, - children: &[TreeNode], - ) -> fmt::Result { - const SIBLING_CONNECTOR: char = '├'; - const LAST_SIBLING_CONNECTOR: char = '└'; - const CHILD_DEPS_CONNECTOR: char = '┬'; - const CHILD_NO_DEPS_CONNECTOR: char = '─'; - const VERTICAL_CONNECTOR: char = '│'; - const EMPTY_CONNECTOR: char = ' '; - - let child_len = children.len(); - for (index, child) in children.iter().enumerate() { - let is_last = index + 1 == child_len; - let sibling_connector = if is_last { - LAST_SIBLING_CONNECTOR - } else { - SIBLING_CONNECTOR - }; - let child_connector = if child.children.is_empty() { - CHILD_NO_DEPS_CONNECTOR - } else { - CHILD_DEPS_CONNECTOR - }; - writeln!( - writer, - "{} {}", - format!("{prefix}{sibling_connector}─{child_connector}"), - child.text - )?; - let child_prefix = format!( - "{}{}{}", - prefix, - if is_last { - EMPTY_CONNECTOR - } else { - VERTICAL_CONNECTOR - }, - EMPTY_CONNECTOR - ); - print_children(writer, &child_prefix, &child.children)?; - } - - Ok(()) - } - - writeln!(writer, "{}", tree_node.text)?; - print_children(writer, "", &tree_node.children)?; - Ok(()) -} - -/// A function that converts a float to a string the represents a human -/// readable version of that number. -pub fn human_size(size: f64) -> String { - let negative = if size.is_sign_positive() { "" } else { "-" }; - let size = size.abs(); - let units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]; - if size < 1_f64 { - return format!("{}{}{}", negative, size, "B"); - } - let delimiter = 1024_f64; - let exponent = std::cmp::min( - (size.ln() / delimiter.ln()).floor() as i32, - (units.len() - 1) as i32, - ); - let pretty_bytes = format!("{:.2}", size / delimiter.powi(exponent)) - .parse::() - .unwrap() - * 1_f64; - let unit = units[exponent as usize]; - format!("{negative}{pretty_bytes}{unit}") +#[derive(Debug, Eq, PartialEq)] +pub struct DependencyInfo { + pub id: usize, + pub children: Vec, + pub size: Option, + pub media_type: Option, } #[instrument( @@ -2067,7 +1963,7 @@ pub fn human_size(size: f64) -> String { )] pub async fn get_dependencies_graph_handler( req: Request, -) -> ApiResult> { +) -> ApiResult> { let scope = req.param_scope()?; let package = req.param_package()?; let version = req.param_version()?; @@ -2084,7 +1980,7 @@ pub async fn get_dependencies_graph_handler( let registry_url = req.data::().unwrap().0.clone(); - tokio::task::spawn_blocking(|| { + let deps = tokio::task::spawn_blocking(|| { analyze_deps_tree( registry_url, scope, @@ -2098,7 +1994,16 @@ pub async fn get_dependencies_graph_handler( .unwrap() .unwrap(); - Ok(vec![]) + let api_deps = deps + .into_iter() + .enumerate() + .map(|(i, dep)| { + assert_eq!(i, dep.1.id); + ApiDependencyGraphItem::from(dep) + }) + .collect::>(); + + Ok(api_deps) } #[instrument( diff --git a/api/src/api/types.rs b/api/src/api/types.rs index f55d0e01..06851b33 100644 --- a/api/src/api/types.rs +++ b/api/src/api/types.rs @@ -81,6 +81,36 @@ impl From for ApiPublishingTask { } } +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct ApiDependencyGraphItem { + pub dependency: super::package::DependencyKind, + pub children: Vec, + pub size: Option, + pub media_type: Option, +} + +impl + From<( + super::package::DependencyKind, + super::package::DependencyInfo, + )> for ApiDependencyGraphItem +{ + fn from( + (kind, info): ( + super::package::DependencyKind, + super::package::DependencyInfo, + ), + ) -> Self { + Self { + dependency: kind, + children: info.children, + size: info.size, + media_type: info.media_type.map(|media_type| media_type.to_string()), + } + } +} + #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ApiUser { From 665311fc069363ed2d7301ee551c208102007f46 Mon Sep 17 00:00:00 2001 From: crowlkats Date: Sun, 8 Dec 2024 11:46:31 +0100 Subject: [PATCH 05/25] hook up backend and frontend --- .../package/(_islands)/DependencyGraph.tsx | 2 +- .../routes/package/dependencies/graph.tsx | 786 +----------------- 2 files changed, 6 insertions(+), 782 deletions(-) diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index 5ac5a288..13417ed5 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -38,7 +38,7 @@ type DependencyGraphKind = | DependencyGraphKindRoot | DependencyGraphKindError; -interface DependencyGraphItem { +export interface DependencyGraphItem { dependency: DependencyGraphKind; children: number[]; size: number | undefined; diff --git a/frontend/routes/package/dependencies/graph.tsx b/frontend/routes/package/dependencies/graph.tsx index f2d7058e..caacedfe 100644 --- a/frontend/routes/package/dependencies/graph.tsx +++ b/frontend/routes/package/dependencies/graph.tsx @@ -1,792 +1,16 @@ // Copyright 2024 the JSR authors. All rights reserved. MIT license. import { HttpError, type RouteConfig } from "fresh"; -import type { Dependency } from "../../../utils/api_types.ts"; import { path } from "../../../utils/api.ts"; import { scopeIAM } from "../../../utils/iam.ts"; import { define } from "../../../util.ts"; import { DependencyGraph, - DependencyGraphProps, + type DependencyGraphItem, } from "../(_islands)/DependencyGraph.tsx"; import { packageDataWithVersion } from "../../../utils/data.ts"; import { PackageHeader } from "../(_components)/PackageHeader.tsx"; import { PackageNav, type Params } from "../(_components)/PackageNav.tsx"; -const dependencies = [ - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "assert", - "version": "0.225.3", - "path": "/assertion_error.ts", - }, - "children": [], - "size": 484, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "assert", - "version": "0.225.3", - "path": "/assert.ts", - }, - "children": [ - 0, - ], - "size": 562, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "root", - "path": "/cookie.ts", - }, - "children": [ - 1, - ], - "size": 11310, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "encoding", - "version": "0.224.3", - "path": "/_validate_binary_like.ts", - }, - "children": [], - "size": 798, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "encoding", - "version": "0.224.3", - "path": "/base64.ts", - }, - "children": [ - 3, - ], - "size": 3336, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "root", - "path": "/etag.ts", - }, - "children": [ - 4, - ], - "size": 6579, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "root", - "path": "/status.ts", - }, - "children": [], - "size": 13575, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "root", - "path": "/_negotiation/common.ts", - }, - "children": [], - "size": 1801, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "root", - "path": "/_negotiation/encoding.ts", - }, - "children": [ - 7, - ], - "size": 4301, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "root", - "path": "/_negotiation/language.ts", - }, - "children": [ - 7, - ], - "size": 4150, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "root", - "path": "/_negotiation/media_type.ts", - }, - "children": [ - 7, - ], - "size": 4970, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "root", - "path": "/negotiation.ts", - }, - "children": [ - 8, - 9, - 10, - ], - "size": 6414, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "async", - "version": "0.224.1", - "path": "/delay.ts", - }, - "children": [], - "size": 1895, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "root", - "path": "/server.ts", - }, - "children": [ - 12, - ], - "size": 25885, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "encoding", - "version": "0.224.3", - "path": "/hex.ts", - }, - "children": [ - 3, - ], - "size": 3097, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "root", - "path": "/unstable_signed_cookie.ts", - }, - "children": [ - 14, - ], - "size": 3687, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "root", - "path": "/server_sent_event_stream.ts", - }, - "children": [], - "size": 2761, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "root", - "path": "/user_agent.ts", - }, - "children": [ - 1, - ], - "size": 36299, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/_common/assert_path.ts", - }, - "children": [], - "size": 307, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/_common/normalize.ts", - }, - "children": [ - 18, - ], - "size": 263, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/_common/constants.ts", - }, - "children": [], - "size": 2020, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/_common/normalize_string.ts", - }, - "children": [ - 20, - ], - "size": 2301, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/posix/_util.ts", - }, - "children": [ - 20, - ], - "size": 391, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/posix/normalize.ts", - }, - "children": [ - 19, - 21, - 22, - ], - "size": 1056, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/posix/join.ts", - }, - "children": [ - 18, - 23, - ], - "size": 721, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/_os.ts", - }, - "children": [], - "size": 705, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/posix/extname.ts", - }, - "children": [ - 20, - 18, - 22, - ], - "size": 2186, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/windows/_util.ts", - }, - "children": [ - 20, - ], - "size": 828, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/windows/extname.ts", - }, - "children": [ - 20, - 18, - 27, - ], - "size": 2342, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/extname.ts", - }, - "children": [ - 25, - 26, - 28, - ], - "size": 547, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/windows/normalize.ts", - }, - "children": [ - 19, - 20, - 21, - 27, - ], - "size": 3786, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/windows/join.ts", - }, - "children": [ - 1, - 18, - 27, - 30, - ], - "size": 2483, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/join.ts", - }, - "children": [ - 25, - 24, - 31, - ], - "size": 510, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/posix/resolve.ts", - }, - "children": [ - 21, - 18, - 22, - ], - "size": 1586, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/_common/relative.ts", - }, - "children": [ - 18, - ], - "size": 287, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/posix/relative.ts", - }, - "children": [ - 22, - 33, - 34, - ], - "size": 3000, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/windows/resolve.ts", - }, - "children": [ - 20, - 21, - 18, - 27, - ], - "size": 4848, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/windows/relative.ts", - }, - "children": [ - 20, - 36, - 34, - ], - "size": 3978, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/relative.ts", - }, - "children": [ - 25, - 35, - 37, - ], - "size": 788, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/resolve.ts", - }, - "children": [ - 25, - 33, - 36, - ], - "size": 528, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "path", - "version": "0.225.1", - "path": "/constants.ts", - }, - "children": [ - 25, - ], - "size": 348, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "media-types", - "version": "1.0.0-rc.1", - "path": "/_util.ts", - }, - "children": [], - "size": 3253, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "media-types", - "version": "1.0.0-rc.1", - "path": "/parse_media_type.ts", - }, - "children": [ - 41, - ], - "size": 3636, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "media-types", - "version": "1.0.0-rc.1", - "path": "/vendor/mime-db.v1.52.0.ts", - }, - "children": [], - "size": 186498, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "media-types", - "version": "1.0.0-rc.1", - "path": "/_db.ts", - }, - "children": [ - 43, - 41, - ], - "size": 1347, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "media-types", - "version": "1.0.0-rc.1", - "path": "/get_charset.ts", - }, - "children": [ - 42, - 41, - 44, - ], - "size": 1497, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "media-types", - "version": "1.0.0-rc.1", - "path": "/format_media_type.ts", - }, - "children": [ - 41, - ], - "size": 2539, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "media-types", - "version": "1.0.0-rc.1", - "path": "/type_by_extension.ts", - }, - "children": [ - 44, - ], - "size": 1203, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "media-types", - "version": "1.0.0-rc.1", - "path": "/content_type.ts", - }, - "children": [ - 42, - 45, - 46, - 44, - 47, - ], - "size": 3552, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "streams", - "version": "0.224.2", - "path": "/byte_slice_stream.ts", - }, - "children": [ - 1, - ], - "size": 2657, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "cli", - "version": "0.224.4", - "path": "/parse_args.ts", - }, - "children": [ - 1, - ], - "size": 22373, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "fmt", - "version": "0.225.2", - "path": "/colors.ts", - }, - "children": [], - "size": 21644, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "root", - "path": "/deno.json", - }, - "children": [], - "size": 461, - "mediaType": "Json", - }, - { - "dependency": { - "type": "jsr", - "scope": "std", - "package": "fmt", - "version": "0.225.2", - "path": "/bytes.ts", - }, - "children": [], - "size": 4665, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "root", - "path": "/file_server.ts", - }, - "children": [ - 24, - 23, - 29, - 32, - 38, - 39, - 40, - 48, - 5, - 6, - 49, - 50, - 51, - 52, - 53, - ], - "size": 25534, - "mediaType": "TypeScript", - }, - { - "dependency": { - "type": "root", - "path": "/mod.ts", - }, - "children": [ - 2, - 5, - 6, - 11, - 13, - 15, - 16, - 17, - 54, - ], - "size": 2380, - "mediaType": "TypeScript", - }, -] as const satisfies DependencyGraphProps["dependencies"]; - export default define.page( function DepsGraph({ data, params, state }) { const iam = scopeIAM(state, data.member); @@ -807,7 +31,7 @@ export default define.page( />
- +
); @@ -844,13 +68,13 @@ export const handler = define.handlers({ }); } - const depsResp = await ctx.state.api.get( - path`/scopes/${pkg.scope}/packages/${pkg.name}/versions/${selectedVersion.version}/dependencies`, + const depsResp = await ctx.state.api.get( + path`/scopes/${pkg.scope}/packages/${pkg.name}/versions/${selectedVersion.version}/dependencies/graph`, ); if (!depsResp.ok) throw depsResp; ctx.state.meta = { - title: `Dependencies - @${pkg.scope}/${pkg.name} - JSR`, + title: `Dependencies Graph - @${pkg.scope}/${pkg.name} - JSR`, description: `@${pkg.scope}/${pkg.name} on JSR${ pkg.description ? `: ${pkg.description}` : "" }`, From c67a6f72ab01c38954c79c0fafcc15d60f6fa9c2 Mon Sep 17 00:00:00 2001 From: crowlkats Date: Sun, 8 Dec 2024 12:13:12 +0100 Subject: [PATCH 06/25] add scroll to zoom and drag to pan --- .../package/(_islands)/DependencyGraph.tsx | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index 13417ed5..35a10b89 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -173,10 +173,37 @@ function GraphControlButton(props: GraphControlButtonProps) { export function DependencyGraph(props: DependencyGraphProps) { const { pan, zoom, reset, ref } = useDigraph(props.dependencies); + const dragActive = useSignal(false); + + function enableDrag() { + dragActive.value = true; + } + function disableDrag() { + dragActive.value = false; + } + + function OnMouseMove(event) { + if (dragActive.value) { + pan(event.movementX, event.movementY); + } + } + + function wheelZoom(event) { + event.preventDefault(); + // TODO: zoom on pointer + zoom(event.deltaY / 250); + } return (
-
+
{/* zoom */} Date: Sun, 8 Dec 2024 13:42:15 +0100 Subject: [PATCH 07/25] feat: add rankdir to digraph --- frontend/routes/package/(_islands)/DependencyGraph.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index 35a10b89..70a0652a 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -51,6 +51,7 @@ export interface DependencyGraphProps { function createDigraph(dependencies: DependencyGraphProps["dependencies"]) { return `digraph "dependencies" { + graph [rankdir="LR"] node [fontname="Courier", shape="box"] ${ @@ -182,13 +183,13 @@ export function DependencyGraph(props: DependencyGraphProps) { dragActive.value = false; } - function OnMouseMove(event) { + function onMouseMove(event: MouseEvent) { if (dragActive.value) { pan(event.movementX, event.movementY); } } - function wheelZoom(event) { + function wheelZoom(event: WheelEvent) { event.preventDefault(); // TODO: zoom on pointer zoom(event.deltaY / 250); @@ -199,7 +200,7 @@ export function DependencyGraph(props: DependencyGraphProps) {
Date: Sun, 8 Dec 2024 18:04:20 +0100 Subject: [PATCH 08/25] cleanup dependency render and fixes --- .../package/(_islands)/DependencyGraph.tsx | 74 ++++++++++--------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index 70a0652a..2b3280fe 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -55,9 +55,9 @@ function createDigraph(dependencies: DependencyGraphProps["dependencies"]) { node [fontname="Courier", shape="box"] ${ - dependencies.map(({ children, dependency }, index) => { + dependencies.map(({ children, size, dependency }, index) => { return [ - ` ${index} ${renderDependency(dependency)}`, + ` ${index} ${renderDependency(dependency, size)}`, ...children.map((child) => ` ${index} -> ${child}`), ].filter(Boolean).join("\n"); }).join("\n") @@ -65,39 +65,43 @@ ${ }`; } -function renderDependency(dependency: DependencyGraphKind) { +function bytesToSize(bytes: number) { + const sizes = ["B", "KB", "MB", "GB", "TB"]; + if (bytes == 0) return "0 B"; + const i = Math.floor(Math.log(bytes) / Math.log(1024)); + return (bytes / Math.pow(1024, i)).toFixed(0) + " " + sizes[i]; +} + +function renderDependency(dependency: DependencyGraphKind, size?: number) { + let href; + let content; + let tooltip; + let color; switch (dependency.type) { - case "jsr": - return renderJsrDependency(dependency); - case "npm": - return renderNpmDependency(dependency); - case "root": - return renderRootDependency(dependency); + case "jsr": { + tooltip = `@${dependency.scope}/${dependency.package}@${dependency.version}`; + href = `/${tooltip}`; + content = tooltip + `\n${dependency.path}\n${bytesToSize(size ?? 0)}`; + color = "#faee4a"; + break; + } + case "npm": { + content = tooltip = `${dependency.package}@${dependency.version}`; + href = `https://www.npmjs.com/package/${dependency.package}`; + color = "#cb3837"; + break; + } + case "root": { + tooltip = content = dependency.path; + color = "#67bef9"; + break; + } case "error": default: return renderErrorDependency(dependency); } -} - -function renderJsrDependency(dependency: DependencyGraphKindJsr) { - const label = - `@${dependency.scope}/${dependency.package}@${dependency.version}`; - const href = `/${label}`; - - return `[href="${href}", label="${label}", tooltip="${label}"]`; -} -function renderNpmDependency(dependency: DependencyGraphKindNpm) { - const label = `${dependency.package}@${dependency.version}`; - const href = `https://www.npmjs.com/package/${dependency.package}`; - - return `[href="${href}", label="${label}", tooltip="${label}"]`; -} - -function renderRootDependency(dependency: DependencyGraphKindRoot) { - const label = dependency.path; - - return `[label="${label}", tooltip="${label}"]`; + return `[href="${href}", label="${content}", tooltip="${tooltip}", style="filled,rounded", color="${color}"]`; } function renderErrorDependency(dependency: DependencyGraphKindError) { @@ -122,10 +126,12 @@ function useDigraph(dependencies: DependencyGraphProps["dependencies"]) { }, [controls]); const zoom = useCallback((zoom: number) => { - controls.value.zoom += zoom; - if (svg.current) { - svg.current.style.transform = - `translate(${controls.value.pan.x}px, ${controls.value.pan.y}px) scale(${controls.value.zoom})`; + if (controls.value.zoom + zoom > 0) { + controls.value.zoom += zoom; + if (svg.current) { + svg.current.style.transform = + `translate(${controls.value.pan.x}px, ${controls.value.pan.y}px) scale(${controls.value.zoom})`; + } } }, [controls]); @@ -196,7 +202,7 @@ export function DependencyGraph(props: DependencyGraphProps) { } return ( -
+
Date: Sun, 8 Dec 2024 18:25:57 +0100 Subject: [PATCH 09/25] cleanup code --- .../package/(_islands)/DependencyGraph.tsx | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index 2b3280fe..044389fe 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -52,7 +52,7 @@ export interface DependencyGraphProps { function createDigraph(dependencies: DependencyGraphProps["dependencies"]) { return `digraph "dependencies" { graph [rankdir="LR"] - node [fontname="Courier", shape="box"] + node [fontname="Courier", shape="box", style="filled,rounded"] ${ dependencies.map(({ children, size, dependency }, index) => { @@ -79,7 +79,8 @@ function renderDependency(dependency: DependencyGraphKind, size?: number) { let color; switch (dependency.type) { case "jsr": { - tooltip = `@${dependency.scope}/${dependency.package}@${dependency.version}`; + tooltip = + `@${dependency.scope}/${dependency.package}@${dependency.version}`; href = `/${tooltip}`; content = tooltip + `\n${dependency.path}\n${bytesToSize(size ?? 0)}`; color = "#faee4a"; @@ -101,7 +102,7 @@ function renderDependency(dependency: DependencyGraphKind, size?: number) { return renderErrorDependency(dependency); } - return `[href="${href}", label="${content}", tooltip="${tooltip}", style="filled,rounded", color="${color}"]`; + return `[href="${href}", label="${content}", tooltip="${tooltip}", color="${color}"]`; } function renderErrorDependency(dependency: DependencyGraphKindError) { @@ -126,12 +127,14 @@ function useDigraph(dependencies: DependencyGraphProps["dependencies"]) { }, [controls]); const zoom = useCallback((zoom: number) => { - if (controls.value.zoom + zoom > 0) { - controls.value.zoom += zoom; - if (svg.current) { - svg.current.style.transform = - `translate(${controls.value.pan.x}px, ${controls.value.pan.y}px) scale(${controls.value.zoom})`; - } + controls.value.zoom = Math.max( + 0.1, + Math.min(controls.value.zoom + zoom, 2.5), + ); + + if (svg.current) { + svg.current.style.transform = + `translate(${controls.value.pan.x}px, ${controls.value.pan.y}px) scale(${controls.value.zoom})`; } }, [controls]); @@ -202,7 +205,7 @@ export function DependencyGraph(props: DependencyGraphProps) { } return ( -
+
Date: Sun, 8 Dec 2024 20:17:59 +0100 Subject: [PATCH 10/25] chore: center graph with sensible defaults on load --- .../package/(_islands)/DependencyGraph.tsx | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index 044389fe..8778f634 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -117,6 +117,22 @@ function useDigraph(dependencies: DependencyGraphProps["dependencies"]) { const svg = useRef(null); const viz = useSignal(undefined); + const center = useCallback(() => { + if (svg.current && ref.current) { + const { width: sWidth, height: sHeight } = svg.current + .getBoundingClientRect(); + const { width: rWidth, height: rHeight } = ref.current + .getBoundingClientRect(); + + controls.value.pan.x = (rWidth - sWidth) / 2; + controls.value.pan.y = (rHeight - sHeight) / 2; + controls.value.zoom = Math.min(rWidth / sWidth, rHeight / sHeight); + + svg.current.style.transform = + `translate(${controls.value.pan.x}px, ${controls.value.pan.y}px) scale(${controls.value.zoom})`; + } + }, []); + const pan = useCallback((x: number, y: number) => { controls.value.pan.x += x; controls.value.pan.y += y; @@ -153,7 +169,9 @@ function useDigraph(dependencies: DependencyGraphProps["dependencies"]) { const digraph = createDigraph(dependencies); svg.current = viz.value.renderSVGElement(digraph); - ref.current.appendChild(svg.current); + ref.current.prepend(svg.current); + + center(); } })(); }, [dependencies]); @@ -205,15 +223,15 @@ export function DependencyGraph(props: DependencyGraphProps) { } return ( -
-
+
{/* zoom */} Date: Sun, 8 Dec 2024 20:42:34 +0100 Subject: [PATCH 11/25] chore: tweak reset --- .../package/(_islands)/DependencyGraph.tsx | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index 8778f634..06411843 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -113,6 +113,7 @@ function renderErrorDependency(dependency: DependencyGraphKindError) { function useDigraph(dependencies: DependencyGraphProps["dependencies"]) { const controls = useSignal({ pan: { x: 0, y: 0 }, zoom: 1 }); + const defaults = useSignal({ pan: { x: 0, y: 0 }, zoom: 1 }); const ref = useRef(null); const svg = useRef(null); const viz = useSignal(undefined); @@ -124,10 +125,10 @@ function useDigraph(dependencies: DependencyGraphProps["dependencies"]) { const { width: rWidth, height: rHeight } = ref.current .getBoundingClientRect(); - controls.value.pan.x = (rWidth - sWidth) / 2; - controls.value.pan.y = (rHeight - sHeight) / 2; - controls.value.zoom = Math.min(rWidth / sWidth, rHeight / sHeight); - + defaults.value.pan.x = (rWidth - sWidth) / 2; + defaults.value.pan.y = (rHeight - sHeight) / 2; + defaults.value.zoom = Math.min(rWidth / sWidth, rHeight / sHeight); + controls.value = { ...defaults.value }; svg.current.style.transform = `translate(${controls.value.pan.x}px, ${controls.value.pan.y}px) scale(${controls.value.zoom})`; } @@ -140,7 +141,7 @@ function useDigraph(dependencies: DependencyGraphProps["dependencies"]) { svg.current.style.transform = `translate(${controls.value.pan.x}px, ${controls.value.pan.y}px) scale(${controls.value.zoom})`; } - }, [controls]); + }, []); const zoom = useCallback((zoom: number) => { controls.value.zoom = Math.max( @@ -152,12 +153,13 @@ function useDigraph(dependencies: DependencyGraphProps["dependencies"]) { svg.current.style.transform = `translate(${controls.value.pan.x}px, ${controls.value.pan.y}px) scale(${controls.value.zoom})`; } - }, [controls]); + }, []); const reset = useCallback(() => { - controls.value = { pan: { x: 0, y: 0 }, zoom: 1 }; + controls.value = { ...defaults.value }; if (svg.current) { - svg.current.style.transform = ""; + svg.current.style.transform = + `translate(${controls.value.pan.x}px, ${controls.value.pan.y}px) scale(${controls.value.zoom})`; } }, []); From 122ed7b9b04eae529ebeb83d0f52e94e23fe7915 Mon Sep 17 00:00:00 2001 From: Michael Herzner Date: Sun, 8 Dec 2024 20:47:16 +0100 Subject: [PATCH 12/25] chore: tweak reset --- frontend/routes/package/(_islands)/DependencyGraph.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index 06411843..0edb745a 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -55,7 +55,7 @@ function createDigraph(dependencies: DependencyGraphProps["dependencies"]) { node [fontname="Courier", shape="box", style="filled,rounded"] ${ - dependencies.map(({ children, size, dependency }, index) => { + dependencies.map(({ children, dependency, size }, index) => { return [ ` ${index} ${renderDependency(dependency, size)}`, ...children.map((child) => ` ${index} -> ${child}`), @@ -82,7 +82,7 @@ function renderDependency(dependency: DependencyGraphKind, size?: number) { tooltip = `@${dependency.scope}/${dependency.package}@${dependency.version}`; href = `/${tooltip}`; - content = tooltip + `\n${dependency.path}\n${bytesToSize(size ?? 0)}`; + content = `${tooltip}\n${dependency.path}\n${bytesToSize(size ?? 0)}`; color = "#faee4a"; break; } @@ -93,7 +93,7 @@ function renderDependency(dependency: DependencyGraphKind, size?: number) { break; } case "root": { - tooltip = content = dependency.path; + content = tooltip = dependency.path; color = "#67bef9"; break; } @@ -128,7 +128,7 @@ function useDigraph(dependencies: DependencyGraphProps["dependencies"]) { defaults.value.pan.x = (rWidth - sWidth) / 2; defaults.value.pan.y = (rHeight - sHeight) / 2; defaults.value.zoom = Math.min(rWidth / sWidth, rHeight / sHeight); - controls.value = { ...defaults.value }; + controls.value = structuredClone(defaults.value); svg.current.style.transform = `translate(${controls.value.pan.x}px, ${controls.value.pan.y}px) scale(${controls.value.zoom})`; } @@ -156,7 +156,7 @@ function useDigraph(dependencies: DependencyGraphProps["dependencies"]) { }, []); const reset = useCallback(() => { - controls.value = { ...defaults.value }; + controls.value = structuredClone(defaults.value); if (svg.current) { svg.current.style.transform = `translate(${controls.value.pan.x}px, ${controls.value.pan.y}px) scale(${controls.value.zoom})`; From a5893387d9c317e7e07a6cfbc760fbe7427affa5 Mon Sep 17 00:00:00 2001 From: Michael Herzner Date: Sun, 8 Dec 2024 23:06:43 +0100 Subject: [PATCH 13/25] chore: improve rendering of deps --- .../package/(_islands)/DependencyGraph.tsx | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index 0edb745a..7238dc97 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -99,16 +99,17 @@ function renderDependency(dependency: DependencyGraphKind, size?: number) { } case "error": default: - return renderErrorDependency(dependency); + content = tooltip = dependency.error; + break; } - return `[href="${href}", label="${content}", tooltip="${tooltip}", color="${color}"]`; -} - -function renderErrorDependency(dependency: DependencyGraphKindError) { - const label = dependency.error; - - return `[label="${label}", tooltip="${label}"]`; + return `[${ + Object + .entries({ href, tooltip, label: content, color }) + .filter(([_, v]) => v) + .map(([k, v]) => `${k}="${v}"`) + .join(", ") + }]`; } function useDigraph(dependencies: DependencyGraphProps["dependencies"]) { From 6fc672ac422a19214437aee1ca2f5a3ac69bec82 Mon Sep 17 00:00:00 2001 From: crowlkats Date: Mon, 9 Dec 2024 11:19:34 +0100 Subject: [PATCH 14/25] grouping of dependencies --- .../package/(_islands)/DependencyGraph.tsx | 163 ++++++++++++++++-- .../routes/package/dependencies/graph.tsx | 6 +- frontend/utils/api_types.ts | 34 ++++ 3 files changed, 186 insertions(+), 17 deletions(-) diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index 7238dc97..647e9daf 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -10,13 +10,18 @@ import { ChevronUp } from "../../../components/icons/ChevronUp.tsx"; import { Minus } from "../../../components/icons/Minus.tsx"; import { Plus } from "../../../components/icons/Plus.tsx"; import { Reset } from "../../../components/icons/Reset.tsx"; +import { DependencyGraphItem } from "../../../utils/api_types.ts"; -interface DependencyGraphKindJsr { +export interface DependencyGraphProps { + dependencies: DependencyGraphItem[]; +} + +interface DependencyGraphKindGroupedJsr { type: "jsr"; scope: string; package: string; version: string; - path: string; + paths: string[]; } interface DependencyGraphKindNpm { type: "npm"; @@ -32,30 +37,157 @@ interface DependencyGraphKindError { error: string; } -type DependencyGraphKind = - | DependencyGraphKindJsr +type GroupedDependencyGraphKind = + | DependencyGraphKindGroupedJsr | DependencyGraphKindNpm | DependencyGraphKindRoot | DependencyGraphKindError; -export interface DependencyGraphItem { - dependency: DependencyGraphKind; +export interface GroupedDependencyGraphItem { + dependency: GroupedDependencyGraphKind; children: number[]; size: number | undefined; mediaType: string | undefined; } -export interface DependencyGraphProps { - dependencies: DependencyGraphItem[]; +interface JsrPackage { + scope: string; + package: string; + version: string; } -function createDigraph(dependencies: DependencyGraphProps["dependencies"]) { +export function groupDependencies( + items: DependencyGraphItem[], +): GroupedDependencyGraphItem[] { + const referencedBy = new Map>(); + for (let i = 0; i < items.length; i++) { + for (const child of items[i].children) { + if (!referencedBy.has(child)) { + referencedBy.set(child, new Set()); + } + referencedBy.get(child)!.add(i); + } + } + + const jsrGroups = new Map(); + + for (let i = 0; i < items.length; i++) { + const item = items[i]; + if (item.dependency.type === "jsr") { + const groupKey = + `${item.dependency.scope}/${item.dependency.package}@${item.dependency.version}`; + const group = jsrGroups.get(groupKey) ?? { + key: { + scope: item.dependency.scope, + package: item.dependency.package, + version: item.dependency.version, + }, + paths: [], + children: [], + size: undefined, + mediaType: undefined, + oldIndices: [], + }; + group.paths.push({ path: item.dependency.path, oldIndex: i }); + group.children.push(...item.children); + if (item.size !== undefined) { + group.size ??= 0; + group.size += item.size; + } + group.oldIndices.push(i); + jsrGroups.set(groupKey, group); + } + } + + const oldIndexToNewIndex = new Map(); + const placedJsrGroups = new Set(); + const out: GroupedDependencyGraphItem[] = []; + + for (let i = 0; i < items.length; i++) { + const item = items[i]; + if (item.dependency.type === "jsr") { + const groupKey = + `${item.dependency.scope}/${item.dependency.package}@${item.dependency.version}`; + const group = jsrGroups.get(groupKey)!; + + if (!placedJsrGroups.has(groupKey)) { + placedJsrGroups.add(groupKey); + + const groupIndicesSet = new Set(group.oldIndices); + const filteredPaths = group.paths.filter(({ oldIndex }) => { + const refs = referencedBy.get(oldIndex)!; + + for (const ref of refs) { + if (!groupIndicesSet.has(ref)) { + return true; + } + } + + return false; // all references are from within the same jsr package + }).map((p) => p.path); + + const uniqueChildren = Array.from(new Set(group.children)); + const newIndex = out.length; + out.push({ + dependency: { + type: "jsr", + scope: group.key.scope, + package: group.key.package, + version: group.key.version, + paths: Array.from(new Set(filteredPaths)), + }, + children: uniqueChildren, + size: group.size, + mediaType: group.mediaType, + }); + + for (const oldIdx of group.oldIndices) { + oldIndexToNewIndex.set(oldIdx, newIndex); + } + } else { + oldIndexToNewIndex.set( + i, + oldIndexToNewIndex.get(jsrGroups.get(groupKey)!.oldIndices[0])!, + ); + } + } else { + out.push({ + dependency: item.dependency, + children: item.children, + size: item.size, + mediaType: item.mediaType, + }); + oldIndexToNewIndex.set(i, out.length - 1); + } + } + + for (let index = 0; index < out.length; index++) { + const newItem = out[index]; + const remappedChildren = newItem.children + .map((childIdx) => oldIndexToNewIndex.get(childIdx)!) + .filter((childNewIdx) => childNewIdx !== index); + newItem.children = Array.from(new Set(remappedChildren)); + } + + return out; +} + +function createDigraph(dependencies: DependencyGraphItem[]) { + const groupedDependencies = groupDependencies(dependencies); + return `digraph "dependencies" { graph [rankdir="LR"] node [fontname="Courier", shape="box", style="filled,rounded"] ${ - dependencies.map(({ children, dependency, size }, index) => { + groupedDependencies.map(({ children, dependency, size }, index) => { return [ ` ${index} ${renderDependency(dependency, size)}`, ...children.map((child) => ` ${index} -> ${child}`), @@ -72,7 +204,10 @@ function bytesToSize(bytes: number) { return (bytes / Math.pow(1024, i)).toFixed(0) + " " + sizes[i]; } -function renderDependency(dependency: DependencyGraphKind, size?: number) { +function renderDependency( + dependency: GroupedDependencyGraphKind, + size?: number, +) { let href; let content; let tooltip; @@ -82,7 +217,9 @@ function renderDependency(dependency: DependencyGraphKind, size?: number) { tooltip = `@${dependency.scope}/${dependency.package}@${dependency.version}`; href = `/${tooltip}`; - content = `${tooltip}\n${dependency.path}\n${bytesToSize(size ?? 0)}`; + content = `${tooltip}\n${dependency.paths.join("\n")}\n${ + bytesToSize(size ?? 0) + }`; color = "#faee4a"; break; } @@ -112,7 +249,7 @@ function renderDependency(dependency: DependencyGraphKind, size?: number) { }]`; } -function useDigraph(dependencies: DependencyGraphProps["dependencies"]) { +function useDigraph(dependencies: DependencyGraphItem[]) { const controls = useSignal({ pan: { x: 0, y: 0 }, zoom: 1 }); const defaults = useSignal({ pan: { x: 0, y: 0 }, zoom: 1 }); const ref = useRef(null); diff --git a/frontend/routes/package/dependencies/graph.tsx b/frontend/routes/package/dependencies/graph.tsx index caacedfe..5732b263 100644 --- a/frontend/routes/package/dependencies/graph.tsx +++ b/frontend/routes/package/dependencies/graph.tsx @@ -3,13 +3,11 @@ import { HttpError, type RouteConfig } from "fresh"; import { path } from "../../../utils/api.ts"; import { scopeIAM } from "../../../utils/iam.ts"; import { define } from "../../../util.ts"; -import { - DependencyGraph, - type DependencyGraphItem, -} from "../(_islands)/DependencyGraph.tsx"; +import { DependencyGraph } from "../(_islands)/DependencyGraph.tsx"; import { packageDataWithVersion } from "../../../utils/data.ts"; import { PackageHeader } from "../(_components)/PackageHeader.tsx"; import { PackageNav, type Params } from "../(_components)/PackageNav.tsx"; +import type { DependencyGraphItem } from "../../../utils/api_types.ts"; export default define.page( function DepsGraph({ data, params, state }) { diff --git a/frontend/utils/api_types.ts b/frontend/utils/api_types.ts index 7108a66d..10f7ac0d 100644 --- a/frontend/utils/api_types.ts +++ b/frontend/utils/api_types.ts @@ -266,3 +266,37 @@ export interface CreatedToken { token: Token; secret: string; } + +interface DependencyGraphKindJsr { + type: "jsr"; + scope: string; + package: string; + version: string; + path: string; +} +interface DependencyGraphKindNpm { + type: "npm"; + package: string; + version: string; +} +interface DependencyGraphKindRoot { + type: "root"; + path: string; +} +interface DependencyGraphKindError { + type: "error"; + error: string; +} + +type DependencyGraphKind = + | DependencyGraphKindJsr + | DependencyGraphKindNpm + | DependencyGraphKindRoot + | DependencyGraphKindError; + +export interface DependencyGraphItem { + dependency: DependencyGraphKind; + children: number[]; + size: number | undefined; + mediaType: string | undefined; +} From 5560a81609baa4fd3aaf4dbfcef2c1a1b0c5944f Mon Sep 17 00:00:00 2001 From: crowlkats Date: Mon, 9 Dec 2024 13:30:04 +0100 Subject: [PATCH 15/25] make more readable --- .../package/(_islands)/DependencyGraph.tsx | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index 647e9daf..026174a8 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -182,18 +182,32 @@ export function groupDependencies( function createDigraph(dependencies: DependencyGraphItem[]) { const groupedDependencies = groupDependencies(dependencies); - return `digraph "dependencies" { - graph [rankdir="LR"] - node [fontname="Courier", shape="box", style="filled,rounded"] + const nodesWithNoParent = new Set( + Object.keys(groupedDependencies).map(Number), + ); -${ - groupedDependencies.map(({ children, dependency, size }, index) => { + const depsGraph = groupedDependencies.map( + ({ children, dependency, size }, index) => { return [ ` ${index} ${renderDependency(dependency, size)}`, - ...children.map((child) => ` ${index} -> ${child}`), + ...children.map((child) => { + nodesWithNoParent.delete(child); + return ` ${index} -> ${child}`; + }), ].filter(Boolean).join("\n"); - }).join("\n") + }, + ).join("\n"); + + return `digraph "dependencies" { + graph [rankdir="LR", concentrate=true] + node [fontname="Courier", shape="box", style="filled,rounded"] + + { + rank=same + ${Array.from(nodesWithNoParent).join("; ")} } + + ${depsGraph} }`; } @@ -308,7 +322,9 @@ function useDigraph(dependencies: DependencyGraphItem[]) { if (ref.current && viz.value) { const digraph = createDigraph(dependencies); - svg.current = viz.value.renderSVGElement(digraph); + svg.current = viz.value.renderSVGElement(digraph, { + engine: "dot", + }); ref.current.prepend(svg.current); center(); From 7a589502296bfa16997633897b26a8180b3b0a6b Mon Sep 17 00:00:00 2001 From: Michael Herzner Date: Mon, 23 Dec 2024 00:14:21 +0100 Subject: [PATCH 16/25] chore: cleanup --- api/src/api/package.rs | 4 +--- .../package/(_islands)/DependencyGraph.tsx | 20 ++++++------------- frontend/utils/api_types.ts | 12 ++++++----- 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/api/src/api/package.rs b/api/src/api/package.rs index a7c316d7..bb2f6a85 100644 --- a/api/src/api/package.rs +++ b/api/src/api/package.rs @@ -3,9 +3,7 @@ use anyhow::Context; use chrono::Utc; use comrak::adapters::SyntaxHighlighterAdapter; use deno_ast::{MediaType, ModuleSpecifier, ParseDiagnostic}; -use deno_graph::source::{ - load_data_url, JsrUrlProvider, LoadOptions, NullFileSystem, -}; +use deno_graph::source::{JsrUrlProvider, LoadOptions, NullFileSystem}; use deno_graph::{ BuildOptions, CapturingModuleAnalyzer, GraphKind, Module, ModuleInfo, Resolution, WorkspaceMember, diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index 026174a8..6720ed95 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -10,7 +10,12 @@ import { ChevronUp } from "../../../components/icons/ChevronUp.tsx"; import { Minus } from "../../../components/icons/Minus.tsx"; import { Plus } from "../../../components/icons/Plus.tsx"; import { Reset } from "../../../components/icons/Reset.tsx"; -import { DependencyGraphItem } from "../../../utils/api_types.ts"; +import type { + DependencyGraphItem, + DependencyGraphKindError, + DependencyGraphKindNpm, + DependencyGraphKindRoot, +} from "../../../utils/api_types.ts"; export interface DependencyGraphProps { dependencies: DependencyGraphItem[]; @@ -23,19 +28,6 @@ interface DependencyGraphKindGroupedJsr { version: string; paths: string[]; } -interface DependencyGraphKindNpm { - type: "npm"; - package: string; - version: string; -} -interface DependencyGraphKindRoot { - type: "root"; - path: string; -} -interface DependencyGraphKindError { - type: "error"; - error: string; -} type GroupedDependencyGraphKind = | DependencyGraphKindGroupedJsr diff --git a/frontend/utils/api_types.ts b/frontend/utils/api_types.ts index 10f7ac0d..45bc377a 100644 --- a/frontend/utils/api_types.ts +++ b/frontend/utils/api_types.ts @@ -267,28 +267,30 @@ export interface CreatedToken { secret: string; } -interface DependencyGraphKindJsr { +export interface DependencyGraphKindJsr { type: "jsr"; scope: string; package: string; version: string; path: string; } -interface DependencyGraphKindNpm { + +export interface DependencyGraphKindNpm { type: "npm"; package: string; version: string; } -interface DependencyGraphKindRoot { +export interface DependencyGraphKindRoot { type: "root"; path: string; } -interface DependencyGraphKindError { + +export interface DependencyGraphKindError { type: "error"; error: string; } -type DependencyGraphKind = +export type DependencyGraphKind = | DependencyGraphKindJsr | DependencyGraphKindNpm | DependencyGraphKindRoot From 9b3c89d4a0ff55b4b7ea938221a9ef9ca82617f0 Mon Sep 17 00:00:00 2001 From: Michael Herzner Date: Mon, 23 Dec 2024 00:34:47 +0100 Subject: [PATCH 17/25] chore: add button to access deps graph --- .../routes/package/dependencies/index.tsx | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/frontend/routes/package/dependencies/index.tsx b/frontend/routes/package/dependencies/index.tsx index 7e6f97d8..e45e3458 100644 --- a/frontend/routes/package/dependencies/index.tsx +++ b/frontend/routes/package/dependencies/index.tsx @@ -82,24 +82,33 @@ export default define.page(function Deps(
) : ( - - {list.map(([name, info]) => ( - - ))} -
+ <> + + {list.map(([name, info]) => ( + + ))} +
+

+ You can find a visualization of the dependencies by clicking the + button below. +

+ + Dependency Graph + + )}
From 657d6dc34cf56935ad20d0269de85edca0654297 Mon Sep 17 00:00:00 2001 From: Michael Herzner Date: Tue, 31 Dec 2024 17:23:59 +0100 Subject: [PATCH 18/25] chore: prevent accidental click on links during pan events --- frontend/deno.lock | 1 + .../package/(_islands)/DependencyGraph.tsx | 26 ++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/frontend/deno.lock b/frontend/deno.lock index 0addb41b..59778b5b 100644 --- a/frontend/deno.lock +++ b/frontend/deno.lock @@ -31,6 +31,7 @@ "npm:@oramacloud/client@1": "1.3.20", "npm:@preact/signals@1.2.1": "1.2.1_preact@10.24.3", "npm:@preact/signals@^1.2.3": "1.3.0_preact@10.24.3", + "npm:@types/node@*": "22.5.4", "npm:@viz-js/viz@^3.11.0": "3.11.0", "npm:autoprefixer@10.4.17": "10.4.17_postcss@8.4.35", "npm:cssnano@6.0.3": "6.0.3_postcss@8.4.35", diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index 6720ed95..92fa7315 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -324,7 +324,7 @@ function useDigraph(dependencies: DependencyGraphItem[]) { })(); }, [dependencies]); - return { pan, zoom, reset, ref }; + return { pan, zoom, reset, svg, ref }; } interface GraphControlButtonProps { @@ -347,18 +347,36 @@ function GraphControlButton(props: GraphControlButtonProps) { ); } +const DRAG_THRESHOLD = 5; + export function DependencyGraph(props: DependencyGraphProps) { - const { pan, zoom, reset, ref } = useDigraph(props.dependencies); + const { pan, zoom, reset, svg, ref } = useDigraph(props.dependencies); const dragActive = useSignal(false); + const dragStart = useSignal({ x: 0, y: 0 }); - function enableDrag() { - dragActive.value = true; + function enableDrag(event: MouseEvent) { + dragStart.value = { x: event.clientX, y: event.clientY }; } + function disableDrag() { dragActive.value = false; + dragStart.value = { x: 0, y: 0 }; + svg.current?.querySelectorAll("a").forEach((link) => { + link.style.pointerEvents = "auto"; + }); } function onMouseMove(event: MouseEvent) { + if (!dragActive.value && (dragStart.value.x || dragStart.value.y)) { + const dx = Math.abs(event.clientX - dragStart.value.x); + const dy = Math.abs(event.clientY - dragStart.value.y); + if (dx > DRAG_THRESHOLD || dy > DRAG_THRESHOLD) { + dragActive.value = true; + svg.current?.querySelectorAll("a").forEach((link) => { + link.style.pointerEvents = "none"; + }); + } + } if (dragActive.value) { pan(event.movementX, event.movementY); } From 82d29aa8dcf13b10edcecfefdfea7a23463c2ddb Mon Sep 17 00:00:00 2001 From: Michael Herzner Date: Tue, 31 Dec 2024 18:04:45 +0100 Subject: [PATCH 19/25] chore: add missing wasm cases to matches --- api/src/api/package.rs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/api/src/api/package.rs b/api/src/api/package.rs index 8d50bf13..56f32a07 100644 --- a/api/src/api/package.rs +++ b/api/src/api/package.rs @@ -1822,10 +1822,10 @@ impl<'a> GraphDependencyCollector<'a> { } } } - Module::Npm(_) => { - return None; - } - Module::Node(_) | Module::External(_) => { + Module::Wasm(_) + | Module::Npm(_) + | Module::Node(_) + | Module::External(_) => { return None; } }; @@ -1834,15 +1834,21 @@ impl<'a> GraphDependencyCollector<'a> { Some(info.id) } else { let maybe_size = match module { - Module::Js(module) => Some(module.size() as u64), - Module::Json(module) => Some(module.size() as u64), - Module::Node(_) | Module::Npm(_) | Module::External(_) => None, + Module::Js(js) => Some(js.size() as u64), + Module::Json(json) => Some(json.size() as u64), + Module::Wasm(_) + | Module::Node(_) + | Module::Npm(_) + | Module::External(_) => None, }; let media_type = match module { Module::Js(js) => Some(js.media_type), Module::Json(json) => Some(json.media_type), - Module::Npm(_) | Module::Node(_) | Module::External(_) => None, + Module::Wasm(_) + | Module::Npm(_) + | Module::Node(_) + | Module::External(_) => None, }; let mut children = vec![]; @@ -1868,6 +1874,7 @@ impl<'a> GraphDependencyCollector<'a> { } } Module::Json(_) + | Module::Wasm(_) | Module::Npm(_) | Module::Node(_) | Module::External(_) => {} From 3a82d66bbed28584adb754e9a1316f886553e521 Mon Sep 17 00:00:00 2001 From: Michael Herzner Date: Tue, 31 Dec 2024 18:19:45 +0100 Subject: [PATCH 20/25] fix: ci --- api/src/api/package.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/api/package.rs b/api/src/api/package.rs index 56f32a07..879fa23d 100644 --- a/api/src/api/package.rs +++ b/api/src/api/package.rs @@ -1972,9 +1972,9 @@ pub async fn get_dependencies_graph_handler( let scope = req.param_scope()?; let package = req.param_package()?; let version = req.param_version()?; - Span::current().record("scope", &field::display(&scope)); - Span::current().record("package", &field::display(&package)); - Span::current().record("version", &field::display(&version)); + Span::current().record("scope", field::display(&scope)); + Span::current().record("package", field::display(&package)); + Span::current().record("version", field::display(&version)); let buckets = req.data::().unwrap().clone(); let gcs_path = From 3f280b0f04283bde812c0678108456c261978800 Mon Sep 17 00:00:00 2001 From: crowlkats Date: Wed, 15 Jan 2025 12:40:55 +0100 Subject: [PATCH 21/25] use exports keys instead of path --- api/src/api/package.rs | 145 ++++++++++++++---- api/src/util.rs | 1 + .../package/(_islands)/DependencyGraph.tsx | 31 +++- frontend/utils/api_types.ts | 7 +- 4 files changed, 147 insertions(+), 37 deletions(-) diff --git a/api/src/api/package.rs b/api/src/api/package.rs index 879fa23d..e3d141e7 100644 --- a/api/src/api/package.rs +++ b/api/src/api/package.rs @@ -2,12 +2,19 @@ use anyhow::Context; use chrono::Utc; use comrak::adapters::SyntaxHighlighterAdapter; -use deno_ast::{MediaType, ModuleSpecifier, ParseDiagnostic}; -use deno_graph::source::{JsrUrlProvider, LoadOptions, NullFileSystem}; -use deno_graph::{ - BuildOptions, CapturingModuleAnalyzer, GraphKind, Module, ModuleInfo, - Resolution, WorkspaceMember, -}; +use deno_ast::MediaType; +use deno_ast::ModuleSpecifier; +use deno_ast::ParseDiagnostic; +use deno_graph::source::JsrUrlProvider; +use deno_graph::source::LoadOptions; +use deno_graph::source::NullFileSystem; +use deno_graph::BuildOptions; +use deno_graph::CapturingModuleAnalyzer; +use deno_graph::GraphKind; +use deno_graph::Module; +use deno_graph::ModuleInfo; +use deno_graph::Resolution; +use deno_graph::WorkspaceMember; use futures::future::Either; use futures::StreamExt; use hyper::body::HttpBody; @@ -20,7 +27,8 @@ use regex::Regex; use routerify::prelude::RequestExt; use routerify::Router; use routerify_query::RequestQueryExt; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; +use serde::Serialize; use sha2::Digest; use std::borrow::Cow; use std::io; @@ -60,6 +68,7 @@ use crate::ids::PackageName; use crate::ids::PackagePath; use crate::ids::ScopeName; use crate::metadata::PackageMetadata; +use crate::metadata::VersionMetadata; use crate::npm::generate_npm_version_manifest; use crate::orama::OramaClient; use crate::provenance; @@ -76,7 +85,9 @@ use crate::util::VersionOrLatest; use crate::NpmUrl; use crate::RegistryUrl; +use super::ApiCreatePackageRequest; use super::ApiDependency; +use super::ApiDependencyGraphItem; use super::ApiDependent; use super::ApiDownloadDataPoint; use super::ApiError; @@ -99,7 +110,6 @@ use super::ApiStats; use super::ApiUpdatePackageGithubRepositoryRequest; use super::ApiUpdatePackageRequest; use super::ApiUpdatePackageVersionRequest; -use super::{ApiCreatePackageRequest, ApiDependencyGraphItem}; const MAX_PUBLISH_TARBALL_SIZE: u64 = 20 * 1024 * 1024; // 20mb @@ -162,7 +172,10 @@ pub fn package_router() -> Router { ) .get( "/:package/versions/:version/dependencies/graph", - util::json(get_dependencies_graph_handler), + util::cache( + CacheDuration::ONE_HOUR, + util::json(get_dependencies_graph_handler), + ), ) .get( "/:package/publishing_tasks", @@ -1539,6 +1552,7 @@ struct DepTreeLoader { package: PackageName, version: crate::ids::Version, bucket: crate::buckets::BucketWithQueue, + exports: Arc>>>, } impl DepTreeLoader { @@ -1566,8 +1580,7 @@ impl DepTreeLoader { crate::gcs_paths::file_path(&scope, &package, &version, &path) .into(), ) - .await - .expect("hello") + .await? else { return Ok(None); }; @@ -1582,6 +1595,8 @@ impl DepTreeLoader { } "http" | "https" => { let bucket = self.bucket.clone(); + let exports = self.exports.clone(); + async move { let jsr_matches = JSR_DEP_PATH_RE.captures(specifier.path()).unwrap(); @@ -1590,22 +1605,45 @@ impl DepTreeLoader { let version = jsr_matches.name("version"); let path = jsr_matches.name("path").unwrap(); - let Some(bytes) = bucket - .download( - format!( - "@{}/{}/{}{}", - scope.as_str(), - package.as_str(), - version.map(|version| version.as_str()).unwrap_or_default(), - path.as_str() - ) - .into(), - ) - .await? - else { + let full_path: Arc = format!( + "@{}/{}/{}{}", + scope.as_str(), + package.as_str(), + version + .as_ref() + .map(|version| version.as_str()) + .unwrap_or_default(), + if path.as_str().starts_with('/') && version.is_none() { + &path.as_str()[1..] + } else { + path.as_str() + } + ) + .into(); + + let Some(bytes) = bucket.download(full_path.clone()).await? else { return Ok(None); }; + if version.is_none() { + if let Some(captures) = JSR_DEP_META_RE.captures(path.as_str()) { + let version = captures.name("version").unwrap(); + let meta = + serde_json::from_slice::(&bytes).unwrap(); + + let mut lock = exports.lock().await; + lock.insert( + format!( + "@{}/{}@{}", + scope.as_str(), + package.as_str(), + version.as_str() + ), + meta.exports, + ); + } + } + Ok(Some(deno_graph::source::LoadResponse::Module { content: bytes.to_vec().into(), specifier: specifier.clone(), @@ -1615,7 +1653,7 @@ impl DepTreeLoader { .boxed() } "jsr" => unreachable!("{specifier}"), - // TODO: handle npm specifiers + // TODO(@crowlKats): handle npm specifiers "npm" | "node" | "bun" => async move { Ok(Some(deno_graph::source::LoadResponse::External { specifier: specifier.clone(), @@ -1701,6 +1739,7 @@ impl deno_graph::ModuleAnalyzer for DepTreeAnalyzer { lazy_static::lazy_static! { static ref JSR_DEP_PATH_RE: Regex = Regex::new(r"/@(?.+?)/(?.+?)(?:/(?.+?))?(?/.+)").unwrap(); + static ref JSR_DEP_META_RE: Regex = Regex::new(r"/(?.+?)_meta.json").unwrap(); } // We have to spawn another tokio runtime, because @@ -1726,7 +1765,7 @@ async fn analyze_deps_tree( base: Url::parse("file:///").unwrap(), name: format!("@{}/{}", scope, package), version: Some(version.0.clone()), - exports, + exports: exports.clone(), }; let module_analyzer = DepTreeAnalyzer::default(); @@ -1736,6 +1775,7 @@ async fn analyze_deps_tree( package, version, bucket, + exports: Default::default(), }; graph .build( @@ -1762,10 +1802,32 @@ async fn analyze_deps_tree( let mut index = 0; let mut dependencies = Default::default(); + let exports_by_identifier = Arc::into_inner(loader.exports) + .unwrap() + .into_inner() + .into_iter() + .map(|(p, exports)| { + // flips export keys->filepaths mapping, and removes leading . in filepaths + // and leading ./ in keys if the key is not the main entrypoint + let reversed_exports = exports + .into_iter() + .map(|(k, v)| { + ( + v[1..].to_string(), + if k == "." { k } else { k[2..].to_string() }, + ) + }) + .collect::>(); + + (p, reversed_exports) + }) + .collect(); + for root in roots { GraphDependencyCollector::collect( &graph, &root, + &exports_by_identifier, &mut index, &mut dependencies, ); @@ -1777,6 +1839,7 @@ async fn analyze_deps_tree( struct GraphDependencyCollector<'a> { graph: &'a deno_graph::ModuleGraph, dependencies: &'a mut IndexMap, + exports: &'a IndexMap>, id_index: &'a mut usize, } @@ -1784,6 +1847,7 @@ impl<'a> GraphDependencyCollector<'a> { pub fn collect( graph: &'a deno_graph::ModuleGraph, root: &'a ModuleSpecifier, + exports: &'a IndexMap>, id_index: &'a mut usize, dependencies: &'a mut IndexMap, ) { @@ -1792,6 +1856,7 @@ impl<'a> GraphDependencyCollector<'a> { Self { graph, dependencies, + exports, id_index, } .build_module_info(root_module) @@ -1810,11 +1875,28 @@ impl<'a> GraphDependencyCollector<'a> { let version = jsr_matches.name("version").unwrap(); let path = jsr_matches.name("path").unwrap(); + let identifier = format!( + "@{}/{}@{}", + scope.as_str(), + package.as_str(), + version.as_str() + ); + + let entrypoint = if let Some(entrypoint) = self + .exports + .get(&identifier) + .and_then(|exports| exports.get(path.as_str())) + { + JsrEntrypoint::Entrypoint(entrypoint.to_string()) + } else { + JsrEntrypoint::Path(path.as_str().to_string()) + }; + DependencyKind::Jsr { scope: scope.as_str().to_string(), package: package.as_str().to_string(), version: version.as_str().to_string(), - path: path.as_str().to_string(), + entrypoint, } } else { DependencyKind::Root { @@ -1932,6 +2014,13 @@ impl<'a> GraphDependencyCollector<'a> { } } +#[derive(Serialize, Deserialize, Hash, Debug, Clone, Eq, PartialEq)] +#[serde(rename_all = "camelCase", tag = "type", content = "value")] +pub enum JsrEntrypoint { + Entrypoint(String), + Path(String), +} + #[derive(Serialize, Deserialize, Hash, Debug, Clone, Eq, PartialEq)] #[serde(rename_all = "camelCase", tag = "type")] pub enum DependencyKind { @@ -1939,7 +2028,7 @@ pub enum DependencyKind { scope: String, package: String, version: String, - path: String, + entrypoint: JsrEntrypoint, }, Npm { package: String, diff --git a/api/src/util.rs b/api/src/util.rs index 228b7675..e8ab55ab 100644 --- a/api/src/util.rs +++ b/api/src/util.rs @@ -114,6 +114,7 @@ where pub struct CacheDuration(pub usize); impl CacheDuration { pub const ONE_MINUTE: CacheDuration = CacheDuration(60); + pub const ONE_HOUR: CacheDuration = CacheDuration(60 * 60); } pub fn cache( diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index 92fa7315..123deb17 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -26,7 +26,7 @@ interface DependencyGraphKindGroupedJsr { scope: string; package: string; version: string; - paths: string[]; + entrypoints: string[]; } type GroupedDependencyGraphKind = @@ -63,7 +63,7 @@ export function groupDependencies( const jsrGroups = new Map { + const filteredEntrypoints = group.entrypoints.filter(({ oldIndex }) => { const refs = referencedBy.get(oldIndex)!; for (const ref of refs) { @@ -123,7 +127,12 @@ export function groupDependencies( } return false; // all references are from within the same jsr package - }).map((p) => p.path); + }).map((p) => { + if (!p.isEntrypoint) { + throw new Error("unreachable"); + } + return p.entrypoint; + }); const uniqueChildren = Array.from(new Set(group.children)); const newIndex = out.length; @@ -133,7 +142,7 @@ export function groupDependencies( scope: group.key.scope, package: group.key.package, version: group.key.version, - paths: Array.from(new Set(filteredPaths)), + entrypoints: Array.from(new Set(filteredEntrypoints)), }, children: uniqueChildren, size: group.size, @@ -223,7 +232,13 @@ function renderDependency( tooltip = `@${dependency.scope}/${dependency.package}@${dependency.version}`; href = `/${tooltip}`; - content = `${tooltip}\n${dependency.paths.join("\n")}\n${ + content = `${tooltip}\n${dependency.entrypoints.map((entrypoint) => { + if (entrypoint == ".") { + return "default entrypoint"; + } else { + return entrypoint; + } + }).join("\n")}\n${ bytesToSize(size ?? 0) }`; color = "#faee4a"; diff --git a/frontend/utils/api_types.ts b/frontend/utils/api_types.ts index 45bc377a..2adf36fa 100644 --- a/frontend/utils/api_types.ts +++ b/frontend/utils/api_types.ts @@ -267,12 +267,17 @@ export interface CreatedToken { secret: string; } +export interface DependencyGraphJsrEntrypoint { + type: "entrypoint" | "path"; + value: string; +} + export interface DependencyGraphKindJsr { type: "jsr"; scope: string; package: string; version: string; - path: string; + entrypoint: DependencyGraphJsrEntrypoint; } export interface DependencyGraphKindNpm { From 1b4061396cab78b19b4deb89f2ab137ef7619cf6 Mon Sep 17 00:00:00 2001 From: crowlkats Date: Wed, 15 Jan 2025 12:42:14 +0100 Subject: [PATCH 22/25] remove testing assrtion --- api/src/api/package.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/api/src/api/package.rs b/api/src/api/package.rs index e3d141e7..615d7267 100644 --- a/api/src/api/package.rs +++ b/api/src/api/package.rs @@ -2069,8 +2069,7 @@ pub async fn get_dependencies_graph_handler( let gcs_path = crate::gcs_paths::version_metadata(&scope, &package, &version).into(); let version_meta = buckets.modules_bucket.download(gcs_path).await?.unwrap(); - let version_meta = - serde_json::from_slice::(&version_meta)?; + let version_meta = serde_json::from_slice::(&version_meta)?; let registry_url = req.data::().unwrap().0.clone(); @@ -2090,11 +2089,7 @@ pub async fn get_dependencies_graph_handler( let api_deps = deps .into_iter() - .enumerate() - .map(|(i, dep)| { - assert_eq!(i, dep.1.id); - ApiDependencyGraphItem::from(dep) - }) + .map(ApiDependencyGraphItem::from) .collect::>(); Ok(api_deps) From bffe10709cbd8b80172dd4e57fbb4752def02b40 Mon Sep 17 00:00:00 2001 From: crowlkats Date: Wed, 15 Jan 2025 13:33:48 +0100 Subject: [PATCH 23/25] fmt --- .../package/(_islands)/DependencyGraph.tsx | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/frontend/routes/package/(_islands)/DependencyGraph.tsx b/frontend/routes/package/(_islands)/DependencyGraph.tsx index 123deb17..a3d2127f 100644 --- a/frontend/routes/package/(_islands)/DependencyGraph.tsx +++ b/frontend/routes/package/(_islands)/DependencyGraph.tsx @@ -63,7 +63,11 @@ export function groupDependencies( const jsrGroups = new Map { - if (entrypoint == ".") { - return "default entrypoint"; - } else { - return entrypoint; - } - }).join("\n")}\n${ - bytesToSize(size ?? 0) - }`; + content = `${tooltip}\n${ + dependency.entrypoints.map((entrypoint) => { + if (entrypoint == ".") { + return "default entrypoint"; + } else { + return entrypoint; + } + }).join("\n") + }\n${bytesToSize(size ?? 0)}`; color = "#faee4a"; break; } From 3c19f682482dde8bc2e4fb59030d7f881b5af97e Mon Sep 17 00:00:00 2001 From: crowlkats Date: Fri, 17 Jan 2025 17:21:16 +0100 Subject: [PATCH 24/25] add test --- api/src/api/package.rs | 91 +++++++++++++++++++++++++++++++++++++++++- api/src/api/types.rs | 2 +- 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/api/src/api/package.rs b/api/src/api/package.rs index 615d7267..4079f019 100644 --- a/api/src/api/package.rs +++ b/api/src/api/package.rs @@ -2068,7 +2068,11 @@ pub async fn get_dependencies_graph_handler( let buckets = req.data::().unwrap().clone(); let gcs_path = crate::gcs_paths::version_metadata(&scope, &package, &version).into(); - let version_meta = buckets.modules_bucket.download(gcs_path).await?.unwrap(); + let version_meta = buckets + .modules_bucket + .download(gcs_path) + .await? + .ok_or(ApiError::PackageVersionNotFound)?; let version_meta = serde_json::from_slice::(&version_meta)?; let registry_url = req.data::().unwrap().0.clone(); @@ -2156,6 +2160,7 @@ mod test { use serde_json::json; use crate::api::ApiDependency; + use crate::api::ApiDependencyGraphItem; use crate::api::ApiDependencyKind; use crate::api::ApiDependent; use crate::api::ApiList; @@ -3589,6 +3594,90 @@ ggHohNAjhbzDaY2iBW/m3NC5dehGUP4T2GBo/cwGhg== assert_eq!(dependents.total, 2); } + #[tokio::test] + async fn test_package_dependencies_graph() { + let mut t = TestSetup::new().await; + + // unpublished package + let mut resp = t + .http() + .get("/api/scopes/scope/packages/foo/versions/0.0.1/dependencies/graph") + .call() + .await + .unwrap(); + resp + .expect_err_code(StatusCode::NOT_FOUND, "packageVersionNotFound") + .await; + + let task = process_tarball_setup(&t, create_mock_tarball("ok")).await; + assert_eq!(task.status, PublishingTaskStatus::Success, "{:?}", task); + + // Empty deps + let mut resp = t + .http() + .get("/api/scopes/scope/packages/foo/versions/1.2.3/dependencies/graph") + .call() + .await + .unwrap(); + let deps: Vec = resp.expect_ok().await; + assert_eq!( + deps, + vec![ApiDependencyGraphItem { + dependency: super::DependencyKind::Root { + path: "/mod.ts".to_string(), + }, + children: vec![], + size: Some(155), + media_type: Some("TypeScript".to_string()), + }] + ); + + // Now publish a package that has a few deps + let package_name = PackageName::try_from("bar").unwrap(); + let version = Version::try_from("1.2.3").unwrap(); + let task = crate::publish::tests::process_tarball_setup2( + &t, + create_mock_tarball("depends_on_ok"), + &package_name, + &version, + false, + ) + .await; + assert_eq!(task.status, PublishingTaskStatus::Success, "{:?}", task); + + let mut resp = t + .http() + .get("/api/scopes/scope/packages/bar/versions/1.2.3/dependencies/graph") + .call() + .await + .unwrap(); + let deps: Vec = resp.expect_ok().await; + assert_eq!( + deps, + vec![ + ApiDependencyGraphItem { + dependency: super::DependencyKind::Jsr { + scope: "scope".to_string(), + package: "foo".to_string(), + version: "1.2.3".to_string(), + entrypoint: super::JsrEntrypoint::Entrypoint(".".to_string()) + }, + children: vec![], + size: Some(155), + media_type: Some("TypeScript".to_string()) + }, + ApiDependencyGraphItem { + dependency: super::DependencyKind::Root { + path: "/mod.ts".to_string() + }, + children: vec![0], + size: Some(117), + media_type: Some("TypeScript".to_string()) + } + ] + ); + } + #[tokio::test] async fn package_delete() { let mut t: TestSetup = TestSetup::new().await; diff --git a/api/src/api/types.rs b/api/src/api/types.rs index 06851b33..50886e07 100644 --- a/api/src/api/types.rs +++ b/api/src/api/types.rs @@ -81,7 +81,7 @@ impl From for ApiPublishingTask { } } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] #[serde(rename_all = "camelCase")] pub struct ApiDependencyGraphItem { pub dependency: super::package::DependencyKind, From 2b2a882b844899fd2083e9e11c3eda3364b2bfa4 Mon Sep 17 00:00:00 2001 From: crowlkats Date: Fri, 17 Jan 2025 17:22:09 +0100 Subject: [PATCH 25/25] increase cache time --- api/src/api/package.rs | 2 +- api/src/util.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/api/package.rs b/api/src/api/package.rs index 4079f019..8fc5fb39 100644 --- a/api/src/api/package.rs +++ b/api/src/api/package.rs @@ -173,7 +173,7 @@ pub fn package_router() -> Router { .get( "/:package/versions/:version/dependencies/graph", util::cache( - CacheDuration::ONE_HOUR, + CacheDuration::ONE_DAY, util::json(get_dependencies_graph_handler), ), ) diff --git a/api/src/util.rs b/api/src/util.rs index e8ab55ab..1c56088f 100644 --- a/api/src/util.rs +++ b/api/src/util.rs @@ -114,7 +114,7 @@ where pub struct CacheDuration(pub usize); impl CacheDuration { pub const ONE_MINUTE: CacheDuration = CacheDuration(60); - pub const ONE_HOUR: CacheDuration = CacheDuration(60 * 60); + pub const ONE_DAY: CacheDuration = CacheDuration(60 * 60 * 24); } pub fn cache(