Merge remote-tracking branch 'origin/main' into refactor/skill-md-builder

# Conflicts:
#	src/commands/install.ts

Author: harlan-zw

package.json

@@ -2,7 +2,7 @@
   "name": "skilld",
   "type": "module",
   "version": "1.7.0",
-  "packageManager": "pnpm@10.33.1",
+  "packageManager": "pnpm@10.33.2",
   "description": "Generate AI agent skills from npm package documentation",
   "author": {
     "name": "Harlan Wilton",
@@ -86,6 +86,7 @@
     "sqlite-vec": "catalog:deps",
     "std-env": "catalog:",
     "tinyglobby": "catalog:deps",
+    "typebox": "catalog:",
     "typescript": "catalog:",
     "unagent": "catalog:",
     "unist-util-visit": "catalog:deps"

pnpm-lock.yaml

@@ -7,7 +7,7 @@ packages:
 overrides:
   global-agent: ^4.1.3
 catalog:
-  '@mariozechner/pi-ai': ^0.68.1
+  '@mariozechner/pi-ai': ^0.70.6
   '@mdream/crawl': ^1.1.0
   '@types/semver': ^7.7.1
   bumpp: ^11.0.1
@@ -20,12 +20,13 @@ catalog:
   semver: ^7.7.4
   std-env: ^4.1.0
   tsx: ^4.21.0
+  typebox: ^1.1.34
   typescript: 6.0.2
   unagent: ^0.0.8
 catalogs:
   deps:
     '@clack/prompts': ^1.2.0
-    '@huggingface/transformers': ^4.1.0
+    '@huggingface/transformers': ^4.2.0
     citty: ^0.2.2
     consola: ^3.4.2
     log-update: ^8.0.0
@@ -34,7 +35,7 @@ catalogs:
     mdast-util-to-string: ^4.0.0
     micromark-extension-frontmatter: ^2.0.0
     mlly: ^1.8.2
-    oxc-parser: ^0.127.0
+    oxc-parser: ^0.128.0
     p-limit: ^7.3.0
     sqlite-vec: ^0.1.9
     tinyglobby: ^0.2.16

pnpm-workspace.yaml

@@ -21,8 +21,8 @@ import { homedir } from 'node:os'
 import { resolve } from 'node:path'
 import { getEnvApiKey, getModel, getModels, getProviders, streamSimple } from '@mariozechner/pi-ai'
 import { getOAuthApiKey, getOAuthProvider, getOAuthProviders } from '@mariozechner/pi-ai/oauth'
-import { Type } from '@sinclair/typebox'
 import { join } from 'pathe'
+import { Type } from 'typebox'
 import { sanitizeMarkdown } from '../../core/sanitize.ts'
 
 export function isPiAiModel(model: string): boolean {

src/agent/clis/pi-ai.ts

@@ -15,7 +15,9 @@ export {
   getPkgKeyFiles,
   getShippedSkills,
   hasShippedDocs,
+  inferDocsTypeFromCache,
   isCached,
+  isReadmeOnlyCache,
   linkCachedDir,
   linkPkg,
   linkPkgNamed,

src/cache/index.ts

@@ -33,6 +33,21 @@ export function isCached(name: string, version: string): boolean {
   return existsSync(getCacheDir(name, version))
 }
 
+/** Check if cache only has docs/README.md (pkg/ already has this) */
+export function isReadmeOnlyCache(cacheDir: string): boolean {
+  const docsDir = join(cacheDir, 'docs')
+  if (!existsSync(docsDir))
+    return false
+  const files = readdirSync(docsDir)
+  return files.length === 1 && files[0] === 'README.md'
+}
+
+export function inferDocsTypeFromCache(cacheDir: string, source?: string): 'llms.txt' | 'readme' | 'docs' {
+  if (source?.includes('llms.txt') || existsSync(join(cacheDir, 'docs', 'llms.txt')))
+    return 'llms.txt'
+  return isReadmeOnlyCache(cacheDir) ? 'readme' : 'docs'
+}
+
 /**
  * Ensure cache directories exist
  */

src/cache/storage.ts

@@ -27,7 +27,9 @@ import {
   getPkgKeyFiles,
   getRepoCacheDir,
   getShippedSkills,
+  inferDocsTypeFromCache,
   isCached,
+  isReadmeOnlyCache,
   linkPkgNamed,
   linkShippedSkill,
   listReferenceFiles,
@@ -38,7 +40,7 @@ import {
 import { promptForAgent, resolveAgent, sharedArgs } from '../cli-helpers.ts'
 import { defaultFeatures, readConfig } from '../core/config.ts'
 import { timedSpinner } from '../core/formatting.ts'
-import { mergeLocks, parsePackages, readLock, syncLockfilesToDirs, writeLock } from '../core/lockfile.ts'
+import { mergeLocks, parsePackageNames, parsePackages, readLock, syncLockfilesToDirs, writeLock } from '../core/lockfile.ts'
 import { readPackageJsonSafe } from '../core/package-json.ts'
 import { toStoragePackageName } from '../core/prefix.ts'
 import { sanitizeMarkdown } from '../core/sanitize.ts'
@@ -199,7 +201,7 @@ export async function installCommand(opts: InstallOptions): Promise<void> {
       for (const pkg of parsePackages(info.packages))
         linkPkgNamed(skillDir, pkg.name, cwd, pkg.version)
       // Only link external docs if package doesn't ship its own and has more than just README
-      if (!pkgHasShippedDocs(pkgName, cwd, version) && !isReadmeOnly(globalCachePath)) {
+      if (!pkgHasShippedDocs(pkgName, cwd, version) && !isReadmeOnlyCache(globalCachePath)) {
         const docsLink = join(referencesPath, 'docs')
         const cachedDocs = join(globalCachePath, 'docs')
         if (existsSync(docsLink))
@@ -368,7 +370,7 @@ export async function installCommand(opts: InstallOptions): Promise<void> {
       for (const pkg of parsePackages(info.packages))
         linkPkgNamed(skillDir, pkg.name, cwd, pkg.version)
       // Link fetched docs unless it's just a README (already in pkg/)
-      if (!isReadmeOnly(globalCachePath)) {
+      if (!isReadmeOnlyCache(globalCachePath)) {
         const docsLink = join(referencesPath, 'docs')
         const cachedDocsDir = join(globalCachePath, 'docs')
         if (existsSync(docsLink))
@@ -514,15 +516,6 @@ function linkPkgSymlink(referencesDir: string, name: string, cwd: string, versio
   symlinkSync(pkgPath, pkgLink, 'junction')
 }
 
-/** Check if cache only has docs/README.md (pkg/ already has this) */
-function isReadmeOnly(cacheDir: string): boolean {
-  const docsDir = join(cacheDir, 'docs')
-  if (!existsSync(docsDir))
-    return false
-  const files = readdirSync(docsDir)
-  return files.length === 1 && files[0] === 'README.md'
-}
-
 /** Check if package ships its own docs folder */
 function pkgHasShippedDocs(name: string, cwd: string, version?: string): boolean {
   const pkgPath = resolvePkgDir(name, cwd, version)
@@ -587,16 +580,12 @@ async function enhanceRegenerated(
       }
     }
 
-    let docsType: 'llms.txt' | 'readme' | 'docs' = 'docs'
-    if (existsSync(join(globalCachePath, 'docs', 'llms.txt')))
-      docsType = 'llms.txt'
-    else if (isReadmeOnly(globalCachePath))
-      docsType = 'readme'
+    const docsType = inferDocsTypeFromCache(globalCachePath)
 
     // Derive dirName from the skill directory name
     const dirName = skillDir.split('/').pop()
 
-    const allPackages = parsePackages(packages).map(p => ({ name: p.name }))
+    const allPackages = parsePackageNames(packages)
     writeGeneratedSkillMd(skillDir, {
       name: pkgName,
       version,
@@ -667,11 +656,7 @@ function regenerateBaseSkillMd(
 
   // Infer docsType from source or cache
   const globalCachePath = getCacheDir(pkgName, version)
-  let docsType: 'llms.txt' | 'readme' | 'docs' = 'docs'
-  if (source?.includes('llms.txt') || existsSync(join(globalCachePath, 'docs', 'llms.txt')))
-    docsType = 'llms.txt'
-  else if (isReadmeOnly(globalCachePath))
-    docsType = 'readme'
+  const docsType = inferDocsTypeFromCache(globalCachePath, source)
 
   // Check cache dirs for issues/discussions/releases (only if feature enabled)
   const feat = readConfig().features ?? defaultFeatures
@@ -686,7 +671,7 @@ function regenerateBaseSkillMd(
   const dirName = skillDir.split('/').pop()
 
   // Build multi-package list from lockfile packages field
-  const allPackages = parsePackages(packages).map(p => ({ name: p.name }))
+  const allPackages = parsePackageNames(packages)
 
   mkdirSync(skillDir, { recursive: true })
   writeGeneratedSkillMd(skillDir, {

src/commands/install.ts

@@ -33,12 +33,13 @@ import {
 } from '../cache/index.ts'
 import { defaultFeatures, readConfig, registerProject } from '../core/config.ts'
 import { formatDuration } from '../core/formatting.ts'
-import { parsePackages, readLock, writeLock } from '../core/lockfile.ts'
+import { parsePackageNames, parsePackages, readLock, writeLock } from '../core/lockfile.ts'
 import { parseFrontmatter } from '../core/markdown.ts'
 import { getSharedSkillsDir, semverDiff, SHARED_SKILLS_DIR } from '../core/shared.ts'
 import { shutdownWorker } from '../retriv/pool.ts'
 import {
   fetchPkgDist,
+  parseGitHubRepoSlug,
   parsePackageSpec,
   readLocalDependencies,
   resolvePackageDocsWithAttempts,
@@ -555,7 +556,7 @@ async function syncBaseSkill(
   const pkgFiles = getPkgKeyFiles(packageName, cwd, version)
 
   // Write base SKILL.md
-  const repoSlug = resolved.repoUrl?.match(/github\.com\/([^/]+\/[^/]+?)(?:\.git)?(?:[/#]|$)/)?.[1]
+  const repoSlug = parseGitHubRepoSlug(resolved.repoUrl)
 
   // Create named symlink for this package
   linkPkgNamed(skillDir, packageName, cwd, version)
@@ -571,7 +572,7 @@ async function syncBaseSkill(
 
   // Read back merged packages from lockfile
   const updatedLock = readLock(baseDir)?.skills[skillDirName]
-  const allPackages = parsePackages(updatedLock?.packages).map(p => ({ name: p.name }))
+  const allPackages = parsePackageNames(updatedLock?.packages)
 
   const skillMd = writeGeneratedSkillMd(skillDir, {
     name: packageName,

src/commands/sync-parallel.ts

@@ -36,7 +36,7 @@ import {
 import { getInstalledGenerators, introLine, isInteractive, promptForAgent, resolveAgent, sharedArgs, suggestPrepareHook } from '../cli-helpers.ts'
 import { defaultFeatures, hasCompletedWizard, readConfig, registerProject } from '../core/config.ts'
 import { timedSpinner } from '../core/formatting.ts'
-import { parsePackages, readLock, removeLockEntry, writeLock } from '../core/lockfile.ts'
+import { parsePackageNames, parsePackages, readLock, removeLockEntry, writeLock } from '../core/lockfile.ts'
 import { parseFrontmatter } from '../core/markdown.ts'
 import { parseSkillInput, resolveSkillName, toStoragePackageName } from '../core/prefix.ts'
 import { getSharedSkillsDir, SHARED_SKILLS_DIR } from '../core/shared.ts'
@@ -45,6 +45,7 @@ import { shutdownWorker } from '../retriv/pool.ts'
 import {
   fetchPkgDist,
   isPrerelease,
+  parseGitHubRepoSlug,
   parsePackageSpec,
   readLocalDependencies,
   resolveCrateDocsWithAttempts,
@@ -460,7 +461,7 @@ async function syncSinglePackage(packageSpec: string, config: SyncConfig): Promi
     linkPkgNamed(skillDir, storagePackageName, cwd, version)
 
     // Merge into lockfile
-    const repoSlug = resolved.repoUrl?.match(/github\.com\/([^/]+\/[^/]+?)(?:\.git)?(?:[/#]|$)/)?.[1]
+    const repoSlug = parseGitHubRepoSlug(resolved.repoUrl)
     writeLock(baseDir, skillDirName, {
       packageName: identityPackageName,
       version,
@@ -472,7 +473,7 @@ async function syncSinglePackage(packageSpec: string, config: SyncConfig): Promi
 
     // Regenerate SKILL.md with all packages listed
     const updatedLock = readLock(baseDir)?.skills[skillDirName]
-    const allPackages = parsePackages(updatedLock?.packages).map(p => ({ name: p.name }))
+    const allPackages = parsePackageNames(updatedLock?.packages)
     const relatedSkills = await findRelatedSkills(storagePackageName, baseDir)
     const existingStorageName = toStoragePackageName(existingLock.packageName!)
     const pkgFiles = getPkgKeyFiles(existingStorageName, cwd, existingLock.version)
@@ -565,7 +566,7 @@ async function syncSinglePackage(packageSpec: string, config: SyncConfig): Promi
   const pkgFiles = getPkgKeyFiles(storagePackageName, cwd, version)
 
   // Write base SKILL.md (no LLM needed)
-  const repoSlug = resolved.repoUrl?.match(/github\.com\/([^/]+\/[^/]+?)(?:\.git)?(?:[/#]|$)/)?.[1]
+  const repoSlug = parseGitHubRepoSlug(resolved.repoUrl)
 
   // Also create named symlink for this package (skip in eject mode)
   if (!config.eject)
@@ -600,7 +601,7 @@ async function syncSinglePackage(packageSpec: string, config: SyncConfig): Promi
 
   // Read back merged packages from lockfile for SKILL.md generation
   const updatedLock = config.eject ? undefined : readLock(baseDir)?.skills[skillDirName]
-  const allPackages = parsePackages(updatedLock?.packages).map(p => ({ name: p.name }))
+  const allPackages = parsePackageNames(updatedLock?.packages)
 
   const isEject = !!config.eject
   const baseSkillMd = writeGeneratedSkillMd(skillDir, {
@@ -1248,7 +1249,7 @@ export async function exportPortablePrompts(packageSpec: string, opts: {
   })
 
   // Write lockfile so skilld list/update/assemble can discover this skill
-  const repoSlug = resolved.repoUrl?.match(/github\.com\/([^/]+\/[^/]+?)(?:\.git)?(?:[/#]|$)/)?.[1]
+  const repoSlug = parseGitHubRepoSlug(resolved.repoUrl)
   writeLock(baseDir, skillDirName, {
     packageName,
     version,

src/commands/sync.ts

@@ -32,6 +32,10 @@ export function parsePackages(packages?: string): Array<{ name: string, version:
   }).filter(p => p.name)
 }
 
+export function parsePackageNames(packages?: string): Array<{ name: string }> {
+  return parsePackages(packages).map(({ name }) => ({ name }))
+}
+
 export function serializePackages(pkgs: Array<{ name: string, version: string }>): string {
   return pkgs.map(p => `${p.name}@${p.version}`).join(', ')
 }