Introduce pkl-doc model version 2

Currently, in order to update a pkl-doc documentation site,
almost the entire existing site is read in order to update metadata
like known versions, known subtypes, and more.

For example, adding a new version of a package requires that the
existing runtime data of all existing versions be updated.
Eventually, this causes the required storage size to balloon
exponentially to the number of versions.

This addresses these limitations by:

1. Updating the runtime data structure to move "known versions" metadata
   to the package level (the same JSON file is used for all versions).
2. Eliminating known subtype and known usage information at a
   cross-package level.
3. Generating the search index by consuming the previously generated
   search index.
4. Generating the main page by consuming the search index.

Because this changes how runtime data is stored, an existing docsite
needs to be migrated.

This also introduces a new migration command, `pkl-doc --migrate`,
which transforms an older version of the website into a newer version.

Author: bioball

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliException.kt

@@ -26,9 +26,13 @@ open class CliException(
    */
   message: String,
 
+  /** The cause */
+  cause: Throwable?,
+
   /** The process exit code to use. */
   val exitCode: Int = 1,
-) : RuntimeException(message) {
+) : RuntimeException(message, cause) {
+  constructor(message: String, exitCode: Int = 1) : this(message, null, exitCode)
 
   override fun toString(): String = message!!
 }
@@ -41,7 +45,11 @@ class CliBugException(
   /** The process exit code to use. */
   exitCode: Int = 1,
 ) :
-  CliException("An unexpected error has occurred. Would you mind filing a bug report?", exitCode) {
+  CliException(
+    "An unexpected error has occurred. Would you mind filing a bug report?",
+    theCause,
+    exitCode,
+  ) {
 
   override fun toString(): String = "$message\n\n${theCause.printStackTraceToString()}"
 }

pkl-doc/src/main/kotlin/org/pkl/doc/CliDocGenerator.kt

@@ -34,7 +34,6 @@ import org.pkl.core.packages.*
  * For the low-level Pkldoc API, see [DocGenerator].
  */
 class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(options.base) {
-
   private val packageResolver =
     PackageResolver.getInstance(securityManager, httpClient, moduleCacheDir)
 
@@ -60,6 +59,17 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
           ),
     )
 
+  private val versions = mutableMapOf<String, Version>()
+
+  private val versionComparator =
+    Comparator<String> { v1, v2 ->
+      versions
+        .getOrPut(v1) { Version.parse(v1) }
+        .compareTo(versions.getOrPut(v2) { Version.parse(v2) })
+    }
+
+  private val docMigrator = DocMigrator(options.outputDir, System.out, versionComparator)
+
   private fun DependencyMetadata.getPackageDependencies(): List<DocPackageInfo.PackageDependency> {
     return buildList {
       for ((_, dependency) in dependencies) {
@@ -87,14 +97,12 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
   }
 
   private fun PackageUri.toDocPackageInfo(): DocPackageInfo {
-    val metadataAndChecksum =
+    val (metadata, checksum) =
       try {
         packageResolver.getDependencyMetadataAndComputeChecksum(this)
       } catch (e: PackageLoadError) {
         throw CliException("Failed to package metadata for $this: ${e.message}")
       }
-    val metadata = metadataAndChecksum.first
-    val checksum = metadataAndChecksum.second
     return DocPackageInfo(
       name = "${uri.authority}${uri.path.substringBeforeLast('@')}",
       moduleNamePrefix = "${metadata.name}.",
@@ -130,6 +138,15 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
   }
 
   override fun doRun() {
+    if (options.migrate) {
+      docMigrator.run()
+      return
+    }
+    if (!docMigrator.isUpToDate) {
+      throw CliException(
+        "pkldoc website model is too old (found: ${docMigrator.docsiteVersion}, required: ${DocMigrator.CURRENT_VERSION}). Run `pkldoc --migrate` to migrate the website."
+      )
+    }
     val docsiteInfoModuleUris = mutableListOf<URI>()
     val packageInfoModuleUris = mutableListOf<URI>()
     val regularModuleUris = mutableListOf<URI>()
@@ -271,10 +288,11 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
           options.normalizedOutputDir,
           options.isTestMode,
           options.noSymlinks,
+          docMigrator,
         )
         .run()
     } catch (e: DocGeneratorException) {
-      throw CliException(e.message!!)
+      throw CliException(e.message!!, e)
     }
   }
 }

pkl-doc/src/main/kotlin/org/pkl/doc/CliDocGeneratorOptions.kt

@@ -47,6 +47,9 @@ constructor(
    * however, will create a full copy instead.
    */
   val noSymlinks: Boolean = false,
+
+  /** Migrate existing pkldoc */
+  val migrate: Boolean = false,
 ) {
   /** [outputDir] after undergoing normalization. */
   val normalizedOutputDir: Path = base.normalizedWorkingDir.resolveSafely(outputDir)

pkl-doc/src/main/kotlin/org/pkl/doc/DocGenerator.kt

@@ -17,6 +17,7 @@ package org.pkl.doc
 
 import java.io.IOException
 import java.net.URI
+import java.nio.file.Files
 import java.nio.file.Path
 import kotlin.io.path.*
 import org.pkl.commons.copyRecursively
@@ -39,11 +40,11 @@ class DocGenerator(
    */
   private val docsiteInfo: DocsiteInfo,
 
-  /** The modules to generate documentation for, grouped by package. */
-  modules: Map<DocPackageInfo, Collection<ModuleSchema>>,
+  /** The packages to generate documentation for. */
+  packages: Map<DocPackageInfo, Collection<ModuleSchema>>,
 
   /**
-   * A function to resolve imports in [modules] and [docsiteInfo].
+   * A function to resolve imports in [packages] and [docsiteInfo].
    *
    * Module `pkl.base` is resolved with this function even if not explicitly imported.
    */
@@ -73,16 +74,23 @@ class DocGenerator(
    * however, will create a full copy instead.
    */
   private val noSymlinks: Boolean = false,
+
+  /**
+   * The doc migrator that is used to determine the latest docsite version, as well as update the
+   * version file.
+   */
+  private val docMigrator: DocMigrator = DocMigrator(outputDir, System.out, versionComparator),
 ) {
   companion object {
     const val CURRENT_DIRECTORY_NAME = "current"
 
-    internal fun List<PackageData>.current(
-      versionComparator: Comparator<String>
+    internal fun determineCurrentPackages(
+      packages: List<PackageData>,
+      descendingVersionComparator: Comparator<String>,
     ): List<PackageData> {
       val comparator =
-        compareBy<PackageData> { it.ref.pkg }.thenBy(versionComparator) { it.ref.version }
-      return asSequence()
+        compareBy<PackageData> { it.ref.pkg }.thenBy(descendingVersionComparator) { it.ref.version }
+      return packages
         // If matching a semver pattern, remove any version that has a prerelease
         // version (e.g. SNAPSHOT in 1.2.3-SNAPSHOT)
         .filter { Version.parseOrNull(it.ref.version)?.preRelease == null }
@@ -94,46 +102,82 @@ class DocGenerator(
 
   private val descendingVersionComparator: Comparator<String> = versionComparator.reversed()
 
-  private val docPackages: List<DocPackage> = modules.map { DocPackage(it.key, it.value.toList()) }
+  private val docPackages: List<DocPackage> =
+    packages.map { DocPackage(it.key, it.value.toList()) }.filter { !it.isUnlisted }
+
+  private fun getCurrentPackages(siteSearchIndex: List<SearchIndexGenerator.PackageIndexEntry>) =
+    buildList {
+      for (entry in siteSearchIndex) {
+        var packageDataFile =
+          outputDir.resolve(entry.packageEntry.url).resolveSibling("package-data.json")
+        if (!Files.exists(packageDataFile)) {
+          // search-index.js in Pkl 0.29 and below did not encode path.
+          // If we get a file does not exist, try again by encoding the path.
+          packageDataFile =
+            outputDir
+              .resolve(entry.packageEntry.url.pathEncoded)
+              .resolveSibling("package-data.json")
+          if (!Files.exists((packageDataFile))) {
+            println(
+              "[Warn] likely corrupted search index; missing $packageDataFile. This entry will be removed in the newly generated index."
+            )
+            continue
+          }
+        }
+        add(PackageData.read(packageDataFile))
+      }
+    }
 
   /** Runs this documentation generator. */
   fun run() {
     try {
+      if (!docMigrator.isUpToDate) {
+        throw DocGeneratorException(
+          "Docsite is not up to date. Expected: ${DocMigrator.CURRENT_VERSION}. Found: ${docMigrator.docsiteVersion}. Use DocMigrator to migrate the site."
+        )
+      }
       val htmlGenerator =
         HtmlGenerator(docsiteInfo, docPackages, importResolver, outputDir, isTestMode)
       val searchIndexGenerator = SearchIndexGenerator(outputDir)
       val packageDataGenerator = PackageDataGenerator(outputDir)
       val runtimeDataGenerator = RuntimeDataGenerator(descendingVersionComparator, outputDir)
 
       for (docPackage in docPackages) {
-        if (docPackage.isUnlisted) continue
-
         docPackage.deletePackageDir()
         htmlGenerator.generate(docPackage)
         searchIndexGenerator.generate(docPackage)
         packageDataGenerator.generate(docPackage)
       }
 
-      val packagesData = packageDataGenerator.readAll()
-      val currentPackagesData = packagesData.current(descendingVersionComparator)
-
-      createCurrentDirectories(currentPackagesData)
-
-      htmlGenerator.generateSite(currentPackagesData)
-      searchIndexGenerator.generateSiteIndex(currentPackagesData)
-      runtimeDataGenerator.deleteDataDir()
-      runtimeDataGenerator.generate(packagesData)
+      val newlyGeneratedPackages = docPackages.map(::PackageData).sortedBy { it.ref.pkg }
+      val currentSearchIndex = searchIndexGenerator.getCurrentSearchIndex()
+      val existingCurrentPackages = getCurrentPackages(currentSearchIndex)
+      val currentPackages =
+        determineCurrentPackages(
+          newlyGeneratedPackages + existingCurrentPackages,
+          descendingVersionComparator,
+        )
+
+      createCurrentDirectories(currentPackages, existingCurrentPackages)
+      searchIndexGenerator.generateSiteIndex(currentPackages)
+      htmlGenerator.generateSite(currentPackages)
+      runtimeDataGenerator.generate(newlyGeneratedPackages)
+      docMigrator.updateDocsiteVersion()
     } catch (e: IOException) {
-      throw DocGeneratorException("I/O error generating documentation.", e)
+      throw DocGeneratorException("I/O error generating documentation: ${e.message}", e)
     }
   }
 
   private fun DocPackage.deletePackageDir() {
     outputDir.resolve(IoUtils.encodePath("$name/$version")).deleteRecursively()
   }
 
-  private fun createCurrentDirectories(currentPackagesData: List<PackageData>) {
-    for (packageData in currentPackagesData) {
+  private fun createCurrentDirectories(
+    currentPackages: List<PackageData>,
+    existingCurrentPackages: List<PackageData>,
+  ) {
+    val packagesToCreate = currentPackages - existingCurrentPackages
+    for (packageData in packagesToCreate) {
       val basePath = outputDir.resolve(packageData.ref.pkg.pathEncoded)
       val src = basePath.resolve(packageData.ref.version)
       val dst = basePath.resolve(CURRENT_DIRECTORY_NAME)
@@ -218,7 +262,7 @@ internal class DocModule(
     get() = schema.moduleName
 
   val path: String by lazy {
-    name.substring(parent.docPackageInfo.moduleNamePrefix.length).replace('.', '/')
+    name.pathEncoded.substring(parent.docPackageInfo.moduleNamePrefix.length).replace('.', '/')
   }
 
   val overview: String?