Introduce pkl-doc model version 2

buildSrc/src/main/kotlin/pklFatJar.gradle.kts

@@ -60,6 +60,7 @@ val relocations =
     // pkl-doc dependencies
     "org.commonmark." to "org.pkl.thirdparty.commonmark.",
     "org.jetbrains." to "org.pkl.thirdparty.jetbrains.",
+    "_COROUTINE." to "org.pkl.thirdparty.kotlinx._COROUTINE.",
 
     // pkl-config-java dependencies
     "io.leangen.geantyref." to "org.pkl.thirdparty.geantyref.",
@@ -99,6 +100,8 @@ tasks.shadowJar {
   exclude("META-INF/maven/**")
   exclude("META-INF/upgrade/**")
 
+  exclude("DebugProbesKt.bin")
+
   val info = project.extensions.getByType<BuildInfo>()
   val minimumJvmTarget = JavaVersion.toVersion(info.jvmTarget)
 

gradle/libs.versions.toml

@@ -34,6 +34,7 @@ kotlin = "2.0.21"
 kotlinPoet = "1.6.+"
 kotlinxHtml = "0.11.0"
 kotlinxSerialization = "1.8.0"
+kotlinxCoroutines = "1.+"
 ktfmt = "0.53"
 # replaces nuValidator's log4j dependency
 # something related to log4j-1.2-api is apparently broken in 2.17.2
@@ -79,6 +80,7 @@ kotlinScripting = { group = "org.jetbrains.kotlin", name = "kotlin-scripting-jsr
 kotlinStdLib = { group = "org.jetbrains.kotlin", name = "kotlin-stdlib-jdk8", version.ref = "kotlin" }
 kotlinxHtml = { group = "org.jetbrains.kotlinx", name = "kotlinx-html-jvm", version.ref = "kotlinxHtml" }
 kotlinxSerializationJson = { group = "org.jetbrains.kotlinx", name = "kotlinx-serialization-json", version.ref = "kotlinxSerialization" }
+kotlinxCoroutinesCore = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "kotlinxCoroutines" }
 log4j12Api = { group = "org.apache.logging.log4j", name = "log4j-1.2-api", version.ref = "log4j" }
 msgpack = { group = "org.msgpack", name = "msgpack-core", version.ref = "msgpack" }
 nuValidator = { group = "nu.validator", name = "validator", version.ref = "nuValidator" }

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliException.kt

@@ -26,9 +26,13 @@ open class CliException(
    */
   message: String,
 
+  /** The cause */
+  cause: Throwable?,
+
   /** The process exit code to use. */
   val exitCode: Int = 1,
-) : RuntimeException(message) {
+) : RuntimeException(message, cause) {
+  constructor(message: String, exitCode: Int = 1) : this(message, null, exitCode)
 
   override fun toString(): String = message!!
 }
@@ -41,7 +45,11 @@ class CliBugException(
   /** The process exit code to use. */
   exitCode: Int = 1,
 ) :
-  CliException("An unexpected error has occurred. Would you mind filing a bug report?", exitCode) {
+  CliException(
+    "An unexpected error has occurred. Would you mind filing a bug report?",
+    theCause,
+    exitCode,
+  ) {
 
   override fun toString(): String = "$message\n\n${theCause.printStackTraceToString()}"
 }

pkl-commons/src/main/kotlin/org/pkl/commons/Control.kt

@@ -0,0 +1,33 @@
+/*
+ * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.commons
+
+import java.util.WeakHashMap
+import kotlin.reflect.KProperty
+
+// Adapted from https://stackoverflow.com/a/38084930
+fun <This, Return> lazyWithReceiver(
+  initializer: This.() -> Return
+): LazyWithReceiver<This, Return> = LazyWithReceiver(initializer)
+
+class LazyWithReceiver<This, out Return>(val initializer: This.() -> Return) {
+  private val values = WeakHashMap<This, Return>()
+
+  private val lock = Object()
+
+  operator fun getValue(thisValue: This, property: KProperty<*>): Return =
+    synchronized(lock) { values.getOrPut(thisValue) { thisValue.initializer() } }
+}

pkl-doc/gradle.lockfile

@@ -88,14 +88,19 @@ org.jetbrains.kotlin:kotlin-serialization-compiler-plugin-embeddable:2.0.21=kotl
 org.jetbrains.kotlin:kotlin-stdlib-jdk7:2.0.21=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.jetbrains.kotlin:kotlin-stdlib-jdk8:2.0.21=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.jetbrains.kotlin:kotlin-stdlib:2.0.21=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinKlibCommonizerClasspath,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.jetbrains.kotlinx:atomicfu:0.23.1=implementationDependenciesMetadata,testImplementationDependenciesMetadata
+org.jetbrains.kotlinx:kotlinx-coroutines-bom:1.10.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.10.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.6.4=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath
+org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.jetbrains.kotlinx:kotlinx-html-jvm:0.11.0=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.jetbrains.kotlinx:kotlinx-serialization-bom:1.8.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.jetbrains.kotlinx:kotlinx-serialization-core-jvm:1.8.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.jetbrains.kotlinx:kotlinx-serialization-core:1.8.0=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.jetbrains.kotlinx:kotlinx-serialization-json-jvm:1.8.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.jetbrains.kotlinx:kotlinx-serialization-json:1.8.0=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
-org.jetbrains:annotations:13.0=compileClasspath,kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinKlibCommonizerClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.jetbrains:annotations:13.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinKlibCommonizerClasspath
+org.jetbrains:annotations:23.0.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.jetbrains:markdown-jvm:0.7.3=runtimeClasspath,testRuntimeClasspath
 org.jetbrains:markdown:0.7.3=runtimeClasspath,testRuntimeClasspath
 org.jspecify:jspecify:1.0.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath

pkl-doc/pkl-doc.gradle.kts

@@ -46,6 +46,7 @@ dependencies {
     // aren't)
     exclude(group = "org.jetbrains.kotlin")
   }
+  implementation(libs.kotlinxCoroutinesCore) { exclude(group = "org.jetbrains.kotlin") }
 
   testImplementation(projects.pklCommonsTest)
   testImplementation(libs.jimfs)

pkl-doc/src/main/kotlin/org/pkl/doc/AbstractGenerator.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.doc
+
+import java.io.OutputStream
+
+abstract class AbstractGenerator(protected val consoleOut: OutputStream) {
+  protected fun writeOutputLine(message: String) {
+    consoleOut.writeLine(message)
+  }
+
+  protected fun writeOutput(message: String) {
+    consoleOut.write(message)
+  }
+}

pkl-doc/src/main/kotlin/org/pkl/doc/ClassPageGenerator.kt

@@ -15,6 +15,7 @@
  */
 package org.pkl.doc
 
+import java.io.OutputStream
 import kotlinx.html.*
 import org.pkl.core.PClass
 
@@ -25,7 +26,16 @@ internal class ClassPageGenerator(
   clazz: PClass,
   pageScope: ClassScope,
   isTestMode: Boolean,
-) : ModuleOrClassPageGenerator<ClassScope>(docsiteInfo, docModule, clazz, pageScope, isTestMode) {
+  consoleOut: OutputStream,
+) :
+  ModuleOrClassPageGenerator<ClassScope>(
+    docsiteInfo,
+    docModule,
+    clazz,
+    pageScope,
+    isTestMode,
+    consoleOut,
+  ) {
   override val html: HTML.() -> Unit = {
     renderHtmlHead()
 

pkl-doc/src/main/kotlin/org/pkl/doc/CliDocGenerator.kt

@@ -15,11 +15,13 @@
  */
 package org.pkl.doc
 
+import java.io.OutputStream
 import java.net.URI
 import java.net.URISyntaxException
 import java.nio.file.Path
 import kotlin.Pair
 import org.pkl.commons.cli.CliBaseOptions.Companion.getProjectFile
+import org.pkl.commons.cli.CliBugException
 import org.pkl.commons.cli.CliCommand
 import org.pkl.commons.cli.CliException
 import org.pkl.commons.toPath
@@ -33,7 +35,11 @@ import org.pkl.core.packages.*
  *
  * For the low-level Pkldoc API, see [DocGenerator].
  */
-class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(options.base) {
+class CliDocGenerator(
+  private val options: CliDocGeneratorOptions,
+  private val consoleOut: OutputStream = System.out,
+) : CliCommand(options.base) {
+  constructor(options: CliDocGeneratorOptions) : this(options, System.out)
 
   private val packageResolver =
     PackageResolver.getInstance(securityManager, httpClient, moduleCacheDir)
@@ -60,6 +66,17 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
           ),
     )
 
+  private val versions = mutableMapOf<String, Version>()
+
+  private val versionComparator =
+    Comparator<String> { v1, v2 ->
+      versions
+        .getOrPut(v1) { Version.parse(v1) }
+        .compareTo(versions.getOrPut(v2) { Version.parse(v2) })
+    }
+
+  private val docMigrator = DocMigrator(options.outputDir, System.out, versionComparator)
+
   private fun DependencyMetadata.getPackageDependencies(): List<DocPackageInfo.PackageDependency> {
     return buildList {
       for ((_, dependency) in dependencies) {
@@ -87,14 +104,12 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
   }
 
   private fun PackageUri.toDocPackageInfo(): DocPackageInfo {
-    val metadataAndChecksum =
+    val (metadata, checksum) =
       try {
         packageResolver.getDependencyMetadataAndComputeChecksum(this)
       } catch (e: PackageLoadError) {
         throw CliException("Failed to package metadata for $this: ${e.message}")
       }
-    val metadata = metadataAndChecksum.first
-    val checksum = metadataAndChecksum.second
     return DocPackageInfo(
       name = "${uri.authority}${uri.path.substringBeforeLast('@')}",
       moduleNamePrefix = "${metadata.name}.",
@@ -130,6 +145,15 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
   }
 
   override fun doRun() {
+    if (options.migrate) {
+      docMigrator.run()
+      return
+    }
+    if (!docMigrator.isUpToDate) {
+      throw CliException(
+        "pkldoc website model is too old (found: ${docMigrator.docsiteVersion}, required: ${DocMigrator.CURRENT_VERSION}). Run `pkldoc --migrate` to migrate the website."
+      )
+    }
     val docsiteInfoModuleUris = mutableListOf<URI>()
     val packageInfoModuleUris = mutableListOf<URI>()
     val regularModuleUris = mutableListOf<URI>()
@@ -271,8 +295,12 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
           options.normalizedOutputDir,
           options.isTestMode,
           options.noSymlinks,
+          consoleOut,
+          docMigrator,
         )
         .run()
+    } catch (e: DocGeneratorBugException) {
+      throw CliBugException(e)
     } catch (e: DocGeneratorException) {
       throw CliException(e.message!!)
     }

pkl-doc/src/main/kotlin/org/pkl/doc/CliDocGeneratorOptions.kt

@@ -47,6 +47,9 @@ constructor(
    * however, will create a full copy instead.
    */
   val noSymlinks: Boolean = false,
+
+  /** Migrate existing pkldoc */
+  val migrate: Boolean = false,
 ) {
   /** [outputDir] after undergoing normalization. */
   val normalizedOutputDir: Path = base.normalizedWorkingDir.resolveSafely(outputDir)