refactor(pnpm): Make Pnpm separate from Npm

Stop inheriting from `Npm` and rely entirely on the output of `pnpm`
commands to figure out the necessary information. For example, do not
re-construct the dependency from the file structure within
`node_modles`, but rely on `pnpm list` instead. This tremendously
reduces complexity and makes the implementation easy to understand and
maintain.

Apart from that it corrects issues with the dependency tree, such as:

1. htmlparser2 now dependends on domutils 1.7.0 instead of 1.5.1, which
   is inline with the dependency tree output by `pnpm list --depth 1000`
2. The cyclic reference from the workspace root project to itself is now
   included.

Signed-off-by: Frank Viernau <[email protected]>

Author: fviernau

plugins/package-managers/node/src/main/kotlin/Npm.kt

@@ -611,7 +611,7 @@ internal fun List<String>.groupLines(vararg markers: String): List<String> {
     }
 }
 
-private fun parseProject(packageJsonFile: File, analysisRoot: File, managerName: String): Project {
+internal fun parseProject(packageJsonFile: File, analysisRoot: File, managerName: String): Project {
     Npm.logger.debug { "Parsing project info from '$packageJsonFile'." }
 
     val packageJson = parsePackageJson(packageJsonFile)

plugins/package-managers/node/src/main/kotlin/pnpm/ModuleInfo.kt

@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.plugins.packagemanagers.node.pnpm
+
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.Json
+
+private val JSON = Json { ignoreUnknownKeys = true }
+
+internal fun parsePnpmList(json: String): List<ModuleInfo> = JSON.decodeFromString(json)
+
+@Serializable
+internal data class ModuleInfo(
+    val name: String,
+    val version: String,
+    val path: String,
+    val private: Boolean,
+    val dependencies: Map<String, Dependency> = emptyMap(),
+    val devDependencies: Map<String, Dependency> = emptyMap(),
+    val optionalDependencies: Map<String, Dependency> = emptyMap()
+) {
+    @Serializable
+    data class Dependency(
+        val from: String,
+        val version: String,
+        val resolved: String? = null,
+        val path: String,
+        val dependencies: Map<String, Dependency> = emptyMap(),
+        val optionalDependencies: Map<String, Dependency> = emptyMap()
+    )
+}

plugins/package-managers/node/src/main/kotlin/pnpm/Pnpm.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,13 +22,33 @@ package org.ossreviewtoolkit.plugins.packagemanagers.node.pnpm
 import java.io.File
 
 import org.ossreviewtoolkit.analyzer.AbstractPackageManagerFactory
+import org.ossreviewtoolkit.analyzer.PackageManager
+import org.ossreviewtoolkit.analyzer.PackageManager.Companion.processPackageVcs
+import org.ossreviewtoolkit.analyzer.PackageManagerResult
+import org.ossreviewtoolkit.downloader.VcsHost
+import org.ossreviewtoolkit.model.DependencyGraph
+import org.ossreviewtoolkit.model.Hash
+import org.ossreviewtoolkit.model.Identifier
+import org.ossreviewtoolkit.model.Package
+import org.ossreviewtoolkit.model.ProjectAnalyzerResult
+import org.ossreviewtoolkit.model.RemoteArtifact
 import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
 import org.ossreviewtoolkit.model.config.RepositoryConfiguration
-import org.ossreviewtoolkit.plugins.packagemanagers.node.Npm
+import org.ossreviewtoolkit.model.utils.DependencyGraphBuilder
+import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackageJson
+import org.ossreviewtoolkit.plugins.packagemanagers.node.parseProject
+import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NON_EXISTING_SEMVER
 import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NodePackageManager
 import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NpmDetection
+import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.expandNpmShortcutUrl
+import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.fixNpmDownloadUrl
+import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.mapNpmLicenses
+import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.parseNpmAuthor
+import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.parseNpmVcsInfo
+import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.splitNpmNamespaceAndName
+import org.ossreviewtoolkit.utils.common.CommandLineTool
 import org.ossreviewtoolkit.utils.common.Os
-import org.ossreviewtoolkit.utils.common.realFile
+import org.ossreviewtoolkit.utils.common.stashDirectories
 
 import org.semver4j.RangesList
 import org.semver4j.RangesListFactory
@@ -41,7 +61,7 @@ class Pnpm(
     analysisRoot: File,
     analyzerConfig: AnalyzerConfiguration,
     repoConfig: RepositoryConfiguration
-) : Npm(name, analysisRoot, analyzerConfig, repoConfig) {
+) : PackageManager(name, analysisRoot, analyzerConfig, repoConfig), CommandLineTool {
     class Factory : AbstractPackageManagerFactory<Pnpm>("PNPM") {
         override val globsForDefinitionFiles = listOf("package.json", "pnpm-lock.yaml")
 
@@ -52,14 +72,44 @@ class Pnpm(
         ) = Pnpm(type, analysisRoot, analyzerConfig, repoConfig)
     }
 
-    override fun hasLockfile(projectDir: File) = NodePackageManager.PNPM.hasLockfile(projectDir)
+    private val handler = PnpmDependencyHandler()
+    private val graphBuilder by lazy { DependencyGraphBuilder(handler) }
 
-    override fun File.isWorkspaceDir() = realFile() in findWorkspaceSubmodules(analysisRoot)
+    override fun resolveDependencies(definitionFile: File, labels: Map<String, String>): List<ProjectAnalyzerResult> =
+        stashDirectories(definitionFile.resolveSibling("node_modules")).use {
+            resolveDependencies(definitionFile)
+        }
+
+    private fun resolveDependencies(definitionFile: File): List<ProjectAnalyzerResult> {
+        val workingDir = definitionFile.parentFile
+        installDependencies(workingDir)
+
+        val workspaceModuleDirs = getWorkspaceModuleDirs(workingDir)
+        handler.setWorkspaceModuleDirs(workspaceModuleDirs)
+
+        val moduleInfosForScope = Scope.entries.associateWith { scope -> listModules(workingDir, scope) }
+
+        return workspaceModuleDirs.map { projectDir ->
+            val project = parseProject(projectDir.resolve("package.json"), analysisRoot, managerName)
+
+            val scopeNames = Scope.entries.mapTo(mutableSetOf()) { scope ->
+                val scopeName = scope.toString()
+                val qualifiedScopeName = DependencyGraph.qualifyScope(project, scope.toString())
+                val moduleInfo = moduleInfosForScope.getValue(scope).single { it.path == projectDir.absolutePath }
 
-    override fun loadWorkspaceSubmodules(moduleDir: File): Set<File> {
-        val process = run(moduleDir, "list", "--recursive", "--depth=-1", "--parseable")
+                moduleInfo.getScopeDependencies(scope).forEach { dependency ->
+                    graphBuilder.addDependency(qualifiedScopeName, dependency)
+                }
 
-        return process.stdout.lines().filter { it.isNotEmpty() }.mapTo(mutableSetOf()) { File(it) }
+                scopeName
+            }
+
+            ProjectAnalyzerResult(
+                project = project.copy(scopeNames = scopeNames),
+                packages = emptySet(),
+                issues = emptyList()
+            )
+        }
     }
 
     override fun command(workingDir: File?) =
@@ -69,18 +119,37 @@ class Pnpm(
             "/home/frank/.nvm/versions/node/v18.12.1/bin/node /home/frank/.nvm/versions/node/v18.12.1/bin/pnpm"
         }
 
+    private fun getWorkspaceModuleDirs(workingDir: File): Set<File> {
+        val json = run(workingDir, "list", "--json", "--only-projects", "--recursive").stdout
+
+        return parsePnpmList(json).mapTo(mutableSetOf()) { File(it.path) }
+    }
+
+    private fun listModules(workingDir: File, scope: Scope): List<ModuleInfo> {
+        val scopeOption = when (scope) {
+            Scope.DEPENDENCIES -> "--prod"
+            Scope.DEV_DEPENDENCIES -> "--dev"
+        }
+
+        val json = run(workingDir, "list", "--json", "--recursive", "--depth", "10000", scopeOption).stdout
+
+        return parsePnpmList(json)
+    }
+
+    override fun createPackageManagerResult(projectResults: Map<File, List<ProjectAnalyzerResult>>) =
+        PackageManagerResult(projectResults, graphBuilder.build(), graphBuilder.packages())
+
     override fun getVersionRequirement(): RangesList = RangesListFactory.create("5.* - 9.*")
 
     override fun mapDefinitionFiles(definitionFiles: List<File>) =
         NpmDetection(definitionFiles).filterApplicable(NodePackageManager.PNPM)
 
-    override fun runInstall(workingDir: File) =
+    private fun installDependencies(workingDir: File) =
         run(
             "install",
             "--ignore-pnpmfile",
             "--ignore-scripts",
             "--frozen-lockfile", // Use the existing lockfile instead of updating an outdated one.
-            "--shamefully-hoist", // Build a similar node_modules structure as NPM and Yarn does.
             workingDir = workingDir
         )
 
@@ -89,3 +158,84 @@ class Pnpm(
         // fixed major version to be sure to get consistent results.
         checkVersion()
 }
+
+private enum class Scope(val descriptor: String) {
+    DEPENDENCIES("dependencies"),
+    DEV_DEPENDENCIES("devDependencies");
+
+    override fun toString(): String = descriptor
+}
+
+private fun ModuleInfo.getScopeDependencies(scope: Scope) =
+    when (scope) {
+        Scope.DEPENDENCIES -> buildList {
+            addAll(dependencies.values)
+            addAll(optionalDependencies.values)
+        }
+
+        Scope.DEV_DEPENDENCIES -> devDependencies.values.toList()
+    }
+
+internal fun parsePackage(packageJsonFile: File): Package {
+    val packageJson = parsePackageJson(packageJsonFile)
+
+    // The "name" and "version" fields are only required if the package is going to be published, otherwise they are
+    // optional, see
+    // - https://docs.npmjs.com/cli/v10/configuring-npm/package-json#name
+    // - https://docs.npmjs.com/cli/v10/configuring-npm/package-json#version
+    // So, projects analyzed by ORT might not have these fields set.
+    val rawName = packageJson.name.orEmpty() // TODO: Fall back to a generated name if the name is unset.
+    val (namespace, name) = splitNpmNamespaceAndName(rawName)
+    val version = packageJson.version ?: NON_EXISTING_SEMVER
+
+    val declaredLicenses = packageJson.licenses.mapNpmLicenses()
+    val authors = parseNpmAuthor(packageJson.authors.firstOrNull()) // TODO: parse all authors.
+
+    var description = packageJson.description.orEmpty()
+    var homepageUrl = packageJson.homepage.orEmpty()
+
+    // Note that all fields prefixed with "_" are considered private to NPM and should not be relied on.
+    var downloadUrl = expandNpmShortcutUrl(packageJson.resolved.orEmpty()).ifEmpty {
+        // If the normalized form of the specified dependency contains a URL as the version, expand and use it.
+        val fromVersion = packageJson.from.orEmpty().substringAfterLast('@')
+        expandNpmShortcutUrl(fromVersion).takeIf { it != fromVersion }.orEmpty()
+    }
+
+    var hash = Hash.create(packageJson.integrity.orEmpty())
+
+    var vcsFromPackage = parseNpmVcsInfo(packageJson)
+
+    val id = Identifier("NPM", namespace, name, version)
+
+    downloadUrl = downloadUrl.fixNpmDownloadUrl()
+
+    val vcsFromDownloadUrl = VcsHost.parseUrl(downloadUrl)
+    if (vcsFromDownloadUrl.url != downloadUrl) {
+        vcsFromPackage = vcsFromPackage.merge(vcsFromDownloadUrl)
+    }
+
+    val module = Package(
+        id = id,
+        authors = authors,
+        declaredLicenses = declaredLicenses,
+        description = description,
+        homepageUrl = homepageUrl,
+        binaryArtifact = RemoteArtifact.EMPTY,
+        sourceArtifact = RemoteArtifact(
+            url = VcsHost.toArchiveDownloadUrl(vcsFromDownloadUrl) ?: downloadUrl,
+            hash = hash
+        ),
+        vcs = vcsFromPackage,
+        vcsProcessed = processPackageVcs(vcsFromPackage, homepageUrl)
+    )
+
+    require(module.id.name.isNotEmpty()) {
+        "Generated package info for '${id.toCoordinates()}' has no name."
+    }
+
+    require(module.id.version.isNotEmpty()) {
+        "Generated package info for '${id.toCoordinates()}' has no version."
+    }
+
+    return module
+}

plugins/package-managers/node/src/main/kotlin/pnpm/PnpmDependencyHandler.kt

@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.plugins.packagemanagers.node.pnpm
+
+import java.io.File
+
+import org.ossreviewtoolkit.model.Identifier
+import org.ossreviewtoolkit.model.Issue
+import org.ossreviewtoolkit.model.Package
+import org.ossreviewtoolkit.model.PackageLinkage
+import org.ossreviewtoolkit.model.utils.DependencyHandler
+import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackageJson
+import org.ossreviewtoolkit.plugins.packagemanagers.node.pnpm.ModuleInfo.Dependency
+import org.ossreviewtoolkit.utils.common.realFile
+
+internal class PnpmDependencyHandler : DependencyHandler<Dependency> {
+    private val workspaceModuleDirs = mutableSetOf<File>()
+
+    private fun Dependency.isProject() = File(path).realFile().absoluteFile in workspaceModuleDirs
+
+    fun setWorkspaceModuleDirs(dirs: Collection<File>) {
+        workspaceModuleDirs.apply {
+            clear()
+            addAll(dirs)
+        }
+    }
+
+    override fun identifierFor(dependency: Dependency): Identifier {
+        val type = "PNPM".takeIf { dependency.isProject() } ?: "NPM"
+        val namespace = dependency.from.substringBeforeLast("/", "")
+        val name = dependency.from.substringAfterLast("/")
+        val version = if (dependency.isProject()) {
+            // TODO: Read each package.json only once?! (use caching).
+            parsePackageJson(File(dependency.path).resolve("package.json")).version.orEmpty()
+        } else {
+            dependency.version.takeUnless { it.startsWith("link:") || it.startsWith("file:") }.orEmpty()
+        }
+
+        return Identifier(type, namespace, name, version)
+    }
+
+    override fun dependenciesFor(dependency: Dependency): List<Dependency> =
+        (dependency.dependencies + dependency.optionalDependencies).values.toList()
+
+    override fun linkageFor(dependency: Dependency): PackageLinkage =
+        PackageLinkage.DYNAMIC.takeUnless { dependency.isProject() } ?: PackageLinkage.PROJECT_DYNAMIC
+
+    override fun createPackage(dependency: Dependency, issues: MutableCollection<Issue>): Package? =
+        dependency.takeUnless { it.isProject() }?.let { parsePackage(File(it.path).resolve("package.json")) }
+}