From 311ade437ee572903bf30b4389e50fb3e955440c Mon Sep 17 00:00:00 2001 From: Nicolas Nobelis Date: Fri, 24 Jan 2025 18:26:21 +0100 Subject: [PATCH 1/2] test(ctrlx-reporter): Improve the functional test This new test does not only check if a report can be generated, it actually verifies the content of the report. Signed-off-by: Nicolas Nobelis --- .../kotlin/CtrlXAutomationReporterFunTest.kt | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/plugins/reporters/ctrlx/src/funTest/kotlin/CtrlXAutomationReporterFunTest.kt b/plugins/reporters/ctrlx/src/funTest/kotlin/CtrlXAutomationReporterFunTest.kt index 6af5dfabe4439..aad913fcf1d51 100644 --- a/plugins/reporters/ctrlx/src/funTest/kotlin/CtrlXAutomationReporterFunTest.kt +++ b/plugins/reporters/ctrlx/src/funTest/kotlin/CtrlXAutomationReporterFunTest.kt @@ -23,15 +23,34 @@ import io.kotest.core.spec.style.StringSpec import io.kotest.engine.spec.tempdir import io.kotest.matchers.collections.haveSize import io.kotest.matchers.collections.shouldBeSingleton +import io.kotest.matchers.collections.shouldHaveSize import io.kotest.matchers.nulls.shouldNotBeNull import io.kotest.matchers.result.shouldBeSuccess import io.kotest.matchers.should +import io.kotest.matchers.shouldBe + +import java.io.File import kotlinx.serialization.json.decodeFromStream +import org.ossreviewtoolkit.model.AnalyzerResult +import org.ossreviewtoolkit.model.AnalyzerRun +import org.ossreviewtoolkit.model.DependencyGraph +import org.ossreviewtoolkit.model.DependencyReference +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.model.OrtResult +import org.ossreviewtoolkit.model.Package +import org.ossreviewtoolkit.model.Project +import org.ossreviewtoolkit.model.Repository +import org.ossreviewtoolkit.model.RootDependencyIndex +import org.ossreviewtoolkit.model.Scope +import org.ossreviewtoolkit.model.VcsInfo +import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.plugins.reporters.ctrlx.CtrlXAutomationReporter.Companion.REPORT_FILENAME import org.ossreviewtoolkit.reporter.ORT_RESULT import org.ossreviewtoolkit.reporter.ReporterInput +import org.ossreviewtoolkit.utils.ort.createOrtTempDir +import org.ossreviewtoolkit.utils.spdx.toSpdx import org.ossreviewtoolkit.utils.test.getAssetFile class CtrlXAutomationReporterFunTest : StringSpec({ @@ -52,4 +71,87 @@ class CtrlXAutomationReporterFunTest : StringSpec({ it shouldBeSuccess outputDir.resolve(REPORT_FILENAME) } } + + "Generating a report works and produces a valid fossinfo.json" { + val reporter = CtrlXAutomationReporter() + val input = createReporterInput() + val outputDir = createOrtTempDir("ctrlx-automation-reporter-test") + + val reporterResult = reporter.generateReport(input, outputDir) + + validateReport(reporterResult) { + components shouldNotBeNull { + this shouldHaveSize 2 + first().name shouldBe "package1" + last().name shouldBe "package2" + } + } + } }) + +private fun validateReport(reporterResult: List>, validate: FossInfo.() -> Unit) { + reporterResult.shouldBeSingleton { result -> + result shouldBeSuccess { file -> + file.name shouldBe "fossinfo.json" + val fossInfo = file.inputStream().use { + CtrlXAutomationReporter.JSON.decodeFromStream(it) + } + + fossInfo.validate() + } + } +} + +private fun createReporterInput(): ReporterInput { + val analyzedVcs = VcsInfo( + type = VcsType.GIT, + revision = "master", + url = "https://github.com/path/first-project.git", + path = "sub/path" + ) + + val package1 = Package.EMPTY.copy( + id = Identifier("Maven:ns:package1:1.0"), + declaredLicenses = setOf("LicenseRef-scancode-broadcom-commercial"), + concludedLicense = "LicenseRef-scancode-broadcom-commercial".toSpdx() + ) + val package2 = Package.EMPTY.copy( + id = Identifier("Maven:ns:package2:1.0"), + declaredLicenses = setOf("MIT"), + concludedLicense = "MIT".toSpdx() + ) + val project = Project.EMPTY.copy( + id = Identifier.EMPTY.copy(name = "test-project"), + scopeDependencies = setOf( + Scope("scope-1", setOf(package1.toReference(), package2.toReference())) + ), + vcs = analyzedVcs, + vcsProcessed = analyzedVcs + ) + + return ReporterInput( + OrtResult( + repository = Repository( + vcs = analyzedVcs, + vcsProcessed = analyzedVcs + ), + analyzer = AnalyzerRun.EMPTY.copy( + result = AnalyzerResult( + projects = setOf(project), + packages = setOf(package1, package2), + dependencyGraphs = mapOf( + "test" to DependencyGraph( + listOf(package1.id, package2.id), + sortedSetOf( + DependencyGraph.DEPENDENCY_REFERENCE_COMPARATOR, + DependencyReference(0), + DependencyReference(1) + ), + mapOf(DependencyGraph.qualifyScope(project.id, "scope-1") to listOf(RootDependencyIndex(0))) + ) + ) + ) + ) + ) + ) +} From 8f46a6d205b3828a4d861845c12b09e4ae1a9e38 Mon Sep 17 00:00:00 2001 From: Nicolas Nobelis Date: Mon, 27 Jan 2025 08:29:20 +0100 Subject: [PATCH 2/2] feat(ctrlx-reporter): Allow license filtering based on classifications The license terms of some proprietary licenses forbid the license to be disclosed. This commit adds a new optional parameter to the CtrlX reporter to specify the categories for which the licenses are included in the report. If a component has a license which has a category not present in this parameter, the license is removed from the component and not visible in the report. If a component has ALL its licenses removed this way, it is not displayed in the report. If the parameter is not set for the reporter, all components and all licenses are present in the report. Signed-off-by: Nicolas Nobelis --- model/src/main/resources/reference.yml | 4 ++ .../kotlin/config/OrtConfigurationTest.kt | 9 ++- .../kotlin/CtrlXAutomationReporterFunTest.kt | 36 ++++++++++- .../main/kotlin/CtrlXAutomationReporter.kt | 64 ++++++++++++++----- 4 files changed, 95 insertions(+), 18 deletions(-) diff --git a/model/src/main/resources/reference.yml b/model/src/main/resources/reference.yml index a036bb4446237..5e94a3bb9711e 100644 --- a/model/src/main/resources/reference.yml +++ b/model/src/main/resources/reference.yml @@ -382,6 +382,10 @@ ort: user: user apiKey: XYZ + CtrlXAutomation: + options: + licenseCategoriesToInclude: 'include-in-disclosure-document' + notifier: mail: hostName: 'localhost' diff --git a/model/src/test/kotlin/config/OrtConfigurationTest.kt b/model/src/test/kotlin/config/OrtConfigurationTest.kt index 92174a372d9a6..7068b1e9f5d4c 100644 --- a/model/src/test/kotlin/config/OrtConfigurationTest.kt +++ b/model/src/test/kotlin/config/OrtConfigurationTest.kt @@ -372,7 +372,7 @@ class OrtConfigurationTest : WordSpec({ with(ortConfig.reporter) { config shouldNotBeNull { - keys shouldContainExactlyInAnyOrder setOf("CycloneDx", "FossId") + keys shouldContainExactlyInAnyOrder setOf("CycloneDx", "FossId", "CtrlXAutomation") get("CycloneDx") shouldNotBeNull { options shouldContainExactly mapOf( @@ -390,6 +390,13 @@ class OrtConfigurationTest : WordSpec({ "apiKey" to "XYZ" ) } + + get("CtrlXAutomation") shouldNotBeNull { + options shouldContainExactly mapOf( + "licenseCategoriesToInclude" to "include-in-disclosure-document" + ) + secrets should beEmpty() + } } } diff --git a/plugins/reporters/ctrlx/src/funTest/kotlin/CtrlXAutomationReporterFunTest.kt b/plugins/reporters/ctrlx/src/funTest/kotlin/CtrlXAutomationReporterFunTest.kt index aad913fcf1d51..abfe9ea9c508f 100644 --- a/plugins/reporters/ctrlx/src/funTest/kotlin/CtrlXAutomationReporterFunTest.kt +++ b/plugins/reporters/ctrlx/src/funTest/kotlin/CtrlXAutomationReporterFunTest.kt @@ -46,10 +46,14 @@ import org.ossreviewtoolkit.model.RootDependencyIndex import org.ossreviewtoolkit.model.Scope import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.licenses.LicenseCategorization +import org.ossreviewtoolkit.model.licenses.LicenseCategory +import org.ossreviewtoolkit.model.licenses.LicenseClassifications import org.ossreviewtoolkit.plugins.reporters.ctrlx.CtrlXAutomationReporter.Companion.REPORT_FILENAME import org.ossreviewtoolkit.reporter.ORT_RESULT import org.ossreviewtoolkit.reporter.ReporterInput import org.ossreviewtoolkit.utils.ort.createOrtTempDir +import org.ossreviewtoolkit.utils.spdx.SpdxSingleLicenseExpression import org.ossreviewtoolkit.utils.spdx.toSpdx import org.ossreviewtoolkit.utils.test.getAssetFile @@ -65,7 +69,7 @@ class CtrlXAutomationReporterFunTest : StringSpec({ "Generating a report works" { val outputDir = tempdir() - val reportFiles = CtrlXAutomationReporter().generateReport(ReporterInput(ORT_RESULT), outputDir) + val reportFiles = CtrlXAutomationReporterFactory.create().generateReport(ReporterInput(ORT_RESULT), outputDir) reportFiles.shouldBeSingleton { it shouldBeSuccess outputDir.resolve(REPORT_FILENAME) @@ -73,7 +77,7 @@ class CtrlXAutomationReporterFunTest : StringSpec({ } "Generating a report works and produces a valid fossinfo.json" { - val reporter = CtrlXAutomationReporter() + val reporter = CtrlXAutomationReporterFactory.create() val input = createReporterInput() val outputDir = createOrtTempDir("ctrlx-automation-reporter-test") @@ -87,6 +91,34 @@ class CtrlXAutomationReporterFunTest : StringSpec({ } } } + + "The reporter should only include licenses with the given category" { + val category = "include-in-disclosure-document" + val categorizations = listOf( + LicenseCategorization( + SpdxSingleLicenseExpression.parse("MIT"), + setOf(category) + ) + ) + val categories = listOf(LicenseCategory(category)) + val input = createReporterInput().copy( + licenseClassifications = LicenseClassifications( + categories = categories, + categorizations = categorizations + ) + ) + val reporter = CtrlXAutomationReporterFactory.create(listOf(category)) + val outputDir = createOrtTempDir("ctrlx-automation-reporter-test") + + val reporterResult = reporter.generateReport(input, outputDir) + + validateReport(reporterResult) { + components shouldNotBeNull { + this shouldHaveSize 1 + first().name shouldBe "package2" + } + } + } }) private fun validateReport(reporterResult: List>, validate: FossInfo.() -> Unit) { diff --git a/plugins/reporters/ctrlx/src/main/kotlin/CtrlXAutomationReporter.kt b/plugins/reporters/ctrlx/src/main/kotlin/CtrlXAutomationReporter.kt index 99185b1bbeb9c..91145017913d5 100644 --- a/plugins/reporters/ctrlx/src/main/kotlin/CtrlXAutomationReporter.kt +++ b/plugins/reporters/ctrlx/src/main/kotlin/CtrlXAutomationReporter.kt @@ -32,13 +32,27 @@ import org.ossreviewtoolkit.reporter.ReporterFactory import org.ossreviewtoolkit.reporter.ReporterInput import org.ossreviewtoolkit.utils.spdx.SpdxConstants import org.ossreviewtoolkit.utils.spdx.SpdxLicense +import org.ossreviewtoolkit.utils.spdx.toSpdx + +data class CtrlXAutomationReporterConfig( + /** + * The categories of the licenses of the packages to include in the report. If a component has a license which has a + * category not present in this parameter, the license is removed from the component and not visible in the report. + * If a component has ALL its licenses removed this way, it is not displayed in the report. If the parameter is not + * set for the reporter, all components and all licenses are present in the report. + */ + val licenseCategoriesToInclude: List? +) @OrtPlugin( displayName = "CtrlX Automation Reporter", description = "A reporter for the ctrlX Automation format.", factory = ReporterFactory::class ) -class CtrlXAutomationReporter(override val descriptor: PluginDescriptor = CtrlXAutomationReporterFactory.descriptor) : +class CtrlXAutomationReporter( + override val descriptor: PluginDescriptor = CtrlXAutomationReporterFactory.descriptor, + private val config: CtrlXAutomationReporterConfig +) : Reporter { companion object { const val REPORT_FILENAME = "fossinfo.json" @@ -54,7 +68,11 @@ class CtrlXAutomationReporter(override val descriptor: PluginDescriptor = CtrlXA override fun generateReport(input: ReporterInput, outputDir: File): List> { val packages = input.ortResult.getPackages(omitExcluded = true) - val components = packages.mapTo(mutableListOf()) { (pkg, _) -> + val licensesToInclude = config.licenseCategoriesToInclude?.flatMap { + input.licenseClassifications.licensesByCategory[it].orEmpty() + }.orEmpty() + + val components = packages.mapNotNullTo(mutableListOf()) { (pkg, _) -> val qualifiedName = when (pkg.id.type) { // At least for NPM packages, CtrlX requires the component name to be prefixed with the scope name, // separated with a slash. Other package managers might require similar handling, but there seems to be @@ -73,25 +91,41 @@ class CtrlXAutomationReporter(override val descriptor: PluginDescriptor = CtrlXA input.ortResult.getPackageLicenseChoices(pkg.id), input.ortResult.getRepositoryLicenseChoices() ) - val licenses = effectiveLicense?.decompose()?.map { + var licenses = effectiveLicense?.decompose()?.map { val name = it.toString() val spdxId = SpdxLicense.forId(name)?.id val text = input.licenseTextProvider.getLicenseText(name) License(name = name, spdx = spdxId, text = text.orEmpty()) } - // The specification requires at least one license. - val componentLicenses = licenses.orEmpty().ifEmpty { listOf(LICENSE_NOASSERTION) } - - Component( - name = qualifiedName, - version = pkg.id.version, - homepage = pkg.homepageUrl.takeUnless { it.isEmpty() }, - copyright = copyrights?.let { CopyrightInformation(it) }, - licenses = componentLicenses, - usage = if (pkg.isModified) Usage.Modified else Usage.AsIs - // TODO: Map the PackageLinkage to an IntegrationMechanism. - ) + var componentShouldBeExcluded = false + + if (config.licenseCategoriesToInclude != null) { + val filteredLicenses = licenses?.filter { it.name.toSpdx() in licensesToInclude } + + if (filteredLicenses != null && filteredLicenses.isEmpty()) { + componentShouldBeExcluded = true + } else { + licenses = filteredLicenses + } + } + + if (componentShouldBeExcluded) { + null + } else { + // The specification requires at least one license. + val componentLicenses = licenses.orEmpty().ifEmpty { listOf(LICENSE_NOASSERTION) } + + Component( + name = qualifiedName, + version = pkg.id.version, + homepage = pkg.homepageUrl.takeUnless { it.isEmpty() }, + copyright = copyrights?.let { CopyrightInformation(it) }, + licenses = componentLicenses, + usage = if (pkg.isModified) Usage.Modified else Usage.AsIs + // TODO: Map the PackageLinkage to an IntegrationMechanism. + ) + } } val reportFileResult = runCatching {