SOLR-17406: Introduce Version Catalogs (#2706)

Introduce project-wide Gradle versions catalogs. Read the updated docs for more information.

Breaking Changes: Any dependencies added from now on are no longer added to versions.props (file removed), but to gradle/libs.versions.toml. When referencing a dependency in a gradle module, use the version catalog instead of hardcoded strings.

Dependencies are now locked with gradlew writeLocks (subject to change).

Author: malliaridis

build-tools/build-infra/build.gradle

@@ -17,7 +17,7 @@
 
 plugins {
   id "java-gradle-plugin"
-  id 'com.diffplug.spotless' version '6.5.2' apply false
+  alias(libs.plugins.diffplug.spotless) apply false
 }
 
 repositories {
@@ -34,12 +34,9 @@ apply from: file('../../gradle/validation/check-environment.gradle')
 tasks.register("checkJdkInternalsExportedToGradle") {}
 apply from: file('../../gradle/validation/spotless.gradle')
 
-// Load common script dependencies.
-apply from: file("../scriptDepVersions.gradle")
-
 java {
-  sourceCompatibility = scriptDepVersions['min-java-version']
-  targetCompatibility = scriptDepVersions['min-java-version']
+  sourceCompatibility = JavaVersion.toVersion(libs.versions.java.min.get())
+  targetCompatibility = JavaVersion.toVersion(libs.versions.java.min.get())
 }
 
 gradlePlugin {
@@ -57,5 +54,5 @@ dependencies {
   implementation gradleApi()
   implementation localGroovy()
 
-  implementation "commons-codec:commons-codec:${scriptDepVersions['commons-codec']}"
+  implementation libs.commonscodec.commonscodec
 }

build-tools/build-infra/settings.gradle

@@ -16,3 +16,12 @@
  */
 
 rootProject.name = 'build-infra'
+
+// Use project's version catalog for centralized dependency management
+dependencyResolutionManagement {
+    versionCatalogs {
+        libs {
+            from(files("../../gradle/libs.versions.toml"))
+        }
+    }
+}

build-tools/missing-doclet/build.gradle

@@ -17,14 +17,13 @@
 
 plugins {
   id 'java-library'
-  id 'com.diffplug.spotless' version '6.5.2' apply false
+  alias(libs.plugins.diffplug.spotless) apply false
 }
 
 repositories {
   mavenCentral()
 }
 
-version = "1.0.0-SNAPSHOT"
 group = "org.apache.solr.tools"
 description = 'Doclet-based javadoc validation'
 
@@ -36,12 +35,9 @@ apply from: file('../../gradle/validation/check-environment.gradle')
 tasks.register("checkJdkInternalsExportedToGradle") {}
 apply from: file('../../gradle/validation/spotless.gradle')
 
-// Load common script dependencies.
-apply from: file("../scriptDepVersions.gradle")
-
 java {
-  sourceCompatibility = scriptDepVersions['min-java-version']
-  targetCompatibility = scriptDepVersions['min-java-version']
+  sourceCompatibility = JavaVersion.toVersion(libs.versions.java.min.get())
+  targetCompatibility = JavaVersion.toVersion(libs.versions.java.min.get())
 }
 
 tasks.withType(JavaCompile).configureEach {

build-tools/missing-doclet/settings.gradle

@@ -16,3 +16,12 @@
  */
 
 rootProject.name = "missing-doclet"
+
+// Use project's version catalog for centralized dependency management
+dependencyResolutionManagement {
+    versionCatalogs {
+        libs {
+            from(files("../../gradle/libs.versions.toml"))
+        }
+    }
+}

build-tools/scriptDepVersions.gradle

@@ -21,21 +21,23 @@ import java.time.format.DateTimeFormatter
 plugins {
   id 'base'
   id 'solr.build-infra'
-  id 'com.palantir.consistent-versions' version '2.16.0'
-  id 'org.owasp.dependencycheck' version '9.0.8'
-  id 'ca.cutterslade.analyze' version '1.10.0'
-  id 'de.thetaphi.forbiddenapis' version '3.7' apply false
-  id 'de.undercouch.download' version '5.5.0' apply false
-  id 'net.ltgt.errorprone' version '3.1.0' apply false
-  id 'com.diffplug.spotless' version '6.5.2' apply false
-  id 'com.github.node-gradle.node' version '7.0.1' apply false
-}
 
-apply from: file('build-tools/scriptDepVersions.gradle')
+  alias(libs.plugins.carrotsearch.dependencychecks)
+  alias(libs.plugins.owasp.dependencycheck)
+  alias(libs.plugins.cutterslade.analyze)
+  alias(libs.plugins.benmanes.versions)
+  alias(libs.plugins.littlerobots.versioncatalogupdate) apply false
+  alias(libs.plugins.thetaphi.forbiddenapis) apply false
+  alias(libs.plugins.undercouch.download) apply false
+  alias(libs.plugins.ltgt.errorprone) apply false
+  alias(libs.plugins.diffplug.spotless) apply false
+  alias(libs.plugins.nodegradle.node) apply false
+  alias(libs.plugins.openapi.generator) apply false
+}
 
 // Declare default Java versions for the entire project and for SolrJ separately
-rootProject.ext.minJavaVersionDefault = JavaVersion.toVersion(scriptDepVersions['min-java-version'])
-rootProject.ext.minJavaVersionSolrJ = JavaVersion.toVersion(scriptDepVersions['min-solrj-java-version'])
+rootProject.ext.minJavaVersionDefault = JavaVersion.toVersion(libs.versions.java.min.get())
+rootProject.ext.minJavaVersionSolrJ = JavaVersion.toVersion(libs.versions.java.solrj.get())
 
 apply from: file('gradle/globals.gradle')
 
@@ -100,7 +102,7 @@ ext {
   }
 
   luceneBaseVersionProvider = project.provider { 
-    def luceneVersion = getVersion('org.apache.lucene:lucene-core')
+    def luceneVersion = libs.versions.apache.lucene.get()
     def m = (luceneVersion =~ /^\d+\.\d+\.\d+\b/)
     if (!m) {
       throw GradleException("Can't strip base version from " + luceneVersion)
@@ -150,7 +152,6 @@ apply from: file('gradle/validation/precommit.gradle')
 apply from: file('gradle/validation/forbidden-apis.gradle')
 apply from: file('gradle/validation/jar-checks.gradle')
 apply from: file('gradle/validation/git-status.gradle')
-apply from: file('gradle/validation/versions-props-sorted.gradle')
 apply from: file('gradle/validation/validate-source-patterns.gradle')
 apply from: file('gradle/validation/rat-sources.gradle')
 apply from: file('gradle/validation/owasp-dependency-check.gradle')
@@ -161,7 +162,7 @@ apply from: file('gradle/validation/validate-log-calls.gradle')
 apply from: file('gradle/validation/check-broken-links.gradle')
 
 apply from: file('gradle/validation/solr.config-file-sanity.gradle')
-
+apply from: file('gradle/validation/dependencies.gradle')
 apply from: file('gradle/validation/spotless.gradle')
 
 // Wire up included builds to some validation tasks.

build.gradle

@@ -16,30 +16,57 @@
 // specific language governing permissions and limitations
 // under the License.
 
-Solr has lots of 3rd party dependencies, defined mainly in `versions.props`.
+Solr has lots of 3rd party dependencies, defined in `gradle/libs.versions.toml`.
 Keeping them up-to-date is crucial for a number of reasons:
 
 * minimizing the risk of critical CVE vulnerabilities by staying on a recent and supported version
 * avoiding "dependency hell", that can arise from falling too far behind
 
-Read the https://github.com/apache/solr/blob/main/help/dependencies.txt[help/dependencies.txt] file for an in-depth explanation of how gradle is deployed in Solr, using
-https://github.com/palantir/gradle-consistent-versions[Gradle consistent-versions] plugin.
+Read the https://github.com/apache/solr/blob/main/help/dependencies.txt[help/dependencies.txt] file for an in-depth
+explanation of how dependencies are managed.
 
 == Manual dependency upgrades
 In order to upgrade a dependency, you need to run through a number of steps:
 
 1. Identify the available versions from e.g. https://search.maven.org[Maven Central]
-2. Update the version in `versions.props` file
-3. Run `./gradlew --write-locks` to re-generate `versions.lock`. Note that this may cause a cascading effect where
+2. Update the version in `gradle/libs.versions.toml` file
+3. Run `./gradlew writeLocks` to re-generate `versions.lock`. Note that this may cause a cascading effect where
    the locked version of other dependencies also change.
-4. Run `./gradlew updateLicenses` to re-generate SHA1 checksums of the new jar files.
-5. Once in a while, a new version of a dependency will transitively bring in brand-new dependencies.
+4. In case of a conflict, resolve the conflict according to `help/dependencies.txt`
+5. Check if there are any constraints that are obsolete after the dependency update
+6. Update the license and notice files of the changed dependencies. See `help/dependencies.txt` for
+   details.
+7. Run `./gradlew updateLicenses` to re-generate SHA1 checksums of the new jar files.
+8. Once in a while, a new version of a dependency will transitively bring in brand-new dependencies.
    You'll need to decide whether to keep or exclude them. See `help/dependencies.txt` for details.
 
+=== Reviewing Constraints
+
+The constraints are defined in gradle/validation/dependencies.gradle. There, if the updated dependency is listed,
+the constraint can be reviewed, updated or removed.
+
+The constraints fall into two "groups". In the first group there are dependency constraints from dependencies
+that our project directly includes and require version alignment to sync the versions across all transitive
+dependencies. In the second group are dependencies that are only present as transitive dependencies.
+There, we try to follow the convention to provide additional information with "which dependencies use what version",
+so that the next person reviewing the constraint does not have to look it up. However, this is quite time-consuming
+to analyze the dependencies and therefore subject to change.
+
+In order to review a constraint, you have to check if the updated dependency is mentioned in any of the constraints,
+either as a reason for another dependency constraint or as the constraint's dependency. Removing temporarily
+a constraint, the task writeLocks will fail if the constraint is still required.
+
+This process and the constraints of dependencies.gradle are not optimal, as it is quite time-consuming and not obvious
+by just looking at it. We just haven't found yet a more efficient way to maintain these constraints.
+
 == Renovate bot Pull Requests
+
+The renovate bot may be replaced in the future with dependabot and this section may only be relevant for older
+versions (<10.0). See https://lists.apache.org/thread/1sb9ttv3lp57z2yod1htx1fykp5sj73z for updates.
+
 A member of the Solr community operates a Github bot running https://github.com/renovatebot/renovate[Renovate], which
 files Pull Requests to Solr with dependency upgrade proposals. The PRs are labeled `dependencies` and do include
-changes resulting from `gradle --write-locks` and `updateLicenses`.
+changes resulting from `./gradlew writeLocks` and `updateLicenses`.
 
 Community members and committers can then review, and if manual changes are needed, help bring the PR to completion.
 For many dependencies, a changelog is included in the PR text, which may help guide the upgrade decision.

dev-docs/dependency-upgrades.adoc

@@ -33,10 +33,10 @@ buildscript {
   }
 
   dependencies {
-    classpath "com.vladsch.flexmark:flexmark:${scriptDepVersions['flexmark']}"
-    classpath "com.vladsch.flexmark:flexmark-ext-abbreviation:${scriptDepVersions['flexmark']}"
-    classpath "com.vladsch.flexmark:flexmark-ext-attributes:${scriptDepVersions['flexmark']}"
-    classpath "com.vladsch.flexmark:flexmark-ext-autolink:${scriptDepVersions['flexmark']}"
+    classpath libs.flexmark.flexmark
+    classpath libs.flexmark.extensions.abbreviation
+    classpath libs.flexmark.extensions.attributes
+    classpath libs.flexmark.extensions.autolink
   }
 }
 

gradle/documentation/markdown.gradle

@@ -45,11 +45,11 @@ configure(project(":solr:documentation")) {
     //   from all Solr javadocs?) then perhaps we can find a way to build this list programatically?
     // - If these javadocs are (only every) consumed by the ref guide only, then these deps & associated tasks
     //   should just be moved to the ref-guide build.gradle
-    javadocs group: 'org.apache.lucene', name: 'lucene-core', classifier: 'javadoc'
-    javadocs group: 'org.apache.lucene', name: 'lucene-analysis-common', classifier: 'javadoc'
-    javadocs group: 'org.apache.lucene', name: 'lucene-analysis-stempel', classifier: 'javadoc'
-    javadocs group: 'org.apache.lucene', name: 'lucene-queryparser', classifier: 'javadoc'
-    javadocs group: 'org.apache.lucene', name: 'lucene-spatial-extras', classifier: 'javadoc'
+    javadocs variantOf(libs.apache.lucene.core) { classifier 'javadoc' }
+    javadocs variantOf(libs.apache.lucene.analysis.common) { classifier 'javadoc' }
+    javadocs variantOf(libs.apache.lucene.analysis.stempel) { classifier 'javadoc' }
+    javadocs variantOf(libs.apache.lucene.queryparser) { classifier 'javadoc' }
+    javadocs variantOf(libs.apache.lucene.spatialextras) { classifier 'javadoc' }
   }
 
 

gradle/documentation/pull-lucene-javadocs.gradle

@@ -26,7 +26,7 @@ configure(rootProject) {
   }
 
   dependencies {
-    javacc "net.java.dev.javacc:javacc:${scriptDepVersions['javacc']}"
+    javacc libs.javacc.javacc
   }
 
   task javacc() {