Without yaml

.gitignore

@@ -80,6 +80,7 @@ protogen/
 
 # Local FDB settings
 fdb-environment.properties
+fdb-environment.yaml
 
 # Docker local support
 run/

build.gradle

@@ -18,6 +18,10 @@
  * limitations under the License.
  */
 
+import org.yaml.snakeyaml.Yaml
+
+import org.apache.tools.ant.taskdefs.condition.Os
+
 buildscript {
     repositories {
         if (Boolean.parseBoolean(mavenLocalEnabled)) {
@@ -29,6 +33,7 @@ buildscript {
 
     dependencies {
         classpath 'org.jboss.tattletale:tattletale:1.2.0.Beta2'
+        classpath libs.snakeyaml
     }
 }
 
@@ -341,6 +346,8 @@ if (!JavaVersion.current().isJava8Compatible()) {
     throw new Exception("Java 8 is required to build fdb-record-layer")
 }
 
+// fdb-environment.properties is the old way we configured the library path and cluster file, it does not scale well
+// to multiple cluster files, so is being replaced with a yaml file.
 def fdbEnvironmentFile = new File("${rootProject.projectDir}/fdb-environment.properties")
 if (fdbEnvironmentFile.exists()) {
     fdbEnvironmentFile.eachLine { line ->
@@ -356,3 +363,15 @@ if (!ext.fdbEnvironment.isEmpty()) {
         }
     }
 }
+
+def fdbEnvironmentYamlFile = new File("${rootProject.projectDir}/fdb-environment.yaml")
+if (fdbEnvironmentYamlFile.exists()) {
+    def libraryPath = new Yaml().loadAll(fdbEnvironmentYamlFile.newInputStream()).first().libraryPath
+    def libraryPathVariable = Os.isFamily(Os.FAMILY_MAC) ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH"
+    allprojects {
+        tasks.withType(Test) { task ->
+            task.environment([(libraryPathVariable): libraryPath])
+            task.environment(FDB_ENVIRONMENT_YAML: fdbEnvironmentYamlFile.absolutePath)
+        }
+    }
+}

docker-local/start.sh

@@ -95,6 +95,13 @@ FDB_CLUSTER_FILE=${FDB_CLUSTER_FILE}
 ${LIBRARY_PATH}=${RUNDIR}
 EOF
 
+cat >${ROOTDIR}/fdb-environment.yaml <<EOF
+# docker-local
+clusterFiles:
+  - ${FDB_CLUSTER_FILE}
+libraryPath: ${RUNDIR}
+EOF
+
 cat ${ROOTDIR}/fdb-environment.properties
 
 echo "Docker-based FDB cluster is now up."

docs/sphinx/source/Building.md

@@ -63,7 +63,24 @@ If you enable the local repo in whatever uses the Record Layer, the following wi
 ./gradlew publishToMavenLocal -PpublishBuild=true
 ```
 
-## Configuring tests for a non-standard FDB cluster file location
+## Configuring tests for a non-standard FDB cluster file location, or to run against multiple clusters
+
+If a file `fdb-environment.yaml` exists in the root of the working directory, it contains the configuration for the C API library
+and a list of cluster files. Most tests will chose randomly from the provided cluster files when testing.
+
+Here is an example file:
+``` yaml
+libraryPath: /Users/scott/fdb/bin/fdb-server-7.3.42-macos_arm64/lib
+clusterFiles:
+  - /Users/scott/fdb/data/fdb-one.cluster
+  - /Users/scott/fdb/data/fdb-two.cluster
+```
+
+
+
+### deprecated properties file
+
+[ this is being replaced by the yaml file described above, to better support multiple cluster files ]
 
 If a file `fdb-environment.properties` exists in the root of the working directory, it contains environment variables that specify where to find
 the local FDB. These settings will apply when running inside IntelliJ as well as from a command line and so are easier to manage than a shell script.

fdb-extensions/fdb-extensions.gradle

@@ -30,6 +30,7 @@ dependencies {
     compileOnly(libs.jsr305)
 
     testImplementation(libs.bundles.test.impl)
+    testImplementation(libs.snakeyaml)
     testRuntimeOnly(libs.bundles.test.runtime)
     testCompileOnly(libs.bundles.test.compileOnly)
     testAnnotationProcessor(libs.autoService)

fdb-extensions/src/test/java/com/apple/foundationdb/test/FDBTestEnvironment.java

@@ -0,0 +1,86 @@
+/*
+ * FDBTestEnvironment.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb.test;
+
+import org.assertj.core.api.Assumptions;
+import org.yaml.snakeyaml.Yaml;
+
+import javax.annotation.Nullable;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Utility class for accessing {@code fdb-environment.yaml}, so that tests can know about fdb clusters
+ * that are available.
+ */
+public final class FDBTestEnvironment {
+    private static final List<String> clusterFiles;
+
+    static {
+        final String fdbEnvironment = System.getenv("FDB_ENVIRONMENT_YAML");
+
+        if (fdbEnvironment != null && !fdbEnvironment.isEmpty()) {
+            clusterFiles = List.copyOf(parseFDBEnvironmentYaml(fdbEnvironment));
+        } else {
+            clusterFiles = Collections.singletonList(null); // List.of does not allow null
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static List<String> parseFDBEnvironmentYaml(final String fdbEnvironment) {
+        Yaml yaml = new Yaml();
+        try (FileInputStream yamlInput = new FileInputStream(fdbEnvironment)) {
+            Object fdbConfig = yaml.load(yamlInput);
+            return (List<String>)((Map<?, ?>)fdbConfig).get("clusterFiles");
+        } catch (IOException e) {
+            throw new IllegalStateException("Could not read fdb-environment.yaml", e);
+        } catch (ClassCastException e) {
+            throw new IllegalStateException("Could not parse fdb environment file " + fdbEnvironment, e);
+        }
+    }
+
+    @Nullable
+    public static String getClusterFile(int i) {
+        return clusterFiles.get(i);
+    }
+
+    public static List<String> allClusterFiles() {
+        return clusterFiles;
+    }
+
+    public static String randomClusterFile() {
+        return clusterFiles.get(ThreadLocalRandom.current().nextInt(clusterFiles.size()));
+    }
+
+    /**
+     * Marks the current test as skipped if there are not the desired number of clusters available to test against.
+     * @param desiredCount the number of clusters to test against
+     */
+    public static void assumeClusterCount(final int desiredCount) {
+        if (desiredCount > 1) {
+            Assumptions.assumeThat(clusterFiles.size()).as("Cluster file count").isGreaterThanOrEqualTo(desiredCount);
+        }
+    }
+}

fdb-extensions/src/test/java/com/apple/foundationdb/test/TestDatabaseExtension.java

@@ -95,7 +95,7 @@ public void beforeAll(final ExtensionContext extensionContext) {
     @Nonnull
     public Database getDatabase() {
         if (db == null) {
-            db = FDB.instance().open();
+            db = FDB.instance().open(FDBTestEnvironment.randomClusterFile());
         }
         return db;
     }

fdb-record-layer-core/fdb-record-layer-core.gradle

@@ -44,6 +44,7 @@ dependencies {
     testImplementation(libs.junit.platform)
     testImplementation(libs.protobuf.util)
     testCompileOnly(libs.spotbugs.annotations)
+    testImplementation(libs.snakeyaml)
 
     testAnnotationProcessor(libs.autoService)
 }

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/BlockingInAsyncDetectionTest.java

@@ -24,6 +24,7 @@
 import com.apple.foundationdb.record.RecordCoreException;
 import com.apple.foundationdb.record.TestHelpers;
 import com.apple.foundationdb.record.test.FDBDatabaseExtension;
+import com.apple.foundationdb.test.FDBTestEnvironment;
 import com.apple.test.Tags;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
@@ -57,7 +58,7 @@ void testBlockingInAsyncException() {
         // Make sure that we aren't holding on to previously created databases
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         assertEquals(BlockingInAsyncDetection.IGNORE_COMPLETE_EXCEPTION_BLOCKING, database.getBlockingInAsyncDetection());
         assertThrows(BlockingInAsyncException.class, () -> callAsyncBlocking(database));
     }
@@ -68,7 +69,7 @@ void testBlockingInAsyncWarning() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.IGNORE_COMPLETE_WARN_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_IN_ASYNC_CONTEXT_MESSAGE,
                 () -> {
                     callAsyncBlocking(database, true);
@@ -82,7 +83,7 @@ void testCompletedBlockingInAsyncWarning() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.WARN_COMPLETE_EXCEPTION_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_IN_ASYNC_CONTEXT_MESSAGE,
                 () -> database.asyncToSync(new FDBStoreTimer(), FDBStoreTimer.Waits.WAIT_ERROR_CHECK,
                         CompletableFuture.supplyAsync(() ->
@@ -95,7 +96,7 @@ void testBlockingCreatingAsyncDetection() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.WARN_COMPLETE_EXCEPTION_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_RETURNING_ASYNC_MESSAGE,
                 () -> returnAnAsync(database, MoreAsyncUtil.delayedFuture(200L, TimeUnit.MILLISECONDS, database.getScheduledExecutor())));
     }
@@ -106,7 +107,7 @@ void testCompletedBlockingCreatingAsyncDetection() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.WARN_COMPLETE_EXCEPTION_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertDidNotLog(FDBDatabase.class, FDBDatabase.BLOCKING_RETURNING_ASYNC_MESSAGE,
                 () -> returnAnAsync(database, CompletableFuture.completedFuture(10L)));
     }

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBDatabaseTest.java

@@ -36,6 +36,7 @@
 import com.apple.foundationdb.record.test.TestKeySpace;
 import com.apple.foundationdb.record.test.TestKeySpacePathManagerExtension;
 import com.apple.foundationdb.subspace.Subspace;
+import com.apple.foundationdb.test.FDBTestEnvironment;
 import com.apple.foundationdb.tuple.Tuple;
 import com.apple.test.BooleanSource;
 import com.apple.test.Tags;
@@ -54,6 +55,7 @@
 
 import javax.annotation.Nonnull;
 import java.io.IOException;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -66,6 +68,7 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -240,7 +243,7 @@ void testJoinNowOnNonCompletedFuture(BlockingInAsyncDetection behavior) {
         if (behavior.throwExceptionOnBlocking()) {
             assertThrows(BlockingInAsyncException.class, () -> database.joinNow(new CompletableFuture<>()));
         } else {
-            FDBDatabase database2 = factory.getDatabase();
+            FDBDatabase database2 = factory.getDatabase(database.getClusterFile());
             TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_FOR_FUTURE_MESSAGE, () -> {
                 long val = database2.joinNow(MoreAsyncUtil.delayedFuture(100, TimeUnit.MILLISECONDS, database2.getScheduledExecutor())
                         .thenApply(vignore -> 1066L));
@@ -494,4 +497,31 @@ void cannotChangeAPIVersionAfterInit() {
             assertEquals(initApiVersion, database.getAPIVersion());
         }
     }
+
+    @Test
+    void canAccessMultipleClusters() {
+        FDBTestEnvironment.assumeClusterCount(2);
+        final FDBDatabase database0 = dbExtension.getDatabase(0);
+        final FDBDatabase database1 = dbExtension.getDatabase(1);
+        final byte[] key = Tuple.from(UUID.randomUUID()).pack();
+        final byte[] value0 = Tuple.from("cluster0").pack();
+        final byte[] value1 = Tuple.from("cluster1").pack();
+        try (FDBRecordContext context0 = database0.openContext()) {
+            context0.ensureActive().set(key, value0);
+            context0.commit();
+        }
+        try (FDBRecordContext context1 = database1.openContext()) {
+            assertNull(context1.ensureActive().get(key).join());
+            context1.ensureActive().set(key, value1);
+            context1.commit();
+        }
+        try (FDBRecordContext context0 = database0.openContext()) {
+            assertArrayEquals(value0, context0.ensureActive().get(key).join());
+            context0.commit();
+        }
+        try (FDBRecordContext context1 = database1.openContext()) {
+            assertArrayEquals(value1, context1.ensureActive().get(key).join());
+            context1.commit();
+        }
+    }
 }

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordContextTest.java

@@ -34,6 +34,7 @@
 import com.apple.foundationdb.record.test.TestKeySpacePathManagerExtension;
 import com.apple.foundationdb.record.util.pair.Pair;
 import com.apple.foundationdb.subspace.Subspace;
+import com.apple.foundationdb.test.FDBTestEnvironment;
 import com.apple.foundationdb.tuple.ByteArrayUtil;
 import com.apple.foundationdb.tuple.ByteArrayUtil2;
 import com.apple.foundationdb.tuple.Tuple;
@@ -433,7 +434,7 @@ public void logWithoutSettingId() {
             FDBDatabaseFactory factory = fdb.getFactory();
             factory.setTransactionIsTracedSupplier(() -> false);
             factory.clear();
-            fdb = factory.getDatabase();
+            fdb = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         }
         try (FDBRecordContext context = fdb.openContext()) {
             RecordCoreException err = assertThrows(RecordCoreException.class, context::logTransaction);
@@ -665,7 +666,8 @@ public void contextExecutor() {
             factory.setContextExecutor(exec -> new ThreadIdRestoringExecutor(exec, myThreadId));
             factory.clear();
 
-            FDBDatabase database = factory.getDatabase();
+
+            FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
             try (FDBRecordContext context = database.openContext()) {
                 context.ensureActive().get(new byte[] { 0 }).thenAccept( value -> {
                     assertEquals(myThreadId, ThreadId.get());

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStorePerformanceTest.java

@@ -38,6 +38,7 @@
 import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath;
 import com.apple.foundationdb.record.test.FDBDatabaseExtension;
 import com.apple.foundationdb.record.test.TestKeySpace;
+import com.apple.foundationdb.test.FDBTestEnvironment;
 import com.apple.foundationdb.tuple.Tuple;
 import com.apple.foundationdb.util.StringUtils;
 import com.apple.test.Tags;
@@ -164,7 +165,7 @@ public void setup() {
     public void createMetaData() {
         FDBDatabaseFactory factory = dbExtension.getDatabaseFactory();
         factory.setDirectoryCacheSize(databaseParameters.pathCache);
-        fdb = factory.getDatabase();
+        fdb = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
 
         RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
         metaDataBuilder.setSplitLongRecords(databaseParameters.splitRecords);