diff --git a/.github/workflows/actions_build.yml b/.github/workflows/actions_build.yml index 26f77ec737..86de3a4aec 100644 --- a/.github/workflows/actions_build.yml +++ b/.github/workflows/actions_build.yml @@ -28,7 +28,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-12, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest] java: [21] include: - java: 8 @@ -75,11 +75,34 @@ jobs: -Porg.gradle.java.installations.paths=${{ steps.setup-build-jdk.outputs.path }},${{ steps.setup-test-jdk.outputs.path }} shell: bash + - name: Summarize the failed tests + if: failure() + run: | + ./gradlew --no-daemon --stacktrace --max-workers=1 reportFailedTests \ + -PnoLint \ + -PflakyTests=false \ + -PbuildJdkVersion=${{ env.BUILD_JDK_VERSION }} \ + -PtestJavaVersion=${{ matrix.java }} \ + ${{ matrix.min-java && format('-PminimumJavaVersion={0}', matrix.min-java) || '' }} \ + -Porg.gradle.java.installations.paths=${{ steps.setup-build-jdk.outputs.path }},${{ steps.setup-jdk.outputs.path }} + + SUMMARY_FILE="build/failed-tests-result.txt" + if test -f "$SUMMARY_FILE"; then + echo '### 🔴 Failed tests' >> $GITHUB_STEP_SUMMARY + cat $SUMMARY_FILE >> $GITHUB_STEP_SUMMARY + fi + shell: bash + + - name: Dump stuck threads + if: always() + run: jps | grep -vi "jps" | awk '{ print $1 }' | xargs -I'{}' jstack -l {} || true + shell: bash + - name: Upload coverage to Codecov if: ${{ matrix.coverage }} uses: codecov/codecov-action@v3 - - name: Collecting the test reports .. + - name: Collect the test reports .. if: failure() run: | find . '(' \ @@ -96,11 +119,6 @@ jobs: path: reports-JVM-${{ matrix.java }}.tar retention-days: 3 - - name: Dump stuck threads - if: always() - run: jps | grep -vi "jps" | awk '{ print $1 }' | xargs -I'{}' jstack -l {} || true - shell: bash - lint: if: github.repository == 'line/centraldogma' runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 06c3b008b0..e8e111470a 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,15 @@ +import org.jsoup.Jsoup +import org.jsoup.nodes.Element +import org.jsoup.select.Elements + +import static java.lang.Math.min + +buildscript { + dependencies { + classpath libs.jsoup + } +} + plugins { alias libs.plugins.nexus.publish alias libs.plugins.osdetector apply false @@ -127,6 +139,85 @@ configure(projectsWithFlags('java')) { tasks.processResources.duplicatesStrategy = DuplicatesStrategy.INCLUDE } +tasks.register("reportFailedTests", TestsReportTask) + +/** + * Summarizes the failed tests and reports as a file with the Markdown syntax. + */ +class TestsReportTask extends DefaultTask { + @OutputFile + final def reportFile = project.file("${project.buildDir}/failed-tests-result.txt") + + @TaskAction + def run() { + // Collect up to 20 error results + int maxErrorSize = 20 + List failedTests = [] + Set handledFiles = [] + + project.allprojects { + tasks.withType(Test) { testTask -> + + def xmlFiles = testTask.reports.junitXml.outputLocation.asFileTree.files + if (xmlFiles.isEmpty()) { + return + } + xmlFiles.each { file -> + if (!handledFiles.add(file.name)) { + return + } + + Elements failures = Jsoup.parse(file, 'UTF-8').select("testsuite failure") + if (failures.isEmpty() || failedTests.size() > maxErrorSize) { + return + } + failures.each { failure -> + Element parent = failure.parent() + String fullMethodName = "${parent.attr("classname")}.${parent.attr("name")}" + String detail = failure.wholeText() + failedTests += [method: fullMethodName, detail: detail] + } + } + } + } + + if (failedTests.isEmpty()) { + return + } + + reportFile.withPrintWriter('UTF-8') { writer -> + failedTests.each { it -> + String method = it.method + String detail = it.detail + + // Create an link to directly create an issue from the error message + String ghIssueTitle = URLEncoder.encode("Test failure: `$method`", "UTF-8") + // 8k is the maximum allowed URL length for GitHub + String ghIssueBody = URLEncoder.encode( + "```\n${detail.substring(0, min(6000, detail.length()))}\n```\n", "UTF-8") + String ghIssueLink = + "https://github.com/line/centraldogma/issues/new?title=$ghIssueTitle&body=$ghIssueBody" + String ghSearchQuery = URLEncoder.encode("is:issue $method", "UTF-8") + String ghSearchLink = "https://github.com/line/centraldogma/issues?q=$ghSearchQuery" + writer.print("- $it.method - [Search similar issues]($ghSearchLink) | ") + writer.println("[Create an issue?]($ghIssueLink)") + + writer.println(" ```") + List lines = detail.split("\n") as List + def summary = lines.take(8) + summary.each { line -> writer.println(" $line") } + writer.println(" ```") + if (lines.size() > 8) { + writer.println("
Full error messages") + writer.println(" 
")
+                    lines.each { line -> writer.println("  $line") }
+                    writer.println("
\n") + } + } + } + } +} + // Configure the Javadoc tasks of all projects. allprojects { tasks.withType(Javadoc) { diff --git a/client/java-armeria-xds/src/test/java/com/linecorp/centraldogma/client/armeria/xds/AuthUpstreamTest.java b/client/java-armeria-xds/src/test/java/com/linecorp/centraldogma/client/armeria/xds/AuthUpstreamTest.java index 3acdfb7e78..de50be2d62 100644 --- a/client/java-armeria-xds/src/test/java/com/linecorp/centraldogma/client/armeria/xds/AuthUpstreamTest.java +++ b/client/java-armeria-xds/src/test/java/com/linecorp/centraldogma/client/armeria/xds/AuthUpstreamTest.java @@ -65,7 +65,7 @@ class AuthUpstreamTest { @Override protected void configure(CentralDogmaBuilder builder) { - builder.administrators(TestAuthMessageUtil.USERNAME); + builder.systemAdministrators(TestAuthMessageUtil.USERNAME); builder.authProviderFactory(new TestAuthProviderFactory()); } diff --git a/common/src/main/java/com/linecorp/centraldogma/common/RedundantChangeException.java b/common/src/main/java/com/linecorp/centraldogma/common/RedundantChangeException.java index 0c8d4ed0cf..a71f91a17a 100644 --- a/common/src/main/java/com/linecorp/centraldogma/common/RedundantChangeException.java +++ b/common/src/main/java/com/linecorp/centraldogma/common/RedundantChangeException.java @@ -16,37 +16,33 @@ package com.linecorp.centraldogma.common; +import static java.util.Objects.requireNonNull; + +import javax.annotation.Nullable; + /** * A {@link CentralDogmaException} that is raised when attempted to push a commit without effective changes. */ public class RedundantChangeException extends CentralDogmaException { private static final long serialVersionUID = 8739464985038079688L; - - /** - * Creates a new instance. - */ - public RedundantChangeException() {} + @Nullable + private final Revision headRevision; /** * Creates a new instance. */ public RedundantChangeException(String message) { super(message); + headRevision = null; } /** * Creates a new instance. */ - public RedundantChangeException(Throwable cause) { - super(cause); - } - - /** - * Creates a new instance. - */ - public RedundantChangeException(String message, Throwable cause) { - super(message, cause); + public RedundantChangeException(Revision headRevision, String message) { + super(message); + this.headRevision = requireNonNull(headRevision, "headRevision"); } /** @@ -57,13 +53,14 @@ public RedundantChangeException(String message, Throwable cause) { */ public RedundantChangeException(String message, boolean writableStackTrace) { super(message, writableStackTrace); + headRevision = null; } /** - * Creates a new instance. + * Returns the head revision of the repository when this exception was raised. */ - protected RedundantChangeException(String message, Throwable cause, boolean enableSuppression, - boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); + @Nullable + public Revision headRevision() { + return headRevision; } } diff --git a/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/MirrorDto.java b/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/MirrorDto.java index 098a990874..11d22d7423 100644 --- a/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/MirrorDto.java +++ b/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/MirrorDto.java @@ -48,6 +48,8 @@ public final class MirrorDto { @Nullable private final String gitignore; private final String credentialId; + @Nullable + private final String zone; @JsonCreator public MirrorDto(@JsonProperty("id") String id, @@ -62,7 +64,8 @@ public MirrorDto(@JsonProperty("id") String id, @JsonProperty("remotePath") String remotePath, @JsonProperty("remoteBranch") String remoteBranch, @JsonProperty("gitignore") @Nullable String gitignore, - @JsonProperty("credentialId") String credentialId) { + @JsonProperty("credentialId") String credentialId, + @JsonProperty("zone") @Nullable String zone) { this.id = requireNonNull(id, "id"); this.enabled = firstNonNull(enabled, true); this.projectName = requireNonNull(projectName, "projectName"); @@ -76,6 +79,7 @@ public MirrorDto(@JsonProperty("id") String id, this.remoteBranch = requireNonNull(remoteBranch, "remoteBranch"); this.gitignore = gitignore; this.credentialId = requireNonNull(credentialId, "credentialId"); + this.zone = zone; } @JsonProperty("id") @@ -145,6 +149,12 @@ public String credentialId() { return credentialId; } + @Nullable + @JsonProperty("zone") + public String zone() { + return zone; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -166,13 +176,14 @@ public boolean equals(Object o) { remotePath.equals(mirrorDto.remotePath) && remoteBranch.equals(mirrorDto.remoteBranch) && Objects.equals(gitignore, mirrorDto.gitignore) && - credentialId.equals(mirrorDto.credentialId); + credentialId.equals(mirrorDto.credentialId) && + Objects.equals(zone, mirrorDto.zone); } @Override public int hashCode() { return Objects.hash(id, projectName, schedule, direction, localRepo, localPath, remoteScheme, - remoteUrl, remotePath, remoteBranch, gitignore, credentialId, enabled); + remoteUrl, remotePath, remoteBranch, gitignore, credentialId, enabled, zone); } @Override @@ -191,6 +202,7 @@ public String toString() { .add("remotePath", remotePath) .add("gitignore", gitignore) .add("credentialId", credentialId) + .add("zone", zone) .toString(); } } diff --git a/common/src/main/java/com/linecorp/centraldogma/internal/jsonpatch/RemoveIfExistsOperation.java b/common/src/main/java/com/linecorp/centraldogma/internal/jsonpatch/RemoveIfExistsOperation.java index 8efe8c0b48..7ef9189c70 100644 --- a/common/src/main/java/com/linecorp/centraldogma/internal/jsonpatch/RemoveIfExistsOperation.java +++ b/common/src/main/java/com/linecorp/centraldogma/internal/jsonpatch/RemoveIfExistsOperation.java @@ -30,10 +30,10 @@ import com.fasterxml.jackson.databind.node.ObjectNode; /** - * JSON Path {@code remove} operation. + * JSON Path {@code removeIfExists} operation. * - *

This operation only takes one pointer ({@code path}) as an argument. It - * is an error condition if no JSON value exists at that pointer.

+ *

This operation only takes one pointer ({@code path}) as an argument. Unlike, {@link RemoveOperation}, it + * does not throw an error if no JSON value exists at that pointer.

*/ public final class RemoveIfExistsOperation extends JsonPatchOperation { diff --git a/dependencies.toml b/dependencies.toml index ed70c06ef2..14e42f6966 100644 --- a/dependencies.toml +++ b/dependencies.toml @@ -3,14 +3,14 @@ # If its classes are exposed in Javadoc, update offline links as well. # [versions] -armeria = "1.30.1" +armeria = "1.31.3" assertj = "3.26.3" awaitility = "4.2.2" -bouncycastle = "1.78.1" +bouncycastle = "1.79" # Don"t upgrade Caffeine to 3.x that requires Java 11. caffeine = "2.9.3" checkstyle = "10.3.3" -controlplane = "1.0.45" +controlplane = "1.0.46" # Ensure that we use the same ZooKeeper version as what Curator depends on. # See: https://github.com/apache/curator/blob/master/pom.xml # (Switch to the right tag to find out the right version.) @@ -20,17 +20,17 @@ cron-utils = "9.2.0" diffutils = "1.3.0" docker = "9.4.0" download = "5.6.0" -dropwizard-metrics = "4.2.26" +dropwizard-metrics = "4.2.28" eddsa = "0.3.0" findbugs = "3.0.2" futures-completable = "0.3.6" -grpc-java = "1.66.0" -guava = "33.2.1-jre" +grpc-java = "1.68.1" +guava = "33.3.1-jre" guava-failureaccess = "1.0.1" hamcrest-library = "2.2" hibernate-validator6 = "6.2.5.Final" hibernate-validator8 = "8.0.1.Final" -jackson = "2.17.2" +jackson = "2.18.1" javassist = "3.30.2-GA" javax-annotation = "1.3.2" javax-inject = "1" @@ -41,7 +41,7 @@ jetty-alpn-agent = "2.0.10" jgit = "5.13.3.202401111512-r" jgit6 = "6.10.0.202406032230-r" junit4 = "4.13.2" -junit5 = "5.11.0" +junit5 = "5.11.3" # Don't upgrade junit-pioneer to 2.x.x that requires Java 11 junit-pioneer = "1.9.1" jsch = "0.1.55" @@ -49,22 +49,23 @@ jsch = "0.1.55" json-path = "2.2.0" # 3.0.0 requires java 17 json-unit = "2.38.0" +jsoup = "1.18.1" # JSoup is only used for Gradle script. jmh-core = "1.37" jmh-gradle-plugin = "0.7.2" jxr = "0.2.1" -kubernetes-client = "6.12.1" +kubernetes-client = "6.13.4" logback12 = { strictly = "1.2.13" } logback15 = { strictly = "1.5.7" } logback = "1.2.13" -micrometer = "1.13.3" -mina-sshd = "2.13.2" +micrometer = "1.13.6" +mina-sshd = "2.14.0" # Don't uprade mockito to 5.x.x that requires Java 11 mockito = "4.11.0" nexus-publish-plugin = "2.0.0" -node-gradle-plugin = "7.0.2" +node-gradle-plugin = "7.1.0" osdetector = "1.7.3" proguard = "7.4.2" -protobuf = "3.25.1" +protobuf = "3.25.5" protobuf-gradle-plugin = "0.8.19" quartz = "2.3.2" reflections = "0.9.11" @@ -78,9 +79,9 @@ slf4j2 = { strictly = "2.0.16" } snappy = "1.1.10.5" sphinx = "2.10.1" spring-boot2 = "2.7.18" -spring-boot3 = "3.3.2" +spring-boot3 = "3.3.5" spring-test-junit5 = "1.5.0" -testcontainers = "1.20.1" +testcontainers = "1.20.3" thrift09 = { strictly = "0.9.3-1" } # Ensure that we use the same ZooKeeper version as what Curator depends on. # See: https://github.com/apache/curator/blob/master/pom.xml @@ -281,6 +282,10 @@ version.ref = "json-unit" module = "net.javacrumbs.json-unit:json-unit-fluent" version.ref = "json-unit" +[libraries.jsoup] +module = "org.jsoup:jsoup" +version.ref = "jsoup" + [libraries.junit4] module = "junit:junit" version.ref = "junit4" diff --git a/gradle.properties b/gradle.properties index e4545c497b..b00cc4bd8e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ group=com.linecorp.centraldogma -version=0.70.1-SNAPSHOT +version=0.72.1-SNAPSHOT projectName=Central Dogma projectUrl=https://line.github.io/centraldogma/ projectDescription=Highly-available version-controlled service configuration repository based on Git, ZooKeeper and HTTP/2 diff --git a/it/mirror-listener/src/test/java/com/linecorp/centraldogma/it/mirror/listener/CustomMirrorListenerTest.java b/it/mirror-listener/src/test/java/com/linecorp/centraldogma/it/mirror/listener/CustomMirrorListenerTest.java index c896f1371d..bf992db847 100644 --- a/it/mirror-listener/src/test/java/com/linecorp/centraldogma/it/mirror/listener/CustomMirrorListenerTest.java +++ b/it/mirror-listener/src/test/java/com/linecorp/centraldogma/it/mirror/listener/CustomMirrorListenerTest.java @@ -89,7 +89,7 @@ void shouldNotifyMirrorEvents() { final Mirror mirror = new AbstractMirror("my-mirror-1", true, EVERY_SECOND, MirrorDirection.REMOTE_TO_LOCAL, Credential.FALLBACK, r, "/", - URI.create("unused://uri"), "/", "", null) { + URI.create("unused://uri"), "/", "", null, null) { @Override protected MirrorResult mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes, Instant triggeredTime) { @@ -114,7 +114,7 @@ protected MirrorResult mirrorRemoteToLocal(File workDir, CommandExecutor executo when(mr.mirrors()).thenReturn(CompletableFuture.completedFuture(ImmutableList.of(mirror))); final MirrorSchedulingService service = new MirrorSchedulingService( - temporaryFolder, pm, new SimpleMeterRegistry(), 1, 1, 1); + temporaryFolder, pm, new SimpleMeterRegistry(), 1, 1, 1, null); final CommandExecutor executor = mock(CommandExecutor.class); service.start(executor); diff --git a/it/mirror/build.gradle b/it/mirror/build.gradle index f333c214a4..2926988efd 100644 --- a/it/mirror/build.gradle +++ b/it/mirror/build.gradle @@ -7,4 +7,6 @@ dependencies { testImplementation libs.jsch testImplementation libs.mina.sshd.core testImplementation libs.mina.sshd.git + testImplementation libs.zookeeper + testImplementation libs.dropwizard.metrics.core } diff --git a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorIntegrationTest.java b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorIntegrationTest.java index 5131b16020..6db28ccdf4 100644 --- a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorIntegrationTest.java +++ b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorIntegrationTest.java @@ -88,7 +88,8 @@ class GitMirrorIntegrationTest { static final CentralDogmaExtension dogma = new CentralDogmaExtension() { @Override protected void configure(CentralDogmaBuilder builder) { - builder.pluginConfigs(new MirroringServicePluginConfig(true, 1, MAX_NUM_FILES, MAX_NUM_BYTES)); + builder.pluginConfigs(new MirroringServicePluginConfig(true, 1, MAX_NUM_FILES, MAX_NUM_BYTES, + false)); } }; diff --git a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LegacyGitMirrorSettingsTest.java b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LegacyGitMirrorSettingsTest.java index 963ec8b992..d099316396 100644 --- a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LegacyGitMirrorSettingsTest.java +++ b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LegacyGitMirrorSettingsTest.java @@ -41,7 +41,7 @@ class LegacyGitMirrorSettingsTest { @Override protected void configure(CentralDogmaBuilder builder) { builder.authProviderFactory(new TestAuthProviderFactory()); - builder.administrators(TestAuthMessageUtil.USERNAME); + builder.systemAdministrators(TestAuthMessageUtil.USERNAME); } }; diff --git a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java index aaff0392f1..f5c48fa557 100644 --- a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java +++ b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java @@ -81,7 +81,8 @@ class LocalToRemoteGitMirrorTest { static final CentralDogmaExtension dogma = new CentralDogmaExtension() { @Override protected void configure(CentralDogmaBuilder builder) { - builder.pluginConfigs(new MirroringServicePluginConfig(true, 1, MAX_NUM_FILES, MAX_NUM_BYTES)); + builder.pluginConfigs( + new MirroringServicePluginConfig(true, 1, MAX_NUM_FILES, MAX_NUM_BYTES, false)); } }; @@ -371,7 +372,7 @@ private void pushMirrorSettings(String localRepo, @Nullable String localPath, @N .push().join(); } catch (CompletionException e) { if (e.getCause() instanceof RedundantChangeException) { - // The same content can be pushed several times. + // The same content can be pushed several times. } else { throw e; } diff --git a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/MirrorRunnerTest.java b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/MirrorRunnerTest.java index 0bb74fe02b..95e7bfe783 100644 --- a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/MirrorRunnerTest.java +++ b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/MirrorRunnerTest.java @@ -49,10 +49,10 @@ class MirrorRunnerTest { - private static final String FOO_PROJ = "foo"; - private static final String BAR_REPO = "bar"; - private static final String PRIVATE_KEY_FILE = "ecdsa_256.openssh"; - private static final String TEST_MIRROR_ID = "test-mirror"; + static final String FOO_PROJ = "foo"; + static final String BAR_REPO = "bar"; + static final String PRIVATE_KEY_FILE = "ecdsa_256.openssh"; + static final String TEST_MIRROR_ID = "test-mirror"; @RegisterExtension static final CentralDogmaExtension dogma = new CentralDogmaExtension() { @@ -60,7 +60,7 @@ class MirrorRunnerTest { @Override protected void configure(CentralDogmaBuilder builder) { builder.authProviderFactory(new TestAuthProviderFactory()); - builder.administrators(USERNAME); + builder.systemAdministrators(USERNAME); } @Override @@ -82,15 +82,15 @@ protected void scaffold(CentralDogma client) { } }; - private BlockingWebClient adminClient; + private BlockingWebClient systemAdminClient; @BeforeEach void setUp() throws Exception { final String adminToken = getAccessToken(dogma.httpClient(), USERNAME, PASSWORD); - adminClient = WebClient.builder(dogma.httpClient().uri()) - .auth(AuthToken.ofOAuth2(adminToken)) - .build() - .blocking(); + systemAdminClient = WebClient.builder(dogma.httpClient().uri()) + .auth(AuthToken.ofOAuth2(adminToken)) + .build() + .blocking(); TestMirrorRunnerListener.reset(); } @@ -98,31 +98,31 @@ void setUp() throws Exception { void triggerMirroring() throws Exception { final PublicKeyCredential credential = getCredential(); ResponseEntity response = - adminClient.prepare() - .post("/api/v1/projects/{proj}/credentials") - .pathParam("proj", FOO_PROJ) - .contentJson(credential) - .asJson(PushResultDto.class) - .execute(); + systemAdminClient.prepare() + .post("/api/v1/projects/{proj}/credentials") + .pathParam("proj", FOO_PROJ) + .contentJson(credential) + .asJson(PushResultDto.class) + .execute(); assertThat(response.status()).isEqualTo(HttpStatus.CREATED); final MirrorDto newMirror = newMirror(); - response = adminClient.prepare() - .post("/api/v1/projects/{proj}/mirrors") - .pathParam("proj", FOO_PROJ) - .contentJson(newMirror) - .asJson(PushResultDto.class) - .execute(); + response = systemAdminClient.prepare() + .post("/api/v1/projects/{proj}/mirrors") + .pathParam("proj", FOO_PROJ) + .contentJson(newMirror) + .asJson(PushResultDto.class) + .execute(); assertThat(response.status()).isEqualTo(HttpStatus.CREATED); for (int i = 0; i < 3; i++) { final ResponseEntity mirrorResponse = - adminClient.prepare() - .post("/api/v1/projects/{proj}/mirrors/{mirrorId}/run") - .pathParam("proj", FOO_PROJ) - .pathParam("mirrorId", TEST_MIRROR_ID) - .asJson(MirrorResult.class) - .execute(); + systemAdminClient.prepare() + .post("/api/v1/projects/{proj}/mirrors/{mirrorId}/run") + .pathParam("proj", FOO_PROJ) + .pathParam("mirrorId", TEST_MIRROR_ID) + .asJson(MirrorResult.class) + .execute(); assertThat(mirrorResponse.status()).isEqualTo(HttpStatus.OK); if (i == 0) { @@ -159,10 +159,11 @@ private static MirrorDto newMirror() { "/", "main", null, - PRIVATE_KEY_FILE); + PRIVATE_KEY_FILE, + null); } - private static PublicKeyCredential getCredential() throws Exception { + static PublicKeyCredential getCredential() throws Exception { final String publicKeyFile = "ecdsa_256.openssh.pub"; final byte[] privateKeyBytes = diff --git a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/TestZoneAwareMirrorListener.java b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/TestZoneAwareMirrorListener.java new file mode 100644 index 0000000000..caf9e3f937 --- /dev/null +++ b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/TestZoneAwareMirrorListener.java @@ -0,0 +1,78 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.it.mirror.git; + +import static com.google.common.base.MoreObjects.firstNonNull; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.linecorp.centraldogma.server.mirror.MirrorListener; +import com.linecorp.centraldogma.server.mirror.MirrorResult; +import com.linecorp.centraldogma.server.mirror.MirrorTask; + +public class TestZoneAwareMirrorListener implements MirrorListener { + + private static final Logger logger = LoggerFactory.getLogger(TestZoneAwareMirrorListener.class); + + static final Map startCount = new ConcurrentHashMap<>(); + static final Map> completions = new ConcurrentHashMap<>(); + static final Map> errors = new ConcurrentHashMap<>(); + + static void reset() { + startCount.clear(); + completions.clear(); + errors.clear(); + } + + private static String key(MirrorTask task) { + return firstNonNull(task.currentZone(), "default"); + } + + @Override + public void onStart(MirrorTask mirror) { + logger.debug("onStart: {}", mirror); + startCount.merge(key(mirror), 1, Integer::sum); + } + + @Override + public void onComplete(MirrorTask mirror, MirrorResult result) { + logger.debug("onComplete: {} -> {}", mirror, result); + final List results = new ArrayList<>(); + results.add(result); + completions.merge(key(mirror), results, (oldValue, newValue) -> { + oldValue.addAll(newValue); + return oldValue; + }); + } + + @Override + public void onError(MirrorTask mirror, Throwable cause) { + logger.debug("onError: {}", mirror, cause); + final List exceptions = new ArrayList<>(); + exceptions.add(cause); + errors.merge(key(mirror), exceptions, (oldValue, newValue) -> { + oldValue.addAll(newValue); + return oldValue; + }); + } +} diff --git a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/ZoneAwareMirrorTest.java b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/ZoneAwareMirrorTest.java new file mode 100644 index 0000000000..37c89b63d7 --- /dev/null +++ b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/ZoneAwareMirrorTest.java @@ -0,0 +1,249 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.it.mirror.git; + +import static com.linecorp.centraldogma.it.mirror.git.MirrorRunnerTest.BAR_REPO; +import static com.linecorp.centraldogma.it.mirror.git.MirrorRunnerTest.FOO_PROJ; +import static com.linecorp.centraldogma.it.mirror.git.MirrorRunnerTest.PRIVATE_KEY_FILE; +import static com.linecorp.centraldogma.it.mirror.git.MirrorRunnerTest.TEST_MIRROR_ID; +import static com.linecorp.centraldogma.it.mirror.git.MirrorRunnerTest.getCredential; +import static com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil.PASSWORD; +import static com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil.USERNAME; +import static com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil.getAccessToken; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; +import static org.awaitility.Awaitility.await; + +import java.net.URI; +import java.util.List; + +import javax.annotation.Nullable; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.FieldSource; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableList; + +import com.linecorp.armeria.client.BlockingWebClient; +import com.linecorp.armeria.client.InvalidHttpResponseException; +import com.linecorp.armeria.client.WebClient; +import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.ResponseEntity; +import com.linecorp.armeria.common.auth.AuthToken; +import com.linecorp.centraldogma.client.CentralDogma; +import com.linecorp.centraldogma.client.CentralDogmaRepository; +import com.linecorp.centraldogma.client.armeria.ArmeriaCentralDogmaBuilder; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.MirrorException; +import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.internal.api.v1.MirrorDto; +import com.linecorp.centraldogma.internal.api.v1.PushResultDto; +import com.linecorp.centraldogma.server.CentralDogmaBuilder; +import com.linecorp.centraldogma.server.ZoneConfig; +import com.linecorp.centraldogma.server.internal.credential.PublicKeyCredential; +import com.linecorp.centraldogma.server.internal.storage.repository.MirrorConfig; +import com.linecorp.centraldogma.server.mirror.MirrorDirection; +import com.linecorp.centraldogma.server.mirror.MirrorResult; +import com.linecorp.centraldogma.server.mirror.MirroringServicePluginConfig; +import com.linecorp.centraldogma.testing.internal.CentralDogmaReplicationExtension; +import com.linecorp.centraldogma.testing.internal.CentralDogmaRuleDelegate; +import com.linecorp.centraldogma.testing.internal.auth.TestAuthProviderFactory; + +class ZoneAwareMirrorTest { + + private static final List ZONES = ImmutableList.of("zone1", "zone2", "zone3"); + + @RegisterExtension + CentralDogmaReplicationExtension cluster = new CentralDogmaReplicationExtension(3) { + @Override + protected void configureEach(int serverId, CentralDogmaBuilder builder) { + builder.authProviderFactory(new TestAuthProviderFactory()); + builder.systemAdministrators(USERNAME); + builder.zone(new ZoneConfig(ZONES.get(serverId - 1), ZONES)); + builder.pluginConfigs(new MirroringServicePluginConfig(true, null, null, null, true)); + } + + @Override + protected boolean runForEachTest() { + return true; + } + }; + + private static int serverPort; + private static String accessToken; + + @BeforeEach + void setUp() throws Exception { + final CentralDogmaRuleDelegate server1 = cluster.servers().get(0); + serverPort = server1.serverAddress().getPort(); + accessToken = getAccessToken( + WebClient.of("http://127.0.0.1:" + serverPort), + USERNAME, PASSWORD); + + final CentralDogma client = + new ArmeriaCentralDogmaBuilder() + .host("127.0.0.1", serverPort) + .accessToken(accessToken) + .build(); + client.createProject(FOO_PROJ).join(); + for (String zone : ZONES) { + client.createRepository(FOO_PROJ, BAR_REPO + '-' + zone).join(); + } + client.createRepository(FOO_PROJ, "bar-default").join(); + client.createRepository(FOO_PROJ, "bar-unknown-zone").join(); + TestZoneAwareMirrorListener.reset(); + } + + @FieldSource("ZONES") + @ParameterizedTest + void shouldRunMirrorTaskOnPinnedZone(String zone) throws Exception { + createMirror(zone); + + await().untilAsserted(() -> { + // Wait for three mirror tasks to run to ensure that all tasks are running in the same zone. + assertThat(TestZoneAwareMirrorListener.startCount.get(zone)).isGreaterThanOrEqualTo(3); + }); + await().untilAsserted(() -> { + final List results = TestZoneAwareMirrorListener.completions.get(zone); + assertThat(results).hasSizeGreaterThan(3); + // Make sure that the mirror was executed in the specified zone. + assertThat(results).allSatisfy(result -> { + assertThat(result.zone()).isEqualTo(zone); + assertThat(result.repoName()).isEqualTo(BAR_REPO + '-' + zone); + }); + }); + assertThat(TestZoneAwareMirrorListener.errors.get(zone)).isNullOrEmpty(); + } + + @Test + void shouldRunUnpinnedMirrorTaskOnDefaultZone() throws Exception { + createMirror(null); + // The default zone is the first zone in the list. + final String defaultZone = ZONES.get(0); + await().untilAsserted(() -> { + // Wait for 3 mirror tasks to be run to verify all jobs are executed in the same zone. + assertThat(TestZoneAwareMirrorListener.startCount.get(defaultZone)).isGreaterThanOrEqualTo(3); + }); + await().untilAsserted(() -> { + final List results = TestZoneAwareMirrorListener.completions.get(defaultZone); + assertThat(results).hasSizeGreaterThan(3); + // Make sure that the mirror was executed in the specified zone. + assertThat(results).allSatisfy(mirrorResult -> { + assertThat(mirrorResult.zone()).isNull(); + assertThat(mirrorResult.repoName()).isEqualTo("bar-default"); + }); + }); + } + + @Test + void shouldRejectUnknownZone() throws Exception { + final String unknownZone = "unknown-zone"; + final InvalidHttpResponseException invalidResponseException = + catchThrowableOfType(InvalidHttpResponseException.class, () -> createMirror(unknownZone)); + assertThat(invalidResponseException.response().status()).isEqualTo(HttpStatus.BAD_REQUEST); + assertThat(invalidResponseException.response().contentUtf8()) + .contains("The zone 'unknown-zone' is not in the zone configuration"); + } + + @Test + void shouldWarnUnknownZoneForScheduledJob() throws Exception { + final CentralDogma client = cluster.servers().get(0).client(); + final CentralDogmaRepository repo = client.forRepo(FOO_PROJ, "meta"); + final String mirrorId = TEST_MIRROR_ID + "-unknown-zone"; + final String unknownZone = "unknown-zone"; + final MirrorConfig mirrorConfig = + new MirrorConfig(mirrorId, + true, + "0/1 * * * * ?", + MirrorDirection.REMOTE_TO_LOCAL, + "bar-unknown-zone", + "/", + URI.create( + "git+ssh://github.com/line/centraldogma-authtest.git/#main"), + null, + "foo", + unknownZone); + final Change change = + Change.ofJsonUpsert("/mirrors/" + mirrorId + ".json", Jackson.writeValueAsString(mirrorConfig)); + repo.commit("Add a mirror having an invalid zone", change) + .push().join(); + + await().untilAsserted(() -> { + // Wait for 3 mirror tasks to be run to verify all jobs are executed in the same zone. + assertThat(TestZoneAwareMirrorListener.startCount.get(unknownZone)).isGreaterThanOrEqualTo(1); + }); + await().untilAsserted(() -> { + final List results = TestZoneAwareMirrorListener.completions.get(unknownZone); + assertThat(results).isNullOrEmpty(); + final List causes = TestZoneAwareMirrorListener.errors.get(unknownZone); + + // Make sure that the mirror was executed in the specified zone. + assertThat(causes).allSatisfy(cause -> { + assertThat(cause).isInstanceOf(MirrorException.class) + .hasMessage("The mirror is pinned to an unknown zone: unknown-zone " + + "(valid zones: " + ZONES + ')'); + }); + }); + } + + private static void createMirror(String zone) throws Exception { + final BlockingWebClient client = WebClient.builder("http://127.0.0.1:" + serverPort) + .auth(AuthToken.ofOAuth2(accessToken)) + .build() + .blocking(); + + final PublicKeyCredential credential = getCredential(); + ResponseEntity response = + client.prepare() + .post("/api/v1/projects/{proj}/credentials") + .pathParam("proj", FOO_PROJ) + .contentJson(credential) + .asJson(PushResultDto.class) + .execute(); + assertThat(response.status()).isEqualTo(HttpStatus.CREATED); + + final MirrorDto newMirror = newMirror(zone); + response = client.prepare() + .post("/api/v1/projects/{proj}/mirrors") + .pathParam("proj", FOO_PROJ) + .contentJson(newMirror) + .asJson(PushResultDto.class) + .execute(); + assertThat(response.status()).isEqualTo(HttpStatus.CREATED); + } + + private static MirrorDto newMirror(@Nullable String zone) { + return new MirrorDto(TEST_MIRROR_ID + '-' + (zone == null ? "default" : zone), + true, + FOO_PROJ, + "0/1 * * * * ?", + "REMOTE_TO_LOCAL", + BAR_REPO + '-' + (zone == null ? "default" : zone), + "/", + "git+ssh", + "github.com/line/centraldogma-authtest.git", + "/", + "main", + null, + PRIVATE_KEY_FILE, + zone); + } +} diff --git a/it/mirror/src/test/resources/META-INF/services/com.linecorp.centraldogma.server.mirror.MirrorListener b/it/mirror/src/test/resources/META-INF/services/com.linecorp.centraldogma.server.mirror.MirrorListener index bb05c0ff8c..a698aaa28f 100644 --- a/it/mirror/src/test/resources/META-INF/services/com.linecorp.centraldogma.server.mirror.MirrorListener +++ b/it/mirror/src/test/resources/META-INF/services/com.linecorp.centraldogma.server.mirror.MirrorListener @@ -1 +1,2 @@ com.linecorp.centraldogma.it.mirror.git.TestMirrorRunnerListener +com.linecorp.centraldogma.it.mirror.git.TestZoneAwareMirrorListener diff --git a/it/server/src/test/java/com/linecorp/centraldogma/it/NonRandomTokenTest.java b/it/server/src/test/java/com/linecorp/centraldogma/it/NonRandomTokenTest.java index 32316af3bd..1caaf48fdd 100644 --- a/it/server/src/test/java/com/linecorp/centraldogma/it/NonRandomTokenTest.java +++ b/it/server/src/test/java/com/linecorp/centraldogma/it/NonRandomTokenTest.java @@ -41,7 +41,7 @@ class NonRandomTokenTest { static final CentralDogmaExtension dogma = new CentralDogmaExtension() { @Override protected void configure(CentralDogmaBuilder builder) { - builder.administrators(TestAuthMessageUtil.USERNAME); + builder.systemAdministrators(TestAuthMessageUtil.USERNAME); builder.authProviderFactory(new TestAuthProviderFactory()); } }; @@ -56,17 +56,17 @@ void createNonRandomToken() throws Exception { assertThat(response.status()).isEqualTo(HttpStatus.OK); final String sessionId = Jackson.readValue(response.content().array(), AccessToken.class) .accessToken(); - final WebClient adminClient = WebClient.builder(client.uri()) - .auth(AuthToken.ofOAuth2(sessionId)).build(); + final WebClient systemAdminClient = WebClient.builder(client.uri()) + .auth(AuthToken.ofOAuth2(sessionId)).build(); final HttpRequest request = HttpRequest.builder() .post("/api/v1/tokens") .content(MediaType.FORM_DATA, - "secret=appToken-secret&isAdmin=true&appId=foo") + "secret=appToken-secret&isSystemAdmin=true&appId=foo") .build(); - AggregatedHttpResponse res = adminClient.execute(request).aggregate().join(); + AggregatedHttpResponse res = systemAdminClient.execute(request).aggregate().join(); assertThat(res.status()).isEqualTo(HttpStatus.CREATED); - res = adminClient.get("/api/v1/tokens").aggregate().join(); + res = systemAdminClient.get("/api/v1/tokens").aggregate().join(); assertThat(res.contentUtf8()).contains("\"secret\":\"appToken-secret\""); } } diff --git a/it/server/src/test/java/com/linecorp/centraldogma/it/ReplicationWriteQuotaTest.java b/it/server/src/test/java/com/linecorp/centraldogma/it/ReplicationWriteQuotaTest.java index b3dc35f5a6..bacf598aa4 100644 --- a/it/server/src/test/java/com/linecorp/centraldogma/it/ReplicationWriteQuotaTest.java +++ b/it/server/src/test/java/com/linecorp/centraldogma/it/ReplicationWriteQuotaTest.java @@ -130,7 +130,7 @@ private static CompletableFuture startNewReplica( int port, int serverId, Map servers) throws IOException { return new CentralDogmaBuilder(tempDir.newFolder().toFile()) .port(port, SessionProtocol.HTTP) - .administrators(TestAuthMessageUtil.USERNAME) + .systemAdministrators(TestAuthMessageUtil.USERNAME) .authProviderFactory(factory) .pluginConfigs(new MirroringServicePluginConfig(false)) .writeQuotaPerRepository(5, 1) diff --git a/it/server/src/test/java/com/linecorp/centraldogma/it/StandaloneWriteQuotaTest.java b/it/server/src/test/java/com/linecorp/centraldogma/it/StandaloneWriteQuotaTest.java index ec8c8db8de..8d46b20747 100644 --- a/it/server/src/test/java/com/linecorp/centraldogma/it/StandaloneWriteQuotaTest.java +++ b/it/server/src/test/java/com/linecorp/centraldogma/it/StandaloneWriteQuotaTest.java @@ -45,7 +45,7 @@ class StandaloneWriteQuotaTest extends WriteQuotaTestBase { static final CentralDogmaExtension dogma = new CentralDogmaExtension() { @Override protected void configure(CentralDogmaBuilder builder) { - builder.administrators(TestAuthMessageUtil.USERNAME); + builder.systemAdministrators(TestAuthMessageUtil.USERNAME); builder.authProviderFactory(new TestAuthProviderFactory()); // Default write quota builder.writeQuotaPerRepository(5, 1); diff --git a/it/server/src/test/java/com/linecorp/centraldogma/it/WriteQuotaTestBase.java b/it/server/src/test/java/com/linecorp/centraldogma/it/WriteQuotaTestBase.java index 38c19b6608..a99a685ecf 100644 --- a/it/server/src/test/java/com/linecorp/centraldogma/it/WriteQuotaTestBase.java +++ b/it/server/src/test/java/com/linecorp/centraldogma/it/WriteQuotaTestBase.java @@ -94,14 +94,16 @@ void updateWriteQuota() throws Exception { assertThat(CompletableFutures.allAsList(futures4).join()).hasSize(8); } - private static QuotaConfig updateWriteQuota(WebClient adminClient, String repoName, QuotaConfig writeQuota) + private static QuotaConfig updateWriteQuota( + WebClient systemAdminClient, String repoName, QuotaConfig writeQuota) throws JsonProcessingException { final String updatePath = "/api/v1/metadata/test_prj/repos/" + repoName + "/quota/write"; final String content = mapper.writeValueAsString(writeQuota); final HttpRequest req = HttpRequest.of(HttpMethod.PATCH, updatePath, MediaType.JSON_PATCH, content); - assertThat(adminClient.execute(req).aggregate().join().status()).isEqualTo(HttpStatus.OK); + assertThat(systemAdminClient.execute(req).aggregate().join().status()).isEqualTo(HttpStatus.OK); - final AggregatedHttpResponse res = adminClient.get("/api/v1/projects/test_prj").aggregate().join(); + final AggregatedHttpResponse res = systemAdminClient.get("/api/v1/projects/test_prj") + .aggregate().join(); final ProjectMetadata meta = Jackson.readValue(res.contentUtf8(), ProjectMetadata.class); return meta.repo(repoName).writeQuota(); } diff --git a/it/xds-member-permission/src/test/java/com/linecorp/centraldogma/server/test/XdsMemberPermissionTest.java b/it/xds-member-permission/src/test/java/com/linecorp/centraldogma/server/test/XdsMemberPermissionTest.java index 08e73924cb..4a5fcee82c 100644 --- a/it/xds-member-permission/src/test/java/com/linecorp/centraldogma/server/test/XdsMemberPermissionTest.java +++ b/it/xds-member-permission/src/test/java/com/linecorp/centraldogma/server/test/XdsMemberPermissionTest.java @@ -54,7 +54,7 @@ class XdsMemberPermissionTest { @Override protected void configure(CentralDogmaBuilder builder) { - builder.administrators(USERNAME) + builder.systemAdministrators(USERNAME) .cors("*") .authProviderFactory(new ShiroAuthProviderFactory(unused -> { final Ini iniConfig = new Ini(); diff --git a/it/zone-leader-plugin/src/test/java/com/linecorp/centraldogma/it/zoneleader/ZoneLeaderPluginTest.java b/it/zone-leader-plugin/src/test/java/com/linecorp/centraldogma/it/zoneleader/ZoneLeaderPluginTest.java index a8374bfceb..7489c0ca81 100644 --- a/it/zone-leader-plugin/src/test/java/com/linecorp/centraldogma/it/zoneleader/ZoneLeaderPluginTest.java +++ b/it/zone-leader-plugin/src/test/java/com/linecorp/centraldogma/it/zoneleader/ZoneLeaderPluginTest.java @@ -27,8 +27,12 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import com.google.common.collect.ImmutableList; + import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.centraldogma.server.CentralDogmaBuilder; +import com.linecorp.centraldogma.server.CentralDogmaConfig; +import com.linecorp.centraldogma.server.ZoneConfig; import com.linecorp.centraldogma.server.plugin.Plugin; import com.linecorp.centraldogma.server.plugin.PluginContext; import com.linecorp.centraldogma.server.plugin.PluginTarget; @@ -38,17 +42,18 @@ class ZoneLeaderPluginTest { private static final List plugins = new ArrayList<>(); private static final int NUM_REPLICAS = 9; + private static final List zones = ImmutableList.of("zone1", "zone2", "zone3"); @RegisterExtension static CentralDogmaReplicationExtension cluster = new CentralDogmaReplicationExtension(NUM_REPLICAS) { @Override protected void configureEach(int serverId, CentralDogmaBuilder builder) { if (serverId <= 3) { - builder.zone("zone1"); + builder.zone(new ZoneConfig("zone1", zones)); } else if (serverId <= 6) { - builder.zone("zone2"); + builder.zone(new ZoneConfig("zone2", zones)); } else { - builder.zone("zone3"); + builder.zone(new ZoneConfig("zone3", zones)); } final ZoneLeaderTestPlugin plugin = new ZoneLeaderTestPlugin(serverId); plugins.add(plugin); @@ -114,7 +119,7 @@ private ZoneLeaderTestPlugin(int serverId) { } @Override - public PluginTarget target() { + public PluginTarget target(CentralDogmaConfig config) { return PluginTarget.ZONE_LEADER_ONLY; } diff --git a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractGitMirror.java b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractGitMirror.java index 225339517a..bff561b3d5 100644 --- a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractGitMirror.java +++ b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractGitMirror.java @@ -125,9 +125,9 @@ abstract class AbstractGitMirror extends AbstractMirror { AbstractGitMirror(String id, boolean enabled, @Nullable Cron schedule, MirrorDirection direction, Credential credential, Repository localRepo, String localPath, URI remoteRepoUri, String remotePath, String remoteBranch, - @Nullable String gitignore) { + @Nullable String gitignore, @Nullable String zone) { super(id, enabled, schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath, - remoteBranch, gitignore); + remoteBranch, gitignore, zone); if (gitignore != null) { ignoreNode = new IgnoreNode(); diff --git a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultGitMirror.java b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultGitMirror.java index 6687d3cc9c..e92a893453 100644 --- a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultGitMirror.java +++ b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultGitMirror.java @@ -47,9 +47,9 @@ final class DefaultGitMirror extends AbstractGitMirror { DefaultGitMirror(String id, boolean enabled, @Nullable Cron schedule, MirrorDirection direction, Credential credential, Repository localRepo, String localPath, URI remoteRepoUri, String remotePath, String remoteBranch, - @Nullable String gitignore) { + @Nullable String gitignore, @Nullable String zone) { super(id, enabled, schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath, - remoteBranch, gitignore); + remoteBranch, gitignore, zone); } @Override diff --git a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirrorProvider.java b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirrorProvider.java index dd6a2baee8..51711b9fca 100644 --- a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirrorProvider.java +++ b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirrorProvider.java @@ -49,7 +49,7 @@ public Mirror newMirror(MirrorContext context) { context.direction(), context.credential(), context.localRepo(), context.localPath(), repositoryUri.uri(), repositoryUri.path(), repositoryUri.branch(), - context.gitignore()); + context.gitignore(), context.zone()); } case SCHEME_GIT_HTTP: case SCHEME_GIT_HTTPS: @@ -60,7 +60,7 @@ public Mirror newMirror(MirrorContext context) { context.direction(), context.credential(), context.localRepo(), context.localPath(), repositoryUri.uri(), repositoryUri.path(), repositoryUri.branch(), - context.gitignore()); + context.gitignore(), context.zone()); } } diff --git a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirror.java b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirror.java index d842d255a7..9421e304ca 100644 --- a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirror.java +++ b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirror.java @@ -85,10 +85,9 @@ final class SshGitMirror extends AbstractGitMirror { SshGitMirror(String id, boolean enabled, @Nullable Cron schedule, MirrorDirection direction, Credential credential, Repository localRepo, String localPath, URI remoteRepoUri, String remotePath, String remoteBranch, - @Nullable String gitignore) { + @Nullable String gitignore, @Nullable String zone) { super(id, enabled, schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath, - remoteBranch, - gitignore); + remoteBranch, gitignore, zone); } @Override diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMetaRepositoryWithMirrorTest.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMetaRepositoryWithMirrorTest.java index bdb586ae6d..fa13c8bba6 100644 --- a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMetaRepositoryWithMirrorTest.java +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMetaRepositoryWithMirrorTest.java @@ -171,9 +171,10 @@ void testMirror(boolean useRawApi) { } else { final List mirrors = ImmutableList.of( new MirrorDto("foo", true, project.name(), DEFAULT_SCHEDULE, "LOCAL_TO_REMOTE", "foo", - "/mirrors/foo", "git+ssh", "foo.com/foo.git", "", "", null, "alice"), + "/mirrors/foo", "git+ssh", "foo.com/foo.git", "", "", null, "alice", null), new MirrorDto("bar", true, project.name(), "0 */10 * * * ?", "REMOTE_TO_LOCAL", "bar", - "", "git+ssh", "bar.com/bar.git", "/some-path", "develop", null, "bob")); + "", "git+ssh", "bar.com/bar.git", "/some-path", "develop", null, "bob", + null)); for (Credential credential : CREDENTIALS) { final Command command = metaRepo.createPushCommand(credential, Author.SYSTEM, false).join(); @@ -181,7 +182,7 @@ void testMirror(boolean useRawApi) { } for (MirrorDto mirror : mirrors) { final Command command = - metaRepo.createPushCommand(mirror, Author.SYSTEM, false).join(); + metaRepo.createPushCommand(mirror, Author.SYSTEM, null, false).join(); pmExtension.executor().execute(command).join(); } } diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingServiceTest.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingServiceTest.java index 05423c90c1..4cfd1327e9 100644 --- a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingServiceTest.java +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingServiceTest.java @@ -75,7 +75,7 @@ void mirroringTaskShouldNeverBeRejected() { final Mirror mirror = new AbstractMirror("my-mirror-1", true, EVERY_SECOND, MirrorDirection.REMOTE_TO_LOCAL, Credential.FALLBACK, r, "/", - URI.create("unused://uri"), "/", "", null) { + URI.create("unused://uri"), "/", "", null, null) { @Override protected MirrorResult mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes, Instant triggeredTime) { @@ -96,7 +96,7 @@ protected MirrorResult mirrorRemoteToLocal(File workDir, CommandExecutor executo when(mr.mirrors()).thenReturn(CompletableFuture.completedFuture(ImmutableList.of(mirror))); final MirrorSchedulingService service = new MirrorSchedulingService( - temporaryFolder, pm, new SimpleMeterRegistry(), 1, 1, 1); + temporaryFolder, pm, new SimpleMeterRegistry(), 1, 1, 1, null); final CommandExecutor executor = mock(CommandExecutor.class); service.start(executor); diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringAndCredentialServiceV1Test.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringAndCredentialServiceV1Test.java index e6e905e459..3ae649b6a1 100644 --- a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringAndCredentialServiceV1Test.java +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringAndCredentialServiceV1Test.java @@ -65,7 +65,7 @@ class MirroringAndCredentialServiceV1Test { @Override protected void configure(CentralDogmaBuilder builder) { builder.authProviderFactory(new TestAuthProviderFactory()); - builder.administrators(USERNAME); + builder.systemAdministrators(USERNAME); } @Override @@ -87,16 +87,16 @@ protected void scaffold(CentralDogma client) { } }; - private BlockingWebClient adminClient; + private BlockingWebClient systemAdminClient; private BlockingWebClient userClient; @BeforeEach void setUp() throws JsonProcessingException { - final String adminToken = getAccessToken(dogma.httpClient(), USERNAME, PASSWORD); - adminClient = WebClient.builder(dogma.httpClient().uri()) - .auth(AuthToken.ofOAuth2(adminToken)) - .build() - .blocking(); + final String systemAdminToken = getAccessToken(dogma.httpClient(), USERNAME, PASSWORD); + systemAdminClient = WebClient.builder(dogma.httpClient().uri()) + .auth(AuthToken.ofOAuth2(systemAdminToken)) + .build() + .blocking(); final String userToken = getAccessToken(dogma.httpClient(), USERNAME2, PASSWORD2); userClient = WebClient.builder(dogma.httpClient().uri()) @@ -132,7 +132,8 @@ private void rejectInvalidRepositoryUri() { "/remote-path/1", "mirror-branch", ".my-env0\n.my-env1", - "public-key-credential"); + "public-key-credential", + null); final AggregatedHttpResponse response = userClient.prepare() .post("/api/v1/projects/{proj}/mirrors") @@ -145,12 +146,12 @@ private void rejectInvalidRepositoryUri() { private void setUpRole() { final ResponseEntity res = - adminClient.prepare() - .post("/api/v1/metadata/{proj}/members") - .pathParam("proj", FOO_PROJ) - .contentJson(ImmutableMap.of("id", USERNAME2, "role", "OWNER")) - .asJson(Revision.class) - .execute(); + systemAdminClient.prepare() + .post("/api/v1/metadata/{proj}/members") + .pathParam("proj", FOO_PROJ) + .contentJson(ImmutableMap.of("id", USERNAME2, "role", "OWNER")) + .asJson(Revision.class) + .execute(); assertThat(res.status()).isEqualTo(HttpStatus.OK); } @@ -180,8 +181,8 @@ private void createAndReadCredential() { assertThat(creationResponse.status()).isEqualTo(HttpStatus.CREATED); assertThat(creationResponse.content().revision().major()).isEqualTo(i + 2); - for (BlockingWebClient client : ImmutableList.of(adminClient, userClient)) { - final boolean isAdmin = client == adminClient; + for (BlockingWebClient client : ImmutableList.of(systemAdminClient, userClient)) { + final boolean isSystemAdmin = client == systemAdminClient; final ResponseEntity fetchResponse = client.prepare() .get("/api/v1/projects/{proj}/credentials/{id}") @@ -196,14 +197,14 @@ private void createAndReadCredential() { if ("password".equals(credentialType)) { final PasswordCredential actual = (PasswordCredential) credentialDto; assertThat(actual.username()).isEqualTo(credential.get("username")); - if (isAdmin) { + if (isSystemAdmin) { assertThat(actual.password()).isEqualTo(credential.get("password")); } else { assertThat(actual.password()).isEqualTo("****"); } } else if ("access_token".equals(credentialType)) { final AccessTokenCredential actual = (AccessTokenCredential) credentialDto; - if (isAdmin) { + if (isSystemAdmin) { assertThat(actual.accessToken()).isEqualTo(credential.get("accessToken")); } else { assertThat(actual.accessToken()).isEqualTo("****"); @@ -212,7 +213,7 @@ private void createAndReadCredential() { final PublicKeyCredential actual = (PublicKeyCredential) credentialDto; assertThat(actual.username()).isEqualTo(credential.get("username")); assertThat(actual.publicKey()).isEqualTo(credential.get("publicKey")); - if (isAdmin) { + if (isSystemAdmin) { assertThat(actual.rawPrivateKey()).isEqualTo(credential.get("privateKey")); assertThat(actual.rawPassphrase()).isEqualTo(credential.get("passphrase")); } else { @@ -247,8 +248,8 @@ private void updateCredential() { .execute(); assertThat(creationResponse.status()).isEqualTo(HttpStatus.OK); - for (BlockingWebClient client : ImmutableList.of(adminClient, userClient)) { - final boolean isAdmin = client == adminClient; + for (BlockingWebClient client : ImmutableList.of(systemAdminClient, userClient)) { + final boolean isSystemAdmin = client == systemAdminClient; final ResponseEntity fetchResponse = client.prepare() .get("/api/v1/projects/{proj}/credentials/{id}") @@ -260,7 +261,7 @@ private void updateCredential() { assertThat(actual.id()).isEqualTo((String) credential.get("id")); assertThat(actual.username()).isEqualTo(credential.get("username")); assertThat(actual.publicKey()).isEqualTo(credential.get("publicKey")); - if (isAdmin) { + if (isSystemAdmin) { assertThat(actual.rawPrivateKey()).isEqualTo(credential.get("privateKey")); assertThat(actual.rawPassphrase()).isEqualTo(credential.get("passphrase")); } else { @@ -291,6 +292,40 @@ private void createAndReadMirror() { final MirrorDto savedMirror = response1.content(); assertThat(savedMirror).isEqualTo(newMirror); } + + // Make sure that the mirror with a port number in the remote URL can be created and read. + final MirrorDto mirrorWithPort = new MirrorDto("mirror-with-port-3", + true, + FOO_PROJ, + "5 * * * * ?", + "REMOTE_TO_LOCAL", + BAR_REPO, + "/updated/local-path/", + "git+https", + "git.com:922/line/centraldogma-test.git", + "/updated/remote-path/", + "updated-mirror-branch", + ".updated-env", + "public-key-credential", + null); + + final ResponseEntity response0 = + userClient.prepare() + .post("/api/v1/projects/{proj}/mirrors") + .pathParam("proj", FOO_PROJ) + .contentJson(mirrorWithPort) + .asJson(PushResultDto.class) + .execute(); + assertThat(response0.status()).isEqualTo(HttpStatus.CREATED); + final ResponseEntity response1 = + userClient.prepare() + .get("/api/v1/projects/{proj}/mirrors/{id}") + .pathParam("proj", FOO_PROJ) + .pathParam("id", mirrorWithPort.id()) + .asJson(MirrorDto.class) + .execute(); + final MirrorDto savedMirror = response1.content(); + assertThat(savedMirror).isEqualTo(mirrorWithPort); } private void updateMirror() { @@ -306,7 +341,8 @@ private void updateMirror() { "/updated/remote-path/", "updated-mirror-branch", ".updated-env", - "access-token-credential"); + "access-token-credential", + null); final ResponseEntity updateResponse = userClient.prepare() .put("/api/v1/projects/{proj}/mirrors/{id}") @@ -376,6 +412,7 @@ private static MirrorDto newMirror(String id) { "/remote-path/" + id + '/', "mirror-branch", ".my-env0\n.my-env1", - "public-key-credential"); + "public-key-credential", + null); } } diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTaskTest.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTaskTest.java index 273b6eac2b..8403171b23 100644 --- a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTaskTest.java +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTaskTest.java @@ -51,9 +51,9 @@ void testSuccessMetrics() { Mirror mirror = newMirror("git://a.com/b.git", DefaultGitMirror.class, "foo", "bar"); mirror = spy(mirror); doReturn(new MirrorResult(mirror.id(), "foo", "bar", MirrorStatus.SUCCESS, "", Instant.now(), - Instant.now())) + Instant.now(), null)) .when(mirror).mirror(any(), any(), anyInt(), anyLong(), any()); - final MirrorTask mirrorTask = new MirrorTask(mirror, User.SYSTEM, Instant.now(), true); + final MirrorTask mirrorTask = new MirrorTask(mirror, User.SYSTEM, Instant.now(), null, true); new InstrumentedMirroringJob(mirrorTask, meterRegistry).run(null, null, 0, 0L); assertThat(MoreMeters.measureAll(meterRegistry)) .contains(entry("mirroring.result#count{direction=LOCAL_TO_REMOTE,localPath=/," + @@ -67,7 +67,7 @@ void testFailureMetrics() { mirror = spy(mirror); final RuntimeException e = new RuntimeException(); doThrow(e).when(mirror).mirror(any(), any(), anyInt(), anyLong(), any()); - final MirrorTask mirrorTask = new MirrorTask(mirror, User.SYSTEM, Instant.now(), true); + final MirrorTask mirrorTask = new MirrorTask(mirror, User.SYSTEM, Instant.now(), null, true); final InstrumentedMirroringJob task = new InstrumentedMirroringJob(mirrorTask, meterRegistry); assertThatThrownBy(() -> task.run(null, null, 0, 0L)) .isSameAs(e); @@ -86,7 +86,7 @@ void testTimerMetrics() { Thread.sleep(1000); return null; }).when(mirror).mirror(any(), any(), anyInt(), anyLong(), any()); - final MirrorTask mirrorTask = new MirrorTask(mirror, User.SYSTEM, Instant.now(), true); + final MirrorTask mirrorTask = new MirrorTask(mirror, User.SYSTEM, Instant.now(), null, true); new InstrumentedMirroringJob(mirrorTask, meterRegistry).run(null, null, 0, 0L); assertThat(MoreMeters.measureAll(meterRegistry)) .hasEntrySatisfying( diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTestUtils.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTestUtils.java index 2df3911b32..f3cf3478d8 100644 --- a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTestUtils.java +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTestUtils.java @@ -59,7 +59,7 @@ static T newMirror(String remoteUri, Cron schedule, final Mirror mirror = new GitMirrorProvider().newMirror( new MirrorContext("mirror-id", true, schedule, MirrorDirection.LOCAL_TO_REMOTE, - credential, repository, "/", URI.create(remoteUri), null)); + credential, repository, "/", URI.create(remoteUri), null, null)); assertThat(mirror).isInstanceOf(mirrorType); assertThat(mirror.direction()).isEqualTo(MirrorDirection.LOCAL_TO_REMOTE); @@ -76,7 +76,7 @@ static void assertMirrorNull(String remoteUri) { final Credential credential = mock(Credential.class); final Mirror mirror = new GitMirrorProvider().newMirror( new MirrorContext("mirror-id", true, EVERY_MINUTE, MirrorDirection.LOCAL_TO_REMOTE, - credential, mock(Repository.class), "/", URI.create(remoteUri), null)); + credential, mock(Repository.class), "/", URI.create(remoteUri), null, null)); assertThat(mirror).isNull(); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java index f58e645239..8d0b05ad31 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java @@ -36,6 +36,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; @@ -131,7 +132,6 @@ import com.linecorp.centraldogma.server.internal.admin.service.DefaultLogoutService; import com.linecorp.centraldogma.server.internal.admin.service.RepositoryService; import com.linecorp.centraldogma.server.internal.admin.service.UserService; -import com.linecorp.centraldogma.server.internal.api.AdministrativeService; import com.linecorp.centraldogma.server.internal.api.ContentServiceV1; import com.linecorp.centraldogma.server.internal.api.CredentialServiceV1; import com.linecorp.centraldogma.server.internal.api.GitHttpService; @@ -140,6 +140,7 @@ import com.linecorp.centraldogma.server.internal.api.MirroringServiceV1; import com.linecorp.centraldogma.server.internal.api.ProjectServiceV1; import com.linecorp.centraldogma.server.internal.api.RepositoryServiceV1; +import com.linecorp.centraldogma.server.internal.api.SystemAdministrativeService; import com.linecorp.centraldogma.server.internal.api.TokenService; import com.linecorp.centraldogma.server.internal.api.WatchService; import com.linecorp.centraldogma.server.internal.api.auth.ApplicationTokenAuthorizer; @@ -225,6 +226,7 @@ public static CentralDogma forConfig(File configFile) throws IOException { private final AtomicInteger numPendingStopRequests = new AtomicInteger(); + private final Map pluginGroups; @Nullable private final PluginGroup pluginsForAllReplicas; @Nullable @@ -257,14 +259,12 @@ public static CentralDogma forConfig(File configFile) throws IOException { CentralDogma(CentralDogmaConfig cfg, MeterRegistry meterRegistry, List plugins) { this.cfg = requireNonNull(cfg, "cfg"); - pluginsForAllReplicas = PluginGroup.loadPlugins( - CentralDogma.class.getClassLoader(), PluginTarget.ALL_REPLICAS, cfg, plugins); - pluginsForLeaderOnly = PluginGroup.loadPlugins( - CentralDogma.class.getClassLoader(), PluginTarget.LEADER_ONLY, cfg, plugins); - pluginsForZoneLeaderOnly = PluginGroup.loadPlugins( - CentralDogma.class.getClassLoader(), PluginTarget.ZONE_LEADER_ONLY, cfg, plugins); + pluginGroups = PluginGroup.loadPlugins(CentralDogma.class.getClassLoader(), cfg, plugins); + pluginsForAllReplicas = pluginGroups.get(PluginTarget.ALL_REPLICAS); + pluginsForLeaderOnly = pluginGroups.get(PluginTarget.LEADER_ONLY); + pluginsForZoneLeaderOnly = pluginGroups.get(PluginTarget.ZONE_LEADER_ONLY); if (pluginsForZoneLeaderOnly != null) { - checkState(!isNullOrEmpty(cfg.zone()), + checkState(cfg.zone() != null, "zone must be specified when zone leader plugins are enabled."); } startStop = new CentralDogmaStartStop(pluginsForAllReplicas); @@ -319,14 +319,19 @@ public ProjectManager projectManager() { * Returns the {@link MirroringService} of the server. * * @return the {@link MirroringService} if the server is started and mirroring is enabled. - * {@link Optional#empty()} otherwise. + * {@code null} otherwise. */ - public Optional mirroringService() { - if (pluginsForLeaderOnly == null) { - return Optional.empty(); - } - return pluginsForLeaderOnly.findFirstPlugin(DefaultMirroringServicePlugin.class) - .map(DefaultMirroringServicePlugin::mirroringService); + @Nullable + public MirroringService mirroringService() { + return pluginGroups.values() + .stream() + .map(group -> { + return group.findFirstPlugin(DefaultMirroringServicePlugin.class); + }) + .filter(Objects::nonNull) + .findFirst() + .map(DefaultMirroringServicePlugin::mirroringService) + .orElse(null); } /** @@ -335,20 +340,8 @@ public Optional mirroringService() { * @param target the {@link PluginTarget} of the {@link Plugin}s to be returned */ public List plugins(PluginTarget target) { - switch (requireNonNull(target, "target")) { - case LEADER_ONLY: - return pluginsForLeaderOnly != null ? ImmutableList.copyOf(pluginsForLeaderOnly.plugins()) - : ImmutableList.of(); - case ALL_REPLICAS: - return pluginsForAllReplicas != null ? ImmutableList.copyOf(pluginsForAllReplicas.plugins()) - : ImmutableList.of(); - case ZONE_LEADER_ONLY: - return pluginsForZoneLeaderOnly != null ? - ImmutableList.copyOf(pluginsForZoneLeaderOnly.plugins()) : ImmutableList.of(); - default: - // Should not reach here. - throw new Error("Unknown plugin target: " + target); - } + requireNonNull(target, "target"); + return pluginGroups.get(target).plugins(); } /** @@ -497,7 +490,8 @@ private CommandExecutor startCommandExecutor( Consumer onReleaseZoneLeadership = null; // TODO(ikhoon): Deduplicate if (pluginsForZoneLeaderOnly != null) { - final String zone = cfg.zone(); + assert cfg.zone() != null; + final String zone = cfg.zone().currentZone(); onTakeZoneLeadership = exec -> { logger.info("Starting plugins on the {} zone leader replica ..", zone); pluginsForZoneLeaderOnly @@ -747,7 +741,7 @@ private AuthProvider createAuthProvider( final AuthProviderParameters parameters = new AuthProviderParameters( // Find application first, then find the session token. new ApplicationTokenAuthorizer(mds::findTokenBySecret).orElse( - new SessionTokenAuthorizer(sessionManager, authCfg.administrators())), + new SessionTokenAuthorizer(sessionManager, authCfg.systemAdministrators())), cfg, sessionManager::generateSessionId, // Propagate login and logout events to the other replicas. @@ -771,6 +765,10 @@ private CommandExecutor newZooKeeperCommandExecutor( final File dataDir = cfg.dataDir(); new File(dataDir, "replica_id").delete(); + String zone = null; + if (config().zone() != null) { + zone = config().zone().currentZone(); + } // TODO(trustin): Provide a way to restart/reload the replicator // so that we can recover from ZooKeeper maintenance automatically. return new ZooKeeperCommandExecutor( @@ -778,7 +776,7 @@ private CommandExecutor newZooKeeperCommandExecutor( new StandaloneCommandExecutor(pm, repositoryWorker, serverStatusManager, sessionManager, /* onTakeLeadership */ null, /* onReleaseLeadership */ null, /* onTakeZoneLeadership */ null, /* onReleaseZoneLeadership */ null), - meterRegistry, pm, config().writeQuotaPerRepository(), config().zone(), + meterRegistry, pm, config().writeQuotaPerRepository(), zone, onTakeLeadership, onReleaseLeadership, onTakeZoneLeadership, onReleaseZoneLeadership); } @@ -818,7 +816,7 @@ private Function authService( final Authorizer tokenAuthorizer = new ApplicationTokenAuthorizer(mds::findTokenBySecret) .orElse(new SessionTokenAuthorizer(sessionManager, - authCfg.administrators())); + authCfg.systemAdministrators())); return AuthService.builder() .add(tokenAuthorizer) .onFailure(new CentralDogmaAuthFailureHandler()) @@ -862,7 +860,7 @@ private void configureHttpApi(ServerBuilder sb, assert statusManager != null; final ContextPathServicesBuilder apiV1ServiceBuilder = sb.contextPath(API_V1_PATH_PREFIX); apiV1ServiceBuilder - .annotatedService(new AdministrativeService(executor, statusManager)) + .annotatedService(new SystemAdministrativeService(executor, statusManager)) .annotatedService(new ProjectServiceV1(projectApiManager, executor)) .annotatedService(new RepositoryServiceV1(executor, mds)) .annotatedService(new CredentialServiceV1(projectApiManager, executor)); @@ -870,7 +868,7 @@ private void configureHttpApi(ServerBuilder sb, if (GIT_MIRROR_ENABLED) { mirrorRunner = new MirrorRunner(projectApiManager, executor, cfg, meterRegistry); apiV1ServiceBuilder.annotatedService(new MirroringServiceV1(projectApiManager, executor, - mirrorRunner)); + mirrorRunner, cfg)); } apiV1ServiceBuilder.annotatedService() diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java index 27401e1158..1e35ced3cc 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java @@ -118,7 +118,7 @@ public final class CentralDogmaBuilder { // AuthConfig properties @Nullable private AuthProviderFactory authProviderFactory; - private final ImmutableSet.Builder administrators = new Builder<>(); + private final ImmutableSet.Builder systemAdministrators = new Builder<>(); private boolean caseSensitiveLoginNames; private String sessionCacheSpec = DEFAULT_SESSION_CACHE_SPEC; private long sessionTimeoutMillis = DEFAULT_SESSION_TIMEOUT_MILLIS; @@ -137,7 +137,7 @@ public final class CentralDogmaBuilder { @Nullable private ManagementConfig managementConfig; @Nullable - private String zone; + private ZoneConfig zoneConfig; /** * Creates a new builder with the specified data directory. @@ -416,22 +416,22 @@ public CentralDogmaBuilder authProviderFactory(AuthProviderFactory authProviderF } /** - * Adds administrators to the set. + * Adds system administrators to the set. */ - public CentralDogmaBuilder administrators(String... administrators) { - requireNonNull(administrators, "administrators"); - for (final String administrator : administrators) { - this.administrators.add(administrator); + public CentralDogmaBuilder systemAdministrators(String... systemAdministrators) { + requireNonNull(systemAdministrators, "systemAdministrators"); + for (final String systemAdministrator : systemAdministrators) { + this.systemAdministrators.add(systemAdministrator); } return this; } /** - * Adds administrators to the set. + * Adds system administrators to the set. */ - public CentralDogmaBuilder administrators(Iterable administrators) { - requireNonNull(administrators, "administrators"); - this.administrators.addAll(administrators); + public CentralDogmaBuilder systemAdministrators(Iterable systemAdministrators) { + requireNonNull(systemAdministrators, "systemAdministrators"); + this.systemAdministrators.addAll(systemAdministrators); return this; } @@ -535,6 +535,13 @@ public CentralDogmaBuilder pluginConfigs(PluginConfig... pluginConfigs) { return this; } + /** + * Returns the {@link PluginConfig}s that have been added. + */ + public List pluginConfigs() { + return pluginConfigs; + } + /** * Adds the {@link Plugin}s. */ @@ -562,11 +569,11 @@ public CentralDogmaBuilder management(ManagementConfig managementConfig) { } /** - * Specifies the zone of the server. + * Specifies the {@link ZoneConfig} of the server. */ - public CentralDogmaBuilder zone(String zone) { - requireNonNull(zone, "zone"); - this.zone = zone; + public CentralDogmaBuilder zone(ZoneConfig zoneConfig) { + requireNonNull(zoneConfig, "zoneConfig"); + this.zoneConfig = zoneConfig; return this; } @@ -580,11 +587,11 @@ public CentralDogma build() { private CentralDogmaConfig buildConfig() { final List ports = !this.ports.isEmpty() ? this.ports : Collections.singletonList(DEFAULT_PORT); - final Set adminSet = administrators.build(); + final Set systemAdminSet = systemAdministrators.build(); final AuthConfig authCfg; if (authProviderFactory != null) { authCfg = new AuthConfig( - authProviderFactory, adminSet, caseSensitiveLoginNames, + authProviderFactory, systemAdminSet, caseSensitiveLoginNames, sessionCacheSpec, sessionTimeoutMillis, sessionValidationSchedule, authProviderProperties != null ? Jackson.valueToTree(authProviderProperties) : null); } else { @@ -601,8 +608,8 @@ private CentralDogmaConfig buildConfig() { requestTimeoutMillis, idleTimeoutMillis, maxFrameLength, numRepositoryWorkers, repositoryCacheSpec, maxRemovedRepositoryAgeMillis, gracefulShutdownTimeout, - webAppEnabled, webAppTitle,replicationConfig, + webAppEnabled, webAppTitle, replicationConfig, null, accessLogFormat, authCfg, quotaConfig, - corsConfig, pluginConfigs, managementConfig, zone); + corsConfig, pluginConfigs, managementConfig, zoneConfig); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java index 0a3c76ea6d..0b3894d45f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaConfig.java @@ -271,7 +271,7 @@ public static CentralDogmaConfig load(String json) throws JsonMappingException, private final ManagementConfig managementConfig; @Nullable - private final String zone; + private final ZoneConfig zoneConfig; CentralDogmaConfig( @JsonProperty(value = "dataDir", required = true) File dataDir, @@ -300,7 +300,7 @@ public static CentralDogmaConfig load(String json) throws JsonMappingException, @JsonProperty("cors") @Nullable CorsConfig corsConfig, @JsonProperty("pluginConfigs") @Nullable List pluginConfigs, @JsonProperty("management") @Nullable ManagementConfig managementConfig, - @JsonProperty("zone") @Nullable String zone) { + @JsonProperty("zone") @Nullable ZoneConfig zoneConfig) { this.dataDir = requireNonNull(dataDir, "dataDir"); this.ports = ImmutableList.copyOf(requireNonNull(ports, "ports")); @@ -349,7 +349,7 @@ public static CentralDogmaConfig load(String json) throws JsonMappingException, pluginConfigMap = this.pluginConfigs.stream().collect( toImmutableMap(PluginConfig::getClass, Function.identity())); this.managementConfig = managementConfig; - this.zone = convertValue(zone, "zone"); + this.zoneConfig = zoneConfig; } /** @@ -589,13 +589,13 @@ public ManagementConfig managementConfig() { } /** - * Returns the zone of the server. + * Returns the zone information of the server. * Note that the zone must be specified to use the {@link PluginTarget#ZONE_LEADER_ONLY} target. */ @Nullable @JsonProperty("zone") - public String zone() { - return zone; + public ZoneConfig zone() { + return zoneConfig; } @Override diff --git a/server/src/main/java/com/linecorp/centraldogma/server/PluginGroup.java b/server/src/main/java/com/linecorp/centraldogma/server/PluginGroup.java index 0962d26732..a68c12d0d3 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/PluginGroup.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/PluginGroup.java @@ -16,10 +16,12 @@ package com.linecorp.centraldogma.server; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static java.util.Objects.requireNonNull; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.ServiceLoader; import java.util.concurrent.CompletableFuture; @@ -27,12 +29,14 @@ import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.stream.Collectors; import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; @@ -63,30 +67,28 @@ final class PluginGroup { * * @param target the {@link PluginTarget} which would be loaded */ + @VisibleForTesting @Nullable static PluginGroup loadPlugins(PluginTarget target, CentralDogmaConfig config) { - return loadPlugins(PluginGroup.class.getClassLoader(), target, config, ImmutableList.of()); + return loadPlugins(PluginGroup.class.getClassLoader(), config, ImmutableList.of()).get(target); } /** * Returns a new {@link PluginGroup} which holds the {@link Plugin}s loaded from the classpath. - * {@code null} is returned if there is no {@link Plugin} whose target equals to the specified + * An empty map is returned if there is no {@link Plugin} whose target equals to the specified * {@code target}. * * @param classLoader which is used to load the {@link Plugin}s - * @param target the {@link PluginTarget} which would be loaded */ - @Nullable - static PluginGroup loadPlugins(ClassLoader classLoader, PluginTarget target, CentralDogmaConfig config, - List plugins) { + static Map loadPlugins(ClassLoader classLoader, CentralDogmaConfig config, + List plugins) { requireNonNull(classLoader, "classLoader"); - requireNonNull(target, "target"); requireNonNull(config, "config"); final ServiceLoader loader = ServiceLoader.load(Plugin.class, classLoader); final ImmutableMap.Builder, Plugin> allPlugins = new ImmutableMap.Builder<>(); for (Plugin plugin : Iterables.concat(plugins, loader)) { - if (target == plugin.target() && plugin.isEnabled(config)) { + if (plugin.isEnabled(config)) { allPlugins.put(plugin.configType(), plugin); } } @@ -94,11 +96,31 @@ static PluginGroup loadPlugins(ClassLoader classLoader, PluginTarget target, Cen // IllegalArgumentException is thrown if there are duplicate keys. final Map, Plugin> pluginMap = allPlugins.build(); if (pluginMap.isEmpty()) { - return null; + return ImmutableMap.of(); } - return new PluginGroup(pluginMap.values(), Executors.newSingleThreadExecutor(new DefaultThreadFactory( - "plugins-for-" + target.name().toLowerCase().replace("_", "-"), true))); + final Map pluginGroups = + pluginMap.values() + .stream() + .collect(Collectors.groupingBy(plugin -> plugin.target(config))) + .entrySet() + .stream() + .collect(toImmutableMap(Entry::getKey, e -> { + final PluginTarget target = e.getKey(); + final List targetPlugins = e.getValue(); + final String poolName = + "plugins-for-" + target.name().toLowerCase().replace("_", "-"); + return new PluginGroup(targetPlugins, + Executors.newSingleThreadExecutor( + new DefaultThreadFactory(poolName, true))); + })); + + pluginGroups.forEach((target, group) -> { + logger.debug("Loaded plugins for target {}: {}", target, + group.plugins().stream().map(plugin -> plugin.getClass().getName()) + .collect(toImmutableList())); + }); + return pluginGroups; } private final List plugins; @@ -119,9 +141,10 @@ List plugins() { /** * Returns the first {@link Plugin} of the specified {@code clazz} as wrapped by an {@link Optional}. */ - Optional findFirstPlugin(Class clazz) { + @Nullable + T findFirstPlugin(Class clazz) { requireNonNull(clazz, "clazz"); - return plugins.stream().filter(clazz::isInstance).map(clazz::cast).findFirst(); + return plugins.stream().filter(clazz::isInstance).map(clazz::cast).findFirst().orElse(null); } /** diff --git a/server/src/main/java/com/linecorp/centraldogma/server/ZoneConfig.java b/server/src/main/java/com/linecorp/centraldogma/server/ZoneConfig.java new file mode 100644 index 0000000000..96bde8aca9 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/ZoneConfig.java @@ -0,0 +1,94 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + */ + +package com.linecorp.centraldogma.server; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.linecorp.centraldogma.server.CentralDogmaConfig.convertValue; +import static java.util.Objects.requireNonNull; + +import java.util.List; +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; + +/** + * A configuration class for the zone. + */ +public final class ZoneConfig { + + private final String currentZone; + private final List allZones; + + /** + * Creates a new instance. + */ + @JsonCreator + public ZoneConfig(@JsonProperty("currentZone") String currentZone, + @JsonProperty("allZones") List allZones) { + requireNonNull(currentZone, "currentZone"); + requireNonNull(allZones, "allZones"); + this.currentZone = convertValue(currentZone, "zone.currentZone"); + this.allZones = allZones; + checkArgument(allZones.contains(currentZone), "The current zone: %s, (expected: one of %s)", + currentZone, allZones); + } + + /** + * Returns the current zone. + */ + @JsonProperty("currentZone") + public String currentZone() { + return currentZone; + } + + /** + * Returns all zones. + */ + @JsonProperty("allZones") + public List allZones() { + return allZones; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ZoneConfig)) { + return false; + } + final ZoneConfig that = (ZoneConfig) o; + return currentZone.equals(that.currentZone) && + allZones.equals(that.allZones); + } + + @Override + public int hashCode() { + return Objects.hash(currentZone, allZones); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("currentZone", currentZone) + .add("allZones", allZones) + .toString(); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/auth/AuthConfig.java b/server/src/main/java/com/linecorp/centraldogma/server/auth/AuthConfig.java index 5f36326f75..6748cad774 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/auth/AuthConfig.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/auth/AuthConfig.java @@ -60,7 +60,7 @@ public final class AuthConfig { private final AuthProviderFactory factory; - private final Set administrators; + private final Set systemAdministrators; private final boolean caseSensitiveLoginNames; private final String sessionCacheSpec; @@ -74,7 +74,7 @@ public final class AuthConfig { * Creates a new instance. * * @param factoryClassName the fully-qualified class name of the {@link AuthProviderFactory} - * @param administrators the login names of the administrators + * @param systemAdministrators the login names of the system administrators * @param caseSensitiveLoginNames the flag whether case-sensitive matching is performed when login names * are compared * @param sessionCacheSpec the cache specification which determines the capacity and behavior of @@ -86,7 +86,7 @@ public final class AuthConfig { @JsonCreator public AuthConfig( @JsonProperty("factoryClassName") String factoryClassName, - @JsonProperty("administrators") @Nullable Set administrators, + @JsonProperty("systemAdministrators") @Nullable Set systemAdministrators, @JsonProperty("caseSensitiveLoginNames") @Nullable Boolean caseSensitiveLoginNames, @JsonProperty("sessionCacheSpec") @Nullable String sessionCacheSpec, @JsonProperty("sessionTimeoutMillis") @Nullable Long sessionTimeoutMillis, @@ -96,7 +96,7 @@ public AuthConfig( .getClassLoader() .loadClass(requireNonNull(factoryClassName, "factoryClassName")) .getDeclaredConstructor().newInstance(), - administrators != null ? ImmutableSet.copyOf(administrators) : ImmutableSet.of(), + systemAdministrators != null ? ImmutableSet.copyOf(systemAdministrators) : ImmutableSet.of(), firstNonNull(caseSensitiveLoginNames, false), firstNonNull(sessionCacheSpec, DEFAULT_SESSION_CACHE_SPEC), firstNonNull(sessionTimeoutMillis, DEFAULT_SESSION_TIMEOUT_MILLIS), @@ -108,7 +108,7 @@ public AuthConfig( * Creates a new instance. * * @param factory the {@link AuthProviderFactory} instance - * @param administrators the login names of the administrators + * @param systemAdministrators the login names of the system administrators * @param caseSensitiveLoginNames the flag whether case-sensitive matching is performed when login names * are compared * @param sessionCacheSpec the cache specification which determines the capacity and behavior of @@ -118,14 +118,14 @@ public AuthConfig( * @param properties the additional properties which are used in the factory */ public AuthConfig(AuthProviderFactory factory, - Set administrators, + Set systemAdministrators, boolean caseSensitiveLoginNames, String sessionCacheSpec, long sessionTimeoutMillis, String sessionValidationSchedule, @Nullable JsonNode properties) { this.factory = requireNonNull(factory, "factory"); - this.administrators = requireNonNull(administrators, "administrators"); + this.systemAdministrators = requireNonNull(systemAdministrators, "systemAdministrators"); this.caseSensitiveLoginNames = caseSensitiveLoginNames; this.sessionCacheSpec = validateCacheSpec(requireNonNull(sessionCacheSpec, "sessionCacheSpec")); checkArgument(sessionTimeoutMillis > 0, @@ -152,11 +152,11 @@ public String factoryClassName() { } /** - * Returns the usernames of the users with administrator rights. + * Returns the usernames of the users with system administrator rights. */ @JsonProperty - public Set administrators() { - return administrators; + public Set systemAdministrators() { + return systemAdministrators; } /** diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommandExecutor.java b/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommandExecutor.java index 66356e8d0f..1ac55ae08c 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommandExecutor.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommandExecutor.java @@ -126,7 +126,7 @@ public final CompletableFuture execute(Command command) { if (!isStarted()) { throw new ReadOnlyException("running in read-only mode. command: " + command); } - if (!writable && !(command instanceof AdministrativeCommand)) { + if (!writable && !(command instanceof SystemAdministrativeCommand)) { // Reject all commands except for AdministrativeCommand when the replica is in read-only mode. // AdministrativeCommand is allowed because it is used to change the read-only mode or migrate // metadata under maintenance mode. diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractPushCommand.java b/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractPushCommand.java index 2905513fc7..b7f83c3a13 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractPushCommand.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractPushCommand.java @@ -106,7 +106,7 @@ public boolean equals(Object obj) { return false; } - final AbstractPushCommand that = (AbstractPushCommand) obj; + final AbstractPushCommand that = (AbstractPushCommand) obj; return super.equals(that) && baseRevision.equals(that.baseRevision) && summary.equals(that.summary) && diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java b/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java index 11b5fa583e..1732648e74 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java @@ -341,6 +341,20 @@ static Command push(@Nullable Long timestamp, Author author, summary, detail, markup, changes); } + /** + * Returns a new {@link Command} that transforms the content at the base revision with + * the specified {@link ContentTransformer} and pushed the result of transformation. + * You can find the result of transformation from {@link CommitResult#changes()}. + */ + static Command transform(@Nullable Long timestamp, Author author, + String projectName, String repositoryName, + Revision baseRevision, String summary, + String detail, Markup markup, + ContentTransformer transformer) { + return TransformCommand.of(timestamp, author, projectName, repositoryName, + baseRevision, summary, detail, markup, transformer); + } + /** * Returns a new {@link Command} which is used to create a new session. * diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java b/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java index a688455339..f9a599cf9a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java @@ -29,6 +29,7 @@ public enum CommandType { REMOVE_REPOSITORY(Void.class), UNREMOVE_REPOSITORY(Void.class), NORMALIZING_PUSH(CommitResult.class), + TRANSFORM(CommitResult.class), PUSH(Revision.class), SAVE_NAMED_QUERY(Void.class), REMOVE_NAMED_QUERY(Void.class), diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/ContentTransformer.java b/server/src/main/java/com/linecorp/centraldogma/server/command/ContentTransformer.java new file mode 100644 index 0000000000..3e8d75d091 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/ContentTransformer.java @@ -0,0 +1,77 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.command; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +import java.util.function.BiFunction; +import java.util.function.Function; + +import com.google.common.base.MoreObjects; + +import com.linecorp.centraldogma.common.EntryType; +import com.linecorp.centraldogma.common.Revision; + +/** + * A {@link Function} which is used for transforming the content at the specified path of the repository. + */ +public class ContentTransformer { + + private final String path; + private final EntryType entryType; + private final BiFunction transformer; + + /** + * Creates a new instance. + */ + public ContentTransformer(String path, EntryType entryType, BiFunction transformer) { + this.path = requireNonNull(path, "path"); + checkArgument(entryType == EntryType.JSON, "entryType: %s (expected: %s)", entryType, EntryType.JSON); + this.entryType = requireNonNull(entryType, "entryType"); + this.transformer = requireNonNull(transformer, "transformer"); + } + + /** + * Returns the path of the content to be transformed. + */ + public String path() { + return path; + } + + /** + * Returns the {@link EntryType} of the content to be transformed. + */ + public EntryType entryType() { + return entryType; + } + + /** + * Returns the {@link Function} which transforms the content. + */ + public BiFunction transformer() { + return transformer; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("path", path) + .add("entryType", entryType) + .add("transformer", transformer) + .toString(); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/ForcePushCommand.java b/server/src/main/java/com/linecorp/centraldogma/server/command/ForcePushCommand.java index 64936b51cc..cae8f4c659 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/ForcePushCommand.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/ForcePushCommand.java @@ -26,7 +26,7 @@ * A {@link Command} which is used to force-push {@code delegate} even the server is in read-only mode. * This command is useful for migrating the repository content during maintenance mode. */ -public final class ForcePushCommand extends AdministrativeCommand { +public final class ForcePushCommand extends SystemAdministrativeCommand { private final Command delegate; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/NormalizableCommit.java b/server/src/main/java/com/linecorp/centraldogma/server/command/NormalizableCommit.java new file mode 100644 index 0000000000..ad29a95108 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/NormalizableCommit.java @@ -0,0 +1,29 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.command; + +/** + * A {@link Command} that can be transformed to a {@link PushAsIsCommand} via {@link #asIs(CommitResult)}. + */ +@FunctionalInterface +public interface NormalizableCommit { + + /** + * Returns a new {@link PushAsIsCommand} which is converted using {@link CommitResult} + * for replicating to other replicas. + */ + PushAsIsCommand asIs(CommitResult commitResult); +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/NormalizingPushCommand.java b/server/src/main/java/com/linecorp/centraldogma/server/command/NormalizingPushCommand.java index 33756196bc..c0401e0e84 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/NormalizingPushCommand.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/NormalizingPushCommand.java @@ -34,7 +34,8 @@ * You can find the normalized changes from the {@link CommitResult#changes()} that is the result of * {@link CommandExecutor#execute(Command)}. */ -public final class NormalizingPushCommand extends AbstractPushCommand { +public final class NormalizingPushCommand extends AbstractPushCommand + implements NormalizableCommit { @JsonCreator NormalizingPushCommand(@JsonProperty("timestamp") @Nullable Long timestamp, @@ -50,11 +51,7 @@ public final class NormalizingPushCommand extends AbstractPushCommand CompletableFuture doExecute(Command command) throws Exceptio .thenApply(CommitResult::revision); } + if (command instanceof TransformCommand) { + return (CompletableFuture) push((TransformCommand) command, true); + } + if (command instanceof CreateSessionCommand) { return (CompletableFuture) createSession((CreateSessionCommand) command); } @@ -288,7 +292,7 @@ private CompletableFuture purgeRepository(PurgeRepositoryCommand c) { }, repositoryWorker); } - private CompletableFuture push(AbstractPushCommand c, boolean normalizing) { + private CompletableFuture push(RepositoryCommand c, boolean normalizing) { if (c.projectName().equals(INTERNAL_PROJECT_DOGMA) || c.repositoryName().equals(Project.REPO_DOGMA) || !writeQuotaEnabled()) { return push0(c, normalizing); @@ -305,7 +309,7 @@ private CompletableFuture push(AbstractPushCommand c, boolean n } private CompletableFuture tryPush( - AbstractPushCommand c, boolean normalizing, @Nullable RateLimiter rateLimiter) { + RepositoryCommand c, boolean normalizing, @Nullable RateLimiter rateLimiter) { if (rateLimiter == null || rateLimiter == UNLIMITED || rateLimiter.tryAcquire()) { return push0(c, normalizing); } else { @@ -314,9 +318,19 @@ private CompletableFuture tryPush( } } - private CompletableFuture push0(AbstractPushCommand c, boolean normalizing) { - return repo(c).commit(c.baseRevision(), c.timestamp(), c.author(), c.summary(), c.detail(), c.markup(), - c.changes(), normalizing); + private CompletableFuture push0(RepositoryCommand c, boolean normalizing) { + if (c instanceof TransformCommand) { + final TransformCommand transformCommand = (TransformCommand) c; + return repo(c).commit(transformCommand.baseRevision(), transformCommand.timestamp(), + transformCommand.author(), transformCommand.summary(), + transformCommand.detail(), transformCommand.markup(), + transformCommand.transformer()); + } + assert c instanceof AbstractPushCommand; + final AbstractPushCommand pushCommand = (AbstractPushCommand) c; + return repo(c).commit(pushCommand.baseRevision(), pushCommand.timestamp(), pushCommand.author(), + pushCommand.summary(), pushCommand.detail(), pushCommand.markup(), + pushCommand.changes(), normalizing); } private CompletableFuture getRateLimiter(String projectName, String repoName) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/AdministrativeCommand.java b/server/src/main/java/com/linecorp/centraldogma/server/command/SystemAdministrativeCommand.java similarity index 79% rename from server/src/main/java/com/linecorp/centraldogma/server/command/AdministrativeCommand.java rename to server/src/main/java/com/linecorp/centraldogma/server/command/SystemAdministrativeCommand.java index 49704002bc..d668924eb2 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/AdministrativeCommand.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/SystemAdministrativeCommand.java @@ -20,9 +20,9 @@ import com.linecorp.centraldogma.common.Author; -abstract class AdministrativeCommand extends RootCommand { - AdministrativeCommand(CommandType commandType, @Nullable Long timestamp, - @Nullable Author author) { +abstract class SystemAdministrativeCommand extends RootCommand { + SystemAdministrativeCommand(CommandType commandType, @Nullable Long timestamp, + @Nullable Author author) { super(commandType, timestamp, author); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/TransformCommand.java b/server/src/main/java/com/linecorp/centraldogma/server/command/TransformCommand.java new file mode 100644 index 0000000000..868b7d81c0 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/TransformCommand.java @@ -0,0 +1,112 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.command; + +import static java.util.Objects.requireNonNull; + +import javax.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.Revision; + +/** + * A {@link Command} that transforms the content at the base revision with + * the specified {@link ContentTransformer} and pushed the result of transformation. + * You can find the result of transformation from {@link CommitResult#changes()}. + * Note that this command is not serialized and deserialized. + */ +public final class TransformCommand extends RepositoryCommand implements NormalizableCommit { + + /** + * Creates a new instance. + */ + public static TransformCommand of(@Nullable Long timestamp, + @Nullable Author author, String projectName, + String repositoryName, Revision baseRevision, + String summary, String detail, Markup markup, + ContentTransformer transformer) { + return new TransformCommand(timestamp, author, projectName, repositoryName, + baseRevision, summary, detail, markup, transformer); + } + + private final Revision baseRevision; + private final String summary; + private final String detail; + private final Markup markup; + private final ContentTransformer transformer; + + private TransformCommand(@Nullable Long timestamp, @Nullable Author author, + String projectName, String repositoryName, Revision baseRevision, + String summary, String detail, Markup markup, + ContentTransformer transformer) { + super(CommandType.TRANSFORM, timestamp, author, projectName, repositoryName); + this.baseRevision = baseRevision; + this.summary = summary; + this.detail = detail; + this.markup = markup; + this.transformer = transformer; + } + + /** + * Returns the base {@link Revision}. + */ + @JsonProperty + public Revision baseRevision() { + return baseRevision; + } + + /** + * Returns the human-readable summary of the commit. + */ + @JsonProperty + public String summary() { + return summary; + } + + /** + * Returns the human-readable detail of the commit. + */ + @JsonProperty + public String detail() { + return detail; + } + + /** + * Returns the {@link Markup} of the {@link #detail()}. + */ + @JsonProperty + public Markup markup() { + return markup; + } + + /** + * Returns the {@link ContentTransformer} which is used for transforming the content. + */ + public ContentTransformer transformer() { + return transformer; + } + + @Override + public PushAsIsCommand asIs(CommitResult commitResult) { + requireNonNull(commitResult, "commitResult"); + return new PushAsIsCommand(timestamp(), author(), projectName(), repositoryName(), + commitResult.revision().backward(1), summary(), detail(), + markup(), commitResult.changes()); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/UpdateServerStatusCommand.java b/server/src/main/java/com/linecorp/centraldogma/server/command/UpdateServerStatusCommand.java index 1184822b5b..ef160a07b1 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/UpdateServerStatusCommand.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/UpdateServerStatusCommand.java @@ -33,7 +33,7 @@ * A {@link Command} which is used to update the status of all servers in the cluster. */ @JsonInclude(Include.NON_NULL) -public final class UpdateServerStatusCommand extends AdministrativeCommand { +public final class UpdateServerStatusCommand extends SystemAdministrativeCommand { private final ServerStatus serverStatus; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/CsrfTokenAuthorizer.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/CsrfTokenAuthorizer.java index fac3340046..ea582bcd32 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/CsrfTokenAuthorizer.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/CsrfTokenAuthorizer.java @@ -39,8 +39,8 @@ public class CsrfTokenAuthorizer implements Authorizer { public CompletionStage authorize(ServiceRequestContext ctx, HttpRequest data) { final OAuth2Token token = AuthTokenExtractors.oAuth2().apply(data.headers()); if (token != null && CsrfToken.ANONYMOUS.equals(token.accessToken())) { - AuthUtil.setCurrentUser(ctx, User.ADMIN); - HttpApiUtil.setVerboseResponses(ctx, User.ADMIN); + AuthUtil.setCurrentUser(ctx, User.SYSTEM_ADMIN); + HttpApiUtil.setVerboseResponses(ctx, User.SYSTEM_ADMIN); return CompletableFuture.completedFuture(true); } else { return CompletableFuture.completedFuture(false); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/SessionTokenAuthorizer.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/SessionTokenAuthorizer.java index e03094da85..937266c166 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/SessionTokenAuthorizer.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/auth/SessionTokenAuthorizer.java @@ -16,7 +16,7 @@ package com.linecorp.centraldogma.server.internal.admin.auth; -import static com.linecorp.centraldogma.server.metadata.User.LEVEL_ADMIN; +import static com.linecorp.centraldogma.server.metadata.User.LEVEL_SYSTEM_ADMIN; import static com.linecorp.centraldogma.server.metadata.User.LEVEL_USER; import static java.util.Objects.requireNonNull; import static java.util.concurrent.CompletableFuture.completedFuture; @@ -41,11 +41,11 @@ public class SessionTokenAuthorizer implements Authorizer { private final SessionManager sessionManager; - private final Set administrators; + private final Set systemAdministrators; - public SessionTokenAuthorizer(SessionManager sessionManager, Set administrators) { + public SessionTokenAuthorizer(SessionManager sessionManager, Set systemAdministrators) { this.sessionManager = requireNonNull(sessionManager, "sessionManager"); - this.administrators = requireNonNull(administrators, "administrators"); + this.systemAdministrators = requireNonNull(systemAdministrators, "systemAdministrators"); } @Override @@ -60,8 +60,9 @@ public CompletionStage authorize(ServiceRequestContext ctx, HttpRequest return false; } final String username = session.username(); - final List roles = administrators.contains(username) ? LEVEL_ADMIN - : LEVEL_USER; + final List roles = + systemAdministrators.contains(username) ? LEVEL_SYSTEM_ADMIN + : LEVEL_USER; final User user = new User(username, roles); ctx.logBuilder().authenticatedUser("user/" + username); AuthUtil.setCurrentUser(ctx, user); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java index e6363cf51b..27265456ac 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java @@ -198,7 +198,7 @@ public CompletableFuture push( CommitMessageDto commitMessage, @RequestConverter(ChangesRequestConverter.class) Iterable> changes) { final User user = AuthUtil.currentUser(ctx); - checkPush(repository.name(), changes, user.isAdmin()); + checkPush(repository.name(), changes, user.isSystemAdmin()); meterRegistry.counter("commits.push", "project", repository.parent().name(), "repository", repository.name()) @@ -444,7 +444,7 @@ public CompletableFuture> mergeFiles( * Checks if the commit is for creating a file and raises a {@link InvalidPushException} if the * given {@code repoName} field is one of {@code meta} and {@code dogma} which are internal repositories. */ - public static void checkPush(String repoName, Iterable> changes, boolean isAdmin) { + public static void checkPush(String repoName, Iterable> changes, boolean isSystemAdmin) { if (Project.REPO_META.equals(repoName)) { final boolean hasChangesOtherThanMetaRepoFiles = Streams.stream(changes).anyMatch(change -> !isMetaFile(change.path())); @@ -453,8 +453,8 @@ public static void checkPush(String repoName, Iterable> changes, boole "The " + Project.REPO_META + " repository is reserved for internal usage."); } - if (isAdmin) { - // Admin may push the legacy files to test the mirror migration. + if (isSystemAdmin) { + // A system admin may push the legacy files to test the mirror migration. } else { for (Change change : changes) { // 'mirrors.json' and 'credentials.json' are disallowed to be created or modified. diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/CredentialServiceV1.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/CredentialServiceV1.java index 032925dec5..6c0080358a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/CredentialServiceV1.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/CredentialServiceV1.java @@ -70,7 +70,7 @@ public CredentialServiceV1(ProjectApiManager projectApiManager, CommandExecutor public CompletableFuture> listCredentials(User loginUser, @Param String projectName) { final CompletableFuture> future = metaRepo(projectName, loginUser).credentials(); - if (loginUser.isAdmin()) { + if (loginUser.isSystemAdmin()) { return future; } return future.thenApply(credentials -> { @@ -91,7 +91,7 @@ public CompletableFuture> listCredentials(User loginUser, public CompletableFuture getCredentialById(User loginUser, @Param String projectName, @Param String id) { final CompletableFuture future = metaRepo(projectName, loginUser).credential(id); - if (loginUser.isAdmin()) { + if (loginUser.isSystemAdmin()) { return future; } return future.thenApply(Credential::withoutSecret); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiExceptionHandler.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiExceptionHandler.java index 4a8acab3da..e0d12559bb 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiExceptionHandler.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiExceptionHandler.java @@ -48,9 +48,10 @@ import com.linecorp.centraldogma.common.RepositoryNotFoundException; import com.linecorp.centraldogma.common.RevisionNotFoundException; import com.linecorp.centraldogma.common.TooManyRequestsException; -import com.linecorp.centraldogma.server.internal.admin.service.TokenNotFoundException; import com.linecorp.centraldogma.server.internal.storage.RequestAlreadyTimedOutException; import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryMetadataException; +import com.linecorp.centraldogma.server.metadata.MemberNotFoundException; +import com.linecorp.centraldogma.server.metadata.TokenNotFoundException; /** * A default {@link ExceptionHandlerFunction} of HTTP API. @@ -96,8 +97,9 @@ public final class HttpApiExceptionHandler implements ServerErrorHandler { (ctx, cause) -> newResponse(ctx, HttpStatus.NOT_FOUND, cause, "Revision %s does not exist.", cause.getMessage())) .put(TokenNotFoundException.class, - (ctx, cause) -> newResponse(ctx, HttpStatus.NOT_FOUND, cause, - "Token '%s' does not exist.", cause.getMessage())) + (ctx, cause) -> newResponse(ctx, HttpStatus.NOT_FOUND, cause, cause.getMessage())) + .put(MemberNotFoundException.class, + (ctx, cause) -> newResponse(ctx, HttpStatus.NOT_FOUND, cause, cause.getMessage())) .put(QueryExecutionException.class, (ctx, cause) -> newResponse(ctx, HttpStatus.BAD_REQUEST, cause)) .put(UnsupportedOperationException.class, diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiUtil.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiUtil.java index a4939bb2d0..9fdd37ff9a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiUtil.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/HttpApiUtil.java @@ -309,7 +309,7 @@ public static void throwUnsafelyIfNonNull(@Nullable Throwable cause) { } public static void setVerboseResponses(ServiceRequestContext ctx, User user) { - ctx.setAttr(VERBOSE_RESPONSES, Flags.verboseResponses() || user.isAdmin()); + ctx.setAttr(VERBOSE_RESPONSES, Flags.verboseResponses() || user.isSystemAdmin()); } private static boolean isVerboseResponse(ServiceRequestContext ctx) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MetadataApiService.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MetadataApiService.java index 4ca811f553..74003c8d6b 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MetadataApiService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MetadataApiService.java @@ -46,8 +46,8 @@ import com.linecorp.centraldogma.internal.jsonpatch.ReplaceOperation; import com.linecorp.centraldogma.server.QuotaConfig; import com.linecorp.centraldogma.server.command.CommandExecutor; -import com.linecorp.centraldogma.server.internal.api.auth.RequiresAdministrator; import com.linecorp.centraldogma.server.internal.api.auth.RequiresProjectRole; +import com.linecorp.centraldogma.server.internal.api.auth.RequiresSystemAdministrator; import com.linecorp.centraldogma.server.metadata.MetadataService; import com.linecorp.centraldogma.server.metadata.ProjectRoles; import com.linecorp.centraldogma.server.metadata.Token; @@ -255,7 +255,7 @@ public CompletableFuture removeTokenRepositoryRole(@Param String proje */ @Patch("/metadata/{projectName}/repos/{repoName}/quota/write") @Consumes("application/json-patch+json") - @RequiresAdministrator + @RequiresSystemAdministrator public CompletableFuture updateWriteQuota(@Param String projectName, @Param String repoName, QuotaConfig quota, diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MirroringServiceV1.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MirroringServiceV1.java index c5a911a634..e8029de3e2 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MirroringServiceV1.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MirroringServiceV1.java @@ -18,14 +18,19 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.linecorp.centraldogma.server.internal.mirror.DefaultMirroringServicePlugin.mirrorConfig; import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.mirrorFile; import java.net.URI; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; + import com.cronutils.model.Cron; +import com.google.common.collect.ImmutableMap; import com.linecorp.armeria.server.annotation.ConsumesJson; import com.linecorp.armeria.server.annotation.Delete; @@ -43,6 +48,8 @@ import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.internal.api.v1.MirrorDto; import com.linecorp.centraldogma.internal.api.v1.PushResultDto; +import com.linecorp.centraldogma.server.CentralDogmaConfig; +import com.linecorp.centraldogma.server.ZoneConfig; import com.linecorp.centraldogma.server.command.Command; import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.command.CommitResult; @@ -52,6 +59,7 @@ import com.linecorp.centraldogma.server.metadata.User; import com.linecorp.centraldogma.server.mirror.Mirror; import com.linecorp.centraldogma.server.mirror.MirrorResult; +import com.linecorp.centraldogma.server.mirror.MirroringServicePluginConfig; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.repository.MetaRepository; @@ -67,12 +75,29 @@ public class MirroringServiceV1 extends AbstractService { private final ProjectApiManager projectApiManager; private final MirrorRunner mirrorRunner; + private final Map mirrorZoneConfig; + @Nullable + private final ZoneConfig zoneConfig; public MirroringServiceV1(ProjectApiManager projectApiManager, CommandExecutor executor, - MirrorRunner mirrorRunner) { + MirrorRunner mirrorRunner, CentralDogmaConfig config) { super(executor); this.projectApiManager = projectApiManager; this.mirrorRunner = mirrorRunner; + zoneConfig = config.zone(); + mirrorZoneConfig = mirrorZoneConfig(config); + } + + private static Map mirrorZoneConfig(CentralDogmaConfig config) { + final MirroringServicePluginConfig mirrorConfig = mirrorConfig(config); + final ImmutableMap.Builder builder = ImmutableMap.builderWithExpectedSize(2); + final boolean zonePinned = mirrorConfig != null && mirrorConfig.zonePinned(); + builder.put("zonePinned", zonePinned); + final ZoneConfig zone = config.zone(); + if (zone != null) { + builder.put("zone", zone); + } + return builder.build(); } /** @@ -154,11 +179,12 @@ public CompletableFuture deleteMirror(@Param String projectName, private CompletableFuture createOrUpdate(String projectName, MirrorDto newMirror, Author author, boolean update) { - return metaRepo(projectName).createPushCommand(newMirror, author, update).thenCompose(command -> { - return executor().execute(command).thenApply(result -> { - return new PushResultDto(result.revision(), command.timestamp()); - }); - }); + return metaRepo(projectName) + .createPushCommand(newMirror, author, zoneConfig, update).thenCompose(command -> { + return executor().execute(command).thenApply(result -> { + return new PushResultDto(result.revision(), command.timestamp()); + }); + }); } /** @@ -175,6 +201,17 @@ public CompletableFuture runMirror(@Param String projectName, @Par return mirrorRunner.run(projectName, mirrorId, user); } + /** + * GET /mirror/config + * + *

Returns the configuration of the mirroring service. + */ + @Get("/mirror/config") + public Map config() { + // TODO(ikhoon): Add more configurations if necessary. + return mirrorZoneConfig; + } + private static MirrorDto convertToMirrorDto(String projectName, Mirror mirror) { final URI remoteRepoUri = mirror.remoteRepoUri(); final Cron schedule = mirror.schedule(); @@ -186,11 +223,11 @@ private static MirrorDto convertToMirrorDto(String projectName, Mirror mirror) { mirror.localRepo().name(), mirror.localPath(), remoteRepoUri.getScheme(), - remoteRepoUri.getHost() + remoteRepoUri.getPath(), + remoteRepoUri.getAuthority() + remoteRepoUri.getPath(), mirror.remotePath(), mirror.remoteBranch(), mirror.gitignore(), - mirror.credential().id()); + mirror.credential().id(), mirror.zone()); } private MetaRepository metaRepo(String projectName) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1.java index 65e56376c2..b512847a8b 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1.java @@ -45,8 +45,8 @@ import com.linecorp.centraldogma.internal.api.v1.CreateProjectRequest; import com.linecorp.centraldogma.internal.api.v1.ProjectDto; import com.linecorp.centraldogma.server.command.CommandExecutor; -import com.linecorp.centraldogma.server.internal.api.auth.RequiresAdministrator; import com.linecorp.centraldogma.server.internal.api.auth.RequiresProjectRole; +import com.linecorp.centraldogma.server.internal.api.auth.RequiresSystemAdministrator; import com.linecorp.centraldogma.server.internal.api.converter.CreateApiResponseConverter; import com.linecorp.centraldogma.server.internal.storage.project.ProjectApiManager; import com.linecorp.centraldogma.server.metadata.Member; @@ -94,13 +94,13 @@ public CompletableFuture> listProjects(@Param @Nullable String } private static ProjectRole getUserRole(Project project, User user) { - if (user.isAdmin()) { + if (user.isSystemAdmin()) { return ProjectRole.OWNER; } final ProjectMetadata metadata = project.metadata(); if (metadata == null) { - // Metadata is null for the internal project which belongs to administrators. + // Metadata is null for the internal project which belongs to system administrators. return ProjectRole.GUEST; } @@ -180,7 +180,7 @@ public CompletableFuture purgeProject(@Param String projectName, Author au */ @Consumes("application/json-patch+json") @Patch("/projects/{projectName}") - @RequiresAdministrator + @RequiresSystemAdministrator public CompletableFuture patchProject(@Param String projectName, JsonNode node, Author author, User user) { checkUnremoveArgument(node); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1.java index aabd43a4cd..b98cabe82b 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1.java @@ -101,7 +101,7 @@ public CompletableFuture> listRepositories(ServiceRequestCon } return project.repos().list().values().stream() - .filter(r -> user.isAdmin() || !Project.REPO_DOGMA.equals(r.name())) + .filter(r -> user.isSystemAdmin() || !Project.REPO_DOGMA.equals(r.name())) .filter(r -> hasOwnerRole || !Project.REPO_META.equals(r.name())) .map(DtoConverter::convert) .collect(toImmutableList()); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/AdministrativeService.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/SystemAdministrativeService.java similarity index 93% rename from server/src/main/java/com/linecorp/centraldogma/server/internal/api/AdministrativeService.java rename to server/src/main/java/com/linecorp/centraldogma/server/internal/api/SystemAdministrativeService.java index 7e8e36acf0..6a261b05e5 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/AdministrativeService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/SystemAdministrativeService.java @@ -27,16 +27,16 @@ import com.linecorp.centraldogma.server.command.Command; import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.internal.api.UpdateServerStatusRequest.Scope; -import com.linecorp.centraldogma.server.internal.api.auth.RequiresAdministrator; +import com.linecorp.centraldogma.server.internal.api.auth.RequiresSystemAdministrator; import com.linecorp.centraldogma.server.management.ServerStatus; import com.linecorp.centraldogma.server.management.ServerStatusManager; @ProducesJson -public final class AdministrativeService extends AbstractService { +public final class SystemAdministrativeService extends AbstractService { private final ServerStatusManager serverStatusManager; - public AdministrativeService(CommandExecutor executor, ServerStatusManager serverStatusManager) { + public SystemAdministrativeService(CommandExecutor executor, ServerStatusManager serverStatusManager) { super(executor); this.serverStatusManager = serverStatusManager; } @@ -62,7 +62,7 @@ public ServerStatus status() { */ @Put("/status") @Consumes("application/json") - @RequiresAdministrator + @RequiresSystemAdministrator public CompletableFuture updateStatus(UpdateServerStatusRequest statusRequest) throws Exception { // TODO(trustin): Consider extracting this into common utility or Armeria. diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/TokenLevelRequest.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/TokenLevelRequest.java index 2f2a111c75..cead450bd8 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/TokenLevelRequest.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/TokenLevelRequest.java @@ -23,7 +23,7 @@ public final class TokenLevelRequest { private final String level; @JsonCreator - TokenLevelRequest(@JsonProperty("level") String level) { + public TokenLevelRequest(@JsonProperty("level") String level) { this.level = level; } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/TokenService.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/TokenService.java index 7abf672e81..6ee120298f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/TokenService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/TokenService.java @@ -48,7 +48,7 @@ import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.internal.Jackson; import com.linecorp.centraldogma.server.command.CommandExecutor; -import com.linecorp.centraldogma.server.internal.api.auth.RequiresAdministrator; +import com.linecorp.centraldogma.server.internal.api.auth.RequiresSystemAdministrator; import com.linecorp.centraldogma.server.internal.api.converter.CreateApiResponseConverter; import com.linecorp.centraldogma.server.metadata.MetadataService; import com.linecorp.centraldogma.server.metadata.Token; @@ -86,7 +86,7 @@ public TokenService(CommandExecutor executor, MetadataService mds) { */ @Get("/tokens") public CompletableFuture> listTokens(User loginUser) { - if (loginUser.isAdmin()) { + if (loginUser.isSystemAdmin()) { return mds.getTokens() .thenApply(tokens -> tokens.appIds().values()); } else { @@ -105,20 +105,24 @@ public CompletableFuture> listTokens(User loginUser) { @StatusCode(201) @ResponseConverter(CreateApiResponseConverter.class) public CompletableFuture> createToken(@Param String appId, + // TODO(minwoox): Remove isAdmin field. @Param @Default("false") boolean isAdmin, + @Param @Default("false") boolean isSystemAdmin, @Param @Nullable String secret, Author author, User loginUser) { - checkArgument(!isAdmin || loginUser.isAdmin(), - "Only administrators are allowed to create an admin-level token."); + final boolean isSystemAdminToken = isSystemAdmin || isAdmin; + checkArgument(!isSystemAdminToken || loginUser.isSystemAdmin(), + "Only system administrators are allowed to create a system admin-level token."); - checkArgument(secret == null || loginUser.isAdmin(), - "Only administrators are allowed to create a new token from the given secret string"); + checkArgument(secret == null || loginUser.isSystemAdmin(), + "Only system administrators are allowed to create a new token from " + + " the given secret string"); final CompletableFuture tokenFuture; if (secret != null) { - tokenFuture = mds.createToken(author, appId, secret, isAdmin); + tokenFuture = mds.createToken(author, appId, secret, isSystemAdminToken); } else { - tokenFuture = mds.createToken(author, appId, isAdmin); + tokenFuture = mds.createToken(author, appId, isSystemAdminToken); } return tokenFuture .thenCompose(unused -> mds.findTokenByAppId(appId)) @@ -150,7 +154,7 @@ public CompletableFuture deleteToken(ServiceRequestContext ctx, *

Purges a token of the specified ID that was deleted before. */ @Delete("/tokens/{appId}/removed") - @RequiresAdministrator + @RequiresSystemAdministrator public CompletableFuture purgeToken(ServiceRequestContext ctx, @Param String appId, Author author, User loginUser) { @@ -198,34 +202,36 @@ public CompletableFuture updateToken(ServiceRequestContext ctx, *

Updates a level of a token of the specified ID. */ @Patch("/tokens/{appId}/level") - @RequiresAdministrator + @RequiresSystemAdministrator public CompletableFuture updateTokenLevel(ServiceRequestContext ctx, @Param String appId, TokenLevelRequest tokenLevelRequest, Author author, User loginUser) { final String newTokenLevel = tokenLevelRequest.level().toLowerCase(); - checkArgument("user".equals(newTokenLevel) || "admin".equals(newTokenLevel), - "token level: %s (expected: user or admin)" + tokenLevelRequest.level()); + checkArgument("user".equals(newTokenLevel) || "admin".equals(newTokenLevel) || + "systemadmin".equals(newTokenLevel), + "token level: %s (expected: user or systemadmin)" + tokenLevelRequest.level()); return getTokenOrRespondForbidden(ctx, appId, loginUser).thenCompose( token -> { - boolean toBeAdmin = false; + boolean toBeSystemAdmin = false; switch (newTokenLevel) { case "user": - if (!token.isAdmin()) { + if (!token.isSystemAdmin()) { throw HttpStatusException.of(HttpStatus.NOT_MODIFIED); } break; case "admin": - if (token.isAdmin()) { + case "systemadmin": + if (token.isSystemAdmin()) { throw HttpStatusException.of(HttpStatus.NOT_MODIFIED); } - toBeAdmin = true; + toBeSystemAdmin = true; break; } - return mds.updateTokenLevel(author, appId, toBeAdmin).thenCompose( + return mds.updateTokenLevel(author, appId, toBeSystemAdmin).thenCompose( unused -> mds.findTokenByAppId(appId).thenApply(Token::withoutSecret)); }); } @@ -233,8 +239,8 @@ public CompletableFuture updateTokenLevel(ServiceRequestContext ctx, private CompletableFuture getTokenOrRespondForbidden(ServiceRequestContext ctx, String appId, User loginUser) { return mds.findTokenByAppId(appId).thenApply(token -> { - // Give permission to the administrators. - if (!loginUser.isAdmin() && + // Give permission to the system administrators. + if (!loginUser.isSystemAdmin() && !token.creation().user().equals(loginUser.id())) { return HttpApiUtil.throwResponse(ctx, HttpStatus.FORBIDDEN, "Unauthorized token: %s", token); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/ApplicationTokenAuthorizer.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/ApplicationTokenAuthorizer.java index d310ee0a06..c15ba7f677 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/ApplicationTokenAuthorizer.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/ApplicationTokenAuthorizer.java @@ -37,9 +37,9 @@ import com.linecorp.armeria.server.auth.AuthTokenExtractors; import com.linecorp.armeria.server.auth.Authorizer; import com.linecorp.centraldogma.server.internal.admin.auth.AuthUtil; -import com.linecorp.centraldogma.server.internal.admin.service.TokenNotFoundException; import com.linecorp.centraldogma.server.internal.api.HttpApiUtil; import com.linecorp.centraldogma.server.metadata.Token; +import com.linecorp.centraldogma.server.metadata.TokenNotFoundException; import com.linecorp.centraldogma.server.metadata.Tokens; import com.linecorp.centraldogma.server.metadata.UserWithToken; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresProjectRoleDecorator.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresProjectRoleDecorator.java index 8287301828..37fbbf6797 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresProjectRoleDecorator.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresProjectRoleDecorator.java @@ -58,7 +58,7 @@ public HttpResponse serve(ServiceRequestContext ctx, HttpRequest req) throws Exc final String projectName = ctx.pathParam("projectName"); checkArgument(!isNullOrEmpty(projectName), "no project name is specified"); - if (user.isAdmin()) { + if (user.isSystemAdmin()) { return unwrap().serve(ctx, req); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresRepositoryRoleDecorator.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresRepositoryRoleDecorator.java index 92e973f6b6..255f1ef262 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresRepositoryRoleDecorator.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresRepositoryRoleDecorator.java @@ -85,21 +85,21 @@ public HttpResponse serve(ServiceRequestContext ctx, HttpRequest req) throws Exc } checkArgument(!isNullOrEmpty(repoName), "no repository name is specified"); - if (user.isAdmin()) { + if (user.isSystemAdmin()) { return unwrap().serve(ctx, req); } if (Project.REPO_DOGMA.equals(repoName)) { - return throwForbiddenResponse(ctx, projectName, repoName, "administrator"); + return throwForbiddenResponse(ctx, projectName, repoName); } return serveUserRepo(ctx, req, user, projectName, maybeRemoveGitSuffix(repoName)); } private static HttpResponse throwForbiddenResponse(ServiceRequestContext ctx, String projectName, - String repoName, String adminOrOwner) { + String repoName) { return HttpApiUtil.throwResponse(ctx, HttpStatus.FORBIDDEN, - "Repository '%s/%s' can be accessed only by an %s.", - projectName, repoName, adminOrOwner); + "Repository '%s/%s' can be accessed only by a system administrator.", + projectName, repoName); } /** diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresAdministrator.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresSystemAdministrator.java similarity index 82% rename from server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresAdministrator.java rename to server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresSystemAdministrator.java index 8878e001bc..bd59ea0e42 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresAdministrator.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresSystemAdministrator.java @@ -22,15 +22,15 @@ import com.linecorp.armeria.server.annotation.Decorator; import com.linecorp.armeria.server.annotation.DecoratorFactory; -import com.linecorp.centraldogma.server.internal.api.auth.RequiresAdministratorDecorator.RequiresAdministratorDecoratorFactory; +import com.linecorp.centraldogma.server.internal.api.auth.RequiresSystemAdministratorDecorator.RequiresSystemAdministratorDecoratorFactory; /** - * A {@link Decorator} to allow a request from an administrator only. + * A {@link Decorator} to allow a request from a system administrator only. */ -@DecoratorFactory(RequiresAdministratorDecoratorFactory.class) +@DecoratorFactory(RequiresSystemAdministratorDecoratorFactory.class) @Retention(RetentionPolicy.RUNTIME) @Target({ ElementType.TYPE, ElementType.METHOD }) -public @interface RequiresAdministrator { +public @interface RequiresSystemAdministrator { /** * A special parameter in order to specify the order of a {@link Decorator}. */ diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresAdministratorDecorator.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresSystemAdministratorDecorator.java similarity index 73% rename from server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresAdministratorDecorator.java rename to server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresSystemAdministratorDecorator.java index dba1fc32b3..da25622dd2 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresAdministratorDecorator.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresSystemAdministratorDecorator.java @@ -31,34 +31,34 @@ import com.linecorp.centraldogma.server.metadata.User; /** - * A {@link Decorator} to allow a request from an administrator only. + * A {@link Decorator} to allow a request from a system administrator only. */ -public final class RequiresAdministratorDecorator extends SimpleDecoratingHttpService { +public final class RequiresSystemAdministratorDecorator extends SimpleDecoratingHttpService { - RequiresAdministratorDecorator(HttpService delegate) { + RequiresSystemAdministratorDecorator(HttpService delegate) { super(delegate); } @Override public HttpResponse serve(ServiceRequestContext ctx, HttpRequest req) throws Exception { final User user = AuthUtil.currentUser(ctx); - if (user.isAdmin()) { + if (user.isSystemAdmin()) { return unwrap().serve(ctx, req); } return HttpApiUtil.throwResponse( ctx, HttpStatus.FORBIDDEN, - "You must be an administrator to perform this operation."); + "You must be a system administrator to perform this operation."); } /** - * A {@link DecoratorFactoryFunction} which creates a {@link RequiresAdministratorDecorator}. + * A {@link DecoratorFactoryFunction} which creates a {@link RequiresSystemAdministratorDecorator}. */ - public static final class RequiresAdministratorDecoratorFactory - implements DecoratorFactoryFunction { + public static final class RequiresSystemAdministratorDecoratorFactory + implements DecoratorFactoryFunction { @Override public Function - newDecorator(RequiresAdministrator parameter) { - return RequiresAdministratorDecorator::new; + newDecorator(RequiresSystemAdministrator parameter) { + return RequiresSystemAdministratorDecorator::new; } } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/converter/HttpApiRequestConverter.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/converter/HttpApiRequestConverter.java index f8bd9579b3..beec74d8fe 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/converter/HttpApiRequestConverter.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/converter/HttpApiRequestConverter.java @@ -70,11 +70,10 @@ public Object convertRequest( checkArgument(!isNullOrEmpty(repositoryName), "repository name should not be null or empty."); - if (Project.REPO_DOGMA.equals(repositoryName) && - !user.isAdmin()) { + if (Project.REPO_DOGMA.equals(repositoryName) && !user.isSystemAdmin()) { return HttpApiUtil.throwResponse( ctx, HttpStatus.FORBIDDEN, - "Repository '%s/%s' can be accessed only by an administrator.", + "Repository '%s/%s' can be accessed only by a system administrator.", projectName, Project.REPO_DOGMA); } // RepositoryNotFoundException would be thrown if there is no project or no repository. diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractMirror.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractMirror.java index 8f5135c3c4..e46fef9d32 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractMirror.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractMirror.java @@ -64,6 +64,8 @@ public abstract class AbstractMirror implements Mirror { @Nullable private final String gitignore; @Nullable + private final String zone; + @Nullable private final Cron schedule; @Nullable private final ExecutionTime executionTime; @@ -72,7 +74,7 @@ public abstract class AbstractMirror implements Mirror { protected AbstractMirror(String id, boolean enabled, @Nullable Cron schedule, MirrorDirection direction, Credential credential, Repository localRepo, String localPath, URI remoteRepoUri, String remotePath, String remoteBranch, - @Nullable String gitignore) { + @Nullable String gitignore, @Nullable String zone) { this.id = requireNonNull(id, "id"); this.enabled = enabled; this.direction = requireNonNull(direction, "direction"); @@ -83,6 +85,7 @@ protected AbstractMirror(String id, boolean enabled, @Nullable Cron schedule, Mi this.remotePath = normalizePath(requireNonNull(remotePath, "remotePath")); this.remoteBranch = requireNonNull(remoteBranch, "remoteBranch"); this.gitignore = gitignore; + this.zone = zone; if (schedule != null) { this.schedule = requireNonNull(schedule, "schedule"); @@ -174,6 +177,12 @@ public final boolean enabled() { return enabled; } + @Nullable + @Override + public String zone() { + return zone; + } + @Override public final MirrorResult mirror(File workDir, CommandExecutor executor, int maxNumFiles, long maxNumBytes, Instant triggeredTime) { @@ -212,7 +221,7 @@ protected abstract MirrorResult mirrorRemoteToLocal( protected final MirrorResult newMirrorResult(MirrorStatus mirrorStatus, @Nullable String description, Instant triggeredTime) { return new MirrorResult(id, localRepo.parent().name(), localRepo.name(), mirrorStatus, description, - triggeredTime, Instant.now()); + triggeredTime, Instant.now(), zone); } @Override diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirror.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirror.java index 73392970f3..a66ab9837a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirror.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirror.java @@ -40,10 +40,10 @@ public final class CentralDogmaMirror extends AbstractMirror { public CentralDogmaMirror(String id, boolean enabled, Cron schedule, MirrorDirection direction, Credential credential, Repository localRepo, String localPath, URI remoteRepoUri, String remoteProject, String remoteRepo, String remotePath, - @Nullable String gitignore) { + @Nullable String gitignore, @Nullable String zone) { // Central Dogma has no notion of 'branch', so we just pass an empty string as a placeholder. super(id, enabled, schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath, - "", gitignore); + "", gitignore, zone); this.remoteProject = requireNonNull(remoteProject, "remoteProject"); this.remoteRepo = requireNonNull(remoteRepo, "remoteRepo"); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServicePlugin.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServicePlugin.java index fc71360940..b3fbbe7b64 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServicePlugin.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServicePlugin.java @@ -26,6 +26,7 @@ import com.google.common.base.MoreObjects; import com.linecorp.centraldogma.server.CentralDogmaConfig; +import com.linecorp.centraldogma.server.ZoneConfig; import com.linecorp.centraldogma.server.mirror.MirroringServicePluginConfig; import com.linecorp.centraldogma.server.plugin.Plugin; import com.linecorp.centraldogma.server.plugin.PluginContext; @@ -33,12 +34,31 @@ public final class DefaultMirroringServicePlugin implements Plugin { + @Nullable + public static MirroringServicePluginConfig mirrorConfig(CentralDogmaConfig config) { + return (MirroringServicePluginConfig) config.pluginConfigMap().get(MirroringServicePluginConfig.class); + } + @Nullable private volatile MirrorSchedulingService mirroringService; + @Nullable + private PluginTarget pluginTarget; + @Override - public PluginTarget target() { - return PluginTarget.LEADER_ONLY; + public PluginTarget target(CentralDogmaConfig config) { + requireNonNull(config, "config"); + if (pluginTarget != null) { + return pluginTarget; + } + + final MirroringServicePluginConfig mirrorConfig = mirrorConfig(config); + if (mirrorConfig != null && mirrorConfig.zonePinned()) { + pluginTarget = PluginTarget.ZONE_LEADER_ONLY; + } else { + pluginTarget = PluginTarget.LEADER_ONLY; + } + return pluginTarget; } @Override @@ -48,27 +68,34 @@ public synchronized CompletionStage start(PluginContext context) { MirrorSchedulingService mirroringService = this.mirroringService; if (mirroringService == null) { final CentralDogmaConfig cfg = context.config(); - final MirroringServicePluginConfig mirroringServicePluginConfig = - (MirroringServicePluginConfig) cfg.pluginConfigMap().get(configType()); + final MirroringServicePluginConfig mirroringServicePluginConfig = mirrorConfig(cfg); final int numThreads; final int maxNumFilesPerMirror; final long maxNumBytesPerMirror; + final ZoneConfig zoneConfig; if (mirroringServicePluginConfig != null) { numThreads = mirroringServicePluginConfig.numMirroringThreads(); maxNumFilesPerMirror = mirroringServicePluginConfig.maxNumFilesPerMirror(); maxNumBytesPerMirror = mirroringServicePluginConfig.maxNumBytesPerMirror(); + if (mirroringServicePluginConfig.zonePinned()) { + zoneConfig = cfg.zone(); + assert zoneConfig != null : "zonePinned is enabled but no zone configuration found"; + } else { + zoneConfig = null; + } } else { numThreads = MirroringServicePluginConfig.INSTANCE.numMirroringThreads(); maxNumFilesPerMirror = MirroringServicePluginConfig.INSTANCE.maxNumFilesPerMirror(); maxNumBytesPerMirror = MirroringServicePluginConfig.INSTANCE.maxNumBytesPerMirror(); + zoneConfig = null; } mirroringService = new MirrorSchedulingService(new File(cfg.dataDir(), "_mirrors"), context.projectManager(), context.meterRegistry(), numThreads, maxNumFilesPerMirror, - maxNumBytesPerMirror); + maxNumBytesPerMirror, zoneConfig); this.mirroringService = mirroringService; } mirroringService.start(context.commandExecutor()); @@ -97,8 +124,9 @@ public MirrorSchedulingService mirroringService() { @Override public String toString() { return MoreObjects.toStringHelper(this) - .add("configType", configType()) - .add("target", target()) + .omitNullValues() + .add("configType", configType().getName()) + .add("target", pluginTarget) .toString(); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorRunner.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorRunner.java index 16a197620f..8e0079ceeb 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorRunner.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorRunner.java @@ -29,6 +29,8 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; + import com.google.common.base.MoreObjects; import com.linecorp.armeria.common.util.SafeCloseable; @@ -56,6 +58,8 @@ public final class MirrorRunner implements SafeCloseable { private final ExecutorService worker; private final Map> inflightRequests = new ConcurrentHashMap<>(); + @Nullable + private final String currentZone; public MirrorRunner(ProjectApiManager projectApiManager, CommandExecutor commandExecutor, CentralDogmaConfig cfg, MeterRegistry meterRegistry) { @@ -70,6 +74,11 @@ public MirrorRunner(ProjectApiManager projectApiManager, CommandExecutor command mirrorConfig = MirroringServicePluginConfig.INSTANCE; } this.mirrorConfig = mirrorConfig; + if (cfg.zone() != null) { + currentZone = cfg.zone().currentZone(); + } else { + currentZone = null; + } final ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor( 0, mirrorConfig.numMirroringThreads(), @@ -91,10 +100,16 @@ private CompletableFuture run(MirrorKey mirrorKey, User user) { final CompletableFuture future = metaRepo(mirrorKey.projectName).mirror(mirrorKey.mirrorId).thenApplyAsync(mirror -> { if (!mirror.enabled()) { - throw new MirrorException("The mirror is disabled: " + mirrorKey); + throw new MirrorException("The mirror is disabled: " + + mirrorKey.projectName + '/' + mirrorKey.mirrorId); } - final MirrorTask mirrorTask = new MirrorTask(mirror, user, Instant.now(), true); + final String zone = mirror.zone(); + if (zone != null && !zone.equals(currentZone)) { + throw new MirrorException("The mirror is not in the current zone: " + currentZone); + } + final MirrorTask mirrorTask = new MirrorTask(mirror, user, Instant.now(), + currentZone, false); final MirrorListener listener = MirrorSchedulingService.mirrorListener; listener.onStart(mirrorTask); try { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingService.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingService.java index deff2d239a..4dc69f7432 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorSchedulingService.java @@ -52,6 +52,7 @@ import com.linecorp.centraldogma.common.MirrorException; import com.linecorp.centraldogma.server.MirroringService; +import com.linecorp.centraldogma.server.ZoneConfig; import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.metadata.User; import com.linecorp.centraldogma.server.mirror.Mirror; @@ -91,6 +92,10 @@ public final class MirrorSchedulingService implements MirroringService { private final int numThreads; private final int maxNumFilesPerMirror; private final long maxNumBytesPerMirror; + @Nullable + private final ZoneConfig zoneConfig; + @Nullable + private final String currentZone; private volatile CommandExecutor commandExecutor; private volatile ListeningScheduledExecutorService scheduler; @@ -101,7 +106,8 @@ public final class MirrorSchedulingService implements MirroringService { @VisibleForTesting public MirrorSchedulingService(File workDir, ProjectManager projectManager, MeterRegistry meterRegistry, - int numThreads, int maxNumFilesPerMirror, long maxNumBytesPerMirror) { + int numThreads, int maxNumFilesPerMirror, long maxNumBytesPerMirror, + @Nullable ZoneConfig zoneConfig) { this.workDir = requireNonNull(workDir, "workDir"); this.projectManager = requireNonNull(projectManager, "projectManager"); @@ -115,6 +121,12 @@ public MirrorSchedulingService(File workDir, ProjectManager projectManager, Mete this.numThreads = numThreads; this.maxNumFilesPerMirror = maxNumFilesPerMirror; this.maxNumBytesPerMirror = maxNumBytesPerMirror; + this.zoneConfig = zoneConfig; + if (zoneConfig != null) { + currentZone = zoneConfig.currentZone(); + } else { + currentZone = null; + } } public boolean isStarted() { @@ -152,7 +164,7 @@ public synchronized void start(CommandExecutor commandExecutor) { })); final ListenableScheduledFuture future = scheduler.scheduleWithFixedDelay( - this::schedulePendingMirrors, + this::scheduleMirrors, TICK.getSeconds(), TICK.getSeconds(), TimeUnit.SECONDS); Futures.addCallback(future, new FutureCallback() { @@ -181,7 +193,7 @@ public synchronized void stop() { } } - private void schedulePendingMirrors() { + private void scheduleMirrors() { final ZonedDateTime now = ZonedDateTime.now(); if (lastExecutionTime == null) { lastExecutionTime = now.minus(TICK); @@ -209,9 +221,31 @@ private void schedulePendingMirrors() { if (m.schedule() == null) { continue; } + if (zoneConfig != null) { + String pinnedZone = m.zone(); + if (pinnedZone == null) { + // Use the first zone if the mirror does not specify a zone. + pinnedZone = zoneConfig.allZones().get(0); + } + if (!pinnedZone.equals(currentZone)) { + // Skip the mirror if it is pinned to a different zone. + if (!zoneConfig.allZones().contains(pinnedZone)) { + // The mirror is pinned to an invalid zone. + final MirrorTask invalidMirror = + new MirrorTask(m, User.SYSTEM, Instant.now(), + pinnedZone, true); + mirrorListener.onStart(invalidMirror); + mirrorListener.onError(invalidMirror, new MirrorException( + "The mirror is pinned to an unknown zone: " + pinnedZone + + " (valid zones: " + zoneConfig.allZones() + ')')); + } + continue; + } + } try { if (m.nextExecutionTime(currentLastExecutionTime).compareTo(now) < 0) { - runAsync(new MirrorTask(m, User.SYSTEM, Instant.now(), true)); + runAsync(new MirrorTask(m, User.SYSTEM, Instant.now(), + currentZone, true)); } } catch (Exception e) { logger.warn("Unexpected exception while mirroring: {}", m, e); @@ -231,7 +265,7 @@ public CompletableFuture mirror() { () -> projectManager.list().values().forEach(p -> { try { p.metaRepo().mirrors().get(5, TimeUnit.SECONDS) - .forEach(m -> run(new MirrorTask(m, User.SYSTEM, Instant.now(), false))); + .forEach(m -> run(new MirrorTask(m, User.SYSTEM, Instant.now(), currentZone, false))); } catch (InterruptedException | TimeoutException | ExecutionException e) { throw new IllegalStateException( "Failed to load mirror list with in 5 seconds. project: " + p.name(), e); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java index d73c7a9bc2..b3e6742218 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java @@ -83,6 +83,7 @@ import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; +import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMultimap; import com.google.common.escape.Escaper; @@ -104,6 +105,7 @@ import com.linecorp.centraldogma.server.command.CommandType; import com.linecorp.centraldogma.server.command.CommitResult; import com.linecorp.centraldogma.server.command.ForcePushCommand; +import com.linecorp.centraldogma.server.command.NormalizableCommit; import com.linecorp.centraldogma.server.command.NormalizingPushCommand; import com.linecorp.centraldogma.server.command.RemoveRepositoryCommand; import com.linecorp.centraldogma.server.command.UpdateServerStatusCommand; @@ -1202,6 +1204,16 @@ List blocks() { public void appendBlock(long blockId) { blocks.add(blockId); } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("replicaId", replicaId) + .add("timestamp", timestamp) + .add("size", size) + .add("blocks", blocks) + .toString(); + } } private long storeLog(ReplicationLog log) { @@ -1317,19 +1329,14 @@ private T blockingExecute(Command command) throws Exception { final T result = delegate.execute(command).get(); final ReplicationLog log; - if (command.type() == CommandType.NORMALIZING_PUSH) { - final NormalizingPushCommand normalizingPushCommand = (NormalizingPushCommand) command; + final Command maybeUnwrapped = unwrapForcePush(command); + if (maybeUnwrapped instanceof NormalizableCommit) { + final NormalizableCommit normalizingPushCommand = (NormalizableCommit) maybeUnwrapped; assert result instanceof CommitResult : result; final CommitResult commitResult = (CommitResult) result; final Command pushAsIsCommand = normalizingPushCommand.asIs(commitResult); - log = new ReplicationLog<>(replicaId(), pushAsIsCommand, commitResult.revision()); - } else if (command.type() == CommandType.FORCE_PUSH && - ((ForcePushCommand) command).delegate().type() == CommandType.NORMALIZING_PUSH) { - final NormalizingPushCommand delegated = - (NormalizingPushCommand) ((ForcePushCommand) command).delegate(); - final CommitResult commitResult = (CommitResult) result; - final Command command0 = Command.forcePush(delegated.asIs(commitResult)); - log = new ReplicationLog<>(replicaId(), command0, commitResult.revision()); + log = new ReplicationLog<>(replicaId(), + maybeWrap(command, pushAsIsCommand), commitResult.revision()); } else { log = new ReplicationLog<>(replicaId(), command, result); } @@ -1349,6 +1356,20 @@ private T blockingExecute(Command command) throws Exception { } } + private static Command unwrapForcePush(Command command) { + if (command.type() == CommandType.FORCE_PUSH) { + return ((ForcePushCommand) command).delegate(); + } + return command; + } + + private static Command maybeWrap(Command oldCommand, Command pushAsIsCommand) { + if (oldCommand.type() == CommandType.FORCE_PUSH) { + return Command.forcePush(pushAsIsCommand); + } + return pushAsIsCommand; + } + private void createParentNodes() throws Exception { if (createdParentNodes) { return; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingServicePlugin.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingServicePlugin.java index b117dc2a39..4f38430225 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingServicePlugin.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingServicePlugin.java @@ -36,7 +36,7 @@ public final class PurgeSchedulingServicePlugin implements Plugin { private volatile PurgeSchedulingService purgeSchedulingService; @Override - public PluginTarget target() { + public PluginTarget target(CentralDogmaConfig config) { return PluginTarget.LEADER_ONLY; } @@ -87,7 +87,7 @@ public PurgeSchedulingService scheduledPurgingService() { public String toString() { return MoreObjects.toStringHelper(this) .omitNullValues() - .add("target", target()) + .add("target", PluginTarget.LEADER_ONLY) .add("purgeSchedulingService", purgeSchedulingService) .toString(); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/project/ProjectApiManager.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/project/ProjectApiManager.java index 29882ab826..d5da31a99f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/project/ProjectApiManager.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/project/ProjectApiManager.java @@ -57,20 +57,20 @@ public ProjectApiManager(ProjectManager projectManager, CommandExecutor commandE public Map listProjects(@Nullable User user) { final Map projects = projectManager.list(); - if (isAdmin()) { + if (isSystemAdmin()) { return projects; } return listProjectsWithoutInternal(projects, user); } - private static boolean isAdmin() { + private static boolean isSystemAdmin() { final User currentUserOrNull = AuthUtil.currentUserOrNull(); if (currentUserOrNull == null) { return false; } - return currentUserOrNull.isAdmin(); + return currentUserOrNull.isSystemAdmin(); } public static Map listProjectsWithoutInternal(Map projects, @@ -147,7 +147,8 @@ public Project getProject(String projectName, @Nullable User user) { if (user == null) { throw new IllegalArgumentException("Cannot access " + projectName); } - if (user.isAdmin()) { + + if (user.isSystemAdmin()) { return project; } final ProjectMetadata metadata = project.metadata(); @@ -165,7 +166,7 @@ private static boolean isInternalProject(String projectName) { } public boolean exists(String projectName) { - if (isInternalProject(projectName) && !isAdmin()) { + if (isInternalProject(projectName) && !isSystemAdmin()) { throw new IllegalArgumentException("Cannot access " + projectName); } return projectManager.exists(projectName); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CentralDogmaMirrorProvider.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CentralDogmaMirrorProvider.java index b23961a8c1..f27c2383a8 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CentralDogmaMirrorProvider.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CentralDogmaMirrorProvider.java @@ -62,6 +62,6 @@ public Mirror newMirror(MirrorContext context) { return new CentralDogmaMirror(context.id(), context.enabled(), context.schedule(), context.direction(), context.credential(), context.localRepo(), context.localPath(), repositoryUri.uri(), remoteProject, remoteRepo, remotePath, - context.gitignore()); + context.gitignore(), context.zone()); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/DefaultMetaRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/DefaultMetaRepository.java index 6480792931..a1c838e2b5 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/DefaultMetaRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/DefaultMetaRepository.java @@ -25,6 +25,8 @@ import java.util.Objects; import java.util.concurrent.CompletableFuture; +import javax.annotation.Nullable; + import com.cronutils.model.Cron; import com.cronutils.model.field.CronField; import com.cronutils.model.field.CronFieldName; @@ -44,6 +46,7 @@ import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.internal.Jackson; import com.linecorp.centraldogma.internal.api.v1.MirrorDto; +import com.linecorp.centraldogma.server.ZoneConfig; import com.linecorp.centraldogma.server.command.Command; import com.linecorp.centraldogma.server.command.CommitResult; import com.linecorp.centraldogma.server.credential.Credential; @@ -237,8 +240,9 @@ private CompletableFuture>> find(String filePattern) { @Override public CompletableFuture> createPushCommand(MirrorDto mirrorDto, Author author, + @Nullable ZoneConfig zoneConfig, boolean update) { - validateMirror(mirrorDto); + validateMirror(mirrorDto, zoneConfig); if (update) { final String summary = "Update the mirror '" + mirrorDto.id() + '\''; return mirror(mirrorDto.id()).thenApply(mirror -> { @@ -290,7 +294,7 @@ private Command newCommand(Credential credential, Author author, S change); } - private static void validateMirror(MirrorDto mirror) { + private static void validateMirror(MirrorDto mirror, @Nullable ZoneConfig zoneConfig) { checkArgument(!Strings.isNullOrEmpty(mirror.id()), "Mirror ID is empty"); final String scheduleString = mirror.schedule(); if (scheduleString != null) { @@ -299,6 +303,13 @@ private static void validateMirror(MirrorDto mirror) { checkArgument(!secondField.getExpression().asString().contains("*"), "The second field of the schedule must be specified. (seconds: *, expected: 0-59)"); } + + final String zone = mirror.zone(); + if (zone != null) { + checkArgument(zoneConfig != null, "Zone configuration is missing"); + checkArgument(zoneConfig.allZones().contains(zone), + "The zone '%s' is not in the zone configuration: %s", zone, zoneConfig); + } } private static MirrorConfig converterToMirrorConfig(MirrorDto mirrorDto) { @@ -315,6 +326,7 @@ private static MirrorConfig converterToMirrorConfig(MirrorDto mirrorDto) { mirrorDto.localPath(), URI.create(remoteUri), mirrorDto.gitignore(), - mirrorDto.credentialId()); + mirrorDto.credentialId(), + mirrorDto.zone()); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/MirrorConfig.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/MirrorConfig.java index 58a7ef94e9..b811f9b666 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/MirrorConfig.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/MirrorConfig.java @@ -81,6 +81,8 @@ public final class MirrorConfig { private final String credentialId; @Nullable private final Cron schedule; + @Nullable + private final String zone; @JsonCreator public MirrorConfig(@JsonProperty("id") String id, @@ -91,7 +93,8 @@ public MirrorConfig(@JsonProperty("id") String id, @JsonProperty("localPath") @Nullable String localPath, @JsonProperty(value = "remoteUri", required = true) URI remoteUri, @JsonProperty("gitignore") @Nullable Object gitignore, - @JsonProperty("credentialId") String credentialId) { + @JsonProperty("credentialId") String credentialId, + @JsonProperty("zone") @Nullable String zone) { this.id = requireNonNull(id, "id"); this.enabled = firstNonNull(enabled, true); if (schedule != null) { @@ -122,6 +125,7 @@ public MirrorConfig(@JsonProperty("id") String id, this.gitignore = null; } this.credentialId = requireNonNull(credentialId, "credentialId"); + this.zone = zone; } @Nullable @@ -132,7 +136,7 @@ Mirror toMirror(Project parent, Iterable credentials) { final MirrorContext mirrorContext = new MirrorContext( id, enabled, schedule, direction, findCredential(credentials, credentialId), - parent.repos().get(localRepo), localPath, remoteUri, gitignore); + parent.repos().get(localRepo), localPath, remoteUri, gitignore, zone); for (MirrorProvider mirrorProvider : MIRROR_PROVIDERS) { final Mirror mirror = mirrorProvider.newMirror(mirrorContext); if (mirror != null) { @@ -209,6 +213,12 @@ public String schedule() { } } + @Nullable + @JsonProperty("zone") + public String zone() { + return zone; + } + @Override public String toString() { return MoreObjects.toStringHelper(this).omitNullValues() @@ -220,6 +230,7 @@ public String toString() { .add("gitignore", gitignore) .add("credentialId", credentialId) .add("schedule", schedule) + .add("zone", zone) .toString(); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java index 51366e7cc6..7ae8102f3a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java @@ -34,6 +34,7 @@ import com.linecorp.centraldogma.common.RevisionRange; import com.linecorp.centraldogma.internal.Util; import com.linecorp.centraldogma.server.command.CommitResult; +import com.linecorp.centraldogma.server.command.ContentTransformer; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.repository.DiffResultType; import com.linecorp.centraldogma.server.storage.repository.FindOption; @@ -184,6 +185,13 @@ public CompletableFuture commit(Revision baseRevision, long commit return unwrap().commit(baseRevision, commitTimeMillis, author, summary, detail, markup, changes); } + @Override + public CompletableFuture commit(Revision baseRevision, long commitTimeMillis, Author author, + String summary, String detail, Markup markup, + ContentTransformer transformer) { + return unwrap().commit(baseRevision, commitTimeMillis, author, summary, detail, markup, transformer); + } + @Override public CompletableFuture findLatestRevision(Revision lastKnownRevision, String pathPattern, boolean errorOnEntryNotFound) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java index 75e7126071..ca3732ebcf 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java @@ -44,6 +44,7 @@ import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.common.RevisionRange; import com.linecorp.centraldogma.server.command.CommitResult; +import com.linecorp.centraldogma.server.command.ContentTransformer; import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.repository.DiffResultType; @@ -327,6 +328,13 @@ public CompletableFuture commit(Revision baseRevision, long commit normalizing); } + @Override + public CompletableFuture commit(Revision baseRevision, long commitTimeMillis, Author author, + String summary, String detail, Markup markup, + ContentTransformer transformer) { + return repo.commit(baseRevision, commitTimeMillis, author, summary, detail, markup, transformer); + } + @Override public String toString() { return toStringHelper(this) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/AbstractChangesApplier.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/AbstractChangesApplier.java new file mode 100644 index 0000000000..31e1cdac73 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/AbstractChangesApplier.java @@ -0,0 +1,136 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.IOException; + +import javax.annotation.Nullable; + +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.dircache.DirCacheBuilder; +import org.eclipse.jgit.dircache.DirCacheEditor; +import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; +import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.Repository; + +import com.fasterxml.jackson.databind.JsonNode; + +import com.linecorp.centraldogma.common.CentralDogmaException; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.server.storage.StorageException; + +abstract class AbstractChangesApplier { + + private static final byte[] EMPTY_BYTE = new byte[0]; + + int apply(Repository jGitRepository, Revision headRevision, + @Nullable ObjectId baseTreeId, DirCache dirCache) { + try (ObjectInserter inserter = jGitRepository.newObjectInserter(); + ObjectReader reader = jGitRepository.newObjectReader()) { + + if (baseTreeId != null) { + // the DirCacheBuilder is to used for doing update operations on the given DirCache object + final DirCacheBuilder builder = dirCache.builder(); + + // Add the tree object indicated by the prevRevision to the temporary DirCache object. + builder.addTree(EMPTY_BYTE, 0, reader, baseTreeId); + builder.finish(); + } + + return doApply(headRevision, dirCache, reader, inserter); + } catch (CentralDogmaException e) { + throw e; + } catch (Exception e) { + throw new StorageException("failed to apply changes on revision: " + headRevision.major(), e); + } + } + + abstract int doApply(Revision headRevision, DirCache dirCache, + ObjectReader reader, ObjectInserter inserter) throws IOException; + + static void applyPathEdit(DirCache dirCache, PathEdit edit) { + final DirCacheEditor e = dirCache.editor(); + e.add(edit); + e.finish(); + } + + // PathEdit implementations which is used when applying changes. + + static final class InsertJson extends PathEdit { + private final ObjectInserter inserter; + private final JsonNode jsonNode; + + InsertJson(String entryPath, ObjectInserter inserter, JsonNode jsonNode) { + super(entryPath); + this.inserter = inserter; + this.jsonNode = jsonNode; + } + + @Override + public void apply(DirCacheEntry ent) { + try { + ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, Jackson.writeValueAsBytes(jsonNode))); + ent.setFileMode(FileMode.REGULAR_FILE); + } catch (IOException e) { + throw new StorageException("failed to create a new JSON blob", e); + } + } + } + + static final class InsertText extends PathEdit { + private final ObjectInserter inserter; + private final String text; + + InsertText(String entryPath, ObjectInserter inserter, String text) { + super(entryPath); + this.inserter = inserter; + this.text = text; + } + + @Override + public void apply(DirCacheEntry ent) { + try { + ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, text.getBytes(UTF_8))); + ent.setFileMode(FileMode.REGULAR_FILE); + } catch (IOException e) { + throw new StorageException("failed to create a new text blob", e); + } + } + } + + static final class CopyOldEntry extends PathEdit { + private final DirCacheEntry oldEntry; + + CopyOldEntry(String entryPath, DirCacheEntry oldEntry) { + super(entryPath); + this.oldEntry = oldEntry; + } + + @Override + public void apply(DirCacheEntry ent) { + ent.setFileMode(oldEntry.getFileMode()); + ent.setObjectId(oldEntry.getObjectId()); + } + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitExecutor.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitExecutor.java new file mode 100644 index 0000000000..b3bccb8833 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CommitExecutor.java @@ -0,0 +1,214 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepository.R_HEADS_MASTER; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepository.doRefUpdate; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepository.newRevWalk; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepository.toTree; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; + +import java.util.List; +import java.util.function.Function; + +import javax.annotation.Nullable; + +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.diff.DiffFormatter; +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.dircache.DirCacheIterator; +import org.eclipse.jgit.lib.CommitBuilder; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.CentralDogmaException; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.ChangeConflictException; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.RedundantChangeException; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.server.command.CommitResult; +import com.linecorp.centraldogma.server.storage.StorageException; + +final class CommitExecutor { + + final GitRepository gitRepository; + private final long commitTimeMillis; + private final Author author; + private final String summary; + private final String detail; + private final Markup markup; + private final boolean allowEmptyCommit; + + CommitExecutor(GitRepository gitRepository, long commitTimeMillis, Author author, + String summary, String detail, Markup markup, boolean allowEmptyCommit) { + this.gitRepository = gitRepository; + this.commitTimeMillis = commitTimeMillis; + this.author = author; + this.summary = summary; + this.detail = detail; + this.markup = markup; + this.allowEmptyCommit = allowEmptyCommit; + } + + Author author() { + return author; + } + + String summary() { + return summary; + } + + void executeInitialCommit() { + commit(null, Revision.INIT, ImmutableList.of()); + } + + CommitResult execute(Revision baseRevision, + Function>> applyingChangesProvider) { + final RevisionAndEntries res; + final Iterable> applyingChanges; + gitRepository.writeLock(); + try { + final Revision normBaseRevision = gitRepository.normalizeNow(baseRevision); + final Revision headRevision = gitRepository.cachedHeadRevision(); + if (headRevision.major() != normBaseRevision.major()) { + throw new ChangeConflictException( + "invalid baseRevision: " + baseRevision + " (expected: " + headRevision + + " or equivalent)"); + } + + applyingChanges = applyingChangesProvider.apply(normBaseRevision); + res = commit(headRevision, headRevision.forward(1), applyingChanges); + + gitRepository.setHeadRevision(res.revision); + } finally { + gitRepository.writeUnLock(); + } + + // Note that the notification is made while no lock is held to avoid the risk of a dead lock. + gitRepository.notifyWatchers(res.revision, res.diffEntries); + return CommitResult.of(res.revision, applyingChanges); + } + + RevisionAndEntries commit(@Nullable Revision headRevision, Revision nextRevision, + Iterable> changes) { + requireNonNull(nextRevision, "nextRevision"); + requireNonNull(changes, "changes"); + + assert (headRevision == null && Iterables.isEmpty(changes)) || + (headRevision != null && headRevision.major() > 0); + assert nextRevision.major() > 0; + + final Repository jGitRepository = gitRepository.jGitRepository(); + try (ObjectInserter inserter = jGitRepository.newObjectInserter(); + ObjectReader reader = jGitRepository.newObjectReader(); + RevWalk revWalk = newRevWalk(reader)) { + final CommitIdDatabase commitIdDatabase = gitRepository.commitIdDatabase(); + + // The staging area that keeps the entries of the new tree. + // It starts with the entries of the tree at the headRevision (or with no entries if the + // headRevision is the initial commit), and then this method will apply the requested changes + // to build the new tree. + final DirCache dirCache = DirCache.newInCore(); + final List diffEntries; + + if (headRevision != null) { + final ObjectId prevTreeId = toTree(commitIdDatabase, revWalk, headRevision); + // Apply the changes and retrieve the list of the affected files. + final int numEdits = new DefaultChangesApplier(changes) + .apply(jGitRepository, headRevision, prevTreeId, dirCache); + // Reject empty commit if necessary. + boolean isEmpty = numEdits == 0; + if (!isEmpty) { + // Even if there are edits, the resulting tree might be identical with the previous tree. + final CanonicalTreeParser p = new CanonicalTreeParser(); + p.reset(reader, prevTreeId); + final DiffFormatter diffFormatter = new DiffFormatter(null); + diffFormatter.setRepository(jGitRepository); + diffEntries = diffFormatter.scan(p, new DirCacheIterator(dirCache)); + isEmpty = diffEntries.isEmpty(); + } else { + diffEntries = ImmutableList.of(); + } + if (!allowEmptyCommit && isEmpty) { + throw new RedundantChangeException( + headRevision, + "changes did not change anything in " + gitRepository.parent().name() + '/' + + gitRepository.name() + " at revision " + headRevision.major() + ": " + changes); + } + } else { + // initial commit. + diffEntries = ImmutableList.of(); + } + + // flush the current index to repository and get the result tree object id. + final ObjectId nextTreeId = dirCache.writeTree(inserter); + + // build a commit object + final PersonIdent personIdent = new PersonIdent(author.name(), author.email(), + commitTimeMillis / 1000L * 1000L, 0); + + final CommitBuilder commitBuilder = new CommitBuilder(); + + commitBuilder.setAuthor(personIdent); + commitBuilder.setCommitter(personIdent); + commitBuilder.setTreeId(nextTreeId); + commitBuilder.setEncoding(UTF_8); + + // Write summary, detail and revision to commit's message as JSON format. + commitBuilder.setMessage(CommitUtil.toJsonString(summary, detail, markup, nextRevision)); + + // if the head commit exists, use it as the parent commit. + if (headRevision != null) { + commitBuilder.setParentId(commitIdDatabase.get(headRevision)); + } + + final ObjectId nextCommitId = inserter.insert(commitBuilder); + inserter.flush(); + + // tagging the revision object, for history lookup purpose. + commitIdDatabase.put(nextRevision, nextCommitId); + doRefUpdate(jGitRepository, revWalk, R_HEADS_MASTER, nextCommitId); + + return new RevisionAndEntries(nextRevision, diffEntries); + } catch (CentralDogmaException | IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new StorageException("failed to push at '" + gitRepository.parent().name() + '/' + + gitRepository.name() + '\'', e); + } + } + + static final class RevisionAndEntries { + final Revision revision; + final List diffEntries; + + RevisionAndEntries(Revision revision, List diffEntries) { + this.revision = revision; + this.diffEntries = diffEntries; + } + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/DefaultChangesApplier.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/DefaultChangesApplier.java new file mode 100644 index 0000000000..86c232801e --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/DefaultChangesApplier.java @@ -0,0 +1,314 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.linecorp.centraldogma.server.internal.storage.repository.git.GitRepository.sanitizeText; +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.StringJoiner; + +import javax.annotation.Nullable; + +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.dircache.DirCacheEditor; +import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath; +import org.eclipse.jgit.dircache.DirCacheEditor.DeleteTree; +import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.ObjectReader; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.google.common.base.MoreObjects; + +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.ChangeConflictException; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.internal.Util; +import com.linecorp.centraldogma.internal.jsonpatch.JsonPatch; + +import difflib.DiffUtils; +import difflib.Patch; + +final class DefaultChangesApplier extends AbstractChangesApplier { + + private final Iterable> changes; + + DefaultChangesApplier(Iterable> changes) { + this.changes = changes; + } + + @Override + int doApply(Revision unused, DirCache dirCache, + ObjectReader reader, ObjectInserter inserter) throws IOException { + int numEdits = 0; + // loop over the specified changes. + for (Change change : changes) { + final String changePath = change.path().substring(1); // Strip the leading '/'. + final DirCacheEntry oldEntry = dirCache.getEntry(changePath); + final byte[] oldContent = oldEntry != null ? reader.open(oldEntry.getObjectId()).getBytes() + : null; + + switch (change.type()) { + case UPSERT_JSON: { + final JsonNode oldJsonNode = oldContent != null ? Jackson.readTree(oldContent) : null; + final JsonNode newJsonNode = firstNonNull((JsonNode) change.content(), + JsonNodeFactory.instance.nullNode()); + + // Upsert only when the contents are really different. + if (!Objects.equals(newJsonNode, oldJsonNode)) { + applyPathEdit(dirCache, new InsertJson(changePath, inserter, newJsonNode)); + numEdits++; + } + break; + } + case UPSERT_TEXT: { + final String sanitizedOldText; + if (oldContent != null) { + sanitizedOldText = sanitizeText(new String(oldContent, UTF_8)); + } else { + sanitizedOldText = null; + } + + final String sanitizedNewText = sanitizeText(change.contentAsText()); + + // Upsert only when the contents are really different. + if (!sanitizedNewText.equals(sanitizedOldText)) { + applyPathEdit(dirCache, new InsertText(changePath, inserter, sanitizedNewText)); + numEdits++; + } + break; + } + case REMOVE: + if (oldEntry != null) { + applyPathEdit(dirCache, new DeletePath(changePath)); + numEdits++; + break; + } + + // The path might be a directory. + if (applyDirectoryEdits(dirCache, changePath, null, change)) { + numEdits++; + } else { + // Was not a directory either; conflict. + reportNonExistentEntry(change); + break; + } + break; + case RENAME: { + final String newPath = + ((String) change.content()).substring(1); // Strip the leading '/'. + + if (dirCache.getEntry(newPath) != null) { + throw new ChangeConflictException("a file exists at the target path: " + change); + } + + if (oldEntry != null) { + if (changePath.equals(newPath)) { + // Redundant rename request - old path and new path are same. + break; + } + + final DirCacheEditor editor = dirCache.editor(); + editor.add(new DeletePath(changePath)); + editor.add(new CopyOldEntry(newPath, oldEntry)); + editor.finish(); + numEdits++; + break; + } + + // The path might be a directory. + if (applyDirectoryEdits(dirCache, changePath, newPath, change)) { + numEdits++; + } else { + // Was not a directory either; conflict. + reportNonExistentEntry(change); + } + break; + } + case APPLY_JSON_PATCH: { + final JsonNode oldJsonNode; + if (oldContent != null) { + oldJsonNode = Jackson.readTree(oldContent); + } else { + oldJsonNode = Jackson.nullNode; + } + + final JsonNode newJsonNode; + try { + newJsonNode = JsonPatch.fromJson((JsonNode) change.content()).apply(oldJsonNode); + } catch (Exception e) { + throw new ChangeConflictException("failed to apply JSON patch: " + change, e); + } + + // Apply only when the contents are really different. + if (!newJsonNode.equals(oldJsonNode)) { + applyPathEdit(dirCache, new InsertJson(changePath, inserter, newJsonNode)); + numEdits++; + } + break; + } + case APPLY_TEXT_PATCH: + final Patch patch = DiffUtils.parseUnifiedDiff( + Util.stringToLines(sanitizeText((String) change.content()))); + + final String sanitizedOldText; + final List sanitizedOldTextLines; + if (oldContent != null) { + sanitizedOldText = sanitizeText(new String(oldContent, UTF_8)); + sanitizedOldTextLines = Util.stringToLines(sanitizedOldText); + } else { + sanitizedOldText = null; + sanitizedOldTextLines = Collections.emptyList(); + } + + final String newText; + try { + final List newTextLines = DiffUtils.patch(sanitizedOldTextLines, patch); + if (newTextLines.isEmpty()) { + newText = ""; + } else { + final StringJoiner joiner = new StringJoiner("\n", "", "\n"); + for (String line : newTextLines) { + joiner.add(line); + } + newText = joiner.toString(); + } + } catch (Exception e) { + throw new ChangeConflictException("failed to apply text patch: " + change, e); + } + + // Apply only when the contents are really different. + if (!newText.equals(sanitizedOldText)) { + applyPathEdit(dirCache, new InsertText(changePath, inserter, newText)); + numEdits++; + } + break; + } + } + return numEdits; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("changes", changes) + .toString(); + } + + /** + * Applies recursive directory edits. + * + * @param oldDir the path to the directory to make a recursive change + * @param newDir the path to the renamed directory, or {@code null} to remove the directory. + * + * @return {@code true} if any edits were made to {@code dirCache}, {@code false} otherwise + */ + private static boolean applyDirectoryEdits(DirCache dirCache, + String oldDir, @Nullable String newDir, Change change) { + + if (!oldDir.endsWith("/")) { + oldDir += '/'; + } + if (newDir != null && !newDir.endsWith("/")) { + newDir += '/'; + } + + final byte[] rawOldDir = Constants.encode(oldDir); + final byte[] rawNewDir = newDir != null ? Constants.encode(newDir) : null; + final int numEntries = dirCache.getEntryCount(); + DirCacheEditor editor = null; + + loop: + for (int i = 0; i < numEntries; i++) { + final DirCacheEntry e = dirCache.getEntry(i); + final byte[] rawPath = e.getRawPath(); + + // Ensure that there are no entries under the newDir; we have a conflict otherwise. + if (rawNewDir != null) { + boolean conflict = true; + if (rawPath.length > rawNewDir.length) { + // Check if there is a file whose path starts with 'newDir'. + for (int j = 0; j < rawNewDir.length; j++) { + if (rawNewDir[j] != rawPath[j]) { + conflict = false; + break; + } + } + } else if (rawPath.length == rawNewDir.length - 1) { + // Check if there is a file whose path is exactly same with newDir without trailing '/'. + for (int j = 0; j < rawNewDir.length - 1; j++) { + if (rawNewDir[j] != rawPath[j]) { + conflict = false; + break; + } + } + } else { + conflict = false; + } + + if (conflict) { + throw new ChangeConflictException("target directory exists already: " + change); + } + } + + // Skip the entries that do not belong to the oldDir. + if (rawPath.length <= rawOldDir.length) { + continue; + } + for (int j = 0; j < rawOldDir.length; j++) { + if (rawOldDir[j] != rawPath[j]) { + continue loop; + } + } + + // Do not create an editor until we find an entry to rename/remove. + // We can tell if there was any matching entries or not from the nullness of editor later. + if (editor == null) { + editor = dirCache.editor(); + editor.add(new DeleteTree(oldDir)); + if (newDir == null) { + // Recursive removal + break; + } + } + + assert newDir != null; // We should get here only when it's a recursive rename. + + final String oldPath = e.getPathString(); + final String newPath = newDir + oldPath.substring(oldDir.length()); + editor.add(new CopyOldEntry(newPath, e)); + } + + if (editor != null) { + editor.finish(); + return true; + } else { + return false; + } + } + + private static void reportNonExistentEntry(Change change) { + throw new ChangeConflictException("non-existent file/directory: " + change); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java index a166720ae6..ec2334e510 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java @@ -16,7 +16,6 @@ package com.linecorp.centraldogma.server.internal.storage.repository.git; -import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkState; import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.context; import static com.linecorp.centraldogma.server.internal.storage.repository.git.FailFastUtil.failFastIfTimedOut; @@ -35,9 +34,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Properties; -import java.util.StringJoiner; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executor; @@ -47,6 +44,7 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.BiConsumer; +import java.util.function.Function; import java.util.function.Supplier; import java.util.regex.Pattern; @@ -55,19 +53,10 @@ import org.eclipse.jgit.diff.DiffEntry; import org.eclipse.jgit.diff.DiffFormatter; import org.eclipse.jgit.dircache.DirCache; -import org.eclipse.jgit.dircache.DirCacheBuilder; -import org.eclipse.jgit.dircache.DirCacheEditor; -import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath; -import org.eclipse.jgit.dircache.DirCacheEditor.DeleteTree; -import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; -import org.eclipse.jgit.dircache.DirCacheEntry; import org.eclipse.jgit.dircache.DirCacheIterator; -import org.eclipse.jgit.lib.CommitBuilder; import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdOwnerMap; -import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.Ref; @@ -89,7 +78,6 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; @@ -100,7 +88,6 @@ import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.CentralDogmaException; import com.linecorp.centraldogma.common.Change; -import com.linecorp.centraldogma.common.ChangeConflictException; import com.linecorp.centraldogma.common.Commit; import com.linecorp.centraldogma.common.Entry; import com.linecorp.centraldogma.common.EntryNotFoundException; @@ -117,6 +104,7 @@ import com.linecorp.centraldogma.internal.jsonpatch.JsonPatch; import com.linecorp.centraldogma.internal.jsonpatch.ReplaceMode; import com.linecorp.centraldogma.server.command.CommitResult; +import com.linecorp.centraldogma.server.command.ContentTransformer; import com.linecorp.centraldogma.server.internal.IsolatedSystemReader; import com.linecorp.centraldogma.server.internal.JGitUtil; import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache; @@ -128,8 +116,6 @@ import com.linecorp.centraldogma.server.storage.repository.Repository; import com.linecorp.centraldogma.server.storage.repository.RepositoryListener; -import difflib.DiffUtils; -import difflib.Patch; import io.netty.channel.EventLoop; /** @@ -141,7 +127,6 @@ class GitRepository implements Repository { static final String R_HEADS_MASTER = Constants.R_HEADS + Constants.MASTER; - private static final byte[] EMPTY_BYTE = new byte[0]; private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL); private static final Field revWalkObjectsField; @@ -262,10 +247,9 @@ class GitRepository implements Repository { // Initialize the commit ID database. commitIdDatabase = new CommitIdDatabase(jGitRepository); - // Insert the initial commit into the master branch. - commit0(null, Revision.INIT, creationTimeMillis, author, - "Create a new repository", "", Markup.PLAINTEXT, - Collections.emptyList(), true); + new CommitExecutor(this, creationTimeMillis, author, "Create a new repository", "", + Markup.PLAINTEXT, true) + .executeInitialCommit(); headRevision = Revision.INIT; success = true; @@ -394,6 +378,10 @@ void internalClose() { close(() -> new CentralDogmaException("should never reach here")); } + CommitIdDatabase commitIdDatabase() { + return commitIdDatabase; + } + @Override public org.eclipse.jgit.lib.Repository jGitRepository() { return jGitRepository; @@ -726,13 +714,11 @@ public CompletableFuture>> previewDiff(Revision baseRevisi final ServiceRequestContext ctx = context(); return CompletableFuture.supplyAsync(() -> { failFastIfTimedOut(this, logger, ctx, "previewDiff", baseRevision); - return blockingPreviewDiff(baseRevision, changes); + return blockingPreviewDiff(baseRevision, new DefaultChangesApplier(changes)); }, repositoryWorker); } - private Map> blockingPreviewDiff(Revision baseRevision, Iterable> changes) { - requireNonNull(baseRevision, "baseRevision"); - requireNonNull(changes, "changes"); + Map> blockingPreviewDiff(Revision baseRevision, AbstractChangesApplier changesApplier) { baseRevision = normalizeNow(baseRevision); readLock(); @@ -742,7 +728,7 @@ private Map> blockingPreviewDiff(Revision baseRevision, Iterab final ObjectId baseTreeId = toTree(revWalk, baseRevision); final DirCache dirCache = DirCache.newInCore(); - final int numEdits = applyChanges(baseRevision, baseTreeId, dirCache, changes); + final int numEdits = changesApplier.apply(jGitRepository, baseRevision, baseTreeId, dirCache); if (numEdits == 0) { return Collections.emptyMap(); } @@ -868,314 +854,49 @@ public CompletableFuture commit( requireNonNull(detail, "detail"); requireNonNull(markup, "markup"); requireNonNull(changes, "changes"); - - final ServiceRequestContext ctx = context(); - return CompletableFuture.supplyAsync(() -> { - failFastIfTimedOut(this, logger, ctx, "commit", baseRevision, author, summary); - return blockingCommit(baseRevision, commitTimeMillis, - author, summary, detail, markup, changes, false, directExecution); - }, repositoryWorker); - } - - private CommitResult blockingCommit( - Revision baseRevision, long commitTimeMillis, Author author, String summary, - String detail, Markup markup, Iterable> changes, boolean allowEmptyCommit, - boolean directExecution) { - - requireNonNull(baseRevision, "baseRevision"); - - final RevisionAndEntries res; - final Iterable> applyingChanges; - writeLock(); - try { - final Revision normBaseRevision = normalizeNow(baseRevision); - final Revision headRevision = cachedHeadRevision(); - if (headRevision.major() != normBaseRevision.major()) { - throw new ChangeConflictException( - "invalid baseRevision: " + baseRevision + " (expected: " + headRevision + - " or equivalent)"); - } - - if (directExecution) { - applyingChanges = blockingPreviewDiff(normBaseRevision, changes).values(); - } else { - applyingChanges = changes; + final CommitExecutor commitExecutor = + new CommitExecutor(this, commitTimeMillis, author, summary, detail, markup, false); + return commit(baseRevision, commitExecutor, normBaseRevision -> { + if (!directExecution) { + return changes; } - res = commit0(headRevision, headRevision.forward(1), commitTimeMillis, - author, summary, detail, markup, applyingChanges, allowEmptyCommit); - - this.headRevision = res.revision; - } finally { - writeUnLock(); - } - - // Note that the notification is made while no lock is held to avoid the risk of a dead lock. - notifyWatchers(res.revision, res.diffEntries); - return CommitResult.of(res.revision, applyingChanges); + return blockingPreviewDiff(normBaseRevision, new DefaultChangesApplier(changes)).values(); + }); } - private RevisionAndEntries commit0(@Nullable Revision prevRevision, Revision nextRevision, - long commitTimeMillis, Author author, String summary, - String detail, Markup markup, - Iterable> changes, boolean allowEmpty) { - + @Override + public CompletableFuture commit(Revision baseRevision, long commitTimeMillis, Author author, + String summary, String detail, Markup markup, + ContentTransformer transformer) { + requireNonNull(baseRevision, "baseRevision"); requireNonNull(author, "author"); requireNonNull(summary, "summary"); - requireNonNull(changes, "changes"); requireNonNull(detail, "detail"); requireNonNull(markup, "markup"); - - assert prevRevision == null || prevRevision.major() > 0; - assert nextRevision.major() > 0; - - try (ObjectInserter inserter = jGitRepository.newObjectInserter(); - ObjectReader reader = jGitRepository.newObjectReader(); - RevWalk revWalk = newRevWalk(reader)) { - - final ObjectId prevTreeId = prevRevision != null ? toTree(revWalk, prevRevision) : null; - - // The staging area that keeps the entries of the new tree. - // It starts with the entries of the tree at the prevRevision (or with no entries if the - // prevRevision is the initial commit), and then this method will apply the requested changes - // to build the new tree. - final DirCache dirCache = DirCache.newInCore(); - - // Apply the changes and retrieve the list of the affected files. - final int numEdits = applyChanges(prevRevision, prevTreeId, dirCache, changes); - - // Reject empty commit if necessary. - final List diffEntries; - boolean isEmpty = numEdits == 0; - if (!isEmpty) { - // Even if there are edits, the resulting tree might be identical with the previous tree. - final CanonicalTreeParser p = new CanonicalTreeParser(); - p.reset(reader, prevTreeId); - final DiffFormatter diffFormatter = new DiffFormatter(null); - diffFormatter.setRepository(jGitRepository); - diffEntries = diffFormatter.scan(p, new DirCacheIterator(dirCache)); - isEmpty = diffEntries.isEmpty(); - } else { - diffEntries = ImmutableList.of(); - } - - if (!allowEmpty && isEmpty) { - throw new RedundantChangeException( - "changes did not change anything in " + parent().name() + '/' + name() + - " at revision " + (prevRevision != null ? prevRevision.major() : 0) + - ": " + changes); - } - - // flush the current index to repository and get the result tree object id. - final ObjectId nextTreeId = dirCache.writeTree(inserter); - - // build a commit object - final PersonIdent personIdent = new PersonIdent(author.name(), author.email(), - commitTimeMillis / 1000L * 1000L, 0); - - final CommitBuilder commitBuilder = new CommitBuilder(); - - commitBuilder.setAuthor(personIdent); - commitBuilder.setCommitter(personIdent); - commitBuilder.setTreeId(nextTreeId); - commitBuilder.setEncoding(UTF_8); - - // Write summary, detail and revision to commit's message as JSON format. - commitBuilder.setMessage(CommitUtil.toJsonString(summary, detail, markup, nextRevision)); - - // if the head commit exists, use it as the parent commit. - if (prevRevision != null) { - commitBuilder.setParentId(commitIdDatabase.get(prevRevision)); - } - - final ObjectId nextCommitId = inserter.insert(commitBuilder); - inserter.flush(); - - // tagging the revision object, for history lookup purpose. - commitIdDatabase.put(nextRevision, nextCommitId); - doRefUpdate(revWalk, R_HEADS_MASTER, nextCommitId); - - return new RevisionAndEntries(nextRevision, diffEntries); - } catch (CentralDogmaException | IllegalArgumentException e) { - throw e; - } catch (Exception e) { - throw new StorageException("failed to push at '" + parent.name() + '/' + name + '\'', e); - } - } - - private int applyChanges(@Nullable Revision baseRevision, @Nullable ObjectId baseTreeId, DirCache dirCache, - Iterable> changes) { - - int numEdits = 0; - - try (ObjectInserter inserter = jGitRepository.newObjectInserter(); - ObjectReader reader = jGitRepository.newObjectReader()) { - - if (baseTreeId != null) { - // the DirCacheBuilder is to used for doing update operations on the given DirCache object - final DirCacheBuilder builder = dirCache.builder(); - - // Add the tree object indicated by the prevRevision to the temporary DirCache object. - builder.addTree(EMPTY_BYTE, 0, reader, baseTreeId); - builder.finish(); - } - - // loop over the specified changes. - for (Change change : changes) { - final String changePath = change.path().substring(1); // Strip the leading '/'. - final DirCacheEntry oldEntry = dirCache.getEntry(changePath); - final byte[] oldContent = oldEntry != null ? reader.open(oldEntry.getObjectId()).getBytes() - : null; - - switch (change.type()) { - case UPSERT_JSON: { - final JsonNode oldJsonNode = oldContent != null ? Jackson.readTree(oldContent) : null; - final JsonNode newJsonNode = firstNonNull((JsonNode) change.content(), - JsonNodeFactory.instance.nullNode()); - - // Upsert only when the contents are really different. - if (!Objects.equals(newJsonNode, oldJsonNode)) { - applyPathEdit(dirCache, new InsertJson(changePath, inserter, newJsonNode)); - numEdits++; - } - break; - } - case UPSERT_TEXT: { - final String sanitizedOldText; - if (oldContent != null) { - sanitizedOldText = sanitizeText(new String(oldContent, UTF_8)); - } else { - sanitizedOldText = null; - } - - final String sanitizedNewText = sanitizeText(change.contentAsText()); - - // Upsert only when the contents are really different. - if (!sanitizedNewText.equals(sanitizedOldText)) { - applyPathEdit(dirCache, new InsertText(changePath, inserter, sanitizedNewText)); - numEdits++; - } - break; - } - case REMOVE: - if (oldEntry != null) { - applyPathEdit(dirCache, new DeletePath(changePath)); - numEdits++; - break; - } - - // The path might be a directory. - if (applyDirectoryEdits(dirCache, changePath, null, change)) { - numEdits++; - } else { - // Was not a directory either; conflict. - reportNonExistentEntry(change); - break; - } - break; - case RENAME: { - final String newPath = - ((String) change.content()).substring(1); // Strip the leading '/'. - - if (dirCache.getEntry(newPath) != null) { - throw new ChangeConflictException("a file exists at the target path: " + change); - } - - if (oldEntry != null) { - if (changePath.equals(newPath)) { - // Redundant rename request - old path and new path are same. - break; - } - - final DirCacheEditor editor = dirCache.editor(); - editor.add(new DeletePath(changePath)); - editor.add(new CopyOldEntry(newPath, oldEntry)); - editor.finish(); - numEdits++; - break; - } - - // The path might be a directory. - if (applyDirectoryEdits(dirCache, changePath, newPath, change)) { - numEdits++; - } else { - // Was not a directory either; conflict. - reportNonExistentEntry(change); - } - break; - } - case APPLY_JSON_PATCH: { - final JsonNode oldJsonNode; - if (oldContent != null) { - oldJsonNode = Jackson.readTree(oldContent); - } else { - oldJsonNode = Jackson.nullNode; - } - - final JsonNode newJsonNode; - try { - newJsonNode = JsonPatch.fromJson((JsonNode) change.content()).apply(oldJsonNode); - } catch (Exception e) { - throw new ChangeConflictException("failed to apply JSON patch: " + change + - " old JSON: " + oldJsonNode, e); - } - - // Apply only when the contents are really different. - if (!newJsonNode.equals(oldJsonNode)) { - applyPathEdit(dirCache, new InsertJson(changePath, inserter, newJsonNode)); - numEdits++; - } - break; - } - case APPLY_TEXT_PATCH: - final Patch patch = DiffUtils.parseUnifiedDiff( - Util.stringToLines(sanitizeText((String) change.content()))); - - final String sanitizedOldText; - final List sanitizedOldTextLines; - if (oldContent != null) { - sanitizedOldText = sanitizeText(new String(oldContent, UTF_8)); - sanitizedOldTextLines = Util.stringToLines(sanitizedOldText); - } else { - sanitizedOldText = null; - sanitizedOldTextLines = Collections.emptyList(); - } - - final String newText; - try { - final List newTextLines = DiffUtils.patch(sanitizedOldTextLines, patch); - if (newTextLines.isEmpty()) { - newText = ""; - } else { - final StringJoiner joiner = new StringJoiner("\n", "", "\n"); - for (String line : newTextLines) { - joiner.add(line); - } - newText = joiner.toString(); - } - } catch (Exception e) { - throw new ChangeConflictException("failed to apply text patch: " + change, e); - } - - // Apply only when the contents are really different. - if (!newText.equals(sanitizedOldText)) { - applyPathEdit(dirCache, new InsertText(changePath, inserter, newText)); - numEdits++; - } - break; - } - } - } catch (CentralDogmaException | IllegalArgumentException e) { - throw e; - } catch (Exception e) { - throw new StorageException("failed to apply changes on revision " + baseRevision, e); - } - return numEdits; + requireNonNull(transformer, "transformer"); + final CommitExecutor commitExecutor = + new CommitExecutor(this, commitTimeMillis, author, summary, detail, markup, false); + return commit(baseRevision, commitExecutor, + normBaseRevision -> blockingPreviewDiff( + normBaseRevision, new TransformingChangesApplier(transformer)).values()); + } + + private CompletableFuture commit( + Revision baseRevision, + CommitExecutor commitExecutor, + Function>> applyingChangesProvider) { + final ServiceRequestContext ctx = context(); + return CompletableFuture.supplyAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "commit", baseRevision, + commitExecutor.author(), commitExecutor.summary()); + return commitExecutor.execute(baseRevision, applyingChangesProvider); + }, repositoryWorker); } /** * Removes {@code \r} and appends {@code \n} on the last line if it does not end with {@code \n}. */ - private static String sanitizeText(String text) { + static String sanitizeText(String text) { if (text.indexOf('\r') >= 0) { text = CR.matcher(text).replaceAll(""); } @@ -1185,113 +906,6 @@ private static String sanitizeText(String text) { return text; } - private static void reportNonExistentEntry(Change change) { - throw new ChangeConflictException("non-existent file/directory: " + change); - } - - private static void applyPathEdit(DirCache dirCache, PathEdit edit) { - final DirCacheEditor e = dirCache.editor(); - e.add(edit); - e.finish(); - } - - /** - * Applies recursive directory edits. - * - * @param oldDir the path to the directory to make a recursive change - * @param newDir the path to the renamed directory, or {@code null} to remove the directory. - * - * @return {@code true} if any edits were made to {@code dirCache}, {@code false} otherwise - */ - private static boolean applyDirectoryEdits(DirCache dirCache, - String oldDir, @Nullable String newDir, Change change) { - - if (!oldDir.endsWith("/")) { - oldDir += '/'; - } - if (newDir != null && !newDir.endsWith("/")) { - newDir += '/'; - } - - final byte[] rawOldDir = Constants.encode(oldDir); - final byte[] rawNewDir = newDir != null ? Constants.encode(newDir) : null; - final int numEntries = dirCache.getEntryCount(); - DirCacheEditor editor = null; - - loop: - for (int i = 0; i < numEntries; i++) { - final DirCacheEntry e = dirCache.getEntry(i); - final byte[] rawPath = e.getRawPath(); - - // Ensure that there are no entries under the newDir; we have a conflict otherwise. - if (rawNewDir != null) { - boolean conflict = true; - if (rawPath.length > rawNewDir.length) { - // Check if there is a file whose path starts with 'newDir'. - for (int j = 0; j < rawNewDir.length; j++) { - if (rawNewDir[j] != rawPath[j]) { - conflict = false; - break; - } - } - } else if (rawPath.length == rawNewDir.length - 1) { - // Check if there is a file whose path is exactly same with newDir without trailing '/'. - for (int j = 0; j < rawNewDir.length - 1; j++) { - if (rawNewDir[j] != rawPath[j]) { - conflict = false; - break; - } - } - } else { - conflict = false; - } - - if (conflict) { - throw new ChangeConflictException("target directory exists already: " + change); - } - } - - // Skip the entries that do not belong to the oldDir. - if (rawPath.length <= rawOldDir.length) { - continue; - } - for (int j = 0; j < rawOldDir.length; j++) { - if (rawOldDir[j] != rawPath[j]) { - continue loop; - } - } - - // Do not create an editor until we find an entry to rename/remove. - // We can tell if there was any matching entries or not from the nullness of editor later. - if (editor == null) { - editor = dirCache.editor(); - editor.add(new DeleteTree(oldDir)); - if (newDir == null) { - // Recursive removal - break; - } - } - - assert newDir != null; // We should get here only when it's a recursive rename. - - final String oldPath = e.getPathString(); - final String newPath = newDir + oldPath.substring(oldDir.length()); - editor.add(new CopyOldEntry(newPath, e)); - } - - if (editor != null) { - editor.finish(); - return true; - } else { - return false; - } - } - - private void doRefUpdate(RevWalk revWalk, String ref, ObjectId commitId) throws IOException { - doRefUpdate(jGitRepository, revWalk, ref, commitId); - } - - @VisibleForTesting static void doRefUpdate(org.eclipse.jgit.lib.Repository jGitRepository, RevWalk revWalk, String ref, ObjectId commitId) throws IOException { @@ -1559,7 +1173,7 @@ public void removeListener(RepositoryListener listener) { listeners.remove(listener); } - private void notifyWatchers(Revision newRevision, List diffEntries) { + void notifyWatchers(Revision newRevision, List diffEntries) { for (DiffEntry entry : diffEntries) { switch (entry.getChangeType()) { case ADD: @@ -1575,10 +1189,14 @@ private void notifyWatchers(Revision newRevision, List diffEntries) { } } - private Revision cachedHeadRevision() { + Revision cachedHeadRevision() { return headRevision; } + void setHeadRevision(Revision headRevision) { + this.headRevision = headRevision; + } + /** * Returns the current revision. */ @@ -1599,6 +1217,10 @@ private Revision uncachedHeadRevision() { } private RevTree toTree(RevWalk revWalk, Revision revision) { + return toTree(commitIdDatabase, revWalk, revision); + } + + static RevTree toTree(CommitIdDatabase commitIdDatabase, RevWalk revWalk, Revision revision) { final ObjectId commitId = commitIdDatabase.get(revision); try { return revWalk.parseCommit(commitId).getTree(); @@ -1613,7 +1235,7 @@ private RevWalk newRevWalk() { return revWalk; } - private static RevWalk newRevWalk(ObjectReader reader) { + static RevWalk newRevWalk(ObjectReader reader) { final RevWalk revWalk = new RevWalk(reader); configureRevWalk(revWalk); return revWalk; @@ -1636,7 +1258,7 @@ private void readUnlock() { rwLock.readLock().unlock(); } - private void writeLock() { + void writeLock() { rwLock.writeLock().lock(); if (closePending.get() != null) { writeUnLock(); @@ -1644,7 +1266,7 @@ private void writeLock() { } } - private void writeUnLock() { + void writeUnLock() { rwLock.writeLock().unlock(); } @@ -1676,7 +1298,7 @@ public void cloneTo(File newRepoDir, BiConsumer progressListen final int batch = 16; final List commits = blockingHistory( new Revision(i), new Revision(Math.min(endRevision.major(), i + batch - 1)), - Repository.ALL_PATH, batch); + ALL_PATH, batch); checkState(!commits.isEmpty(), "empty commits"); if (previousNonEmptyRevision == null) { @@ -1689,19 +1311,21 @@ public void cloneTo(File newRepoDir, BiConsumer progressListen final Revision baseRevision = revision.backward(1); final Collection> changes = - diff(previousNonEmptyRevision, revision, Repository.ALL_PATH).join().values(); + diff(previousNonEmptyRevision, revision, ALL_PATH).join().values(); try { - newRepo.blockingCommit( - baseRevision, c.when(), c.author(), c.summary(), c.detail(), c.markup(), - changes, /* allowEmptyCommit */ false, false); + new CommitExecutor(newRepo, c.when(), c.author(), c.summary(), + c.detail(), c.markup(), false) + .execute(baseRevision, normBaseRevision -> blockingPreviewDiff( + normBaseRevision, new DefaultChangesApplier(changes)).values()); previousNonEmptyRevision = revision; } catch (RedundantChangeException e) { // NB: We allow an empty commit here because an old version of Central Dogma had a bug // which allowed the creation of an empty commit. - newRepo.blockingCommit( - baseRevision, c.when(), c.author(), c.summary(), c.detail(), c.markup(), - changes, /* allowEmptyCommit */ true, false); + new CommitExecutor(newRepo, c.when(), c.author(), c.summary(), + c.detail(), c.markup(), true) + .execute(baseRevision, normBaseRevision -> blockingPreviewDiff( + normBaseRevision, new DefaultChangesApplier(changes)).values()); } progressListener.accept(i, endRevision.major()); @@ -1732,73 +1356,4 @@ public String toString() { .add("dir", jGitRepository.getDirectory()) .toString(); } - - private static final class RevisionAndEntries { - final Revision revision; - final List diffEntries; - - RevisionAndEntries(Revision revision, List diffEntries) { - this.revision = revision; - this.diffEntries = diffEntries; - } - } - - // PathEdit implementations which is used when applying changes. - - private static final class InsertText extends PathEdit { - private final ObjectInserter inserter; - private final String text; - - InsertText(String entryPath, ObjectInserter inserter, String text) { - super(entryPath); - this.inserter = inserter; - this.text = text; - } - - @Override - public void apply(DirCacheEntry ent) { - try { - ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, text.getBytes(UTF_8))); - ent.setFileMode(FileMode.REGULAR_FILE); - } catch (IOException e) { - throw new StorageException("failed to create a new text blob", e); - } - } - } - - private static final class InsertJson extends PathEdit { - private final ObjectInserter inserter; - private final JsonNode jsonNode; - - InsertJson(String entryPath, ObjectInserter inserter, JsonNode jsonNode) { - super(entryPath); - this.inserter = inserter; - this.jsonNode = jsonNode; - } - - @Override - public void apply(DirCacheEntry ent) { - try { - ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, Jackson.writeValueAsBytes(jsonNode))); - ent.setFileMode(FileMode.REGULAR_FILE); - } catch (IOException e) { - throw new StorageException("failed to create a new JSON blob", e); - } - } - } - - private static final class CopyOldEntry extends PathEdit { - private final DirCacheEntry oldEntry; - - CopyOldEntry(String entryPath, DirCacheEntry oldEntry) { - super(entryPath); - this.oldEntry = oldEntry; - } - - @Override - public void apply(DirCacheEntry ent) { - ent.setFileMode(oldEntry.getFileMode()); - ent.setObjectId(oldEntry.getObjectId()); - } - } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/TransformingChangesApplier.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/TransformingChangesApplier.java new file mode 100644 index 0000000000..20c208852a --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/TransformingChangesApplier.java @@ -0,0 +1,82 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +import java.io.IOException; +import java.util.Objects; + +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.ObjectReader; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.google.common.base.MoreObjects; + +import com.linecorp.centraldogma.common.CentralDogmaException; +import com.linecorp.centraldogma.common.ChangeConflictException; +import com.linecorp.centraldogma.common.EntryType; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.server.command.ContentTransformer; + +final class TransformingChangesApplier extends AbstractChangesApplier { + + private final ContentTransformer transformer; + + TransformingChangesApplier(ContentTransformer transformer) { + checkArgument(transformer.entryType() == EntryType.JSON, + "transformer: %s (expected: JSON type)", transformer); + //noinspection unchecked + this.transformer = (ContentTransformer) transformer; + } + + @Override + int doApply(Revision headRevision, DirCache dirCache, + ObjectReader reader, ObjectInserter inserter) throws IOException { + final String changePath = transformer.path().substring(1); // Strip the leading '/'. + final DirCacheEntry oldEntry = dirCache.getEntry(changePath); + final byte[] oldContent = oldEntry != null ? reader.open(oldEntry.getObjectId()).getBytes() + : null; + final JsonNode oldJsonNode = oldContent != null ? Jackson.readTree(oldContent) + : JsonNodeFactory.instance.nullNode(); + try { + final JsonNode newJsonNode = transformer.transformer().apply(headRevision, oldJsonNode.deepCopy()); + requireNonNull(newJsonNode, "transformer.transformer().apply() returned null"); + if (!Objects.equals(newJsonNode, oldJsonNode)) { + applyPathEdit(dirCache, new InsertJson(changePath, inserter, newJsonNode)); + return 1; + } + } catch (CentralDogmaException e) { + throw e; + } catch (Exception e) { + throw new ChangeConflictException("failed to transform the content: " + oldJsonNode + + " transformer: " + transformer, e); + } + return 0; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("transformer", transformer) + .toString(); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/MemberNotFoundException.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/MemberNotFoundException.java new file mode 100644 index 0000000000..54efc34309 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/MemberNotFoundException.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.metadata; + +import com.linecorp.centraldogma.common.CentralDogmaException; + +/** + * A {@link CentralDogmaException} that is raised when failed to find a {@link Member}. + */ +public final class MemberNotFoundException extends CentralDogmaException { + + private static final long serialVersionUID = 914551040812058495L; + + MemberNotFoundException(String memberId, String projectName) { + super("failed to find member " + memberId + " in '" + projectName + '\''); + } + + MemberNotFoundException(String memberId, String projectName, String repoName) { + super("failed to find member " + memberId + " in '" + projectName + '/' + repoName + '\''); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java index 9a259a3fc5..2a09bb7e1b 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java @@ -17,6 +17,7 @@ package com.linecorp.centraldogma.server.metadata; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.linecorp.centraldogma.internal.jsonpatch.JsonPatchOperation.asJsonArray; import static com.linecorp.centraldogma.internal.jsonpatch.JsonPatchUtil.encodeSegment; import static com.linecorp.centraldogma.server.internal.storage.project.ProjectApiManager.listProjectsWithoutInternal; @@ -29,6 +30,7 @@ import java.util.Collection; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -40,6 +42,7 @@ import com.fasterxml.jackson.core.JsonPointer; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.spotify.futures.CompletableFutures; import com.linecorp.armeria.common.annotation.Nullable; @@ -48,19 +51,17 @@ import com.linecorp.centraldogma.common.Change; import com.linecorp.centraldogma.common.ChangeConflictException; import com.linecorp.centraldogma.common.ProjectRole; +import com.linecorp.centraldogma.common.RedundantChangeException; import com.linecorp.centraldogma.common.RepositoryExistsException; import com.linecorp.centraldogma.common.RepositoryRole; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.internal.Jackson; import com.linecorp.centraldogma.internal.jsonpatch.AddOperation; -import com.linecorp.centraldogma.internal.jsonpatch.JsonPatchOperation; -import com.linecorp.centraldogma.internal.jsonpatch.RemoveIfExistsOperation; import com.linecorp.centraldogma.internal.jsonpatch.RemoveOperation; import com.linecorp.centraldogma.internal.jsonpatch.ReplaceOperation; import com.linecorp.centraldogma.internal.jsonpatch.TestAbsenceOperation; import com.linecorp.centraldogma.server.QuotaConfig; import com.linecorp.centraldogma.server.command.CommandExecutor; -import com.linecorp.centraldogma.server.internal.admin.service.TokenNotFoundException; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.project.ProjectManager; @@ -241,46 +242,64 @@ public CompletableFuture addMember(Author author, String projectName, } /** - * Removes the specified {@code member} from the {@link ProjectMetadata} in the specified - * {@code projectName}. It also removes the {@link RepositoryRole} of the specified {@code member} - * from every {@link RepositoryMetadata}. + * Removes the specified {@code user} from the {@link ProjectMetadata} in the specified + * {@code projectName}. It also removes the {@link RepositoryRole} of the specified {@code user} from every + * {@link RepositoryMetadata}. */ - public CompletableFuture removeMember(Author author, String projectName, User member) { + public CompletableFuture removeMember(Author author, String projectName, User user) { requireNonNull(author, "author"); requireNonNull(projectName, "projectName"); - requireNonNull(member, "member"); + requireNonNull(user, "user"); + final String memberId = user.id(); final String commitSummary = - "Remove the member '" + member.id() + "' from the project '" + projectName + '\''; - return metadataRepo.push( - projectName, Project.REPO_DOGMA, author, commitSummary, - () -> fetchMetadata(projectName).thenApply( - metadataWithRevision -> { - final ImmutableList.Builder patches = ImmutableList.builder(); - metadataWithRevision - .object().repos().values() - .stream().filter(r -> r.roles().users().containsKey(member.id())) - .forEach(r -> patches.add(new RemoveOperation( - repositoryUserRolePointer(r.name(), member.id())))); - // e.g. - // { - // "repos": ... - // "members: { - // "my-user@linecorp.com": { - // "login": "my-user@linecorp.com", - // "role": "OWNER", - // "creation": ... - // }, - // ... - // } - // } - patches.add(new RemoveOperation(JsonPointer.compile("/members" + - encodeSegment(member.id())))); - final Change change = - Change.ofJsonPatch(METADATA_JSON, Jackson.valueToTree(patches.build())); - return HolderWithRevision.of(change, metadataWithRevision.revision()); - }) - ); + "Remove the member '" + memberId + "' from the project '" + projectName + '\''; + + final ProjectMetadataTransformer transformer = + new ProjectMetadataTransformer((headRevision, projectMetadata) -> { + projectMetadata.member(memberId); // Raises an exception if the member does not exist. + final Map newMembers = + projectMetadata.members().entrySet().stream() + .filter(entry -> !entry.getKey().equals(memberId)) + .collect(toImmutableMap(Entry::getKey, Entry::getValue)); + + final ImmutableMap newRepos = + removeMemberFromRepositories(projectMetadata, memberId); + return new ProjectMetadata(projectMetadata.name(), + newRepos, + newMembers, + projectMetadata.tokens(), + projectMetadata.creation(), + projectMetadata.removal()); + }); + return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, transformer); + } + + private static ImmutableMap removeMemberFromRepositories( + ProjectMetadata projectMetadata, String memberId) { + final ImmutableMap.Builder reposBuilder = + ImmutableMap.builderWithExpectedSize(projectMetadata.repos().size()); + for (Entry entry : projectMetadata.repos().entrySet()) { + final RepositoryMetadata repositoryMetadata = entry.getValue(); + final Roles roles = repositoryMetadata.roles(); + final Map users = roles.users(); + if (users.get(memberId) != null) { + final ImmutableMap newUsers = + users.entrySet().stream() + .filter(e -> !e.getKey().equals(memberId)) + .collect(toImmutableMap(Entry::getKey, Entry::getValue)); + final Roles newRoles = new Roles(roles.projectRoles(), newUsers, roles.tokens()); + reposBuilder.put(entry.getKey(), + new RepositoryMetadata(repositoryMetadata.name(), + newRoles, + repositoryMetadata.creation(), + repositoryMetadata.removal(), + repositoryMetadata.writeQuota())); + } else { + reposBuilder.put(entry); + } + } + return reposBuilder.build(); } /** @@ -432,32 +451,24 @@ public CompletableFuture updateRepositoryProjectRoles(Author author, "Can't give a role to guest for internal repository: " + repoName); } } - - // e.g. - // { - // "repos": { - // "my-repo": { - // "name": "my-repo", - // "roles": { - // "users": { - // "my-user": "READ" - // } - // "tokens": ... - // "projects": { - // "member": "READ", - // "guest": null - // } - // } - // } - // } - // } - final JsonPointer path = JsonPointer.compile("/repos" + encodeSegment(repoName) + "/roles/projects"); - final Change change = Change.ofJsonPatch(METADATA_JSON, - new ReplaceOperation(path, Jackson.valueToTree(projectRoles)) - .toJsonNode()); final String commitSummary = "Update the project roles of the '" + repoName + "' in the project '" + projectName + '\''; - return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, change); + final RepositoryMetadataTransformer transformer = new RepositoryMetadataTransformer( + repoName, (headRevision, repositoryMetadata) -> { + if (repositoryMetadata.roles().projectRoles().equals(projectRoles)) { + throw new RedundantChangeException( + headRevision, + "the project roles of '" + projectName + '/' + repoName + "' isn't changed."); + } + final Roles newRoles = new Roles(projectRoles, repositoryMetadata.roles().users(), + repositoryMetadata.roles().tokens()); + return new RepositoryMetadata(repositoryMetadata.name(), + newRoles, + repositoryMetadata.creation(), + repositoryMetadata.removal(), + repositoryMetadata.writeQuota()); + }); + return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, transformer); } /** @@ -479,8 +490,7 @@ public CompletableFuture addToken(Author author, String projectName, requireNonNull(role, "role"); return getTokens().thenCompose(tokens -> { - final Token token = tokens.appIds().get(appId); - checkArgument(token != null, "Token not found: " + appId); + tokens.get(appId); // Will raise an exception if not found. final TokenRegistration registration = new TokenRegistration(appId, role, UserAndTimestamp.of(author)); @@ -519,27 +529,58 @@ public CompletableFuture removeToken(Author author, String projectName private CompletableFuture removeToken(String projectName, Author author, String appId, boolean quiet) { final String commitSummary = "Remove the token '" + appId + "' from the project '" + projectName + '\''; - return metadataRepo.push( - projectName, Project.REPO_DOGMA, author, commitSummary, - () -> fetchMetadata(projectName).thenApply(metadataWithRevision -> { - final ImmutableList.Builder patches = ImmutableList.builder(); - final ProjectMetadata metadata = metadataWithRevision.object(); - metadata.repos().values() - .stream().filter(repo -> repo.roles().tokens().containsKey(appId)) - .forEach(r -> patches.add( - new RemoveOperation(repositoryTokenRolePointer(r.name(), appId)))); - if (quiet) { - patches.add(new RemoveIfExistsOperation(JsonPointer.compile("/tokens" + - encodeSegment(appId)))); - } else { - patches.add(new RemoveOperation(JsonPointer.compile("/tokens" + - encodeSegment(appId)))); - } - final Change change = - Change.ofJsonPatch(METADATA_JSON, Jackson.valueToTree(patches.build())); - return HolderWithRevision.of(change, metadataWithRevision.revision()); - }) - ); + final ProjectMetadataTransformer transformer = + new ProjectMetadataTransformer((headRevision, projectMetadata) -> { + final Map tokens = projectMetadata.tokens(); + final Map newTokens; + if (tokens.get(appId) == null) { + if (!quiet) { + throw new TokenNotFoundException( + "failed to find the token " + appId + " in project " + projectName); + } + newTokens = tokens; + } else { + newTokens = tokens.entrySet() + .stream() + .filter(entry -> !entry.getKey().equals(appId)) + .collect(toImmutableMap(Entry::getKey, Entry::getValue)); + } + + final ImmutableMap newRepos = + removeTokenFromRepositories(appId, projectMetadata); + return new ProjectMetadata(projectMetadata.name(), + newRepos, + projectMetadata.members(), + newTokens, + projectMetadata.creation(), + projectMetadata.removal()); + }); + return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, transformer); + } + + private static ImmutableMap removeTokenFromRepositories( + String appId, ProjectMetadata projectMetadata) { + final ImmutableMap.Builder builder = + ImmutableMap.builderWithExpectedSize(projectMetadata.repos().size()); + for (Entry entry : projectMetadata.repos().entrySet()) { + final RepositoryMetadata repositoryMetadata = entry.getValue(); + final Roles roles = repositoryMetadata.roles(); + if (roles.tokens().get(appId) != null) { + final Map newTokens = + roles.tokens().entrySet().stream() + .filter(e -> !e.getKey().equals(appId)) + .collect(toImmutableMap(Entry::getKey, Entry::getValue)); + final Roles newRoles = new Roles(roles.projectRoles(), roles.users(), newTokens); + builder.put(entry.getKey(), new RepositoryMetadata(repositoryMetadata.name(), + newRoles, + repositoryMetadata.creation(), + repositoryMetadata.removal(), + repositoryMetadata.writeQuota())); + } else { + builder.put(entry); + } + } + return builder.build(); } /** @@ -551,7 +592,6 @@ public CompletableFuture updateTokenRole(Author author, String project requireNonNull(projectName, "projectName"); requireNonNull(token, "token"); requireNonNull(role, "role"); - final TokenRegistration registration = new TokenRegistration(token.appId(), role, UserAndTimestamp.of(author)); final JsonPointer path = JsonPointer.compile("/tokens" + encodeSegment(registration.id())); @@ -580,10 +620,29 @@ public CompletableFuture addUserRepositoryRole(Author author, String p return getProject(projectName).thenCompose(project -> { ensureProjectMember(project, member); final String commitSummary = "Add repository role of '" + member.id() + - "' as '" + role + "' to the '" + projectName + '/' + repoName + '\n'; - return addRepositoryRoleAtPointer(author, projectName, - repositoryUserRolePointer(repoName, member.id()), - role, commitSummary); + "' as '" + role + "' to '" + projectName + '/' + repoName + '\n'; + final RepositoryMetadataTransformer transformer = new RepositoryMetadataTransformer( + repoName, (headRevision, repositoryMetadata) -> { + final Roles roles = repositoryMetadata.roles(); + if (roles.users().get(member.id()) != null) { + throw new ChangeConflictException( + "the member " + member.id() + " is already added to '" + + projectName + '/' + repoName + '\''); + } + + final ImmutableMap newUsers = + ImmutableMap.builderWithExpectedSize(roles.users().size() + 1) + .putAll(roles.users()) + .put(member.id(), role) + .build(); + final Roles newRoles = new Roles(roles.projectRoles(), newUsers, roles.tokens()); + return new RepositoryMetadata(repositoryMetadata.name(), + newRoles, + repositoryMetadata.creation(), + repositoryMetadata.removal(), + repositoryMetadata.writeQuota()); + }); + return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, transformer); }); } @@ -599,10 +658,27 @@ public CompletableFuture removeUserRepositoryRole(Author author, Strin requireNonNull(member, "member"); final String memberId = member.id(); - return removeRepositoryRoleAtPointer(author, projectName, - repositoryUserRolePointer(repoName, memberId), - "Remove repository role of the '" + memberId + - "' from the '" + projectName + '/' + repoName + '\''); + final RepositoryMetadataTransformer transformer = new RepositoryMetadataTransformer( + repoName, (headRevision, repositoryMetadata) -> { + final Roles roles = repositoryMetadata.roles(); + if (roles.users().get(memberId) == null) { + throw new MemberNotFoundException(memberId, projectName, repoName); + } + + final Map newUsers = + roles.users().entrySet().stream() + .filter(entry -> !entry.getKey().equals(memberId)) + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + final Roles newRoles = new Roles(roles.projectRoles(), newUsers, roles.tokens()); + return new RepositoryMetadata(repositoryMetadata.name(), + newRoles, + repositoryMetadata.creation(), + repositoryMetadata.removal(), + repositoryMetadata.writeQuota()); + }); + final String commitSummary = "Remove repository role of the '" + memberId + + "' from '" + projectName + '/' + repoName + '\''; + return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, transformer); } /** @@ -619,11 +695,48 @@ public CompletableFuture updateUserRepositoryRole(Author author, Strin requireNonNull(role, "role"); final String memberId = member.id(); - return replaceRepositoryRoleAtPointer( - author, projectName, - repositoryUserRolePointer(repoName, memberId), role, - "Update repository role of the '" + memberId + - "' as '" + role + "' for the '" + projectName + '/' + repoName + '\''); + final RepositoryMetadataTransformer transformer = new RepositoryMetadataTransformer( + repoName, (headRevision, repositoryMetadata) -> { + final Roles roles = repositoryMetadata.roles(); + final RepositoryRole oldRepositoryRole = roles.users().get(memberId); + if (oldRepositoryRole == null) { + throw new MemberNotFoundException(memberId, projectName, repoName); + } + + if (oldRepositoryRole == role) { + throw new RedundantChangeException( + headRevision, + "the repository role of " + memberId + " in '" + projectName + '/' + repoName + + "' isn't changed."); + } + + final Map newUsers = + updateRepositoryRole(role, roles.users(), memberId); + final Roles newRoles = new Roles(roles.projectRoles(), newUsers, roles.tokens()); + return new RepositoryMetadata(repositoryMetadata.name(), + newRoles, + repositoryMetadata.creation(), + repositoryMetadata.removal(), + repositoryMetadata.writeQuota()); + }); + final String commitSummary = "Update repository role of the '" + memberId + "' as '" + role + + "' for '" + projectName + '/' + repoName + '\''; + return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, transformer); + } + + private static Map updateRepositoryRole( + RepositoryRole repositoryRole, Map repositoryRoles, + String id) { + final ImmutableMap.Builder builder = + ImmutableMap.builderWithExpectedSize(repositoryRoles.size()); + for (Entry entry : repositoryRoles.entrySet()) { + if (entry.getKey().equals(id)) { + builder.put(id, repositoryRole); + } else { + builder.put(entry); + } + } + return builder.build(); } /** @@ -641,11 +754,29 @@ public CompletableFuture addTokenRepositoryRole(Author author, String return getProject(projectName).thenCompose(project -> { ensureProjectToken(project, appId); - return addRepositoryRoleAtPointer( - author, projectName, - repositoryTokenRolePointer(repoName, appId), role, - "Add repository role of the token '" + appId + - "' as '" + role + "' to the '" + projectName + '/' + repoName + '\''); + final String commitSummary = "Add repository role of the token '" + appId + "' as '" + role + + "' to '" + projectName + '/' + repoName + "'\n"; + final RepositoryMetadataTransformer transformer = new RepositoryMetadataTransformer( + repoName, (headRevision, repositoryMetadata) -> { + final Roles roles = repositoryMetadata.roles(); + if (roles.tokens().get(appId) != null) { + throw new ChangeConflictException( + "the token " + appId + " is already added to '" + + projectName + '/' + repoName + '\''); + } + + final Map newTokens = + ImmutableMap.builderWithExpectedSize(roles.tokens().size() + 1) + .putAll(roles.tokens()) + .put(appId, role).build(); + final Roles newRoles = new Roles(roles.projectRoles(), roles.users(), newTokens); + return new RepositoryMetadata(repositoryMetadata.name(), + newRoles, + repositoryMetadata.creation(), + repositoryMetadata.removal(), + repositoryMetadata.writeQuota()); + }); + return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, transformer); }); } @@ -660,10 +791,29 @@ public CompletableFuture removeTokenRepositoryRole(Author author, Stri requireNonNull(repoName, "repoName"); requireNonNull(appId, "appId"); - return removeRepositoryRoleAtPointer(author, projectName, - repositoryTokenRolePointer(repoName, appId), - "Remove repository role of the token '" + appId + - "' from the '" + projectName + '/' + repoName + '\''); + final RepositoryMetadataTransformer transformer = new RepositoryMetadataTransformer( + repoName, (headRevision, repositoryMetadata) -> { + final Roles roles = repositoryMetadata.roles(); + if (roles.tokens().get(appId) == null) { + throw new ChangeConflictException( + "the token " + appId + " doesn't exist at '" + + projectName + '/' + repoName + '\''); + } + + final Map newTokens = + roles.tokens().entrySet().stream() + .filter(entry -> !entry.getKey().equals(appId)) + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + final Roles newRoles = new Roles(roles.projectRoles(), roles.users(), newTokens); + return new RepositoryMetadata(repositoryMetadata.name(), + newRoles, + repositoryMetadata.creation(), + repositoryMetadata.removal(), + repositoryMetadata.writeQuota()); + }); + final String commitSummary = "Remove repository role of the token '" + appId + + "' from '" + projectName + '/' + repoName + '\''; + return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, transformer); } /** @@ -679,11 +829,35 @@ public CompletableFuture updateTokenRepositoryRole(Author author, Stri requireNonNull(appId, "appId"); requireNonNull(role, "role"); - return replaceRepositoryRoleAtPointer( - author, projectName, - repositoryTokenRolePointer(repoName, appId), role, - "Update repository role of the token '" + appId + - "' as '" + role + "' for the '" + projectName + '/' + repoName + '\''); + final RepositoryMetadataTransformer transformer = new RepositoryMetadataTransformer( + repoName, (headRevision, repositoryMetadata) -> { + final Roles roles = repositoryMetadata.roles(); + final RepositoryRole oldRepositoryRole = roles.tokens().get(appId); + if (oldRepositoryRole == null) { + throw new TokenNotFoundException( + "the token " + appId + " doesn't exist at '" + + projectName + '/' + repoName + '\''); + } + + if (oldRepositoryRole == role) { + throw new RedundantChangeException( + headRevision, + "the permission of " + appId + " in '" + projectName + '/' + repoName + + "' isn't changed."); + } + + final Map newTokens = + updateRepositoryRole(role, roles.tokens(), appId); + final Roles newRoles = new Roles(roles.projectRoles(), roles.users(), newTokens); + return new RepositoryMetadata(repositoryMetadata.name(), + newRoles, + repositoryMetadata.creation(), + repositoryMetadata.removal(), + repositoryMetadata.writeQuota()); + }); + final String commitSummary = "Update repository role of the token '" + appId + + "' for '" + projectName + '/' + repoName + '\''; + return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, transformer); } /** @@ -706,37 +880,6 @@ public CompletableFuture updateWriteQuota( return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, change); } - private CompletableFuture addRepositoryRoleAtPointer(Author author, - String projectName, JsonPointer path, - RepositoryRole role, - String commitSummary) { - final Change change = - Change.ofJsonPatch(METADATA_JSON, - asJsonArray(new TestAbsenceOperation(path), - new AddOperation(path, Jackson.valueToTree(role)))); - return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, change); - } - - /** - * Removes {@link RepositoryRole} at the specified {@code path}. - */ - private CompletableFuture removeRepositoryRoleAtPointer(Author author, String projectName, - JsonPointer path, String commitSummary) { - final Change change = Change.ofJsonPatch(METADATA_JSON, - new RemoveOperation(path).toJsonNode()); - return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, change); - } - - private CompletableFuture replaceRepositoryRoleAtPointer(Author author, - String projectName, JsonPointer path, - RepositoryRole role, - String commitSummary) { - final Change change = - Change.ofJsonPatch(METADATA_JSON, - new ReplaceOperation(path, Jackson.valueToTree(role)).toJsonNode()); - return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, change); - } - /** * Finds {@link RepositoryRole} of the specified {@link User} or {@link UserWithToken} * from the specified {@code repoName} in the specified {@code projectName}. If the {@link User} @@ -747,7 +890,7 @@ public CompletableFuture findRepositoryRole(String projectName, requireNonNull(projectName, "projectName"); requireNonNull(repoName, "repoName"); requireNonNull(user, "user"); - if (user.isAdmin()) { + if (user.isSystemAdmin()) { return CompletableFuture.completedFuture(RepositoryRole.ADMIN); } if (user instanceof UserWithToken) { @@ -830,7 +973,7 @@ public CompletableFuture findProjectRole(String projectName, User u requireNonNull(projectName, "projectName"); requireNonNull(user, "user"); - if (user.isAdmin()) { + if (user.isSystemAdmin()) { return CompletableFuture.completedFuture(ProjectRole.OWNER); } return getProject(projectName).thenApply(project -> { @@ -862,11 +1005,11 @@ public CompletableFuture createToken(Author author, String appId) { } /** - * Creates a new {@link Token} with the specified {@code appId}, {@code isAdmin} and an auto-generated + * Creates a new {@link Token} with the specified {@code appId}, {@code isSystemAdmin} and an auto-generated * secret. */ - public CompletableFuture createToken(Author author, String appId, boolean isAdmin) { - return createToken(author, appId, SECRET_PREFIX + UUID.randomUUID(), isAdmin); + public CompletableFuture createToken(Author author, String appId, boolean isSystemAdmin) { + return createToken(author, appId, SECRET_PREFIX + UUID.randomUUID(), isSystemAdmin); } /** @@ -877,17 +1020,17 @@ public CompletableFuture createToken(Author author, String appId, Stri } /** - * Creates a new {@link Token} with the specified {@code appId}, {@code secret} and {@code isAdmin}. + * Creates a new {@link Token} with the specified {@code appId}, {@code secret} and {@code isSystemAdmin}. */ public CompletableFuture createToken(Author author, String appId, String secret, - boolean isAdmin) { + boolean isSystemAdmin) { requireNonNull(author, "author"); requireNonNull(appId, "appId"); requireNonNull(secret, "secret"); checkArgument(secret.startsWith(SECRET_PREFIX), "secret must start with: " + SECRET_PREFIX); - final Token newToken = new Token(appId, secret, isAdmin, UserAndTimestamp.of(author)); + final Token newToken = new Token(appId, secret, isSystemAdmin, UserAndTimestamp.of(author)); final JsonPointer appIdPath = JsonPointer.compile("/appIds" + encodeSegment(newToken.id())); final String newTokenSecret = newToken.secret(); assert newTokenSecret != null; @@ -910,23 +1053,36 @@ public CompletableFuture destroyToken(Author author, String appId) { requireNonNull(author, "author"); requireNonNull(appId, "appId"); - return tokenRepo.push(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, author, - "Delete the token: " + appId, - () -> tokenRepo - .fetch(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, TOKEN_JSON) - .thenApply(tokens -> { - final JsonPointer deletionPath = - JsonPointer.compile("/appIds" + encodeSegment(appId) + - "/deletion"); - final Change change = Change.ofJsonPatch( - TOKEN_JSON, - asJsonArray(new TestAbsenceOperation(deletionPath), - new AddOperation(deletionPath, - Jackson.valueToTree( - UserAndTimestamp.of( - author))))); - return HolderWithRevision.of(change, tokens.revision()); - })); + final String commitSummary = "Destroy the token: " + appId; + final UserAndTimestamp userAndTimestamp = UserAndTimestamp.of(author); + + final TokensTransformer transformer = new TokensTransformer((headRevision, tokens) -> { + final Token token = tokens.get(appId); // Raise an exception if not found. + if (token.deletion() != null) { + throw new ChangeConflictException("The token is already destroyed: " + appId); + } + + final String secret = token.secret(); + assert secret != null; + final Token newToken = new Token(token.appId(), secret, token.isSystemAdmin(), + token.isSystemAdmin(), token.creation(), + token.deactivation(), userAndTimestamp); + return new Tokens(newAppIds(tokens, appId, newToken), tokens.secrets()); + }); + return tokenRepo.push(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, author, commitSummary, transformer); + } + + private static Map newAppIds(Tokens tokens, String appId, Token newToken) { + final ImmutableMap.Builder appIdsBuilder = + ImmutableMap.builderWithExpectedSize(tokens.appIds().size()); + for (Entry entry : tokens.appIds().entrySet()) { + if (!entry.getKey().equals(appId)) { + appIdsBuilder.put(entry); + } else { + appIdsBuilder.put(appId, newToken); + } + } + return appIdsBuilder.build(); } /** @@ -939,7 +1095,7 @@ public Revision purgeToken(Author author, String appId) { requireNonNull(appId, "appId"); final Collection projects = listProjectsWithoutInternal(projectManager.list(), - User.ADMIN).values(); + User.SYSTEM_ADMIN).values(); // Remove the token from projects that only have the token. for (Project project : projects) { final ProjectMetadata projectMetadata = fetchMetadata(project.name()).join().object(); @@ -953,23 +1109,22 @@ public Revision purgeToken(Author author, String appId) { } } - return tokenRepo.push(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, author, "Remove the token: " + appId, - () -> tokenRepo.fetch(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, TOKEN_JSON) - .thenApply(tokens -> { - final Token token = tokens.object().get(appId); - final JsonPointer appIdPath = - JsonPointer.compile("/appIds" + encodeSegment(appId)); - final String secret = token.secret(); - assert secret != null; - final JsonPointer secretPath = - JsonPointer.compile( - "/secrets" + encodeSegment(secret)); - final Change change = Change.ofJsonPatch( - TOKEN_JSON, - asJsonArray(new RemoveOperation(appIdPath), - new RemoveIfExistsOperation(secretPath))); - return HolderWithRevision.of(change, tokens.revision()); - })).join(); + final String commitSummary = "Remove the token: " + appId; + + final TokensTransformer transformer = new TokensTransformer((headRevision, tokens) -> { + final Token token = tokens.get(appId); + final Map newAppIds = tokens.appIds().entrySet().stream() + .filter(entry -> !entry.getKey().equals(appId)) + .collect(toImmutableMap(Entry::getKey, Entry::getValue)); + final String secret = token.secret(); + assert secret != null; + final Map newSecrets = + tokens.secrets().entrySet().stream().filter(entry -> !entry.getKey().equals(secret)) + .collect(toImmutableMap(Entry::getKey, Entry::getValue)); + return new Tokens(newAppIds, newSecrets); + }); + return tokenRepo.push(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, author, commitSummary, transformer) + .join(); } /** @@ -979,28 +1134,23 @@ public CompletableFuture activateToken(Author author, String appId) { requireNonNull(author, "author"); requireNonNull(appId, "appId"); - return tokenRepo.push(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, author, - "Enable the token: " + appId, - () -> tokenRepo - .fetch(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, TOKEN_JSON) - .thenApply(tokens -> { - final Token token = tokens.object().get(appId); - final JsonPointer removalPath = - JsonPointer.compile("/appIds" + encodeSegment(appId) + - "/deactivation"); - final String secret = token.secret(); - assert secret != null; - final JsonPointer secretPath = - JsonPointer.compile("/secrets" + - encodeSegment(secret)); - final Change change = Change.ofJsonPatch( - TOKEN_JSON, - asJsonArray(new RemoveOperation(removalPath), - new AddOperation(secretPath, - Jackson.valueToTree(appId)))); - return HolderWithRevision.of(change, tokens.revision()); - }) - ); + final String commitSummary = "Enable the token: " + appId; + + final TokensTransformer transformer = new TokensTransformer((headRevision, tokens) -> { + final Token token = tokens.get(appId); // Raise an exception if not found. + if (token.deactivation() == null) { + throw new RedundantChangeException(headRevision, "The token is already activated: " + appId); + } + final String secret = token.secret(); + assert secret != null; + final Map newSecrets = + ImmutableMap.builderWithExpectedSize(tokens.secrets().size() + 1) + .putAll(tokens.secrets()) + .put(secret, appId).build(); + final Token newToken = new Token(token.appId(), secret, token.isSystemAdmin(), token.creation()); + return new Tokens(newAppIds(tokens, appId, newToken), newSecrets); + }); + return tokenRepo.push(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, author, commitSummary, transformer); } /** @@ -1010,62 +1160,47 @@ public CompletableFuture deactivateToken(Author author, String appId) requireNonNull(author, "author"); requireNonNull(appId, "appId"); - return tokenRepo.push(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, author, - "Disable the token: " + appId, - () -> tokenRepo - .fetch(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, TOKEN_JSON) - .thenApply(tokens -> { - final Token token = tokens.object().get(appId); - final JsonPointer removalPath = - JsonPointer.compile("/appIds" + encodeSegment(appId) + - "/deactivation"); - final String secret = token.secret(); - assert secret != null; - final JsonPointer secretPath = - JsonPointer.compile("/secrets" + - encodeSegment(secret)); - final Change change = Change.ofJsonPatch( - TOKEN_JSON, - asJsonArray(new TestAbsenceOperation(removalPath), - new AddOperation(removalPath, Jackson.valueToTree( - UserAndTimestamp.of(author))), - new RemoveOperation(secretPath))); - return HolderWithRevision.of(change, tokens.revision()); - })); + final String commitSummary = "Deactivate the token: " + appId; + final UserAndTimestamp userAndTimestamp = UserAndTimestamp.of(author); + + final TokensTransformer transformer = new TokensTransformer((headRevision, tokens) -> { + final Token token = tokens.get(appId); + if (token.deactivation() != null) { + throw new RedundantChangeException(headRevision, "The token is already deactivated: " + appId); + } + final String secret = token.secret(); + assert secret != null; + final Token newToken = new Token(token.appId(), secret, token.isSystemAdmin(), + token.isSystemAdmin(), token.creation(), userAndTimestamp, null); + final Map newAppIds = newAppIds(tokens, appId, newToken); + final Map newSecrets = + tokens.secrets().entrySet().stream().filter(entry -> !entry.getKey().equals(secret)) + .collect(toImmutableMap(Entry::getKey, Entry::getValue)); + return new Tokens(newAppIds, newSecrets); + }); + return tokenRepo.push(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, author, commitSummary, transformer); } /** * Update the {@link Token} of the specified {@code appId} to user or admin. */ - public CompletableFuture updateTokenLevel(Author author, String appId, boolean toBeAdmin) { + public CompletableFuture updateTokenLevel(Author author, String appId, boolean toBeSystemAdmin) { requireNonNull(author, "author"); requireNonNull(appId, "appId"); + final String commitSummary = + "Update the token level: " + appId + " to " + (toBeSystemAdmin ? "admin" : "user"); + final TokensTransformer transformer = new TokensTransformer((headRevision, tokens) -> { + final Token token = tokens.get(appId); // Raise an exception if not found. + if (toBeSystemAdmin == token.isSystemAdmin()) { + throw new RedundantChangeException( + headRevision, + "The token is already " + (toBeSystemAdmin ? "admin" : "user")); + } - return tokenRepo.push(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, author, - "Update the token level: " + appId + " to " + (toBeAdmin ? "admin" : "user"), - () -> tokenRepo.fetch(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, TOKEN_JSON) - .thenApply(tokens -> { - final Tokens tokens0 = tokens.object(); - final Token token = tokens0.get(appId); - if (token == null) { - throw new TokenNotFoundException("App ID: " + appId); - } - if (toBeAdmin == token.isAdmin()) { - throw new IllegalArgumentException( - "The token is already " + - (toBeAdmin ? "admin" : "user")); - } - final JsonPointer path = JsonPointer.compile( - "/appIds" + encodeSegment(appId)); - - final Change change = Change.ofJsonPatch( - TOKEN_JSON, - new ReplaceOperation( - path, - Jackson.valueToTree(token.withAdmin( - toBeAdmin))).toJsonNode()); - return HolderWithRevision.of(change, tokens.revision()); - })); + return new Tokens(newAppIds(tokens, appId, token.withSystemAdmin(toBeSystemAdmin)), + tokens.secrets()); + }); + return tokenRepo.push(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, author, commitSummary, transformer); } /** @@ -1094,8 +1229,9 @@ private static void ensureProjectMember(ProjectMetadata project, User user) { requireNonNull(project, "project"); requireNonNull(user, "user"); - checkArgument(project.members().values().stream().anyMatch(member -> member.login().equals(user.id())), - user.id() + " is not a member of the project '" + project.name() + '\''); + if (project.members().values().stream().noneMatch(member -> member.login().equals(user.id()))) { + throw new MemberNotFoundException(user.id(), project.name()); + } } /** @@ -1105,42 +1241,9 @@ private static void ensureProjectToken(ProjectMetadata project, String appId) { requireNonNull(project, "project"); requireNonNull(appId, "appId"); - checkArgument(project.tokens().containsKey(appId), - appId + " is not a token of the project '" + project.name() + '\''); - } - - /** - * Generates the path of {@link JsonPointer} of repository role of the specified {@code memberId} in the - * specified {@code repoName}. - */ - private static JsonPointer repositoryUserRolePointer(String repoName, String memberId) { - // e.g. - // { - // "repos": { - // "my-repo": { - // "name": "my-repo", - // "roles": { - // "users": { - // "my-user@linecorp.com": "READ" - // } - // "tokens": { - // "my-token": "WRITE" - // } - // "projects": {...} - // } - // } - // } - // } - return JsonPointer.compile("/repos" + encodeSegment(repoName) + "/roles/users" + - encodeSegment(memberId)); - } - - /** - * Generates the path of {@link JsonPointer} of repository role of the specified token {@code appId} - * in the specified {@code repoName}. - */ - private static JsonPointer repositoryTokenRolePointer(String repoName, String appId) { - return JsonPointer.compile("/repos" + encodeSegment(repoName) + "/roles/tokens" + - encodeSegment(appId)); + if (!project.tokens().containsKey(appId)) { + throw new TokenNotFoundException( + appId + " is not a token of the project '" + project.name() + '\''); + } } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/PerRolePermissions.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/PerRolePermissions.java index 213f83087b..4cf2eae32f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/PerRolePermissions.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/PerRolePermissions.java @@ -19,7 +19,6 @@ import static java.util.Objects.requireNonNull; import java.util.Collection; -import java.util.EnumSet; import java.util.Objects; import java.util.Set; @@ -28,6 +27,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import com.linecorp.centraldogma.common.ProjectRole; @@ -36,16 +36,14 @@ /** * A default permission for a {@link Repository}. */ -public class PerRolePermissions { +public final class PerRolePermissions { /** * {@link Permission}s for administrators. */ - public static final Collection ALL_PERMISSION = EnumSet.allOf(Permission.class); - - public static final Collection READ_WRITE = EnumSet.of(Permission.READ, Permission.WRITE); - public static final Collection READ_ONLY = EnumSet.of(Permission.READ); - public static final Collection NO_PERMISSION = EnumSet.noneOf(Permission.class); + public static final Collection READ_WRITE = ImmutableList.of(Permission.READ, Permission.WRITE); + public static final Collection READ_ONLY = ImmutableList.of(Permission.READ); + public static final Collection NO_PERMISSION = ImmutableList.of(); /** * The default permission. diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/ProjectMetadata.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/ProjectMetadata.java index 36354b0e54..2a1fec8f60 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/ProjectMetadata.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/ProjectMetadata.java @@ -29,7 +29,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.MoreObjects; -import com.linecorp.centraldogma.common.EntryNotFoundException; import com.linecorp.centraldogma.common.RepositoryNotFoundException; import com.linecorp.centraldogma.server.storage.project.Project; @@ -163,7 +162,7 @@ public Member member(String memberId) { if (member != null) { return member; } - throw new EntryNotFoundException(memberId); + throw new MemberNotFoundException(memberId, name()); } /** diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/ProjectMetadataTransformer.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/ProjectMetadataTransformer.java new file mode 100644 index 0000000000..574f5fa05a --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/ProjectMetadataTransformer.java @@ -0,0 +1,47 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.metadata; + +import static com.linecorp.centraldogma.server.metadata.MetadataService.METADATA_JSON; + +import java.util.function.BiFunction; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; + +import com.linecorp.centraldogma.common.EntryType; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.server.command.ContentTransformer; + +class ProjectMetadataTransformer extends ContentTransformer { + + ProjectMetadataTransformer(BiFunction transformer) { + super(METADATA_JSON, EntryType.JSON, + (headRevision, jsonNode) -> Jackson.valueToTree( + transformer.apply(headRevision, projectMetadata(jsonNode)))); + } + + private static ProjectMetadata projectMetadata(JsonNode node) { + try { + return Jackson.treeToValue(node, ProjectMetadata.class); + } catch (JsonParseException | JsonMappingException e) { + // Should never reach here. + throw new Error(); + } + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/ProjectRoles.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/ProjectRoles.java index 6b7d45843b..ad2580aa11 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/ProjectRoles.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/ProjectRoles.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.centraldogma.common.RepositoryRole; @@ -27,10 +28,15 @@ */ public final class ProjectRoles { + private static final ProjectRoles EMPTY = new ProjectRoles(null, null); + /** * Returns a new {@link ProjectRoles} with the specified {@link RepositoryRole}s. */ public static ProjectRoles of(@Nullable RepositoryRole member, @Nullable RepositoryRole guest) { + if (member == null && guest == null) { + return EMPTY; + } return new ProjectRoles(member, guest); } @@ -68,6 +74,23 @@ public RepositoryRole guest() { return guest; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ProjectRoles)) { + return false; + } + final ProjectRoles that = (ProjectRoles) o; + return member == that.member && guest == that.guest; + } + + @Override + public int hashCode() { + return Objects.hashCode(member, guest); + } + @Override public String toString() { return MoreObjects.toStringHelper(this).omitNullValues() diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadata.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadata.java index 915d2a3555..ae2d5d06cb 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadata.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadata.java @@ -18,6 +18,8 @@ import static java.util.Objects.requireNonNull; +import java.util.Objects; + import javax.annotation.Nullable; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -157,6 +159,27 @@ public QuotaConfig writeQuota() { return writeQuota; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + final RepositoryMetadata that = (RepositoryMetadata) o; + return name.equals(that.name) && + roles.equals(that.roles) && + creation.equals(that.creation) && Objects.equals(removal, that.removal) && + Objects.equals(writeQuota, that.writeQuota); + } + + @Override + public int hashCode() { + return Objects.hash(name, roles, creation, removal, writeQuota); + } + @Override public String toString() { return MoreObjects.toStringHelper(this) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadataDeserializer.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadataDeserializer.java index 31445b9580..7c3e31f89a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadataDeserializer.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadataDeserializer.java @@ -54,6 +54,7 @@ public RepositoryMetadata deserialize(JsonParser p, DeserializationContext ctxt) final Roles roles; if (perRolePermissionsNode != null) { + assert jsonNode.get("roles") == null; // legacy format final PerRolePermissions perRolePermissions = Jackson.treeToValue(perRolePermissionsNode, PerRolePermissions.class); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadataTransformer.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadataTransformer.java new file mode 100644 index 0000000000..7e2029af95 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadataTransformer.java @@ -0,0 +1,57 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.metadata; + +import java.util.Map.Entry; +import java.util.function.BiFunction; + +import com.google.common.collect.ImmutableMap; + +import com.linecorp.centraldogma.common.Revision; + +final class RepositoryMetadataTransformer extends ProjectMetadataTransformer { + + RepositoryMetadataTransformer(String repoName, + BiFunction transformer) { + super((headRevision, projectMetadata) -> { + final RepositoryMetadata repositoryMetadata = projectMetadata.repo(repoName); + assert repositoryMetadata.name().equals(repoName); + final RepositoryMetadata newRepositoryMetadata = + transformer.apply(headRevision, repositoryMetadata); + return newProjectMetadata(projectMetadata, newRepositoryMetadata); + }); + } + + private static ProjectMetadata newProjectMetadata(ProjectMetadata projectMetadata, + RepositoryMetadata repositoryMetadata) { + final ImmutableMap.Builder builder = + ImmutableMap.builderWithExpectedSize(projectMetadata.repos().size()); + for (Entry entry : projectMetadata.repos().entrySet()) { + if (entry.getKey().equals(repositoryMetadata.name())) { + builder.put(entry.getKey(), repositoryMetadata); + } else { + builder.put(entry); + } + } + final ImmutableMap newRepos = builder.build(); + return new ProjectMetadata(projectMetadata.name(), + newRepos, + projectMetadata.members(), + projectMetadata.tokens(), + projectMetadata.creation(), + projectMetadata.removal()); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositorySupport.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositorySupport.java index 7cea106684..0b565b057a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositorySupport.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositorySupport.java @@ -16,13 +16,10 @@ package com.linecorp.centraldogma.server.metadata; -import static com.linecorp.armeria.common.util.Functions.voidFunction; import static java.util.Objects.requireNonNull; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; import java.util.function.Function; -import java.util.function.Supplier; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableList; @@ -30,7 +27,6 @@ import com.linecorp.armeria.common.util.Exceptions; import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.Change; -import com.linecorp.centraldogma.common.ChangeConflictException; import com.linecorp.centraldogma.common.Entry; import com.linecorp.centraldogma.common.Markup; import com.linecorp.centraldogma.common.RedundantChangeException; @@ -39,6 +35,7 @@ import com.linecorp.centraldogma.server.command.Command; import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.command.CommitResult; +import com.linecorp.centraldogma.server.command.ContentTransformer; import com.linecorp.centraldogma.server.storage.project.ProjectManager; import com.linecorp.centraldogma.server.storage.repository.Repository; @@ -109,59 +106,32 @@ private CompletableFuture push(String projectName, String repoName, Au requireNonNull(commitSummary, "commitSummary"); requireNonNull(changes, "changes"); - return executor.execute( - Command.push(author, projectName, repoName, revision, commitSummary, "", - Markup.PLAINTEXT, changes)) + return executor.execute(Command.push(author, projectName, repoName, revision, commitSummary, "", + Markup.PLAINTEXT, changes)) .thenApply(CommitResult::revision); } - CompletableFuture push(String projectName, String repoName, Author author, String commitSummary, - Supplier>>> changeSupplier) { + CompletableFuture push(String projectName, String repoName, + Author author, String commitSummary, + ContentTransformer transformer) { requireNonNull(projectName, "projectName"); requireNonNull(repoName, "repoName"); requireNonNull(author, "author"); requireNonNull(commitSummary, "commitSummary"); - requireNonNull(changeSupplier, "changeSupplier"); - - final CompletableFuture future = new CompletableFuture<>(); - push(projectName, repoName, author, commitSummary, changeSupplier, future); - return future; - } - - private void push(String projectName, String repoName, Author author, String commitSummary, - Supplier>>> changeSupplier, - CompletableFuture future) { - changeSupplier.get().thenAccept(changeWithRevision -> { - final Revision revision = changeWithRevision.revision(); - final Change change = changeWithRevision.object(); - - push(projectName, repoName, author, commitSummary, change, revision) - .thenAccept(future::complete) - .exceptionally(voidFunction(cause -> { - cause = Exceptions.peel(cause); - if (cause instanceof ChangeConflictException) { - final Revision latestRevision; - try { - latestRevision = projectManager().get(projectName).repos().get(repoName) - .normalizeNow(Revision.HEAD); - } catch (Throwable cause1) { - future.completeExceptionally(cause1); - return; - } - - if (revision.equals(latestRevision)) { - future.completeExceptionally(cause); - return; - } - // Try again. - push(projectName, repoName, author, commitSummary, changeSupplier, future); - } else if (cause instanceof RedundantChangeException) { - future.complete(revision); - } else { - future.completeExceptionally(cause); - } - })); - }).exceptionally(voidFunction(future::completeExceptionally)); + requireNonNull(transformer, "transformer"); + + return executor.execute(Command.transform(null, author, projectName, repoName, Revision.HEAD, + commitSummary, "", Markup.PLAINTEXT, transformer)) + .thenApply(CommitResult::revision) + .exceptionally(cause -> { + final Throwable peeled = Exceptions.peel(cause); + if (peeled instanceof RedundantChangeException) { + final Revision revision = ((RedundantChangeException) peeled).headRevision(); + assert revision != null; + return revision; + } + return Exceptions.throwUnsafely(peeled); + }); } Revision normalize(Repository repository) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/Roles.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/Roles.java index 73ada7ca00..469a769b7e 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/Roles.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/Roles.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; import com.linecorp.centraldogma.common.RepositoryRole; @@ -72,6 +73,25 @@ public Map tokens() { return tokens; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Roles)) { + return false; + } + final Roles other = (Roles) o; + return projectRoles.equals(other.projectRoles) && + users.equals(other.users) && + tokens.equals(other.tokens); + } + + @Override + public int hashCode() { + return Objects.hashCode(projectRoles, users, tokens); + } + @Override public String toString() { return MoreObjects.toStringHelper(this).omitNullValues() diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/Token.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/Token.java index 358c51eceb..8d642fe794 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/Token.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/Token.java @@ -49,9 +49,9 @@ public final class Token implements Identifiable { private final String secret; /** - * Specifies whether this token is for administrators. + * Specifies whether this token is for system administrators. */ - private final boolean isAdmin; + private final boolean isSystemAdmin; /** * Specifies when this token is created by whom. @@ -67,8 +67,8 @@ public final class Token implements Identifiable { @Nullable private final UserAndTimestamp deletion; - Token(String appId, String secret, boolean isAdmin, UserAndTimestamp creation) { - this(appId, secret, isAdmin, isAdmin, creation, null, null); + Token(String appId, String secret, boolean isSystemAdmin, UserAndTimestamp creation) { + this(appId, secret, null, isSystemAdmin, creation, null, null); } /** @@ -86,16 +86,16 @@ public Token(@JsonProperty("appId") String appId, assert isAdmin != null || isSystemAdmin != null; this.appId = Util.validateFileName(appId, "appId"); this.secret = Util.validateFileName(secret, "secret"); - this.isAdmin = isSystemAdmin != null ? isSystemAdmin : isAdmin; + this.isSystemAdmin = isSystemAdmin != null ? isSystemAdmin : isAdmin; this.creation = requireNonNull(creation, "creation"); this.deactivation = deactivation; this.deletion = deletion; } - private Token(String appId, boolean isAdmin, UserAndTimestamp creation, + private Token(String appId, boolean isSystemAdmin, UserAndTimestamp creation, @Nullable UserAndTimestamp deactivation, @Nullable UserAndTimestamp deletion) { this.appId = Util.validateFileName(appId, "appId"); - this.isAdmin = isAdmin; + this.isSystemAdmin = isSystemAdmin; this.creation = requireNonNull(creation, "creation"); this.deactivation = deactivation; this.deletion = deletion; @@ -125,11 +125,11 @@ public String secret() { } /** - * Returns whether this token has administrative privileges. + * Returns whether this token has system administrative privileges. */ @JsonProperty - public boolean isAdmin() { - return isAdmin; + public boolean isSystemAdmin() { + return isSystemAdmin; } /** @@ -179,7 +179,7 @@ public String toString() { // Do not add "secret" to prevent it from logging. return MoreObjects.toStringHelper(this).omitNullValues() .add("appId", appId()) - .add("isAdmin", isAdmin()) + .add("isSystemAdmin", isSystemAdmin()) .add("creation", creation()) .add("deactivation", deactivation()) .add("deletion", deletion()) @@ -190,19 +190,19 @@ public String toString() { * Returns a new {@link Token} instance without its secret. */ public Token withoutSecret() { - return new Token(appId(), isAdmin(), creation(), deactivation(), deletion()); + return new Token(appId(), isSystemAdmin(), creation(), deactivation(), deletion()); } /** - * Returns a new {@link Token} instance with {@code isAdmin}. + * Returns a new {@link Token} instance without its secret. * This method must be called by the token whose secret is not null. */ - public Token withAdmin(boolean isAdmin) { - if (isAdmin == isAdmin()) { + public Token withSystemAdmin(boolean isSystemAdmin) { + if (isSystemAdmin == isSystemAdmin()) { return this; } final String secret = secret(); assert secret != null; - return new Token(appId(), secret, isAdmin, isAdmin, creation(), deactivation(), deletion()); + return new Token(appId(), secret, null, isSystemAdmin, creation(), deactivation(), deletion()); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/service/TokenNotFoundException.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/TokenNotFoundException.java similarity index 53% rename from server/src/main/java/com/linecorp/centraldogma/server/internal/admin/service/TokenNotFoundException.java rename to server/src/main/java/com/linecorp/centraldogma/server/metadata/TokenNotFoundException.java index dfc041cf4c..f0b839066f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/admin/service/TokenNotFoundException.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/TokenNotFoundException.java @@ -14,28 +14,18 @@ * under the License. */ -package com.linecorp.centraldogma.server.internal.admin.service; +package com.linecorp.centraldogma.server.metadata; -public class TokenNotFoundException extends RuntimeException { +import com.linecorp.centraldogma.common.CentralDogmaException; - private static final long serialVersionUID = 7795045154004749414L; +/** + * A {@link CentralDogmaException} that is raised when failed to find a {@link Token}. + */ +public final class TokenNotFoundException extends CentralDogmaException { - public TokenNotFoundException() {} + private static final long serialVersionUID = 7795045154004749414L; - public TokenNotFoundException(String message) { + TokenNotFoundException(String message) { super(message); } - - public TokenNotFoundException(Throwable cause) { - super(cause); - } - - public TokenNotFoundException(String message, Throwable cause) { - super(message, cause); - } - - protected TokenNotFoundException(String message, Throwable cause, boolean enableSuppression, - boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/Tokens.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/Tokens.java index 6f008358ad..f5d2f4527f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/Tokens.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/Tokens.java @@ -33,8 +33,6 @@ import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableMap; -import com.linecorp.centraldogma.server.internal.admin.service.TokenNotFoundException; - /** * Holds a token map and a secret map for fast lookup. */ diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/TokensTransformer.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/TokensTransformer.java new file mode 100644 index 0000000000..12b6ab2e39 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/TokensTransformer.java @@ -0,0 +1,49 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.metadata; + +import static com.linecorp.centraldogma.server.metadata.MetadataService.TOKEN_JSON; + +import java.util.function.BiFunction; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; + +import com.linecorp.centraldogma.common.EntryType; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.server.command.ContentTransformer; + +class TokensTransformer extends ContentTransformer { + + TokensTransformer(BiFunction transformer) { + super(TOKEN_JSON, EntryType.JSON, + (headRevision, jsonNode) -> Jackson.valueToTree( + transformer.apply(headRevision, tokens(jsonNode)))); + } + + private static Tokens tokens(JsonNode node) { + final Tokens tokens; + try { + tokens = Jackson.treeToValue(node, Tokens.class); + } catch (JsonParseException | JsonMappingException e) { + // Should never reach here. + throw new Error(e); + } + return tokens; + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/User.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/User.java index ce5147f316..8f9a6820e1 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/User.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/User.java @@ -42,22 +42,23 @@ public class User implements Identifiable, Serializable { private static final long serialVersionUID = -5429782019985526549L; private static final String LEVEL_USER_STR = "LEVEL_USER"; - private static final String LEVEL_ADMIN_STR = "LEVEL_ADMIN"; + private static final String LEVEL_SYSTEM_ADMIN_STR = "LEVEL_SYSTEM_ADMIN"; // System-wide roles for a user. It is different from the role in a project. public static final List LEVEL_USER = ImmutableList.of(LEVEL_USER_STR); - public static final List LEVEL_ADMIN = ImmutableList.of(LEVEL_ADMIN_STR, LEVEL_USER_STR); + public static final List LEVEL_SYSTEM_ADMIN = + ImmutableList.of(LEVEL_SYSTEM_ADMIN_STR, LEVEL_USER_STR); public static final User DEFAULT = new User("user@localhost.localdomain", LEVEL_USER); - public static final User ADMIN = new User("admin@localhost.localdomain", LEVEL_ADMIN); - public static final User SYSTEM = new User("system@localhost.localdomain", LEVEL_ADMIN); + public static final User SYSTEM_ADMIN = new User("admin@localhost.localdomain", LEVEL_SYSTEM_ADMIN); + public static final User SYSTEM = new User("system@localhost.localdomain", LEVEL_SYSTEM_ADMIN); private final String login; private final String name; private final String email; private final List roles; - private final boolean isAdmin; + private final boolean isSystemAdmin; /** * Creates a new instance. @@ -71,7 +72,7 @@ public User(@JsonProperty("login") String login, this.name = requireNonNull(name, "name"); this.email = requireNonNull(email, "email"); this.roles = ImmutableList.copyOf(requireNonNull(roles, "roles")); - isAdmin = roles.stream().anyMatch(LEVEL_ADMIN_STR::equals); + isSystemAdmin = roles.stream().anyMatch(LEVEL_SYSTEM_ADMIN_STR::equals); } /** @@ -95,7 +96,7 @@ public User(String login, List roles) { name = Util.emailToUsername(email, "login"); this.roles = ImmutableList.copyOf(roles); - isAdmin = roles.stream().anyMatch(LEVEL_ADMIN_STR::equals); + isSystemAdmin = roles.stream().anyMatch(LEVEL_SYSTEM_ADMIN_STR::equals); } /** @@ -136,10 +137,10 @@ public String id() { } /** - * Returns {@code true} if this user has administrative privileges. + * Returns {@code true} if this user has system administrative privileges. */ - public boolean isAdmin() { - return isAdmin; + public boolean isSystemAdmin() { + return isSystemAdmin; } @Override @@ -167,7 +168,7 @@ public String toString() { .add("name", name()) .add("email", email()) .add("roles", roles()) - .add("isAdmin", isAdmin()) + .add("isSystemAdmin", isSystemAdmin()) .toString(); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/UserWithToken.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/UserWithToken.java index 8d3e5e69a8..d115ced519 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/UserWithToken.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/UserWithToken.java @@ -45,8 +45,8 @@ public Token token() { } @Override - public boolean isAdmin() { - return token.isAdmin(); + public boolean isSystemAdmin() { + return token.isSystemAdmin(); } @Override diff --git a/server/src/main/java/com/linecorp/centraldogma/server/mirror/Mirror.java b/server/src/main/java/com/linecorp/centraldogma/server/mirror/Mirror.java index d4644f7366..61720ef2cf 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/mirror/Mirror.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/mirror/Mirror.java @@ -100,6 +100,12 @@ public interface Mirror { */ boolean enabled(); + /** + * Returns the zone where this {@link Mirror} is pinned to. + */ + @Nullable + String zone(); + /** * Performs the mirroring task. * diff --git a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorContext.java b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorContext.java index 56dae01833..0390b56666 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorContext.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorContext.java @@ -44,13 +44,15 @@ public final class MirrorContext { private final URI remoteUri; @Nullable private final String gitignore; + @Nullable + private final String zone; /** * Creates a new instance. */ public MirrorContext(String id, boolean enabled, @Nullable Cron schedule, MirrorDirection direction, Credential credential, Repository localRepo, String localPath, URI remoteUri, - @Nullable String gitignore) { + @Nullable String gitignore, @Nullable String zone) { this.id = requireNonNull(id, "id"); this.enabled = enabled; this.schedule = schedule; @@ -60,6 +62,7 @@ public MirrorContext(String id, boolean enabled, @Nullable Cron schedule, Mirror this.localPath = requireNonNull(localPath, "localPath"); this.remoteUri = requireNonNull(remoteUri, "remoteUri"); this.gitignore = gitignore; + this.zone = zone; } /** @@ -128,6 +131,14 @@ public String gitignore() { return gitignore; } + /** + * Returns the zone where this mirror is pinned. + */ + @Nullable + public String zone() { + return zone; + } + @Override public String toString() { return MoreObjects.toStringHelper(this).omitNullValues() @@ -140,6 +151,7 @@ public String toString() { .add("localPath", localPath) .add("remoteUri", remoteUri) .add("gitignore", gitignore) + .add("zone", zone) .toString(); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorResult.java b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorResult.java index 429bb2baed..7342af9398 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorResult.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorResult.java @@ -42,6 +42,8 @@ public final class MirrorResult { private final String description; private final Instant triggeredTime; private final Instant completedTime; + @Nullable + private final String zone; /** * Creates a new instance. @@ -53,7 +55,8 @@ public MirrorResult(@JsonProperty("mirrorId") String mirrorId, @JsonProperty("mirrorStatus") MirrorStatus mirrorStatus, @JsonProperty("description") @Nullable String description, @JsonProperty("triggeredTime") Instant triggeredTime, - @JsonProperty("completedTime") Instant completedTime) { + @JsonProperty("completedTime") Instant completedTime, + @JsonProperty("zone") @Nullable String zone) { this.mirrorId = requireNonNull(mirrorId, "mirrorId"); this.projectName = requireNonNull(projectName, "projectName"); this.repoName = requireNonNull(repoName, "repoName"); @@ -61,6 +64,7 @@ public MirrorResult(@JsonProperty("mirrorId") String mirrorId, this.description = description; this.triggeredTime = requireNonNull(triggeredTime, "triggeredTime"); this.completedTime = requireNonNull(completedTime, "completedTime"); + this.zone = zone; } /** @@ -120,6 +124,15 @@ public Instant completedTime() { return completedTime; } + /** + * Returns the zone where the mirroring operation was performed. + */ + @Nullable + @JsonProperty("zone") + public String zone() { + return zone; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -135,13 +148,14 @@ public boolean equals(Object o) { mirrorStatus == that.mirrorStatus && Objects.equals(description, that.description) && triggeredTime.equals(that.triggeredTime) && - completedTime.equals(that.completedTime); + completedTime.equals(that.completedTime) && + Objects.equals(zone, that.zone); } @Override public int hashCode() { return Objects.hash(mirrorId, projectName, repoName, mirrorStatus, description, - triggeredTime, completedTime); + triggeredTime, completedTime, zone); } @Override @@ -155,6 +169,7 @@ public String toString() { .add("description", description) .add("triggeredTime", triggeredTime) .add("completedTime", completedTime) + .add("zone", zone) .toString(); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorTask.java b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorTask.java index d2884e15f3..70233da338 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorTask.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorTask.java @@ -19,8 +19,11 @@ import java.time.Instant; import java.util.Objects; +import javax.annotation.Nullable; + import com.google.common.base.MoreObjects; +import com.linecorp.centraldogma.server.ZoneConfig; import com.linecorp.centraldogma.server.metadata.User; import com.linecorp.centraldogma.server.storage.project.Project; @@ -32,15 +35,19 @@ public final class MirrorTask { private final Mirror mirror; private final User triggeredBy; private final Instant triggeredTime; + @Nullable + private final String currentZone; private final boolean scheduled; /** * Creates a new instance. */ - public MirrorTask(Mirror mirror, User triggeredBy, Instant triggeredTime, boolean scheduled) { + public MirrorTask(Mirror mirror, User triggeredBy, Instant triggeredTime, @Nullable String currentZone, + boolean scheduled) { this.mirror = mirror; this.triggeredTime = triggeredTime; this.triggeredBy = triggeredBy; + this.currentZone = currentZone; this.scheduled = scheduled; } @@ -72,6 +79,15 @@ public Instant triggeredTime() { return triggeredTime; } + /** + * Returns the current zone where the {@link Mirror} is running. + * This value is {@code null} if the {@link ZoneConfig} is not available. + */ + @Nullable + public String currentZone() { + return currentZone; + } + /** * Returns whether the {@link Mirror} is triggered by a scheduler. */ diff --git a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirroringServicePluginConfig.java b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirroringServicePluginConfig.java index b76b4c7f7b..3aa4afde0b 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirroringServicePluginConfig.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirroringServicePluginConfig.java @@ -32,7 +32,7 @@ public final class MirroringServicePluginConfig extends AbstractPluginConfig { public static final MirroringServicePluginConfig INSTANCE = - new MirroringServicePluginConfig(true, null, null, null); + new MirroringServicePluginConfig(true, null, null, null, false); static final int DEFAULT_NUM_MIRRORING_THREADS = 16; static final int DEFAULT_MAX_NUM_FILES_PER_MIRROR = 8192; @@ -41,12 +41,13 @@ public final class MirroringServicePluginConfig extends AbstractPluginConfig { private final int numMirroringThreads; private final int maxNumFilesPerMirror; private final long maxNumBytesPerMirror; + private final boolean zonePinned; /** * Creates a new instance. */ public MirroringServicePluginConfig(boolean enabled) { - this(enabled, null, null, null); + this(enabled, null, null, null, false); } /** @@ -57,7 +58,8 @@ public MirroringServicePluginConfig( @JsonProperty("enabled") @Nullable Boolean enabled, @JsonProperty("numMirroringThreads") @Nullable Integer numMirroringThreads, @JsonProperty("maxNumFilesPerMirror") @Nullable Integer maxNumFilesPerMirror, - @JsonProperty("maxNumBytesPerMirror") @Nullable Long maxNumBytesPerMirror) { + @JsonProperty("maxNumBytesPerMirror") @Nullable Long maxNumBytesPerMirror, + @JsonProperty("zonePinned") boolean zonePinned) { super(enabled); this.numMirroringThreads = firstNonNull(numMirroringThreads, DEFAULT_NUM_MIRRORING_THREADS); checkArgument(this.numMirroringThreads > 0, @@ -68,6 +70,7 @@ public MirroringServicePluginConfig( this.maxNumBytesPerMirror = firstNonNull(maxNumBytesPerMirror, DEFAULT_MAX_NUM_BYTES_PER_MIRROR); checkArgument(this.maxNumBytesPerMirror > 0, "maxNumBytesPerMirror: %s (expected: > 0)", this.maxNumBytesPerMirror); + this.zonePinned = zonePinned; } /** @@ -94,12 +97,21 @@ public long maxNumBytesPerMirror() { return maxNumBytesPerMirror; } + /** + * Returns whether a {@link Mirror} is pinned to a specific zone. + */ + @JsonProperty("zonePinned") + public boolean zonePinned() { + return zonePinned; + } + @Override public String toString() { return MoreObjects.toStringHelper(this) .add("numMirroringThreads", numMirroringThreads) .add("maxNumFilesPerMirror", maxNumFilesPerMirror) .add("maxNumBytesPerMirror", maxNumBytesPerMirror) + .add("zonePinned", zonePinned) .toString(); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/plugin/AllReplicasPlugin.java b/server/src/main/java/com/linecorp/centraldogma/server/plugin/AllReplicasPlugin.java index 7c4ecf1a20..2f0605ef4c 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/plugin/AllReplicasPlugin.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/plugin/AllReplicasPlugin.java @@ -15,8 +15,11 @@ */ package com.linecorp.centraldogma.server.plugin; +import com.linecorp.centraldogma.server.CentralDogmaConfig; + /** - * A Base class for {@link Plugin} whose {@link #target()} is {@link PluginTarget#ALL_REPLICAS}. + * A Base class for {@link Plugin} whose {@link #target(CentralDogmaConfig)} is + * {@link PluginTarget#ALL_REPLICAS}. */ public abstract class AllReplicasPlugin implements Plugin { @@ -26,7 +29,7 @@ public abstract class AllReplicasPlugin implements Plugin { public void init(PluginInitContext pluginInitContext) {} @Override - public final PluginTarget target() { + public final PluginTarget target(CentralDogmaConfig config) { return PluginTarget.ALL_REPLICAS; } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/plugin/Plugin.java b/server/src/main/java/com/linecorp/centraldogma/server/plugin/Plugin.java index f6306e2918..f8bb613be2 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/plugin/Plugin.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/plugin/Plugin.java @@ -27,7 +27,7 @@ public interface Plugin { /** * Returns the {@link PluginTarget} which specifies the replicas that this {@link Plugin} is applied to. */ - PluginTarget target(); + PluginTarget target(CentralDogmaConfig config); /** * Invoked when this {@link Plugin} is supposed to be started. diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/project/Project.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/project/Project.java index 97ec01c216..fb3f6c14d0 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/project/Project.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/project/Project.java @@ -32,12 +32,12 @@ /** * A top-level element in Central Dogma storage model. A project has {@code "dogma"} and {@code "meta"} - * repositories by default which contain project configuration files accessible by administrators + * repositories by default which contain project configuration files accessible by system administrators * and project owners respectively. */ public interface Project { /** - * The repository that contains project configuration files, which are accessible by administrators. + * The repository that contains project configuration files, which are accessible by system administrators. */ String REPO_DOGMA = "dogma"; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/MetaRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/MetaRepository.java index 49526fdc68..f82864a8d5 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/MetaRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/MetaRepository.java @@ -19,8 +19,11 @@ import java.util.List; import java.util.concurrent.CompletableFuture; +import javax.annotation.Nullable; + import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.internal.api.v1.MirrorDto; +import com.linecorp.centraldogma.server.ZoneConfig; import com.linecorp.centraldogma.server.command.Command; import com.linecorp.centraldogma.server.command.CommitResult; import com.linecorp.centraldogma.server.credential.Credential; @@ -63,6 +66,7 @@ default CompletableFuture> mirrors() { * Create a push {@link Command} for the {@link MirrorDto}. */ CompletableFuture> createPushCommand(MirrorDto mirrorDto, Author author, + @Nullable ZoneConfig zoneConfig, boolean update); /** diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java index bd6f6136dc..b374ca42dd 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java @@ -54,6 +54,7 @@ import com.linecorp.centraldogma.common.RevisionRange; import com.linecorp.centraldogma.internal.HistoryConstants; import com.linecorp.centraldogma.server.command.CommitResult; +import com.linecorp.centraldogma.server.command.ContentTransformer; import com.linecorp.centraldogma.server.internal.replication.ReplicationLog; import com.linecorp.centraldogma.server.storage.StorageException; import com.linecorp.centraldogma.server.storage.project.Project; @@ -404,6 +405,14 @@ CompletableFuture commit(Revision baseRevision, long commitTimeMil Author author, String summary, String detail, Markup markup, Iterable> changes, boolean directExecution); + /** + * Adds the content that is transformed by the specified {@link ContentTransformer} to + * this {@link Repository}. + */ + CompletableFuture commit(Revision baseRevision, long commitTimeMillis, + Author author, String summary, String detail, Markup markup, + ContentTransformer transformer); + /** * Get a list of {@link Commit} for given pathPattern. * diff --git a/server/src/test/java/com/linecorp/centraldogma/server/PluginGroupTest.java b/server/src/test/java/com/linecorp/centraldogma/server/PluginGroupTest.java index 5602b614f7..b63ed4ce63 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/PluginGroupTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/PluginGroupTest.java @@ -41,7 +41,7 @@ void confirmPluginsForAllReplicasLoaded() { final CentralDogmaConfig cfg = mock(CentralDogmaConfig.class); final PluginGroup group = PluginGroup.loadPlugins(PluginTarget.ALL_REPLICAS, cfg); assertThat(group).isNotNull(); - confirmPluginStartStop(group.findFirstPlugin(NoopPluginForAllReplicas.class).orElse(null)); + confirmPluginStartStop(group.findFirstPlugin(NoopPluginForAllReplicas.class)); } @Test @@ -49,7 +49,7 @@ void confirmPluginsForLeaderLoaded() { final CentralDogmaConfig cfg = mock(CentralDogmaConfig.class); final PluginGroup group = PluginGroup.loadPlugins(PluginTarget.LEADER_ONLY, cfg); assertThat(group).isNotNull(); - confirmPluginStartStop(group.findFirstPlugin(NoopPluginForLeader.class).orElse(null)); + confirmPluginStartStop(group.findFirstPlugin(NoopPluginForLeader.class)); } @Test @@ -58,19 +58,19 @@ void confirmDefaultMirroringServiceLoadedDependingOnConfig() { when(cfg.pluginConfigMap()).thenReturn(ImmutableMap.of()); final PluginGroup group1 = PluginGroup.loadPlugins(PluginTarget.LEADER_ONLY, cfg); assertThat(group1).isNotNull(); - assertThat(group1.findFirstPlugin(DefaultMirroringServicePlugin.class)).isPresent(); + assertThat(group1.findFirstPlugin(DefaultMirroringServicePlugin.class)).isNotNull(); when(cfg.pluginConfigMap()).thenReturn(ImmutableMap.of( MirroringServicePluginConfig.class, new MirroringServicePluginConfig(true))); final PluginGroup group2 = PluginGroup.loadPlugins(PluginTarget.LEADER_ONLY, cfg); assertThat(group2).isNotNull(); - assertThat(group2.findFirstPlugin(DefaultMirroringServicePlugin.class)).isPresent(); + assertThat(group2.findFirstPlugin(DefaultMirroringServicePlugin.class)).isNotNull(); when(cfg.pluginConfigMap()).thenReturn(ImmutableMap.of( MirroringServicePluginConfig.class, new MirroringServicePluginConfig(false))); final PluginGroup group3 = PluginGroup.loadPlugins(PluginTarget.LEADER_ONLY, cfg); assertThat(group3).isNotNull(); - assertThat(group3.findFirstPlugin(DefaultMirroringServicePlugin.class)).isNotPresent(); + assertThat(group3.findFirstPlugin(DefaultMirroringServicePlugin.class)).isNull(); } /** @@ -83,12 +83,12 @@ void confirmScheduledPurgingServiceLoadedDependingOnConfig() { when(cfg.maxRemovedRepositoryAgeMillis()).thenReturn(1L); final PluginGroup group1 = PluginGroup.loadPlugins(PluginTarget.LEADER_ONLY, cfg); assertThat(group1).isNotNull(); - assertThat(group1.findFirstPlugin(PurgeSchedulingServicePlugin.class)).isPresent(); + assertThat(group1.findFirstPlugin(PurgeSchedulingServicePlugin.class)).isNotNull(); when(cfg.maxRemovedRepositoryAgeMillis()).thenReturn(0L); final PluginGroup group2 = PluginGroup.loadPlugins(PluginTarget.LEADER_ONLY, cfg); assertThat(group2).isNotNull(); - assertThat(group2.findFirstPlugin(PurgeSchedulingServicePlugin.class)).isNotPresent(); + assertThat(group2.findFirstPlugin(PurgeSchedulingServicePlugin.class)).isNull(); } private static void confirmPluginStartStop(@Nullable AbstractNoopPlugin plugin) { diff --git a/server/src/test/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutorTest.java b/server/src/test/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutorTest.java index a097f6d5ba..52fe27dfc5 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutorTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutorTest.java @@ -21,6 +21,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.List; +import java.util.function.BiFunction; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -34,6 +35,7 @@ import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.EntryType; import com.linecorp.centraldogma.common.Markup; import com.linecorp.centraldogma.common.ReadOnlyException; import com.linecorp.centraldogma.common.Revision; @@ -98,7 +100,8 @@ void jsonUpsertPushCommandConvertedIntoJsonPatchWhenApplicable() { Markup.PLAINTEXT, change)) .join(); // The same json upsert. - assertThat(commitResult).isEqualTo(CommitResult.of(new Revision(2), ImmutableList.of(change))); + final Revision previousRevision = commitResult.revision(); + assertThat(commitResult).isEqualTo(CommitResult.of(previousRevision, ImmutableList.of(change))); // Json upsert is converted into json patch. change = Change.ofJsonUpsert("/foo.json", "{\"a\": \"c\"}"); @@ -108,7 +111,7 @@ void jsonUpsertPushCommandConvertedIntoJsonPatchWhenApplicable() { Markup.PLAINTEXT, change)) .join(); - assertThat(commitResult.revision()).isEqualTo(new Revision(3)); + assertThat(commitResult.revision()).isEqualTo(previousRevision.forward(1)); final List> changes = commitResult.changes(); assertThat(changes).hasSize(1); assertThatJson(changes.get(0).content()).isEqualTo( @@ -116,14 +119,14 @@ void jsonUpsertPushCommandConvertedIntoJsonPatchWhenApplicable() { "\"path\":\"/a\"," + "\"oldValue\":\"b\"," + "\"value\":\"c\"}" + - "]"); + ']'); change = Change.ofJsonUpsert("/foo.json", "{\"a\": \"d\"}"); // PushAsIs just uses the json upsert. final Revision revision = executor.execute( new PushAsIsCommand(0L, Author.SYSTEM, TEST_PRJ, TEST_REPO2, Revision.HEAD, "", "", Markup.PLAINTEXT, ImmutableList.of(change))).join(); - assertThat(revision).isEqualTo(new Revision(4)); + assertThat(revision).isEqualTo(previousRevision.forward(2)); } @Test @@ -148,6 +151,8 @@ void shouldPerformAdministrativeCommandWithReadOnly() throws JsonParseException .join() .contentAsJson(); assertThat(json.get("a").asText()).isEqualTo("b"); + executor.execute(Command.updateServerStatus(ServerStatus.WRITABLE)).join(); + assertThat(executor.isWritable()).isTrue(); } @Test @@ -169,4 +174,46 @@ void createInternalProject() { .join(); assertThat(commitResult).isEqualTo(CommitResult.of(new Revision(2), ImmutableList.of(change))); } + + @Test + void transformCommandConvertedIntoJsonPatch() { + final StandaloneCommandExecutor executor = (StandaloneCommandExecutor) extension.executor(); + + // Initial commit. + final Change change = Change.ofJsonUpsert("/bar.json", "{\"a\": \"b\"}"); + CommitResult commitResult = + executor.execute(Command.push( + Author.SYSTEM, TEST_PRJ, TEST_REPO2, Revision.HEAD, "", "", + Markup.PLAINTEXT, change)) + .join(); + // The same json upsert. + final Revision previousRevision = commitResult.revision(); + assertThat(commitResult).isEqualTo(CommitResult.of(previousRevision, ImmutableList.of(change))); + + final BiFunction transformer = (revision, jsonNode) -> { + if (jsonNode.has("a")) { + ((ObjectNode) jsonNode).put("a", "c"); + } + return jsonNode; + }; + final ContentTransformer contentTransformer = + new ContentTransformer<>("/bar.json", EntryType.JSON, transformer); + + commitResult = + executor.execute(Command.transform( + null, Author.SYSTEM, TEST_PRJ, TEST_REPO2, Revision.HEAD, "", "", + Markup.PLAINTEXT, contentTransformer)).join(); + + // Json upsert is converted into json patch. + + assertThat(commitResult.revision()).isEqualTo(previousRevision.forward(1)); + final List> changes = commitResult.changes(); + assertThat(changes).hasSize(1); + assertThatJson(changes.get(0).content()).isEqualTo( + "[{\"op\":\"safeReplace\"," + + "\"path\":\"/a\"," + + "\"oldValue\":\"b\"," + + "\"value\":\"c\"}" + + ']'); + } } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1ListProjectTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1ListProjectTest.java index ae7b4f2055..e3b93c20ce 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1ListProjectTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1ListProjectTest.java @@ -66,7 +66,7 @@ class ProjectServiceV1ListProjectTest { @Override protected void configure(CentralDogmaBuilder builder) { - builder.administrators(TestAuthMessageUtil.USERNAME); + builder.systemAdministrators(TestAuthMessageUtil.USERNAME); builder.authProviderFactory(new TestAuthProviderFactory()); } @@ -76,18 +76,18 @@ protected boolean runForEachTest() { } }; - private BlockingWebClient adminClient; + private BlockingWebClient systemAdminClient; private BlockingWebClient normalClient; @BeforeEach void setUp() throws JsonProcessingException { final URI uri = dogma.httpClient().uri(); - adminClient = WebClient.builder(uri) - .auth(AuthToken.ofOAuth2(sessionId(dogma.httpClient(), - TestAuthMessageUtil.USERNAME, - TestAuthMessageUtil.PASSWORD))) - .build() - .blocking(); + systemAdminClient = WebClient.builder(uri) + .auth(AuthToken.ofOAuth2(sessionId(dogma.httpClient(), + TestAuthMessageUtil.USERNAME, + TestAuthMessageUtil.PASSWORD))) + .build() + .blocking(); normalClient = WebClient.builder(uri) .auth(AuthToken.ofOAuth2(sessionId(dogma.httpClient(), TestAuthMessageUtil.USERNAME2, @@ -104,7 +104,7 @@ void listProjects() { createProject(normalClient, "trustin"); createProject(normalClient, "hyangtack"); createProject(normalClient, "minwoox"); - createProject(adminClient, "jrhee17"); + createProject(systemAdminClient, "jrhee17"); AggregatedHttpResponse aRes = normalClient.get(PROJECTS_PREFIX); assertThat(aRes.headers().status()).isEqualTo(HttpStatus.OK); @@ -156,7 +156,7 @@ void listProjects() { .isEqualTo(String.format(normalUserExpect, "")); // Admin fetches internal project "dogma" as well. - aRes = adminClient.get(PROJECTS_PREFIX); + aRes = systemAdminClient.get(PROJECTS_PREFIX); assertThat(aRes.headers().status()).isEqualTo(HttpStatus.OK); final String adminUserExpect = @@ -258,8 +258,8 @@ void listRemovedProjects() throws IOException { @Test void userRoleWithLoginUser() { - createProject(adminClient, "trustin"); - createProject(adminClient, "hyangtack"); + createProject(systemAdminClient, "trustin"); + createProject(systemAdminClient, "hyangtack"); final Map projects = getProjects(normalClient); assertThat(projects).hasSize(2); @@ -268,11 +268,11 @@ void userRoleWithLoginUser() { .containsExactlyInAnyOrder(ProjectRole.GUEST, ProjectRole.GUEST); AggregatedHttpResponse aRes = - adminClient.prepare() - .post("/api/v1/metadata/trustin/members") - .contentJson(new IdAndProjectRole( - TestAuthMessageUtil.USERNAME2, ProjectRole.MEMBER)) - .execute(); + systemAdminClient.prepare() + .post("/api/v1/metadata/trustin/members") + .contentJson(new IdAndProjectRole( + TestAuthMessageUtil.USERNAME2, ProjectRole.MEMBER)) + .execute(); assertThat(aRes.status()).isEqualTo(HttpStatus.OK); await().untilAsserted(() -> { final Map projects0 = getProjects(normalClient); @@ -280,11 +280,11 @@ void userRoleWithLoginUser() { assertThat(projects0.get("hyangtack").userRole()).isEqualTo(ProjectRole.GUEST); }); - aRes = adminClient.prepare() - .post("/api/v1/metadata/hyangtack/members") - .contentJson(new IdAndProjectRole( - TestAuthMessageUtil.USERNAME2, ProjectRole.OWNER)) - .execute(); + aRes = systemAdminClient.prepare() + .post("/api/v1/metadata/hyangtack/members") + .contentJson(new IdAndProjectRole( + TestAuthMessageUtil.USERNAME2, ProjectRole.OWNER)) + .execute(); assertThat(aRes.status()).isEqualTo(HttpStatus.OK); await().untilAsserted(() -> { final Map projects0 = getProjects(normalClient); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1Test.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1Test.java index d5d7e999ee..7621b52e3f 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1Test.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1Test.java @@ -64,25 +64,25 @@ class ProjectServiceV1Test { @Override protected void configure(CentralDogmaBuilder builder) { - builder.administrators(TestAuthMessageUtil.USERNAME); + builder.systemAdministrators(TestAuthMessageUtil.USERNAME); builder.authProviderFactory(new TestAuthProviderFactory()); } }; private static final ObjectMapper mapper = new ObjectMapper(); - private BlockingWebClient adminClient; + private BlockingWebClient systemAdminClient; private BlockingWebClient normalClient; @BeforeEach void setUp() throws JsonProcessingException, UnknownHostException { final URI uri = dogma.httpClient().uri(); - adminClient = WebClient.builder(uri) - .auth(AuthToken.ofOAuth2(sessionId(dogma.httpClient(), - TestAuthMessageUtil.USERNAME, - TestAuthMessageUtil.PASSWORD))) - .build() - .blocking(); + systemAdminClient = WebClient.builder(uri) + .auth(AuthToken.ofOAuth2(sessionId(dogma.httpClient(), + TestAuthMessageUtil.USERNAME, + TestAuthMessageUtil.PASSWORD))) + .build() + .blocking(); normalClient = WebClient.builder(uri) .auth(AuthToken.ofOAuth2(sessionId(dogma.httpClient(), TestAuthMessageUtil.USERNAME2, @@ -155,9 +155,9 @@ void removeProject() { .status()).isEqualTo(HttpStatus.NO_CONTENT); // Cannot remove internal dogma project. - assertThat(adminClient.delete(PROJECTS_PREFIX + "/dogma") - .headers() - .status()).isEqualTo(HttpStatus.BAD_REQUEST); + assertThat(systemAdminClient.delete(PROJECTS_PREFIX + "/dogma") + .headers() + .status()).isEqualTo(HttpStatus.BAD_REQUEST); } @Test @@ -169,14 +169,14 @@ void removeAbsentProject() { @Test void purgeProject() { removeProject(); - assertThat(adminClient.delete(PROJECTS_PREFIX + "/foo/removed") - .headers() - .status()).isEqualTo(HttpStatus.NO_CONTENT); + assertThat(systemAdminClient.delete(PROJECTS_PREFIX + "/foo/removed") + .headers() + .status()).isEqualTo(HttpStatus.NO_CONTENT); // Illegal access to the internal project. - assertThat(adminClient.delete(PROJECTS_PREFIX + "/dogma/removed") - .headers() - .status()).isEqualTo(HttpStatus.BAD_REQUEST); + assertThat(systemAdminClient.delete(PROJECTS_PREFIX + "/dogma/removed") + .headers() + .status()).isEqualTo(HttpStatus.BAD_REQUEST); } @Test @@ -190,7 +190,7 @@ void unremoveProject() { HttpHeaderNames.CONTENT_TYPE, MediaType.JSON_PATCH); final String unremovePatch = "[{\"op\":\"replace\",\"path\":\"/status\",\"value\":\"active\"}]"; - final AggregatedHttpResponse aRes = adminClient.execute(headers, unremovePatch); + final AggregatedHttpResponse aRes = systemAdminClient.execute(headers, unremovePatch); assertThat(ResponseHeaders.of(aRes.headers()).status()).isEqualTo(HttpStatus.OK); final String expectedJson = '{' + @@ -214,7 +214,7 @@ void unremoveAbsentProject() { "application/json-patch+json"); final String unremovePatch = "[{\"op\":\"replace\",\"path\":\"/status\",\"value\":\"active\"}]"; - final AggregatedHttpResponse aRes = adminClient.execute(headers, unremovePatch); + final AggregatedHttpResponse aRes = systemAdminClient.execute(headers, unremovePatch); assertThat(ResponseHeaders.of(aRes.headers()).status()).isEqualTo(HttpStatus.NOT_FOUND); } } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/AdministrativeServiceTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/SystemAdministrativeServiceTest.java similarity index 99% rename from server/src/test/java/com/linecorp/centraldogma/server/internal/api/AdministrativeServiceTest.java rename to server/src/test/java/com/linecorp/centraldogma/server/internal/api/SystemAdministrativeServiceTest.java index 824e4bd200..5cdf0ef14c 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/AdministrativeServiceTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/SystemAdministrativeServiceTest.java @@ -36,7 +36,7 @@ import com.linecorp.centraldogma.server.management.ServerStatus; import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension; -class AdministrativeServiceTest { +class SystemAdministrativeServiceTest { @RegisterExtension final CentralDogmaExtension dogma = new CentralDogmaExtension() { diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/TokenServiceTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/TokenServiceTest.java index ecd07d4203..74d19095cc 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/TokenServiceTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/TokenServiceTest.java @@ -75,14 +75,14 @@ class TokenServiceTest { @Override protected void configure(CentralDogmaBuilder builder) { - builder.administrators(TestAuthMessageUtil.USERNAME); + builder.systemAdministrators(TestAuthMessageUtil.USERNAME); builder.authProviderFactory(new TestAuthProviderFactory()); } }; - private static final Author adminAuthor = Author.ofEmail("admin@localhost.com"); + private static final Author systemAdminAuthor = Author.ofEmail("systemAdmin@localhost.com"); private static final Author guestAuthor = Author.ofEmail("guest@localhost.com"); - private static final User admin = new User("admin@localhost.com", User.LEVEL_ADMIN); + private static final User systemAdmin = new User("systemAdmin@localhost.com", User.LEVEL_SYSTEM_ADMIN); private static final User guest = new User("guest@localhost.com"); private static final JsonNode activation = Jackson.valueToTree( ImmutableList.of( @@ -97,7 +97,7 @@ protected void configure(CentralDogmaBuilder builder) { private static TokenService tokenService; private static MetadataService metadataService; - private static WebClient adminClient; + private static WebClient systemAdminClient; // ctx is only used for getting the blocking task executor. private final ServiceRequestContext ctx = @@ -112,11 +112,11 @@ static String sessionId(WebClient webClient, String username, String password) @BeforeAll static void setUp() throws JsonMappingException, JsonParseException { final URI uri = dogma.httpClient().uri(); - adminClient = WebClient.builder(uri) - .auth(AuthToken.ofOAuth2(sessionId(dogma.httpClient(), - TestAuthMessageUtil.USERNAME, - TestAuthMessageUtil.PASSWORD))) - .build(); + systemAdminClient = WebClient.builder(uri) + .auth(AuthToken.ofOAuth2(sessionId(dogma.httpClient(), + TestAuthMessageUtil.USERNAME, + TestAuthMessageUtil.PASSWORD))) + .build(); metadataService = new MetadataService(manager.projectManager(), manager.executor()); tokenService = new TokenService(manager.executor(), metadataService); } @@ -126,18 +126,19 @@ public void tearDown() { final Tokens tokens = metadataService.getTokens().join(); tokens.appIds().forEach((appId, token) -> { if (!token.isDeleted()) { - metadataService.destroyToken(adminAuthor, appId); + metadataService.destroyToken(systemAdminAuthor, appId); } - metadataService.purgeToken(adminAuthor, appId); + metadataService.purgeToken(systemAdminAuthor, appId); }); } @Test - void adminToken() { - final Token token = tokenService.createToken("forAdmin1", true, null, adminAuthor, admin).join() + void systemAdminToken() { + final Token token = tokenService.createToken("forAdmin1", true, true, null, + systemAdminAuthor, systemAdmin).join() .content(); assertThat(token.isActive()).isTrue(); - assertThatThrownBy(() -> tokenService.createToken("forAdmin2", true, null, guestAuthor, guest) + assertThatThrownBy(() -> tokenService.createToken("forAdmin2", true, true, null, guestAuthor, guest) .join()) .isInstanceOf(IllegalArgumentException.class); @@ -146,30 +147,32 @@ void adminToken() { metadataService.addToken(Author.SYSTEM, "myPro", "forAdmin1", ProjectRole.OWNER).join(); assertThat(metadataService.getProject("myPro").join().tokens().containsKey("forAdmin1")).isTrue(); - final Collection tokens = tokenService.listTokens(admin).join(); + final Collection tokens = tokenService.listTokens(systemAdmin).join(); assertThat(tokens.stream().filter(t -> !StringUtil.isNullOrEmpty(t.secret()))).hasSize(1); assertThatThrownBy(() -> tokenService.deleteToken(ctx, "forAdmin1", guestAuthor, guest) .join()) .hasCauseInstanceOf(HttpResponseException.class); - assertThat(tokenService.deleteToken(ctx, "forAdmin1", adminAuthor, admin).thenCompose( - unused -> tokenService.purgeToken(ctx, "forAdmin1", adminAuthor, admin)).join()).satisfies( - t -> { + assertThat(tokenService.deleteToken(ctx, "forAdmin1", systemAdminAuthor, systemAdmin) + .thenCompose(unused -> tokenService.purgeToken( + ctx, "forAdmin1", systemAdminAuthor, systemAdmin)).join()) + .satisfies(t -> { assertThat(t.appId()).isEqualTo(token.appId()); - assertThat(t.isAdmin()).isEqualTo(token.isAdmin()); + assertThat(t.isSystemAdmin()).isEqualTo(token.isSystemAdmin()); assertThat(t.creation()).isEqualTo(token.creation()); assertThat(t.isDeleted()).isTrue(); }); - assertThat(tokenService.listTokens(admin).join().size()).isEqualTo(0); + assertThat(tokenService.listTokens(systemAdmin).join().size()).isEqualTo(0); assertThat(metadataService.getProject("myPro").join().tokens().size()).isEqualTo(0); } @Test void userToken() { - final Token userToken1 = tokenService.createToken("forUser1", false, null, adminAuthor, admin) + final Token userToken1 = tokenService.createToken("forUser1", false, false, null, systemAdminAuthor, + systemAdmin) .join().content(); - final Token userToken2 = tokenService.createToken("forUser2", false, null, guestAuthor, guest) + final Token userToken2 = tokenService.createToken("forUser2", false, false, null, guestAuthor, guest) .join().content(); assertThat(userToken1.isActive()).isTrue(); assertThat(userToken2.isActive()).isTrue(); @@ -178,117 +181,126 @@ void userToken() { assertThat(tokens.stream().filter(token -> !StringUtil.isNullOrEmpty(token.secret())).count()) .isEqualTo(0); - assertThat(tokenService.deleteToken(ctx, "forUser1", adminAuthor, admin).thenCompose( - unused -> tokenService.purgeToken(ctx, "forUser1", adminAuthor, admin)).join()).satisfies(t -> { - assertThat(t.appId()).isEqualTo(userToken1.appId()); - assertThat(t.isAdmin()).isEqualTo(userToken1.isAdmin()); - assertThat(t.creation()).isEqualTo(userToken1.creation()); - assertThat(t.deactivation()).isEqualTo(userToken1.deactivation()); - }); - assertThat(tokenService.deleteToken(ctx, "forUser2", guestAuthor, guest).thenCompose( - unused -> tokenService.purgeToken(ctx, "forUser2", guestAuthor, guest)).join()).satisfies(t -> { - assertThat(t.appId()).isEqualTo(userToken2.appId()); - assertThat(t.isAdmin()).isEqualTo(userToken2.isAdmin()); - assertThat(t.creation()).isEqualTo(userToken2.creation()); - assertThat(t.deactivation()).isEqualTo(userToken2.deactivation()); - }); + assertThat(tokenService.deleteToken(ctx, "forUser1", systemAdminAuthor, systemAdmin) + .thenCompose(unused -> tokenService.purgeToken( + ctx, "forUser1", systemAdminAuthor, systemAdmin)).join()) + .satisfies(t -> { + assertThat(t.appId()).isEqualTo(userToken1.appId()); + assertThat(t.isSystemAdmin()).isEqualTo(userToken1.isSystemAdmin()); + assertThat(t.creation()).isEqualTo(userToken1.creation()); + assertThat(t.deactivation()).isEqualTo(userToken1.deactivation()); + }); + assertThat(tokenService.deleteToken(ctx, "forUser2", guestAuthor, guest) + .thenCompose(unused -> tokenService.purgeToken( + ctx, "forUser2", guestAuthor, guest)).join()) + .satisfies(t -> { + assertThat(t.appId()).isEqualTo(userToken2.appId()); + assertThat(t.isSystemAdmin()).isEqualTo(userToken2.isSystemAdmin()); + assertThat(t.creation()).isEqualTo(userToken2.creation()); + assertThat(t.deactivation()).isEqualTo(userToken2.deactivation()); + }); } @Test void nonRandomToken() { - final Token token = tokenService.createToken("forAdmin1", true, "appToken-secret", adminAuthor, - admin) + final Token token = tokenService.createToken("forAdmin1", true, true, "appToken-secret", + systemAdminAuthor, + systemAdmin) .join().content(); assertThat(token.isActive()).isTrue(); - final Collection tokens = tokenService.listTokens(admin).join(); + final Collection tokens = tokenService.listTokens(systemAdmin).join(); assertThat(tokens.stream().filter(t -> !StringUtil.isNullOrEmpty(t.secret()))).hasSize(1); - assertThatThrownBy(() -> tokenService.createToken("forUser1", true, "appToken-secret", guestAuthor, - guest) + assertThatThrownBy(() -> tokenService.createToken("forUser1", true, true, + "appToken-secret", guestAuthor, guest) .join()) .isInstanceOf(IllegalArgumentException.class); final ServiceRequestContext ctx = ServiceRequestContext.of( HttpRequest.of(HttpMethod.DELETE, "/tokens/{appId}/removed")); - tokenService.deleteToken(this.ctx, "forAdmin1", adminAuthor, admin).thenCompose( - unused -> tokenService.purgeToken(ctx, "forAdmin1", adminAuthor, admin)).join(); + tokenService.deleteToken(this.ctx, "forAdmin1", systemAdminAuthor, systemAdmin).thenCompose( + unused -> tokenService.purgeToken(ctx, "forAdmin1", systemAdminAuthor, systemAdmin)).join(); } @Test public void updateToken() { - final Token token = tokenService.createToken("forUpdate", true, null, adminAuthor, admin).join() + final Token token = tokenService.createToken("forUpdate", true, true, null, + systemAdminAuthor, systemAdmin).join() .content(); assertThat(token.isActive()).isTrue(); - tokenService.updateToken(ctx, "forUpdate", deactivation, adminAuthor, admin).join(); + tokenService.updateToken(ctx, "forUpdate", deactivation, systemAdminAuthor, systemAdmin).join(); final Token deactivatedToken = metadataService.findTokenByAppId("forUpdate").join(); assertThat(deactivatedToken.isActive()).isFalse(); - tokenService.updateToken(ctx, "forUpdate", activation, adminAuthor, admin).join(); + tokenService.updateToken(ctx, "forUpdate", activation, systemAdminAuthor, systemAdmin).join(); final Token activatedToken = metadataService.findTokenByAppId("forUpdate").join(); assertThat(activatedToken.isActive()).isTrue(); assertThatThrownBy( () -> tokenService.updateToken(ctx, "forUpdate", Jackson.valueToTree( - ImmutableList.of(ImmutableMap.of())), adminAuthor, admin).join()) + ImmutableList.of(ImmutableMap.of())), systemAdminAuthor, systemAdmin).join()) .isInstanceOf(CompletionException.class) .hasCauseInstanceOf(IllegalArgumentException.class); - tokenService.deleteToken(ctx, "forUpdate", adminAuthor, admin).join(); + tokenService.deleteToken(ctx, "forUpdate", systemAdminAuthor, systemAdmin).join(); final Token deletedToken = metadataService.findTokenByAppId("forUpdate").join(); assertThat(deletedToken.isDeleted()).isTrue(); assertThatThrownBy( - () -> tokenService.updateToken(ctx, "forUpdate", activation, adminAuthor, admin).join()) + () -> tokenService.updateToken(ctx, "forUpdate", activation, + systemAdminAuthor, systemAdmin).join()) .isInstanceOf(CompletionException.class) .hasCauseInstanceOf(IllegalArgumentException.class); } @Test void updateTokenLevel() { - final Token token = tokenService.createToken("forUpdate", false, null, adminAuthor, admin).join() + final Token token = tokenService.createToken("forUpdate", false, false, null, + systemAdminAuthor, systemAdmin).join() .content(); assertThat(token.isActive()).isTrue(); - final Token userToken = tokenService.updateTokenLevel(ctx, "forUpdate", new TokenLevelRequest("ADMIN"), - adminAuthor, admin) + final Token userToken = tokenService.updateTokenLevel( + ctx, "forUpdate", new TokenLevelRequest("SYSTEMADMIN"), + systemAdminAuthor, systemAdmin) .join(); - assertThat(userToken.isAdmin()).isTrue(); + assertThat(userToken.isSystemAdmin()).isTrue(); final Token adminToken = tokenService.updateTokenLevel(ctx, "forUpdate", new TokenLevelRequest("USER"), - adminAuthor, admin) + systemAdminAuthor, systemAdmin) .join(); - assertThat(adminToken.isAdmin()).isFalse(); + assertThat(adminToken.isSystemAdmin()).isFalse(); assertThatThrownBy( () -> tokenService.updateTokenLevel(ctx, "forUpdate", new TokenLevelRequest("INVALID"), - adminAuthor, admin).join()) + systemAdminAuthor, systemAdmin).join()) .isInstanceOf(IllegalArgumentException.class); } @Test void createTokenAndUpdateLevel() throws JsonParseException { - assertThat(adminClient.post(API_V1_PATH_PREFIX + "tokens", - QueryParams.of("appId", "forUpdate", "isAdmin", false), - HttpData.empty()) - .aggregate() - .join() - .headers() - .get(HttpHeaderNames.LOCATION)).isEqualTo("/tokens/forUpdate"); + assertThat(systemAdminClient.post(API_V1_PATH_PREFIX + "tokens", + QueryParams.of("appId", "forUpdate", "isSystemAdmin", false), + HttpData.empty()) + .aggregate() + .join() + .headers() + .get(HttpHeaderNames.LOCATION)).isEqualTo("/tokens/forUpdate"); final RequestHeaders headers = RequestHeaders.of(HttpMethod.PATCH, API_V1_PATH_PREFIX + "tokens/forUpdate/level", HttpHeaderNames.CONTENT_TYPE, MediaType.JSON); - final String body = "{\"level\":\"ADMIN\"}"; - final AggregatedHttpResponse response = adminClient.execute(headers, body).aggregate().join(); + final String body = "{\"level\":\"SYSTEMADMIN\"}"; + final AggregatedHttpResponse response = systemAdminClient.execute(headers, body).aggregate().join(); final JsonNode jsonNode = Jackson.readTree(response.contentUtf8()); assertThat(jsonNode.get("appId").asText()).isEqualTo("forUpdate"); - assertThat(jsonNode.get("admin").asBoolean()).isEqualTo(true); + assertThat(jsonNode.get("systemAdmin").asBoolean()).isEqualTo(true); - final AggregatedHttpResponse response2 = adminClient.execute(headers, body).aggregate().join(); + final AggregatedHttpResponse response2 = systemAdminClient.execute(headers, body).aggregate().join(); assertThat(response2.status()).isEqualTo(HttpStatus.NOT_MODIFIED); } } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirrorTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirrorTest.java index d47fbf0215..da64562f60 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirrorTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirrorTest.java @@ -123,7 +123,7 @@ static T newMirror(String remoteUri, Cron schedule, final Mirror mirror = new CentralDogmaMirrorProvider().newMirror( new MirrorContext(mirrorId, true, schedule, MirrorDirection.LOCAL_TO_REMOTE, - credential, repository, "/", URI.create(remoteUri), null)); + credential, repository, "/", URI.create(remoteUri), null, null)); assertThat(mirror).isInstanceOf(mirrorType); assertThat(mirror.id()).isEqualTo(mirrorId); @@ -141,7 +141,7 @@ static void assertMirrorNull(String remoteUri) { final Credential credential = mock(Credential.class); final Mirror mirror = new CentralDogmaMirrorProvider().newMirror( new MirrorContext("mirror-id", true, EVERY_MINUTE, MirrorDirection.LOCAL_TO_REMOTE, - credential, mock(Repository.class), "/", URI.create(remoteUri), null)); + credential, mock(Repository.class), "/", URI.create(remoteUri), null, null)); assertThat(mirror).isNull(); } } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/ReplicationLogTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/ReplicationLogTest.java index bc82e83f44..effefa96be 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/ReplicationLogTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/ReplicationLogTest.java @@ -30,9 +30,7 @@ import com.linecorp.centraldogma.server.command.CommitResult; import com.linecorp.centraldogma.server.command.NormalizingPushCommand; import com.linecorp.centraldogma.server.command.PushAsIsCommand; -import com.linecorp.centraldogma.testing.internal.FlakyTest; -@FlakyTest class ReplicationLogTest { private static final Author AUTHOR = new Author("foo", "bar@baz.com"); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutorTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutorTest.java index 99a7baeb9d..75f6da9683 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutorTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutorTest.java @@ -42,6 +42,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; @@ -49,9 +50,14 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.function.ThrowingConsumer; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.github.benmanes.caffeine.cache.Cache; import com.google.common.collect.ImmutableList; @@ -59,15 +65,19 @@ import com.linecorp.armeria.common.metric.MoreMeters; import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.EntryType; import com.linecorp.centraldogma.common.Markup; import com.linecorp.centraldogma.common.ReadOnlyException; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.server.QuotaConfig; import com.linecorp.centraldogma.server.command.Command; +import com.linecorp.centraldogma.server.command.CommandType; import com.linecorp.centraldogma.server.command.CommitResult; +import com.linecorp.centraldogma.server.command.ContentTransformer; import com.linecorp.centraldogma.server.command.ForcePushCommand; import com.linecorp.centraldogma.server.command.NormalizingPushCommand; import com.linecorp.centraldogma.server.command.PushAsIsCommand; +import com.linecorp.centraldogma.server.command.TransformCommand; import com.linecorp.centraldogma.server.management.ServerStatus; import com.linecorp.centraldogma.testing.internal.FlakyTest; @@ -75,9 +85,10 @@ class ZooKeeperCommandExecutorTest { private static final Logger logger = LoggerFactory.getLogger(ZooKeeperCommandExecutorTest.class); - - private static final Change pushChange = Change.ofTextUpsert("/foo", "bar"); - private static final Change normalizedChange = Change.ofTextUpsert("/foo", "bar_normalized"); + private static final Change pushChange = Change.ofJsonUpsert("/foo.json", "{\"a\":\"b\"}"); + private static final Change normalizedChange = + Change.ofJsonPatch("/foo.json", + "[{\"op\":\"safeReplace\",\"path\":\"/a\",\"oldValue\":\"b\",\"value\":\"c\"}]"); @Test void testLogWatch() throws Exception { @@ -276,9 +287,10 @@ void hierarchicalQuorums() throws Throwable { } } - @Test @Timeout(120) - void hierarchicalQuorumsWithFailOver() throws Throwable { + @ValueSource(booleans = { true, false }) + @ParameterizedTest + void hierarchicalQuorumsWithFailOver(boolean normalizingPushCommand) throws Throwable { try (Cluster cluster = Cluster.builder() .numReplicas(9) .numGroup(3) @@ -313,18 +325,7 @@ void hierarchicalQuorumsWithFailOver() throws Throwable { cluster.get(1).commandExecutor().stop().join(); cluster.get(4).commandExecutor().stop().join(); - final Command normalizingPushCommand = - Command.push(0L, Author.SYSTEM, "project", "repo1", new Revision(1), - "summary", "detail", - Markup.PLAINTEXT, - ImmutableList.of(pushChange)); - - assert normalizingPushCommand instanceof NormalizingPushCommand; - final PushAsIsCommand asIsCommand = ((NormalizingPushCommand) normalizingPushCommand).asIs( - CommitResult.of(new Revision(2), ImmutableList.of(normalizedChange))); - - assertThat(replica1.commandExecutor().execute(normalizingPushCommand).join().revision()) - .isEqualTo(new Revision(2)); + final PushAsIsCommand asIsCommand = executeCommand(replica1, normalizingPushCommand); final ReplicationLog commandResult2 = replica1.commandExecutor().loadLog(1, false).get(); assertThat(commandResult2.command()).isEqualTo(asIsCommand); assertThat(commandResult2.result()).isInstanceOf(Revision.class); @@ -359,6 +360,47 @@ void hierarchicalQuorumsWithFailOver() throws Throwable { } } + private static PushAsIsCommand executeCommand(Replica replica1, boolean normalizingPush) { + if (normalizingPush) { + final Command normalizingPushCommand = + Command.push(0L, Author.SYSTEM, "project", "repo1", new Revision(1), + "summary", "detail", + Markup.PLAINTEXT, + ImmutableList.of(pushChange)); + + assert normalizingPushCommand instanceof NormalizingPushCommand; + final PushAsIsCommand asIsCommand = ((NormalizingPushCommand) normalizingPushCommand).asIs( + CommitResult.of(new Revision(2), ImmutableList.of(normalizedChange))); + + assertThat(replica1.commandExecutor().execute(normalizingPushCommand).join().revision()) + .isEqualTo(new Revision(2)); + return asIsCommand; + } else { + final BiFunction transformer = (revision, jsonNode) -> { + final JsonNode oldContent = pushChange.content(); + assertThat(jsonNode).isEqualTo(oldContent); + final JsonNode newContent = oldContent.deepCopy(); + ((ObjectNode) newContent).put("a", "c"); + return newContent; + }; + final ContentTransformer contentTransformer = new ContentTransformer<>( + pushChange.path(), EntryType.JSON, transformer); + final Command transformCommand = + Command.transform(0L, Author.SYSTEM, "project", "repo1", new Revision(1), + "summary", "detail", + Markup.PLAINTEXT, contentTransformer); + + assert transformCommand instanceof TransformCommand; + final PushAsIsCommand asIsCommand = + ((TransformCommand) transformCommand).asIs( + CommitResult.of(new Revision(2), ImmutableList.of(normalizedChange))); + + assertThat(replica1.commandExecutor().execute(transformCommand).join().revision()) + .isEqualTo(new Revision(2)); + return asIsCommand; + } + } + @Test void hierarchicalQuorums_writingOnZeroWeightReplica() throws Throwable { try (Cluster cluster = Cluster.builder() @@ -628,12 +670,28 @@ static Function, CompletableFuture> newMockDelegate() { argument = ((ForcePushCommand) argument).delegate(); } if (argument instanceof NormalizingPushCommand) { - if (((NormalizingPushCommand) argument).changes().equals( + final NormalizingPushCommand normalizingPushCommand = + (NormalizingPushCommand) argument; + assertThat(normalizingPushCommand.type()).isSameAs(CommandType.NORMALIZING_PUSH); + if (normalizingPushCommand.changes().equals( ImmutableList.of(pushChange))) { return completedFuture( CommitResult.of(revision, ImmutableList.of(normalizedChange))); } } + + if (argument instanceof TransformCommand) { + final TransformCommand pushCommand = + (TransformCommand) argument; + assertThat(pushCommand.type()).isSameAs(CommandType.TRANSFORM); + final BiFunction transformer = + (BiFunction) pushCommand.transformer() + .transformer(); + final JsonNode applied = transformer.apply(null, pushChange.content()); + assertThat(applied).isEqualTo(JsonNodeFactory.instance.objectNode().put("a", "c")); + return completedFuture( + CommitResult.of(revision, ImmutableList.of(normalizedChange))); + } return completedFuture(CommitResult.of(revision, ImmutableList.of())); }); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataApiServiceTest.java b/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataApiServiceTest.java index c140f06564..8a59ef2b5f 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataApiServiceTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataApiServiceTest.java @@ -59,13 +59,13 @@ class MetadataApiServiceTest { @Override protected void configure(CentralDogmaBuilder builder) { - builder.administrators(TestAuthMessageUtil.USERNAME); + builder.systemAdministrators(TestAuthMessageUtil.USERNAME); builder.authProviderFactory(new TestAuthProviderFactory()); } }; @SuppressWarnings("NotNullFieldNotInitialized") - static WebClient adminClient; + static WebClient systemAdminClient; @BeforeAll static void setUp() throws JsonMappingException, JsonParseException { @@ -76,26 +76,28 @@ static void setUp() throws JsonMappingException, JsonParseException { assertThat(response.status()).isEqualTo(HttpStatus.OK); final String sessionId = Jackson.readValue(response.content().array(), AccessToken.class) .accessToken(); - adminClient = WebClient.builder(dogma.httpClient().uri()) - .auth(AuthToken.ofOAuth2(sessionId)).build(); + systemAdminClient = WebClient.builder(dogma.httpClient().uri()) + .auth(AuthToken.ofOAuth2(sessionId)).build(); RequestHeaders headers = RequestHeaders.of(HttpMethod.POST, PROJECTS_PREFIX, HttpHeaderNames.CONTENT_TYPE, MediaType.JSON); String body = "{\"name\": \"" + PROJECT_NAME + "\"}"; // Create a project. - assertThat(adminClient.execute(headers, body).aggregate().join().status()).isSameAs(HttpStatus.CREATED); + assertThat(systemAdminClient.execute(headers, body).aggregate().join().status()) + .isSameAs(HttpStatus.CREATED); headers = RequestHeaders.of(HttpMethod.POST, PROJECTS_PREFIX + '/' + PROJECT_NAME + "/repos", HttpHeaderNames.CONTENT_TYPE, MediaType.JSON); body = "{\"name\": \"" + REPOSITORY_NAME + "\"}"; // Create a repository. - assertThat(adminClient.execute(headers, body).aggregate().join().status()).isSameAs(HttpStatus.CREATED); + assertThat(systemAdminClient.execute(headers, body).aggregate().join().status()) + .isSameAs(HttpStatus.CREATED); // Create a token final HttpRequest request = HttpRequest.builder() .post("/api/v1/tokens") .content(MediaType.FORM_DATA, "appId=" + APP_ID) .build(); - assertThat(adminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.CREATED); + assertThat(systemAdminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.CREATED); } @Test @@ -110,13 +112,14 @@ void addUpdateAndRemoveProjectMember() throws JsonProcessingException { .patch("/api/v1/metadata/" + PROJECT_NAME + "/members/" + MEMBER_ID) .content(MediaType.JSON_PATCH, Jackson.writeValueAsString(jsonPatch)) .build(); - assertThat(adminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); + assertThat(systemAdminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); // Remove the member request = HttpRequest.builder() .delete("/api/v1/metadata/" + PROJECT_NAME + "/members/" + MEMBER_ID) .build(); - assertThat(adminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.NO_CONTENT); + assertThat(systemAdminClient.execute(request).aggregate().join().status()) + .isSameAs(HttpStatus.NO_CONTENT); } private static void addProjectMember() { @@ -128,7 +131,7 @@ private static void addProjectMember() { "\"role\":\"MEMBER\"" + '}') .build(); - assertThat(adminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); + assertThat(systemAdminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); } @Test @@ -145,13 +148,14 @@ void addUpdateAndRemoveProjectToken() throws JsonProcessingException { .patch("/api/v1/metadata/" + PROJECT_NAME + "/tokens/app_id") .content(MediaType.JSON_PATCH, Jackson.writeValueAsString(jsonPatch)) .build(); - assertThat(adminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); + assertThat(systemAdminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); // Remove the token request = HttpRequest.builder() .delete("/api/v1/metadata/" + PROJECT_NAME + "/tokens/app_id") .build(); - assertThat(adminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.NO_CONTENT); + assertThat(systemAdminClient.execute(request).aggregate().join().status()) + .isSameAs(HttpStatus.NO_CONTENT); } private static void addProjectToken() { @@ -163,7 +167,7 @@ private static void addProjectToken() { "\"role\":\"MEMBER\"" + '}') .build(); - assertThat(adminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); + assertThat(systemAdminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); } @Test @@ -178,7 +182,7 @@ void addUpdateAndRemoveRepositoryUser() throws JsonProcessingException { "\"role\":\"READ\"" + '}') .build(); - assertThat(adminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); + assertThat(systemAdminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); ProjectMetadata projectMetadata = projectMetadata(); assertThat(projectMetadata.repo(REPOSITORY_NAME).roles().users().get(MEMBER_ID)) @@ -188,7 +192,8 @@ void addUpdateAndRemoveRepositoryUser() throws JsonProcessingException { request = HttpRequest.builder() .delete("/api/v1/metadata/" + PROJECT_NAME + "/members/" + MEMBER_ID) .build(); - assertThat(adminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.NO_CONTENT); + assertThat(systemAdminClient.execute(request).aggregate().join().status()) + .isSameAs(HttpStatus.NO_CONTENT); projectMetadata = projectMetadata(); assertThat(projectMetadata.repo(REPOSITORY_NAME).roles().users().get(MEMBER_ID)).isNull(); } @@ -205,7 +210,7 @@ void addUpdateAndRemoveRepositoryToken() throws JsonProcessingException { "\"role\":\"READ\"" + '}') .build(); - assertThat(adminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); + assertThat(systemAdminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); ProjectMetadata projectMetadata = projectMetadata(); assertThat(projectMetadata.repo(REPOSITORY_NAME).roles().tokens().get(APP_ID)) @@ -215,7 +220,8 @@ void addUpdateAndRemoveRepositoryToken() throws JsonProcessingException { request = HttpRequest.builder() .delete("/api/v1/metadata/" + PROJECT_NAME + "/tokens/" + APP_ID) .build(); - assertThat(adminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.NO_CONTENT); + assertThat(systemAdminClient.execute(request).aggregate().join().status()) + .isSameAs(HttpStatus.NO_CONTENT); projectMetadata = projectMetadata(); assertThat(projectMetadata.repo(REPOSITORY_NAME).roles().tokens().get(APP_ID)).isNull(); } @@ -227,11 +233,11 @@ void grantRoleToMemberForMetaRepository() throws Exception { HttpRequest request = HttpRequest.builder() .post("/api/v1/tokens") .content(MediaType.FORM_DATA, - "secret=" + memberToken + "&isAdmin=false&appId=foo") + "secret=" + memberToken + "&isSystemAdmin=false&appId=foo") .build(); - AggregatedHttpResponse res = adminClient.execute(request).aggregate().join(); + AggregatedHttpResponse res = systemAdminClient.execute(request).aggregate().join(); assertThat(res.status()).isEqualTo(HttpStatus.CREATED); - res = adminClient.get("/api/v1/tokens").aggregate().join(); + res = systemAdminClient.get("/api/v1/tokens").aggregate().join(); assertThat(res.contentUtf8()).contains("\"secret\":\"" + memberToken + '"'); // Add as a member to the project @@ -243,7 +249,7 @@ void grantRoleToMemberForMetaRepository() throws Exception { "\"role\":\"MEMBER\"" + '}') .build(); - res = adminClient.execute(request).aggregate().join(); + res = systemAdminClient.execute(request).aggregate().join(); assertThat(res.status()).isSameAs(HttpStatus.OK); final WebClient memberClient = WebClient.builder(dogma.httpClient().uri()) @@ -263,7 +269,7 @@ void grantRoleToMemberForMetaRepository() throws Exception { " \"guest\": null" + '}') .build(); - assertThat(adminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); + assertThat(systemAdminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); // Now the member can access the meta repository. res = memberClient.get("/api/v1/projects/" + PROJECT_NAME + "/repos/meta/list").aggregate().join(); @@ -278,7 +284,7 @@ void grantRoleToMemberForMetaRepository() throws Exception { " \"guest\": null" + '}') .build(); - assertThat(adminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); + assertThat(systemAdminClient.execute(request).aggregate().join().status()).isSameAs(HttpStatus.OK); // Now the member cannot access the meta repository. res = memberClient.get("/api/v1/projects/" + PROJECT_NAME + "/repos/meta/list").aggregate().join(); @@ -288,8 +294,8 @@ void grantRoleToMemberForMetaRepository() throws Exception { private static ProjectMetadata projectMetadata() throws JsonParseException, JsonMappingException { return Jackson.readValue( - adminClient.prepare() - .get("/api/v1/projects/" + PROJECT_NAME) - .execute().aggregate().join().contentUtf8(), ProjectMetadata.class); + systemAdminClient.prepare() + .get("/api/v1/projects/" + PROJECT_NAME) + .execute().aggregate().join().contentUtf8(), ProjectMetadata.class); } } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataServiceTest.java b/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataServiceTest.java index 0d9c2e6984..735e0a3414 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataServiceTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataServiceTest.java @@ -38,6 +38,7 @@ import com.linecorp.centraldogma.common.RepositoryExistsException; import com.linecorp.centraldogma.common.RepositoryNotFoundException; import com.linecorp.centraldogma.common.RepositoryRole; +import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.server.QuotaConfig; import com.linecorp.centraldogma.server.command.Command; import com.linecorp.centraldogma.testing.internal.ProjectManagerExtension; @@ -179,7 +180,8 @@ void repositoryProjectRoles() { assertThat(repositoryMetadata.roles().projectRoles().member()).isSameAs(RepositoryRole.WRITE); assertThat(repositoryMetadata.roles().projectRoles().guest()).isEqualTo(RepositoryRole.WRITE); - mds.updateRepositoryProjectRoles(author, project1, repo1, DEFAULT_PROJECT_ROLES).join(); + final Revision revision = + mds.updateRepositoryProjectRoles(author, project1, repo1, DEFAULT_PROJECT_ROLES).join(); repositoryMetadata = getRepo1(mds); assertThat(repositoryMetadata.roles().projectRoles().member()).isSameAs(RepositoryRole.WRITE); @@ -188,6 +190,11 @@ void repositoryProjectRoles() { assertThat(mds.findRepositoryRole(project1, repo1, owner).join()).isSameAs(RepositoryRole.ADMIN); assertThat(mds.findRepositoryRole(project1, repo1, guest).join()).isNull(); + // Updating the same role is ok. + assertThat(mds.updateRepositoryProjectRoles( + author, project1, repo1, DEFAULT_PROJECT_ROLES).join()) + .isEqualTo(revision); + assertThatThrownBy(() -> mds.updateRepositoryProjectRoles( author, project1, REPO_DOGMA, ProjectRoles.of(RepositoryRole.WRITE, RepositoryRole.WRITE)) .join()) @@ -209,16 +216,26 @@ void userRepositoryRole() { // Not a member yet. assertThatThrownBy(() -> mds.addUserRepositoryRole(author, project1, repo1, user1, RepositoryRole.READ) .join()) - .hasCauseInstanceOf(IllegalArgumentException.class); + .hasCauseInstanceOf(MemberNotFoundException.class); // Be a member of the project. mds.addMember(author, project1, user1, ProjectRole.MEMBER).join(); + // Try once more. + assertThatThrownBy(() -> mds.addMember(author, project1, user1, ProjectRole.MEMBER).join()) + // TODO(minwoox): Consider removing JSON path operation from MetadataService completely. + .hasCauseInstanceOf(ChangeConflictException.class); + + // invalid repo. + assertThatThrownBy(() -> mds.addUserRepositoryRole( + author, project1, "invalid-repo", user1, RepositoryRole.READ).join()) + .hasCauseInstanceOf(RepositoryNotFoundException.class); // A member of the project has no role. assertThat(mds.findRepositoryRole(project1, repo1, user1).join()).isNull(); // Add 'user1' to user repository role. - mds.addUserRepositoryRole(author, project1, repo1, user1, RepositoryRole.READ).join(); + final Revision revision = mds.addUserRepositoryRole(author, project1, repo1, user1, RepositoryRole.READ) + .join(); // Fail due to duplicated addition. assertThatThrownBy(() -> mds.addUserRepositoryRole(author, project1, repo1, user1, RepositoryRole.READ) @@ -227,13 +244,26 @@ void userRepositoryRole() { assertThat(mds.findRepositoryRole(project1, repo1, user1).join()).isSameAs(RepositoryRole.READ); - mds.updateUserRepositoryRole(author, project1, repo1, user1, RepositoryRole.WRITE).join(); + assertThat(mds.updateUserRepositoryRole(author, project1, repo1, user1, RepositoryRole.WRITE) + .join().major()) + .isEqualTo(revision.major() + 1); assertThat(mds.findRepositoryRole(project1, repo1, user1).join()).isSameAs(RepositoryRole.WRITE); - mds.removeUserRepositoryRole(author, project1, repo1, user1).join(); + // Updating the same operation will return the same revision. + assertThat(mds.updateUserRepositoryRole(author, project1, repo1, user1, RepositoryRole.WRITE) + .join().major()) + .isEqualTo(revision.major() + 1); + + // Update invalid user + assertThatThrownBy(() -> mds.updateUserRepositoryRole(author, project1, repo1, + user2, RepositoryRole.WRITE).join()) + .hasCauseInstanceOf(MemberNotFoundException.class); + + assertThat(mds.removeUserRepositoryRole(author, project1, repo1, user1).join().major()) + .isEqualTo(revision.major() + 2); assertThatThrownBy(() -> mds.removeUserRepositoryRole(author, project1, repo1, user1).join()) - .hasCauseInstanceOf(ChangeConflictException.class); + .hasCauseInstanceOf(MemberNotFoundException.class); assertThat(mds.findRepositoryRole(project1, repo1, user1).join()).isNull(); } @@ -244,17 +274,21 @@ void tokenRepositoryRole() { mds.addRepo(author, project1, repo1, ProjectRoles.of(null, null)).join(); mds.createToken(author, app1).join(); + // Try once more. + assertThatThrownBy(() -> mds.createToken(author, app1).join()) + .hasCauseInstanceOf(ChangeConflictException.class); + final Token token = mds.findTokenByAppId(app1).join(); assertThat(token).isNotNull(); // Token 'app2' is not created yet. assertThatThrownBy(() -> mds.addToken(author, project1, app2, ProjectRole.MEMBER).join()) - .hasCauseInstanceOf(IllegalArgumentException.class); + .hasCauseInstanceOf(TokenNotFoundException.class); - // Not a member yet. + // Token is not registered to the project yet. assertThatThrownBy(() -> mds.addTokenRepositoryRole(author, project1, repo1, app1, RepositoryRole.READ) .join()) - .hasCauseInstanceOf(IllegalArgumentException.class); + .hasCauseInstanceOf(TokenNotFoundException.class); assertThat(mds.findRepositoryRole(project1, repo1, app1).join()).isNull(); @@ -264,10 +298,26 @@ void tokenRepositoryRole() { assertThat(mds.findRepositoryRole(project1, repo1, app1).join()).isSameAs(RepositoryRole.READ); - mds.updateTokenRepositoryRole(author, project1, repo1, app1, RepositoryRole.WRITE).join(); + // Try once more + assertThatThrownBy(() -> mds.addToken(author, project1, app1, ProjectRole.MEMBER).join()) + .hasCauseInstanceOf(ChangeConflictException.class); + assertThatThrownBy(() -> mds.addTokenRepositoryRole(author, project1, repo1, app1, RepositoryRole.READ) + .join()) + .hasCauseInstanceOf(ChangeConflictException.class); + final Revision revision = + mds.updateTokenRepositoryRole(author, project1, repo1, app1, RepositoryRole.WRITE).join(); assertThat(mds.findRepositoryRole(project1, repo1, app1).join()).isSameAs(RepositoryRole.WRITE); + // Update invalid token + assertThatThrownBy(() -> mds.updateTokenRepositoryRole(author, project1, repo1, app2, + RepositoryRole.WRITE).join()) + .hasCauseInstanceOf(TokenNotFoundException.class); + + // Update again with the same permission. + assertThat(mds.updateTokenRepositoryRole(author, project1, repo1, app1, RepositoryRole.WRITE).join()) + .isEqualTo(revision); + mds.removeTokenRepositoryRole(author, project1, repo1, app1).join(); assertThatThrownBy(() -> mds.removeTokenRepositoryRole(author, project1, repo1, app1).join()) .hasCauseInstanceOf(ChangeConflictException.class); @@ -297,6 +347,10 @@ void removeMember() { assertThat(mds.findRepositoryRole(project1, repo1, user1).join()).isNull(); assertThat(mds.findRepositoryRole(project1, repo1, user2).join()).isSameAs(RepositoryRole.READ); + + // Remove 'user1' again. + assertThatThrownBy(() -> mds.removeMember(author, project1, user1).join()) + .hasCauseInstanceOf(MemberNotFoundException.class); } @Test @@ -320,6 +374,10 @@ void removeToken() { assertThat(mds.findRepositoryRole(project1, repo1, app1).join()).isNull(); assertThat(mds.findRepositoryRole(project1, repo1, app2).join()).isSameAs(RepositoryRole.READ); + + // Remove 'app1' again. + assertThatThrownBy(() -> mds.removeToken(author, project1, app1).join()) + .hasCauseInstanceOf(TokenNotFoundException.class); } @Test @@ -345,6 +403,10 @@ void destroyToken() { assertThat(mds.findRepositoryRole(project1, repo1, app1).join()).isNull(); assertThat(mds.findRepositoryRole(project1, repo1, app2).join()).isSameAs(RepositoryRole.READ); + + // Remove 'app1' again. + assertThatThrownBy(() -> mds.destroyToken(author, app1).join()) + .hasCauseInstanceOf(TokenNotFoundException.class); } @Test @@ -357,14 +419,20 @@ void tokenActivationAndDeactivation() { assertThat(token).isNotNull(); assertThat(token.creation().user()).isEqualTo(owner.id()); - mds.deactivateToken(author, app1).join(); + final Revision revision = mds.deactivateToken(author, app1).join(); token = mds.getTokens().join().get(app1); assertThat(token.isActive()).isFalse(); assertThat(token.deactivation()).isNotNull(); assertThat(token.deactivation().user()).isEqualTo(owner.id()); - mds.activateToken(author, app1).join(); + // Executing the same operation will return the same revision. + assertThat(mds.deactivateToken(author, app1).join()).isEqualTo(revision); + + assertThat(mds.activateToken(author, app1).join().major()).isEqualTo(revision.major() + 1); assertThat(mds.getTokens().join().get(app1).isActive()).isTrue(); + + // Executing the same operation will return the same revision. + assertThat(mds.activateToken(author, app1).join().major()).isEqualTo(revision.major() + 1); } @Test @@ -394,15 +462,17 @@ void updateUser() { mds.createToken(author, app1).join(); token = mds.getTokens().join().get(app1); assertThat(token).isNotNull(); - assertThat(token.isAdmin()).isFalse(); + assertThat(token.isSystemAdmin()).isFalse(); - mds.updateTokenLevel(author, app1, true).join(); + final Revision revision = mds.updateTokenLevel(author, app1, true).join(); token = mds.getTokens().join().get(app1); - assertThat(token.isAdmin()).isTrue(); + assertThat(token.isSystemAdmin()).isTrue(); + assertThat(mds.updateTokenLevel(author, app1, true).join()).isEqualTo(revision); - mds.updateTokenLevel(author, app1, false).join(); + assertThat(mds.updateTokenLevel(author, app1, false).join()).isEqualTo(revision.forward(1)); token = mds.getTokens().join().get(app1); - assertThat(token.isAdmin()).isFalse(); + assertThat(token.isSystemAdmin()).isFalse(); + assertThat(mds.updateTokenLevel(author, app1, false).join()).isEqualTo(revision.forward(1)); } private static RepositoryMetadata getRepo1(MetadataService mds) { diff --git a/server/src/test/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadataTest.java b/server/src/test/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadataTest.java new file mode 100644 index 0000000000..5866d0dc71 --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadataTest.java @@ -0,0 +1,109 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.metadata; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import com.google.common.collect.ImmutableMap; + +import com.linecorp.centraldogma.common.RepositoryRole; +import com.linecorp.centraldogma.internal.Jackson; + +class RepositoryMetadataTest { + + @Test + void deserializeLegacyFormat() throws Exception { + final String format = '{' + + " \"name\": \"minu-test\"," + + " \"perRolePermissions\": {" + + " \"owner\": [" + + " \"READ\"," + + " \"WRITE\"" + + " ]," + + " \"member\": [\"READ\"]," + + " \"guest\": []" + + " }," + + " \"perUserPermissions\": {" + + " \"bar@dogma.com\": [" + + " \"READ\"," + + " \"WRITE\"" + + " ]," + + " \"foo@dogma.com\": [" + + " \"READ\"" + + " ]" + + " }," + + " \"perTokenPermissions\": {" + + " \"goodman\": [" + + " \"READ\"" + + " ]" + + " }," + + " \"creation\": {" + + " \"user\": \"minu.song@dogma.com\"," + + " \"timestamp\": \"2024-08-19T02:47:23.370762417Z\"" + + " }" + + '}'; + final RepositoryMetadata repositoryMetadata = Jackson.readValue(format, RepositoryMetadata.class); + validate(repositoryMetadata); + // The legacy format is serialized into the new format. + assertThat(Jackson.writeValueAsString(repositoryMetadata)).isEqualTo( + Jackson.writeValueAsString(Jackson.readTree(newFormat()))); + } + + @Test + void deserializeNewFormat() throws Exception { + final String format = newFormat(); + validate(Jackson.readValue(format, RepositoryMetadata.class)); + } + + private static String newFormat() { + return '{' + + " \"name\": \"minu-test\"," + + " \"roles\": {" + + " \"projects\": {" + + " \"member\": \"READ\"," + + " \"guest\": null" + + " }," + + " \"users\": {" + + " \"bar@dogma.com\": \"WRITE\"," + + " \"foo@dogma.com\": \"READ\"" + + " }," + + " \"tokens\": {" + + " \"goodman\": \"READ\"" + + " }" + + " }," + + " \"creation\": {" + + " \"user\": \"minu.song@dogma.com\"," + + " \"timestamp\": \"2024-08-19T02:47:23.370762417Z\"" + + " }" + + '}'; + } + + private static void validate(RepositoryMetadata repositoryMetadata) { + assertThat(repositoryMetadata.id()).isEqualTo("minu-test"); + assertThat(repositoryMetadata.name()).isEqualTo("minu-test"); // id and name are the same. + assertThat(repositoryMetadata.roles().projectRoles()) + .isEqualTo(ProjectRoles.of(RepositoryRole.READ, null)); + assertThat(repositoryMetadata.roles().users()) + .isEqualTo(ImmutableMap.of("foo@dogma.com", RepositoryRole.READ, + "bar@dogma.com", RepositoryRole.WRITE)); + assertThat(repositoryMetadata.roles().tokens()) + .isEqualTo(ImmutableMap.of("goodman", RepositoryRole.READ)); + assertThat(repositoryMetadata.creation()) + .isEqualTo(new UserAndTimestamp("minu.song@dogma.com", "2024-08-19T02:47:23.370762417Z")); + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/metadata/TokenTest.java b/server/src/test/java/com/linecorp/centraldogma/server/metadata/TokenTest.java index 30fd885db7..ce8bc67c6c 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/metadata/TokenTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/metadata/TokenTest.java @@ -15,28 +15,88 @@ */ package com.linecorp.centraldogma.server.metadata; +import static com.linecorp.centraldogma.internal.jsonpatch.JsonPatchOperation.asJsonArray; +import static com.linecorp.centraldogma.server.metadata.MetadataService.TOKEN_JSON; import static org.assertj.core.api.Assertions.assertThat; +import java.util.Collection; + +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonPointer; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.linecorp.armeria.common.HttpMethod; +import com.linecorp.armeria.common.HttpRequest; +import com.linecorp.armeria.server.ServiceRequestContext; +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.internal.jsonpatch.AddOperation; +import com.linecorp.centraldogma.internal.jsonpatch.TestAbsenceOperation; +import com.linecorp.centraldogma.server.internal.api.TokenLevelRequest; +import com.linecorp.centraldogma.server.internal.api.TokenService; +import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.repository.Repository; +import com.linecorp.centraldogma.testing.internal.ProjectManagerExtension; class TokenTest { + @RegisterExtension + static final ProjectManagerExtension manager = new ProjectManagerExtension(); + private static final String APP_ID = "foo-id"; private static final String APP_SECRET = "appToken-foo"; + private static final Author AUTHOR = Author.ofEmail("systemAdmin@localhost.com"); + private static final User USER = new User("systemAdmin@localhost.com", User.LEVEL_SYSTEM_ADMIN); + + private static final ServiceRequestContext CTX = + ServiceRequestContext.builder(HttpRequest.of(HttpMethod.GET, "/")).build(); + + private static TokenService tokenService; + private static MetadataService metadataService; + + @BeforeAll + static void setUp() throws JsonParseException { + metadataService = new MetadataService(manager.projectManager(), manager.executor()); + tokenService = new TokenService(manager.executor(), metadataService); + + // Put the legacy token. + final Repository dogmaRepository = + manager.projectManager().get(InternalProjectInitializer.INTERNAL_PROJECT_DOGMA).repos() + .get(Project.REPO_DOGMA); + final JsonPointer appIdPath = JsonPointer.compile("/appIds/" + APP_ID); + final JsonPointer secretPath = JsonPointer.compile("/secrets/" + APP_SECRET); + final Change change = Change.ofJsonPatch( + TOKEN_JSON, + asJsonArray(new TestAbsenceOperation(appIdPath), + new TestAbsenceOperation(secretPath), + new AddOperation(appIdPath, Jackson.readTree(tokenJson(true))), + new AddOperation(secretPath, Jackson.valueToTree(APP_ID)))); + + dogmaRepository.commit(Revision.HEAD, System.currentTimeMillis(), AUTHOR, + "Add the legacy token", change).join(); + } @Test void deserializeToken() throws Exception { final String legacyTokenJson = tokenJson(true); final Token legacyToken = Jackson.readValue(legacyTokenJson, Token.class); assertThat(legacyToken.appId()).isEqualTo(APP_ID); - assertThat(legacyToken.isAdmin()).isTrue(); + assertThat(legacyToken.isSystemAdmin()).isTrue(); final String tokenJson = tokenJson(false); final Token token = Jackson.readValue(tokenJson, Token.class); assertThat(token.appId()).isEqualTo(APP_ID); - assertThat(token.isAdmin()).isTrue(); + assertThat(token.isSystemAdmin()).isTrue(); } private static String tokenJson(boolean legacy) { @@ -48,4 +108,51 @@ private static String tokenJson(boolean legacy) { " \"timestamp\": \"2018-04-10T09:58:20.032Z\"" + "}}"; } + + @Test + void updateToken() throws JsonParseException { + final Collection tokens = tokenService.listTokens(USER).join(); + assertThat(tokens.size()).isOne(); + final Token token = Iterables.getFirst(tokens, null); + assertThat(token.appId()).isEqualTo(APP_ID); + assertThat(token.isSystemAdmin()).isTrue(); + assertThat(token.isActive()).isTrue(); + + final JsonNode deactivation = Jackson.valueToTree( + ImmutableList.of( + ImmutableMap.of("op", "replace", + "path", "/status", + "value", "inactive"))); + + tokenService.updateToken(CTX, APP_ID, deactivation, AUTHOR, USER).join(); + Token updated = metadataService.findTokenByAppId(APP_ID).join(); + assertThat(updated.appId()).isEqualTo(APP_ID); + assertThat(updated.isSystemAdmin()).isTrue(); + assertThat(updated.isActive()).isFalse(); + + final JsonNode activation = Jackson.valueToTree( + ImmutableList.of( + ImmutableMap.of("op", "replace", + "path", "/status", + "value", "active"))); + + tokenService.updateToken(CTX, APP_ID, activation, AUTHOR, USER).join(); + updated = metadataService.findTokenByAppId(APP_ID).join(); + assertThat(updated.appId()).isEqualTo(APP_ID); + assertThat(updated.isSystemAdmin()).isTrue(); + assertThat(updated.isActive()).isTrue(); + } + + @Test + void updateTokenLevel() { + final Token userToken = + tokenService.updateTokenLevel(CTX, APP_ID, new TokenLevelRequest("USER"), AUTHOR, USER) + .join(); + assertThat(userToken.isSystemAdmin()).isFalse(); + + final Token updatedToken = + tokenService.updateTokenLevel(CTX, APP_ID, new TokenLevelRequest("SYSTEMADMIN"), + AUTHOR, USER).join(); + assertThat(updatedToken.isSystemAdmin()).isTrue(); + } } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/plugin/NoopPluginForAllReplicas.java b/server/src/test/java/com/linecorp/centraldogma/server/plugin/NoopPluginForAllReplicas.java index d9c624e70c..d4809bacff 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/plugin/NoopPluginForAllReplicas.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/plugin/NoopPluginForAllReplicas.java @@ -15,9 +15,11 @@ */ package com.linecorp.centraldogma.server.plugin; +import com.linecorp.centraldogma.server.CentralDogmaConfig; + public class NoopPluginForAllReplicas extends AbstractNoopPlugin { @Override - public PluginTarget target() { + public PluginTarget target(CentralDogmaConfig config) { return PluginTarget.ALL_REPLICAS; } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/plugin/NoopPluginForLeader.java b/server/src/test/java/com/linecorp/centraldogma/server/plugin/NoopPluginForLeader.java index f8d207a0b0..f1055a2365 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/plugin/NoopPluginForLeader.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/plugin/NoopPluginForLeader.java @@ -15,9 +15,11 @@ */ package com.linecorp.centraldogma.server.plugin; +import com.linecorp.centraldogma.server.CentralDogmaConfig; + public class NoopPluginForLeader extends AbstractNoopPlugin { @Override - public PluginTarget target() { + public PluginTarget target(CentralDogmaConfig config) { return PluginTarget.LEADER_ONLY; } diff --git a/site/build.gradle b/site/build.gradle index fcb30e5ff7..9c56061902 100644 --- a/site/build.gradle +++ b/site/build.gradle @@ -15,10 +15,14 @@ if (project.hasProperty("noSite")) { apply plugin: "kr.motd.sphinx" +def osdetector = project.rootProject.osdetector +def skipSphinx = osdetector.os == 'osx' && osdetector.arch == 'aarch_64' + sphinx { group = 'Documentation' description = 'Generates the Sphinx web site.' sourceDirectory "${project.projectDir}/src/sphinx" + skip = skipSphinx } task javadoc(type: Javadoc, diff --git a/site/src/sphinx/auth.rst b/site/src/sphinx/auth.rst index 2a32aad229..a66fc31139 100644 --- a/site/src/sphinx/auth.rst +++ b/site/src/sphinx/auth.rst @@ -31,7 +31,7 @@ The authentication configuration consists of the following properties: ... "authentication": { "factoryClassName": "the fully-qualified class name of an AuthenticationProviderFactory", - "administrators": [], + "systemAdministrators": [], "caseSensitiveLoginNames": false, "sessionCacheSpec": "maximumSize=8192,expireAfterWrite=604800s", "sessionTimeoutMillis": 604800000, @@ -46,9 +46,10 @@ The authentication configuration consists of the following properties: ``com.linecorp.centraldogma.server.auth.saml.SamlAuthProviderFactory`` or ``com.linecorp.centraldogma.server.auth.shiro.ShiroAuthProviderFactory``. -- ``administrators`` (string array) +- ``systemAdministrators`` (string array) - - login names of the administrators. A user who has a login name specified here will get the administrator role. + - login names of the system administrators. A user who has a login name specified here will get the + system administrator role. - ``caseSensitiveLoginNames`` (boolean) @@ -89,7 +90,7 @@ the authentication to. ... "authentication": { "factoryClassName": "com.linecorp.centraldogma.server.auth.saml.SamlAuthProviderFactory", - "administrators": [], + "systemAdministrators": [], "caseSensitiveLoginNames": false, "sessionCacheSpec": "maximumSize=8192,expireAfterWrite=604800s", "sessionTimeoutMillis": 604800000, @@ -225,7 +226,7 @@ in the ``properties`` property. ... "authentication": { "factoryClassName": "com.linecorp.centraldogma.server.auth.shiro.ShiroAuthProviderFactory", - "administrators": [], + "systemAdministrators": [], "caseSensitiveLoginNames": false, "sessionCacheSpec": "maximumSize=8192,expireAfterWrite=604800s", "sessionTimeoutMillis": 604800000, @@ -270,10 +271,10 @@ When you add a user as a member of the project, you need to choose the role of t There are 4 user role types in the access control system of Central Dogma, but you can choose one of ``Owner`` and ``Member`` role in the UI. More information about the role is as follows. -- ``Administrator`` +- ``System Administrator`` - - the user that all permissions are assigned to, a.k.a 'super user'. Only an administrator can restore - removed project. The administrators can be configured in ``conf/dogma.json`` as described the above. + - the user that all permissions are assigned to, a.k.a 'super user'. Only a system administrator can restore + removed project. The system administrators can be configured in ``conf/dogma.json`` as described the above. - ``Owner`` of a project @@ -335,8 +336,8 @@ request comes from. Anyone who is logged into the Central Dogma can create a new ``Application Token``, and the token is shared for everyone. So any owner of a project can add any token to their project. However only both the token -creator and the administrator are allowed to deactivate and/or remove the token. +creator and the system administrator are allowed to deactivate and/or remove the token. -There are two levels of a token, which are ``Admin`` and ``User``. ``Admin`` level token can be created by -only the administrators. A client who sends a request with the token is allowed to access administrator-level -APIs. +There are two levels of a token, which are ``System Admin`` and ``User``. ``System Admin`` level token can be +created by only the system administrators. A client who sends a request with the token is allowed to access +system administrator-level APIs. diff --git a/site/src/sphinx/mirroring.rst b/site/src/sphinx/mirroring.rst index a611e86c8d..ee3c70a473 100644 --- a/site/src/sphinx/mirroring.rst +++ b/site/src/sphinx/mirroring.rst @@ -57,7 +57,8 @@ Setting up a mirroring task "gitignore": [ "/credential.txt", "private_dir" - ] + ], + "zone": "zone1" } - ``id`` (string) @@ -117,6 +118,15 @@ Setting up a mirroring task of strings where each line represents a single pattern. The file pattern expressed in gitignore is relative to the path of ``remoteUri``. +- ``zone`` (string, optional) + + - the zone where the mirroring task is executed. + + - If unspecified: + + - a mirroring task is executed in the first zone of ``zone.allZones`` configuration. + - if ``zone.allZones`` is not configured, a mirroring task is executed in the leader replica. + Setting up a credential ^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/site/src/sphinx/setup-configuration.rst b/site/src/sphinx/setup-configuration.rst index 53a1ef209a..50cd47e79c 100644 --- a/site/src/sphinx/setup-configuration.rst +++ b/site/src/sphinx/setup-configuration.rst @@ -263,16 +263,24 @@ Core properties - the path of the management service. If not specified, the management service is mounted at ``/internal/management``. -- ``zone`` (string) +- ``zone`` - - the zone name of the server. If not specified, ``PluginTarget.ZONE_LEADER_ONLY`` can't be used. + - the zone information of the server. If not specified, ``PluginTarget.ZONE_LEADER_ONLY`` can't be used. - - If the value starts with ``env:``, the environment variable is used as the zone name. + - ``currentZone`` (string) + + - the current zone name. If the value starts with ``env:``, the environment variable is used as the zone name. For example, if the value is ``env:ZONE_NAME``, the environment variable named ``ZONE_NAME`` is used as the zone name. - You can also dynamically load a zone name by implementing :api:`com.linecorp.centraldogma.server.ConfigValueConverter`. + - ``allZones`` (string array) + + - the list of zone names. + + - the current zone name must be included in the list of zone names. + .. _replication: Configuring replication @@ -528,6 +536,7 @@ with ``pluginConfigs`` property in ``dogma.json`` as follows. "numMirroringThreads": null, "maxNumFilesPerMirror": null, "maxNumBytesPerMirror": null, + "zonePinned": false } ] } @@ -555,6 +564,11 @@ properties that can be configured: this, Central Dogma will reject to mirror the Git repository. If ``null``, the default value of '33554432 bytes' (32 MiB) is used. +- ``zonePinned`` (boolean) + + - whether the mirroring plugin is pinned to a specific zone. If ``true``, a mirroring task will be executed + only in the specified zone. If ``false``, the plugin will be executed in the leader replica. + For more information about mirroring, refer to :ref:`mirroring`. .. _hiding_sensitive_property_values: diff --git a/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/CentralDogmaReplicationExtension.java b/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/CentralDogmaReplicationExtension.java index 76d52fcff7..8894792f12 100644 --- a/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/CentralDogmaReplicationExtension.java +++ b/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/CentralDogmaReplicationExtension.java @@ -114,12 +114,19 @@ private List newDogmaCluster(int numReplicas) throws I @Override protected void configure(CentralDogmaBuilder builder) { builder.port(new InetSocketAddress(NetUtil.LOCALHOST4, dogmaPort), SessionProtocol.HTTP) - .administrators(TestAuthMessageUtil.USERNAME) + .systemAdministrators(TestAuthMessageUtil.USERNAME) .authProviderFactory(factory) - .pluginConfigs(new MirroringServicePluginConfig(false)) .gracefulShutdownTimeout(new GracefulShutdownTimeout(0, 0)) .replication(new ZooKeeperReplicationConfig(serverId, zooKeeperServers)); configureEach(serverId, builder); + final boolean isMirrorConfigured = + builder.pluginConfigs() + .stream() + .anyMatch(pluginCfg -> pluginCfg instanceof MirroringServicePluginConfig); + if (!isMirrorConfigured) { + // Disable the mirroring service when it is not explicitly configured. + builder.pluginConfigs(new MirroringServicePluginConfig(false)); + } } @Override diff --git a/testing/common/src/main/java/com/linecorp/centraldogma/testing/internal/CentralDogmaRuleDelegate.java b/testing/common/src/main/java/com/linecorp/centraldogma/testing/internal/CentralDogmaRuleDelegate.java index 30c76f034c..081ada1822 100644 --- a/testing/common/src/main/java/com/linecorp/centraldogma/testing/internal/CentralDogmaRuleDelegate.java +++ b/testing/common/src/main/java/com/linecorp/centraldogma/testing/internal/CentralDogmaRuleDelegate.java @@ -16,6 +16,8 @@ package com.linecorp.centraldogma.testing.internal; +import static com.google.common.base.Preconditions.checkState; + import java.io.File; import java.io.IOError; import java.net.InetSocketAddress; @@ -217,7 +219,9 @@ public ProjectManager projectManager() { * @throws IllegalStateException if Central Dogma did not start yet */ public final MirroringService mirroringService() { - return dogma().mirroringService().get(); + final MirroringService mirroringService = dogma().mirroringService(); + checkState(mirroringService != null, "Mirroring service not available"); + return mirroringService; } /** diff --git a/webapp/src/dogma/common/components/ProjectSearchBox.tsx b/webapp/src/dogma/common/components/ProjectSearchBox.tsx index 3a1eb5f189..6dd8675f7a 100644 --- a/webapp/src/dogma/common/components/ProjectSearchBox.tsx +++ b/webapp/src/dogma/common/components/ProjectSearchBox.tsx @@ -41,7 +41,7 @@ const DropdownIndicator = ( const ProjectSearchBox = ({ id, size, placeholder, autoFocus }: ProjectSearchBoxProps) => { const { colorMode } = useColorMode(); - const { data, isLoading } = useGetProjectsQuery({ admin: false }); + const { data, isLoading } = useGetProjectsQuery({ systemAdmin: false }); const projects = data || []; const projectOptions: ProjectOptionType[] = projects.map((project: ProjectDto) => ({ value: project.name, diff --git a/webapp/src/dogma/common/components/UserRole.tsx b/webapp/src/dogma/common/components/UserRole.tsx index ba6f5609b2..12cc662ab2 100644 --- a/webapp/src/dogma/common/components/UserRole.tsx +++ b/webapp/src/dogma/common/components/UserRole.tsx @@ -10,7 +10,7 @@ function badgeColor(role: string) { case 'member': return 'green'; case 'owner': - case 'admin': + case 'system admin': return 'blue'; default: return 'gray'; diff --git a/webapp/src/dogma/features/api/apiSlice.ts b/webapp/src/dogma/features/api/apiSlice.ts index 2f3da459da..7b85ca7ddb 100644 --- a/webapp/src/dogma/features/api/apiSlice.ts +++ b/webapp/src/dogma/features/api/apiSlice.ts @@ -37,7 +37,7 @@ export type ApiAction = { }; export type GetProjects = { - admin: boolean; + systemAdmin: boolean; }; export type GetHistory = { @@ -75,6 +75,16 @@ export type TitleDto = { hostname: string; }; +export type ZoneDto = { + currentZone: string; + allZones: string[]; +}; + +export type MirrorConfig = { + zonePinned: boolean; + zone: ZoneDto; +}; + export const apiSlice = createApi({ reducerPath: 'api', baseQuery: fetchBaseQuery({ @@ -94,7 +104,7 @@ export const apiSlice = createApi({ async queryFn(arg, _queryApi, _extraOptions, fetchWithBQ) { const projects = await fetchWithBQ('/api/v1/projects'); if (projects.error) return { error: projects.error as FetchBaseQueryError }; - if (arg.admin) { + if (arg.systemAdmin) { const removedProjects = await fetchWithBQ('/api/v1/projects?status=removed'); if (removedProjects.error) return { error: removedProjects.error as FetchBaseQueryError }; return { @@ -361,6 +371,12 @@ export const apiSlice = createApi({ }), invalidatesTags: ['Metadata'], }), + getMirrorConfig: builder.query({ + query: () => ({ + url: `/api/v1/mirror/config`, + method: 'GET', + }), + }), getCredentials: builder.query({ query: (projectName) => `/api/v1/projects/${projectName}/credentials`, providesTags: ['Metadata'], @@ -396,7 +412,6 @@ export const apiSlice = createApi({ }), getTitle: builder.query({ query: () => ({ - baseUrl: '', url: `/title`, method: 'GET', }), @@ -446,6 +461,7 @@ export const { useUpdateMirrorMutation, useDeleteMirrorMutation, useRunMirrorMutation, + useGetMirrorConfigQuery, // Credential useGetCredentialsQuery, useGetCredentialQuery, diff --git a/webapp/src/dogma/features/auth/ProjectRole.tsx b/webapp/src/dogma/features/auth/ProjectRole.tsx index ce031c614f..7ab919f4af 100644 --- a/webapp/src/dogma/features/auth/ProjectRole.tsx +++ b/webapp/src/dogma/features/auth/ProjectRole.tsx @@ -15,7 +15,7 @@ type WithProjectRoleProps = { export function findUserRole(user: UserDto, metadata: ProjectMetadataDto) { let role: ProjectRole; if (metadata && user) { - if (user.admin) { + if (user.systemAdmin) { role = 'OWNER'; } else { role = metadata.members[user.email]?.role as ProjectRole; diff --git a/webapp/src/dogma/features/auth/UserDto.ts b/webapp/src/dogma/features/auth/UserDto.ts index 446dcf6b26..dcfdfc9856 100644 --- a/webapp/src/dogma/features/auth/UserDto.ts +++ b/webapp/src/dogma/features/auth/UserDto.ts @@ -19,5 +19,5 @@ export interface UserDto { name: string; email: string; roles: string[]; - admin: boolean; + systemAdmin: boolean; } diff --git a/webapp/src/dogma/features/auth/authSlice.ts b/webapp/src/dogma/features/auth/authSlice.ts index 16b203c965..fde9d43118 100644 --- a/webapp/src/dogma/features/auth/authSlice.ts +++ b/webapp/src/dogma/features/auth/authSlice.ts @@ -133,7 +133,7 @@ const anonymousUser: UserDto = { name: 'Anonymous', email: 'anonymous@localhost', roles: [], - admin: false, + systemAdmin: false, }; export const authSlice = createSlice({ diff --git a/webapp/src/dogma/features/file/FileList.tsx b/webapp/src/dogma/features/file/FileList.tsx index cb641a0bee..1b0b5a2726 100644 --- a/webapp/src/dogma/features/file/FileList.tsx +++ b/webapp/src/dogma/features/file/FileList.tsx @@ -125,7 +125,7 @@ const FileList = ({ ); return ( - []} data={data} /> + []} data={data} /> { error, isLoading, } = useGetProjectsQuery({ - admin: user?.admin || false, + systemAdmin: user?.systemAdmin || false, }); let filteredProjects = projects; @@ -164,8 +164,8 @@ export const Projects = () => { } } - if (user.admin) { - // Restore project button for admin users. + if (user.systemAdmin) { + // Restore project button for system admin users. return ; } else { return null; diff --git a/webapp/src/dogma/features/project/settings/credentials/CredentialView.tsx b/webapp/src/dogma/features/project/settings/credentials/CredentialView.tsx index 79d71454e0..0fa04825c0 100644 --- a/webapp/src/dogma/features/project/settings/credentials/CredentialView.tsx +++ b/webapp/src/dogma/features/project/settings/credentials/CredentialView.tsx @@ -61,14 +61,14 @@ interface SecretViewerProps { const SecretViewer = ({ dispatch, secretProvider }: SecretViewerProps) => { const [showSecret, setShowSecret] = useState(false); - const admin = useAppSelector((state) => state.auth.user.admin); + const systemAdmin = useAppSelector((state) => state.auth.user.systemAdmin); return ( {showSecret ? secretProvider() : '****'} - {admin ? ( + {systemAdmin ? (