Skip to content

[MBUILDCACHE-104] Allow multiple cache entries per checksum #175

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

[MBUILDCACHE-104] Allow multiple cache entries per checksum #175

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
001fffb
Allow multiple cache entries per checksum
reda-alaoui Aug 20, 2024
4f70bff
Cache clearing broken by zone handling
reda-alaoui Sep 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 38 additions & 19 deletions src/main/java/org/apache/maven/buildcache/BuildCacheMojosExecutionStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@

import java.io.File;
import java.nio.file.Path;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -106,7 +108,7 @@ public void execute(

// execute clean bound goals before restoring to not interfere/slowdown clean
CacheState cacheState = DISABLED;
CacheResult result = CacheResult.empty();
Map<Zone, CacheResult> results = new LinkedHashMap<>();
boolean skipCache = cacheConfig.isSkipCache() || MavenProjectInput.isSkipCache(project);
boolean cacheIsDisabled = MavenProjectInput.isCacheDisabled(project);
// Forked execution should be thought as a part of originating mojo internal implementation
Expand All @@ -125,15 +127,21 @@ public void execute(
"Cache is explicitly disabled on project level for {}", getVersionlessProjectKey(project));
}
if (cacheState == INITIALIZED || skipCache) {
result = cacheController.findCachedBuild(session, project, mojoExecutions, skipCache);
for (Zone zone : cacheConfig.getInputZones()) {
results.put(
zone,
cacheController.findCachedBuild(session, project, mojoExecutions, zone, skipCache));
}
}
}

boolean restorable = result.isSuccess() || result.isPartialSuccess();
boolean restored = false; // if partially restored need to save increment
if (restorable) {
CacheResult bestResult = results.values().stream()
.max(Comparator.comparing(CacheResult::isRestorable))
.orElseGet(CacheResult::empty);
if (bestResult.isRestorable()) {
CacheRestorationStatus cacheRestorationStatus =
restoreProject(result, mojoExecutions, mojoExecutionRunner, cacheConfig);
restoreProject(bestResult, mojoExecutions, mojoExecutionRunner, cacheConfig);
restored = CacheRestorationStatus.SUCCESS == cacheRestorationStatus;
executeExtraCleanPhaseIfNeeded(cacheRestorationStatus, cleanPhase, mojoExecutionRunner);
}
Expand All @@ -147,21 +155,32 @@ public void execute(
}
}

if (cacheState == INITIALIZED && (!result.isSuccess() || !restored)) {
if (cacheConfig.isSkipSave()) {
LOGGER.info("Cache saving is disabled.");
} else if (cacheConfig.isMandatoryClean()
&& lifecyclePhasesHelper
.getCleanSegment(project, mojoExecutions)
.isEmpty()) {
LOGGER.info("Cache storing is skipped since there was no \"clean\" phase.");
} else {
final Map<String, MojoExecutionEvent> executionEvents = mojoListener.getProjectExecutions(project);
cacheController.save(result, mojoExecutions, executionEvents);
if (cacheState == INITIALIZED) {
final Map<String, MojoExecutionEvent> executionEvents = mojoListener.getProjectExecutions(project);
for (Zone outputZone : cacheConfig.getOutputZones()) {
CacheResult zoneResult = results.get(outputZone);
if (bestResult.isSuccess()
&& restored
&& (bestResult.getInputZone().equals(outputZone)
|| zoneResult != null && zoneResult.isSuccess())) {
continue;
}
if (cacheConfig.isSkipSave()) {
LOGGER.info("Cache saving is disabled.");
break;
}
if (cacheConfig.isMandatoryClean()
&& lifecyclePhasesHelper
.getCleanSegment(project, mojoExecutions)
.isEmpty()) {
LOGGER.info("Cache storing is skipped since there was no \"clean\" phase.");
break;
}
cacheController.save(bestResult, mojoExecutions, executionEvents, outputZone);
}
}

if (cacheConfig.isFailFast() && !result.isSuccess() && !skipCache && !forkedExecution) {
if (cacheConfig.isFailFast() && !bestResult.isSuccess() && !skipCache && !forkedExecution) {
throw new LifecycleExecutionException(
"Failed to restore project[" + getVersionlessProjectKey(project)
+ "] from cache, failing build.",
Expand All @@ -178,8 +197,8 @@ public void execute(
* we execute an extra clean phase to remove any potential partially restored files
*
* @param cacheRestorationStatus the restoration status
* @param cleanPhase clean phase mojos
* @param mojoExecutionRunner mojo runner
* @param cleanPhase clean phase mojos
* @param mojoExecutionRunner mojo runner
* @throws LifecycleExecutionException
*/
private void executeExtraCleanPhaseIfNeeded(
Expand Down
9 changes: 7 additions & 2 deletions src/main/java/org/apache/maven/buildcache/CacheController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,19 @@
public interface CacheController {

CacheResult findCachedBuild(
MavenSession session, MavenProject project, List<MojoExecution> mojoExecutions, boolean skipCache);
MavenSession session,
MavenProject project,
List<MojoExecution> mojoExecutions,
Zone inputZone,
boolean skipCache);

ArtifactRestorationReport restoreProjectArtifacts(CacheResult cacheResult);

void save(
CacheResult cacheResult,
List<MojoExecution> mojoExecutions,
Map<String, MojoExecutionEvent> executionEvents);
Map<String, MojoExecutionEvent> executionEvents,
Zone outputZone);

boolean isForcedExecution(MavenProject project, MojoExecution execution);

Expand Down
87 changes: 50 additions & 37 deletions src/main/java/org/apache/maven/buildcache/CacheControllerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,11 @@ public CacheControllerImpl(
@Override
@Nonnull
public CacheResult findCachedBuild(
MavenSession session, MavenProject project, List<MojoExecution> mojoExecutions, boolean skipCache) {
MavenSession session,
MavenProject project,
List<MojoExecution> mojoExecutions,
Zone inputZone,
boolean skipCache) {
final String highestPhase = lifecyclePhasesHelper.resolveHighestLifecyclePhase(project, mojoExecutions);

if (!lifecyclePhasesHelper.isLaterPhaseThanClean(highestPhase)) {
Expand All @@ -177,74 +181,81 @@ public CacheResult findCachedBuild(

String projectName = getVersionlessProjectKey(project);

ProjectsInputInfo inputInfo = projectInputCalculator.calculateInput(project);
ProjectsInputInfo inputInfo = projectInputCalculator.calculateInput(project, inputZone);

final CacheContext context = new CacheContext(project, inputInfo, session);

CacheResult result = empty(context);
CacheResult result = empty(context, inputZone);
if (!skipCache) {

LOGGER.info("Attempting to restore project {} from build cache", projectName);
LOGGER.info("Attempting to restore project {} from build cache zone {}", projectName, inputZone);

// remote build first
if (cacheConfig.isRemoteCacheEnabled()) {
result = findCachedBuild(mojoExecutions, context);
result = findCachedBuild(mojoExecutions, context, inputZone);
if (!result.isSuccess() && result.getContext() != null) {
LOGGER.info("Remote cache is incomplete or missing, trying local build for {}", projectName);
}
}

if (!result.isSuccess() && result.getContext() != null) {
CacheResult localBuild = findLocalBuild(mojoExecutions, context);
CacheResult localBuild = findLocalBuild(mojoExecutions, context, inputZone);
if (localBuild.isSuccess() || (localBuild.isPartialSuccess() && !result.isPartialSuccess())) {
result = localBuild;
} else {
LOGGER.info(
"Local build was not found by checksum {} for {}", inputInfo.getChecksum(), projectName);
"Local build was not found by checksum {} for {} and zone {}",
inputInfo.getChecksum(),
projectName,
inputZone);
}
}
} else {
LOGGER.info(
"Project {} is marked as requiring force rebuild, will skip lookup in build cache", projectName);
}
cacheResults.put(getVersionlessProjectKey(project), result);
cacheResults.put(getVersionlessProjectKey(project, inputZone), result);

return result;
}

private CacheResult findCachedBuild(List<MojoExecution> mojoExecutions, CacheContext context) {
private CacheResult findCachedBuild(List<MojoExecution> mojoExecutions, CacheContext context, Zone inputZone) {
Optional<Build> cachedBuild = Optional.empty();
try {
cachedBuild = localCache.findBuild(context);
cachedBuild = localCache.findBuild(context, inputZone);
if (cachedBuild.isPresent()) {
return analyzeResult(context, mojoExecutions, cachedBuild.get());
return analyzeResult(context, mojoExecutions, inputZone, cachedBuild.get());
}
} catch (Exception e) {
LOGGER.error("Cannot read cached remote build", e);
}
return cachedBuild.map(build -> failure(build, context)).orElseGet(() -> empty(context));
return cachedBuild.map(build -> failure(build, context, inputZone)).orElseGet(() -> empty(context, inputZone));
}

private CacheResult findLocalBuild(List<MojoExecution> mojoExecutions, CacheContext context) {
private CacheResult findLocalBuild(List<MojoExecution> mojoExecutions, CacheContext context, Zone inputZone) {
Optional<Build> localBuild = Optional.empty();
try {
localBuild = localCache.findLocalBuild(context);
localBuild = localCache.findLocalBuild(context, inputZone);
if (localBuild.isPresent()) {
return analyzeResult(context, mojoExecutions, localBuild.get());
return analyzeResult(context, mojoExecutions, inputZone, localBuild.get());
}
} catch (Exception e) {
LOGGER.error("Cannot read local build", e);
}
return localBuild.map(build -> failure(build, context)).orElseGet(() -> empty(context));
return localBuild.map(build -> failure(build, context, inputZone)).orElseGet(() -> empty(context, inputZone));
}

private CacheResult analyzeResult(CacheContext context, List<MojoExecution> mojoExecutions, Build build) {
private CacheResult analyzeResult(
CacheContext context, List<MojoExecution> mojoExecutions, Zone inputZone, Build build) {
try {
final ProjectsInputInfo inputInfo = context.getInputInfo();
String projectName = getVersionlessProjectKey(context.getProject());

LOGGER.info(
"Found cached build, restoring {} from cache by checksum {}", projectName, inputInfo.getChecksum());
"Found cached build, restoring {} from cache zone {} by checksum {}",
projectName,
inputZone,
inputInfo.getChecksum());
LOGGER.debug("Cached build details: {}", build);

final String cacheImplementationVersion = build.getCacheImplementationVersion();
Expand All @@ -263,12 +274,12 @@ private CacheResult analyzeResult(CacheContext context, List<MojoExecution> mojo
"Cached build doesn't contains all requested plugin executions "
+ "(missing: {}), cannot restore",
missingMojos);
return failure(build, context);
return failure(build, context, inputZone);
}

if (!isCachedSegmentPropertiesPresent(context.getProject(), build, cachedSegment)) {
LOGGER.info("Cached build violates cache rules, cannot restore");
return failure(build, context);
return failure(build, context, inputZone);
}

final String highestRequestPhase =
Expand All @@ -281,15 +292,15 @@ private CacheResult analyzeResult(CacheContext context, List<MojoExecution> mojo
projectName,
build.getHighestCompletedGoal(),
highestRequestPhase);
return partialSuccess(build, context);
return partialSuccess(build, context, inputZone);
}

return success(build, context);
return success(build, context, inputZone);

} catch (Exception e) {
LOGGER.error("Failed to restore project", e);
localCache.clearCache(context);
return failure(build, context);
localCache.clearCache(context, inputZone);
return failure(build, context, inputZone);
}
}

Expand Down Expand Up @@ -389,8 +400,8 @@ public ArtifactRestorationReport restoreProjectArtifacts(CacheResult cacheResult
// Set this value before trying the restoration, to keep a trace of the attempt if it fails
restorationReport.setRestoredFilesInProjectDirectory(true);
// generated sources artifact
final Path attachedArtifactFile =
localCache.getArtifactFile(context, cacheResult.getSource(), attachedArtifactInfo);
final Path attachedArtifactFile = localCache.getArtifactFile(
context, cacheResult.getSource(), cacheResult.getInputZone(), attachedArtifactInfo);
restoreGeneratedSources(attachedArtifactInfo, attachedArtifactFile, project);
}
} else {
Expand Down Expand Up @@ -462,7 +473,8 @@ private Future<File> createDownloadTask(
String originalVersion) {
final FutureTask<File> downloadTask = new FutureTask<>(() -> {
LOGGER.debug("Downloading artifact {}", artifact.getArtifactId());
final Path artifactFile = localCache.getArtifactFile(context, cacheResult.getSource(), artifact);
final Path artifactFile =
localCache.getArtifactFile(context, cacheResult.getSource(), cacheResult.getInputZone(), artifact);

if (!Files.exists(artifactFile)) {
throw new FileNotFoundException("Missing file for cached build, cannot restore. File: " + artifactFile);
Expand All @@ -482,7 +494,8 @@ private Future<File> createDownloadTask(
public void save(
CacheResult cacheResult,
List<MojoExecution> mojoExecutions,
Map<String, MojoExecutionEvent> executionEvents) {
Map<String, MojoExecutionEvent> executionEvents,
Zone outputZone) {
CacheContext context = cacheResult.getContext();

if (context == null || context.getInputInfo() == null) {
Expand Down Expand Up @@ -524,21 +537,21 @@ public void save(
hashFactory.getAlgorithm());
populateGitInfo(build, session);
build.getDto().set_final(cacheConfig.isSaveToRemoteFinal());
cacheResults.put(getVersionlessProjectKey(project), rebuilded(cacheResult, build));
cacheResults.put(getVersionlessProjectKey(project, outputZone), rebuilded(cacheResult, build));

localCache.beforeSave(context);
localCache.beforeSave(context, outputZone);

// if package phase presence means new artifacts were packaged
if (project.hasLifecyclePhase("package")) {
if (projectArtifact.getFile() != null) {
localCache.saveArtifactFile(cacheResult, projectArtifact);
localCache.saveArtifactFile(cacheResult, outputZone, projectArtifact);
}
for (org.apache.maven.artifact.Artifact attachedArtifact : attachedArtifacts) {
if (attachedArtifact.getFile() != null) {
boolean storeArtifact =
isOutputArtifact(attachedArtifact.getFile().getName());
if (storeArtifact) {
localCache.saveArtifactFile(cacheResult, attachedArtifact);
localCache.saveArtifactFile(cacheResult, outputZone, attachedArtifact);
} else {
LOGGER.debug(
"Skipping attached project artifact '{}' = "
Expand All @@ -549,7 +562,7 @@ public void save(
}
}

localCache.saveBuildInfo(cacheResult, build);
localCache.saveBuildInfo(cacheResult, outputZone, build);

if (cacheConfig.isBaselineDiffEnabled()) {
produceDiffReport(cacheResult, build);
Expand All @@ -558,7 +571,7 @@ public void save(
} catch (Exception e) {
LOGGER.error("Failed to save project, cleaning cache. Project: {}", project, e);
try {
localCache.clearCache(context);
localCache.clearCache(context, outputZone);
} catch (Exception ex) {
LOGGER.error("Failed to clean cache due to unexpected error:", ex);
}
Expand All @@ -567,7 +580,7 @@ public void save(

public void produceDiffReport(CacheResult cacheResult, Build build) {
MavenProject project = cacheResult.getContext().getProject();
Optional<Build> baselineHolder = remoteCache.findBaselineBuild(project);
Optional<Build> baselineHolder = remoteCache.findBaselineBuild(project, cacheResult.getInputZone());
if (baselineHolder.isPresent()) {
Build baseline = baselineHolder.get();
String outputDirectory = project.getBuild().getDirectory();
Expand Down Expand Up @@ -841,10 +854,10 @@ public void saveCacheReport(MavenSession session) {
projectReport.setLifecycleMatched(checksumMatched && result.isSuccess());
projectReport.setSource(String.valueOf(result.getSource()));
if (result.getSource() == CacheSource.REMOTE) {
projectReport.setUrl(remoteCache.getResourceUrl(context, BUILDINFO_XML));
projectReport.setUrl(remoteCache.getResourceUrl(context, result.getInputZone(), BUILDINFO_XML));
} else if (result.getSource() == CacheSource.BUILD && cacheConfig.isSaveToRemote()) {
projectReport.setSharedToRemote(true);
projectReport.setUrl(remoteCache.getResourceUrl(context, BUILDINFO_XML));
projectReport.setUrl(remoteCache.getResourceUrl(context, result.getInputZone(), BUILDINFO_XML));
}
cacheReport.addProject(projectReport);
}
Expand Down
Loading