diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/DecompressorValue.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/DecompressorValue.java index af45af69ad0136..cfc6e6d2b2e207 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/DecompressorValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/DecompressorValue.java @@ -112,11 +112,13 @@ static Decompressor getDecompressor(Path archivePath) throws RepositoryFunctionE return TarBz2Function.INSTANCE; } else if (baseName.endsWith(".ar") || baseName.endsWith(".deb")) { return ArFunction.INSTANCE; + } else if (baseName.endsWith(".7z")) { + return SevenZDecompressor.INSTANCE; } else { throw new RepositoryFunctionException( Starlark.errorf( "Expected a file with a .zip, .jar, .war, .aar, .nupkg, .whl, .tar, .tar.gz, .tgz," - + " .tar.xz, , .tar.zst, .tzst, .tar.bz2, .tbz, .ar or .deb suffix (got %s)", + + " .tar.xz, , .tar.zst, .tzst, .tar.bz2, .tbz, .ar, .deb or .7z suffix (got %s)", archivePath), Transience.PERSISTENT); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/SevenZDecompressor.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/SevenZDecompressor.java new file mode 100644 index 00000000000000..3ce0460bffb062 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/SevenZDecompressor.java @@ -0,0 +1,105 @@ +package com.google.devtools.build.lib.bazel.repository.decompressor; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.io.ByteStreams; +import com.google.devtools.build.lib.bazel.repository.RepositoryFunctionException; +import com.google.devtools.build.lib.bazel.repository.decompressor.DecompressorValue.Decompressor; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import org.apache.commons.compress.archivers.sevenz.SevenZArchiveEntry; +import org.apache.commons.compress.archivers.sevenz.SevenZFile; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Creates a repository by decompressing a 7-zip file. This implementation generally follows the + * logic from {@link ZipDecompressor} with the exception that the 7z format does not support file + * permissions or symbolic links. + */ +public class SevenZDecompressor implements Decompressor { + public static final Decompressor INSTANCE = new SevenZDecompressor(); + + /** Decompresses the file to directory {@link DecompressorDescriptor#destinationPath()} */ + @Override + @Nullable + public Path decompress(DecompressorDescriptor descriptor) + throws IOException, RepositoryFunctionException, InterruptedException { + Path destinationDirectory = descriptor.destinationPath(); + Optional prefix = descriptor.prefix(); + Map renameFiles = descriptor.renameFiles(); + boolean foundPrefix = false; + + try (SevenZFile sevenZFile = + SevenZFile.builder().setFile(descriptor.archivePath().getPathFile()).get()) { + Iterable entries = sevenZFile.getEntries(); + for (SevenZArchiveEntry entry : entries) { + String entryName = entry.getName(); + entryName = renameFiles.getOrDefault(entryName, entryName); + StripPrefixedPath entryPath = + StripPrefixedPath.maybeDeprefix(entryName.getBytes(UTF_8), prefix); + foundPrefix = foundPrefix || entryPath.foundPrefix(); + if (entryPath.skip()) { + continue; + } + extract7zEntry(sevenZFile, entry, destinationDirectory, entryPath.getPathFragment()); + } + + if (prefix.isPresent() && !foundPrefix) { + Set prefixes = new HashSet<>(); + for (SevenZArchiveEntry entry : entries) { + StripPrefixedPath entryPath = + StripPrefixedPath.maybeDeprefix(entry.getName().getBytes(UTF_8), Optional.empty()); + CouldNotFindPrefixException.maybeMakePrefixSuggestion(entryPath.getPathFragment()) + .ifPresent(prefixes::add); + } + throw new CouldNotFindPrefixException(prefix.get(), prefixes); + } + } + return destinationDirectory; + } + + private static void extract7zEntry( + SevenZFile sevenZFile, + SevenZArchiveEntry entry, + Path destinationDirectory, + PathFragment strippedRelativePath) + throws IOException, InterruptedException { + if (strippedRelativePath.isAbsolute()) { + throw new IOException( + String.format( + "Failed to extract %s, 7-zipped paths cannot be absolute", strippedRelativePath)); + } + // Sanity/security check - at this point, uplevel references (..) should be resolved. + // There shouldn't be any remaining uplevel references, otherwise, the extracted file could + // "escape" the destination directory. + if (strippedRelativePath.containsUplevelReferences()) { + throw new IOException( + String.format( + "Failed to extract %s, 7-zipped entry contains uplevel references (..)", + strippedRelativePath)); + } + Path outputPath = destinationDirectory.getRelative(strippedRelativePath); + outputPath.getParentDirectory().createDirectoryAndParents(); + boolean isDirectory = entry.isDirectory(); + if (isDirectory) { + outputPath.createDirectoryAndParents(); + } else { + try (InputStream input = sevenZFile.getInputStream(entry); + OutputStream output = outputPath.getOutputStream()) { + ByteStreams.copy(input, output); + if (Thread.interrupted()) { + throw new InterruptedException(); + } + } + outputPath.setLastModifiedTime(entry.getLastModifiedTime().toMillis()); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkBaseExternalContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkBaseExternalContext.java index 7ab9d76e9a116e..60c9f6c938e4ac 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkBaseExternalContext.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkBaseExternalContext.java @@ -906,7 +906,7 @@ public Object download( determined from the file extension of the URL. If the file has no \ extension, you can explicitly specify either "zip", "jar", "war", \ "aar", "nupkg", "whl", "tar", "tar.gz", "tgz", "tar.xz", "txz", ".tar.zst", \ - ".tzst", "tar.bz2", ".tbz", ".ar", or ".deb" here. + ".tzst", "tar.bz2", ".tbz", ".ar", ".deb", or ".7z" here. """), @Param( name = "strip_prefix", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/BUILD index 76acc32a6b2ab4..5daf737791d7bb 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/BUILD @@ -17,6 +17,7 @@ java_library( name = "DecompressorTests_lib", srcs = glob(["*.java"]), data = [ + "test_decompress_archive.7z", "test_decompress_archive.tar.gz", "test_decompress_archive.zip", "test_files.ar", @@ -27,6 +28,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/clock", "//src/main/java/com/google/devtools/build/lib/unix", "//src/main/java/com/google/devtools/build/lib/util:os", + "//src/main/java/com/google/devtools/build/lib/util:string_encoding", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/DecompressorValueTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/DecompressorValueTest.java index f6096a9605394a..d8cf6d38845ff9 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/DecompressorValueTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/DecompressorValueTest.java @@ -67,6 +67,8 @@ public void testKnownFileExtensionsDoNotThrow() throws Exception { unused = DecompressorValue.getDecompressor(path); path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.deb"); unused = DecompressorValue.getDecompressor(path); + path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.7z"); + unused = DecompressorValue.getDecompressor(path); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/SevenZDecompressorTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/SevenZDecompressorTest.java new file mode 100644 index 00000000000000..35a03d0962cd0a --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/SevenZDecompressorTest.java @@ -0,0 +1,120 @@ +package com.google.devtools.build.lib.bazel.repository.decompressor; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.bazel.repository.decompressor.TestArchiveDescriptor.INNER_FOLDER_NAME; +import static com.google.devtools.build.lib.bazel.repository.decompressor.TestArchiveDescriptor.ROOT_FOLDER_NAME; + +import com.google.devtools.build.lib.vfs.Dirent; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.Symlinks; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests .7z decompression. */ +@RunWith(JUnit4.class) +public class SevenZDecompressorTest { + @Rule public TestName name = new TestName(); + + /** + * .7z file, created with one file: + * + *
    + *
  • root_folder/another_folder/regularFile + *
+ * + * Compressed with command "7zz a test_decompress_archive.7z root_folder" + */ + private static final String ARCHIVE_NAME = "test_decompress_archive.7z"; + + private static final String REGULAR_FILENAME = "regularFile"; + + /** Provides a test filesystem descriptor for a test. NOTE: unique per individual test ONLY. */ + private TestArchiveDescriptor archiveDescriptor() throws Exception { + return new TestArchiveDescriptor( + ARCHIVE_NAME, + /* outDirName= */ this.getClass().getSimpleName() + "_" + name.getMethodName(), + /* withHardLinks= */ false); + } + + /** Test decompressing a .7z file without stripping a prefix */ + @Test + public void testDecompressWithoutPrefix() throws Exception { + Path outputDir = decompress(archiveDescriptor().createDescriptorBuilder().build()); + + Path fileDir = outputDir.getRelative(ROOT_FOLDER_NAME).getRelative(INNER_FOLDER_NAME); + List files = + fileDir.readdir(Symlinks.NOFOLLOW).stream() + .map(Dirent::getName) + .collect(Collectors.toList()); + assertThat(files).contains(REGULAR_FILENAME); + assertThat(fileDir.getRelative(REGULAR_FILENAME).getFileSize()).isNotEqualTo(0); + } + + /** Test decompressing a .7z file and stripping a prefix. */ + @Test + public void testDecompressWithPrefix() throws Exception { + DecompressorDescriptor.Builder descriptorBuilder = + archiveDescriptor().createDescriptorBuilder().setPrefix(ROOT_FOLDER_NAME); + Path outputDir = decompress(descriptorBuilder.build()); + Path fileDir = outputDir.getRelative(INNER_FOLDER_NAME); + + List files = + fileDir.readdir(Symlinks.NOFOLLOW).stream() + .map(Dirent::getName) + .collect(Collectors.toList()); + assertThat(files).contains(REGULAR_FILENAME); + } + + /** Test decompressing a .7z with entries being renamed during the extraction process. */ + @Test + public void testDecompressWithRenamedFiles() throws Exception { + String innerDirName = ROOT_FOLDER_NAME + "/" + INNER_FOLDER_NAME; + + HashMap renameFiles = new HashMap<>(); + renameFiles.put(innerDirName + "/" + REGULAR_FILENAME, innerDirName + "/renamedFile"); + DecompressorDescriptor.Builder descriptorBuilder = + archiveDescriptor().createDescriptorBuilder().setRenameFiles(renameFiles); + Path outputDir = decompress(descriptorBuilder.build()); + + Path fileDir = outputDir.getRelative(ROOT_FOLDER_NAME).getRelative(INNER_FOLDER_NAME); + List files = + fileDir.readdir(Symlinks.NOFOLLOW).stream() + .map((Dirent::getName)) + .collect(Collectors.toList()); + assertThat(files).contains("renamedFile"); + assertThat(fileDir.getRelative("renamedFile").getFileSize()).isNotEqualTo(0); + } + + /** Test that entry renaming is applied prior to prefix stripping. */ + @Test + public void testDecompressWithRenamedFilesAndPrefix() throws Exception { + String innerDirName = ROOT_FOLDER_NAME + "/" + INNER_FOLDER_NAME; + + HashMap renameFiles = new HashMap<>(); + renameFiles.put(innerDirName + "/" + REGULAR_FILENAME, innerDirName + "/renamedFile"); + DecompressorDescriptor.Builder descriptorBuilder = + archiveDescriptor() + .createDescriptorBuilder() + .setPrefix(ROOT_FOLDER_NAME) + .setRenameFiles(renameFiles); + Path outputDir = decompress(descriptorBuilder.build()); + + Path fileDir = outputDir.getRelative(INNER_FOLDER_NAME); + List files = + fileDir.readdir(Symlinks.NOFOLLOW).stream() + .map((Dirent::getName)) + .collect(Collectors.toList()); + assertThat(files).contains("renamedFile"); + assertThat(fileDir.getRelative("renamedFile").getFileSize()).isNotEqualTo(0); + } + + private Path decompress(DecompressorDescriptor descriptor) throws Exception { + return new SevenZDecompressor().decompress(descriptor); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/test_decompress_archive.7z b/src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/test_decompress_archive.7z new file mode 100644 index 00000000000000..faf568abd28cb8 Binary files /dev/null and b/src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/test_decompress_archive.7z differ diff --git a/src/test/tools/bzlmod/MODULE.bazel.lock b/src/test/tools/bzlmod/MODULE.bazel.lock index 897491f757498a..1a550bc8708ac7 100644 --- a/src/test/tools/bzlmod/MODULE.bazel.lock +++ b/src/test/tools/bzlmod/MODULE.bazel.lock @@ -181,7 +181,7 @@ "moduleExtensions": { "@@pybind11_bazel+//:internal_configure.bzl%internal_configure_extension": { "general": { - "bzlTransitiveDigest": "jLp6l9wb9jlGqTRAfZaf1n+yOu/Lh64vzVRBMg3xiX8=", + "bzlTransitiveDigest": "X6LjWsFJ2Lt4rTFqqPZXOgcO+G9HyBOTNQ+Z4iS0m84=", "usagesDigest": "D1r3lfzMuUBFxgG8V6o0bQTLMk3GkaGOaPzw53wrwyw=", "recordedFileInputs": { "@@pybind11_bazel+//MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34" @@ -211,7 +211,7 @@ }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { - "bzlTransitiveDigest": "jzbq/qMpM8xW1zVnA2SSSZvRDl5COpD09LbqXgCxvrY=", + "bzlTransitiveDigest": "2Uvmven5Eh3Sc+TsuBRbsFlsQXIIlg02oPwrWY9Vtbc=", "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, diff --git a/tools/build_defs/repo/http.bzl b/tools/build_defs/repo/http.bzl index 8749584a3e655a..bdb870521804ba 100644 --- a/tools/build_defs/repo/http.bzl +++ b/tools/build_defs/repo/http.bzl @@ -337,7 +337,8 @@ repository. Files are symlinked after remote files are downloaded and patches (` By default, the archive type is determined from the file extension of the URL. If the file has no extension, you can explicitly specify one of the following: `"zip"`, `"jar"`, `"war"`, `"aar"`, `"tar"`, `"tar.gz"`, `"tgz"`, -`"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `"tar.bz2"`, `"ar"`, or `"deb"`.""", +`"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `"tar.bz2"`, `"ar"`, `"deb"`, or +`"7z"`.""", ), "patches": attr.label_list( default = [], @@ -450,7 +451,7 @@ and makes its targets available for binding. It supports the following file extensions: `"zip"`, `"jar"`, `"war"`, `"aar"`, `"tar"`, `"tar.gz"`, `"tgz"`, `"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `tar.bz2`, `"ar"`, -or `"deb"`. +`"deb"`, or `"7z"`. Examples: Suppose the current repository contains the source code for a chat program,