Skip to content

Fix for #2872. Makes exporttable command volume aware. #3228

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AlbertWhitlock wants to merge 10 commits into
base: main
Choose a base branch
from
Open

Fix for #2872. Makes exporttable command volume aware. #3228

Changes from 3 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
439816f
Initial commit for #2872. Make exporttable command volume aware.
AlbertWhitlock Mar 6, 2023
477fd4e
Fixed syntax error. Added {} to if.
AlbertWhitlock Mar 6, 2023
92e5ffc
Improved way to collect volume name and changed naming of new distcp.…
AlbertWhitlock Mar 13, 2023
5df4df5
Cleaned up class and started test case
AlbertWhitlock Apr 11, 2023
aa66552
Added IT class and improved WriteExportFiles.java
AlbertWhitlock May 23, 2023
71f9e52
Restored original ShellServerIT. Renamed IT class. Modified checks in…
AlbertWhitlock Jun 5, 2023
6e19c8a
removed ExportTableCommandWitlMultipleVolumes.java
AlbertWhitlock Jun 13, 2023
3e2ebf6
Merge branch 'main' of https://github.com/apache/accumulo into acc2872
AlbertWhitlock Jun 13, 2023
fced61f
Made suggested change to exportTable method
AlbertWhitlock Jun 26, 2023
fe5fd30
Changes to old test and added a new test file
AlbertWhitlock Sep 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 29 additions & 5 deletions ...ager/src/main/java/org/apache/accumulo/manager/tableOps/tableExport/WriteExportFiles.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

Expand Down Expand Up @@ -154,6 +156,9 @@ public void undo(long tid, Manager env) {
public static void exportTable(VolumeManager fs, ServerContext context, String tableName,
TableId tableID, String exportDir) throws Exception {

// int count = 0;
Set<String> volumeSet = new TreeSet<>();

fs.mkdirs(new Path(exportDir));
Path exportMetaFilePath = fs.getFileSystemByPath(new Path(exportDir))
.makeQualified(new Path(exportDir, Constants.EXPORT_FILE));
Expand Down Expand Up @@ -186,8 +191,23 @@ public static void exportTable(VolumeManager fs, ServerContext context, String t
dataOut.close();
dataOut = null;

createDistcpFile(fs, exportDir, exportMetaFilePath, uniqueFiles);
// make a set of unique volumes from the map
for (String fileString : uniqueFiles.values()) {
String[] fileSegmentArray = fileString.split("/");
volumeSet.add(fileSegmentArray[2]);
}

// for each unique volume: get every matching entry in the map and send to createDistcpFile
// method
for (String volumeString : volumeSet) {
Set<String> sortedVolumeSet = new TreeSet<>();
for (String rFileString : uniqueFiles.values()) {
if (rFileString.contains(volumeString)) {
sortedVolumeSet.add(rFileString);
}
}
createDistcpFile(fs, exportDir, exportMetaFilePath, sortedVolumeSet, volumeString);
}
} finally {
if (dataOut != null) {
dataOut.close();
Expand All @@ -196,12 +216,16 @@ public static void exportTable(VolumeManager fs, ServerContext context, String t
}

private static void createDistcpFile(VolumeManager fs, String exportDir, Path exportMetaFilePath,
Map<String,String> uniqueFiles) throws IOException {
BufferedWriter distcpOut = new BufferedWriter(
new OutputStreamWriter(fs.create(new Path(exportDir, "distcp.txt")), UTF_8));
Set<String> uniqueFiles, String volumeName) throws IOException {
if (volumeName.contains(":")) {
volumeName = volumeName.replace(":", "-");
}

BufferedWriter distcpOut = new BufferedWriter(new OutputStreamWriter(
fs.create(new Path(exportDir, "distcp-" + volumeName + ".txt")), UTF_8));

try {
for (String file : uniqueFiles.values()) {
for (String file : uniqueFiles) {
distcpOut.append(file);
distcpOut.newLine();
}
Expand Down