Skip to content

Commit 4749ea1

Browse files
MMcMMMcM
committed
Move the {compression,encryption}Enabled state gotten from properties from the directory to its serializer.
1 parent 8c71285 commit 4749ea1

File tree

5 files changed

+43
-28
lines changed

5 files changed

+43
-28
lines changed

‎fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/directory/FDBDirectory.java‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -166,9 +166,6 @@ public class FDBDirectory extends Directory {
166166

167167
private final Cache<ComparablePair<Long, Integer>, CompletableFuture<byte[]>> blockCache;
168168

169-
private final boolean compressionEnabled;
170-
private final boolean encryptionEnabled;
171-
172169
// The shared cache is initialized when first listing the directory, if a manager is present, and cleared before writing.
173170
@Nullable
174171
private final FDBDirectorySharedCacheManager sharedCacheManager;
@@ -230,9 +227,9 @@ private FDBDirectory(@Nonnull Subspace subspace, @Nullable Map<String, String> i
230227
.removalListener(notification -> cacheRemovalCallback())
231228
.build();
232229
this.fileSequenceCounter = new AtomicLong(-1);
233-
this.compressionEnabled = Objects.requireNonNullElse(agilityContext.getPropertyValue(LuceneRecordContextProperties.LUCENE_INDEX_COMPRESSION_ENABLED), false);
234-
this.encryptionEnabled = Objects.requireNonNullElse(agilityContext.getPropertyValue(LuceneRecordContextProperties.LUCENE_INDEX_ENCRYPTION_ENABLED), false);
235-
this.serializer = new LuceneSerializer(agilityContext.getPropertyValue(LuceneRecordContextProperties.LUCENE_INDEX_KEY_MANAGER));
230+
this.serializer = new LuceneSerializer(Objects.requireNonNullElse(agilityContext.getPropertyValue(LuceneRecordContextProperties.LUCENE_INDEX_COMPRESSION_ENABLED), false),
231+
Objects.requireNonNullElse(agilityContext.getPropertyValue(LuceneRecordContextProperties.LUCENE_INDEX_ENCRYPTION_ENABLED), false),
232+
agilityContext.getPropertyValue(LuceneRecordContextProperties.LUCENE_INDEX_KEY_MANAGER));
236233
this.fileReferenceMapSupplier = Suppliers.memoize(this::loadFileReferenceCacheForMemoization);
237234
this.sharedCacheManager = sharedCacheManager;
238235
this.sharedCacheKey = sharedCacheKey;
@@ -405,7 +402,7 @@ public static boolean isStoredFieldsFile(String name) {
405402
*/
406403
public void writeFDBLuceneFileReference(@Nonnull String name, @Nonnull FDBLuceneFileReference reference) {
407404
final byte[] fileReferenceBytes = reference.getBytes();
408-
final byte[] encodedBytes = Objects.requireNonNull(serializer.encode(fileReferenceBytes, compressionEnabled, encryptionEnabled));
405+
final byte[] encodedBytes = Objects.requireNonNull(serializer.encode(fileReferenceBytes));
409406
agilityContext.recordSize(LuceneEvents.SizeEvents.LUCENE_WRITE_FILE_REFERENCE, encodedBytes.length);
410407
if (LOGGER.isTraceEnabled()) {
411408
LOGGER.trace(getLogMessage("Write lucene file reference",
@@ -427,7 +424,7 @@ public void writeFDBLuceneFileReference(@Nonnull String name, @Nonnull FDBLucene
427424
* @return the actual data size written to database with potential compression and encryption applied
428425
*/
429426
public int writeData(final long id, final int block, @Nonnull final byte[] value) {
430-
final byte[] encodedBytes = Objects.requireNonNull(serializer.encode(value, compressionEnabled, encryptionEnabled));
427+
final byte[] encodedBytes = Objects.requireNonNull(serializer.encode(value));
431428
agilityContext.increment(LuceneEvents.Counts.LUCENE_BLOCK_WRITES);
432429
//This may not be correct transactionally
433430
agilityContext.recordSize(LuceneEvents.SizeEvents.LUCENE_WRITE, encodedBytes.length);
@@ -915,10 +912,10 @@ public void rename(@Nonnull final String source, @Nonnull final String dest) thr
915912
.addLogInfo(LogMessageKeys.SOURCE_FILE, source)
916913
.addLogInfo(LogMessageKeys.INDEX_TYPE, LuceneIndexTypes.LUCENE)
917914
.addLogInfo(LogMessageKeys.SUBSPACE, subspace)
918-
.addLogInfo(LuceneLogMessageKeys.COMPRESSION_SUPPOSED, compressionEnabled)
919-
.addLogInfo(LuceneLogMessageKeys.ENCRYPTION_SUPPOSED, encryptionEnabled);
915+
.addLogInfo(LuceneLogMessageKeys.COMPRESSION_SUPPOSED, serializer.isCompressionEnabled())
916+
.addLogInfo(LuceneLogMessageKeys.ENCRYPTION_SUPPOSED, serializer.isEncryptionEnabled());
920917
}
921-
byte[] encodedBytes = serializer.encode(value.getBytes(), compressionEnabled, encryptionEnabled);
918+
byte[] encodedBytes = serializer.encode(value.getBytes());
922919
agilityContext.set(metaSubspace.pack(dest), encodedBytes);
923920
agilityContext.clear(key);
924921

@@ -1057,8 +1054,8 @@ private String getLogMessage(@Nonnull String staticMsg, @Nullable final Object..
10571054
private KeyValueLogMessage getKeyValueLogMessage(final @Nonnull String staticMsg, final Object... keysAndValues) {
10581055
return KeyValueLogMessage.build(staticMsg, keysAndValues)
10591056
.addKeyAndValue(LogMessageKeys.SUBSPACE, subspace)
1060-
.addKeyAndValue(LuceneLogMessageKeys.COMPRESSION_SUPPOSED, compressionEnabled)
1061-
.addKeyAndValue(LuceneLogMessageKeys.ENCRYPTION_SUPPOSED, encryptionEnabled);
1057+
.addKeyAndValue(LuceneLogMessageKeys.COMPRESSION_SUPPOSED, serializer.isCompressionEnabled())
1058+
.addKeyAndValue(LuceneLogMessageKeys.ENCRYPTION_SUPPOSED, serializer.isEncryptionEnabled());
10621059
}
10631060

10641061
/**

‎fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/directory/LuceneSerializer.java‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,26 +53,44 @@ public class LuceneSerializer {
5353
private static final byte COMPRESSION_VERSION_FOR_HIGH_COMPRESSION = 0;
5454
private static final int ENCRYPTION_KEY_SHIFT = 3;
5555

56+
private final boolean compressionEnabled;
57+
private final boolean encryptionEnabled;
5658
@Nullable
5759
private final SerializationKeyManager keyManager;
5860

59-
public LuceneSerializer(@Nullable SerializationKeyManager keyManager) {
61+
public LuceneSerializer(boolean compressionEnabled,
62+
boolean encryptionEnabled, @Nullable SerializationKeyManager keyManager) {
63+
this.compressionEnabled = compressionEnabled;
64+
this.encryptionEnabled = encryptionEnabled;
6065
this.keyManager = keyManager;
6166
}
6267

68+
public boolean isCompressionEnabled() {
69+
return compressionEnabled;
70+
}
71+
72+
public boolean isEncryptionEnabled() {
73+
return encryptionEnabled;
74+
}
75+
76+
@Nullable
77+
public SerializationKeyManager getKeyManager() {
78+
return keyManager;
79+
}
80+
6381
@Nullable
64-
public byte[] encode(@Nullable byte[] data, boolean compress, boolean encrypt) {
82+
public byte[] encode(@Nullable byte[] data) {
6583
if (data == null) {
6684
return null;
6785
}
6886

6987
final CompressedAndEncryptedSerializerState state = new CompressedAndEncryptedSerializerState();
7088
long prefix = 0;
71-
if (compress) {
89+
if (compressionEnabled) {
7290
prefix |= ENCODING_COMPRESSED;
7391
state.setCompressed(true);
7492
}
75-
if (encrypt) {
93+
if (encryptionEnabled) {
7694
if (keyManager == null) {
7795
throw new RecordCoreException("cannot encrypt Lucene blocks without keys");
7896
}
@@ -96,8 +114,8 @@ public byte[] encode(@Nullable byte[] data, boolean compress, boolean encrypt) {
96114

97115
if (LOGGER.isTraceEnabled()) {
98116
LOGGER.trace(KeyValueLogMessage.of("Encoded lucene data",
99-
LuceneLogMessageKeys.COMPRESSION_SUPPOSED, compress,
100-
LuceneLogMessageKeys.ENCRYPTION_SUPPOSED, encrypt,
117+
LuceneLogMessageKeys.COMPRESSION_SUPPOSED, compressionEnabled,
118+
LuceneLogMessageKeys.ENCRYPTION_SUPPOSED, encryptionEnabled,
101119
LuceneLogMessageKeys.COMPRESSED_EVENTUALLY, state.isCompressed(),
102120
LuceneLogMessageKeys.ENCRYPTED_EVENTUALLY, state.isEncrypted(),
103121
LuceneLogMessageKeys.ORIGINAL_DATA_SIZE, data.length,

‎fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneSerializerTest.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
class LuceneSerializerTest {
4545
@Test
4646
void testEncodingWithoutCompression() throws InvalidProtocolBufferException {
47-
final LuceneSerializer serializer = new LuceneSerializer(null);
47+
final LuceneSerializer serializer = new LuceneSerializer(true, false, null);
4848
final ByteString content = RandomUtil.randomByteString(ThreadLocalRandom.current(), 100);
4949
final LuceneFileSystemProto.LuceneFileReference reference = LuceneFileSystemProto.LuceneFileReference.newBuilder()
5050
.setId(1)
@@ -53,7 +53,7 @@ void testEncodingWithoutCompression() throws InvalidProtocolBufferException {
5353
.setContent(content)
5454
.build();
5555
final byte[] originalValue = reference.toByteArray();
56-
final byte[] encodedValue = serializer.encode(originalValue, true, false);
56+
final byte[] encodedValue = serializer.encode(originalValue);
5757
final byte[] decodedValue = serializer.decode(encodedValue);
5858

5959
final byte[] expectedEncodedValue = new byte[originalValue.length + 1];
@@ -74,7 +74,7 @@ void testEncodingWithoutCompression() throws InvalidProtocolBufferException {
7474

7575
@Test
7676
void testEncodingWithCompression() throws InvalidProtocolBufferException {
77-
final LuceneSerializer serializer = new LuceneSerializer(null);
77+
final LuceneSerializer serializer = new LuceneSerializer(true, false, null);
7878
final String duplicateMsg = "abcdefghijklmnopqrstuvwxyz";
7979
final String content = "content_" + duplicateMsg + "_" + duplicateMsg;
8080
final LuceneFileSystemProto.LuceneFileReference reference = LuceneFileSystemProto.LuceneFileReference.newBuilder()
@@ -84,7 +84,7 @@ void testEncodingWithCompression() throws InvalidProtocolBufferException {
8484
.setContent(ByteString.copyFromUtf8(content))
8585
.build();
8686
final byte[] value = reference.toByteArray();
87-
final byte[] encodedValue = serializer.encode(value, true, false);
87+
final byte[] encodedValue = serializer.encode(value);
8888
final byte[] decodedValue = serializer.decode(encodedValue);
8989

9090
// The encoded value's size is smaller than the original one due to compression
@@ -107,7 +107,7 @@ void testEncodingWithEncryption(boolean compressToo) throws Exception {
107107
keyGen.init(128);
108108
SecretKey key = keyGen.generateKey();
109109
final SerializationKeyManager keyManager = new FixedZeroKeyManager(key, null, null);
110-
final LuceneSerializer serializer = new LuceneSerializer(keyManager);
110+
final LuceneSerializer serializer = new LuceneSerializer(compressToo, true, keyManager);
111111
final ByteString content = RandomUtil.randomByteString(ThreadLocalRandom.current(), 100);
112112
final LuceneFileSystemProto.LuceneFileReference reference = LuceneFileSystemProto.LuceneFileReference.newBuilder()
113113
.setId(1)
@@ -116,7 +116,7 @@ void testEncodingWithEncryption(boolean compressToo) throws Exception {
116116
.setContent(content)
117117
.build();
118118
final byte[] value = reference.toByteArray();
119-
final byte[] encodedValue = serializer.encode(value, compressToo, true);
119+
final byte[] encodedValue = serializer.encode(value);
120120
final byte[] decodedValue = serializer.decode(encodedValue);
121121
Assertions.assertArrayEquals(value, decodedValue);
122122
final LuceneFileSystemProto.LuceneFileReference decryptedReference = LuceneFileSystemProto.LuceneFileReference.parseFrom(decodedValue);

‎fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/FDBDirectoryFailuresTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ public void testWriteSeekData() throws Exception {
121121
assertTrue(ex.getCause() instanceof TimeoutException);
122122

123123
directory.getCallerContext().commit();
124-
assertCorrectMetricSize(LuceneEvents.SizeEvents.LUCENE_WRITE, 1, directory.getSerializer().encode(data, true, false).length);
124+
assertCorrectMetricSize(LuceneEvents.SizeEvents.LUCENE_WRITE, 1, directory.getSerializer().encode(data).length);
125125
}
126126

127127
@Test

‎fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/FDBDirectoryTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ public void testWriteLuceneFileReference() {
131131

132132
LuceneSerializer serializer = directory.getSerializer();
133133
assertCorrectMetricSize(LuceneEvents.SizeEvents.LUCENE_WRITE_FILE_REFERENCE, 2,
134-
serializer.encode(reference1.getBytes(), true, false).length + serializer.encode(reference2.getBytes(), true, false).length);
134+
serializer.encode(reference1.getBytes()).length + serializer.encode(reference2.getBytes()).length);
135135
}
136136

137137
@Test
@@ -160,7 +160,7 @@ public void testWriteSeekData() throws Exception {
160160
directory.getFDBLuceneFileReferenceAsync("testReference2"), 1).get(), "seek data should exist");
161161

162162
directory.getCallerContext().commit();
163-
assertCorrectMetricSize(LuceneEvents.SizeEvents.LUCENE_WRITE, 1, directory.getSerializer().encode(data, true, false).length);
163+
assertCorrectMetricSize(LuceneEvents.SizeEvents.LUCENE_WRITE, 1, directory.getSerializer().encode(data).length);
164164
}
165165

166166
@Test

0 commit comments

Comments
 (0)