Skip to content

Improve serialization of big integer in Lucene.java #19348

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+61 −71
Open

Improve serialization of big integer in Lucene.java #19348

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Migrate usages of deprecated `Operations#union` from Lucene ([#19397](https://github.com/opensearch-project/OpenSearch/pull/19397))
- Delegate primitive write methods with ByteSizeCachingDirectory wrapped IndexOutput ([#19432](https://github.com/opensearch-project/OpenSearch/pull/19432))
- Bump opensearch-protobufs dependency to 0.18.0 and update transport-grpc module compatibility ([#19447](https://github.com/opensearch-project/OpenSearch/issues/19447))
- Improve serialization of BigInteger in Lucene.java ([19348](https://github.com/opensearch-project/OpenSearch/pull/19348)))

### Fixed
- Fix unnecessary refreshes on update preparation failures ([#15261](https://github.com/opensearch-project/OpenSearch/issues/15261))
Expand Down
131 changes: 60 additions & 71 deletions server/src/main/java/org/opensearch/common/lucene/Lucene.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ private static SegmentInfos readSegmentInfos(String segmentsFileName, Directory
*/
public static SegmentInfos pruneUnreferencedFiles(String segmentsFileName, Directory directory) throws IOException {
final SegmentInfos si = readSegmentInfos(segmentsFileName, directory);
try (Lock writeLock = directory.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
try (Lock ignored = directory.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
int foundSegmentFiles = 0;
for (final String file : directory.listAll()) {
/*
Expand All @@ -222,7 +222,7 @@ public static SegmentInfos pruneUnreferencedFiles(String segmentsFileName, Direc
}
final IndexCommit cp = getIndexCommit(si, directory);
try (
IndexWriter writer = new IndexWriter(
IndexWriter ignored = new IndexWriter(
directory,
new IndexWriterConfig(Lucene.STANDARD_ANALYZER).setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
.setIndexCommit(cp)
Expand All @@ -249,15 +249,15 @@ public static IndexCommit getIndexCommit(SegmentInfos si, Directory directory) t
* this operation fails.
*/
public static void cleanLuceneIndex(Directory directory) throws IOException {
try (Lock writeLock = directory.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
try (Lock ignored = directory.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
for (final String file : directory.listAll()) {
if (file.startsWith(IndexFileNames.SEGMENTS)) {
directory.deleteFile(file); // remove all segment_N files
}
}
}
try (
IndexWriter writer = new IndexWriter(
IndexWriter ignored = new IndexWriter(
directory,
new IndexWriterConfig(Lucene.STANDARD_ANALYZER).setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
.setMergePolicy(NoMergePolicy.INSTANCE) // no merges
Expand Down Expand Up @@ -388,7 +388,7 @@ private static Comparable readTypedValue(StreamInput in) throws IOException {
case 7 -> in.readShort();
case 8 -> in.readBoolean();
case 9 -> in.readBytesRef();
case 10 -> new BigInteger(in.readString());
case 10 -> new BigInteger(in.readByteArray());
default -> throw new IOException("Can't match type [" + type + "]");
};
}
Expand All @@ -405,9 +405,8 @@ public static void writeTotalHits(StreamOutput out, TotalHits totalHits) throws
}

public static void writeTopDocs(StreamOutput out, TopDocsAndMaxScore topDocs) throws IOException {
if (topDocs.topDocs instanceof CollapseTopFieldDocs) {
if (topDocs.topDocs instanceof CollapseTopFieldDocs collapseDocs) {
out.writeByte((byte) 2);
CollapseTopFieldDocs collapseDocs = (CollapseTopFieldDocs) topDocs.topDocs;

writeTotalHits(out, topDocs.topDocs.totalHits);
out.writeFloat(topDocs.maxScore);
Expand All @@ -421,9 +420,8 @@ public static void writeTopDocs(StreamOutput out, TopDocsAndMaxScore topDocs) th
writeFieldDoc(out, (FieldDoc) doc);
writeSortValue(out, collapseDocs.collapseValues[i]);
}
} else if (topDocs.topDocs instanceof TopFieldDocs) {
} else if (topDocs.topDocs instanceof TopFieldDocs topFieldDocs) {
out.writeByte((byte) 1);
TopFieldDocs topFieldDocs = (TopFieldDocs) topDocs.topDocs;

writeTotalHits(out, topDocs.topDocs.totalHits);
out.writeFloat(topDocs.maxScore);
Expand Down Expand Up @@ -459,57 +457,58 @@ private static void writeMissingValue(StreamOutput out, Object missingValue) thr

private static Object readMissingValue(StreamInput in) throws IOException {
final byte id = in.readByte();
switch (id) {
case 0:
return in.readGenericValue();
case 1:
return SortField.STRING_FIRST;
case 2:
return SortField.STRING_LAST;
default:
throw new IOException("Unknown missing value id: " + id);
}
return switch (id) {
case 0 -> in.readGenericValue();
case 1 -> SortField.STRING_FIRST;
case 2 -> SortField.STRING_LAST;
default -> throw new IOException("Unknown missing value id: " + id);
};
}

public static void writeSortValue(StreamOutput out, Object field) throws IOException {
if (field == null) {
out.writeByte((byte) 0);
} else {
Class type = field.getClass();
if (type == String.class) {
switch (field) {
case null -> out.writeByte((byte) 0);
case String s -> {
out.writeByte((byte) 1);
out.writeString((String) field);
} else if (type == Integer.class) {
out.writeString(s);
}
case Integer i -> {
out.writeByte((byte) 2);
out.writeInt((Integer) field);
} else if (type == Long.class) {
out.writeInt(i);
}
case Long l -> {
out.writeByte((byte) 3);
out.writeLong((Long) field);
} else if (type == Float.class) {
out.writeLong(l);
}
case Float f -> {
out.writeByte((byte) 4);
out.writeFloat((Float) field);
} else if (type == Double.class) {
out.writeFloat(f);
}
case Double d -> {
out.writeByte((byte) 5);
out.writeDouble((Double) field);
} else if (type == Byte.class) {
out.writeDouble(d);
}
case Byte b -> {
out.writeByte((byte) 6);
out.writeByte((Byte) field);
} else if (type == Short.class) {
out.writeByte(b);
}
case Short s -> {
out.writeByte((byte) 7);
out.writeShort((Short) field);
} else if (type == Boolean.class) {
out.writeShort(s);
}
case Boolean b -> {
out.writeByte((byte) 8);
out.writeBoolean((Boolean) field);
} else if (type == BytesRef.class) {
out.writeBoolean(b);
}
case BytesRef b -> {
out.writeByte((byte) 9);
out.writeBytesRef((BytesRef) field);
} else if (type == BigInteger.class) {
// TODO: improve serialization of BigInteger
out.writeBytesRef(b);
}
case BigInteger i -> {
out.writeByte((byte) 10);
out.writeString(field.toString());
} else {
throw new IOException("Can't handle sort field value of type [" + type + "]");
out.writeByteArray(i.toByteArray());
}
default -> throw new IOException("Can't handle sort field value of type [" + field.getClass() + "]");
}
}

Expand Down Expand Up @@ -577,13 +576,12 @@ public static void writeSortField(StreamOutput out, SortField sortField) throws
);
newSortField.setMissingValue(sortField.getMissingValue());
sortField = newSortField;
} else if (sortField instanceof NonPruningSortField) {
} else if (sortField instanceof NonPruningSortField nonPruningSortField) {
// There are 2 cases of how NonPruningSortField wraps around its underlying sort field.
// Which are through the SortField class or SortedSetSortField class
// We will serialize the sort field based on the type of underlying sort field
// Here the underlying sort field is SortedSetSortField, therefore, we will follow the
// logic in serializing SortedSetSortField and also unwrap the SortField case.
NonPruningSortField nonPruningSortField = (NonPruningSortField) sortField;
if (nonPruningSortField.getDelegate().getClass() == SortedSetSortField.class) {
SortField newSortField = new SortField(
nonPruningSortField.getField(),
Expand Down Expand Up @@ -620,16 +618,12 @@ public static void writeSortField(StreamOutput out, SortField sortField) throws

private static Number readExplanationValue(StreamInput in) throws IOException {
final int numberType = in.readByte();
switch (numberType) {
case 0:
return in.readFloat();
case 1:
return in.readDouble();
case 2:
return in.readZLong();
default:
throw new IOException("Unexpected number type: " + numberType);
}
return switch (numberType) {
case 0 -> in.readFloat();
case 1 -> in.readDouble();
case 2 -> in.readZLong();
default -> throw new IOException("Unexpected number type: " + numberType);
};
}

public static Explanation readExplanation(StreamInput in) throws IOException {
Expand Down Expand Up @@ -678,7 +672,7 @@ public static boolean indexExists(final Directory directory) throws IOException

/**
* Returns {@code true} iff the given exception or
* one of it's causes is an instance of {@link CorruptIndexException},
* one of its causes is an instance of {@link CorruptIndexException},
* {@link IndexFormatTooOldException}, or {@link IndexFormatTooNewException} otherwise {@code false}.
*/
public static boolean isCorruptionException(Throwable t) {
Expand All @@ -694,20 +688,15 @@ public static Version parseVersionLenient(String toParse, Version defaultValue)

/**
* Tries to extract a segment reader from the given index reader.
* If no SegmentReader can be extracted an {@link IllegalStateException} is thrown.
* Throws {@link IllegalStateException} if segment reader cannot be extracted.
*/
public static SegmentReader segmentReader(LeafReader reader) {
if (reader instanceof SegmentReader) {
return (SegmentReader) reader;
} else if (reader instanceof FilterLeafReader) {
final FilterLeafReader fReader = (FilterLeafReader) reader;
return segmentReader(FilterLeafReader.unwrap(fReader));
} else if (reader instanceof FilterCodecReader) {
final FilterCodecReader fReader = (FilterCodecReader) reader;
return segmentReader(FilterCodecReader.unwrap(fReader));
}
// hard fail - we can't get a SegmentReader
throw new IllegalStateException("Can not extract segment reader from given index reader [" + reader + "]");
return switch (reader) {
case SegmentReader sReader -> sReader;
case FilterLeafReader fReader -> segmentReader(FilterLeafReader.unwrap(fReader));
case FilterCodecReader fcReader -> segmentReader(FilterCodecReader.unwrap(fcReader));
default -> throw new IllegalStateException("Can not extract segment reader from given index reader [" + reader + "]");
};
}

@SuppressForbidden(reason = "Version#parseLeniently() used in a central place")
Expand Down
Loading