Skip to content

nested document structures may lead to repetitions of the same group. #709

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

nested document structures may lead to repetitions of the same group. #709

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3ffdfd5
adding inner hash class to calculate hash to be able to identify dupl…
Sep 17, 2014
fbd91fd
Merge branch 'feature-fixduplicate'
Sep 17, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions src/main/java/org/xbib/elasticsearch/plugin/jdbc/PlainIndexableObject.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,16 @@ protected boolean checkCollapsedMapLength(Map<String, Object> map) {
@SuppressWarnings({"unchecked"})
protected XContentBuilder toXContent(XContentBuilder builder, Params params, List list) throws IOException {
builder.startArray();
HashValueCalculator hashUtil = new HashValueCalculator();
long prev = 0;
long current = 0;
for (Object o : list) {
current = hashUtil.calculate(o);
if ( current == prev ) {
continue;
} else {
prev = current;
}
if (o instanceof Values) {
Values v = (Values) o;
v.toXContent(builder, ToXContent.EMPTY_PARAMS);
Expand Down Expand Up @@ -233,4 +242,58 @@ public int hashCode() {
return hash;
}

/**
* Implementing hash calculator on es graph object. it is used to identify same graps in the structure
*/
private class HashValueCalculator {

public long calculate(Object o) throws IOException {
long hash = 0;
if (o instanceof Values) {
Values v = (Values) o;
hash = calculate(v);
} else if (o instanceof Map) {
hash = calculate((Map<String, Object>) o);
} else if (o instanceof List) {
hash = calculate((List) o);
} else {
try {
hash = hash + o.hashCode();
} catch (Exception e) {
throw new IOException("unknown object class:" + o.getClass().getName());
}
}
return hash;
}

public long calculate(Values values) throws IOException {
long hash = 0;
if ( values != null && !values.isNull() ) {
for ( Object o : values.getValues()) {
hash = hash + o.hashCode();
}
}
return hash;
}

public long calculate(List list) throws IOException {
long hash = 0;
for (Object o : list) {
hash = hash + calculate(o);
}
return hash;
}

public long calculate(Map<String, Object> map) throws IOException {
long hash = 0;
for (Map.Entry<String, Object> k : map.entrySet()) {
Object o = k.getValue();
if (ignoreNull && (o == null || (o instanceof Values) && ((Values) o).isNull())) {
continue;
}
hash = hash + calculate(o);
}
return hash;
}
}
}
4 changes: 3 additions & 1 deletion src/test/java/org/xbib/elasticsearch/river/jdbc/support/ValueListenerTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@ public void testArrays() throws Exception {
List<String> row1 = Arrays.asList("4679", "Tesla, Abe and Elba", "2014-01-06 00:00:00", "3917", "Idris Elba", "9450", "/web/q/g/h/57436356.jpg");
List<String> row2 = Arrays.asList("4679", "Tesla, Abe and Elba", "2014-01-06 00:00:00", "3917", "Idris Elba", "9965", "/web/i/s/q/GS3193626.jpg");
List<String> row3 = Arrays.asList("4679", "Tesla, Abe and Elba", "2014-01-06 00:00:00", "3917", "Idris Elba", "9451", "/web/i/s/q/GS3193626.jpg");
List<String> row4 = Arrays.asList("4679", "Tesla, Abe and Elba", "2014-01-06 00:00:00", "3918", "Idris Elba", "9451", "/web/i/s/q/GS3193626.jpg");
MockRiverMouth output = new MockRiverMouth();
new StringKeyValueStreamListener()
.output(output)
Expand All @@ -239,9 +240,10 @@ public void testArrays() throws Exception {
.values(row1)
.values(row2)
.values(row3)
.values(row4)
.end();
assertEquals(output.data().toString(),
"{[null/null/null/4679]->{blog={name=\"Tesla, Abe and Elba\", published=\"2014-01-06 00:00:00\", association=[{id=\"3917\", name=\"Idris Elba\"}, {id=\"3917\", name=\"Idris Elba\"}, {id=\"3917\", name=\"Idris Elba\"}], attachment=[{id=\"9450\", name=\"/web/q/g/h/57436356.jpg\"}, {id=\"9965\", name=\"/web/i/s/q/GS3193626.jpg\"}, {id=\"9451\", name=\"/web/i/s/q/GS3193626.jpg\"}]}}={\"blog\":{\"name\":\"Tesla, Abe and Elba\",\"published\":\"2014-01-06 00:00:00\",\"association\":[{\"id\":\"3917\",\"name\":\"Idris Elba\"},{\"id\":\"3917\",\"name\":\"Idris Elba\"},{\"id\":\"3917\",\"name\":\"Idris Elba\"}],\"attachment\":[{\"id\":\"9450\",\"name\":\"/web/q/g/h/57436356.jpg\"},{\"id\":\"9965\",\"name\":\"/web/i/s/q/GS3193626.jpg\"},{\"id\":\"9451\",\"name\":\"/web/i/s/q/GS3193626.jpg\"}]}}}"
"{[null/null/null/4679]->{blog={name=\"Tesla, Abe and Elba\", published=\"2014-01-06 00:00:00\", association=[{id=\"3917\", name=\"Idris Elba\"}, {id=\"3917\", name=\"Idris Elba\"}, {id=\"3917\", name=\"Idris Elba\"}, {id=\"3918\", name=\"Idris Elba\"}], attachment=[{id=\"9450\", name=\"/web/q/g/h/57436356.jpg\"}, {id=\"9965\", name=\"/web/i/s/q/GS3193626.jpg\"}, {id=\"9451\", name=\"/web/i/s/q/GS3193626.jpg\"}, {id=\"9451\", name=\"/web/i/s/q/GS3193626.jpg\"}]}}={\"blog\":{\"name\":\"Tesla, Abe and Elba\",\"published\":\"2014-01-06 00:00:00\",\"association\":[{\"id\":\"3917\",\"name\":\"Idris Elba\"},{\"id\":\"3918\",\"name\":\"Idris Elba\"}],\"attachment\":[{\"id\":\"9450\",\"name\":\"/web/q/g/h/57436356.jpg\"},{\"id\":\"9965\",\"name\":\"/web/i/s/q/GS3193626.jpg\"},{\"id\":\"9451\",\"name\":\"/web/i/s/q/GS3193626.jpg\"}]}}}"
);
}

Expand Down