From 960c599e3bea98943eecae28e75de4abd68761a6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 14:36:34 +0000 Subject: [PATCH 01/29] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a4625fba2ab..2d0a8eb30c0 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ - 2.3.1 + 2.3.2-SNAPSHOT false UTF-8 From 3b93f97bcbe2f2b064cd399ce9badd325dee78d9 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:21:59 +0000 Subject: [PATCH 02/29] Gh-3334: Fix Gremlin Reusing Accumulo Iterators (#3335) * move to set where applicable * update test * Fix gremlin select step * remove deprecated parameter * use safe add of labels --- .../gchq/gaffer/tinkerpop/GafferPopGraph.java | 64 +++++++++--------- .../traversal/step/GafferPopHasStep.java | 2 +- .../traversal/step/GafferPopVertexStep.java | 5 +- .../process/traversal/step/LazyFoldStep.java | 66 ------------------- .../GafferPopVertexStepStrategy.java | 11 ++-- .../traversal/util/GafferVertexUtils.java | 45 +++++++------ .../gaffer/tinkerpop/GafferPopGraphTest.java | 2 +- .../GafferPopGraphStepStrategyCypherIT.java | 2 +- .../rest/controller/GremlinController.java | 7 +- 9 files changed, 70 insertions(+), 134 deletions(-) delete mode 100644 library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/LazyFoldStep.java diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java index e416951b825..af32329b2e9 100755 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java @@ -412,7 +412,6 @@ public void addEdge(final GafferPopEdge edge) { public Iterator vertices(final Object... vertexIds) { final boolean getAll = null == vertexIds || 0 == vertexIds.length; final OperationChain> getOperation; - final Iterable orphanVertices; LOGGER.debug(GET_DEBUG_MSG, variables.getElementsLimit()); if (getAll) { @@ -430,29 +429,28 @@ public Iterator vertices(final Object... vertexIds) { .then(new Limit(variables.getElementsLimit(), true)) .build(); } - // Run requested chain on the graph - final Iterable result = execute(getOperation); + // Run requested chain on the graph and buffer result to set to avoid reusing iterator + final Set result = new HashSet<>(IterableUtils.toList(execute(getOperation))); // Translate results to Gafferpop elements final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); - final Iterable translatedResults = () -> StreamSupport.stream(result.spliterator(), false) + final Set translatedResults = StreamSupport.stream(result.spliterator(), false) .map(generator::_apply) .filter(Vertex.class::isInstance) .map(e -> (Vertex) e) .limit(variables.getElementsLimit()) - .iterator(); + .collect(Collectors.toSet()); - if (IterableUtils.size(translatedResults) >= variables.getElementsLimit()) { + if (translatedResults.size() >= variables.getElementsLimit()) { LOGGER.warn( "Result size is greater than or equal to configured limit ({}). Results may have been truncated", variables.getElementsLimit()); } // Check for seeds that are not entities but are vertices on an edge (orphan vertices) - orphanVertices = GafferVertexUtils.getOrphanVertices(result, this, vertexIds); - Iterable chainedIterable = IterableUtils.chainedIterable(translatedResults, orphanVertices); + translatedResults.addAll(GafferVertexUtils.getOrphanVertices(result, this, vertexIds)); - return chainedIterable.iterator(); + return translatedResults.iterator(); } /** @@ -597,19 +595,19 @@ public Iterator edges(final Object... elementIds) { .build(); } - // Run requested chain on the graph - final Iterable result = execute(getOperation); + // Run requested chain on the graph and buffer to set to avoid reusing iterator + final Set result = new HashSet<>(IterableUtils.toList(execute(getOperation))); // Translate results to Gafferpop elements final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); - final Iterable translatedResults = () -> StreamSupport.stream(result.spliterator(), false) + final Set translatedResults = StreamSupport.stream(result.spliterator(), false) .map(generator::_apply) .filter(Edge.class::isInstance) .map(e -> (Edge) e) .limit(variables.getElementsLimit()) - .iterator(); + .collect(Collectors.toSet()); - if (IterableUtils.size(translatedResults) >= variables.getElementsLimit()) { + if (translatedResults.size() >= variables.getElementsLimit()) { LOGGER.warn( "Result size is greater than or equal to configured limit ({}). Results may have been truncated", variables.getElementsLimit()); @@ -811,20 +809,19 @@ private Iterator verticesWithSeedsAndView(final List result = execute(getOperation); + // Run operation on graph buffer to set to avoid reusing iterator + final Set result = new HashSet<>(IterableUtils.toList(execute(getOperation))); // Translate results to Gafferpop elements final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); - final Iterable translatedResults = () -> StreamSupport.stream(result.spliterator(), false) + final Set translatedResults = StreamSupport.stream(result.spliterator(), false) .map(generator::_apply) .filter(GafferPopVertex.class::isInstance) .map(e -> (GafferPopVertex) e) .limit(variables.getElementsLimit()) - .iterator(); + .collect(Collectors.toSet()); return translatedResults.iterator(); - } private Iterator adjVerticesWithSeedsAndView(final List seeds, final Direction direction, final View view) { @@ -840,30 +837,29 @@ private Iterator adjVerticesWithSeedsAndView(final List see .build()) .build()); - List seedList = StreamSupport.stream(getAdjEntitySeeds.spliterator(), false).collect(Collectors.toList()); + List seedList = IterableUtils.toList(getAdjEntitySeeds); // GetAdjacentIds provides list of entity seeds so run a GetElements to get the actual Entities - final Iterable result = execute(new OperationChain.Builder() - .first(new GetElements.Builder() - .input(seedList) - .build()) - .build()); + final Set result = new HashSet<>(IterableUtils.toList( + execute(new OperationChain.Builder() + .first(new GetElements.Builder() + .input(seedList) + .build()) + .build()))); // Translate results to Gafferpop elements final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); - final Iterable translatedResults = () -> StreamSupport.stream(result.spliterator(), false) + final Set translatedResults = StreamSupport.stream(result.spliterator(), false) .map(generator::_apply) .filter(Vertex.class::isInstance) .map(e -> (Vertex) e) - .iterator(); + .collect(Collectors.toSet()); // Check for seeds that are not entities but are vertices on an edge (orphan vertices) - Iterable chainedIterable = translatedResults; for (final EntityId seed : seedList) { - Iterable orphanVertices = GafferVertexUtils.getOrphanVertices(result, this, seed.getVertex()); - chainedIterable = IterableUtils.chainedIterable(chainedIterable, orphanVertices); + translatedResults.addAll(GafferVertexUtils.getOrphanVertices(result, this, seed.getVertex())); } - return chainedIterable.iterator(); + return translatedResults.iterator(); } private Iterator edgesWithSeedsAndView(final List seeds, final Direction direction, final View view) { @@ -900,16 +896,16 @@ private Iterator edgesWithSeedsAndView(final List seeds, fina } // Run requested chain on the graph - final Iterable result = execute(getOperation); + final Set result = new HashSet<>(IterableUtils.toList(execute(getOperation))); // Translate results to Gafferpop elements final GafferPopElementGenerator generator = new GafferPopElementGenerator(this, true); - final Iterable translatedResults = () -> StreamSupport.stream(result.spliterator(), false) + final Set translatedResults = StreamSupport.stream(result.spliterator(), false) .map(generator::_apply) .filter(Edge.class::isInstance) .map(e -> (Edge) e) .limit(variables.getElementsLimit()) - .iterator(); + .collect(Collectors.toSet()); return translatedResults.iterator(); } diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopHasStep.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopHasStep.java index dcf36bcf715..9403537572a 100644 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopHasStep.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopHasStep.java @@ -37,7 +37,7 @@ public class GafferPopHasStep extends HasStep { public GafferPopHasStep(final HasStep originalHasStep) { super(originalHasStep.getTraversal()); LOGGER.debug("Running custom HasStep on GafferPopGraph"); - + originalHasStep.getLabels().forEach(this::addLabel); originalHasStep.getHasContainers().forEach(this::addHasContainer); } diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopVertexStep.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopVertexStep.java index 0d4a06341f2..d3dd9ed4188 100644 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopVertexStep.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopVertexStep.java @@ -62,7 +62,7 @@ * (GafferPop) g.V().out() = [v2, v3, v4, v5] * */ -public class GafferPopVertexStep extends FlatMapStep, E> +public class GafferPopVertexStep extends FlatMapStep, E> implements AutoCloseable, Configuring { private static final Logger LOGGER = LoggerFactory.getLogger(GafferPopVertexStep.class); @@ -77,6 +77,7 @@ public GafferPopVertexStep(final VertexStep originalVertexStep) { this.edgeLabels = originalVertexStep.getEdgeLabels(); this.returnClass = originalVertexStep.getReturnClass(); this.traversal = originalVertexStep.getTraversal(); + originalVertexStep.getLabels().forEach(this::addLabel); } @Override @@ -90,7 +91,7 @@ public void configure(final Object... keyValues) { } @Override - protected Iterator flatMap(final Traverser.Admin> traverser) { + protected Iterator flatMap(final Traverser.Admin> traverser) { return Vertex.class.isAssignableFrom(returnClass) ? (Iterator) this.vertices(traverser.get()) : (Iterator) this.edges(traverser.get()); diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/LazyFoldStep.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/LazyFoldStep.java deleted file mode 100644 index f08cd3a3ce7..00000000000 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/LazyFoldStep.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2024 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package uk.gov.gchq.gaffer.tinkerpop.process.traversal.step; - -import org.apache.tinkerpop.gremlin.process.traversal.Step; -import org.apache.tinkerpop.gremlin.process.traversal.Traversal; -import org.apache.tinkerpop.gremlin.process.traversal.Traverser.Admin; -import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep; -import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException; - -import java.util.Iterator; - -/** - * A Lazy implementation of the FoldStep - * Folds to a lazy iterable - */ -public class LazyFoldStep extends AbstractStep> { - - private boolean done = false; - - public LazyFoldStep(final Traversal.Admin> traversal) { - super(traversal); - } - - @Override - public Admin> processNextStart() { - if (done) { - throw FastNoSuchElementException.instance(); - } - - // Do nothing on subsequent calls - this.done = true; - - Iterable lazyIterable = () -> new Iterator() { - @Override - public boolean hasNext() { - return LazyFoldStep.this.starts.hasNext(); - } - - @Override - public S next() { - if (!hasNext()) { - throw FastNoSuchElementException.instance(); - } - - return LazyFoldStep.this.starts.next().get(); - } - }; - - return this.getTraversal().getTraverserGenerator().generate(lazyIterable, (Step) this, 1L); - } -} diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/strategy/optimisation/GafferPopVertexStepStrategy.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/strategy/optimisation/GafferPopVertexStepStrategy.java index fb5972ee083..0bd8412112a 100644 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/strategy/optimisation/GafferPopVertexStepStrategy.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/strategy/optimisation/GafferPopVertexStepStrategy.java @@ -18,6 +18,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traversal.Admin; import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.FoldStep; import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep; import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy; import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper; @@ -27,7 +28,8 @@ import org.slf4j.LoggerFactory; import uk.gov.gchq.gaffer.tinkerpop.process.traversal.step.GafferPopVertexStep; -import uk.gov.gchq.gaffer.tinkerpop.process.traversal.step.LazyFoldStep; + +import java.util.List; /** * Optimisation strategy to reduce the number of Gaffer operations performed. @@ -49,14 +51,13 @@ public void apply(final Admin traversal) { LOGGER.debug("Inserting FoldStep and replacing VertexStep"); // Replace vertex step - final GafferPopVertexStep listVertexStep = new GafferPopVertexStep<>( - originalVertexStep); + final GafferPopVertexStep listVertexStep = new GafferPopVertexStep<>(originalVertexStep); TraversalHelper.replaceStep(originalVertexStep, listVertexStep, traversal); // Add in a fold step before the new VertexStep so that the input is the list of // all vertices - LazyFoldStep lazyFoldStep = new LazyFoldStep<>(originalVertexStep.getTraversal()); - TraversalHelper.insertBeforeStep(lazyFoldStep, listVertexStep, traversal); + FoldStep> foldStep = new FoldStep<>(originalVertexStep.getTraversal()); + TraversalHelper.insertBeforeStep(foldStep, listVertexStep, traversal); }); } diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/util/GafferVertexUtils.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/util/GafferVertexUtils.java index be409317801..11dea281d87 100644 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/util/GafferVertexUtils.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/util/GafferVertexUtils.java @@ -27,8 +27,8 @@ import uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph; import uk.gov.gchq.gaffer.tinkerpop.GafferPopVertex; -import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -42,18 +42,19 @@ private GafferVertexUtils() { } /** - * Util method to extract vertices that are vertices on an edge but do not have an + * Util method to extract vertices that are vertices on an edge but do not have + * an * associated {@link Vertex} or {@link Entity} the current graph. * These vertices exist only on an edge. * * - * @param result The results from a Gaffer query - * @param graph The GafferPop graph being queried + * @param result The results from a Gaffer query + * @param graph The GafferPop graph being queried * @param vertexIds The vertexIds that have been used as seeds in the query - * @return Iterable of 'orphan' {@link Vertex}'s + * @return {@link Collection} of 'orphan' {@link Vertex}'s */ - public static Iterable getOrphanVertices(final Iterable result, final GafferPopGraph graph, final Object... vertexIds) { + public static Collection getOrphanVertices(final Iterable result, final GafferPopGraph graph, final Object... vertexIds) { // Check for any vertex ID seeds that are not returned as Entities List orphanVertexIds = Arrays.stream(vertexIds) .filter(id -> StreamSupport.stream(result.spliterator(), false) @@ -73,21 +74,23 @@ public static Iterable getOrphanVertices(final Iterable extractOrphanVerticesFromEdges(final Iterable result, final GafferPopGraph graph, final List orphanVertexIds) { - List orphanVertices = new ArrayList<>(); - StreamSupport.stream(result.spliterator(), false) - .filter(Edge.class::isInstance) - .map(e -> (Edge) e) - .forEach(e -> { - if (orphanVertexIds.contains(e.getSource()) || orphanVertexIds.equals(e.getSource())) { - orphanVertices.add(new GafferPopVertex(GafferPopGraph.ID_LABEL, GafferCustomTypeFactory.parseForGraphSONv3(e.getSource()), graph)); - } - if (orphanVertexIds.contains(e.getDestination()) || orphanVertexIds.equals(e.getDestination())) { - orphanVertices.add(new GafferPopVertex(GafferPopGraph.ID_LABEL, GafferCustomTypeFactory.parseForGraphSONv3(e.getDestination()), graph)); - } - }); - return orphanVertices; + private static Collection extractOrphanVerticesFromEdges(final Iterable result, final GafferPopGraph graph, final List orphanVertexIds) { + return StreamSupport.stream(result.spliterator(), false) + .filter(Edge.class::isInstance) + .map(e -> (Edge) e) + .map(e -> { + if (orphanVertexIds.contains(e.getSource()) || orphanVertexIds.equals(e.getSource())) { + return new GafferPopVertex(GafferPopGraph.ID_LABEL, GafferCustomTypeFactory.parseForGraphSONv3(e.getSource()), graph); + } + if (orphanVertexIds.contains(e.getDestination()) || orphanVertexIds.equals(e.getDestination())) { + return new GafferPopVertex(GafferPopGraph.ID_LABEL, GafferCustomTypeFactory.parseForGraphSONv3(e.getDestination()), graph); + } + return e; + }) + .filter(Vertex.class::isInstance) + .map(v -> (Vertex) v) + .collect(Collectors.toSet()); } } diff --git a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphTest.java b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphTest.java index 468f7285de9..503fbf82e99 100644 --- a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphTest.java +++ b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphTest.java @@ -507,7 +507,7 @@ void shouldGetAllEdges() { final Iterator edges = graph.edges(); // Then - assertThat(edges).toIterable().containsExactly(createdEdge, knowsEdge); + assertThat(edges).toIterable().containsExactlyInAnyOrder(createdEdge, knowsEdge); } @Test diff --git a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/strategy/GafferPopGraphStepStrategyCypherIT.java b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/strategy/GafferPopGraphStepStrategyCypherIT.java index c4040fea4a9..24f7f8bc7da 100644 --- a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/strategy/GafferPopGraphStepStrategyCypherIT.java +++ b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/strategy/GafferPopGraphStepStrategyCypherIT.java @@ -45,7 +45,7 @@ class GafferPopGraphStepStrategyCypherIT { @BeforeAll public static void beforeAll() { - GafferPopGraph gafferPopGraph = GafferPopModernTestUtils.createModernGraph(GafferPopGraphStepStrategyCypherIT.class, StoreType.MAP); + GafferPopGraph gafferPopGraph = GafferPopModernTestUtils.createModernGraph(GafferPopGraphStepStrategyCypherIT.class, StoreType.ACCUMULO); g = gafferPopGraph.traversal(); } diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java index 14f387ffce0..6be0fd9a842 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java @@ -181,13 +181,14 @@ public ResponseEntity execute( @PostMapping(path = "/cypher/explain", consumes = TEXT_PLAIN_VALUE, produces = APPLICATION_JSON_VALUE) @io.swagger.v3.oas.annotations.Operation( summary = "Explain a Cypher Query Executed via Gremlin", - description = "Translates a Cypher query to Gremlin and outputs an explanation of what Gaffer operations" + + description = "Translates a Cypher query to Gremlin and outputs an explanation of what Gaffer operations " + "were executed on the graph, note will always append a '.toList()' to the translation") public String cypherExplain(@RequestHeader final HttpHeaders httpHeaders, @RequestBody final String cypherQuery) { final CypherAst ast = CypherAst.parse(cypherQuery); // Translate the cypher to gremlin, always add a .toList() otherwise Gremlin wont execute it as its lazy - final String translation = ast.buildTranslation(Translator.builder().gremlinGroovy().enableCypherExtensions().build()) + ".toList()"; + final String translation = ast.buildTranslation( + Translator.builder().gremlinGroovy().enableCypherExtensions().build()) + ".toList()"; JSONObject response = runGremlinQuery(translation).get1(); response.put(EXPLAIN_GREMLIN_KEY, translation); @@ -207,7 +208,7 @@ public String cypherExplain(@RequestHeader final HttpHeaders httpHeaders, @Reque @PostMapping(path = "/cypher/execute", consumes = TEXT_PLAIN_VALUE, produces = APPLICATION_NDJSON_VALUE) @io.swagger.v3.oas.annotations.Operation( summary = "Run a Cypher Query", - description = "Translates a Cypher query to Gremlin and executes it returning a GraphSONv3 JSON result." + + description = "Translates a Cypher query to Gremlin and executes it returning a GraphSONv3 JSON result. " + "Note will always append a '.toList()' to the translation") public ResponseEntity cypherExecute( @RequestHeader final HttpHeaders httpHeaders, From 7726ff3f7c5873c01859978168887d685f099461 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:54:17 +0000 Subject: [PATCH 03/29] Gh-3340: Misleading vertices in Gremlin (#3341) * make orphaned search optional and increase limit * testing updates * typo * sonar fixes --- .../gchq/gaffer/tinkerpop/GafferPopGraph.java | 133 ++++++++++-------- .../tinkerpop/GafferPopGraphVariables.java | 36 ++++- .../traversal/step/GafferPopGraphStep.java | 6 +- .../tinkerpop/GafferPopFederationTests.java | 12 +- .../gaffer/tinkerpop/GafferPopGraphIT.java | 16 ++- .../gaffer/tinkerpop/GafferPopGraphTest.java | 15 +- .../test/resources/gafferpop-test.properties | 3 +- 7 files changed, 139 insertions(+), 82 deletions(-) diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java index af32329b2e9..29d9f6159c7 100755 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java @@ -52,6 +52,7 @@ import uk.gov.gchq.gaffer.operation.impl.get.GetAllElements; import uk.gov.gchq.gaffer.operation.impl.get.GetElements; import uk.gov.gchq.gaffer.operation.io.Input; +import uk.gov.gchq.gaffer.operation.io.Output; import uk.gov.gchq.gaffer.store.operation.GetSchema; import uk.gov.gchq.gaffer.store.schema.Schema; import uk.gov.gchq.gaffer.tinkerpop.generator.GafferEdgeGenerator; @@ -86,6 +87,10 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; +import static uk.gov.gchq.gaffer.tinkerpop.GafferPopGraphVariables.DEFAULT_GET_ELEMENTS_LIMIT; +import static uk.gov.gchq.gaffer.tinkerpop.GafferPopGraphVariables.DEFAULT_HAS_STEP_FILTER_STAGE; + + /** * A GafferPopGraph is an implementation of * {@link org.apache.tinkerpop.gremlin.structure.Graph}. @@ -187,29 +192,25 @@ public class GafferPopGraph implements org.apache.tinkerpop.gremlin.structure.Gr */ public static final String GET_ELEMENTS_LIMIT = "gaffer.elements.getlimit"; - /** - * Default value for the max number of elements returned by getElements - */ - public static final int DEFAULT_GET_ELEMENTS_LIMIT = 5000; - /** * Configuration key for when to apply HasStep filtering */ public static final String HAS_STEP_FILTER_STAGE = "gaffer.elements.hasstepfilterstage"; - public enum HasStepFilterStage { - PRE_AGGREGATION, - POST_AGGREGATION, - POST_TRANSFORM - } - /** - * Default to pre-aggregation filtering for HasStep predicates + * Configuration key to set if orphaned vertices (e.g. vertices without an entity) + * should be included in the result by default */ - public static final HasStepFilterStage DEFAULT_HAS_STEP_FILTER_STAGE = HasStepFilterStage.PRE_AGGREGATION; + public static final String INCLUDE_ORPHANED_VERTICES = "gaffer.includeOrphanedVertices"; + /** + * Set default user ID to use if not set by the user factory. + */ public static final String USER_ID = "gaffer.userId"; + /** + * Set default data auths if not set by the user factory. + */ public static final String DATA_AUTHS = "gaffer.dataAuths"; /** @@ -410,27 +411,41 @@ public void addEdge(final GafferPopEdge edge) { */ @Override public Iterator vertices(final Object... vertexIds) { + LOGGER.debug(GET_DEBUG_MSG, variables.getElementsLimit()); final boolean getAll = null == vertexIds || 0 == vertexIds.length; - final OperationChain> getOperation; + final Output> getOperation; - LOGGER.debug(GET_DEBUG_MSG, variables.getElementsLimit()); if (getAll) { - getOperation = new Builder() - .first(new GetAllElements.Builder() - .view(createAllEntitiesView()) - .build()) - .then(new Limit(variables.getElementsLimit(), true)) - .build(); + final GetAllElements.Builder builder = new GetAllElements.Builder(); + // If we are not including orphans then apply the all entities view + if (!variables.getIncludeOrphanedVertices()) { + builder.view(createAllEntitiesView()); + } + getOperation = builder.build(); + } else { - getOperation = new Builder() - .first(new GetElements.Builder() - .input(getElementSeeds(Arrays.asList(vertexIds))) - .build()) - .then(new Limit(variables.getElementsLimit(), true)) - .build(); + final GetElements.Builder builder = new GetElements.Builder() + .input(getElementSeeds(Arrays.asList(vertexIds))); + // If we are not including orphans then apply the all entities view + if (!variables.getIncludeOrphanedVertices()) { + builder.view(createAllEntitiesView()); + } + getOperation = builder.build(); } + // Run requested chain on the graph and buffer result to set to avoid reusing iterator - final Set result = new HashSet<>(IterableUtils.toList(execute(getOperation))); + final OperationChain> chain = new Builder() + .first(getOperation) + .then(new Limit<>(variables.getElementsLimit(), true)) + .build(); + final Set result = new HashSet<>(IterableUtils.toList(execute(chain))); + + // Warn of truncation + if (result.size() >= variables.getElementsLimit()) { + LOGGER.warn( + "Result size is greater than or equal to configured limit ({}). Results may have been truncated", + variables.getElementsLimit()); + } // Translate results to Gafferpop elements final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); @@ -438,17 +453,12 @@ public Iterator vertices(final Object... vertexIds) { .map(generator::_apply) .filter(Vertex.class::isInstance) .map(e -> (Vertex) e) - .limit(variables.getElementsLimit()) .collect(Collectors.toSet()); - if (translatedResults.size() >= variables.getElementsLimit()) { - LOGGER.warn( - "Result size is greater than or equal to configured limit ({}). Results may have been truncated", - variables.getElementsLimit()); - } - // Check for seeds that are not entities but are vertices on an edge (orphan vertices) - translatedResults.addAll(GafferVertexUtils.getOrphanVertices(result, this, vertexIds)); + if (variables.getIncludeOrphanedVertices()) { + translatedResults.addAll(GafferVertexUtils.getOrphanVertices(result, this, vertexIds)); + } return translatedResults.iterator(); } @@ -756,6 +766,8 @@ public void setDefaultVariables(final GafferPopGraphVariables variables) { configuration().getInteger(GET_ELEMENTS_LIMIT, DEFAULT_GET_ELEMENTS_LIMIT)); variables.set(GafferPopGraphVariables.HAS_STEP_FILTER_STAGE, configuration().getString(HAS_STEP_FILTER_STAGE, DEFAULT_HAS_STEP_FILTER_STAGE.toString())); + variables.set(GafferPopGraphVariables.INCLUDE_ORPHANED_VERTICES, + configuration().getBoolean(INCLUDE_ORPHANED_VERTICES, false)); variables.set(GafferPopGraphVariables.LAST_OPERATION_CHAIN, new OperationChain()); } @@ -782,24 +794,15 @@ private Iterator verticesWithSeedsAndView(final List> getOperation; + final Output> getOperation; LOGGER.debug(GET_DEBUG_MSG, variables.getElementsLimit()); if (getAll) { - getOperation = new Builder() - .first(new GetAllElements.Builder() - .view(entitiesView) - .build()) - .then(new Limit<>(variables.getElementsLimit(), true)) - .build(); + getOperation = new GetAllElements.Builder().view(entitiesView).build(); } else { - getOperation = new Builder() - .first(new GetElements.Builder() - .input(seeds) - .view(entitiesView) - .build()) - .then(new Limit<>(variables.getElementsLimit(), true)) - .build(); - + getOperation = new GetElements.Builder() + .input(seeds) + .view(entitiesView) + .build(); if (null == entitiesView || entitiesView.getEntityGroups().contains(ID_LABEL)) { seeds.forEach(seed -> { if (seed instanceof EntitySeed) { @@ -810,7 +813,11 @@ private Iterator verticesWithSeedsAndView(final List result = new HashSet<>(IterableUtils.toList(execute(getOperation))); + final OperationChain> chain = new Builder() + .first(getOperation) + .then(new Limit<>(variables.getElementsLimit(), true)) + .build(); + final Set result = new HashSet<>(IterableUtils.toList(execute(chain))); // Translate results to Gafferpop elements final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); @@ -818,7 +825,6 @@ private Iterator verticesWithSeedsAndView(final List (GafferPopVertex) e) - .limit(variables.getElementsLimit()) .collect(Collectors.toSet()); return translatedResults.iterator(); @@ -829,7 +835,8 @@ private Iterator adjVerticesWithSeedsAndView(final List see throw new UnsupportedOperationException("There could be a lot of vertices, so please add some seeds"); } - final Iterable getAdjEntitySeeds = execute(new OperationChain.Builder() + final Iterable getAdjEntitySeeds = execute( + new OperationChain.Builder() .first(new GetAdjacentIds.Builder() .input(seeds) .view(view) @@ -839,13 +846,15 @@ private Iterator adjVerticesWithSeedsAndView(final List see List seedList = IterableUtils.toList(getAdjEntitySeeds); + final GetElements.Builder builder = new GetElements.Builder().input(seedList); + // If we are not including orphans then apply the all entities view + if (!variables.getIncludeOrphanedVertices()) { + builder.view(createAllEntitiesView()); + } + // GetAdjacentIds provides list of entity seeds so run a GetElements to get the actual Entities final Set result = new HashSet<>(IterableUtils.toList( - execute(new OperationChain.Builder() - .first(new GetElements.Builder() - .input(seedList) - .build()) - .build()))); + execute(new OperationChain.Builder().first(builder.build()).build()))); // Translate results to Gafferpop elements final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); @@ -856,9 +865,11 @@ private Iterator adjVerticesWithSeedsAndView(final List see .collect(Collectors.toSet()); // Check for seeds that are not entities but are vertices on an edge (orphan vertices) - for (final EntityId seed : seedList) { - translatedResults.addAll(GafferVertexUtils.getOrphanVertices(result, this, seed.getVertex())); + if (variables.getIncludeOrphanedVertices()) { + translatedResults.addAll( + GafferVertexUtils.getOrphanVertices(result, this, seedList.stream().map(EntityId::getVertex).toArray(Object[]::new))); } + return translatedResults.iterator(); } diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphVariables.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphVariables.java index 01ede3dbec2..99135a38de4 100644 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphVariables.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphVariables.java @@ -33,6 +33,8 @@ public final class GafferPopGraphVariables implements Graph.Variables { private static final Logger LOGGER = LoggerFactory.getLogger(GafferPopGraphVariables.class); private static final String VAR_UPDATE_ERROR_STRING = "Ignoring update variable: {}, incorrect value type: {}"; + // KEYS + /** * Variable key for the {@link Map} of Gaffer operation options. */ @@ -54,7 +56,8 @@ public final class GafferPopGraphVariables implements Graph.Variables { public static final String USER = "user"; /** - * The max number of elements that can be returned by GetElements + * The max number of elements that can be returned by a single GetElements + * or GetAllElements */ public static final String GET_ELEMENTS_LIMIT = "getElementsLimit"; @@ -73,6 +76,29 @@ public final class GafferPopGraphVariables implements Graph.Variables { */ public static final String LAST_OPERATION_CHAIN = "lastOperation"; + /** + * The key to set if orphaned vertices (e.g. vertices without an entity) + * should be included in the result + */ + public static final String INCLUDE_ORPHANED_VERTICES = "includeOrphanedVertices"; + + // DEFAULTS + + /** + * Default value for the max number of elements returned by getElements + */ + public static final int DEFAULT_GET_ELEMENTS_LIMIT = 20000; + + /** + * Default to pre-aggregation filtering for HasStep predicates + */ + public static final HasStepFilterStage DEFAULT_HAS_STEP_FILTER_STAGE = HasStepFilterStage.PRE_AGGREGATION; + + public enum HasStepFilterStage { + PRE_AGGREGATION, + POST_AGGREGATION, + POST_TRANSFORM + } private final Map variables; @@ -121,6 +147,10 @@ public void set(final String key, final Object value) { } break; + case INCLUDE_ORPHANED_VERTICES: + variables.put(key, Boolean.valueOf(value.toString())); + break; + default: variables.put(key, value); break; @@ -169,6 +199,10 @@ public OperationChain getLastOperationChain() { return (OperationChain) variables.get(LAST_OPERATION_CHAIN); } + public boolean getIncludeOrphanedVertices() { + return Boolean.parseBoolean(variables.get(INCLUDE_ORPHANED_VERTICES).toString()); + } + public String toString() { return StringFactory.graphVariablesString(this); } diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java index fafee9b8148..23aac9a8039 100644 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java @@ -36,8 +36,8 @@ import uk.gov.gchq.gaffer.data.elementdefinition.view.View; import uk.gov.gchq.gaffer.data.elementdefinition.view.ViewElementDefinition; import uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph; -import uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph.HasStepFilterStage; import uk.gov.gchq.gaffer.tinkerpop.GafferPopGraphVariables; +import uk.gov.gchq.gaffer.tinkerpop.GafferPopGraphVariables.HasStepFilterStage; import uk.gov.gchq.gaffer.tinkerpop.process.traversal.step.util.GafferPopHasContainer; import uk.gov.gchq.koryphe.impl.predicate.Exists; @@ -248,8 +248,8 @@ private ViewElementDefinition createElementDefFromPredicates(final String filter hasStepFilterStage = HasStepFilterStage.valueOf(filterStage); } catch (final IllegalArgumentException e) { LOGGER.warn("Unknown hasStepFilterStage: {}. Defaulting to {}", - filterStage, GafferPopGraph.DEFAULT_HAS_STEP_FILTER_STAGE); - hasStepFilterStage = GafferPopGraph.DEFAULT_HAS_STEP_FILTER_STAGE; + filterStage, GafferPopGraphVariables.DEFAULT_HAS_STEP_FILTER_STAGE); + hasStepFilterStage = GafferPopGraphVariables.DEFAULT_HAS_STEP_FILTER_STAGE; } switch (hasStepFilterStage) { diff --git a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopFederationTests.java b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopFederationTests.java index 8baed6e705e..24f83e74bd2 100644 --- a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopFederationTests.java +++ b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopFederationTests.java @@ -292,7 +292,8 @@ void shouldSeedWithVertexOnlyEdge() throws OperationException { // Edge has a vertex but not an entity in the graph - Gaffer only feature getGraph().addEdge(new GafferPopEdge("knows", GafferPopModernTestUtils.MARKO.getId(), "7", getGraph())); - List result = g.V("7").toList(); + // Need to enable orphaned vertices on the query + List result = g.with(GafferPopGraphVariables.INCLUDE_ORPHANED_VERTICES, "true").V("7").toList(); assertThat(result) .extracting(r -> r.id()) .contains("7"); @@ -305,9 +306,12 @@ void shouldTraverseEdgeWithVertexOnlySeed() throws OperationException { // Edge has a vertex but not an entity in the graph - Gaffer only feature getGraph().addEdge(new GafferPopEdge("knows", GafferPopModernTestUtils.MARKO.getId(), "7", getGraph())); - List> result = g.V("7").inE().outV().elementMap().toList(); - assertThat(result) - .containsExactly(MARKO.getPropertyMap()); + // Need to enable orphaned vertices on the query + List> result = g.with(GafferPopGraphVariables.INCLUDE_ORPHANED_VERTICES, "true") + .V("7").inE().outV().elementMap().toList(); + + assertThat(result).containsExactly(MARKO.getPropertyMap()); + reset(); } diff --git a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java index 07701d92a76..83dcdcb2841 100644 --- a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java +++ b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java @@ -308,7 +308,8 @@ void shouldSeedWithVertexOnlyEdge(String graph, GraphTraversalSource g) { mapStore.addEdge(new GafferPopEdge("knows", GafferPopModernTestUtils.MARKO.getId(), VERTEX_ONLY_ID_STRING, mapStore)); accumuloStore.addEdge(new GafferPopEdge("knows", GafferPopModernTestUtils.MARKO.getId(), VERTEX_ONLY_ID_STRING, accumuloStore)); - List result = g.V(VERTEX_ONLY_ID_STRING).toList(); + // Must enable orphaned vertices for this traversal + List result = g.with(GafferPopGraphVariables.INCLUDE_ORPHANED_VERTICES, "true").V(VERTEX_ONLY_ID_STRING).toList(); assertThat(result) .extracting(r -> r.id()) .contains(VERTEX_ONLY_ID_STRING); @@ -318,28 +319,31 @@ void shouldSeedWithVertexOnlyEdge(String graph, GraphTraversalSource g) { @ParameterizedTest(name = TEST_NAME_FORMAT) @MethodSource("provideTraversals") void shouldTraverseEdgeWithVertexOnlyEdge(String graph, GraphTraversalSource g) { + // Must enable orphaned vertices for this traversal + GraphTraversalSource gWithOption = g.with(GafferPopGraphVariables.INCLUDE_ORPHANED_VERTICES, "true"); // Edge has a two vertices with no entities in the graph - Gaffer only feature // [8 - knows -> 7] mapStore.addEdge(new GafferPopEdge("knows", "8", VERTEX_ONLY_ID_STRING, mapStore)); accumuloStore.addEdge(new GafferPopEdge("knows", "8", VERTEX_ONLY_ID_STRING, accumuloStore)); - List result = g.V(VERTEX_ONLY_ID_STRING).inE().inV().toList(); + + List result = gWithOption.V(VERTEX_ONLY_ID_STRING).inE().inV().toList(); assertThat(result) .extracting(r -> r.id()) .contains(VERTEX_ONLY_ID_STRING); - List result2 = g.V(VERTEX_ONLY_ID_STRING).inE().outV().toList(); + List result2 = gWithOption.V(VERTEX_ONLY_ID_STRING).inE().outV().toList(); assertThat(result2) .extracting(r -> r.id()) .contains("8"); - List result3 = g.V("8").outE().inV().toList(); + List result3 = gWithOption.V("8").outE().inV().toList(); assertThat(result3) .extracting(r -> r.id()) .contains(VERTEX_ONLY_ID_STRING); - List result4 = g.V("8").outE().outV().toList(); + List result4 = gWithOption.V("8").outE().outV().toList(); assertThat(result4) .extracting(r -> r.id()) .contains("8"); - List resultLabel = g.V("8").out("knows").toList(); + List resultLabel = gWithOption.V("8").out("knows").toList(); assertThat(resultLabel) .extracting(r -> r.id()) .containsOnly(VERTEX_ONLY_ID_STRING); diff --git a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphTest.java b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphTest.java index 503fbf82e99..d08089366ee 100644 --- a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphTest.java +++ b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphTest.java @@ -30,7 +30,7 @@ import uk.gov.gchq.gaffer.operation.OperationChain; import uk.gov.gchq.gaffer.operation.impl.add.AddElementsFromSocket; import uk.gov.gchq.gaffer.operation.impl.get.GetElements; -import uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph.HasStepFilterStage; +import uk.gov.gchq.gaffer.tinkerpop.GafferPopGraphVariables.HasStepFilterStage; import uk.gov.gchq.gaffer.tinkerpop.util.GafferPopTestUtil; import uk.gov.gchq.gaffer.tinkerpop.util.GafferPopTstvTestUtils; import uk.gov.gchq.gaffer.user.User; @@ -77,10 +77,11 @@ void shouldConstructGafferPopGraphWithOnlyConfig() { // Then final Map variables = graph.variables().asMap(); assertThat(variables) - .hasSize(5) + .hasSize(6) .containsEntry(GafferPopGraphVariables.USER, expectedUser) .containsEntry(GafferPopGraphVariables.GET_ELEMENTS_LIMIT, 1) .containsEntry(GafferPopGraphVariables.HAS_STEP_FILTER_STAGE, HasStepFilterStage.POST_TRANSFORM.toString()) + .containsEntry(GafferPopGraphVariables.INCLUDE_ORPHANED_VERTICES, false) .containsKey(GafferPopGraphVariables.OP_OPTIONS); final Map opOptions = (Map) variables.get(GafferPopGraphVariables.OP_OPTIONS); @@ -102,10 +103,11 @@ void shouldConstructGafferPopGraphWithConfigFile() { // Then final Map variables = graph.variables().asMap(); assertThat(variables) - .hasSize(5) + .hasSize(6) .containsEntry(GafferPopGraphVariables.USER, expectedUser) .containsEntry(GafferPopGraphVariables.GET_ELEMENTS_LIMIT, 2) .containsEntry(GafferPopGraphVariables.HAS_STEP_FILTER_STAGE, HasStepFilterStage.POST_AGGREGATION.toString()) + .containsEntry(GafferPopGraphVariables.INCLUDE_ORPHANED_VERTICES, true) .containsKey(GafferPopGraphVariables.OP_OPTIONS); final Map opOptions = (Map) variables.get(GafferPopGraphVariables.OP_OPTIONS); @@ -126,12 +128,13 @@ void shouldConstructGafferPopGraph() { // Then final Map variables = graph.variables().asMap(); assertThat(variables) - .hasSize(5) + .hasSize(6) .containsEntry(GafferPopGraphVariables.USER, expectedUser) .containsEntry(GafferPopGraphVariables.GET_ELEMENTS_LIMIT, - GafferPopGraph.DEFAULT_GET_ELEMENTS_LIMIT) + GafferPopGraphVariables.DEFAULT_GET_ELEMENTS_LIMIT) + .containsEntry(GafferPopGraphVariables.INCLUDE_ORPHANED_VERTICES, false) .containsEntry(GafferPopGraphVariables.HAS_STEP_FILTER_STAGE, - GafferPopGraph.DEFAULT_HAS_STEP_FILTER_STAGE.toString()) + GafferPopGraphVariables.DEFAULT_HAS_STEP_FILTER_STAGE.toString()) .containsKey(GafferPopGraphVariables.OP_OPTIONS); diff --git a/library/tinkerpop/src/test/resources/gafferpop-test.properties b/library/tinkerpop/src/test/resources/gafferpop-test.properties index b2a7d349ef0..7647fb2ab5e 100644 --- a/library/tinkerpop/src/test/resources/gafferpop-test.properties +++ b/library/tinkerpop/src/test/resources/gafferpop-test.properties @@ -1,5 +1,5 @@ # -# Copyright 2016-2023 Crown Copyright +# Copyright 2016-2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,4 +21,5 @@ gaffer.userId=user01 gaffer.operation.options=key1:value1 gaffer.elements.getlimit=2 gaffer.elements.hasstepfilterstage=POST_AGGREGATION +gaffer.includeOrphanedVertices=true From 1a91fcf8f6f10b1d181750d69c1daedc9c8e30c9 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:37:19 +0000 Subject: [PATCH 04/29] Gh-3342: Handle different groups in View federated poc (#3343) * New utils class * integrate checks for views and schema compatibility * update unit testing for new class * tidy * address comments * typo --- .../gchq/gaffer/tinkerpop/GafferPopEdge.java | 5 +- .../federated/simple/FederatedUtils.java | 167 ++++++++++++++++++ .../handler/FederatedOperationHandler.java | 19 +- .../handler/FederatedOutputHandler.java | 49 ++++- .../federated/simple/FederatedUtilsTest.java | 156 ++++++++++++++++ 5 files changed, 378 insertions(+), 18 deletions(-) create mode 100644 store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedUtils.java create mode 100644 store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/FederatedUtilsTest.java diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopEdge.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopEdge.java index be0a7fdf10c..a70792f6062 100644 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopEdge.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopEdge.java @@ -16,6 +16,7 @@ package uk.gov.gchq.gaffer.tinkerpop; +import org.apache.commons.collections4.IterableUtils; import org.apache.tinkerpop.gremlin.structure.Direction; import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.Property; @@ -36,6 +37,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.Optional; @@ -215,8 +217,7 @@ private Vertex getVertex(final GafferPopVertex vertex) { .build()) .build(); - Iterable result = graph().execute(findBasedOnID); - + final Set result = new HashSet<>(IterableUtils.toList(graph().execute(findBasedOnID))); final GafferPopElementGenerator generator = new GafferPopElementGenerator(graph()); Optional foundEntity = StreamSupport.stream(result.spliterator(), false) diff --git a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedUtils.java b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedUtils.java new file mode 100644 index 00000000000..e2ac2e20efb --- /dev/null +++ b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedUtils.java @@ -0,0 +1,167 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.gchq.gaffer.federated.simple; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import uk.gov.gchq.gaffer.data.elementdefinition.view.View; +import uk.gov.gchq.gaffer.federated.simple.operation.handler.FederatedOperationHandler; +import uk.gov.gchq.gaffer.graph.GraphSerialisable; +import uk.gov.gchq.gaffer.operation.Operation; +import uk.gov.gchq.gaffer.operation.OperationChain; +import uk.gov.gchq.gaffer.operation.graph.OperationView; +import uk.gov.gchq.gaffer.store.schema.Schema; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Utility class with static methods to help support the federated store. + */ +public final class FederatedUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(FederatedUtils.class); + + private FederatedUtils() { + // utility class + } + + /** + * Checks if graphs share any groups between their schemas. + * + * @param graphs Graphs to check. + * @return Do they share groups. + */ + public static boolean doGraphsShareGroups(final List graphs) { + // Check if any of the graphs have common groups in their schemas + List schemas = graphs.stream() + .map(GraphSerialisable::getSchema) + .collect(Collectors.toList()); + + // Compare all schemas against each other + for (int i = 0; i < schemas.size() - 1; i++) { + for (int j = i + 1; j < schemas.size(); j++) { + // Compare each schema against the others to see if common groups + if (!Collections.disjoint(schemas.get(i).getGroups(), schemas.get(j).getGroups())) { + LOGGER.debug("Found common schema groups between requested graphs"); + return true; + } + } + } + return false; + } + + /** + * Get a version of the operation chain that satisfies the schema for the + * requested graph. This will ensure the operation will pass validation on + * the sub graph end as there is no universal way to skip validation. + * + * @param operation The operation. + * @param graphSerialisable The graph. + * @param depth Current recursion depth of this method. + * @param depthLimit Limit to the recursion depth. + * @return A valid version of the operation chain. + */ + public static OperationChain getValidOperationForGraph(final Operation operation, final GraphSerialisable graphSerialisable, final int depth, final int depthLimit) { + LOGGER.debug("Creating valid operation for graph, depth is: {}", depth); + final Collection updatedOperations = new ArrayList<>(); + + // Fix the view so it will pass schema validation + if (operation instanceof OperationView + && ((OperationView) operation).getView() != null + && ((OperationView) operation).getView().hasGroups()) { + + // Update the view for the graph + ((OperationView) operation).setView( + getValidViewForGraph(((OperationView) operation).getView(), graphSerialisable)); + + updatedOperations.add(operation); + + // Recursively go into operation chains to make sure everything is fixed + } else if (operation instanceof OperationChain) { + for (final Operation op : ((OperationChain) operation).getOperations()) { + // Resolve if haven't hit the depth limit for validation + if (depth < depthLimit) { + updatedOperations.addAll(getValidOperationForGraph(op, graphSerialisable, depth + 1, depthLimit).getOperations()); + } else { + LOGGER.warn( + "Hit depth limit of {} making the operation valid for graph. The View may be invalid for Graph: {}", + depthLimit, + graphSerialisable.getGraphId()); + updatedOperations.add(op); + } + } + } else { + updatedOperations.add(operation); + } + + // Create and return the fixed chain for the graph + OperationChain newChain = new OperationChain<>(); + newChain.setOptions(operation.getOptions()); + newChain.updateOperations(updatedOperations); + + return newChain; + } + + /** + * Returns a {@link View} that contains groups only relevant to the graph. If + * the supplied view does not require modification it will just be returned. + * + * @param view The view to make valid. + * @param graphSerialisable The relevant graph. + * @return A version of the view valid for the graph. + */ + public static View getValidViewForGraph(final View view, final GraphSerialisable graphSerialisable) { + final Schema schema = graphSerialisable.getSchema(); + + // Figure out all the groups relevant to the graph + final Set validEntities = new HashSet<>(view.getEntityGroups()); + final Set validEdges = new HashSet<>(view.getEdgeGroups()); + validEntities.retainAll(schema.getEntityGroups()); + validEdges.retainAll(schema.getEdgeGroups()); + + if (!validEntities.equals(view.getEntityGroups()) || !validEdges.equals(view.getEdgeGroups())) { + // Need to make changes to the view so start by cloning the view + // and clearing all the edges and entities + final View.Builder builder = new View.Builder() + .merge(view) + .entities(Collections.emptyMap()) + .edges(Collections.emptyMap()); + validEntities.forEach(e -> builder.entity(e, view.getEntity(e))); + validEdges.forEach(e -> builder.edge(e, view.getEdge(e))); + final View newView = builder.build(); + // If the View has no groups left after fixing then this is likely an issue so throw + if (!newView.hasEntities() && !newView.hasEdges()) { + throw new IllegalArgumentException(String.format( + "No groups specified in View are relevant to Graph: '%1$s'. " + + "Please refine your Graphs/View or specify following option to skip execution on offending Graph: '%2$s' ", + graphSerialisable.getGraphId(), + FederatedOperationHandler.OPT_SKIP_FAILED_EXECUTE)); + } + return newView; + } + + // Nothing to do return unmodified view + return view; + } + +} diff --git a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOperationHandler.java b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOperationHandler.java index 734fb3f7ee0..2f4e06932b8 100644 --- a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOperationHandler.java +++ b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOperationHandler.java @@ -22,6 +22,7 @@ import uk.gov.gchq.gaffer.cache.exception.CacheOperationException; import uk.gov.gchq.gaffer.federated.simple.FederatedStore; +import uk.gov.gchq.gaffer.federated.simple.FederatedUtils; import uk.gov.gchq.gaffer.federated.simple.access.GraphAccess; import uk.gov.gchq.gaffer.graph.GraphSerialisable; import uk.gov.gchq.gaffer.operation.Operation; @@ -90,9 +91,16 @@ public class FederatedOperationHandler

implements Operation */ public static final String OPT_SEPARATE_RESULTS = "federated.separateResults"; + /** + * Depth should go to when making an operation chain relevant to specified + * graphs e.g. fix the View. + */ + public static final String OPT_FIX_OP_LIMIT = "federated.fixOperationLimit"; + @Override public Object doOperation(final P operation, final Context context, final Store store) throws OperationException { LOGGER.debug("Running operation: {}", operation); + final int fixLimit = Integer.parseInt(operation.getOption(OPT_FIX_OP_LIMIT, "5")); // If the operation has output wrap and return using sub class handler if (operation instanceof Output) { @@ -112,20 +120,17 @@ public Object doOperation(final P operation, final Context context, final Store // Execute the operation chain on each graph for (final GraphSerialisable gs : graphsToExecute) { try { - gs.getGraph().execute(operation, context.getUser()); - } catch (final OperationException | UnsupportedOperationException e) { + gs.getGraph().execute( + FederatedUtils.getValidOperationForGraph(operation, gs, 0, fixLimit), + context.getUser()); + } catch (final OperationException | UnsupportedOperationException | IllegalArgumentException e) { // Optionally skip this error if user has specified to do so LOGGER.error("Operation failed on graph: {}", gs.getGraphId()); if (!Boolean.parseBoolean(operation.getOption(OPT_SKIP_FAILED_EXECUTE, "false"))) { throw e; } LOGGER.info("Continuing operation execution on sub graphs"); - } catch (final IllegalArgumentException e) { - // An operation may fail validation for a sub graph this is not really an error. - // We can just continue to execute on the rest of the graphs - LOGGER.warn("Operation contained invalid arguments for a sub graph, skipped execution on graph: {}", gs.getGraphId()); } - } // Assume no output, we've already checked above diff --git a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOutputHandler.java b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOutputHandler.java index 457377e7e0a..99adb3c5e81 100644 --- a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOutputHandler.java +++ b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOutputHandler.java @@ -20,9 +20,11 @@ import org.slf4j.LoggerFactory; import uk.gov.gchq.gaffer.federated.simple.FederatedStore; +import uk.gov.gchq.gaffer.federated.simple.FederatedUtils; import uk.gov.gchq.gaffer.federated.simple.merge.DefaultResultAccumulator; import uk.gov.gchq.gaffer.federated.simple.merge.FederatedResultAccumulator; import uk.gov.gchq.gaffer.graph.GraphSerialisable; +import uk.gov.gchq.gaffer.operation.OperationChain; import uk.gov.gchq.gaffer.operation.OperationException; import uk.gov.gchq.gaffer.operation.io.Output; import uk.gov.gchq.gaffer.store.Context; @@ -44,8 +46,10 @@ public class FederatedOutputHandler

, O> @Override public O doOperation(final P operation, final Context context, final Store store) throws OperationException { + final int fixLimit = Integer.parseInt(operation.getOption(OPT_FIX_OP_LIMIT, "5")); List graphsToExecute = this.getGraphsToExecuteOn(operation, context, (FederatedStore) store); + // No-op if (graphsToExecute.isEmpty()) { return null; } @@ -54,18 +58,15 @@ public O doOperation(final P operation, final Context context, final Store store List graphResults = new ArrayList<>(); for (final GraphSerialisable gs : graphsToExecute) { try { - graphResults.add(gs.getGraph().execute(operation, context.getUser())); - } catch (final OperationException | UnsupportedOperationException e) { + OperationChain fixedChain = FederatedUtils.getValidOperationForGraph(operation, gs, 0, fixLimit); + graphResults.add(gs.getGraph().execute(fixedChain, context.getUser())); + } catch (final OperationException | UnsupportedOperationException | IllegalArgumentException e) { // Optionally skip this error if user has specified to do so LOGGER.error("Operation failed on graph: {}", gs.getGraphId()); if (!Boolean.parseBoolean(operation.getOption(OPT_SKIP_FAILED_EXECUTE, "false"))) { throw e; } LOGGER.info("Continuing operation execution on sub graphs"); - } catch (final IllegalArgumentException e) { - // An operation may fail validation for a sub graph this is not really an error. - // We can just continue to execute on the rest of the graphs - LOGGER.warn("Operation contained invalid arguments for a sub graph, skipped execution on graph: {}", gs.getGraphId()); } } @@ -80,15 +81,45 @@ public O doOperation(final P operation, final Context context, final Store store combinedProps.putAll(operation.getOptions()); } + // Set up the result accumulator + FederatedResultAccumulator resultAccumulator = getResultAccumulator((FederatedStore) store, operation, graphsToExecute); + + // Should now have a list of objects so need to reduce to just one + return graphResults.stream().reduce(resultAccumulator::apply).orElse(graphResults.get(0)); + } + + + /** + * Sets up a {@link FederatedResultAccumulator} for the specified operation + * and graphs. + * + * @param store The federated store. + * @param operation The original operation. + * @param graphsToExecute The graphs executed on. + * @return A set up accumulator. + */ + protected FederatedResultAccumulator getResultAccumulator(final FederatedStore store, final P operation, final List graphsToExecute) { + // Merge the store props with the operation options for setting up the + // accumulator + Properties combinedProps = store.getProperties().getProperties(); + if (operation.getOptions() != null) { + combinedProps.putAll(operation.getOptions()); + } + // Set up the result accumulator FederatedResultAccumulator resultAccumulator = new DefaultResultAccumulator<>(combinedProps); - resultAccumulator.setSchema(((FederatedStore) store).getSchema(graphsToExecute)); + resultAccumulator.setSchema(store.getSchema(graphsToExecute)); + // Check if user has specified to aggregate if (operation.containsOption(OPT_AGGREGATE_ELEMENTS)) { resultAccumulator.setAggregateElements(Boolean.parseBoolean(operation.getOption(OPT_AGGREGATE_ELEMENTS))); } - // Should now have a list of objects so need to reduce to just one - return graphResults.stream().reduce(resultAccumulator::apply).orElse(graphResults.get(0)); + // Turn aggregation off if there are no shared groups + if (!FederatedUtils.doGraphsShareGroups(graphsToExecute)) { + resultAccumulator.setAggregateElements(false); + } + + return resultAccumulator; } } diff --git a/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/FederatedUtilsTest.java b/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/FederatedUtilsTest.java new file mode 100644 index 00000000000..b7197b4df25 --- /dev/null +++ b/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/FederatedUtilsTest.java @@ -0,0 +1,156 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.gchq.gaffer.federated.simple; + +import org.junit.jupiter.api.Test; + +import uk.gov.gchq.gaffer.data.elementdefinition.view.View; +import uk.gov.gchq.gaffer.graph.GraphConfig; +import uk.gov.gchq.gaffer.graph.GraphSerialisable; +import uk.gov.gchq.gaffer.operation.Operation; +import uk.gov.gchq.gaffer.operation.OperationChain; +import uk.gov.gchq.gaffer.operation.graph.OperationView; +import uk.gov.gchq.gaffer.operation.impl.get.GetAllElements; +import uk.gov.gchq.gaffer.store.StoreProperties; +import uk.gov.gchq.gaffer.store.schema.Schema; +import uk.gov.gchq.gaffer.store.schema.SchemaEdgeDefinition; +import uk.gov.gchq.gaffer.store.schema.SchemaEntityDefinition; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +class FederatedUtilsTest { + + @Test + void shouldRemoveGroupFromViewIfNotInSchema() { + // Given + String entityInSchema = "entityInSchema"; + String edgeInSchema = "edgeInSchema"; + View testView = new View.Builder() + .entity(entityInSchema) + .entity("entityNotInSchema") + .edge(edgeInSchema) + .edge("edgeNotInSchema") + .build(); + GraphSerialisable graph = new GraphSerialisable( + new GraphConfig("test"), + new Schema.Builder() + .entity(entityInSchema, new SchemaEntityDefinition()) + .edge(edgeInSchema, new SchemaEdgeDefinition()).build(), + new StoreProperties()); + + // When + View fixedView = FederatedUtils.getValidViewForGraph(testView, graph); + + // Then + assertThat(fixedView.getEntityGroups()).containsOnly(entityInSchema); + assertThat(fixedView.getEdgeGroups()).containsOnly(edgeInSchema); + } + + @Test + void shouldPreventExecutionIfNoGroupsInViewAreRelevant() { + // Given + View testView = new View.Builder() + .entity("entityNotInSchema") + .edge("edgeNotInSchema") + .build(); + GraphSerialisable graph = new GraphSerialisable( + new GraphConfig("test"), + new Schema.Builder() + .entity("entityInSchema", new SchemaEntityDefinition()) + .edge("edgeInSchema", new SchemaEdgeDefinition()).build(), + new StoreProperties()); + + // When + assertThatExceptionOfType(IllegalArgumentException.class) + .isThrownBy(() -> FederatedUtils.getValidViewForGraph(testView, graph)); + } + + @Test + void shouldChangeViewsOfNestedOperations() { + String entityInSchema = "entityInSchema"; + String edgeInSchema = "edgeInSchema"; + GraphSerialisable graph = new GraphSerialisable( + new GraphConfig("test"), + new Schema.Builder() + .entity(entityInSchema, new SchemaEntityDefinition()) + .edge(edgeInSchema, new SchemaEdgeDefinition()).build(), + new StoreProperties()); + + // View with some mixed groups in + View testView = new View.Builder() + .entity(entityInSchema) + .entity("entityNotInSchema") + .edge(edgeInSchema) + .edge("edgeNotInSchema") + .build(); + + // Build nested operation chain with multiple views + OperationChain nestedViewChain = new OperationChain.Builder() + .first(new GetAllElements.Builder().view(testView).build()) + .then(new OperationChain.Builder() + .first(new GetAllElements.Builder().view(testView).build()) + .then(new OperationChain.Builder() + .first(new GetAllElements.Builder().view(testView).build()) + .build()) + .build()) + .build(); + + // Get a fixed operation chain + List newChain = FederatedUtils.getValidOperationForGraph(nestedViewChain, graph, 0, 5).flatten(); + List fixedOperations = newChain.stream() + .filter(op -> op instanceof OperationView) + .map(op -> (OperationView) op) + .collect(Collectors.toList()); + + // Check all the views only contain relevant groups + fixedOperations.stream() + .map(OperationView::getView) + .forEach(v -> { + assertThat(v.getEntityGroups()).containsOnly(entityInSchema); + assertThat(v.getEdgeGroups()).containsOnly(edgeInSchema); + }); + } + + @Test + void shouldDetectIfGraphsShareGroups() { + // Given + String sharedGroup = "sharedGroup"; + + GraphSerialisable graph1 = new GraphSerialisable( + new GraphConfig("graph1"), + new Schema.Builder().entity(sharedGroup, new SchemaEntityDefinition()).build(), + new StoreProperties()); + GraphSerialisable graph2 = new GraphSerialisable( + new GraphConfig("graph2"), + new Schema.Builder().entity(sharedGroup, new SchemaEntityDefinition()).build(), + new StoreProperties()); + GraphSerialisable graph3 = new GraphSerialisable( + new GraphConfig("graph3"), + new Schema.Builder().entity("notShared", new SchemaEntityDefinition()).build(), + new StoreProperties()); + + // When/Then + assertThat(FederatedUtils.doGraphsShareGroups(Arrays.asList(graph1, graph2))).isTrue(); + assertThat(FederatedUtils.doGraphsShareGroups(Arrays.asList(graph1, graph3))).isFalse(); + } + +} From a9009166c950862a318f1821eceef160185bb287 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:32:38 +0000 Subject: [PATCH 05/29] Gh-3344: Change graph permissions operation in federated POC (#3345) * new change graph access operation * unit tests --- .../federated/simple/FederatedStore.java | 21 ++- .../simple/operation/ChangeGraphAccess.java | 173 ++++++++++++++++++ .../misc/ChangeGraphAccessHandler.java | 56 ++++++ .../operation/ChangeGraphAccessTest.java | 116 ++++++++++++ 4 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/ChangeGraphAccess.java create mode 100644 store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/misc/ChangeGraphAccessHandler.java create mode 100644 store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/operation/ChangeGraphAccessTest.java diff --git a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedStore.java b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedStore.java index 4ef634fb56a..9b29dabd6b2 100644 --- a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedStore.java +++ b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedStore.java @@ -32,6 +32,7 @@ import uk.gov.gchq.gaffer.data.element.id.EntityId; import uk.gov.gchq.gaffer.federated.simple.access.GraphAccess; import uk.gov.gchq.gaffer.federated.simple.operation.AddGraph; +import uk.gov.gchq.gaffer.federated.simple.operation.ChangeGraphAccess; import uk.gov.gchq.gaffer.federated.simple.operation.ChangeGraphId; import uk.gov.gchq.gaffer.federated.simple.operation.FederatedOperationChainValidator; import uk.gov.gchq.gaffer.federated.simple.operation.GetAllGraphIds; @@ -44,6 +45,7 @@ import uk.gov.gchq.gaffer.federated.simple.operation.handler.get.GetAllGraphIdsHandler; import uk.gov.gchq.gaffer.federated.simple.operation.handler.get.GetAllGraphInfoHandler; import uk.gov.gchq.gaffer.federated.simple.operation.handler.get.GetSchemaHandler; +import uk.gov.gchq.gaffer.federated.simple.operation.handler.misc.ChangeGraphAccessHandler; import uk.gov.gchq.gaffer.federated.simple.operation.handler.misc.ChangeGraphIdHandler; import uk.gov.gchq.gaffer.federated.simple.operation.handler.misc.RemoveGraphHandler; import uk.gov.gchq.gaffer.graph.GraphSerialisable; @@ -132,7 +134,8 @@ public class FederatedStore extends Store { new SimpleEntry<>(GetSchema.class, new GetSchemaHandler()), new SimpleEntry<>(ChangeGraphId.class, new ChangeGraphIdHandler()), new SimpleEntry<>(GetAllGraphInfo.class, new GetAllGraphInfoHandler()), - new SimpleEntry<>(RemoveGraph.class, new RemoveGraphHandler())) + new SimpleEntry<>(RemoveGraph.class, new RemoveGraphHandler()), + new SimpleEntry<>(ChangeGraphAccess.class, new ChangeGraphAccessHandler())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); /** @@ -289,6 +292,22 @@ public void changeGraphId(final String graphToUpdateId, final String newGraphId) } } + /** + * Updates a graph access by overwriting the access for that graph + * stored in the cache. + * + * @param graphId The graph ID to update. + * @param newAccess The new graph access. + * @throws CacheOperationException If issue updating the cache. + */ + public void changeGraphAccess(final String graphId, final GraphAccess newAccess) throws CacheOperationException { + final GraphSerialisable graph = getGraph(graphId); + // Create the new pair + final Pair graphAndAccessPair = new ImmutablePair<>(graph, newAccess); + // Add to the cache this will overwrite any existing value + graphCache.getCache().put(graphId, graphAndAccessPair); + } + /** * Gets a merged schema based on the graphs specified. * diff --git a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/ChangeGraphAccess.java b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/ChangeGraphAccess.java new file mode 100644 index 00000000000..8bad1209fcf --- /dev/null +++ b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/ChangeGraphAccess.java @@ -0,0 +1,173 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.gchq.gaffer.federated.simple.operation; + +import org.apache.commons.lang3.exception.CloneFailedException; + +import uk.gov.gchq.gaffer.access.predicate.AccessPredicate; +import uk.gov.gchq.gaffer.operation.Operation; +import uk.gov.gchq.koryphe.Since; +import uk.gov.gchq.koryphe.Summary; + +import java.util.Map; + +@Since("2.4.0") +@Summary("Changes the access controls on a Graph") +public class ChangeGraphAccess implements Operation { + + private String graphId; + private String owner; + private Boolean isPublic; + private AccessPredicate readPredicate; + private AccessPredicate writePredicate; + private Map options; + + // Getters + /** + * Get the graph ID of the graph that will be changed. + * + * @return the graph ID + */ + public String getGraphId() { + return graphId; + } + + public String getOwner() { + return owner; + } + + public Boolean isPublic() { + return isPublic; + } + + public AccessPredicate getReadPredicate() { + return readPredicate; + } + + public AccessPredicate getWritePredicate() { + return writePredicate; + } + + // Setters + /** + * Set the graph ID of the current graph. + * + * @param graphId the graph ID + */ + public void setGraphId(final String graphId) { + this.graphId = graphId; + } + + public void setOwner(final String owner) { + this.owner = owner; + } + + public void setIsPublic(final Boolean isPublic) { + this.isPublic = isPublic; + } + + public void setReadPredicate(final AccessPredicate readPredicate) { + this.readPredicate = readPredicate; + } + + public void setWritePredicate(final AccessPredicate writePredicate) { + this.writePredicate = writePredicate; + } + + @Override + public Map getOptions() { + return options; + } + + @Override + public void setOptions(final Map options) { + this.options = options; + } + + @Override + public Operation shallowClone() throws CloneFailedException { + return new ChangeGraphAccess.Builder() + .graphId(graphId) + .owner(owner) + .isPublic(isPublic) + .readPredicate(readPredicate) + .writePredicate(writePredicate) + .options(options) + .build(); + } + + public static class Builder extends Operation.BaseBuilder { + public Builder() { + super(new ChangeGraphAccess()); + } + + /** + * Set the current graph ID + * + * @param graphId the graph ID of the graph to alter + * @return The builder + */ + public Builder graphId(final String graphId) { + _getOp().setGraphId(graphId); + return _self(); + } + + /** + * Set the new owner of the Graph. + * + * @param owner The owner. + * @return The builder + */ + public Builder owner(final String owner) { + _getOp().setOwner(owner); + return _self(); + } + + /** + * Set if graph is public. + * + * @param isPublic Is the graph public. + * @return The builder + */ + public Builder isPublic(final Boolean isPublic) { + _getOp().setIsPublic(isPublic); + return _self(); + } + + /** + * Set the read predicate for the graph. + * + * @param readPredicate The read predicate. + * @return The builder. + */ + public Builder readPredicate(final AccessPredicate readPredicate) { + _getOp().setReadPredicate(readPredicate); + return _self(); + } + + /** + * Set the write predicate. + * + * @param writePredicate The write predicate. + * @return The builder. + */ + public Builder writePredicate(final AccessPredicate writePredicate) { + _getOp().setWritePredicate(writePredicate); + return _self(); + } + } +} diff --git a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/misc/ChangeGraphAccessHandler.java b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/misc/ChangeGraphAccessHandler.java new file mode 100644 index 00000000000..a6831949def --- /dev/null +++ b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/misc/ChangeGraphAccessHandler.java @@ -0,0 +1,56 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.gchq.gaffer.federated.simple.operation.handler.misc; + +import uk.gov.gchq.gaffer.cache.exception.CacheOperationException; +import uk.gov.gchq.gaffer.federated.simple.FederatedStore; +import uk.gov.gchq.gaffer.federated.simple.access.GraphAccess; +import uk.gov.gchq.gaffer.federated.simple.operation.ChangeGraphAccess; +import uk.gov.gchq.gaffer.operation.OperationException; +import uk.gov.gchq.gaffer.store.Context; +import uk.gov.gchq.gaffer.store.Store; +import uk.gov.gchq.gaffer.store.operation.handler.OperationHandler; + +public class ChangeGraphAccessHandler implements OperationHandler { + + @Override + public Object doOperation(final ChangeGraphAccess operation, final Context context, final Store store) throws OperationException { + try { + // Check user for write access as we're modifying the graph + GraphAccess existingAccess = ((FederatedStore) store).getGraphAccess(operation.getGraphId()); + if (!existingAccess.hasWriteAccess(context.getUser(), store.getProperties().getAdminAuth())) { + throw new OperationException( + "User: '" + context.getUser().getUserId() + "' does not have write permissions for Graph: " + operation.getGraphId()); + } + + // Create the new access object based on what was specified + GraphAccess.Builder accessBuilder = new GraphAccess.Builder() + .owner(operation.getOwner() != null ? operation.getOwner() : existingAccess.getOwner()) + .isPublic(operation.isPublic() != null ? operation.isPublic() : existingAccess.isPublic()) + .readAccessPredicate(operation.getReadPredicate() != null ? operation.getReadPredicate() : existingAccess.getReadAccessPredicate()) + .writeAccessPredicate(operation.getWritePredicate() != null ? operation.getWritePredicate() : existingAccess.getWriteAccessPredicate()); + + // Update the access + ((FederatedStore) store).changeGraphAccess(operation.getGraphId(), accessBuilder.build()); + } catch (final CacheOperationException e) { + throw new OperationException("Error changing the Graph Access for: " + operation.getGraphId(), e); + } + + return null; + } + +} diff --git a/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/operation/ChangeGraphAccessTest.java b/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/operation/ChangeGraphAccessTest.java new file mode 100644 index 00000000000..2da8069c9d4 --- /dev/null +++ b/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/operation/ChangeGraphAccessTest.java @@ -0,0 +1,116 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.gchq.gaffer.federated.simple.operation; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import uk.gov.gchq.gaffer.access.predicate.NoAccessPredicate; +import uk.gov.gchq.gaffer.access.predicate.UnrestrictedAccessPredicate; +import uk.gov.gchq.gaffer.cache.CacheServiceLoader; +import uk.gov.gchq.gaffer.cache.exception.CacheOperationException; +import uk.gov.gchq.gaffer.federated.simple.FederatedStore; +import uk.gov.gchq.gaffer.federated.simple.access.GraphAccess; +import uk.gov.gchq.gaffer.graph.GraphConfig; +import uk.gov.gchq.gaffer.operation.OperationException; +import uk.gov.gchq.gaffer.store.Context; +import uk.gov.gchq.gaffer.store.StoreException; +import uk.gov.gchq.gaffer.store.StoreProperties; +import uk.gov.gchq.gaffer.store.schema.Schema; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; + +class ChangeGraphAccessTest { + + @AfterEach + void reset() { + CacheServiceLoader.shutdown(); + } + + @Test + void shouldChangeAccessOfGraph() throws StoreException, OperationException, CacheOperationException { + final String federatedGraphId = "federated"; + final String graphId = "shouldChangeAccessOfGraph"; + + final FederatedStore federatedStore = new FederatedStore(); + federatedStore.initialise(federatedGraphId, null, new StoreProperties()); + + // Add a graph with no restrictions + final AddGraph addGraph = new AddGraph.Builder() + .graphConfig(new GraphConfig(graphId)) + .schema(new Schema()) + .properties(new StoreProperties().getProperties()) + .owner("oldOwner") + .isPublic(true) + .readPredicate(new UnrestrictedAccessPredicate()) + .writePredicate(new UnrestrictedAccessPredicate()) + .build(); + + // Change the graph access + final ChangeGraphAccess changeGraphAccess = new ChangeGraphAccess.Builder() + .graphId(graphId) + .owner("newOwner") + .isPublic(false) + .readPredicate(new NoAccessPredicate()) + .writePredicate(new NoAccessPredicate()) + .build(); + + // When + federatedStore.execute(addGraph, new Context()); + federatedStore.execute(changeGraphAccess, new Context()); + + GraphAccess updatedAccess = federatedStore.getGraphAccess(graphId); + + // Then + assertThat(updatedAccess.getOwner()).isEqualTo("newOwner"); + assertThat(updatedAccess.isPublic()).isFalse(); + assertThat(updatedAccess.getReadAccessPredicate()).isInstanceOf(NoAccessPredicate.class); + assertThat(updatedAccess.getWriteAccessPredicate()).isInstanceOf(NoAccessPredicate.class); + } + + @Test + void shouldNotChangeAccessOfAccessControlledGraph() throws StoreException, OperationException { + final String federatedGraphId = "federated"; + final String graphId = "shouldNotChangeAccessOfAccessControlledGraph"; + + final FederatedStore federatedStore = new FederatedStore(); + federatedStore.initialise(federatedGraphId, null, new StoreProperties()); + + // Add a graph that no one can edit + final AddGraph addGraph = new AddGraph.Builder() + .graphConfig(new GraphConfig(graphId)) + .schema(new Schema()) + .properties(new StoreProperties().getProperties()) + .isPublic(false) + .writePredicate(new NoAccessPredicate()) + .build(); + + final ChangeGraphAccess changeGraphAccess = new ChangeGraphAccess.Builder() + .graphId(graphId) + .isPublic(true) + .build(); + + // When + federatedStore.execute(addGraph, new Context()); + + // Then + assertThatExceptionOfType(OperationException.class) + .isThrownBy(() -> federatedStore.execute(changeGraphAccess, new Context())) + .withMessageContaining("does not have write permissions"); + } +} From d41a74bedb9c2bfd0187c5624de5f63c74e67d1c Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Thu, 9 Jan 2025 15:10:38 +0000 Subject: [PATCH 06/29] Gh-2478: JobTracker tracks all operations, not just jobs (#3347) * job tracker fixes * fix test and copyright * checkstyle * sonarcloud fixes * checkstyle * Address comments Co-authored-by: tb06904 <141412860+tb06904@users.noreply.github.com> --------- Co-authored-by: tb06904 <141412860+tb06904@users.noreply.github.com> --- .../java/uk/gov/gchq/gaffer/store/Store.java | 12 +---- .../uk/gov/gchq/gaffer/store/StoreTest.java | 12 ++--- .../service/v2/AbstractOperationService.java | 7 ++- .../rest/service/v2/IOperationServiceV2.java | 5 +- .../rest/service/v2/OperationServiceV2.java | 11 ++-- .../rest/service/v2/JobServiceV2IT.java | 30 +++++------ .../rest/service/v2/OperationServiceV2IT.java | 49 ++++++++--------- .../rest/controller/OperationController.java | 12 ++--- .../controller/OperationControllerIT.java | 52 +++++++------------ 9 files changed, 75 insertions(+), 115 deletions(-) diff --git a/core/store/src/main/java/uk/gov/gchq/gaffer/store/Store.java b/core/store/src/main/java/uk/gov/gchq/gaffer/store/Store.java index d153548bacd..40d37b067a1 100644 --- a/core/store/src/main/java/uk/gov/gchq/gaffer/store/Store.java +++ b/core/store/src/main/java/uk/gov/gchq/gaffer/store/Store.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 Crown Copyright + * Copyright 2016-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -386,15 +386,7 @@ public O execute(final Output operation, final Context context) throws Op } protected O execute(final OperationChain operation, final Context context) throws OperationException { - try { - addOrUpdateJobDetail(operation, context, null, JobStatus.RUNNING); - final O result = (O) handleOperation(operation, context); - addOrUpdateJobDetail(operation, context, null, JobStatus.FINISHED); - return result; - } catch (final Throwable t) { - addOrUpdateJobDetail(operation, context, t.getMessage(), JobStatus.FAILED); - throw t; - } + return (O) handleOperation(operation, context); } /** diff --git a/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java b/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java index c01e7241d8c..27ae65325b7 100644 --- a/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java +++ b/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 Crown Copyright + * Copyright 2016-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,6 @@ import uk.gov.gchq.gaffer.cache.CacheServiceLoader; import uk.gov.gchq.gaffer.cache.ICache; import uk.gov.gchq.gaffer.cache.ICacheService; -import uk.gov.gchq.gaffer.cache.exception.CacheOperationException; import uk.gov.gchq.gaffer.cache.impl.HashMapCacheService; import uk.gov.gchq.gaffer.commonutil.TestGroups; import uk.gov.gchq.gaffer.commonutil.TestPropertyNames; @@ -164,12 +163,12 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static uk.gov.gchq.gaffer.jobtracker.JobTracker.JOB_TRACKER_CACHE_SERVICE_NAME; import static uk.gov.gchq.gaffer.store.StoreTrait.INGEST_AGGREGATION; import static uk.gov.gchq.gaffer.store.StoreTrait.ORDERED; @@ -257,12 +256,10 @@ public void after() { } @Test - public void shouldExecuteOperationWhenJobTrackerCacheIsBroken(@Mock final StoreProperties storeProperties) throws Exception { + void shouldNotCreateJobWhenExecutingOperation(@Mock final StoreProperties storeProperties) throws Exception { // Given ICache mockICache = Mockito.mock(ICache.class); - doThrow(new CacheOperationException("Stubbed class")).when(mockICache).put(any(), any()); ICacheService mockICacheService = Mockito.spy(ICacheService.class); - given(mockICacheService.getCache(any())).willReturn(mockICache); Field field = CacheServiceLoader.class.getDeclaredField("SERVICES"); field.setAccessible(true); @@ -278,8 +275,7 @@ public void shouldExecuteOperationWhenJobTrackerCacheIsBroken(@Mock final StoreP // Then verify(addElementsHandler).doOperation(addElements, context, store); - verify(mockICacheService, Mockito.atLeast(1)).getCache(any()); - verify(mockICache, Mockito.atLeast(1)).put(any(), any()); + verifyNoInteractions(mockICacheService, mockICache); } @Test diff --git a/rest-api/common-rest/src/main/java/uk/gov/gchq/gaffer/rest/service/v2/AbstractOperationService.java b/rest-api/common-rest/src/main/java/uk/gov/gchq/gaffer/rest/service/v2/AbstractOperationService.java index 36269935a9d..f8aeb5c1eca 100644 --- a/rest-api/common-rest/src/main/java/uk/gov/gchq/gaffer/rest/service/v2/AbstractOperationService.java +++ b/rest-api/common-rest/src/main/java/uk/gov/gchq/gaffer/rest/service/v2/AbstractOperationService.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 Crown Copyright + * Copyright 2020-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,6 @@ package uk.gov.gchq.gaffer.rest.service.v2; import uk.gov.gchq.gaffer.commonutil.CloseableUtil; -import uk.gov.gchq.gaffer.commonutil.pair.Pair; import uk.gov.gchq.gaffer.core.exception.GafferRuntimeException; import uk.gov.gchq.gaffer.core.exception.Status; import uk.gov.gchq.gaffer.graph.GraphRequest; @@ -97,7 +96,7 @@ protected void postOperationHook(final OperationChain opChain, final Context } @SuppressWarnings({"ThrowFromFinallyBlock", "PMD.UseTryWithResources"}) - protected Pair _execute(final Operation operation, final Context context) { + protected O _execute(final Operation operation, final Context context) { OperationChain opChain = (OperationChain) OperationChain.wrap(operation); @@ -119,7 +118,7 @@ protected Pair _execute(final Operation operation, final Context } } - return new Pair<>(result.getResult(), result.getContext().getJobId()); + return result.getResult(); } protected Operation generateExampleJson(final Class opClass) throws IllegalAccessException, InstantiationException { diff --git a/rest-api/core-rest/src/main/java/uk/gov/gchq/gaffer/rest/service/v2/IOperationServiceV2.java b/rest-api/core-rest/src/main/java/uk/gov/gchq/gaffer/rest/service/v2/IOperationServiceV2.java index c0839f6631d..4156138eb43 100644 --- a/rest-api/core-rest/src/main/java/uk/gov/gchq/gaffer/rest/service/v2/IOperationServiceV2.java +++ b/rest-api/core-rest/src/main/java/uk/gov/gchq/gaffer/rest/service/v2/IOperationServiceV2.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2020 Crown Copyright + * Copyright 2016-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,8 +42,6 @@ import static uk.gov.gchq.gaffer.rest.ServiceConstants.GAFFER_MEDIA_TYPE_HEADER; import static uk.gov.gchq.gaffer.rest.ServiceConstants.GAFFER_MEDIA_TYPE_HEADER_DESCRIPTION; import static uk.gov.gchq.gaffer.rest.ServiceConstants.INTERNAL_SERVER_ERROR; -import static uk.gov.gchq.gaffer.rest.ServiceConstants.JOB_ID_HEADER; -import static uk.gov.gchq.gaffer.rest.ServiceConstants.JOB_ID_HEADER_DESCRIPTION; import static uk.gov.gchq.gaffer.rest.ServiceConstants.OK; import static uk.gov.gchq.gaffer.rest.ServiceConstants.OPERATION_NOT_FOUND; import static uk.gov.gchq.gaffer.rest.ServiceConstants.OPERATION_NOT_IMPLEMENTED; @@ -91,7 +89,6 @@ public interface IOperationServiceV2 { produces = (APPLICATION_JSON + "," + TEXT_PLAIN), response = Object.class, responseHeaders = { - @ResponseHeader(name = JOB_ID_HEADER, description = JOB_ID_HEADER_DESCRIPTION), @ResponseHeader(name = GAFFER_MEDIA_TYPE_HEADER, description = GAFFER_MEDIA_TYPE_HEADER_DESCRIPTION) }) @ApiResponses(value = {@ApiResponse(code = 200, message = OK, response = Object.class), diff --git a/rest-api/core-rest/src/main/java/uk/gov/gchq/gaffer/rest/service/v2/OperationServiceV2.java b/rest-api/core-rest/src/main/java/uk/gov/gchq/gaffer/rest/service/v2/OperationServiceV2.java index c18011f6855..c07467c5f8f 100644 --- a/rest-api/core-rest/src/main/java/uk/gov/gchq/gaffer/rest/service/v2/OperationServiceV2.java +++ b/rest-api/core-rest/src/main/java/uk/gov/gchq/gaffer/rest/service/v2/OperationServiceV2.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 Crown Copyright + * Copyright 2017-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,6 @@ import uk.gov.gchq.gaffer.commonutil.CloseableUtil; import uk.gov.gchq.gaffer.commonutil.exception.UnauthorisedException; -import uk.gov.gchq.gaffer.commonutil.pair.Pair; import uk.gov.gchq.gaffer.core.exception.Error; import uk.gov.gchq.gaffer.core.exception.Status; import uk.gov.gchq.gaffer.operation.Operation; @@ -45,7 +44,6 @@ import static uk.gov.gchq.gaffer.jsonserialisation.JSONSerialiser.createDefaultMapper; import static uk.gov.gchq.gaffer.rest.ServiceConstants.GAFFER_MEDIA_TYPE; import static uk.gov.gchq.gaffer.rest.ServiceConstants.GAFFER_MEDIA_TYPE_HEADER; -import static uk.gov.gchq.gaffer.rest.ServiceConstants.JOB_ID_HEADER; /** * An implementation of {@link IOperationServiceV2}. By default it will use a singleton @@ -85,10 +83,9 @@ public Response getOperationDetails() { @Override public Response execute(final Operation operation) { - final Pair resultAndJobId = _execute(operation, userFactory.createContext()); - return Response.ok(resultAndJobId.getFirst()) + final Object result = _execute(operation, userFactory.createContext()); + return Response.ok(result) .header(GAFFER_MEDIA_TYPE_HEADER, GAFFER_MEDIA_TYPE) - .header(JOB_ID_HEADER, resultAndJobId.getSecond()) .build(); } @@ -109,7 +106,7 @@ public Response executeChunkedChain(final OperationChain opChain) { // create thread to write chunks to the chunked output object Thread thread = new Thread(() -> { try { - final Object result = _execute(opChain, context).getFirst(); + final Object result = _execute(opChain, context); chunkResult(result, output); } catch (final Exception e) { throw new RuntimeException(e); diff --git a/rest-api/core-rest/src/test/java/uk/gov/gchq/gaffer/rest/service/v2/JobServiceV2IT.java b/rest-api/core-rest/src/test/java/uk/gov/gchq/gaffer/rest/service/v2/JobServiceV2IT.java index 75726cee287..71db1530f4e 100644 --- a/rest-api/core-rest/src/test/java/uk/gov/gchq/gaffer/rest/service/v2/JobServiceV2IT.java +++ b/rest-api/core-rest/src/test/java/uk/gov/gchq/gaffer/rest/service/v2/JobServiceV2IT.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2020 Crown Copyright + * Copyright 2019-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,14 +35,12 @@ import java.util.List; import java.util.concurrent.TimeUnit; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; -public class JobServiceV2IT extends AbstractRestApiV2IT { +class JobServiceV2IT extends AbstractRestApiV2IT { @Test - public void shouldCorrectlyDoAndThenCancelScheduledJob() throws IOException, InterruptedException { + void shouldCorrectlyDoAndThenCancelScheduledJob() throws IOException, InterruptedException { // When final Repeat repeat = new Repeat(1, 2, TimeUnit.SECONDS); Job job = new Job(repeat, new OperationChain.Builder().first(new GetAllElements()).build()); @@ -52,7 +50,7 @@ public void shouldCorrectlyDoAndThenCancelScheduledJob() throws IOException, Int }); // Then - assertEquals(201, jobSchedulingResponse.getStatus()); + assertThat(jobSchedulingResponse.getStatus()).isEqualTo(201); String parentJobId = jobSchedulingDetail.getJobId(); // Wait for first scheduled to run @@ -65,10 +63,10 @@ public void shouldCorrectlyDoAndThenCancelScheduledJob() throws IOException, Int for (JobDetail jobDetail : jobDetails) { if (null != jobDetail.getParentJobId() && jobDetail.getParentJobId().equals(parentJobId)) { - assertEquals(JobStatus.FINISHED, jobDetail.getStatus()); + assertThat(jobDetail.getStatus()).isEqualTo(JobStatus.FINISHED); } if (jobDetail.getJobId().equals(parentJobId)) { - assertEquals(JobStatus.SCHEDULED_PARENT, jobDetail.getStatus()); + assertThat(jobDetail.getStatus()).isEqualTo(JobStatus.SCHEDULED_PARENT); } } @@ -81,13 +79,13 @@ public void shouldCorrectlyDoAndThenCancelScheduledJob() throws IOException, Int for (JobDetail jobDetail : jobDetailsAfterCancelled) { if (parentJobId.equals(jobDetail.getJobId())) { - assertEquals(JobStatus.CANCELLED, jobDetail.getStatus()); + assertThat(jobDetail.getStatus()).isEqualTo(JobStatus.CANCELLED); } } } @Test - public void shouldNotKeepScheduledJobsRunningAfterRestartWhenUsingInMemoryCache() throws IOException { + void shouldNotKeepScheduledJobsRunningAfterRestartWhenUsingInMemoryCache() throws IOException { // Given - schedule Job final Repeat repeat = new Repeat(1, 2, TimeUnit.SECONDS); Job job = new Job(repeat, new OperationChain.Builder().first(new GetAllElements()).build()); @@ -104,8 +102,8 @@ public void shouldNotKeepScheduledJobsRunningAfterRestartWhenUsingInMemoryCache( }); // then - assert parent is of Scheduled parent - assertEquals(JobStatus.SCHEDULED_PARENT, - allJobDetails.stream().filter(jobDetail -> jobDetail.getJobId().equals(parentJobId)).findFirst().get().getStatus()); + JobStatus jobStatus = allJobDetails.stream().filter(jobDetail -> jobDetail.getJobId().equals(parentJobId)).findFirst().get().getStatus(); + assertThat(jobStatus).isEqualTo(JobStatus.SCHEDULED_PARENT); // Restart server to check Job still scheduled client.stopServer(); @@ -119,15 +117,15 @@ public void shouldNotKeepScheduledJobsRunningAfterRestartWhenUsingInMemoryCache( }); // Then - assert parent job id is not present - assertTrue(allJobDetails2.stream().noneMatch(jobDetail -> jobDetail.getJobId().equals(parentJobId))); + assertThat(allJobDetails2.stream().noneMatch(jobDetail -> jobDetail.getJobId().equals(parentJobId))).isTrue(); } @Test - public void shouldReturnJobIdHeader() throws IOException { + void shouldNotReturnJobIdHeader() throws IOException { // When final Response response = client.executeOperation(new GetAllElements()); // Then - assertNotNull(response.getHeaderString(ServiceConstants.JOB_ID_HEADER)); + assertThat(response.getHeaders().toString()).doesNotContain(ServiceConstants.JOB_ID_HEADER); } } diff --git a/rest-api/core-rest/src/test/java/uk/gov/gchq/gaffer/rest/service/v2/OperationServiceV2IT.java b/rest-api/core-rest/src/test/java/uk/gov/gchq/gaffer/rest/service/v2/OperationServiceV2IT.java index ea5f944f055..f41ac2cd8df 100644 --- a/rest-api/core-rest/src/test/java/uk/gov/gchq/gaffer/rest/service/v2/OperationServiceV2IT.java +++ b/rest-api/core-rest/src/test/java/uk/gov/gchq/gaffer/rest/service/v2/OperationServiceV2IT.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2021 Crown Copyright + * Copyright 2015-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -51,25 +51,21 @@ import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; import static uk.gov.gchq.gaffer.core.exception.Status.SERVICE_UNAVAILABLE; public class OperationServiceV2IT extends OperationServiceIT { @Test - public void shouldReturnJobIdHeader() throws IOException { + void shouldNotReturnJobIdHeader() throws IOException { // When final Response response = client.executeOperation(new GetAllElements()); // Then - assertNotNull(response.getHeaderString(ServiceConstants.JOB_ID_HEADER)); + assertThat(response.getHeaderString(ServiceConstants.JOB_ID_HEADER)).isNull(); } @Test - public void shouldReturn403WhenUnauthorised() throws IOException { + void shouldReturn403WhenUnauthorised() throws IOException { // Given final Graph graph = new Graph.Builder() .config(StreamUtil.graphConfig(this.getClass())) @@ -82,11 +78,11 @@ public void shouldReturn403WhenUnauthorised() throws IOException { final Response response = client.executeOperation(new GetAllElements()); // Then - assertEquals(403, response.getStatus()); + assertThat(response.getStatus()).isEqualTo(403); } @Test - public void shouldPropagateStatusInformationContainedInOperationExceptionsThrownByOperationHandlers() + void shouldPropagateStatusInformationContainedInOperationExceptionsThrownByOperationHandlers() throws IOException { // Given final StoreProperties storeProperties = StoreProperties.loadStoreProperties(StreamUtil.STORE_PROPERTIES); @@ -103,11 +99,11 @@ public void shouldPropagateStatusInformationContainedInOperationExceptionsThrown final Response response = client.executeOperation(new GetAllJobDetails()); // Then - assertEquals(SERVICE_UNAVAILABLE.getStatusCode(), response.getStatus()); + assertThat(response.getStatus()).isEqualTo(SERVICE_UNAVAILABLE.getStatusCode()); } @Test - public void shouldReturnSameJobIdInHeaderAsGetAllJobDetailsOperation() throws IOException { + void shouldNotHaveJobIdInHeader() throws IOException { // Given final Graph graph = new Graph.Builder() .config(StreamUtil.graphConfig(this.getClass())) @@ -121,11 +117,12 @@ public void shouldReturnSameJobIdInHeaderAsGetAllJobDetailsOperation() throws IO final Response response = client.executeOperation(new GetAllJobDetails()); // Then - assertTrue(response.readEntity(String.class).contains(response.getHeaderString("job-id"))); + assertThat(response.getHeaders()).isNotEmpty(); + assertThat(response.readEntity(String.class)).doesNotContain("job-id"); } @Test - public void shouldReturnAllOperationsAsOperationDetails() throws IOException, ClassNotFoundException { + void shouldReturnAllOperationsAsOperationDetails() throws IOException, ClassNotFoundException { // Given final Set> expectedOperations = client.getDefaultGraphFactory().getGraph() .getSupportedOperations(); @@ -146,7 +143,7 @@ public void shouldReturnAllOperationsAsOperationDetails() throws IOException, Cl } @Test - public void shouldReturnOperationDetailSummaryOfClass() throws Exception { + void shouldReturnOperationDetailSummaryOfClass() throws Exception { // Given final String expectedSummary = "\"summary\":\"Gets elements related to provided seeds\""; @@ -154,11 +151,11 @@ public void shouldReturnOperationDetailSummaryOfClass() throws Exception { final Response response = client.getOperationDetails(GetElements.class); // Then - assertTrue(response.readEntity(String.class).contains(expectedSummary)); + assertThat(response.readEntity(String.class)).contains(expectedSummary); } @Test - public void shouldReturnOutputClassForOperationWithOutput() throws Exception { + void shouldReturnOutputClassForOperationWithOutput() throws Exception { // Given final String expectedOutputString = "\"outputClassName\":\"java.lang.Iterable\""; @@ -166,11 +163,11 @@ public void shouldReturnOutputClassForOperationWithOutput() throws Exception { final Response response = client.getOperationDetails(GetElements.class); // Then - assertTrue(response.readEntity(String.class).contains(expectedOutputString)); + assertThat(response.readEntity(String.class)).contains(expectedOutputString); } @Test - public void shouldNotIncludeAnyOutputClassForOperationWithoutOutput() throws Exception { + void shouldNotIncludeAnyOutputClassForOperationWithoutOutput() throws Exception { // Given final String outputClassNameString = "\"outputClassName\""; @@ -178,11 +175,11 @@ public void shouldNotIncludeAnyOutputClassForOperationWithoutOutput() throws Exc final Response response = client.getOperationDetails(DiscardOutput.class); // Then - assertFalse(response.readEntity(String.class).contains(outputClassNameString)); + assertThat(response.readEntity(String.class)).doesNotContain(outputClassNameString); } @Test - public void shouldReturnOptionsAndSummariesForEnumFields() throws Exception { + void shouldReturnOptionsAndSummariesForEnumFields() throws Exception { // When final Response response = client.getOperationDetails(GetElements.class); @@ -197,11 +194,11 @@ public void shouldReturnOptionsAndSummariesForEnumFields() throws Exception { new OperationFieldPojo("directedType", "java.lang.String", false, "Is the Edge directed?", Sets.newHashSet("DIRECTED", "UNDIRECTED", "EITHER")), new OperationFieldPojo("views", "java.util.List", false, null, null) ); - assertEquals(fields, opDetails.getFields()); + assertThat(opDetails.getFields()).isEqualTo(fields); } @Test - public void shouldAllowUserWithAuthThroughHeaders() throws IOException { + void shouldAllowUserWithAuthThroughHeaders() throws IOException { // Given System.setProperty(SystemProperty.USER_FACTORY_CLASS, TestUserFactory.class.getName()); client.stopServer(); @@ -221,11 +218,11 @@ public void shouldAllowUserWithAuthThroughHeaders() throws IOException { final Response response = ((RestApiV2TestClient) client).executeOperationChainChunkedWithHeaders(opChain, "ListUser"); // Then - assertEquals(200, response.getStatus()); + assertThat(response.getStatus()).isEqualTo(200); } @Test - public void shouldNotAllowUserWithNoAuthThroughHeaders() throws IOException { + void shouldNotAllowUserWithNoAuthThroughHeaders() throws IOException { // Given System.setProperty(SystemProperty.USER_FACTORY_CLASS, TestUserFactory.class.getName()); client.stopServer(); @@ -246,7 +243,7 @@ public void shouldNotAllowUserWithNoAuthThroughHeaders() throws IOException { "BasicUser"); // Then - assertEquals(500, response.getStatus()); + assertThat(response.getStatus()).isEqualTo(500); } @Override diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/OperationController.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/OperationController.java index 0d6e3ce0241..529f434b6d6 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/OperationController.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/OperationController.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2024 Crown Copyright + * Copyright 2020-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,7 +36,6 @@ import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; import uk.gov.gchq.gaffer.commonutil.CloseableUtil; -import uk.gov.gchq.gaffer.commonutil.pair.Pair; import uk.gov.gchq.gaffer.core.exception.GafferRuntimeException; import uk.gov.gchq.gaffer.core.exception.Status; import uk.gov.gchq.gaffer.operation.Operation; @@ -56,7 +55,6 @@ import static uk.gov.gchq.gaffer.jsonserialisation.JSONSerialiser.createDefaultMapper; import static uk.gov.gchq.gaffer.rest.ServiceConstants.GAFFER_MEDIA_TYPE; import static uk.gov.gchq.gaffer.rest.ServiceConstants.GAFFER_MEDIA_TYPE_HEADER; -import static uk.gov.gchq.gaffer.rest.ServiceConstants.JOB_ID_HEADER; @RestController @Tag(name = "operations") @@ -211,11 +209,10 @@ public ResponseEntity execute(@RequestHeader final HttpHeaders httpHeade })) @RequestBody final Operation operation) { userFactory.setHttpHeaders(httpHeaders); - final Pair resultAndJobId = _execute(operation, userFactory.createContext()); + final Object result = _execute(operation, userFactory.createContext()); return ResponseEntity.ok() .header(GAFFER_MEDIA_TYPE_HEADER, GAFFER_MEDIA_TYPE) - .header(JOB_ID_HEADER, resultAndJobId.getSecond()) - .body(resultAndJobId.getFirst()); + .body(result); } @PostMapping(path = "/execute/chunked", consumes = APPLICATION_JSON_VALUE, produces = APPLICATION_JSON_VALUE) @@ -227,8 +224,7 @@ public ResponseEntity executeChunked( userFactory.setHttpHeaders(httpHeaders); final StreamingResponseBody responseBody = response -> { try { - final Pair resultAndJobId = _execute(operation, userFactory.createContext()); - final Object result = resultAndJobId.getFirst(); + final Object result = _execute(operation, userFactory.createContext()); if (result instanceof Iterable) { final Iterable itr = (Iterable) result; try { diff --git a/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/integration/controller/OperationControllerIT.java b/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/integration/controller/OperationControllerIT.java index dddbdb24357..6ebb42032ae 100644 --- a/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/integration/controller/OperationControllerIT.java +++ b/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/integration/controller/OperationControllerIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2024 Crown Copyright + * Copyright 2020-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,21 +45,17 @@ import uk.gov.gchq.koryphe.impl.binaryoperator.StringConcat; import uk.gov.gchq.koryphe.util.ReflectionUtil; -import java.io.IOException; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; import static uk.gov.gchq.gaffer.core.exception.Status.SERVICE_UNAVAILABLE; import static uk.gov.gchq.gaffer.jsonserialisation.JSONSerialiser.createDefaultMapper; -public class OperationControllerIT extends AbstractRestApiIT { +class OperationControllerIT extends AbstractRestApiIT { @Autowired private GraphFactory graphFactory; // This will be a Mock (see application-test.properties) @@ -70,7 +66,7 @@ private MockGraphFactory getGraphFactory() { @Test - public void shouldReturnHelpfulErrorMessageIfJsonIsIncorrect() { + void shouldReturnHelpfulErrorMessageIfJsonIsIncorrect() { // Given Graph graph = new Graph.Builder() .config(StreamUtil.graphConfig(this.getClass())) @@ -83,7 +79,7 @@ public void shouldReturnHelpfulErrorMessageIfJsonIsIncorrect() { // When String request = "{\"class\"\"GetAllElements\"}"; - LinkedMultiValueMap headers = new LinkedMultiValueMap(); + LinkedMultiValueMap headers = new LinkedMultiValueMap(); headers.add("Content-Type", "application/json;charset=utf-8"); final ResponseEntity response = post("/graph/operations/execute", @@ -91,13 +87,13 @@ public void shouldReturnHelpfulErrorMessageIfJsonIsIncorrect() { Error.class); // Then - assertEquals(400, response.getStatusCode().value()); - assertEquals(400, response.getBody().getStatusCode()); - assertTrue(response.getBody().getSimpleMessage().contains("was expecting a colon to separate field name and value")); + assertThat(response.getStatusCode().value()).isEqualTo(400); + assertThat(response.getBody().getStatusCode()).isEqualTo(400); + assertThat(response.getBody().getSimpleMessage()).contains("was expecting a colon to separate field name and value"); } @Test - public void shouldReturnHelpfulErrorMessageIfOperationIsUnsupported() { + void shouldReturnHelpfulErrorMessageIfOperationIsUnsupported() { // Given Graph graph = new Graph.Builder() .config(StreamUtil.graphConfig(this.getClass())) @@ -113,13 +109,12 @@ public void shouldReturnHelpfulErrorMessageIfOperationIsUnsupported() { Error.class); // Then - - assertNotNull(response.getBody().getSimpleMessage()); - assertTrue(response.getBody().getSimpleMessage().contains("GetAllGraphIds is not supported by the MapStore")); + assertThat(response.getBody().getSimpleMessage()).isNotNull(); + assertThat(response.getBody().getSimpleMessage()).contains("GetAllGraphIds is not supported by the MapStore"); } @Test - public void shouldReturn403WhenUnauthorised() throws IOException { + void shouldReturn403WhenUnauthorised() { // Given Graph graph = new Graph.Builder() .config(StreamUtil.graphConfig(this.getClass())) @@ -135,12 +130,12 @@ public void shouldReturn403WhenUnauthorised() throws IOException { Error.class); // Then - assertEquals(403, response.getStatusCode().value()); - assertEquals(403, response.getBody().getStatusCode()); + assertThat(response.getStatusCode().value()).isEqualTo(403); + assertThat(response.getBody().getStatusCode()).isEqualTo(403); } @Test - public void shouldPropagateStatusInformationContainedInOperationExceptionsThrownByOperationHandlers() throws IOException { + void shouldPropagateStatusInformationContainedInOperationExceptionsThrownByOperationHandlers() { // Given final StoreProperties storeProperties = new MapStoreProperties(); storeProperties.set(StoreProperties.JOB_TRACKER_ENABLED, Boolean.FALSE.toString()); @@ -158,11 +153,11 @@ public void shouldPropagateStatusInformationContainedInOperationExceptionsThrown new GetAllJobDetails(), Error.class); // Then - assertEquals(SERVICE_UNAVAILABLE.getStatusCode(), response.getStatusCode().value()); + assertThat(response.getStatusCode().value()).isEqualTo(SERVICE_UNAVAILABLE.getStatusCode()); } @Test - public void shouldReturnSameJobIdInHeaderAsGetAllJobDetailsOperation() throws IOException { + void shouldNotReturnJobIdInHeader() { // Given StoreProperties properties = new MapStoreProperties(); properties.setJobTrackerEnabled(true); @@ -182,18 +177,11 @@ public void shouldReturnSameJobIdInHeaderAsGetAllJobDetailsOperation() throws IO Set.class); // Then - try { - assertTrue(response.getBody().toString().contains(response.getHeaders().get("job-id").get(0))); - } catch (final AssertionError e) { - System.out.println("Job ID was not found in the Header"); - System.out.println("Header was: " + response.getHeaders().get("job-id")); - System.out.println("Body was: " + response.getBody()); - throw e; - } + assertThat(response.getHeaders().toString()).doesNotContain("job-id"); } @Test - public void shouldCorrectlyStreamExecuteChunked() throws Exception { + void shouldCorrectlyStreamExecuteChunked() throws Exception { // Given final Schema schema = new Schema.Builder() .entity("g1", new SchemaEntityDefinition.Builder() @@ -241,11 +229,11 @@ public void shouldCorrectlyStreamExecuteChunked() throws Exception { // Then String expected = mapper.writeValueAsString(ent1) + "\r\n" + mapper.writeValueAsString(ent2) + "\r\n"; - assertEquals(expected, response.getBody()); + assertThat(response.getBody()).isEqualTo(expected); } @Test - public void shouldCorrectlySerialiseAllOperationDetails() throws IOException { + void shouldCorrectlySerialiseAllOperationDetails() { // Given final Graph graph = new Graph.Builder() .config(StreamUtil.graphConfig(this.getClass())) From 4eee4dea9d92aac6e8bde9be4fd5bb529fc2e647 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:14:16 +0000 Subject: [PATCH 07/29] Gh-3351: Better GetWalks support on federated POC (#3352) * make getwalks get handled correctly * force options on all sub operations * typo * preserve outer options on getwalks * typo * dont flatten getwalks op * move block * set opts early * add missing all edge check * integration tests * test typo --- .../gchq/gaffer/operation/impl/GetWalks.java | 12 +- .../operation/handler/GetWalksHandler.java | 32 +++-- .../handler/OperationChainHandler.java | 14 +- .../federated/simple/FederatedStore.java | 7 +- .../federated/simple/access/GraphAccess.java | 6 +- .../handler/EitherOperationHandler.java | 13 +- .../federated/simple/FederatedStoreIT.java | 73 +++++++++- .../simple/util/FederatedModernTestUtils.java | 135 ++++++++++++++++++ 8 files changed, 269 insertions(+), 23 deletions(-) create mode 100644 store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/util/FederatedModernTestUtils.java diff --git a/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/GetWalks.java b/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/GetWalks.java index a27026367bc..a8e053c845c 100644 --- a/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/GetWalks.java +++ b/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/GetWalks.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 Crown Copyright + * Copyright 2017-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,6 +41,7 @@ import uk.gov.gchq.koryphe.ValidationResult; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.function.Predicate; @@ -153,6 +154,11 @@ public ValidationResult validate() { return result; } + @Override + public List flatten() { + return Arrays.asList(this); + } + @JsonIgnore public int getNumberOfGetEdgeOperations() { return getNumberOfGetEdgeOperations(operations); @@ -166,7 +172,7 @@ private int getNumberOfGetEdgeOperations(final Operation op) { hops += getNumberOfGetEdgeOperations(((Operations) op).getOperations()); } else if (op instanceof GetElements) { final GetElements getElements = (GetElements) op; - if (null != getElements.getView() && getElements.getView().hasEdges()) { + if (null != getElements.getView() && (getElements.getView().hasEdges() || getElements.getView().isAllEdges())) { hops += 1; } } @@ -187,7 +193,7 @@ private int getNumberOfGetEdgeOperationsWithoutRepeats(final Operation op) { hops += getNumberOfGetEdgeOperationsWithoutRepeats(((Operations) op).getOperations()); } else if (op instanceof GetElements) { final GetElements getElements = (GetElements) op; - if (null != getElements.getView() && getElements.getView().hasEdges()) { + if (null != getElements.getView() && (getElements.getView().hasEdges() || getElements.getView().isAllEdges())) { hops += 1; } } diff --git a/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/handler/GetWalksHandler.java b/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/handler/GetWalksHandler.java index 67b2a905f2c..697a672f8f5 100644 --- a/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/handler/GetWalksHandler.java +++ b/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/handler/GetWalksHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 Crown Copyright + * Copyright 2017-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -52,6 +52,7 @@ import uk.gov.gchq.koryphe.iterable.LimitedIterable; import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -98,6 +99,7 @@ public class GetWalksHandler implements OutputOperationHandler> { private Integer maxHops = null; private boolean prune = true; + private java.util.Map operationOptions = new HashMap<>(); @Override public Iterable doOperation(final GetWalks getWalks, final Context context, final Store store) throws OperationException { @@ -113,6 +115,7 @@ public Iterable doOperation(final GetWalks getWalks, final Context context final Integer resultLimit = getWalks.getResultsLimit(); final int hops = getWalks.getNumberOfGetEdgeOperations(); + operationOptions = getWalks.getOptions(); final LimitedIterable limitedInputItr = new LimitedIterable<>(getWalks.getInput(), 0, resultLimit, false); @@ -134,14 +137,24 @@ public Iterable doOperation(final GetWalks getWalks, final Context context for (final OperationChain> operation : getWalks.getOperations()) { if (isWhileOperation(operation)) { seeds = executeWhileOperation( - operation, seeds, resultLimit, - context, store, hops, adjacencyMaps, entityMaps - ); + (While) operation.getOperations().get(0), + seeds, + resultLimit, + context, + store, + hops, + adjacencyMaps, + entityMaps); } else { seeds = executeOperation( - operation, seeds, resultLimit, - context, store, hops, adjacencyMaps, entityMaps - ); + operation, + seeds, + resultLimit, + context, + store, + hops, + adjacencyMaps, + entityMaps); } } @@ -181,7 +194,7 @@ private boolean isWhileOperation(final OperationChain> operati && operation.getOperations().get(0) instanceof While; } - private List executeWhileOperation(final OperationChain> operation, + private List executeWhileOperation(final While> whileOp, final List seeds, final Integer resultLimit, final Context context, @@ -190,7 +203,6 @@ private List executeWhileOperation(final OperationChain> op final AdjacencyMaps adjacencyMaps, final EntityMaps entityMaps) throws OperationException { List resultSeeds = seeds; - final While whileOp = (While) operation.getOperations().get(0); if (null != whileOp.getOperation()) { validateWhileOperation(whileOp); final OperationHandler opHandler = store.getOperationHandler(While.class); @@ -278,6 +290,8 @@ private Iterable executeOperation(final Output> opera .then(OperationChain.wrap(operation)) .build(); + convertedOp.setOptions(operationOptions); + // Execute an the operation chain on the supplied store and cache // the seeds in memory using an ArrayList. return new LimitedIterable<>(store.execute(convertedOp, context), 0, resultLimit, false); diff --git a/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/handler/OperationChainHandler.java b/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/handler/OperationChainHandler.java index f866460215e..357fa583a42 100644 --- a/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/handler/OperationChainHandler.java +++ b/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/handler/OperationChainHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 Crown Copyright + * Copyright 2017-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ import uk.gov.gchq.koryphe.ValidationResult; import java.util.List; +import java.util.Map; import static uk.gov.gchq.gaffer.store.operation.handler.util.OperationHandlerUtil.updateOperationInput; @@ -40,6 +41,10 @@ * @param the output type of the operation chain */ public class OperationChainHandler implements OutputOperationHandler, OUT> { + /** + * Context variable to apply the operation options on the chain to all sub operations + */ + public static final String APPLY_CHAIN_OPS_TO_ALL = "applyChainOptionsToAll"; private final OperationChainValidator opChainValidator; private final List opChainOptimisers; @@ -70,6 +75,7 @@ public OUT doOperation(final OperationChain operationChain, final Context c } public OperationChain prepareOperationChain(final OperationChain operationChain, final Context context, final Store store) { + Map options = operationChain.getOptions(); final ValidationResult validationResult = opChainValidator.validate(operationChain, context .getUser(), store); if (!validationResult.isValid()) { @@ -81,6 +87,12 @@ public OperationChain prepareOperationChain(final OperationChain opera for (final OperationChainOptimiser opChainOptimiser : opChainOptimisers) { optimisedOperationChain = opChainOptimiser.optimise(optimisedOperationChain); } + + // Optionally re-apply the chain level options to all sub operations too + if (context.getVariable(APPLY_CHAIN_OPS_TO_ALL) != null) { + optimisedOperationChain.getOperations().forEach(op -> op.setOptions(options)); + } + return optimisedOperationChain; } diff --git a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedStore.java b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedStore.java index 9b29dabd6b2..8bd4372c0aa 100644 --- a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedStore.java +++ b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedStore.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -59,6 +59,7 @@ import uk.gov.gchq.gaffer.operation.Operation; import uk.gov.gchq.gaffer.operation.OperationChain; import uk.gov.gchq.gaffer.operation.OperationException; +import uk.gov.gchq.gaffer.operation.impl.GetWalks; import uk.gov.gchq.gaffer.operation.impl.add.AddElements; import uk.gov.gchq.gaffer.operation.impl.delete.DeleteElements; import uk.gov.gchq.gaffer.operation.impl.get.GetAdjacentIds; @@ -77,6 +78,7 @@ import uk.gov.gchq.gaffer.store.operation.GetTraits; import uk.gov.gchq.gaffer.store.operation.OperationChainValidator; import uk.gov.gchq.gaffer.store.operation.handler.GetGraphCreatedTimeHandler; +import uk.gov.gchq.gaffer.store.operation.handler.GetWalksHandler; import uk.gov.gchq.gaffer.store.operation.handler.OperationChainHandler; import uk.gov.gchq.gaffer.store.operation.handler.OperationHandler; import uk.gov.gchq.gaffer.store.operation.handler.OutputOperationHandler; @@ -135,7 +137,8 @@ public class FederatedStore extends Store { new SimpleEntry<>(ChangeGraphId.class, new ChangeGraphIdHandler()), new SimpleEntry<>(GetAllGraphInfo.class, new GetAllGraphInfoHandler()), new SimpleEntry<>(RemoveGraph.class, new RemoveGraphHandler()), - new SimpleEntry<>(ChangeGraphAccess.class, new ChangeGraphAccessHandler())) + new SimpleEntry<>(ChangeGraphAccess.class, new ChangeGraphAccessHandler()), + new SimpleEntry<>(GetWalks.class, new GetWalksHandler())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); /** diff --git a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/access/GraphAccess.java b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/access/GraphAccess.java index c9aa8d1e268..44fd70cb371 100644 --- a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/access/GraphAccess.java +++ b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/access/GraphAccess.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,10 +24,12 @@ import static uk.gov.gchq.gaffer.federated.simple.FederatedStore.FEDERATED_STORE_SYSTEM_USER; +import java.io.Serializable; + /** * Access control for a Graph that as been added through a federated store. */ -public class GraphAccess implements AccessControlledResource { +public class GraphAccess implements AccessControlledResource, Serializable { // Default accesses applied to a graph can be overridden using builder private boolean isPublic = true; private String owner = User.UNKNOWN_USER_ID; diff --git a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/EitherOperationHandler.java b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/EitherOperationHandler.java index 66de41a466c..e60fdbd4891 100644 --- a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/EitherOperationHandler.java +++ b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/EitherOperationHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,6 +29,7 @@ import uk.gov.gchq.gaffer.store.operation.handler.OperationHandler; import uk.gov.gchq.gaffer.store.operation.handler.named.AddToCacheHandler; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -56,15 +57,17 @@ public EitherOperationHandler(final OperationHandler standardHandler) { public Object doOperation(final O operation, final Context context, final Store store) throws OperationException { LOGGER.debug("Checking if Operation should be handled locally or on sub graphs: {}", operation); - // Check inside operation chain for if all the operations are handled by a federated store + // Check inside operation chain for if any operations are handled locally by a federated store if (operation instanceof OperationChain) { Set> storeSpecificOps = ((FederatedStore) store).getStoreSpecificOperations(); List> chainOps = ((OperationChain) operation).flatten().stream() .map(Operation::getClass) .collect(Collectors.toList()); - if (storeSpecificOps.containsAll(chainOps)) { - return new OperationChainHandler<>(store.getOperationChainValidator(), store.getOperationChainOptimisers()) - .doOperation((OperationChain) operation, context, store); + // Use default chain handler to handle each operation rather than forwarding the whole chain + if (!Collections.disjoint(storeSpecificOps, chainOps)) { + LOGGER.debug("Operation chain contains some operations that should not be forwarded"); + context.setVariable(OperationChainHandler.APPLY_CHAIN_OPS_TO_ALL, true); + return standardHandler.doOperation(operation, context, store); } } diff --git a/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/FederatedStoreIT.java b/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/FederatedStoreIT.java index e38331306fc..613f956c24e 100644 --- a/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/FederatedStoreIT.java +++ b/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/FederatedStoreIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,12 +24,15 @@ import uk.gov.gchq.gaffer.data.element.Element; import uk.gov.gchq.gaffer.data.element.Entity; import uk.gov.gchq.gaffer.data.element.Properties; +import uk.gov.gchq.gaffer.data.elementdefinition.view.View; +import uk.gov.gchq.gaffer.data.graph.Walk; import uk.gov.gchq.gaffer.federated.simple.access.GraphAccess; import uk.gov.gchq.gaffer.federated.simple.operation.AddGraph; import uk.gov.gchq.gaffer.federated.simple.operation.GetAllGraphIds; import uk.gov.gchq.gaffer.federated.simple.operation.GetAllGraphInfo; import uk.gov.gchq.gaffer.federated.simple.operation.handler.FederatedOperationHandler; import uk.gov.gchq.gaffer.federated.simple.operation.handler.get.GetAllGraphInfoHandler; +import uk.gov.gchq.gaffer.federated.simple.util.FederatedModernTestUtils; import uk.gov.gchq.gaffer.federated.simple.util.FederatedTestUtils; import uk.gov.gchq.gaffer.graph.Graph; import uk.gov.gchq.gaffer.graph.GraphConfig; @@ -38,8 +41,12 @@ import uk.gov.gchq.gaffer.named.operation.NamedOperation; import uk.gov.gchq.gaffer.operation.OperationChain; import uk.gov.gchq.gaffer.operation.OperationException; +import uk.gov.gchq.gaffer.operation.data.EntitySeed; +import uk.gov.gchq.gaffer.operation.graph.SeededGraphFilters; +import uk.gov.gchq.gaffer.operation.impl.GetWalks; import uk.gov.gchq.gaffer.operation.impl.add.AddElements; import uk.gov.gchq.gaffer.operation.impl.get.GetAllElements; +import uk.gov.gchq.gaffer.operation.impl.get.GetElements; import uk.gov.gchq.gaffer.operation.impl.get.GetGraphCreatedTime; import uk.gov.gchq.gaffer.store.Context; import uk.gov.gchq.gaffer.store.StoreException; @@ -52,6 +59,7 @@ import static uk.gov.gchq.gaffer.federated.simple.util.FederatedTestUtils.StoreType; import java.util.AbstractMap.SimpleEntry; +import java.util.ArrayList; import java.util.Arrays; import java.util.Map; import java.util.Set; @@ -451,4 +459,67 @@ void shouldAddAndExecuteNamedOperationsFromSubGraphCache() throws OperationExcep .withMessageContaining(namedOpName); } + @Test + void shouldGetWalksAcrossFederatedGraphs() throws OperationException { + // Given + final Graph federatedGraph = FederatedModernTestUtils.setUpSimpleFederatedGraph(FederatedStoreIT.class, new MapStoreProperties()); + + final GetWalks getWalks = new GetWalks.Builder() + .operations(new GetElements.Builder() + .inOutType(SeededGraphFilters.IncludeIncomingOutgoingType.OUTGOING) + .view(new View.Builder().allEdges(true).build()) + .build(), + new GetElements.Builder() + .inOutType(SeededGraphFilters.IncludeIncomingOutgoingType.OUTGOING) + .view(new View.Builder().allEdges(true).build()) + .build()) + .input(new EntitySeed("1")) + .build(); + + // When + OperationChain> getAllElements = new OperationChain.Builder() + .first(getWalks) + .option(FederatedOperationHandler.OPT_GRAPH_IDS, "createdGraph,knowsGraph") + .build(); + + // Then + final Iterable res = federatedGraph.execute(getAllElements, new Context()); + final ArrayList walkList = new ArrayList<>(); + for (final Walk walk : res) { + walkList.add(walk.getVerticesOrdered().stream().map(Object::toString).collect(Collectors.joining(""))); + } + assertThat(walkList).containsExactlyInAnyOrder("143", "145"); + } + + @Test + void shouldGetWalksUsingDefaultGraphIds() throws OperationException { + // Given + final Graph federatedGraph = FederatedModernTestUtils.setUpSimpleFederatedGraph(FederatedStoreIT.class, new MapStoreProperties()); + + final GetWalks getWalks = new GetWalks.Builder() + .operations(new GetElements.Builder() + .inOutType(SeededGraphFilters.IncludeIncomingOutgoingType.OUTGOING) + .view(new View.Builder().allEdges(true).build()) + .build(), + new GetElements.Builder() + .inOutType(SeededGraphFilters.IncludeIncomingOutgoingType.OUTGOING) + .view(new View.Builder().allEdges(true).build()) + .build()) + .input(new EntitySeed("1")) + .build(); + + // When + OperationChain> getAllElements = new OperationChain.Builder() + .first(getWalks) + .build(); + + // Then + final Iterable res = federatedGraph.execute(getAllElements, new Context()); + final ArrayList walkList = new ArrayList<>(); + for (final Walk walk : res) { + walkList.add(walk.getVerticesOrdered().stream().map(Object::toString).collect(Collectors.joining(""))); + } + assertThat(walkList).containsExactlyInAnyOrder("143", "145"); + } + } diff --git a/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/util/FederatedModernTestUtils.java b/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/util/FederatedModernTestUtils.java new file mode 100644 index 00000000000..61d662cf492 --- /dev/null +++ b/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/util/FederatedModernTestUtils.java @@ -0,0 +1,135 @@ +/* + * Copyright 2025 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.gchq.gaffer.federated.simple.util; + +import uk.gov.gchq.gaffer.commonutil.StreamUtil; +import uk.gov.gchq.gaffer.data.element.Edge; +import uk.gov.gchq.gaffer.data.element.Element; +import uk.gov.gchq.gaffer.data.element.Entity; +import uk.gov.gchq.gaffer.federated.simple.FederatedStoreProperties; +import uk.gov.gchq.gaffer.federated.simple.operation.AddGraph; +import uk.gov.gchq.gaffer.federated.simple.operation.handler.FederatedOperationHandler; +import uk.gov.gchq.gaffer.graph.Graph; +import uk.gov.gchq.gaffer.graph.GraphConfig; +import uk.gov.gchq.gaffer.operation.OperationException; +import uk.gov.gchq.gaffer.operation.impl.add.AddElements; +import uk.gov.gchq.gaffer.store.Context; +import uk.gov.gchq.gaffer.store.StoreProperties; +import uk.gov.gchq.gaffer.store.schema.Schema; + +import java.util.ArrayList; +import java.util.List; + +import static uk.gov.gchq.gaffer.federated.simple.FederatedStoreProperties.PROP_DEFAULT_GRAPH_IDS; +import static uk.gov.gchq.gaffer.federated.simple.FederatedStoreProperties.PROP_DEFAULT_MERGE_ELEMENTS; + +public final class FederatedModernTestUtils { + public static final String FEDERATED_GRAPH_ID = "simpleFederatedGraph"; + public static final String CREATED_GRAPH_ID = "createdGraph"; + public static final String KNOWS_GRAPH_ID = "knowsGraph"; + private static final Context CONTEXT = new Context(); + + private FederatedModernTestUtils() { + // utility class + } + + public static Graph setUpSimpleFederatedGraph(Class clazz, StoreProperties properties) + throws OperationException { + FederatedStoreProperties fedProperties = new FederatedStoreProperties(); + fedProperties.set(PROP_DEFAULT_GRAPH_IDS, CREATED_GRAPH_ID + "," + KNOWS_GRAPH_ID); + fedProperties.set(PROP_DEFAULT_MERGE_ELEMENTS, "true"); + final Graph federatedGraph = new Graph.Builder() + .config(new GraphConfig.Builder() + .graphId(FEDERATED_GRAPH_ID) + .build()) + .addStoreProperties(fedProperties) + .build(); + + federatedGraph.execute( + new AddGraph.Builder() + .graphConfig(new GraphConfig(KNOWS_GRAPH_ID)) + .schema(Schema.fromJson(StreamUtil.openStreams(clazz, "/modern/schema"))) + .properties(properties.getProperties()) + .build(), + CONTEXT); + + federatedGraph.execute( + new AddGraph.Builder() + .graphConfig(new GraphConfig(CREATED_GRAPH_ID)) + .schema(Schema.fromJson(StreamUtil.openStreams(clazz, "/modern/schema"))) + .properties(properties.getProperties()) + .build(), + CONTEXT); + + setupKnowsGraph(federatedGraph); + setupCreatedGraph(federatedGraph); + + return federatedGraph; + } + + private static void setupKnowsGraph(final Graph federatedGraph) throws OperationException { + List knowsGraphElements = new ArrayList<>(); + + knowsGraphElements.add(getEntity("1", "person", "marko")); + knowsGraphElements.add(getEntity("4", "person", "josh")); + knowsGraphElements.add(getEntity("2", "person", "vadas")); + knowsGraphElements.add(getEntity("6", "person", "peter")); + knowsGraphElements.add(getEdge("1", "4", "knows")); + knowsGraphElements.add(getEdge("1", "2", "knows")); + + federatedGraph.execute(new AddElements.Builder() + .input(knowsGraphElements.toArray(new Element[0])) + .option(FederatedOperationHandler.OPT_GRAPH_IDS, KNOWS_GRAPH_ID) + .build(), CONTEXT); + } + + private static void setupCreatedGraph(final Graph federatedGraph) throws OperationException { + List createdGraphElements = new ArrayList<>(); + + createdGraphElements.add(getEntity("3", "software", "lop")); + createdGraphElements.add(getEntity("5", "software", "ripple")); + createdGraphElements.add(getEntity("1", "person", "marko")); + createdGraphElements.add(getEntity("4", "person", "josh")); + createdGraphElements.add(getEntity("6", "person", "peter")); + createdGraphElements.add(getEdge("1", "3", "created")); + createdGraphElements.add(getEdge("4", "3", "created")); + createdGraphElements.add(getEdge("6", "3", "created")); + createdGraphElements.add(getEdge("4", "5", "created")); + + federatedGraph.execute(new AddElements.Builder() + .input(createdGraphElements.toArray(new Element[0])) + .option(FederatedOperationHandler.OPT_GRAPH_IDS, CREATED_GRAPH_ID) + .build(), CONTEXT); + } + + private static Entity getEntity(final String vertex, final String group, final String name) { + return new Entity.Builder() + .group(group) + .vertex(vertex) + .property("name", name) + .build(); + } + + private static Edge getEdge(final String source, final String dest, final String group) { + return new Edge.Builder() + .group(group) + .source(source) + .dest(dest) + .directed(true) + .build(); + } +} From ac5f6ac2d09f2c2a17db01054d2c378e660b4a3c Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Wed, 15 Jan 2025 16:30:27 +0000 Subject: [PATCH 08/29] dont reset user on graphstep --- .../gchq/gaffer/tinkerpop/GafferPopGraph.java | 12 ++- .../traversal/step/GafferPopGraphStep.java | 4 +- .../rest/controller/GremlinController.java | 78 ++++++++----------- .../rest/handler/GremlinWebSocketHandler.java | 45 +++++------ 4 files changed, 61 insertions(+), 78 deletions(-) diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java index 29d9f6159c7..7942b3ff556 100755 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java @@ -286,7 +286,7 @@ public GafferPopGraph(final Configuration configuration, final Graph graph) { // Set the graph variables to current config variables = new GafferPopGraphVariables(); - setDefaultVariables(variables); + setDefaultVariables(false); serviceRegistry = new ServiceRegistry(); serviceRegistry.registerService(new GafferPopNamedOperationServiceFactory(this)); @@ -756,12 +756,16 @@ public T execute(final OperationChain opChain) { * Sets the {@link GafferPopGraphVariables} to default values for this * graph * - * @param variables The variables + * @param preserveUser optionally preserve the graph User as it may have been set + * externally */ - public void setDefaultVariables(final GafferPopGraphVariables variables) { + public void setDefaultVariables(final boolean preserveUser) { LOGGER.debug("Resetting graph variables to defaults"); + if (!preserveUser) { + LOGGER.debug("Resetting graph user variable"); + variables.set(GafferPopGraphVariables.USER, defaultUser); + } variables.set(GafferPopGraphVariables.OP_OPTIONS, Collections.unmodifiableMap(opOptions)); - variables.set(GafferPopGraphVariables.USER, defaultUser); variables.set(GafferPopGraphVariables.GET_ELEMENTS_LIMIT, configuration().getInteger(GET_ELEMENTS_LIMIT, DEFAULT_GET_ELEMENTS_LIMIT)); variables.set(GafferPopGraphVariables.HAS_STEP_FILTER_STAGE, diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java index 23aac9a8039..5122f28cbcb 100644 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java @@ -74,8 +74,8 @@ public GafferPopGraphStep(final GraphStep originalGraphStep) { // Save reference to the graph GafferPopGraph graph = (GafferPopGraph) originalGraphStep.getTraversal().getGraph().get(); - // Restore variables to defaults before parsing options - graph.setDefaultVariables((GafferPopGraphVariables) graph.variables()); + // Restore variables to defaults before parsing options, preserve user as may have been set by REST API + graph.setDefaultVariables(true); // Find any options on the traversal Optional optionsStrategy = originalGraphStep.getTraversal().getStrategies().getStrategy(OptionsStrategy.class); diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java index 6be0fd9a842..4dbfdc92411 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java @@ -24,17 +24,14 @@ import org.apache.tinkerpop.gremlin.groovy.engine.GremlinExecutor; import org.apache.tinkerpop.gremlin.jsr223.ConcurrentBindings; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; -import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONVersion; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONXModuleV3; -import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph; import org.json.JSONObject; import org.opencypher.gremlin.server.jsr223.CypherPlugin; import org.opencypher.gremlin.translation.CypherAst; import org.opencypher.gremlin.translation.translator.Translator; -import org.opencypher.v9_0.util.SyntaxException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -66,6 +63,7 @@ import java.util.LinkedList; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -98,23 +96,32 @@ public class GremlinController { public static final String EXPLAIN_GREMLIN_KEY = "gremlin"; private static final Logger LOGGER = LoggerFactory.getLogger(GremlinController.class); - private static final String GENERAL_ERROR_MSG = "Failed to evaluate Gremlin query: "; + private static final String GENERAL_ERROR_MSG = "Gremlin query failed: "; private final ConcurrentBindings bindings = new ConcurrentBindings(); - private final ExecutorService executorService = Context.taskWrapping(Executors.newFixedThreadPool(4)); - private final Long requestTimeout; + private final ExecutorService executorService = Context.taskWrapping(Executors.newFixedThreadPool(1)); + private final GremlinExecutor gremlinExecutor; private final AbstractUserFactory userFactory; - private final Graph graph; + private final GafferPopGraph graph; private final Map> plugins = new HashMap<>(); @Autowired public GremlinController(final GraphTraversalSource g, final AbstractUserFactory userFactory, final Long requestTimeout) { + // Check we actually have a graph instance to use + if (g.getGraph() instanceof GafferPopGraph) { + graph = (GafferPopGraph) g.getGraph(); + } else { + throw new GafferRuntimeException("There is no GafferPop Graph configured"); + } bindings.putIfAbsent("g", g); - graph = g.getGraph(); this.userFactory = userFactory; - this.requestTimeout = requestTimeout; // Add cypher plugin so cypher functions can be used in queries plugins.put(CypherPlugin.class.getName(), new HashMap<>()); + gremlinExecutor = GremlinExecutor.build() + .addPlugins("gremlin-groovy", plugins) + .evaluationTimeout(requestTimeout) + .executorService(executorService) + .globalBindings(bindings).create(); } /** @@ -159,7 +166,7 @@ public ResponseEntity execute( Object result = runGremlinQuery(gremlinQuery).get0(); // Write to output stream for response responseBody = os -> GRAPHSON_V3_WRITER.writeObject(os, result); - } catch (final GafferRuntimeException e) { + } catch (final Exception e) { status = HttpStatus.INTERNAL_SERVER_ERROR; responseBody = os -> os.write( new JSONObject().put("simpleMessage", e.getMessage()).toString().getBytes(StandardCharsets.UTF_8)); @@ -229,7 +236,7 @@ public ResponseEntity cypherExecute( Object result = runGremlinQuery(translation).get0(); // Write to output stream for response responseBody = os -> GRAPHSON_V3_WRITER.writeObject(os, result); - } catch (final GafferRuntimeException | SyntaxException e) { + } catch (final Exception e) { status = HttpStatus.INTERNAL_SERVER_ERROR; responseBody = os -> os.write( new JSONObject().put("simpleMessage", e.getMessage()).toString().getBytes(StandardCharsets.UTF_8)); @@ -286,14 +293,7 @@ public static JSONObject getGafferPopExplanation(final GafferPopGraph graph) { * @param httpHeaders Headers for user auth */ private void preExecuteSetUp(final HttpHeaders httpHeaders) { - // Check we actually have a graph instance to use - GafferPopGraph gafferPopGraph; - if (graph instanceof EmptyGraph) { - throw new GafferRuntimeException("There is no GafferPop Graph configured"); - } else { - gafferPopGraph = (GafferPopGraph) graph; - } - gafferPopGraph.setDefaultVariables((GafferPopGraphVariables) gafferPopGraph.variables()); + graph.setDefaultVariables(false); // Hooks for user auth userFactory.setHttpHeaders(httpHeaders); graph.variables().set(GafferPopGraphVariables.USER, userFactory.createUser()); @@ -306,71 +306,55 @@ private void preExecuteSetUp(final HttpHeaders httpHeaders) { * @return A pair tuple with result and explain in. */ private Tuple2 runGremlinQuery(final String gremlinQuery) { - GafferPopGraph gafferPopGraph = (GafferPopGraph) graph; - // OpenTelemetry hooks Span span = OtelUtil.startSpan( this.getClass().getName(), "Gremlin Request: " + UUID.nameUUIDFromBytes(gremlinQuery.getBytes(StandardCharsets.UTF_8))); span.setAttribute(OtelUtil.GREMLIN_QUERY_ATTRIBUTE, gremlinQuery); - User user = ((GafferPopGraphVariables) gafferPopGraph.variables()).getUser(); - String userId; + User user = ((GafferPopGraphVariables) graph.variables()).getUser(); if (user != null) { - userId = user.getUserId(); + span.setAttribute(OtelUtil.USER_ATTRIBUTE, user.getUserId()); } else { LOGGER.warn("Could not find Gaffer user for OTEL. Using default."); - userId = "unknownGremlinUser"; + span.setAttribute(OtelUtil.USER_ATTRIBUTE, "unknownGremlinUser"); } - span.setAttribute(OtelUtil.USER_ATTRIBUTE, userId); // tuple to hold the result and explain Tuple2 pair = new Tuple2<>(); pair.put1(new JSONObject()); - try (Scope scope = span.makeCurrent(); - GremlinExecutor gremlinExecutor = getGremlinExecutor()) { + try (Scope scope = span.makeCurrent()) { // Execute the query Object result = gremlinExecutor.eval(gremlinQuery).get(); // Store the result and explain for returning pair.put0(result); - pair.put1(getGafferPopExplanation(gafferPopGraph)); + pair.put1(getGafferPopExplanation(graph)); // Provide an debug explanation for the query that just ran span.addEvent("Request complete"); LOGGER.debug("{}", pair.get1()); // Reset the vars - gafferPopGraph.setDefaultVariables((GafferPopGraphVariables) gafferPopGraph.variables()); + graph.setDefaultVariables(false); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); span.setStatus(StatusCode.ERROR, e.getMessage()); span.recordException(e); throw new GafferRuntimeException(GENERAL_ERROR_MSG + e.getMessage(), e); + } catch (final ExecutionException e) { + span.setStatus(StatusCode.ERROR, e.getCause().getMessage()); + span.recordException(e.getCause()); + throw new GafferRuntimeException(GENERAL_ERROR_MSG + e.getCause().getMessage(), e); } catch (final Exception e) { span.setStatus(StatusCode.ERROR, e.getMessage()); span.recordException(e); - throw new GafferRuntimeException(GENERAL_ERROR_MSG + e.getMessage(), e); - } finally { + throw e; + } finally { span.end(); } return pair; } - - /** - * Returns a new gremlin executor. It's the responsibility of the caller to - * ensure it is closed. - * - * @return Gremlin executor. - */ - private GremlinExecutor getGremlinExecutor() { - return GremlinExecutor.build() - .addPlugins("gremlin-groovy", plugins) - .evaluationTimeout(requestTimeout) - .executorService(executorService) - .globalBindings(bindings).create(); - } - } diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java index 63865214d48..a123188a1b3 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java @@ -27,7 +27,6 @@ import org.apache.tinkerpop.gremlin.jsr223.ConcurrentBindings; import org.apache.tinkerpop.gremlin.process.remote.traversal.DefaultRemoteTraverser; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; -import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.util.MessageSerializer; import org.apache.tinkerpop.gremlin.util.Tokens; import org.apache.tinkerpop.gremlin.util.function.FunctionUtils; @@ -48,6 +47,7 @@ import org.springframework.web.socket.handler.BinaryWebSocketHandler; import uk.gov.gchq.gaffer.commonutil.otel.OtelUtil; +import uk.gov.gchq.gaffer.core.exception.GafferRuntimeException; import uk.gov.gchq.gaffer.rest.controller.GremlinController; import uk.gov.gchq.gaffer.rest.factory.spring.AbstractUserFactory; import uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph; @@ -87,11 +87,11 @@ public class GremlinWebSocketHandler extends BinaryWebSocketHandler { new SimpleEntry<>(SerTokens.MIME_JSON, new GraphSONMessageSerializerV3())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - private final ExecutorService executorService = Context.taskWrapping(Executors.newFixedThreadPool(4)); + private final ExecutorService executorService = Context.taskWrapping(Executors.newFixedThreadPool(1)); + private final GremlinExecutor gremlinExecutor; private final ConcurrentBindings bindings = new ConcurrentBindings(); - private final Long requestTimeout; private final AbstractUserFactory userFactory; - private final Graph graph; + private final GafferPopGraph graph; private final Map> plugins = new HashMap<>(); /** @@ -102,12 +102,21 @@ public class GremlinWebSocketHandler extends BinaryWebSocketHandler { * @param requestTimeout The timeout for gremlin requests */ public GremlinWebSocketHandler(final GraphTraversalSource g, final AbstractUserFactory userFactory, final Long requestTimeout) { + // Check we actually have a graph instance to use + if (g.getGraph() instanceof GafferPopGraph) { + graph = (GafferPopGraph) g.getGraph(); + } else { + throw new GafferRuntimeException("There is no GafferPop Graph configured"); + } bindings.putIfAbsent("g", g); - graph = g.getGraph(); this.userFactory = userFactory; - this.requestTimeout = requestTimeout; // Add cypher plugin so cypher functions can be used in queries plugins.put(CypherPlugin.class.getName(), new HashMap<>()); + gremlinExecutor = GremlinExecutor.build() + .addPlugins("gremlin-groovy", plugins) + .evaluationTimeout(requestTimeout) + .executorService(executorService) + .globalBindings(bindings).create(); } @Override @@ -149,8 +158,7 @@ private ResponseMessage handleGremlinRequest(final WebSocketSession session, fin span.setAttribute(OtelUtil.GREMLIN_QUERY_ATTRIBUTE, request.getArgs().get(Tokens.ARGS_GREMLIN).toString()); // Execute the query - try (Scope scope = span.makeCurrent(); - GremlinExecutor gremlinExecutor = getGremlinExecutor()) { + try (Scope scope = span.makeCurrent()) { // Set current headers for potential authorisation then set the user userFactory.setHttpHeaders(session.getHandshakeHeaders()); User user = userFactory.createUser(); @@ -175,10 +183,12 @@ private ResponseMessage handleGremlinRequest(final WebSocketSession session, fin // Provide an debug explanation for the query that just ran span.addEvent("Request complete"); - if (graph instanceof GafferPopGraph) { - JSONObject gafferOperationChain = GremlinController.getGafferPopExplanation((GafferPopGraph) graph); + if (LOGGER.isDebugEnabled()) { + JSONObject gafferOperationChain = GremlinController.getGafferPopExplanation(graph); LOGGER.debug("{}", gafferOperationChain); } + // Reset the vars as request is complete + graph.setDefaultVariables(false); // Build the response responseMessage = ResponseMessage.build(requestId) @@ -242,19 +252,4 @@ private ByteBuf convertToByteBuf(final BinaryMessage message) { } } - /** - * Returns a new gremlin executor. It's the responsibility of the caller to - * ensure it is closed. - * - * @return Gremlin executor. - */ - private GremlinExecutor getGremlinExecutor() { - return GremlinExecutor.build() - .globalBindings(bindings) - .addPlugins("gremlin-groovy", plugins) - .evaluationTimeout(requestTimeout) - .executorService(executorService) - .create(); - } - } From 656c0b025adc806cd14a0c639e8159e69ee740b3 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Wed, 15 Jan 2025 16:30:46 +0000 Subject: [PATCH 09/29] misc otel updates --- .../uk/gov/gchq/gaffer/commonutil/otel/OtelUtil.java | 1 + .../src/main/java/uk/gov/gchq/gaffer/graph/Graph.java | 3 ++- .../store/operation/handler/OperationChainHandler.java | 10 +++++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/core/common-util/src/main/java/uk/gov/gchq/gaffer/commonutil/otel/OtelUtil.java b/core/common-util/src/main/java/uk/gov/gchq/gaffer/commonutil/otel/OtelUtil.java index 7cf3e93ab11..ae23398457a 100644 --- a/core/common-util/src/main/java/uk/gov/gchq/gaffer/commonutil/otel/OtelUtil.java +++ b/core/common-util/src/main/java/uk/gov/gchq/gaffer/commonutil/otel/OtelUtil.java @@ -25,6 +25,7 @@ public final class OtelUtil { public static final String JOB_ID_ATTRIBUTE = "gaffer.jobId"; public static final String GRAPH_ID_ATTRIBUTE = "gaffer.graphId"; public static final String VIEW_ATTRIBUTE = "gaffer.view"; + public static final String OP_OPTIONS_ATTRIBUTE = "gaffer.operation.options"; public static final String GREMLIN_QUERY_ATTRIBUTE = "gaffer.gremlin.query"; private static boolean openTelemetryActive = false; diff --git a/core/graph/src/main/java/uk/gov/gchq/gaffer/graph/Graph.java b/core/graph/src/main/java/uk/gov/gchq/gaffer/graph/Graph.java index 02d616b425a..e50f06e494b 100644 --- a/core/graph/src/main/java/uk/gov/gchq/gaffer/graph/Graph.java +++ b/core/graph/src/main/java/uk/gov/gchq/gaffer/graph/Graph.java @@ -320,10 +320,11 @@ private GraphResult _execute(final StoreExecuter storeExecuter, final // OpenTelemetry hooks Span span = OtelUtil.startSpan( this.getClass().getName(), - "Graph Request: " + clonedOpChain.toOverviewString()); + "Graph Request: " + clonedContext.getJobId()); span.setAttribute(OtelUtil.GRAPH_ID_ATTRIBUTE, getGraphId()); span.setAttribute(OtelUtil.JOB_ID_ATTRIBUTE, clonedContext.getJobId()); span.setAttribute(OtelUtil.USER_ATTRIBUTE, clonedContext.getUser().getUserId()); + span.setAttribute(OtelUtil.OP_OPTIONS_ATTRIBUTE, clonedOpChain.getOptions().toString()); O result = null; // Sets the span to current so parent child spans are auto linked diff --git a/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/handler/OperationChainHandler.java b/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/handler/OperationChainHandler.java index 357fa583a42..1ad1ee943a2 100644 --- a/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/handler/OperationChainHandler.java +++ b/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/handler/OperationChainHandler.java @@ -58,8 +58,16 @@ public OUT doOperation(final OperationChain operationChain, final Context c // OpenTelemetry hooks Span span = OtelUtil.startSpan(this.getClass().getName(), op.getClass().getName()); span.setAttribute(OtelUtil.JOB_ID_ATTRIBUTE, context.getJobId()); + span.setAttribute(OtelUtil.OP_OPTIONS_ATTRIBUTE, (op.getOptions() != null) ? op.getOptions().toString() : "[]"); + // Extract the view if (op instanceof OperationView && ((OperationView) op).getView() != null) { - span.setAttribute(OtelUtil.VIEW_ATTRIBUTE, ((OperationView) op).getView().toString()); + String strView = ((OperationView) op).getView().toString(); + // Truncate the view if its too long + if (strView.length() > 2048) { + span.setAttribute(OtelUtil.VIEW_ATTRIBUTE, strView.substring(0, 2048)); + } else { + span.setAttribute(OtelUtil.VIEW_ATTRIBUTE, strView); + } } // Sets the span to current so parent child spans are auto linked From a8c955df9a269239733e6d379eeab10b13d12469 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Wed, 15 Jan 2025 18:37:21 +0000 Subject: [PATCH 10/29] multi instances when on rest api --- .../gchq/gaffer/tinkerpop/GafferPopGraph.java | 22 ++++-- .../traversal/step/GafferPopGraphStep.java | 3 - .../gaffer/tinkerpop/GafferPopGraphIT.java | 73 ++++++++++++++---- .../gaffer/rest/config/GremlinConfig.java | 9 +-- .../rest/config/GremlinWebSocketConfig.java | 9 ++- .../rest/controller/GremlinController.java | 75 +++++++++---------- .../rest/handler/GremlinWebSocketHandler.java | 51 +++++++------ 7 files changed, 145 insertions(+), 97 deletions(-) diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java index 7942b3ff556..3703d2d1fae 100755 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java @@ -286,7 +286,7 @@ public GafferPopGraph(final Configuration configuration, final Graph graph) { // Set the graph variables to current config variables = new GafferPopGraphVariables(); - setDefaultVariables(false); + setDefaultVariables(); serviceRegistry = new ServiceRegistry(); serviceRegistry.registerService(new GafferPopNamedOperationServiceFactory(this)); @@ -299,6 +299,16 @@ public GafferPopGraph(final Configuration configuration, final Graph graph) { GlobalCache.registerStrategies(this.getClass(), traversalStrategies); } + /** + * Return a new instance of the graph usually so a different set + * of graph variables can be used for a query. + * + * @return Identical instance this graph. + */ + public GafferPopGraph newInstance() { + return new GafferPopGraph(configuration, graph); + } + private static Graph createGraph(final Configuration configuration) { final String graphId = configuration.getString(GRAPH_ID); if (null == graphId) { @@ -745,6 +755,7 @@ public T execute(final OperationChain opChain) { try { LOGGER.info("GafferPop operation chain called: {}", opChain.toOverviewString()); + LOGGER.info("USER IS: " + variables.getUser().getUserId()); return graph.execute(opChain, variables.getUser()); } catch (final Exception e) { LOGGER.error("Operation chain failed: {}", e.getMessage()); @@ -756,15 +767,10 @@ public T execute(final OperationChain opChain) { * Sets the {@link GafferPopGraphVariables} to default values for this * graph * - * @param preserveUser optionally preserve the graph User as it may have been set - * externally */ - public void setDefaultVariables(final boolean preserveUser) { + public void setDefaultVariables() { LOGGER.debug("Resetting graph variables to defaults"); - if (!preserveUser) { - LOGGER.debug("Resetting graph user variable"); - variables.set(GafferPopGraphVariables.USER, defaultUser); - } + variables.set(GafferPopGraphVariables.USER, defaultUser); variables.set(GafferPopGraphVariables.OP_OPTIONS, Collections.unmodifiableMap(opOptions)); variables.set(GafferPopGraphVariables.GET_ELEMENTS_LIMIT, configuration().getInteger(GET_ELEMENTS_LIMIT, DEFAULT_GET_ELEMENTS_LIMIT)); diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java index 5122f28cbcb..9cb703b9023 100644 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java @@ -74,9 +74,6 @@ public GafferPopGraphStep(final GraphStep originalGraphStep) { // Save reference to the graph GafferPopGraph graph = (GafferPopGraph) originalGraphStep.getTraversal().getGraph().get(); - // Restore variables to defaults before parsing options, preserve user as may have been set by REST API - graph.setDefaultVariables(true); - // Find any options on the traversal Optional optionsStrategy = originalGraphStep.getTraversal().getStrategies().getStrategy(OptionsStrategy.class); if (optionsStrategy.isPresent()) { diff --git a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java index 83dcdcb2841..f2147732672 100644 --- a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java +++ b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java @@ -19,6 +19,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Element; import org.apache.tinkerpop.gremlin.structure.T; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.junit.jupiter.api.BeforeAll; @@ -26,12 +27,16 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import io.opentelemetry.context.Context; import uk.gov.gchq.gaffer.cache.CacheServiceLoader; import uk.gov.gchq.gaffer.tinkerpop.util.GafferPopTestUtil.StoreType; import uk.gov.gchq.gaffer.tinkerpop.util.modern.GafferPopModernTestUtils; +import uk.gov.gchq.gaffer.user.User; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -77,7 +82,7 @@ void shouldGetAllVertices(String graph, GraphTraversalSource g) { final List result = g.V().toList(); assertThat(result) - .extracting(r -> r.id()) + .extracting(Element::id) .containsExactlyInAnyOrder( MARKO.getId(), VADAS.getId(), @@ -94,7 +99,7 @@ void shouldTruncateGetAllVertices(String graph, GraphTraversalSource g) { assertThat(result) .hasSize(2) - .extracting(r -> r.id()) + .extracting(Element::id) .containsAnyOf( MARKO.getId(), VADAS.getId(), @@ -110,7 +115,7 @@ void shouldGetVerticesById(String graph, GraphTraversalSource g) { final List result = g.V(MARKO.getId(), RIPPLE.getId()).toList(); assertThat(result) - .extracting(r -> r.id()) + .extracting(Element::id) .containsExactlyInAnyOrder(MARKO.getId(), RIPPLE.getId()); } @@ -132,7 +137,7 @@ void shouldGetVerticesByLabel(String graph, GraphTraversalSource g) { final List result = g.V().hasLabel(PERSON).toList(); assertThat(result) - .extracting(r -> r.id()) + .extracting(Element::id) .containsExactlyInAnyOrder( MARKO.getId(), VADAS.getId(), @@ -147,7 +152,7 @@ void shouldGetAllOutgoingEdgesFromVertex(String graph, GraphTraversalSource g) { final List result = g.V(MARKO.getId()).outE().toList(); assertThat(result) - .extracting(r -> r.id()) + .extracting(Element::id) .containsExactlyInAnyOrder( MARKO.knows(VADAS), MARKO.knows(JOSH), @@ -228,7 +233,7 @@ void shouldGetAllEdges(String graph, GraphTraversalSource g) { final List result = g.E().toList(); assertThat(result) - .extracting(r -> r.id()) + .extracting(Element::id) .containsExactlyInAnyOrder( MARKO.knows(JOSH), MARKO.knows(VADAS), @@ -245,7 +250,7 @@ void shouldTruncateGetAllEdges(String graph, GraphTraversalSource g) { assertThat(result) .hasSize(2) - .extracting(r -> r.id()) + .extracting(Element::id) .containsAnyOf( MARKO.knows(JOSH), MARKO.knows(VADAS), @@ -271,7 +276,7 @@ void shouldGetEdgeById(String graph, GraphTraversalSource g) { assertThat(result) .withFailMessage("(%s) Edge ID: %s returned %s", graph, id, result) - .extracting(r -> r.id()) + .extracting(Element::id) .containsExactlyInAnyOrder(MARKO.knows(VADAS)); }); } @@ -295,7 +300,7 @@ void shouldAddE(String graph, GraphTraversalSource g) { final List result = g.E().toList(); assertThat(result) - .extracting(r -> r.id()) + .extracting(Element::id) .contains(VADAS.knows(PETER)); reset(); } @@ -311,7 +316,7 @@ void shouldSeedWithVertexOnlyEdge(String graph, GraphTraversalSource g) { // Must enable orphaned vertices for this traversal List result = g.with(GafferPopGraphVariables.INCLUDE_ORPHANED_VERTICES, "true").V(VERTEX_ONLY_ID_STRING).toList(); assertThat(result) - .extracting(r -> r.id()) + .extracting(Element::id) .contains(VERTEX_ONLY_ID_STRING); reset(); } @@ -329,27 +334,65 @@ void shouldTraverseEdgeWithVertexOnlyEdge(String graph, GraphTraversalSource g) List result = gWithOption.V(VERTEX_ONLY_ID_STRING).inE().inV().toList(); assertThat(result) - .extracting(r -> r.id()) + .extracting(Element::id) .contains(VERTEX_ONLY_ID_STRING); List result2 = gWithOption.V(VERTEX_ONLY_ID_STRING).inE().outV().toList(); assertThat(result2) - .extracting(r -> r.id()) + .extracting(Element::id) .contains("8"); List result3 = gWithOption.V("8").outE().inV().toList(); assertThat(result3) - .extracting(r -> r.id()) + .extracting(Element::id) .contains(VERTEX_ONLY_ID_STRING); List result4 = gWithOption.V("8").outE().outV().toList(); assertThat(result4) - .extracting(r -> r.id()) + .extracting(Element::id) .contains("8"); List resultLabel = gWithOption.V("8").out("knows").toList(); assertThat(resultLabel) - .extracting(r -> r.id()) + .extracting(Element::id) .containsOnly(VERTEX_ONLY_ID_STRING); reset(); } + @ParameterizedTest(name = TEST_NAME_FORMAT) + @MethodSource("provideTraversals") + void shouldPreserveExternallySetUser(String graph, GraphTraversalSource g) { + ExecutorService executorService = Context.taskWrapping(Executors.newFixedThreadPool(2)); + + User testUser = new User("shouldPreserveExternallySetUser"); + User testUser2 = new User("shouldPreserveExternallySetUser2"); + + executorService.submit(() -> { + GraphTraversalSource g2 = ((GafferPopGraph) g.getGraph()).newInstance().traversal(); + g2.getGraph().variables().set(GafferPopGraphVariables.USER, testUser); + g2.V().toList(); + assertThat(((GafferPopGraphVariables) g.getGraph().variables()).getUser()).isEqualTo(testUser); + }); + + executorService.submit(() -> { + GraphTraversalSource g2 = ((GafferPopGraph) g.getGraph()).newInstance().traversal(); + g2.getGraph().variables().set(GafferPopGraphVariables.USER, testUser2); + g2.V().toList(); + assertThat(((GafferPopGraphVariables) g.getGraph().variables()).getUser()).isEqualTo(testUser2); + }); + + // Set the user + // g.getGraph().variables().set(GafferPopGraphVariables.USER, testUser); + + // // Run a Query + // g.V().toList(); + + // Ensure user is still set + //assertThat(((GafferPopGraphVariables) g.getGraph().variables()).getUser()).isEqualTo(testUser); + + // Reset vars + ((GafferPopGraph) g.getGraph()).setDefaultVariables(); + assertThat(((GafferPopGraphVariables) g.getGraph().variables()).getUser()).isNotEqualTo(testUser); + reset(); + } + + void reset() { // reset cache for federation CacheServiceLoader.shutdown(); diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinConfig.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinConfig.java index d9d09a36ce9..432d5888475 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinConfig.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinConfig.java @@ -19,8 +19,6 @@ import org.apache.commons.configuration2.PropertiesConfiguration; import org.apache.commons.configuration2.builder.fluent.Configurations; import org.apache.commons.configuration2.ex.ConfigurationException; -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; -import org.apache.tinkerpop.gremlin.structure.Graph; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Bean; @@ -49,10 +47,9 @@ public class GremlinConfig { private static final String REQUEST_TIMEOUT_KEY = "gaffer.rest.timeout"; @Bean - public GraphTraversalSource graphTraversalSource(final GraphFactory graphFactory) { - // Obtain the graph traversal - Graph graph = GafferPopGraph.open(findPropertiesFile(graphFactory), graphFactory.getGraph()); - return graph.traversal(); + public GafferPopGraph gafferPopGraph(final GraphFactory graphFactory) { + // Obtain the graph + return GafferPopGraph.open(findPropertiesFile(graphFactory), graphFactory.getGraph()); } @Bean diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinWebSocketConfig.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinWebSocketConfig.java index 8877af1bb60..53600256ea7 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinWebSocketConfig.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinWebSocketConfig.java @@ -25,25 +25,26 @@ import uk.gov.gchq.gaffer.rest.factory.spring.AbstractUserFactory; import uk.gov.gchq.gaffer.rest.handler.GremlinWebSocketHandler; +import uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph; @Configuration @EnableWebSocket public class GremlinWebSocketConfig implements WebSocketConfigurer { - private final GraphTraversalSource g; + private final GafferPopGraph graph; private final AbstractUserFactory userFactory; private final Long requestTimeout; @Autowired - public GremlinWebSocketConfig(final GraphTraversalSource g, final AbstractUserFactory userFactory, final Long requestTimeout) { - this.g = g; + public GremlinWebSocketConfig(final GafferPopGraph graph, final AbstractUserFactory userFactory, final Long requestTimeout) { + this.graph = graph; this.userFactory = userFactory; this.requestTimeout = requestTimeout; } @Override public void registerWebSocketHandlers(final WebSocketHandlerRegistry registry) { - registry.addHandler(new GremlinWebSocketHandler(g, userFactory, requestTimeout), "/gremlin"); + registry.addHandler(new GremlinWebSocketHandler(graph, userFactory, requestTimeout), "/gremlin"); } } diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java index 4dbfdc92411..720af5468d8 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java @@ -98,30 +98,19 @@ public class GremlinController { private static final Logger LOGGER = LoggerFactory.getLogger(GremlinController.class); private static final String GENERAL_ERROR_MSG = "Gremlin query failed: "; - private final ConcurrentBindings bindings = new ConcurrentBindings(); - private final ExecutorService executorService = Context.taskWrapping(Executors.newFixedThreadPool(1)); - private final GremlinExecutor gremlinExecutor; + private final ExecutorService executorService = Context.taskWrapping(Executors.newFixedThreadPool(8)); private final AbstractUserFactory userFactory; + private final Long requestTimeout; private final GafferPopGraph graph; private final Map> plugins = new HashMap<>(); @Autowired - public GremlinController(final GraphTraversalSource g, final AbstractUserFactory userFactory, final Long requestTimeout) { - // Check we actually have a graph instance to use - if (g.getGraph() instanceof GafferPopGraph) { - graph = (GafferPopGraph) g.getGraph(); - } else { - throw new GafferRuntimeException("There is no GafferPop Graph configured"); - } - bindings.putIfAbsent("g", g); + public GremlinController(final GafferPopGraph graph, final AbstractUserFactory userFactory, final Long requestTimeout) { + this.graph = graph; this.userFactory = userFactory; + this.requestTimeout = requestTimeout; // Add cypher plugin so cypher functions can be used in queries plugins.put(CypherPlugin.class.getName(), new HashMap<>()); - gremlinExecutor = GremlinExecutor.build() - .addPlugins("gremlin-groovy", plugins) - .evaluationTimeout(requestTimeout) - .executorService(executorService) - .globalBindings(bindings).create(); } /** @@ -136,8 +125,7 @@ public GremlinController(final GraphTraversalSource g, final AbstractUserFactory summary = "Explain a Gremlin Query", description = "Runs a Gremlin query and outputs an explanation of what Gaffer operations were executed on the graph") public String explain(@RequestHeader final HttpHeaders httpHeaders, @RequestBody final String gremlinQuery) { - preExecuteSetUp(httpHeaders); - return runGremlinQuery(gremlinQuery).get1().toString(); + return runGremlinQuery(gremlinQuery, httpHeaders).get1().toString(); } /** @@ -157,13 +145,11 @@ public String explain(@RequestHeader final HttpHeaders httpHeaders, @RequestBody public ResponseEntity execute( @RequestHeader final HttpHeaders httpHeaders, @RequestBody final String gremlinQuery) throws IOException { - // Set up - preExecuteSetUp(httpHeaders); HttpStatus status = HttpStatus.OK; StreamingResponseBody responseBody; try { - Object result = runGremlinQuery(gremlinQuery).get0(); + Object result = runGremlinQuery(gremlinQuery, httpHeaders).get0(); // Write to output stream for response responseBody = os -> GRAPHSON_V3_WRITER.writeObject(os, result); } catch (final Exception e) { @@ -197,7 +183,7 @@ public String cypherExplain(@RequestHeader final HttpHeaders httpHeaders, @Reque final String translation = ast.buildTranslation( Translator.builder().gremlinGroovy().enableCypherExtensions().build()) + ".toList()"; - JSONObject response = runGremlinQuery(translation).get1(); + JSONObject response = runGremlinQuery(translation, httpHeaders).get1(); response.put(EXPLAIN_GREMLIN_KEY, translation); return response.toString(); } @@ -220,9 +206,6 @@ public String cypherExplain(@RequestHeader final HttpHeaders httpHeaders, @Reque public ResponseEntity cypherExecute( @RequestHeader final HttpHeaders httpHeaders, @RequestBody final String cypherQuery) throws IOException { - // Set up - preExecuteSetUp(httpHeaders); - HttpStatus status = HttpStatus.OK; StreamingResponseBody responseBody; @@ -233,7 +216,7 @@ public ResponseEntity cypherExecute( final String translation = ast.buildTranslation( Translator.builder().gremlinGroovy().enableCypherExtensions().build()) + ".toList()"; // Run Query - Object result = runGremlinQuery(translation).get0(); + Object result = runGremlinQuery(translation, httpHeaders).get0(); // Write to output stream for response responseBody = os -> GRAPHSON_V3_WRITER.writeObject(os, result); } catch (final Exception e) { @@ -290,13 +273,19 @@ public static JSONObject getGafferPopExplanation(final GafferPopGraph graph) { * Do some basic pre execute set up so the graph is ready for the gremlin * request to be executed. * - * @param httpHeaders Headers for user auth + * @param GafferPopGraph The graph structure to use */ - private void preExecuteSetUp(final HttpHeaders httpHeaders) { - graph.setDefaultVariables(false); - // Hooks for user auth - userFactory.setHttpHeaders(httpHeaders); - graph.variables().set(GafferPopGraphVariables.USER, userFactory.createUser()); + private GremlinExecutor setUpExecutor(final GafferPopGraph graphInstance) { + final ConcurrentBindings bindings = new ConcurrentBindings(); + final GraphTraversalSource g = graphInstance.traversal(); + + // Set up the executor + bindings.putIfAbsent("g", g); + return GremlinExecutor.build() + .addPlugins("gremlin-groovy", plugins) + .evaluationTimeout(requestTimeout) + .executorService(executorService) + .globalBindings(bindings).create(); } /** @@ -305,13 +294,21 @@ private void preExecuteSetUp(final HttpHeaders httpHeaders) { * @param gremlinQuery The Gremlin groovy query. * @return A pair tuple with result and explain in. */ - private Tuple2 runGremlinQuery(final String gremlinQuery) { + private Tuple2 runGremlinQuery(final String gremlinQuery, final HttpHeaders httpHeaders) { + // We can't reuse the existing graph instance as we need to set variables + // specific to this query only. + final GafferPopGraph graphInstance = graph.newInstance(); + + // Hooks for user auth + userFactory.setHttpHeaders(httpHeaders); + User user = userFactory.createUser(); + graph.variables().set(GafferPopGraphVariables.USER, user); + // OpenTelemetry hooks Span span = OtelUtil.startSpan( this.getClass().getName(), "Gremlin Request: " + UUID.nameUUIDFromBytes(gremlinQuery.getBytes(StandardCharsets.UTF_8))); span.setAttribute(OtelUtil.GREMLIN_QUERY_ATTRIBUTE, gremlinQuery); - User user = ((GafferPopGraphVariables) graph.variables()).getUser(); if (user != null) { span.setAttribute(OtelUtil.USER_ATTRIBUTE, user.getUserId()); } else { @@ -323,7 +320,8 @@ private Tuple2 runGremlinQuery(final String gremlinQuery) { Tuple2 pair = new Tuple2<>(); pair.put1(new JSONObject()); - try (Scope scope = span.makeCurrent()) { + try (Scope scope = span.makeCurrent(); + GremlinExecutor gremlinExecutor = setUpExecutor(graphInstance)) { // Execute the query Object result = gremlinExecutor.eval(gremlinQuery).get(); @@ -334,10 +332,6 @@ private Tuple2 runGremlinQuery(final String gremlinQuery) { // Provide an debug explanation for the query that just ran span.addEvent("Request complete"); LOGGER.debug("{}", pair.get1()); - - // Reset the vars - graph.setDefaultVariables(false); - } catch (final InterruptedException e) { Thread.currentThread().interrupt(); span.setStatus(StatusCode.ERROR, e.getMessage()); @@ -350,8 +344,9 @@ private Tuple2 runGremlinQuery(final String gremlinQuery) { } catch (final Exception e) { span.setStatus(StatusCode.ERROR, e.getMessage()); span.recordException(e); - throw e; + throw new GafferRuntimeException(GENERAL_ERROR_MSG + e.getMessage(), e); } finally { + graphInstance.setDefaultVariables(); span.end(); } diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java index a123188a1b3..063c8100cfd 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java @@ -47,7 +47,6 @@ import org.springframework.web.socket.handler.BinaryWebSocketHandler; import uk.gov.gchq.gaffer.commonutil.otel.OtelUtil; -import uk.gov.gchq.gaffer.core.exception.GafferRuntimeException; import uk.gov.gchq.gaffer.rest.controller.GremlinController; import uk.gov.gchq.gaffer.rest.factory.spring.AbstractUserFactory; import uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph; @@ -87,10 +86,9 @@ public class GremlinWebSocketHandler extends BinaryWebSocketHandler { new SimpleEntry<>(SerTokens.MIME_JSON, new GraphSONMessageSerializerV3())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - private final ExecutorService executorService = Context.taskWrapping(Executors.newFixedThreadPool(1)); - private final GremlinExecutor gremlinExecutor; - private final ConcurrentBindings bindings = new ConcurrentBindings(); + private final ExecutorService executorService = Context.taskWrapping(Executors.newFixedThreadPool(8)); private final AbstractUserFactory userFactory; + private final Long requestTimeout; private final GafferPopGraph graph; private final Map> plugins = new HashMap<>(); @@ -101,22 +99,12 @@ public class GremlinWebSocketHandler extends BinaryWebSocketHandler { * @param userFactory The user factory * @param requestTimeout The timeout for gremlin requests */ - public GremlinWebSocketHandler(final GraphTraversalSource g, final AbstractUserFactory userFactory, final Long requestTimeout) { - // Check we actually have a graph instance to use - if (g.getGraph() instanceof GafferPopGraph) { - graph = (GafferPopGraph) g.getGraph(); - } else { - throw new GafferRuntimeException("There is no GafferPop Graph configured"); - } - bindings.putIfAbsent("g", g); + public GremlinWebSocketHandler(final GafferPopGraph graph, final AbstractUserFactory userFactory, final Long requestTimeout) { + this.graph = graph; this.userFactory = userFactory; + this.requestTimeout = requestTimeout; // Add cypher plugin so cypher functions can be used in queries plugins.put(CypherPlugin.class.getName(), new HashMap<>()); - gremlinExecutor = GremlinExecutor.build() - .addPlugins("gremlin-groovy", plugins) - .evaluationTimeout(requestTimeout) - .executorService(executorService) - .globalBindings(bindings).create(); } @Override @@ -139,6 +127,25 @@ protected void handleBinaryMessage(final WebSocketSession session, final BinaryM sendBinaryResponse(session, serialiser, handleGremlinRequest(session, request)); } + /** + * Do some basic pre execute set up so the graph is ready for the gremlin + * request to be executed. + * + * @param GafferPopGraph The graph structure to use + */ + private GremlinExecutor setUpExecutor(final GafferPopGraph graphInstance) { + final ConcurrentBindings bindings = new ConcurrentBindings(); + final GraphTraversalSource g = graphInstance.traversal(); + + // Set up the executor + bindings.putIfAbsent("g", g); + return GremlinExecutor.build() + .addPlugins("gremlin-groovy", plugins) + .evaluationTimeout(requestTimeout) + .executorService(executorService) + .globalBindings(bindings).create(); + } + /** * Extracts the relevant information from a {@link RequestMessage} and validates * the Gremlin query requested before executing on the current graph. Formulates @@ -152,13 +159,17 @@ private ResponseMessage handleGremlinRequest(final WebSocketSession session, fin final UUID requestId = request.getRequestId(); ResponseMessage responseMessage; LOGGER.info("QUERY IS: {} ", request.getArgs().get(Tokens.ARGS_GREMLIN)); + // We can't reuse the existing graph instance as we need to set variables + // specific to this query only. + final GafferPopGraph graphInstance = graph.newInstance(); // OpenTelemetry hooks Span span = OtelUtil.startSpan(this.getClass().getName(), "Gremlin Request: " + requestId.toString()); span.setAttribute(OtelUtil.GREMLIN_QUERY_ATTRIBUTE, request.getArgs().get(Tokens.ARGS_GREMLIN).toString()); // Execute the query - try (Scope scope = span.makeCurrent()) { + try (Scope scope = span.makeCurrent(); + GremlinExecutor gremlinExecutor = setUpExecutor(graph)) { // Set current headers for potential authorisation then set the user userFactory.setHttpHeaders(session.getHandshakeHeaders()); User user = userFactory.createUser(); @@ -187,9 +198,6 @@ private ResponseMessage handleGremlinRequest(final WebSocketSession session, fin JSONObject gafferOperationChain = GremlinController.getGafferPopExplanation(graph); LOGGER.debug("{}", gafferOperationChain); } - // Reset the vars as request is complete - graph.setDefaultVariables(false); - // Build the response responseMessage = ResponseMessage.build(requestId) .code(ResponseStatusCode.SUCCESS) @@ -209,6 +217,7 @@ private ResponseMessage handleGremlinRequest(final WebSocketSession session, fin span.setStatus(StatusCode.ERROR, e.getMessage()); span.recordException(e); } finally { + graphInstance.setDefaultVariables(); span.end(); } From 1f0d16ec52e8db3f4641988e89bda6394df0162a Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Thu, 16 Jan 2025 11:43:19 +0000 Subject: [PATCH 11/29] tweaks so vars are still reset --- .../gchq/gaffer/tinkerpop/GafferPopGraph.java | 128 ++++++++++-------- .../traversal/step/GafferPopGraphStep.java | 2 + .../gaffer/tinkerpop/GafferPopGraphIT.java | 41 ------ 3 files changed, 72 insertions(+), 99 deletions(-) diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java index 3703d2d1fae..350502bd48d 100755 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java @@ -457,7 +457,7 @@ public Iterator vertices(final Object... vertexIds) { variables.getElementsLimit()); } - // Translate results to Gafferpop elements + // Translate results to GafferPop elements final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); final Set translatedResults = StreamSupport.stream(result.spliterator(), false) .map(generator::_apply) @@ -677,60 +677,6 @@ public Iterator edgesWithView(final Object id, final Direction direction, return edgesWithView(Collections.singletonList(id), direction, view); } - /** - * This performs a GetElements operation filtering edges by direction and view. - * - * @param ids vertex IDs or edge IDs to be queried for. - * Supports input as a {@link Vertex}, {@link Edge}, List of Edge IDs or individual Vertex IDs. - * @param direction {@link Direction} of edges to return. - * @param view Gaffer {@link View} to filter edges by - * @return iterator of {@link GafferPopEdge}s. - * @see #edges(Object...) - */ - public Iterator edgesWithView(final Iterable ids, final Direction direction, final View view) { - return edgesWithSeedsAndView(getElementSeeds(ids), direction, view); - } - - @Override - public C compute(final Class graphComputerClass) throws IllegalArgumentException { - throw Exceptions.graphComputerNotSupported(); - } - - @Override - public GraphComputer compute() throws IllegalArgumentException { - throw Exceptions.graphComputerNotSupported(); - } - - @Override - public Transaction tx() { - throw Exceptions.transactionsNotSupported(); - } - - @Override - public Variables variables() { - return this.variables; - } - - @Override - public Configuration configuration() { - return configuration; - } - - @Override - public void close() throws Exception { - serviceRegistry.close(); - } - - @Override - public ServiceRegistry getServiceRegistry() { - return serviceRegistry; - } - - @Override - public Features features() { - return features; - } - public T execute(final OperationChain opChain) { // Set options at opChain level opChain.setOptions(variables.getOperationOptions()); @@ -765,12 +711,16 @@ public T execute(final OperationChain opChain) { /** * Sets the {@link GafferPopGraphVariables} to default values for this - * graph + * graph optionally preserving the previous user. * + * @param preserveUser keep the current set user. */ - public void setDefaultVariables() { + public void setDefaultVariables(final boolean preserveUser) { LOGGER.debug("Resetting graph variables to defaults"); - variables.set(GafferPopGraphVariables.USER, defaultUser); + if (!preserveUser) { + LOGGER.debug("Resetting graph user to default"); + variables.set(GafferPopGraphVariables.USER, defaultUser); + } variables.set(GafferPopGraphVariables.OP_OPTIONS, Collections.unmodifiableMap(opOptions)); variables.set(GafferPopGraphVariables.GET_ELEMENTS_LIMIT, configuration().getInteger(GET_ELEMENTS_LIMIT, DEFAULT_GET_ELEMENTS_LIMIT)); @@ -781,6 +731,14 @@ public void setDefaultVariables() { variables.set(GafferPopGraphVariables.LAST_OPERATION_CHAIN, new OperationChain()); } + /** + * Sets the {@link GafferPopGraphVariables} to default values for this + * graph. + */ + public void setDefaultVariables() { + setDefaultVariables(false); + } + /** * Get the underlying Gaffer graph this GafferPop graph is connected to. * @@ -790,6 +748,60 @@ public Graph getGafferGraph() { return graph; } + /** + * This performs a GetElements operation filtering edges by direction and view. + * + * @param ids vertex IDs or edge IDs to be queried for. + * Supports input as a {@link Vertex}, {@link Edge}, List of Edge IDs or individual Vertex IDs. + * @param direction {@link Direction} of edges to return. + * @param view Gaffer {@link View} to filter edges by + * @return iterator of {@link GafferPopEdge}s. + * @see #edges(Object...) + */ + public Iterator edgesWithView(final Iterable ids, final Direction direction, final View view) { + return edgesWithSeedsAndView(getElementSeeds(ids), direction, view); + } + + @Override + public C compute(final Class graphComputerClass) throws IllegalArgumentException { + throw Exceptions.graphComputerNotSupported(); + } + + @Override + public GraphComputer compute() throws IllegalArgumentException { + throw Exceptions.graphComputerNotSupported(); + } + + @Override + public Transaction tx() { + throw Exceptions.transactionsNotSupported(); + } + + @Override + public Variables variables() { + return this.variables; + } + + @Override + public Configuration configuration() { + return configuration; + } + + @Override + public void close() throws Exception { + serviceRegistry.close(); + } + + @Override + public ServiceRegistry getServiceRegistry() { + return serviceRegistry; + } + + @Override + public Features features() { + return features; + } + private Iterator verticesWithSeedsAndView(final List seeds, final View view) { final boolean getAll = null == seeds || seeds.isEmpty(); final LinkedList idVertices = new LinkedList<>(); diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java index 9cb703b9023..188eabe55cd 100644 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java @@ -73,6 +73,8 @@ public GafferPopGraphStep(final GraphStep originalGraphStep) { // Save reference to the graph GafferPopGraph graph = (GafferPopGraph) originalGraphStep.getTraversal().getGraph().get(); + // Reset vars on the graph but preserving the user in case it was set externally + graph.setDefaultVariables(true); // Find any options on the traversal Optional optionsStrategy = originalGraphStep.getTraversal().getStrategies().getStrategy(OptionsStrategy.class); diff --git a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java index f2147732672..dc63bf686e8 100644 --- a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java +++ b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java @@ -27,16 +27,12 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import io.opentelemetry.context.Context; import uk.gov.gchq.gaffer.cache.CacheServiceLoader; import uk.gov.gchq.gaffer.tinkerpop.util.GafferPopTestUtil.StoreType; import uk.gov.gchq.gaffer.tinkerpop.util.modern.GafferPopModernTestUtils; -import uk.gov.gchq.gaffer.user.User; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -355,43 +351,6 @@ void shouldTraverseEdgeWithVertexOnlyEdge(String graph, GraphTraversalSource g) reset(); } - @ParameterizedTest(name = TEST_NAME_FORMAT) - @MethodSource("provideTraversals") - void shouldPreserveExternallySetUser(String graph, GraphTraversalSource g) { - ExecutorService executorService = Context.taskWrapping(Executors.newFixedThreadPool(2)); - - User testUser = new User("shouldPreserveExternallySetUser"); - User testUser2 = new User("shouldPreserveExternallySetUser2"); - - executorService.submit(() -> { - GraphTraversalSource g2 = ((GafferPopGraph) g.getGraph()).newInstance().traversal(); - g2.getGraph().variables().set(GafferPopGraphVariables.USER, testUser); - g2.V().toList(); - assertThat(((GafferPopGraphVariables) g.getGraph().variables()).getUser()).isEqualTo(testUser); - }); - - executorService.submit(() -> { - GraphTraversalSource g2 = ((GafferPopGraph) g.getGraph()).newInstance().traversal(); - g2.getGraph().variables().set(GafferPopGraphVariables.USER, testUser2); - g2.V().toList(); - assertThat(((GafferPopGraphVariables) g.getGraph().variables()).getUser()).isEqualTo(testUser2); - }); - - // Set the user - // g.getGraph().variables().set(GafferPopGraphVariables.USER, testUser); - - // // Run a Query - // g.V().toList(); - - // Ensure user is still set - //assertThat(((GafferPopGraphVariables) g.getGraph().variables()).getUser()).isEqualTo(testUser); - - // Reset vars - ((GafferPopGraph) g.getGraph()).setDefaultVariables(); - assertThat(((GafferPopGraphVariables) g.getGraph().variables()).getUser()).isNotEqualTo(testUser); - reset(); - } - void reset() { // reset cache for federation From 89d808f7ce9567916c3649ab3ce571cfc03abcbb Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Thu, 16 Jan 2025 11:43:43 +0000 Subject: [PATCH 12/29] test tweaks --- .../rest/config/GremlinWebSocketConfig.java | 1 - .../rest/controller/GremlinController.java | 44 +++++++++++-------- .../rest/handler/GremlinWebSocketHandler.java | 16 ++++--- .../controller/GremlinControllerTest.java | 11 ++--- .../handler/GremlinWebSocketIT.java | 12 +++-- 5 files changed, 43 insertions(+), 41 deletions(-) diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinWebSocketConfig.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinWebSocketConfig.java index 53600256ea7..f71d900f108 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinWebSocketConfig.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinWebSocketConfig.java @@ -16,7 +16,6 @@ package uk.gov.gchq.gaffer.rest.config; -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; import org.springframework.web.socket.config.annotation.EnableWebSocket; diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java index 720af5468d8..55ab478c855 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java @@ -21,6 +21,8 @@ import io.opentelemetry.context.Context; import io.opentelemetry.context.Scope; import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.tinkerpop.gremlin.groovy.engine.GremlinExecutor; import org.apache.tinkerpop.gremlin.jsr223.ConcurrentBindings; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; @@ -55,7 +57,6 @@ import uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph; import uk.gov.gchq.gaffer.tinkerpop.GafferPopGraphVariables; import uk.gov.gchq.gaffer.user.User; -import uk.gov.gchq.koryphe.tuple.n.Tuple2; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -125,7 +126,7 @@ public GremlinController(final GafferPopGraph graph, final AbstractUserFactory u summary = "Explain a Gremlin Query", description = "Runs a Gremlin query and outputs an explanation of what Gaffer operations were executed on the graph") public String explain(@RequestHeader final HttpHeaders httpHeaders, @RequestBody final String gremlinQuery) { - return runGremlinQuery(gremlinQuery, httpHeaders).get1().toString(); + return runGremlinQuery(gremlinQuery, httpHeaders).getRight().toString(); } /** @@ -149,7 +150,7 @@ public ResponseEntity execute( HttpStatus status = HttpStatus.OK; StreamingResponseBody responseBody; try { - Object result = runGremlinQuery(gremlinQuery, httpHeaders).get0(); + Object result = runGremlinQuery(gremlinQuery, httpHeaders).getLeft(); // Write to output stream for response responseBody = os -> GRAPHSON_V3_WRITER.writeObject(os, result); } catch (final Exception e) { @@ -183,7 +184,7 @@ public String cypherExplain(@RequestHeader final HttpHeaders httpHeaders, @Reque final String translation = ast.buildTranslation( Translator.builder().gremlinGroovy().enableCypherExtensions().build()) + ".toList()"; - JSONObject response = runGremlinQuery(translation, httpHeaders).get1(); + JSONObject response = runGremlinQuery(translation, httpHeaders).getRight(); response.put(EXPLAIN_GREMLIN_KEY, translation); return response.toString(); } @@ -216,7 +217,7 @@ public ResponseEntity cypherExecute( final String translation = ast.buildTranslation( Translator.builder().gremlinGroovy().enableCypherExtensions().build()) + ".toList()"; // Run Query - Object result = runGremlinQuery(translation, httpHeaders).get0(); + Object result = runGremlinQuery(translation, httpHeaders).getLeft(); // Write to output stream for response responseBody = os -> GRAPHSON_V3_WRITER.writeObject(os, result); } catch (final Exception e) { @@ -238,15 +239,15 @@ public ResponseEntity cypherExecute( * query may be absent from the operation chains in the explain as it may have * been done in the Tinkerpop framework instead. * - * @param graph The GafferPop graph + * @param graphInstance The GafferPop graph instance. * @return A JSON payload with an overview and full JSON representation of the * chain in. */ - public static JSONObject getGafferPopExplanation(final GafferPopGraph graph) { + public static JSONObject getGafferPopExplanation(final GafferPopGraph graphInstance) { JSONObject result = new JSONObject(); // Get the last operation chain that ran LinkedList operations = new LinkedList<>(); - ((GafferPopGraphVariables) graph.variables()) + ((GafferPopGraphVariables) graphInstance.variables()) .getLastOperationChain() .getOperations() .forEach(op -> { @@ -273,7 +274,8 @@ public static JSONObject getGafferPopExplanation(final GafferPopGraph graph) { * Do some basic pre execute set up so the graph is ready for the gremlin * request to be executed. * - * @param GafferPopGraph The graph structure to use + * @param graphInstance The graph instance to bind the traversal to. + * @return The set-up {@link GremlinExecutor}. */ private GremlinExecutor setUpExecutor(final GafferPopGraph graphInstance) { final ConcurrentBindings bindings = new ConcurrentBindings(); @@ -289,12 +291,15 @@ private GremlinExecutor setUpExecutor(final GafferPopGraph graphInstance) { } /** - * Executes a given Gremlin query and returns the result along with an explanation. + * Executes a given Gremlin query and returns the result along with an + * explanation. * * @param gremlinQuery The Gremlin groovy query. - * @return A pair tuple with result and explain in. + * @param httpHeaders The headers from the request for the + * {@link AbstractUserFactory}. + * @return A pair with result left and explain right. */ - private Tuple2 runGremlinQuery(final String gremlinQuery, final HttpHeaders httpHeaders) { + private Pair runGremlinQuery(final String gremlinQuery, final HttpHeaders httpHeaders) { // We can't reuse the existing graph instance as we need to set variables // specific to this query only. final GafferPopGraph graphInstance = graph.newInstance(); @@ -302,7 +307,7 @@ private Tuple2 runGremlinQuery(final String gremlinQuery, fi // Hooks for user auth userFactory.setHttpHeaders(httpHeaders); User user = userFactory.createUser(); - graph.variables().set(GafferPopGraphVariables.USER, user); + graphInstance.variables().set(GafferPopGraphVariables.USER, user); // OpenTelemetry hooks Span span = OtelUtil.startSpan( @@ -316,9 +321,8 @@ private Tuple2 runGremlinQuery(final String gremlinQuery, fi span.setAttribute(OtelUtil.USER_ATTRIBUTE, "unknownGremlinUser"); } - // tuple to hold the result and explain - Tuple2 pair = new Tuple2<>(); - pair.put1(new JSONObject()); + // Pair to hold the result and explain + MutablePair pair = new MutablePair<>(null, new JSONObject()); try (Scope scope = span.makeCurrent(); GremlinExecutor gremlinExecutor = setUpExecutor(graphInstance)) { @@ -326,12 +330,14 @@ private Tuple2 runGremlinQuery(final String gremlinQuery, fi Object result = gremlinExecutor.eval(gremlinQuery).get(); // Store the result and explain for returning - pair.put0(result); - pair.put1(getGafferPopExplanation(graph)); + pair.setLeft(result); + pair.setRight(getGafferPopExplanation(graphInstance)); // Provide an debug explanation for the query that just ran span.addEvent("Request complete"); - LOGGER.debug("{}", pair.get1()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("{}", pair.getRight()); + } } catch (final InterruptedException e) { Thread.currentThread().interrupt(); span.setStatus(StatusCode.ERROR, e.getMessage()); diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java index 063c8100cfd..8f6df9f094f 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java @@ -95,9 +95,10 @@ public class GremlinWebSocketHandler extends BinaryWebSocketHandler { /** * Constructor * - * @param g The graph traversal source - * @param userFactory The user factory - * @param requestTimeout The timeout for gremlin requests + * @param graph The {@link GafferPopGraph} this is initialised to use the + * correct underlying Gaffer graph. + * @param userFactory The user factory. + * @param requestTimeout The timeout for gremlin requests. */ public GremlinWebSocketHandler(final GafferPopGraph graph, final AbstractUserFactory userFactory, final Long requestTimeout) { this.graph = graph; @@ -131,7 +132,8 @@ protected void handleBinaryMessage(final WebSocketSession session, final BinaryM * Do some basic pre execute set up so the graph is ready for the gremlin * request to be executed. * - * @param GafferPopGraph The graph structure to use + * @param graphInstance The graph instance to bind the traversal to. + * @return The set-up {@link GremlinExecutor}. */ private GremlinExecutor setUpExecutor(final GafferPopGraph graphInstance) { final ConcurrentBindings bindings = new ConcurrentBindings(); @@ -169,11 +171,11 @@ private ResponseMessage handleGremlinRequest(final WebSocketSession session, fin // Execute the query try (Scope scope = span.makeCurrent(); - GremlinExecutor gremlinExecutor = setUpExecutor(graph)) { + GremlinExecutor gremlinExecutor = setUpExecutor(graphInstance)) { // Set current headers for potential authorisation then set the user userFactory.setHttpHeaders(session.getHandshakeHeaders()); User user = userFactory.createUser(); - graph.variables().set(GafferPopGraphVariables.USER, user); + graphInstance.variables().set(GafferPopGraphVariables.USER, user); span.setAttribute(OtelUtil.USER_ATTRIBUTE, user.getUserId()); // Run the query using the gremlin executor service @@ -195,7 +197,7 @@ private ResponseMessage handleGremlinRequest(final WebSocketSession session, fin // Provide an debug explanation for the query that just ran span.addEvent("Request complete"); if (LOGGER.isDebugEnabled()) { - JSONObject gafferOperationChain = GremlinController.getGafferPopExplanation(graph); + JSONObject gafferOperationChain = GremlinController.getGafferPopExplanation(graphInstance); LOGGER.debug("{}", gafferOperationChain); } // Build the response diff --git a/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/controller/GremlinControllerTest.java b/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/controller/GremlinControllerTest.java index 34387c8ef07..9f59238754c 100644 --- a/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/controller/GremlinControllerTest.java +++ b/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/controller/GremlinControllerTest.java @@ -16,8 +16,6 @@ package uk.gov.gchq.gaffer.rest.controller; -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; -import org.apache.tinkerpop.gremlin.structure.Graph; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -65,9 +63,8 @@ class GremlinControllerTest { @TestConfiguration static class TestConfig { @Bean - public GraphTraversalSource g() { - Graph graph = GafferPopModernTestUtils.createModernGraph(TestConfig.class, StoreType.MAP); - return graph.traversal(); + public GafferPopGraph graph() { + return GafferPopModernTestUtils.createModernGraph(TestConfig.class, StoreType.MAP); } @Bean @@ -85,7 +82,7 @@ public Long timeout() { private MockMvc mockMvc; @Autowired - private GraphTraversalSource g; + private GafferPopGraph graph; @Test void shouldExecuteValidGremlinQuery() throws Exception { @@ -93,7 +90,7 @@ void shouldExecuteValidGremlinQuery() throws Exception { // Create the expected output OutputStream expectedOutput = new ByteArrayOutputStream(); - GremlinController.GRAPHSON_V3_WRITER.writeObject(expectedOutput, Arrays.asList(MARKO.toVertex((GafferPopGraph) g.getGraph()))); + GremlinController.GRAPHSON_V3_WRITER.writeObject(expectedOutput, Arrays.asList(MARKO.toVertex((GafferPopGraph) graph))); // When MvcResult result = mockMvc diff --git a/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/integration/handler/GremlinWebSocketIT.java b/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/integration/handler/GremlinWebSocketIT.java index bae3b653253..993c9375200 100644 --- a/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/integration/handler/GremlinWebSocketIT.java +++ b/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/integration/handler/GremlinWebSocketIT.java @@ -19,8 +19,6 @@ import org.apache.tinkerpop.gremlin.driver.Client; import org.apache.tinkerpop.gremlin.driver.Cluster; import org.apache.tinkerpop.gremlin.driver.Result; -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; -import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV3; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -37,6 +35,7 @@ import uk.gov.gchq.gaffer.rest.factory.spring.AbstractUserFactory; import uk.gov.gchq.gaffer.rest.factory.spring.UnknownUserFactory; +import uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph; import uk.gov.gchq.gaffer.tinkerpop.util.GafferPopTestUtil.StoreType; import uk.gov.gchq.gaffer.tinkerpop.util.modern.GafferPopModernTestUtils; @@ -70,9 +69,8 @@ static class TestConfig { @Bean @Profile("test") - public GraphTraversalSource g() { - Graph graph = GafferPopModernTestUtils.createModernGraph(TestConfig.class, StoreType.MAP); - return graph.traversal(); + public GafferPopGraph graph() { + return GafferPopModernTestUtils.createModernGraph(TestConfig.class, StoreType.MAP); } @Bean @@ -92,7 +90,7 @@ public Long timeout() { private Integer port; @Autowired - private GraphTraversalSource g; + private GafferPopGraph graph; private Client client; @@ -161,7 +159,7 @@ void shouldAcceptGremlinQueryUsingCustomCypherFunctions() { // Then assertThat(results) - .map(result -> result.getObject()) + .map(Result::getObject) .containsExactlyInAnyOrder( String.valueOf(MARKO.getAge()), String.valueOf(VADAS.getAge()), From d039e978e2cb15b7eb6043eea37738cf1d680370 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Fri, 17 Jan 2025 15:26:35 +0000 Subject: [PATCH 13/29] minor tweaks --- .../gchq/gaffer/commonutil/otel/OtelUtil.java | 2 +- .../java/uk/gov/gchq/gaffer/graph/Graph.java | 2 +- .../gchq/gaffer/tinkerpop/GafferPopGraph.java | 48 ++++++------------- .../traversal/step/GafferPopGraphStep.java | 2 +- .../gaffer/tinkerpop/GafferPopGraphIT.java | 2 +- .../gaffer/rest/config/GremlinConfig.java | 2 +- .../rest/config/GremlinWebSocketConfig.java | 2 +- .../rest/controller/GremlinController.java | 18 ++++--- .../rest/handler/GremlinWebSocketHandler.java | 7 ++- .../controller/GremlinControllerTest.java | 2 +- .../handler/GremlinWebSocketIT.java | 2 +- 11 files changed, 33 insertions(+), 56 deletions(-) diff --git a/core/common-util/src/main/java/uk/gov/gchq/gaffer/commonutil/otel/OtelUtil.java b/core/common-util/src/main/java/uk/gov/gchq/gaffer/commonutil/otel/OtelUtil.java index ae23398457a..b4f2881c339 100644 --- a/core/common-util/src/main/java/uk/gov/gchq/gaffer/commonutil/otel/OtelUtil.java +++ b/core/common-util/src/main/java/uk/gov/gchq/gaffer/commonutil/otel/OtelUtil.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/graph/src/main/java/uk/gov/gchq/gaffer/graph/Graph.java b/core/graph/src/main/java/uk/gov/gchq/gaffer/graph/Graph.java index e50f06e494b..d170d80cc3e 100644 --- a/core/graph/src/main/java/uk/gov/gchq/gaffer/graph/Graph.java +++ b/core/graph/src/main/java/uk/gov/gchq/gaffer/graph/Graph.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 Crown Copyright + * Copyright 2017-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java index 350502bd48d..34d952ea637 100755 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 Crown Copyright + * Copyright 2016-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -253,11 +253,11 @@ public enum DefaultIdManager { private final Graph graph; private final Configuration configuration; - private final GafferPopGraphVariables variables; - private final GafferPopGraphFeatures features; - private final Map opOptions; private final User defaultUser; - private final ServiceRegistry serviceRegistry; + private final GafferPopGraphVariables variables = new GafferPopGraphVariables(); + private final GafferPopGraphFeatures features = new GafferPopGraphFeatures(); + private final Map opOptions = new HashMap<>(); + private final ServiceRegistry serviceRegistry = new ServiceRegistry(); private static final Logger LOGGER = LoggerFactory.getLogger(GafferPopGraph.class); private static final String GET_DEBUG_MSG = "Requested a GetElements, results will be truncated to: {}."; @@ -270,8 +270,6 @@ public GafferPopGraph(final Configuration configuration) { public GafferPopGraph(final Configuration configuration, final Graph graph) { this.configuration = configuration; this.graph = graph; - features = new GafferPopGraphFeatures(); - opOptions = new HashMap<>(); if (configuration().containsKey(OP_OPTIONS)) { for (final String option : configuration().getStringArray(OP_OPTIONS)) { final String[] parts = option.split(":"); @@ -285,10 +283,8 @@ public GafferPopGraph(final Configuration configuration, final Graph graph) { .build(); // Set the graph variables to current config - variables = new GafferPopGraphVariables(); setDefaultVariables(); - serviceRegistry = new ServiceRegistry(); serviceRegistry.registerService(new GafferPopNamedOperationServiceFactory(this)); // Add and register custom traversals @@ -364,21 +360,6 @@ public Vertex addVertex(final Object... keyValues) { idValue = ElementHelper.getIdValue(keyValues).orElseThrow(() -> new IllegalArgumentException("ID is required")); } - /* - * TODO: Check the ID type is relevant for the group (a.k.a label) in the schema and auto convert - * as the some Standard tinkerpop tests add data for the same group but with a different - * Object type for the ID. Using a String ID manager might be the most flexible for these - * tests. - * Basic idea of auto converting the type is below: - * - * String idSchemaType = graph.getSchema().getEntity(label).getVertex(); - * String idTypeName = graph.getSchema().getType(idSchemaType).getFullClassString(); - * if (!idTypeName.equals(idValue.getClass().getName())) { - * LOGGER.warn("Vertex ID is not the correct type for the schema: " + idValue); - * idValue = graph.getSchema().getType(idSchemaType).getClazz().cast(idValue); - * } - */ - final GafferPopVertex vertex = new GafferPopVertex(label, idValue, this); ElementHelper.attachProperties(vertex, VertexProperty.Cardinality.list, keyValues); addVertex(vertex); @@ -701,7 +682,6 @@ public T execute(final OperationChain opChain) { try { LOGGER.info("GafferPop operation chain called: {}", opChain.toOverviewString()); - LOGGER.info("USER IS: " + variables.getUser().getUserId()); return graph.execute(opChain, variables.getUser()); } catch (final Exception e) { LOGGER.error("Operation chain failed: {}", e.getMessage()); @@ -711,7 +691,15 @@ public T execute(final OperationChain opChain) { /** * Sets the {@link GafferPopGraphVariables} to default values for this - * graph optionally preserving the previous user. + * graph. + */ + public void setDefaultVariables() { + setDefaultVariables(false); + } + + /** + * Sets the {@link GafferPopGraphVariables} to default values for this + * graph optionally preserving the current user. * * @param preserveUser keep the current set user. */ @@ -731,14 +719,6 @@ public void setDefaultVariables(final boolean preserveUser) { variables.set(GafferPopGraphVariables.LAST_OPERATION_CHAIN, new OperationChain()); } - /** - * Sets the {@link GafferPopGraphVariables} to default values for this - * graph. - */ - public void setDefaultVariables() { - setDefaultVariables(false); - } - /** * Get the underlying Gaffer graph this GafferPop graph is connected to. * diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java index 188eabe55cd..bea42a1e26c 100644 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/process/traversal/step/GafferPopGraphStep.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java index dc63bf686e8..91fb1225863 100644 --- a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java +++ b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 Crown Copyright + * Copyright 2017-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinConfig.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinConfig.java index 432d5888475..5c9393ec9d4 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinConfig.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinConfig.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinWebSocketConfig.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinWebSocketConfig.java index f71d900f108..86cf9f21e2b 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinWebSocketConfig.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/config/GremlinWebSocketConfig.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java index 55ab478c855..41b153524dd 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/controller/GremlinController.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,6 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.tinkerpop.gremlin.groovy.engine.GremlinExecutor; import org.apache.tinkerpop.gremlin.jsr223.ConcurrentBindings; -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONVersion; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter; @@ -60,8 +59,9 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.HashMap; -import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.ExecutionException; @@ -98,7 +98,7 @@ public class GremlinController { private static final Logger LOGGER = LoggerFactory.getLogger(GremlinController.class); private static final String GENERAL_ERROR_MSG = "Gremlin query failed: "; - + // Shared thread pool for executing gremlin queries in private final ExecutorService executorService = Context.taskWrapping(Executors.newFixedThreadPool(8)); private final AbstractUserFactory userFactory; private final Long requestTimeout; @@ -246,7 +246,7 @@ public ResponseEntity cypherExecute( public static JSONObject getGafferPopExplanation(final GafferPopGraph graphInstance) { JSONObject result = new JSONObject(); // Get the last operation chain that ran - LinkedList operations = new LinkedList<>(); + List operations = new ArrayList<>(); ((GafferPopGraphVariables) graphInstance.variables()) .getLastOperationChain() .getOperations() @@ -279,10 +279,9 @@ public static JSONObject getGafferPopExplanation(final GafferPopGraph graphInsta */ private GremlinExecutor setUpExecutor(final GafferPopGraph graphInstance) { final ConcurrentBindings bindings = new ConcurrentBindings(); - final GraphTraversalSource g = graphInstance.traversal(); // Set up the executor - bindings.putIfAbsent("g", g); + bindings.putIfAbsent("g", graphInstance.traversal()); return GremlinExecutor.build() .addPlugins("gremlin-groovy", plugins) .evaluationTimeout(requestTimeout) @@ -335,9 +334,8 @@ private Pair runGremlinQuery(final String gremlinQuery, fina // Provide an debug explanation for the query that just ran span.addEvent("Request complete"); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("{}", pair.getRight()); - } + LOGGER.debug("{}", pair.getRight()); + } catch (final InterruptedException e) { Thread.currentThread().interrupt(); span.setStatus(StatusCode.ERROR, e.getMessage()); diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java index 8f6df9f094f..b9353f5ed9a 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -85,7 +85,7 @@ public class GremlinWebSocketHandler extends BinaryWebSocketHandler { new SimpleEntry<>(SerTokens.MIME_GRAPHSON_V3, new GraphSONMessageSerializerV3()), new SimpleEntry<>(SerTokens.MIME_JSON, new GraphSONMessageSerializerV3())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - + // Shared thread pool for executing gremlin queries in private final ExecutorService executorService = Context.taskWrapping(Executors.newFixedThreadPool(8)); private final AbstractUserFactory userFactory; private final Long requestTimeout; @@ -137,10 +137,9 @@ protected void handleBinaryMessage(final WebSocketSession session, final BinaryM */ private GremlinExecutor setUpExecutor(final GafferPopGraph graphInstance) { final ConcurrentBindings bindings = new ConcurrentBindings(); - final GraphTraversalSource g = graphInstance.traversal(); // Set up the executor - bindings.putIfAbsent("g", g); + bindings.putIfAbsent("g", graphInstance.traversal()); return GremlinExecutor.build() .addPlugins("gremlin-groovy", plugins) .evaluationTimeout(requestTimeout) diff --git a/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/controller/GremlinControllerTest.java b/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/controller/GremlinControllerTest.java index 9f59238754c..325122304de 100644 --- a/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/controller/GremlinControllerTest.java +++ b/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/controller/GremlinControllerTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/integration/handler/GremlinWebSocketIT.java b/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/integration/handler/GremlinWebSocketIT.java index 993c9375200..aa2cebc3ff6 100644 --- a/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/integration/handler/GremlinWebSocketIT.java +++ b/rest-api/spring-rest/src/test/java/uk/gov/gchq/gaffer/rest/integration/handler/GremlinWebSocketIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 46b32d3a6b249c7024237aa2fd7a6614364d3ca7 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Fri, 17 Jan 2025 16:16:53 +0000 Subject: [PATCH 14/29] checkstyle --- .../uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java index b9353f5ed9a..36029f5c5d3 100644 --- a/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java +++ b/rest-api/spring-rest/src/main/java/uk/gov/gchq/gaffer/rest/handler/GremlinWebSocketHandler.java @@ -26,7 +26,6 @@ import org.apache.tinkerpop.gremlin.groovy.engine.GremlinExecutor; import org.apache.tinkerpop.gremlin.jsr223.ConcurrentBindings; import org.apache.tinkerpop.gremlin.process.remote.traversal.DefaultRemoteTraverser; -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.util.MessageSerializer; import org.apache.tinkerpop.gremlin.util.Tokens; import org.apache.tinkerpop.gremlin.util.function.FunctionUtils; From d7964996675d7fc1f362de45e292a0cde9e5ff00 Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:57:19 +0000 Subject: [PATCH 15/29] add count and logging to DeleteElements --- .../uk/gov/gchq/gaffer/graph/GraphTest.java | 4 +- .../gaffer/integration/store/TestStore.java | 4 +- .../operation/impl/delete/DeleteElements.java | 17 +++++-- .../java/uk/gov/gchq/gaffer/store/Store.java | 2 +- .../uk/gov/gchq/gaffer/store/StoreTest.java | 6 +-- .../gaffer/store/integration/StoreIT.java | 4 +- .../handler/TestAddToGraphLibraryImpl.java | 4 +- .../gaffer/accumulostore/AccumuloStore.java | 4 +- .../handler/DeleteElementsHandler.java | 30 +++++++++--- .../delete/AbstractDeletedElementsIT.java | 2 +- .../handler/DeleteElementsHandlerTest.java | 9 ++-- .../gaffer/federatedstore/FederatedStore.java | 6 +-- .../impl/FederatedDeleteElementsTest.java | 24 ++++----- .../impl/FederatedGetTraitsHandlerTest.java | 4 +- .../uk/gov/gchq/gaffer/mapstore/MapStore.java | 4 +- .../mapstore/impl/DeleteElementsHandler.java | 30 +++++++++--- .../impl/DeleteElementsHandlerTest.java | 49 +++++++++++-------- .../gchq/gaffer/proxystore/ProxyStore.java | 4 +- .../federated/simple/FederatedStore.java | 4 +- 19 files changed, 133 insertions(+), 78 deletions(-) diff --git a/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java b/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java index ae42ec9d588..f07330b4590 100644 --- a/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java +++ b/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 Crown Copyright + * Copyright 2016-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -2744,7 +2744,7 @@ protected OperationHandler getAddElementsHandler() { } @Override - protected OperationHandler getDeleteElementsHandler() { + protected OutputOperationHandler getDeleteElementsHandler() { return null; } diff --git a/core/graph/src/test/java/uk/gov/gchq/gaffer/integration/store/TestStore.java b/core/graph/src/test/java/uk/gov/gchq/gaffer/integration/store/TestStore.java index 7a3180ee22d..66c7ea242c2 100644 --- a/core/graph/src/test/java/uk/gov/gchq/gaffer/integration/store/TestStore.java +++ b/core/graph/src/test/java/uk/gov/gchq/gaffer/integration/store/TestStore.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 Crown Copyright + * Copyright 2016-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -86,7 +86,7 @@ protected OperationHandler getAddElementsHandler() { } @Override - protected OperationHandler getDeleteElementsHandler() { + protected OutputOperationHandler getDeleteElementsHandler() { return null; } diff --git a/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/delete/DeleteElements.java b/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/delete/DeleteElements.java index cb42b220cff..10ae74b4578 100644 --- a/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/delete/DeleteElements.java +++ b/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/delete/DeleteElements.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,8 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.core.type.TypeReference; + import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; @@ -25,7 +27,10 @@ import uk.gov.gchq.gaffer.data.element.Element; import uk.gov.gchq.gaffer.operation.Operation; import uk.gov.gchq.gaffer.operation.Validatable; +import uk.gov.gchq.gaffer.operation.io.InputOutput; import uk.gov.gchq.gaffer.operation.io.MultiInput; +import uk.gov.gchq.gaffer.operation.io.Output; +import uk.gov.gchq.gaffer.operation.serialisation.TypeReferenceImpl; import uk.gov.gchq.koryphe.Since; import uk.gov.gchq.koryphe.Summary; @@ -50,7 +55,8 @@ @Summary("Deletes elements") public class DeleteElements implements Validatable, - MultiInput { + MultiInput, + InputOutput, Long> { private boolean validate = true; private boolean skipInvalidElements; private Iterable elements; @@ -92,6 +98,11 @@ public void setInput(final Iterable elements) { this.elements = elements; } + @Override + public TypeReference getOutputTypeReference() { + return new TypeReferenceImpl.Long(); + } + @Override public Map getOptions() { return options; @@ -154,7 +165,7 @@ public String toString() { public static class Builder extends Operation.BaseBuilder implements Validatable.Builder, - MultiInput.Builder { + MultiInput.Builder, InputOutput.Builder, Long, Builder> { public Builder() { super(new DeleteElements()); } diff --git a/core/store/src/main/java/uk/gov/gchq/gaffer/store/Store.java b/core/store/src/main/java/uk/gov/gchq/gaffer/store/Store.java index 40d37b067a1..03e55df93c7 100644 --- a/core/store/src/main/java/uk/gov/gchq/gaffer/store/Store.java +++ b/core/store/src/main/java/uk/gov/gchq/gaffer/store/Store.java @@ -899,7 +899,7 @@ public List getOperationChainOptimisers() { * * @return the implementation of the handler for {@link uk.gov.gchq.gaffer.operation.impl.delete.DeleteElements} */ - protected abstract OperationHandler getDeleteElementsHandler(); + protected abstract OutputOperationHandler getDeleteElementsHandler(); /** * Get this Store's implementation of the handler for {@link uk.gov.gchq.gaffer.operation.DeleteAllData}. diff --git a/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java b/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java index 27ae65325b7..18e5bce7993 100644 --- a/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java +++ b/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java @@ -1185,7 +1185,7 @@ protected OperationHandler getAddElementsHandler() { @Override - protected OperationHandler getDeleteElementsHandler() { + protected OutputOperationHandler getDeleteElementsHandler() { return null; } @@ -1286,7 +1286,7 @@ protected OperationHandler getAddElementsHandler() { @Override - protected OperationHandler getDeleteElementsHandler() { + protected OutputOperationHandler getDeleteElementsHandler() { return null; } @@ -1396,7 +1396,7 @@ protected OperationHandler getAddElementsHandler() { } @Override - protected OperationHandler getDeleteElementsHandler() { + protected OutputOperationHandler getDeleteElementsHandler() { return null; } diff --git a/core/store/src/test/java/uk/gov/gchq/gaffer/store/integration/StoreIT.java b/core/store/src/test/java/uk/gov/gchq/gaffer/store/integration/StoreIT.java index 64f4d663311..a6806cdfa09 100644 --- a/core/store/src/test/java/uk/gov/gchq/gaffer/store/integration/StoreIT.java +++ b/core/store/src/test/java/uk/gov/gchq/gaffer/store/integration/StoreIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 Crown Copyright + * Copyright 2016-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -145,7 +145,7 @@ protected OperationHandler getAddElementsHandler() { } @Override - protected OperationHandler getDeleteElementsHandler() { + protected OutputOperationHandler getDeleteElementsHandler() { return null; } diff --git a/core/store/src/test/java/uk/gov/gchq/gaffer/store/operation/handler/TestAddToGraphLibraryImpl.java b/core/store/src/test/java/uk/gov/gchq/gaffer/store/operation/handler/TestAddToGraphLibraryImpl.java index 1ce44a44781..8e5f943ad2f 100644 --- a/core/store/src/test/java/uk/gov/gchq/gaffer/store/operation/handler/TestAddToGraphLibraryImpl.java +++ b/core/store/src/test/java/uk/gov/gchq/gaffer/store/operation/handler/TestAddToGraphLibraryImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 Crown Copyright + * Copyright 2017-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -74,7 +74,7 @@ protected Class getRequiredParentSerialiserClass() { } @Override - protected OperationHandler getDeleteElementsHandler() { + protected OutputOperationHandler getDeleteElementsHandler() { return null; } } diff --git a/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/AccumuloStore.java b/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/AccumuloStore.java index 8e2ea59d92f..67131661c8d 100644 --- a/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/AccumuloStore.java +++ b/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/AccumuloStore.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 Crown Copyright + * Copyright 2016-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -443,7 +443,7 @@ protected OutputOperationHandler> getGetTraitsHandler } @Override - protected OperationHandler getDeleteElementsHandler() { + protected OutputOperationHandler getDeleteElementsHandler() { return new DeleteElementsHandler(); } diff --git a/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java b/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java index 0ab2b3df2b9..d53b30c4d06 100644 --- a/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java +++ b/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,9 @@ package uk.gov.gchq.gaffer.accumulostore.operation.handler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import uk.gov.gchq.gaffer.accumulostore.AccumuloStore; import uk.gov.gchq.gaffer.data.element.Element; import uk.gov.gchq.gaffer.operation.OperationException; @@ -24,19 +27,24 @@ import uk.gov.gchq.gaffer.store.Store; import uk.gov.gchq.gaffer.store.StoreException; import uk.gov.gchq.gaffer.store.ValidatedElements; -import uk.gov.gchq.gaffer.store.operation.handler.OperationHandler; +import uk.gov.gchq.gaffer.store.operation.handler.OutputOperationHandler; + +import java.util.ArrayList; +import java.util.List; -public class DeleteElementsHandler implements OperationHandler { +public class DeleteElementsHandler implements OutputOperationHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(DeleteElementsHandler.class); + private Long elementCount = 0L; @Override - public Object doOperation(final DeleteElements operation, + public Long doOperation(final DeleteElements operation, final Context context, final Store store) throws OperationException { deleteElements(operation, (AccumuloStore) store); - return null; + return elementCount; } - private void deleteElements(final DeleteElements operation, final AccumuloStore store) + private Long deleteElements(final DeleteElements operation, final AccumuloStore store) throws OperationException { try { final Iterable validatedElements; @@ -46,9 +54,19 @@ private void deleteElements(final DeleteElements operation, final AccumuloStore } else { validatedElements = operation.getInput(); } + + final List elementsToDelete = new ArrayList<>(); + for (final Element el : validatedElements) { + elementsToDelete.add(el.toString()); + elementCount++; + } + LOGGER.debug("Deleting elements: {}", elementsToDelete); + store.deleteElements(validatedElements); } catch (final StoreException e) { throw new OperationException("Failed to delete elements", e); } + + return elementCount; } } diff --git a/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/integration/delete/AbstractDeletedElementsIT.java b/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/integration/delete/AbstractDeletedElementsIT.java index 2835ed6ae3d..dc854b3d0ab 100644 --- a/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/integration/delete/AbstractDeletedElementsIT.java +++ b/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/integration/delete/AbstractDeletedElementsIT.java @@ -114,7 +114,7 @@ public void shouldNotReturnDeletedElements() throws Exception { assertElements((Iterable) elements, resultBefore); // When - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed("1")) .build()) diff --git a/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandlerTest.java b/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandlerTest.java index 1b7e0e05894..f2ce829cba9 100644 --- a/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandlerTest.java +++ b/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandlerTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ import uk.gov.gchq.gaffer.store.StoreException; import uk.gov.gchq.gaffer.store.ValidatedElements; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyIterable; @@ -45,7 +46,8 @@ void shouldDoOperationValidated() throws Exception { .build(); AccumuloStore store = Mockito.mock(AccumuloStore.class); - handler.doOperation(op, new Context(), store); + final Object elementCount = handler.doOperation(op, new Context(), store); + assertThat(elementCount).isEqualTo("Elements deleted: 1"); verify(store).deleteElements(any(ValidatedElements.class)); } @@ -60,7 +62,8 @@ void shouldDoOperationNotValidated() throws Exception { .build(); AccumuloStore store = Mockito.mock(AccumuloStore.class); - handler.doOperation(op, new Context(), store); + final Object elementCount = handler.doOperation(op, new Context(), store); + assertThat(elementCount).isEqualTo("Elements deleted: 1"); verify(store).deleteElements(anyIterable()); } diff --git a/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/FederatedStore.java b/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/FederatedStore.java index 6b8d4780c8b..5f58b9aff63 100644 --- a/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/FederatedStore.java +++ b/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/FederatedStore.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 Crown Copyright + * Copyright 2017-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -550,8 +550,8 @@ protected OutputOperationHandler> ge } @Override - protected OperationHandler getDeleteElementsHandler() { - return new FederatedNoOutputHandler<>(); + protected OutputOperationHandler getDeleteElementsHandler() { + return new FederatedOutputHandler<>(); } @Override diff --git a/store-implementation/federated-store/src/test/java/uk/gov/gchq/gaffer/federatedstore/operation/handler/impl/FederatedDeleteElementsTest.java b/store-implementation/federated-store/src/test/java/uk/gov/gchq/gaffer/federatedstore/operation/handler/impl/FederatedDeleteElementsTest.java index badad7f056b..dec71f3d071 100644 --- a/store-implementation/federated-store/src/test/java/uk/gov/gchq/gaffer/federatedstore/operation/handler/impl/FederatedDeleteElementsTest.java +++ b/store-implementation/federated-store/src/test/java/uk/gov/gchq/gaffer/federatedstore/operation/handler/impl/FederatedDeleteElementsTest.java @@ -86,7 +86,7 @@ public void setUp() throws Exception { @Test void shouldDeleteEntityFromSingleGraphWithMapStore() throws Exception { // When - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed("1")) .view(new View.Builder().entity(GROUP_BASIC_ENTITY).build()) @@ -108,7 +108,7 @@ void shouldDeleteEntityFromSingleGraphWithMapStore() throws Exception { @Test void shouldDeleteEdgeFromSingleGraphWithMapStore() throws Exception { // When - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EdgeSeed("1", "2")) .view(new View.Builder().edge(GROUP_BASIC_EDGE).build()) @@ -130,7 +130,7 @@ void shouldDeleteEdgeFromSingleGraphWithMapStore() throws Exception { @Test void shouldDeleteEntityAndEdgesFromSingleGraphhWithMapStore() throws Exception { // Given/When - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed("2")) .build()) @@ -152,7 +152,7 @@ void shouldDeleteEntityAndEdgesFromSingleGraphhWithMapStore() throws Exception { @Test void shouldDeleteEntityFromBothGraphsWithMapStore() throws Exception { // When - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed("1")) .view(new View.Builder().entity(GROUP_BASIC_ENTITY).build()) @@ -174,7 +174,7 @@ void shouldDeleteEntityFromBothGraphsWithMapStore() throws Exception { @Test void shouldDeleteEdgeFromBothGraphsWithMapStore() throws Exception { // When - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EdgeSeed("1", "2")) .view(new View.Builder().edge(GROUP_BASIC_EDGE).build()) @@ -196,7 +196,7 @@ void shouldDeleteEdgeFromBothGraphsWithMapStore() throws Exception { @Test void shouldDeleteEntityAndEdgesFromBothGraphshWithMapStore() throws Exception { // Given/When - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed("3")) .build()) @@ -217,7 +217,7 @@ void shouldDeleteEntityAndEdgesFromBothGraphshWithMapStore() throws Exception { @Test void shouldDeleteEntityFromSingleGraphWithAccumuloStore() throws Exception { // When - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed("1")) .view(new View.Builder().entity(GROUP_BASIC_ENTITY).build()) @@ -239,7 +239,7 @@ void shouldDeleteEntityFromSingleGraphWithAccumuloStore() throws Exception { @Test void shouldDeleteEdgeFromSingleGraphWithAccumuloStore() throws Exception { // When - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EdgeSeed("1", "2")) .view(new View.Builder().edge(GROUP_BASIC_EDGE).build()) @@ -261,7 +261,7 @@ void shouldDeleteEdgeFromSingleGraphWithAccumuloStore() throws Exception { @Test void shouldDeleteEntityAndEdgesFromSingleGraphhWithAccumuloStore() throws Exception { // Given/When - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed("2")) .build()) @@ -283,7 +283,7 @@ void shouldDeleteEntityAndEdgesFromSingleGraphhWithAccumuloStore() throws Except @Test void shouldDeleteEntityFromBothGraphsWithAccumuloStore() throws Exception { // When - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed("1")) .view(new View.Builder().entity(GROUP_BASIC_ENTITY).build()) @@ -305,7 +305,7 @@ void shouldDeleteEntityFromBothGraphsWithAccumuloStore() throws Exception { @Test void shouldDeleteEdgeFromBothGraphsWithAccumuloStore() throws Exception { // When - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EdgeSeed("1", "2")) .view(new View.Builder().edge(GROUP_BASIC_EDGE).build()) @@ -327,7 +327,7 @@ void shouldDeleteEdgeFromBothGraphsWithAccumuloStore() throws Exception { @Test void shouldDeleteEntityAndEdgesFromBothGraphshWithAccumuloStore() throws Exception { // Given/When - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed("3")) .build()) diff --git a/store-implementation/federated-store/src/test/java/uk/gov/gchq/gaffer/federatedstore/operation/handler/impl/FederatedGetTraitsHandlerTest.java b/store-implementation/federated-store/src/test/java/uk/gov/gchq/gaffer/federatedstore/operation/handler/impl/FederatedGetTraitsHandlerTest.java index b06a9b35a70..58084c3920e 100644 --- a/store-implementation/federated-store/src/test/java/uk/gov/gchq/gaffer/federatedstore/operation/handler/impl/FederatedGetTraitsHandlerTest.java +++ b/store-implementation/federated-store/src/test/java/uk/gov/gchq/gaffer/federatedstore/operation/handler/impl/FederatedGetTraitsHandlerTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2024 Crown Copyright + * Copyright 2018-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -288,7 +288,7 @@ protected OperationHandler getAddElementsHandler() { } @Override - protected OperationHandler getDeleteElementsHandler() { + protected OutputOperationHandler getDeleteElementsHandler() { return null; } diff --git a/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/MapStore.java b/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/MapStore.java index 528632ce895..9ab0624a971 100644 --- a/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/MapStore.java +++ b/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/MapStore.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 Crown Copyright + * Copyright 2017-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -176,7 +176,7 @@ protected OperationHandler getAddElementsHandler() { } @Override - protected OperationHandler getDeleteElementsHandler() { + protected OutputOperationHandler getDeleteElementsHandler() { return new DeleteElementsHandler(); } diff --git a/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java b/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java index 99e05619494..6f13d4b1797 100644 --- a/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java +++ b/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,32 +31,44 @@ import uk.gov.gchq.gaffer.store.Context; import uk.gov.gchq.gaffer.store.Store; import uk.gov.gchq.gaffer.store.ValidatedElements; -import uk.gov.gchq.gaffer.store.operation.handler.OperationHandler; +import uk.gov.gchq.gaffer.store.operation.handler.OutputOperationHandler; import uk.gov.gchq.gaffer.store.schema.Schema; import uk.gov.gchq.gaffer.store.util.AggregatorUtil; +import java.util.ArrayList; +import java.util.List; + /** - * An {@link OperationHandler} for the {@link DeleteElements} operation on the - * {@link MapStore}. + * An {@link OutputOperationHandler} for the {@link DeleteElements} operation on the + * {@link MapStore}. Returns a string of how many elements have been deleted. */ -public class DeleteElementsHandler implements OperationHandler { +public class DeleteElementsHandler implements OutputOperationHandler { private static final Logger LOGGER = LoggerFactory.getLogger(DeleteElementsHandler.class); + private Long elementCount = 0L; @Override - public Object doOperation(final DeleteElements deleteElements, final Context context, final Store store) { + public Long doOperation(final DeleteElements deleteElements, final Context context, final Store store) { Iterable elements = deleteElements.getInput(); if (deleteElements.isValidate()) { elements = new ValidatedElements(elements, store.getSchema(), deleteElements.isSkipInvalidElements()); } deleteElements(elements, (MapStore) store); - return null; + return elementCount; } - private void deleteElements(final Iterable elements, final MapStore mapStore) { + private Long deleteElements(final Iterable elements, final MapStore mapStore) { final MapImpl mapImpl = mapStore.getMapImpl(); final Schema schema = mapStore.getSchema(); + final List elementsToDelete = new ArrayList<>(); + for (final Element el : elements) { + elementsToDelete.add(el.toString()); + elementCount++; + } + + LOGGER.debug("Deleting elements: {}", elementsToDelete); + final int bufferSize = mapStore.getProperties().getIngestBufferSize(); if (bufferSize < 1) { @@ -67,6 +79,8 @@ private void deleteElements(final Iterable elements, final Ma // Stream of lists that gets each batch Streams.toBatches(elements, bufferSize).forEach(batch -> deleteBatch(mapImpl, schema, AggregatorUtil.ingestAggregate(batch, schema))); } + + return elementCount; } private void deleteBatch(final MapImpl mapImpl, final Schema schema, final Iterable elements) { diff --git a/store-implementation/map-store/src/test/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandlerTest.java b/store-implementation/map-store/src/test/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandlerTest.java index a1cca81557b..dcf80460534 100644 --- a/store-implementation/map-store/src/test/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandlerTest.java +++ b/store-implementation/map-store/src/test/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandlerTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -76,7 +76,7 @@ void shouldDeleteEntityOnlyForAggregatedGraph() throws OperationException { // Given/When // Delete Vertex A - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed("A")) .view(ENTITY_VIEW) @@ -84,7 +84,8 @@ void shouldDeleteEntityOnlyForAggregatedGraph() throws OperationException { .then(new DeleteElements()) .build(); - aggregatedGraph.execute(chain, USER); + final Object deletedCount = aggregatedGraph.execute(chain, USER); + assertThat(deletedCount).isEqualTo(1L); // Then @@ -111,7 +112,7 @@ void shouldDeleteEdgeOnlyForAggregatedGraph() throws OperationException { // Given/When // Delete Edge B->C - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EdgeSeed("B", "C")) .view(EDGE_VIEW) @@ -119,7 +120,8 @@ void shouldDeleteEdgeOnlyForAggregatedGraph() throws OperationException { .then(new DeleteElements()) .build(); - aggregatedGraph.execute(chain, USER); + final Object elementCount = aggregatedGraph.execute(chain, USER); + assertThat(elementCount).isEqualTo(1L); // Then @@ -146,7 +148,7 @@ void shouldDeleteEntityOnlyForNonAggregatedGraph() throws OperationException { // Given/When // Delete Vertex A - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed("A")) .view(ENTITY_VIEW) @@ -154,7 +156,8 @@ void shouldDeleteEntityOnlyForNonAggregatedGraph() throws OperationException { .then(new DeleteElements()) .build(); - nonAggregatedGraph.execute(chain, USER); + final Object elementCount = nonAggregatedGraph.execute(chain, USER); + assertThat(elementCount).isEqualTo(1L); // Then @@ -182,7 +185,7 @@ void shouldDeleteEdgeOnlyForNonAggregatedGraph() throws OperationException { // When // Delete Edge B->C - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EdgeSeed("B", "C")) .view(EDGE_VIEW) @@ -190,7 +193,8 @@ void shouldDeleteEdgeOnlyForNonAggregatedGraph() throws OperationException { .then(new DeleteElements()) .build(); - nonAggregatedGraph.execute(chain, USER); + final Object elementCount = nonAggregatedGraph.execute(chain, USER); + assertThat(elementCount).isEqualTo(1L); // Then @@ -217,14 +221,15 @@ void shouldDeleteEntityAndEdgeForAggregatedGraph() throws OperationException { // Given/When // Delete Vertex A and its edges - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed("A")) .build()) .then(new DeleteElements()) .build(); - aggregatedGraph.execute(chain, USER); + final Object elementCount = aggregatedGraph.execute(chain, USER); + assertThat(elementCount).isEqualTo(2L); // Then @@ -249,14 +254,15 @@ void shouldDeleteEntityAndEdgeForNonAggregatedGraph() throws OperationException // Given/When // Delete Vertex A and its edges - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed("A")) .build()) .then(new DeleteElements()) .build(); - nonAggregatedGraph.execute(chain, USER); + final Object elementCount = nonAggregatedGraph.execute(chain, USER); + assertThat(elementCount).isEqualTo(2L); // Then @@ -281,14 +287,15 @@ void shouldDeleteEntityAndAllEdgesForAggregatedGraph() throws OperationException // Given/When // Delete Vertex B and its edges - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed("B")) .build()) .then(new DeleteElements()) .build(); - aggregatedGraph.execute(chain, USER); + final Object elementCount = aggregatedGraph.execute(chain, USER); + assertThat(elementCount).isEqualTo(3L); // Then @@ -313,14 +320,15 @@ void shouldDeleteEntityAndAllEdgesForNonAggregatedGraph() throws OperationExcept // Given/When // Delete Vertex B and its edges - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed("B")) .build()) .then(new DeleteElements()) .build(); - nonAggregatedGraph.execute(chain, USER); + final Object elementCount = nonAggregatedGraph.execute(chain, USER); + assertThat(elementCount).isEqualTo(3L); // Then @@ -345,13 +353,14 @@ void shouldDeleteAll() throws OperationException { // Given/When // Delete all - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetAllElements.Builder() .build()) .then(new DeleteElements()) .build(); - aggregatedGraph.execute(chain, USER); + final Object elementCount = aggregatedGraph.execute(chain, USER); + assertThat(elementCount).isEqualTo(5L); // Then @@ -384,7 +393,7 @@ void shouldDeleteElementsInBatches() throws StoreException, OperationException { // When // Delete all - final OperationChain chain = new OperationChain.Builder() + final OperationChain chain = new OperationChain.Builder() .first(new GetAllElements.Builder() .build()) .then(new DeleteElements()) diff --git a/store-implementation/proxy-store/src/main/java/uk/gov/gchq/gaffer/proxystore/ProxyStore.java b/store-implementation/proxy-store/src/main/java/uk/gov/gchq/gaffer/proxystore/ProxyStore.java index dd49f4cb214..8dfaf951429 100644 --- a/store-implementation/proxy-store/src/main/java/uk/gov/gchq/gaffer/proxystore/ProxyStore.java +++ b/store-implementation/proxy-store/src/main/java/uk/gov/gchq/gaffer/proxystore/ProxyStore.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 Crown Copyright + * Copyright 2016-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -366,7 +366,7 @@ protected OperationHandler getAddElementsHandler() { } @Override - protected OperationHandler getDeleteElementsHandler() { + protected OutputOperationHandler getDeleteElementsHandler() { return null; } diff --git a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedStore.java b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedStore.java index 8bd4372c0aa..80c01f077cb 100644 --- a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedStore.java +++ b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedStore.java @@ -441,8 +441,8 @@ protected OperationHandler getAddElementsHandler() { } @Override - protected OperationHandler getDeleteElementsHandler() { - return new FederatedOperationHandler<>(); + protected OutputOperationHandler getDeleteElementsHandler() { + return new FederatedOutputHandler<>(); } @Override From 43b0d99cd805e3768f203bcdb12c445ac6a63da1 Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Tue, 21 Jan 2025 11:28:46 +0000 Subject: [PATCH 16/29] sonarcloud --- .../uk/gov/gchq/gaffer/graph/GraphTest.java | 72 ++++++++----------- .../operation/impl/delete/DeleteElements.java | 1 - .../uk/gov/gchq/gaffer/store/StoreTest.java | 4 +- 3 files changed, 33 insertions(+), 44 deletions(-) diff --git a/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java b/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java index f07330b4590..80c68e3d165 100644 --- a/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java +++ b/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java @@ -342,7 +342,7 @@ public void shouldCreateNewContextInstanceWhenExecuteOperation(@Mock final Store @Test public void shouldCreateNewContextInstanceWhenExecuteOutputOperation(@Mock final Store store) - throws OperationException, IOException { + throws OperationException { // Given final Schema schema = new Schema(); given(store.getProperties()).willReturn(new StoreProperties()); @@ -365,7 +365,7 @@ public void shouldCreateNewContextInstanceWhenExecuteOutputOperation(@Mock final @Test public void shouldCreateNewContextInstanceWhenExecuteJob(@Mock final Store store) - throws OperationException, IOException { + throws OperationException { // Given final Schema schema = new Schema(); given(store.getProperties()).willReturn(new StoreProperties()); @@ -1065,7 +1065,7 @@ public void shouldNotSetGraphViewOnOperationWhenOperationIsNotAGet(@Mock final S } @Test - public void shouldThrowExceptionIfStoreClassPropertyIsNotSet() throws OperationException { + public void shouldThrowExceptionIfStoreClassPropertyIsNotSet() { try { new Graph.Builder() .config(new GraphConfig.Builder() @@ -1097,15 +1097,10 @@ public void shouldThrowExceptionIfGraphIdIsMissing() { } @Test - public void shouldThrowExceptionIfSchemaIsInvalid() throws OperationException { + public void shouldThrowExceptionIfSchemaIsInvalid() { final StoreProperties storeProperties = new StoreProperties(); storeProperties.setStoreClass(TestStoreImpl.class.getName()); - try { - new Graph.Builder() - .config(new GraphConfig.Builder() - .graphId(GRAPH_ID) - .build()) - .addSchema(new Schema.Builder() + final Schema invalidSchema = new Schema.Builder() .type("int", new TypeDefinition.Builder() .clazz(Integer.class) .aggregateFunction(new Sum()) @@ -1126,13 +1121,17 @@ public void shouldThrowExceptionIfSchemaIsInvalid() throws OperationException { .vertex("string") .property("p2", "int") .build()) - .build()) + .build(); + final GraphConfig config = new GraphConfig.Builder() + .graphId(GRAPH_ID) + .build(); + assertThatExceptionOfType(SchemaException.class) + .isThrownBy(() -> new Graph.Builder() + .config(config) + .addSchema(invalidSchema) .storeProperties(storeProperties) - .build(); - fail("exception expected"); - } catch (final SchemaException e) { - assertNotNull(e.getMessage()); - } + .build()) + .withMessageContaining("Schema is not valid"); } @Test @@ -1254,7 +1253,7 @@ private void writeToFile(final String schemaFile, final File dir) throws IOExcep } @Test - public void shouldThrowExceptionIfGraphIdIsInvalid(@Mock final StoreProperties properties) throws Exception { + public void shouldThrowExceptionIfGraphIdIsInvalid(@Mock final StoreProperties properties) { try { new Graph.Builder() .config(new GraphConfig.Builder() @@ -1355,8 +1354,7 @@ public void shouldBuildGraphUsingGraphIdAndLookupSchema() throws Exception { @Test public void shouldAddHooksVarArgsAndGetGraphHooks(@Mock final GraphHook graphHook1, - @Mock final Log4jLogger graphHook2) - throws Exception { + @Mock final Log4jLogger graphHook2) { // Given final StoreProperties storeProperties = new StoreProperties(); storeProperties.setStoreClass(TestStoreImpl.class.getName()); @@ -1382,8 +1380,7 @@ public void shouldAddHooksVarArgsAndGetGraphHooks(@Mock final GraphHook graphHoo @Test public void shouldAddHookAndGetGraphHooks(@Mock final GraphHook graphHook1, - @Mock final Log4jLogger graphHook3) - throws Exception { + @Mock final Log4jLogger graphHook3) { // Given final StoreProperties storeProperties = new StoreProperties(); storeProperties.setStoreClass(TestStore.class.getName()); @@ -1417,8 +1414,7 @@ public void shouldAddHookAndGetGraphHooks(@Mock final GraphHook graphHook1, @Test public void shouldAddNamedViewResolverHookAfterNamedOperationResolver(@Mock final GraphHook graphHook1, - @Mock final Log4jLogger graphHook2) - throws Exception { + @Mock final Log4jLogger graphHook2) { // Given final StoreProperties storeProperties = new StoreProperties(); storeProperties.setStoreClass(TestStore.class.getName()); @@ -1511,7 +1507,7 @@ public void shouldAddHookFromPathAndGetGraphHooks() throws Exception { } @Test - public void shouldBuildGraphFromConfigFile() throws Exception { + public void shouldBuildGraphFromConfigFile() { // Given final StoreProperties storeProperties = new StoreProperties(); storeProperties.setStoreClass(TestStoreImpl.class.getName()); @@ -1542,7 +1538,7 @@ public void shouldBuildGraphFromConfigFile() throws Exception { @Test public void shouldBuildGraphFromConfigAndMergeConfigWithExistingConfig(@Mock final GraphLibrary library1, @Mock final GraphLibrary library2, @Mock final GraphHook hook1, @Mock final GraphHook hook2, @Mock final GraphHook hook3) - throws Exception { + { // Given final StoreProperties storeProperties = new StoreProperties(); storeProperties.setStoreClass(TestStoreImpl.class.getName()); @@ -1592,7 +1588,7 @@ public void shouldBuildGraphFromConfigAndMergeConfigWithExistingConfig(@Mock fin @Test public void shouldBuildGraphFromConfigAndOverrideFields(@Mock final GraphLibrary library1, @Mock final GraphLibrary library2, @Mock final GraphHook hook1, @Mock final GraphHook hook2, @Mock final GraphHook hook3) - throws Exception { + { // Given final StoreProperties storeProperties = new StoreProperties(); storeProperties.setStoreClass(TestStoreImpl.class.getName()); @@ -1642,7 +1638,7 @@ public void shouldBuildGraphFromConfigAndOverrideFields(@Mock final GraphLibrary } @Test - public void shouldReturnClonedViewFromConfig() throws Exception { + public void shouldReturnClonedViewFromConfig() { // Given final StoreProperties storeProperties = new StoreProperties(); storeProperties.setStoreClass(TestStoreImpl.class.getName()); @@ -1840,8 +1836,7 @@ public void shouldCorrectlySetViewForNestedOperationChain() throws OperationExce } @Test - public void shouldThrowExceptionOnExecuteWithANullContext(@Mock final OperationChain opChain) - throws OperationException { + public void shouldThrowExceptionOnExecuteWithANullContext(@Mock final OperationChain opChain) { // Given final Context context = null; @@ -1860,8 +1855,7 @@ public void shouldThrowExceptionOnExecuteWithANullContext(@Mock final OperationC } @Test - public void shouldThrowExceptionOnExecuteJobWithANullContext(@Mock final OperationChain opChain) - throws OperationException { + public void shouldThrowExceptionOnExecuteJobWithANullContext(@Mock final OperationChain opChain) { // Given final Context context = null; @@ -1880,8 +1874,7 @@ public void shouldThrowExceptionOnExecuteJobWithANullContext(@Mock final Operati } @Test - public void shouldThrowExceptionOnExecuteWithANullUser(@Mock final OperationChain opChain) - throws OperationException { + public void shouldThrowExceptionOnExecuteWithANullUser(@Mock final OperationChain opChain) { // Given final User user = null; @@ -1900,8 +1893,7 @@ public void shouldThrowExceptionOnExecuteWithANullUser(@Mock final OperationChai } @Test - public void shouldThrowExceptionOnExecuteJobWithANullUser(@Mock final OperationChain opChain) - throws OperationException { + public void shouldThrowExceptionOnExecuteJobWithANullUser(@Mock final OperationChain opChain) { // Given final User user = null; @@ -1920,8 +1912,7 @@ public void shouldThrowExceptionOnExecuteJobWithANullUser(@Mock final OperationC } @Test - public void shouldThrowExceptionOnExecuteJobUsingJobWithANullContext(@Mock final OperationChain opChain) - throws OperationException { + public void shouldThrowExceptionOnExecuteJobUsingJobWithANullContext(@Mock final OperationChain opChain) { // Given final Context context = null; @@ -1942,7 +1933,7 @@ public void shouldThrowExceptionOnExecuteJobUsingJobWithANullContext(@Mock final } @Test - public void shouldThrowExceptionOnExecuteJobUsingJobWithANullOperation() throws OperationException { + public void shouldThrowExceptionOnExecuteJobUsingJobWithANullOperation() { // Given final Context context = new Context(); @@ -1963,7 +1954,7 @@ public void shouldThrowExceptionOnExecuteJobUsingJobWithANullOperation() throws } @Test - public void shouldThrowExceptionOnExecuteJobUsingJobWithANullJob() throws OperationException { + public void shouldThrowExceptionOnExecuteJobUsingJobWithANullJob() { // Given final Context context = new Context(); @@ -1984,8 +1975,7 @@ public void shouldThrowExceptionOnExecuteJobUsingJobWithANullJob() throws Operat } @Test - public void shouldThrowExceptionOnExecuteJobUsingJobWithANullUser(@Mock final OperationChain opChain) - throws OperationException { + public void shouldThrowExceptionOnExecuteJobUsingJobWithANullUser(@Mock final OperationChain opChain) { // Given final User user = null; diff --git a/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/delete/DeleteElements.java b/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/delete/DeleteElements.java index 10ae74b4578..6a6995493e9 100644 --- a/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/delete/DeleteElements.java +++ b/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/delete/DeleteElements.java @@ -29,7 +29,6 @@ import uk.gov.gchq.gaffer.operation.Validatable; import uk.gov.gchq.gaffer.operation.io.InputOutput; import uk.gov.gchq.gaffer.operation.io.MultiInput; -import uk.gov.gchq.gaffer.operation.io.Output; import uk.gov.gchq.gaffer.operation.serialisation.TypeReferenceImpl; import uk.gov.gchq.koryphe.Since; import uk.gov.gchq.koryphe.Summary; diff --git a/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java b/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java index 18e5bce7993..0f97f616817 100644 --- a/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java +++ b/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java @@ -420,7 +420,7 @@ public void shouldCloseOperationIfExceptionThrown(@Mock final StoreProperties pr } @Test - public void shouldThrowExceptionIfOperationChainIsInvalid(@Mock final StoreProperties properties) throws OperationException, StoreException { + public void shouldThrowExceptionIfOperationChainIsInvalid(@Mock final StoreProperties properties) throws StoreException { // Given final Schema schema = createSchemaMock(); final OperationChain opChain = new OperationChain<>(); @@ -806,7 +806,7 @@ public void shouldExecuteOperationChainJob(@Mock final StoreProperties propertie @Test public void shouldExecuteOperationJobAndWrapJobOperationInChain(@Mock final StoreProperties properties) - throws OperationException, InterruptedException, StoreException, SerialisationException { + throws OperationException, InterruptedException, StoreException { // Given final Operation operation = new GetVariables.Builder().variableNames(Lists.newArrayList()).build(); given(properties.getJobExecutorThreadCount()).willReturn(1); From 1c0e6fa80373492e900f16335437330fa8a8bfd1 Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Tue, 21 Jan 2025 11:39:26 +0000 Subject: [PATCH 17/29] more sonarcloud --- .../uk/gov/gchq/gaffer/graph/GraphTest.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java b/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java index 80c68e3d165..7fc4655f018 100644 --- a/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java +++ b/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java @@ -1253,7 +1253,7 @@ private void writeToFile(final String schemaFile, final File dir) throws IOExcep } @Test - public void shouldThrowExceptionIfGraphIdIsInvalid(@Mock final StoreProperties properties) { + void shouldThrowExceptionIfGraphIdIsInvalid(@Mock final StoreProperties properties) { try { new Graph.Builder() .config(new GraphConfig.Builder() @@ -1507,7 +1507,7 @@ public void shouldAddHookFromPathAndGetGraphHooks() throws Exception { } @Test - public void shouldBuildGraphFromConfigFile() { + void shouldBuildGraphFromConfigFile() { // Given final StoreProperties storeProperties = new StoreProperties(); storeProperties.setStoreClass(TestStoreImpl.class.getName()); @@ -1638,7 +1638,7 @@ public void shouldBuildGraphFromConfigAndOverrideFields(@Mock final GraphLibrary } @Test - public void shouldReturnClonedViewFromConfig() { + void shouldReturnClonedViewFromConfig() { // Given final StoreProperties storeProperties = new StoreProperties(); storeProperties.setStoreClass(TestStoreImpl.class.getName()); @@ -1836,7 +1836,7 @@ public void shouldCorrectlySetViewForNestedOperationChain() throws OperationExce } @Test - public void shouldThrowExceptionOnExecuteWithANullContext(@Mock final OperationChain opChain) { + void shouldThrowExceptionOnExecuteWithANullContext(@Mock final OperationChain opChain) { // Given final Context context = null; @@ -1855,7 +1855,7 @@ public void shouldThrowExceptionOnExecuteWithANullContext(@Mock final OperationC } @Test - public void shouldThrowExceptionOnExecuteJobWithANullContext(@Mock final OperationChain opChain) { + void shouldThrowExceptionOnExecuteJobWithANullContext(@Mock final OperationChain opChain) { // Given final Context context = null; @@ -1874,7 +1874,7 @@ public void shouldThrowExceptionOnExecuteJobWithANullContext(@Mock final Operati } @Test - public void shouldThrowExceptionOnExecuteWithANullUser(@Mock final OperationChain opChain) { + void shouldThrowExceptionOnExecuteWithANullUser(@Mock final OperationChain opChain) { // Given final User user = null; @@ -1893,7 +1893,7 @@ public void shouldThrowExceptionOnExecuteWithANullUser(@Mock final OperationChai } @Test - public void shouldThrowExceptionOnExecuteJobWithANullUser(@Mock final OperationChain opChain) { + void shouldThrowExceptionOnExecuteJobWithANullUser(@Mock final OperationChain opChain) { // Given final User user = null; @@ -1912,7 +1912,7 @@ public void shouldThrowExceptionOnExecuteJobWithANullUser(@Mock final OperationC } @Test - public void shouldThrowExceptionOnExecuteJobUsingJobWithANullContext(@Mock final OperationChain opChain) { + void shouldThrowExceptionOnExecuteJobUsingJobWithANullContext(@Mock final OperationChain opChain) { // Given final Context context = null; @@ -1933,7 +1933,7 @@ public void shouldThrowExceptionOnExecuteJobUsingJobWithANullContext(@Mock final } @Test - public void shouldThrowExceptionOnExecuteJobUsingJobWithANullOperation() { + void shouldThrowExceptionOnExecuteJobUsingJobWithANullOperation() { // Given final Context context = new Context(); @@ -1954,7 +1954,7 @@ public void shouldThrowExceptionOnExecuteJobUsingJobWithANullOperation() { } @Test - public void shouldThrowExceptionOnExecuteJobUsingJobWithANullJob() { + void shouldThrowExceptionOnExecuteJobUsingJobWithANullJob() { // Given final Context context = new Context(); @@ -1975,7 +1975,7 @@ public void shouldThrowExceptionOnExecuteJobUsingJobWithANullJob() { } @Test - public void shouldThrowExceptionOnExecuteJobUsingJobWithANullUser(@Mock final OperationChain opChain) { + void shouldThrowExceptionOnExecuteJobUsingJobWithANullUser(@Mock final OperationChain opChain) { // Given final User user = null; From bdc8bc00b5c15cc1a9554617b2f66d09a793c40f Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Tue, 21 Jan 2025 11:58:34 +0000 Subject: [PATCH 18/29] checkstyle --- .../src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java | 6 ++---- .../src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java b/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java index 7fc4655f018..e937196cce2 100644 --- a/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java +++ b/core/graph/src/test/java/uk/gov/gchq/gaffer/graph/GraphTest.java @@ -1537,8 +1537,7 @@ void shouldBuildGraphFromConfigFile() { @Test public void shouldBuildGraphFromConfigAndMergeConfigWithExistingConfig(@Mock final GraphLibrary library1, @Mock final GraphLibrary library2, - @Mock final GraphHook hook1, @Mock final GraphHook hook2, @Mock final GraphHook hook3) - { + @Mock final GraphHook hook1, @Mock final GraphHook hook2, @Mock final GraphHook hook3) { // Given final StoreProperties storeProperties = new StoreProperties(); storeProperties.setStoreClass(TestStoreImpl.class.getName()); @@ -1587,8 +1586,7 @@ public void shouldBuildGraphFromConfigAndMergeConfigWithExistingConfig(@Mock fin @Test public void shouldBuildGraphFromConfigAndOverrideFields(@Mock final GraphLibrary library1, @Mock final GraphLibrary library2, - @Mock final GraphHook hook1, @Mock final GraphHook hook2, @Mock final GraphHook hook3) - { + @Mock final GraphHook hook1, @Mock final GraphHook hook2, @Mock final GraphHook hook3) { // Given final StoreProperties storeProperties = new StoreProperties(); storeProperties.setStoreClass(TestStoreImpl.class.getName()); diff --git a/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java b/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java index 0f97f616817..48fe8981ff4 100644 --- a/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java +++ b/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java @@ -40,7 +40,6 @@ import uk.gov.gchq.gaffer.data.element.LazyEntity; import uk.gov.gchq.gaffer.data.element.id.EntityId; import uk.gov.gchq.gaffer.data.elementdefinition.exception.SchemaException; -import uk.gov.gchq.gaffer.exception.SerialisationException; import uk.gov.gchq.gaffer.jobtracker.Job; import uk.gov.gchq.gaffer.jobtracker.JobDetail; import uk.gov.gchq.gaffer.jobtracker.JobStatus; From 2b911f222b8625e5fdbe06de71465a0d984f8289 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Tue, 21 Jan 2025 12:03:49 +0000 Subject: [PATCH 19/29] tweak set usage --- .../gchq/gaffer/tinkerpop/GafferPopGraph.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java index 34d952ea637..72a31ffa2a7 100755 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraph.java @@ -429,7 +429,7 @@ public Iterator vertices(final Object... vertexIds) { .first(getOperation) .then(new Limit<>(variables.getElementsLimit(), true)) .build(); - final Set result = new HashSet<>(IterableUtils.toList(execute(chain))); + final List result = IterableUtils.toList(execute(chain)); // Warn of truncation if (result.size() >= variables.getElementsLimit()) { @@ -440,11 +440,11 @@ public Iterator vertices(final Object... vertexIds) { // Translate results to GafferPop elements final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); - final Set translatedResults = StreamSupport.stream(result.spliterator(), false) + final List translatedResults = StreamSupport.stream(result.spliterator(), false) .map(generator::_apply) .filter(Vertex.class::isInstance) .map(e -> (Vertex) e) - .collect(Collectors.toSet()); + .collect(Collectors.toList()); // Check for seeds that are not entities but are vertices on an edge (orphan vertices) if (variables.getIncludeOrphanedVertices()) { @@ -597,16 +597,16 @@ public Iterator edges(final Object... elementIds) { } // Run requested chain on the graph and buffer to set to avoid reusing iterator - final Set result = new HashSet<>(IterableUtils.toList(execute(getOperation))); + final List result = IterableUtils.toList(execute(getOperation)); // Translate results to Gafferpop elements final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); - final Set translatedResults = StreamSupport.stream(result.spliterator(), false) + final List translatedResults = StreamSupport.stream(result.spliterator(), false) .map(generator::_apply) .filter(Edge.class::isInstance) .map(e -> (Edge) e) .limit(variables.getElementsLimit()) - .collect(Collectors.toSet()); + .collect(Collectors.toList()); if (translatedResults.size() >= variables.getElementsLimit()) { LOGGER.warn( @@ -819,15 +819,15 @@ private Iterator verticesWithSeedsAndView(final List(variables.getElementsLimit(), true)) .build(); - final Set result = new HashSet<>(IterableUtils.toList(execute(chain))); + final List result = IterableUtils.toList(execute(chain)); // Translate results to Gafferpop elements final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); - final Set translatedResults = StreamSupport.stream(result.spliterator(), false) + final List translatedResults = StreamSupport.stream(result.spliterator(), false) .map(generator::_apply) .filter(GafferPopVertex.class::isInstance) .map(e -> (GafferPopVertex) e) - .collect(Collectors.toSet()); + .collect(Collectors.toList()); return translatedResults.iterator(); } @@ -855,16 +855,16 @@ private Iterator adjVerticesWithSeedsAndView(final List see } // GetAdjacentIds provides list of entity seeds so run a GetElements to get the actual Entities - final Set result = new HashSet<>(IterableUtils.toList( - execute(new OperationChain.Builder().first(builder.build()).build()))); + final List result = IterableUtils.toList( + execute(new OperationChain.Builder().first(builder.build()).build())); // Translate results to Gafferpop elements final GafferPopElementGenerator generator = new GafferPopElementGenerator(this); - final Set translatedResults = StreamSupport.stream(result.spliterator(), false) + final List translatedResults = StreamSupport.stream(result.spliterator(), false) .map(generator::_apply) .filter(Vertex.class::isInstance) .map(e -> (Vertex) e) - .collect(Collectors.toSet()); + .collect(Collectors.toList()); // Check for seeds that are not entities but are vertices on an edge (orphan vertices) if (variables.getIncludeOrphanedVertices()) { @@ -909,16 +909,16 @@ private Iterator edgesWithSeedsAndView(final List seeds, fina } // Run requested chain on the graph - final Set result = new HashSet<>(IterableUtils.toList(execute(getOperation))); + final List result = IterableUtils.toList(execute(getOperation)); // Translate results to Gafferpop elements final GafferPopElementGenerator generator = new GafferPopElementGenerator(this, true); - final Set translatedResults = StreamSupport.stream(result.spliterator(), false) + final List translatedResults = StreamSupport.stream(result.spliterator(), false) .map(generator::_apply) .filter(Edge.class::isInstance) .map(e -> (Edge) e) .limit(variables.getElementsLimit()) - .collect(Collectors.toSet()); + .collect(Collectors.toList()); return translatedResults.iterator(); } From b536dc6adc0c135627d540e3960c7fd472227968 Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Tue, 21 Jan 2025 13:37:46 +0000 Subject: [PATCH 20/29] sonarcloud --- .../src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java | 2 +- .../operation/handler/DeleteElementsHandlerTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java b/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java index 48fe8981ff4..6e614b7c279 100644 --- a/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java +++ b/core/store/src/test/java/uk/gov/gchq/gaffer/store/StoreTest.java @@ -439,7 +439,7 @@ public void shouldThrowExceptionIfOperationChainIsInvalid(@Mock final StorePrope } @Test - public void shouldCallDoUnhandledOperationWhenDoOperationWithUnknownOperationClass(@Mock final StoreProperties properties) throws Exception { + void shouldCallDoUnhandledOperationWhenDoOperationWithUnknownOperationClass(@Mock final StoreProperties properties) throws Exception { // Given final Schema schema = createSchemaMock(); final Operation operation = new SetVariable.Builder().variableName("aVariable").input("inputString").build(); diff --git a/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandlerTest.java b/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandlerTest.java index f2ce829cba9..3447b8c66ac 100644 --- a/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandlerTest.java +++ b/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandlerTest.java @@ -47,7 +47,7 @@ void shouldDoOperationValidated() throws Exception { AccumuloStore store = Mockito.mock(AccumuloStore.class); final Object elementCount = handler.doOperation(op, new Context(), store); - assertThat(elementCount).isEqualTo("Elements deleted: 1"); + assertThat(elementCount).isEqualTo(1L); verify(store).deleteElements(any(ValidatedElements.class)); } @@ -63,7 +63,7 @@ void shouldDoOperationNotValidated() throws Exception { AccumuloStore store = Mockito.mock(AccumuloStore.class); final Object elementCount = handler.doOperation(op, new Context(), store); - assertThat(elementCount).isEqualTo("Elements deleted: 1"); + assertThat(elementCount).isEqualTo(1L); verify(store).deleteElements(anyIterable()); } From c78b623c6ce6517a7938e6d5b5617282c4feac6c Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Tue, 21 Jan 2025 13:41:56 +0000 Subject: [PATCH 21/29] copyright --- .../integration/delete/AbstractDeletedElementsIT.java | 2 +- .../operation/handler/impl/FederatedDeleteElementsTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/integration/delete/AbstractDeletedElementsIT.java b/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/integration/delete/AbstractDeletedElementsIT.java index dc854b3d0ab..14688945e1f 100644 --- a/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/integration/delete/AbstractDeletedElementsIT.java +++ b/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/integration/delete/AbstractDeletedElementsIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2024 Crown Copyright + * Copyright 2018-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/store-implementation/federated-store/src/test/java/uk/gov/gchq/gaffer/federatedstore/operation/handler/impl/FederatedDeleteElementsTest.java b/store-implementation/federated-store/src/test/java/uk/gov/gchq/gaffer/federatedstore/operation/handler/impl/FederatedDeleteElementsTest.java index dec71f3d071..38c8c5db433 100644 --- a/store-implementation/federated-store/src/test/java/uk/gov/gchq/gaffer/federatedstore/operation/handler/impl/FederatedDeleteElementsTest.java +++ b/store-implementation/federated-store/src/test/java/uk/gov/gchq/gaffer/federatedstore/operation/handler/impl/FederatedDeleteElementsTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From c5617788b1b98fc362410c8af904ecdeeb29479e Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Tue, 21 Jan 2025 14:48:53 +0000 Subject: [PATCH 22/29] address comment --- .../accumulostore/operation/handler/DeleteElementsHandler.java | 2 +- .../uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java b/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java index d53b30c4d06..41a94280419 100644 --- a/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java +++ b/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java @@ -58,9 +58,9 @@ private Long deleteElements(final DeleteElements operation, final AccumuloStore final List elementsToDelete = new ArrayList<>(); for (final Element el : validatedElements) { elementsToDelete.add(el.toString()); - elementCount++; } LOGGER.debug("Deleting elements: {}", elementsToDelete); + elementCount = (long) elementsToDelete.size(); store.deleteElements(validatedElements); } catch (final StoreException e) { diff --git a/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java b/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java index 6f13d4b1797..6d2f1a7a563 100644 --- a/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java +++ b/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java @@ -64,8 +64,8 @@ private Long deleteElements(final Iterable elements, final Ma final List elementsToDelete = new ArrayList<>(); for (final Element el : elements) { elementsToDelete.add(el.toString()); - elementCount++; } + elementCount = (long) elementsToDelete.size(); LOGGER.debug("Deleting elements: {}", elementsToDelete); From 8bac723f607042bcae0ae78366dbd64989a69f6b Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Tue, 21 Jan 2025 14:54:07 +0000 Subject: [PATCH 23/29] refactor --- .../operation/handler/DeleteElementsHandler.java | 15 ++++++--------- .../mapstore/impl/DeleteElementsHandler.java | 16 ++++++---------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java b/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java index 41a94280419..545ed5b92d4 100644 --- a/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java +++ b/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java @@ -34,17 +34,16 @@ public class DeleteElementsHandler implements OutputOperationHandler { private static final Logger LOGGER = LoggerFactory.getLogger(DeleteElementsHandler.class); - private Long elementCount = 0L; + final List deletedElements = new ArrayList<>(); @Override public Long doOperation(final DeleteElements operation, final Context context, final Store store) throws OperationException { - deleteElements(operation, (AccumuloStore) store); - return elementCount; + return (long) deleteElements(operation, (AccumuloStore) store).size(); } - private Long deleteElements(final DeleteElements operation, final AccumuloStore store) + private List deleteElements(final DeleteElements operation, final AccumuloStore store) throws OperationException { try { final Iterable validatedElements; @@ -55,18 +54,16 @@ private Long deleteElements(final DeleteElements operation, final AccumuloStore validatedElements = operation.getInput(); } - final List elementsToDelete = new ArrayList<>(); for (final Element el : validatedElements) { - elementsToDelete.add(el.toString()); + deletedElements.add(el.toString()); } - LOGGER.debug("Deleting elements: {}", elementsToDelete); - elementCount = (long) elementsToDelete.size(); + LOGGER.debug("Deleting elements: {}", deletedElements); store.deleteElements(validatedElements); } catch (final StoreException e) { throw new OperationException("Failed to delete elements", e); } - return elementCount; + return deletedElements; } } diff --git a/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java b/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java index 6d2f1a7a563..62af8b91041 100644 --- a/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java +++ b/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java @@ -44,7 +44,7 @@ */ public class DeleteElementsHandler implements OutputOperationHandler { private static final Logger LOGGER = LoggerFactory.getLogger(DeleteElementsHandler.class); - private Long elementCount = 0L; + final List deletedElements = new ArrayList<>(); @Override public Long doOperation(final DeleteElements deleteElements, final Context context, final Store store) { @@ -53,21 +53,17 @@ public Long doOperation(final DeleteElements deleteElements, final Context conte elements = new ValidatedElements(elements, store.getSchema(), deleteElements.isSkipInvalidElements()); } - deleteElements(elements, (MapStore) store); - return elementCount; + return (long) deleteElements(elements, (MapStore) store).size(); } - private Long deleteElements(final Iterable elements, final MapStore mapStore) { + private List deleteElements(final Iterable elements, final MapStore mapStore) { final MapImpl mapImpl = mapStore.getMapImpl(); final Schema schema = mapStore.getSchema(); - final List elementsToDelete = new ArrayList<>(); for (final Element el : elements) { - elementsToDelete.add(el.toString()); + deletedElements.add(el.toString()); } - elementCount = (long) elementsToDelete.size(); - - LOGGER.debug("Deleting elements: {}", elementsToDelete); + LOGGER.debug("Deleting elements: {}", deletedElements); final int bufferSize = mapStore.getProperties().getIngestBufferSize(); @@ -80,7 +76,7 @@ private Long deleteElements(final Iterable elements, final Ma Streams.toBatches(elements, bufferSize).forEach(batch -> deleteBatch(mapImpl, schema, AggregatorUtil.ingestAggregate(batch, schema))); } - return elementCount; + return deletedElements; } private void deleteBatch(final MapImpl mapImpl, final Schema schema, final Iterable elements) { From b263383f4978c69bd098681807bf76c0e2115117 Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:29:10 +0000 Subject: [PATCH 24/29] fix --- .../operation/handler/DeleteElementsHandler.java | 8 ++++---- .../gchq/gaffer/mapstore/impl/DeleteElementsHandler.java | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java b/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java index 545ed5b92d4..34e39e47578 100644 --- a/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java +++ b/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/operation/handler/DeleteElementsHandler.java @@ -34,17 +34,17 @@ public class DeleteElementsHandler implements OutputOperationHandler { private static final Logger LOGGER = LoggerFactory.getLogger(DeleteElementsHandler.class); - final List deletedElements = new ArrayList<>(); @Override public Long doOperation(final DeleteElements operation, final Context context, final Store store) throws OperationException { - return (long) deleteElements(operation, (AccumuloStore) store).size(); + return deleteElements(operation, (AccumuloStore) store); } - private List deleteElements(final DeleteElements operation, final AccumuloStore store) + private Long deleteElements(final DeleteElements operation, final AccumuloStore store) throws OperationException { + final List deletedElements = new ArrayList<>(); try { final Iterable validatedElements; if (operation.isValidate()) { @@ -64,6 +64,6 @@ private List deleteElements(final DeleteElements operation, final Accumu throw new OperationException("Failed to delete elements", e); } - return deletedElements; + return (long) deletedElements.size(); } } diff --git a/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java b/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java index 62af8b91041..11946a28584 100644 --- a/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java +++ b/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/impl/DeleteElementsHandler.java @@ -44,7 +44,6 @@ */ public class DeleteElementsHandler implements OutputOperationHandler { private static final Logger LOGGER = LoggerFactory.getLogger(DeleteElementsHandler.class); - final List deletedElements = new ArrayList<>(); @Override public Long doOperation(final DeleteElements deleteElements, final Context context, final Store store) { @@ -53,12 +52,13 @@ public Long doOperation(final DeleteElements deleteElements, final Context conte elements = new ValidatedElements(elements, store.getSchema(), deleteElements.isSkipInvalidElements()); } - return (long) deleteElements(elements, (MapStore) store).size(); + return deleteElements(elements, (MapStore) store); } - private List deleteElements(final Iterable elements, final MapStore mapStore) { + private Long deleteElements(final Iterable elements, final MapStore mapStore) { final MapImpl mapImpl = mapStore.getMapImpl(); final Schema schema = mapStore.getSchema(); + final List deletedElements = new ArrayList<>(); for (final Element el : elements) { deletedElements.add(el.toString()); @@ -76,7 +76,7 @@ private List deleteElements(final Iterable elements, Streams.toBatches(elements, bufferSize).forEach(batch -> deleteBatch(mapImpl, schema, AggregatorUtil.ingestAggregate(batch, schema))); } - return deletedElements; + return (long) deletedElements.size(); } private void deleteBatch(final MapImpl mapImpl, final Schema schema, final Iterable elements) { From 3ec4c9bdeb2872c2d98e29f1b9d9e295b1608d25 Mon Sep 17 00:00:00 2001 From: p29876 <165825455+p29876@users.noreply.github.com> Date: Wed, 29 Jan 2025 14:45:56 +0000 Subject: [PATCH 25/29] Gh-3350: Simple fed store fixes (#3358) * Gh-3350 Fed store fixes * checkstyle * debug * undo --------- Co-authored-by: tb06904 <141412860+tb06904@users.noreply.github.com> --- .../predicate/user/NoAccessUserPredicate.java | 6 +- .../user/UnrestrictedAccessUserPredicate.java | 7 ++- .../federated/simple/FederatedUtils.java | 9 ++- .../handler/FederatedOperationHandler.java | 62 ++++++++++--------- .../handler/FederatedOutputHandler.java | 4 +- .../handler/SeparateOutputHandler.java | 8 ++- 6 files changed, 55 insertions(+), 41 deletions(-) diff --git a/core/access/src/main/java/uk/gov/gchq/gaffer/access/predicate/user/NoAccessUserPredicate.java b/core/access/src/main/java/uk/gov/gchq/gaffer/access/predicate/user/NoAccessUserPredicate.java index 29d42045a9f..b2d7dafe78e 100644 --- a/core/access/src/main/java/uk/gov/gchq/gaffer/access/predicate/user/NoAccessUserPredicate.java +++ b/core/access/src/main/java/uk/gov/gchq/gaffer/access/predicate/user/NoAccessUserPredicate.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Crown Copyright + * Copyright 2020-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,9 +21,11 @@ import uk.gov.gchq.koryphe.Summary; import uk.gov.gchq.koryphe.predicate.KoryphePredicate; +import java.io.Serializable; + @Since("1.13.1") @Summary("Predicate which never allows user access") -public class NoAccessUserPredicate extends KoryphePredicate { +public class NoAccessUserPredicate extends KoryphePredicate implements Serializable { @Override public boolean test(final User user) { return false; diff --git a/core/access/src/main/java/uk/gov/gchq/gaffer/access/predicate/user/UnrestrictedAccessUserPredicate.java b/core/access/src/main/java/uk/gov/gchq/gaffer/access/predicate/user/UnrestrictedAccessUserPredicate.java index 67bcde6b08c..a8bc560a190 100644 --- a/core/access/src/main/java/uk/gov/gchq/gaffer/access/predicate/user/UnrestrictedAccessUserPredicate.java +++ b/core/access/src/main/java/uk/gov/gchq/gaffer/access/predicate/user/UnrestrictedAccessUserPredicate.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Crown Copyright + * Copyright 2020-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package uk.gov.gchq.gaffer.access.predicate.user; import uk.gov.gchq.gaffer.user.User; @@ -20,9 +21,11 @@ import uk.gov.gchq.koryphe.Summary; import uk.gov.gchq.koryphe.predicate.KoryphePredicate; +import java.io.Serializable; + @Since("1.13.1") @Summary("A predicate which always allows a user access") -public class UnrestrictedAccessUserPredicate extends KoryphePredicate { +public class UnrestrictedAccessUserPredicate extends KoryphePredicate implements Serializable { @Override public boolean test(final User user) { return true; diff --git a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedUtils.java b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedUtils.java index e2ac2e20efb..5182b00c5b7 100644 --- a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedUtils.java +++ b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/FederatedUtils.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -91,10 +91,9 @@ public static OperationChain getValidOperationForGraph(final Operation operation && ((OperationView) operation).getView().hasGroups()) { // Update the view for the graph - ((OperationView) operation).setView( - getValidViewForGraph(((OperationView) operation).getView(), graphSerialisable)); - - updatedOperations.add(operation); + OperationView fixedOp = (OperationView) operation.shallowClone(); + fixedOp.setView(getValidViewForGraph(fixedOp.getView(), graphSerialisable)); + updatedOperations.add((Operation) fixedOp); // Recursively go into operation chains to make sure everything is fixed } else if (operation instanceof OperationChain) { diff --git a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOperationHandler.java b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOperationHandler.java index 2f4e06932b8..cf9aac4d956 100644 --- a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOperationHandler.java +++ b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOperationHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,12 +32,10 @@ import uk.gov.gchq.gaffer.store.Store; import uk.gov.gchq.gaffer.store.operation.handler.OperationHandler; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.LinkedList; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; /** * Main default handler for federated operations. Handles delegation to selected @@ -97,10 +95,15 @@ public class FederatedOperationHandler

implements Operation */ public static final String OPT_FIX_OP_LIMIT = "federated.fixOperationLimit"; + /** + * Default depth limit for fixing an operation chain. + */ + public static final int DFLT_FIX_OP_LIMIT = 5; + @Override public Object doOperation(final P operation, final Context context, final Store store) throws OperationException { LOGGER.debug("Running operation: {}", operation); - final int fixLimit = Integer.parseInt(operation.getOption(OPT_FIX_OP_LIMIT, "5")); + final int fixLimit = Integer.parseInt(operation.getOption(OPT_FIX_OP_LIMIT, String.valueOf(DFLT_FIX_OP_LIMIT))); // If the operation has output wrap and return using sub class handler if (operation instanceof Output) { @@ -153,40 +156,43 @@ public Object doOperation(final P operation, final Context context, final Store */ protected List getGraphsToExecuteOn(final Operation operation, final Context context, final FederatedStore store) throws OperationException { - // Use default graph IDs as fallback - List graphIds = store.getDefaultGraphIds(); - List graphsToExecute = new LinkedList<>(); - // If user specified graph IDs for this chain parse as comma separated list - if (operation.containsOption(OPT_GRAPH_IDS)) { - graphIds = Arrays.asList(operation.getOption(OPT_GRAPH_IDS).split(",")); - } else if (operation.containsOption(OPT_SHORT_GRAPH_IDS)) { - graphIds = Arrays.asList(operation.getOption(OPT_SHORT_GRAPH_IDS).split(",")); - } + List specifiedGraphIds = new ArrayList<>(); + List graphsToExecute = new ArrayList<>(); + // If user specified graph IDs for this chain parse as comma separated list + if (operation.containsOption(OPT_SHORT_GRAPH_IDS)) { + specifiedGraphIds.addAll(Arrays.asList(operation.getOption(OPT_SHORT_GRAPH_IDS).split(","))); + // Check legacy option + } else if (operation.containsOption(OPT_GRAPH_IDS)) { + specifiedGraphIds.addAll(Arrays.asList(operation.getOption(OPT_GRAPH_IDS).split(","))); // If a user has specified to just exclude some graphs then run all but them - if (operation.containsOption(OPT_EXCLUDE_GRAPH_IDS)) { - // Get all the IDs - graphIds = StreamSupport.stream(store.getAllGraphsAndAccess().spliterator(), false) - .map(Pair::getLeft) - .map(GraphSerialisable::getGraphId) - .collect(Collectors.toList()); - + } else if (operation.containsOption(OPT_EXCLUDE_GRAPH_IDS)) { + store.getAllGraphsAndAccess().forEach(pair -> specifiedGraphIds.add(pair.getLeft().getGraphId())); // Exclude the ones the user has specified - Arrays.asList(operation.getOption(OPT_EXCLUDE_GRAPH_IDS).split(",")).forEach(graphIds::remove); + Arrays.asList(operation.getOption(OPT_EXCLUDE_GRAPH_IDS).split(",")).forEach(specifiedGraphIds::remove); + } + + // Use default graph IDs as a fallback + if (specifiedGraphIds.isEmpty()) { + specifiedGraphIds.addAll(store.getDefaultGraphIds()); } - try { - // Get the corresponding graph serialisable - for (final String id : graphIds) { - LOGGER.debug("Will execute on Graph: {}", id); + // Get the corresponding graph serialisables + for (final String id : specifiedGraphIds) { + try { Pair pair = store.getGraphAccessPair(id); + // Check the user has access to the graph if (pair.getRight().hasReadAccess(context.getUser(), store.getProperties().getAdminAuth())) { + LOGGER.debug("User has access, will execute on Graph: '{}'", id); + // Create a new graph object from the serialised info graphsToExecute.add(pair.getLeft()); + } else { + LOGGER.warn("User does not have access, to Graph: '{}' it will be skipped", id); } + } catch (final CacheOperationException e) { + throw new OperationException("Failed to get Graph from cache: '" + id + "'", e); } - } catch (final CacheOperationException e) { - throw new OperationException("Failed to get Graphs from cache", e); } // Keep graphs sorted so results returned are predictable between runs diff --git a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOutputHandler.java b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOutputHandler.java index 99adb3c5e81..f5929ae60ac 100644 --- a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOutputHandler.java +++ b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOutputHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,7 +46,7 @@ public class FederatedOutputHandler

, O> @Override public O doOperation(final P operation, final Context context, final Store store) throws OperationException { - final int fixLimit = Integer.parseInt(operation.getOption(OPT_FIX_OP_LIMIT, "5")); + final int fixLimit = Integer.parseInt(operation.getOption(OPT_FIX_OP_LIMIT, String.valueOf(DFLT_FIX_OP_LIMIT))); List graphsToExecute = this.getGraphsToExecuteOn(operation, context, (FederatedStore) store); // No-op diff --git a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/SeparateOutputHandler.java b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/SeparateOutputHandler.java index a9e9bd49479..50b63876c3b 100644 --- a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/SeparateOutputHandler.java +++ b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/SeparateOutputHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,9 @@ import org.slf4j.LoggerFactory; import uk.gov.gchq.gaffer.federated.simple.FederatedStore; +import uk.gov.gchq.gaffer.federated.simple.FederatedUtils; import uk.gov.gchq.gaffer.graph.GraphSerialisable; +import uk.gov.gchq.gaffer.operation.OperationChain; import uk.gov.gchq.gaffer.operation.OperationException; import uk.gov.gchq.gaffer.operation.io.Output; import uk.gov.gchq.gaffer.store.Context; @@ -39,6 +41,7 @@ public class SeparateOutputHandler

, O> extends FederatedOper @Override public Map doOperation(final P operation, final Context context, final Store store) throws OperationException { + final int fixLimit = Integer.parseInt(operation.getOption(OPT_FIX_OP_LIMIT, String.valueOf(DFLT_FIX_OP_LIMIT))); List graphsToExecute = this.getGraphsToExecuteOn(operation, context, (FederatedStore) store); if (graphsToExecute.isEmpty()) { @@ -50,7 +53,8 @@ public Map doOperation(final P operation, final Context context, fina Map results = new HashMap<>(); for (final GraphSerialisable gs : graphsToExecute) { try { - results.put(gs.getGraphId(), gs.getGraph().execute(operation, context.getUser())); + OperationChain fixedChain = FederatedUtils.getValidOperationForGraph(operation, gs, 0, fixLimit); + results.put(gs.getGraphId(), gs.getGraph().execute(fixedChain, context.getUser())); } catch (final OperationException | UnsupportedOperationException e) { // Optionally skip this error if user has specified to do so LOGGER.error("Operation failed on graph: {}", gs.getGraphId()); From 72aeb9b025c251e826e9f24065a0712108df5423 Mon Sep 17 00:00:00 2001 From: p29876 <165825455+p29876@users.noreply.github.com> Date: Thu, 30 Jan 2025 13:25:36 +0000 Subject: [PATCH 26/29] Gh-3359: Reduce number of vertex lookups for edges (#3360) * gh-3359 Reduce number of vertex lookups for edges Stop inV/outV performing a full vertex lookup Add new methods for doing this when we want to traverse an Edge * headers * tidy and add tests * javadoc --- .../gchq/gaffer/tinkerpop/GafferPopEdge.java | 29 ++++++++++++---- .../gaffer/tinkerpop/GafferPopGraphIT.java | 33 +++++++++++++++++++ 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopEdge.java b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopEdge.java index a70792f6062..7e1b66979f3 100644 --- a/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopEdge.java +++ b/library/tinkerpop/src/main/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopEdge.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 Crown Copyright + * Copyright 2016-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -128,15 +128,17 @@ public Property propertyWithoutUpdate(final String key, final V value) { return newProperty; } + // This is the method TinkerPop uses when traversing edges + // lookup the 'full' vertices rather than returning the 'dummy' ID vertices @Override public Iterator vertices(final Direction direction) { switch (direction) { case OUT: - return IteratorUtils.of(outVertex()); + return IteratorUtils.of(lookupVertex(outVertex)); case IN: - return IteratorUtils.of(inVertex()); + return IteratorUtils.of(lookupVertex(inVertex)); default: - return IteratorUtils.of(outVertex(), inVertex()); + return IteratorUtils.of(lookupVertex(outVertex), lookupVertex(inVertex)); } } @@ -156,14 +158,27 @@ public String toString() { return StringFactory.edgeString(this); } + + /** + * Gets the outgoing vertex of the edge. + * + * Note: the returned vertex will not have any properties set - only the ID + * if you need the 'full' vertex use {@link #vertices(Direction)} + */ @Override public Vertex outVertex() { - return getVertex(outVertex); + return outVertex; } + /** + * Gets the incoming vertex of the edge. + * + * Note: the returned vertex will not have any properties set - only the ID + * if you need the 'full' vertex use {@link #vertices(Direction)} + */ @Override public Vertex inVertex() { - return getVertex(inVertex); + return inVertex; } /** @@ -209,7 +224,7 @@ private static GafferPopVertex getValidVertex(final Object vertex, final GafferP * @param vertex The vertex object or ID * @return A valid Vertex based on the supplied object or ID. */ - private Vertex getVertex(final GafferPopVertex vertex) { + private Vertex lookupVertex(final GafferPopVertex vertex) { OperationChain> findBasedOnID = new OperationChain.Builder() .first(new GetElements.Builder() .input(new EntitySeed(vertex.id())) diff --git a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java index 91fb1225863..39fa2858ae7 100644 --- a/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java +++ b/library/tinkerpop/src/test/java/uk/gov/gchq/gaffer/tinkerpop/GafferPopGraphIT.java @@ -37,6 +37,7 @@ import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; +import static uk.gov.gchq.gaffer.tinkerpop.util.modern.GafferPopModernTestUtils.AGE; import static uk.gov.gchq.gaffer.tinkerpop.util.modern.GafferPopModernTestUtils.CREATED; import static uk.gov.gchq.gaffer.tinkerpop.util.modern.GafferPopModernTestUtils.JOSH; import static uk.gov.gchq.gaffer.tinkerpop.util.modern.GafferPopModernTestUtils.KNOWS; @@ -277,6 +278,38 @@ void shouldGetEdgeById(String graph, GraphTraversalSource g) { }); } + @ParameterizedTest(name = TEST_NAME_FORMAT) + @MethodSource("provideTraversals") + void shouldLookupInVertex(String graph, GraphTraversalSource g) { + final List result = g.E("[1,knows,2]").inV().toList(); + + assertThat(result) + .hasSize(1); + + // Check that properties are set on the returned vertex + // i.e. a vertex lookup has been performed + Vertex vadas = result.get(0); + assertThat(vadas.id()).isEqualTo(VADAS.getId()); + assertThat(vadas.property(NAME).value()).isEqualTo(VADAS.getName()); + assertThat(vadas.property(AGE).value()).isEqualTo(VADAS.getAge()); + } + + @ParameterizedTest(name = TEST_NAME_FORMAT) + @MethodSource("provideTraversals") + void shouldLookupOutVertex(String graph, GraphTraversalSource g) { + final List result = g.E("[1,knows,2]").outV().toList(); + + assertThat(result) + .hasSize(1); + + // Check that properties are set on the returned vertex + // i.e. a vertex lookup has been performed + Vertex marko = result.get(0); + assertThat(marko.id()).isEqualTo(MARKO.getId()); + assertThat(marko.property(NAME).value()).isEqualTo(MARKO.getName()); + assertThat(marko.property(AGE).value()).isEqualTo(MARKO.getAge()); + } + @ParameterizedTest(name = TEST_NAME_FORMAT) @MethodSource("provideTraversals") void shouldAddV(String graph, GraphTraversalSource g) { From f370d3734756aeb4ae5a8017ea0e05c285626d4d Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Tue, 4 Feb 2025 10:52:10 +0000 Subject: [PATCH 27/29] gh-3366: Address JoinIT failures (#3370) * fix joinITs and add defaultGraphIds to AllGraphInfo * copyright --------- Co-authored-by: p29876 <165825455+p29876@users.noreply.github.com> --- .../merge/operator/ElementAggregateOperator.java | 4 ++-- .../operation/handler/get/GetAllGraphInfoHandler.java | 5 ++++- .../simple/integration/FederatedStoreITs.java | 11 +++-------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/merge/operator/ElementAggregateOperator.java b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/merge/operator/ElementAggregateOperator.java index 40fda531dbd..fd6655e57e6 100644 --- a/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/merge/operator/ElementAggregateOperator.java +++ b/store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/merge/operator/ElementAggregateOperator.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -67,7 +67,7 @@ public Iterable apply(final Iterable update, final Iterable doOperation(final GetAllGraphInfo operation, final Context context, final Store store) @@ -70,6 +71,8 @@ public Map doOperation(final GetAllGraphInfo operation, final Co allGraphInfo.put(graph.getConfig().getGraphId(), graphInfo); }); + allGraphInfo.put(DEFAULT_GRAPH_IDS, ((FederatedStore) store).getDefaultGraphIds()); + return allGraphInfo; } } diff --git a/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/integration/FederatedStoreITs.java b/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/integration/FederatedStoreITs.java index 6f1fdd35d70..f2f1c50a465 100644 --- a/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/integration/FederatedStoreITs.java +++ b/store-implementation/simple-federated-store/src/test/java/uk/gov/gchq/gaffer/federated/simple/integration/FederatedStoreITs.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2024-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,15 +48,10 @@ public class FederatedStoreITs extends AbstractStoreITs { new SimpleEntry<>("shouldGetAllElementsWithFilterWithoutSummarisation", "GetAllElementsIT - count value is duplicated, elements correct otherwise"), new SimpleEntry<>("shouldImportFromFileThenCorrectlyExportToFile", "ImportExportCsvIT - Investigate further"), - new SimpleEntry<>("shouldRightKeyOuterJoin", "JoinIT - Missing results"), - new SimpleEntry<>("shouldLeftKeyInnerJoin", "JoinIT - Missing results"), - new SimpleEntry<>("shouldRightKeyInnerJoin", "JoinIT - Missing results"), - new SimpleEntry<>("shouldRightKeyFullJoin", "JoinIT - Missing results"), - new SimpleEntry<>("shouldLeftKeyOuterJoin", "JoinIT - Missing results"), new SimpleEntry<>("shouldReturnDuplicateEdgesWhenNoAggregationIsUsed", - "NoAggregationIT - Need to ensure that when schema has aggregation false that this is applied"), + "NoAggregationIT - Will return one from each graph as they contain the exact same elements"), new SimpleEntry<>("shouldReturnDuplicateEntitiesWhenNoAggregationIsUsed", - "NoAggregationIT - Need to ensure that when schema has aggregation false that this is applied"), + "NoAggregationIT - Will return one from each graph as they contain the exact same elements"), new SimpleEntry<>("shouldAggregateOnlyRequiredGroupsWithQueryTimeAggregation", "PartAggregationIT - Investigate further"), new SimpleEntry<>("shouldAggregateOnlyRequiredGroups", "PartAggregationIT - Investigate further"), new SimpleEntry<>("shouldApplyPostOpAggregation", "SchemaMigrationIT - Need to apply schema aggregation choices")) From 0c98f2cfd723e5dd01eca1840f1b5247478137de Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Tue, 4 Feb 2025 11:49:33 +0000 Subject: [PATCH 28/29] Gh-3368: Update Koryphe version (#3369) * update koryphe version * test fix and copyright --------- Co-authored-by: wb36499 <166839644+wb36499@users.noreply.github.com> --- NOTICES | 2 +- .../gchq/gaffer/store/operation/handler/MapHandlerTest.java | 6 ++++-- pom.xml | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/NOTICES b/NOTICES index 8362f7a1385..400f2aee8c4 100644 --- a/NOTICES +++ b/NOTICES @@ -21,7 +21,7 @@ and their licenses, below. For information on the dependencies of these dependen projects below. -Koryphe (uk.gov.gchq.koryphe:koryphe:2.5.2): +Koryphe (uk.gov.gchq.koryphe:koryphe:2.6.0): - Apache License, Version 2.0 diff --git a/core/store/src/test/java/uk/gov/gchq/gaffer/store/operation/handler/MapHandlerTest.java b/core/store/src/test/java/uk/gov/gchq/gaffer/store/operation/handler/MapHandlerTest.java index a0a42738198..e2bdbcd5088 100644 --- a/core/store/src/test/java/uk/gov/gchq/gaffer/store/operation/handler/MapHandlerTest.java +++ b/core/store/src/test/java/uk/gov/gchq/gaffer/store/operation/handler/MapHandlerTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2021 Crown Copyright + * Copyright 2017-2025 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package uk.gov.gchq.gaffer.store.operation.handler; import com.google.common.collect.Lists; @@ -255,11 +256,12 @@ public void shouldFlatMapMultipleObjects() throws OperationException { // Given final MapHandler>, Iterable> handler = new MapHandler<>(); + final Function>, Iterable> iterableConcat = new IterableConcat<>(); final Map>, Iterable> operation = new Map.Builder>>() .input(Arrays.asList( Arrays.asList(1, 2), Arrays.asList(3, 4))) - .first(new IterableConcat<>()) + .first((Function>, Iterable>) iterableConcat) .build(); // When diff --git a/pom.xml b/pom.xml index 2d0a8eb30c0..0777852f637 100644 --- a/pom.xml +++ b/pom.xml @@ -1,6 +1,6 @@ - 2.3.2-SNAPSHOT + 2.3.2 false UTF-8