Skip to content

Commit 241a084

Browse files
lampajrlampajr
committed
Move dataset views mgmt into Horreum logic
As part of this refactoring I removed table view_recalc_queue and: Triggers - dsv_after_delete - dsv_after_update - recalc_dataset_view - dsv_after_insert Functions: - dsv_after_vc_delete_func - dsv_after_vc_update_func - recalc_dataset_view - dsv_after_lv_insert_func Signed-off-by: Andrea Lamparelli <[email protected]>
1 parent 5925c2f commit 241a084

6 files changed

Lines changed: 108 additions & 13 deletions

File tree

‎horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/DatasetServiceImpl.java‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
import jakarta.ws.rs.DefaultValue;
1919

2020
import org.hibernate.Hibernate;
21+
import org.hibernate.ScrollMode;
22+
import org.hibernate.ScrollableResults;
2123
import org.hibernate.Session;
2224
import org.hibernate.query.NativeQuery;
2325
import org.hibernate.type.StandardBasicTypes;
@@ -445,9 +447,8 @@ void calculateLabelValues(int testId, int datasetId, int queryLabelId, boolean i
445447
"Evaluation of label %s failed: '%s' Code:<pre>%s</pre>", row[0], e.getMessage(), jsCode),
446448
(out) -> logMessage(datasetId, PersistentLogDAO.DEBUG, "Output while calculating labels: <pre>%s</pre>", out));
447449

448-
//Create new dataset views from the recently created label values
449-
em.createNativeQuery("DELETE FROM dataset_view WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
450-
em.createNativeQuery("call calc_dataset_view(?1);").setParameter(1, datasetId).executeUpdate();
450+
// create new dataset views from the recently created label values
451+
calcDatasetViews(datasetId);
451452

452453
createFingerprint(datasetId, testId);
453454
mediator.updateLabels(new Dataset.LabelsUpdatedEvent(testId, datasetId, isRecalculation));
@@ -456,10 +457,48 @@ void calculateLabelValues(int testId, int datasetId, int queryLabelId, boolean i
456457
testId, new Dataset.LabelsUpdatedEvent(testId, datasetId, isRecalculation)));
457458
}
458459

460+
@Transactional
461+
public void calcDatasetViews(int datasetId) {
462+
// TODO(user) move calc_dataset_view into Horreum business logic see https://github.com/hibernate/hibernate-orm/pull/7457
463+
em.createNativeQuery("DELETE FROM dataset_view WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
464+
em.createNativeQuery("call calc_dataset_view(?1, NULL);").setParameter(1, datasetId).executeUpdate();
465+
}
466+
467+
@Transactional
468+
@SuppressWarnings("unchecked")
469+
public void calcDatasetViewsByTestAndView(int testId, int viewId) {
470+
// delete all dataset views associated to the provided viewId and testId
471+
// for new views it won't delete anything
472+
em.createNativeQuery(
473+
"DELETE FROM dataset_view WHERE view_id = ?1 AND dataset_id IN (SELECT id FROM dataset WHERE testid = ?2)")
474+
.setParameter(1, viewId)
475+
.setParameter(2, testId)
476+
.executeUpdate();
477+
478+
// re-create dataset views associated to the provided viewId
479+
try (ScrollableResults<Integer> datasetIds = em
480+
.createNativeQuery("SELECT id FROM dataset WHERE testid = ?1")
481+
.setParameter(1, testId)
482+
.unwrap(NativeQuery.class)
483+
.setReadOnly(false)
484+
.setFetchSize(100)
485+
.scroll(ScrollMode.FORWARD_ONLY)) {
486+
while (datasetIds.next()) {
487+
int datasetId = datasetIds.get();
488+
log.tracef("Recalculate dataset views for view %d and dataset %d", viewId, datasetId);
489+
em.createNativeQuery("call calc_dataset_view(?1, ?2);")
490+
.setParameter(1, datasetId)
491+
.setParameter(2, viewId)
492+
.executeUpdate();
493+
}
494+
}
495+
}
496+
459497
@Transactional
460498
public void deleteDataset(int datasetId) {
461499
em.createNativeQuery("DELETE FROM label_values WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
462500
em.createNativeQuery("DELETE FROM dataset_schemas WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
501+
em.createNativeQuery("DELETE FROM dataset_view WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
463502
em.createNativeQuery("DELETE FROM fingerprint WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
464503
em.createNativeQuery("DELETE FROM dataset WHERE id = ?1").setParameter(1, datasetId).executeUpdate();
465504
}

‎horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/RunServiceImpl.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1082,9 +1082,8 @@ public Map<Integer, String> updateSchema(int id, String path, String schemaUri)
10821082
@Transactional
10831083
@Override
10841084
public List<Integer> recalculateDatasets(int runId) {
1085-
transform(runId, true);
1086-
return session.createNativeQuery("SELECT id FROM dataset WHERE runid = ? ORDER BY ordinal", Integer.class)
1087-
.setParameter(1, runId).getResultList();
1085+
log.infof("Transforming run id %d", runId);
1086+
return transform(runId, true);
10881087
}
10891088

10901089
@RolesAllowed(Roles.ADMIN)

‎horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/TestServiceImpl.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -669,6 +669,7 @@ public void recalculateDatasets(int testId) {
669669
status.finished++;
670670
status.datasets += newDatasets;
671671
if (status.finished == status.totalRuns) {
672+
log.infof("Datasets recalculation for test %d (%s) completed", testId, test.name);
672673
recalculations.remove(testId, status);
673674
}
674675
}

‎horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/UIServiceImpl.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ public class UIServiceImpl implements UIService {
2525
@Inject
2626
TestServiceImpl testService;
2727

28+
@Inject
29+
DatasetServiceImpl datasetService;
30+
2831
@Override
2932
@RolesAllowed("tester")
3033
@WithRoles
@@ -64,6 +67,9 @@ private View doUpdate(TestDAO test, ViewDAO view) {
6467
test.views.add(view);
6568
test.persist();
6669
em.flush();
70+
71+
// update datasets views
72+
datasetService.calcDatasetViewsByTestAndView(test.id, view.id);
6773
return ViewMapper.from(view);
6874
}
6975

@@ -80,6 +86,8 @@ public void deleteView(int testId, int viewId) {
8086
if (!test.views.removeIf(v -> v.id == viewId)) {
8187
throw ServiceException.badRequest("Test does not contain this view!");
8288
}
89+
// remove dataset views records linked to this view
90+
em.createNativeQuery("DELETE FROM dataset_view WHERE view_id = ?1").setParameter(1, viewId).executeUpdate();
8391
// the orphan removal doesn't work for some reason, we need to remove if manually
8492
ViewDAO.deleteById(viewId);
8593
test.persist();

‎horreum-backend/src/main/resources/db/changeLog.xml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4757,5 +4757,43 @@
47574757
WHERE backendconfig.id = updated.id;
47584758
</sql>
47594759
</changeSet>
4760+
<changeSet id="127" author="lampajr">
4761+
<validCheckSum>ANY</validCheckSum>
4762+
<sql>
4763+
-- drop triggers
4764+
DROP TRIGGER IF EXISTS dsv_after_delete ON viewcomponent;
4765+
DROP TRIGGER IF EXISTS dsv_after_update ON viewcomponent;
4766+
DROP TRIGGER IF EXISTS recalc_dataset_view ON view_recalc_queue;
4767+
DROP TRIGGER IF EXISTS dsv_after_insert ON label_values;
4768+
4769+
-- drop functions
4770+
DROP FUNCTION dsv_after_vc_delete_func;
4771+
DROP FUNCTION dsv_after_vc_update_func;
4772+
DROP FUNCTION recalc_dataset_view;
4773+
DROP FUNCTION dsv_after_lv_insert_func;
47604774

4775+
-- drop view_recalc_queue table as not needed anymore
4776+
DROP TABLE view_recalc_queue;
4777+
4778+
-- still need db procedure until https://hibernate.atlassian.net/browse/HHH-17314 is fixed in quarkus
4779+
-- viewId can be NULL
4780+
CREATE OR REPLACE PROCEDURE calc_dataset_view(datasetId bigint, viewId bigint DEFAULT NULL) AS $$
4781+
BEGIN
4782+
WITH view_agg AS (
4783+
SELECT
4784+
vc.view_id, vc.id as vcid, array_agg(DISTINCT label.id) as label_ids, jsonb_object_agg(label.name, lv.value) as value FROM dataset_schemas ds
4785+
JOIN label ON label.schema_id = ds.schema_id
4786+
JOIN viewcomponent vc ON vc.labels ? label.name
4787+
JOIN label_values lv ON lv.label_id = label.id AND lv.dataset_id = ds.dataset_id
4788+
WHERE ds.dataset_id = datasetId AND (viewId IS NULL OR vc.view_id = viewId)
4789+
AND vc.view_id IN (SELECT view.id FROM view JOIN dataset ON view.test_id = dataset.testid WHERE dataset.id = datasetId)
4790+
GROUP BY vc.view_id, vcid
4791+
)
4792+
INSERT INTO dataset_view (dataset_id, view_id, label_ids, value)
4793+
SELECT datasetId, view_id, array_agg(DISTINCT label_id), jsonb_object_agg(vcid, value) FROM view_agg, unnest(label_ids) as label_id
4794+
GROUP BY view_id;
4795+
END
4796+
$$ LANGUAGE plpgsql;
4797+
</sql>
4798+
</changeSet>
47614799
</databaseChangeLog>

‎horreum-backend/src/test/java/io/hyperfoil/tools/horreum/svc/DatasetServiceTest.java‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,15 @@
3434
import io.hyperfoil.tools.horreum.entity.data.*;
3535
import io.hyperfoil.tools.horreum.entity.data.ViewComponentDAO;
3636
import io.hyperfoil.tools.horreum.mapper.LabelMapper;
37+
import io.hyperfoil.tools.horreum.mapper.ViewMapper;
3738
import io.hyperfoil.tools.horreum.server.CloseMe;
3839
import io.hyperfoil.tools.horreum.test.HorreumTestProfile;
3940
import io.hyperfoil.tools.horreum.test.PostgresResource;
4041
import io.quarkus.test.common.QuarkusTestResource;
4142
import io.quarkus.test.junit.QuarkusTest;
4243
import io.quarkus.test.junit.TestProfile;
4344
import io.quarkus.test.oidc.server.OidcWiremockTestResource;
45+
import io.restassured.response.Response;
4446

4547
@QuarkusTest
4648
@QuarkusTestResource(PostgresResource.class)
@@ -380,13 +382,21 @@ public void testDatasetView() {
380382
assertTrue(ids.contains(labelA));
381383
assertTrue(ids.contains(labelB));
382384

383-
Util.withTx(tm, () -> {
384-
try (CloseMe ignored = roleManager.withRoles(Arrays.asList(TESTER_ROLES))) {
385-
int vcs = em.createNativeQuery("UPDATE viewcomponent SET labels = '[\"a\",\"b\"]'").executeUpdate();
386-
assertEquals(2, vcs);
387-
}
388-
return null;
389-
});
385+
// there must be only one result here!
386+
ViewDAO dbView = (ViewDAO) ViewDAO.findAll().list().get(0);
387+
assertEquals(2, dbView.components.size());
388+
389+
View updateView = ViewMapper.from(dbView);
390+
ArrayNode vcLabels = (ArrayNode) updateView.components.stream().filter(vc -> vc.labels.size() == 1).findFirst()
391+
.orElseThrow().labels;
392+
vcLabels.add("b");
393+
394+
Response response = jsonRequest()
395+
.auth()
396+
.oauth2(getTesterToken())
397+
.body(updateView)
398+
.post("/api/ui/view");
399+
assertEquals(200, response.getStatusCode());
390400

391401
JsonNode updated = fetchDatasetsByTest(test.id);
392402
JsonNode updatedView = updated.get("datasets").get(0).get("view");

0 commit comments

Comments
 (0)