Skip to content

Commit 22d9d78

Browse files
iagaponenkoiagaponenko
committed
NO SUBCHUNK MATERIALIZATION: Temporary commit
The work in progress. Still working on qana::RelationGraph to move away from query rewriting for materialized sub-chunks and inject constaints for sub-chunks on the chunk and chunk overlap tables. The uit test still fails in: qproc/testQueryAnaGeneral:246 qproc/testQueryAnaGeneral:1296 More details can be found in the file a.txt that is located at the home folder of the package.
1 parent 90cc74e commit 22d9d78

File tree

6 files changed

+90
-46
lines changed

6 files changed

+90
-46
lines changed

‎src/qana/RelationGraph.cc‎

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -79,10 +79,12 @@ using query::QueryTemplate;
7979
using query::SelectStmt;
8080
using query::TableRef;
8181
using query::TableRefList;
82+
using query::ValueExpr;
8283
using query::ValueExprPtr;
8384
using query::ValueExprPtrVector;
8485
using query::ValueFactor;
8586
using query::ValueFactorPtr;
87+
using query::WhereClause;
8688

8789
std::ostream& operator<<(std::ostream& out, Vertex const& vertex) {
8890
out << "Vertex("
@@ -155,6 +157,20 @@ void Vertex::insert(Edge const& e) {
155157
}
156158
}
157159

160+
std::shared_ptr<CompPredicate> Vertex::makeSubChunkCompPredicate() const {
161+
auto const table = tr.hasAlias() ? tr.getAlias() : info->table;
162+
return std::make_shared<CompPredicate>(ValueExpr::newColumnExpr("", table, "", SUB_CHUNK_COLUMN),
163+
CompPredicate::EQUALS_OP,
164+
ValueExpr::newSimple(ValueFactor::newConstFactor(SUBCHUNK_TAG)));
165+
}
166+
167+
std::shared_ptr<CompPredicate> Vertex::makeOverlapCompPredicate() const {
168+
auto const table = tr.hasAlias() ? tr.getAlias() : info->table + "FullOverlap_" + CHUNK_TAG;
169+
return std::make_shared<CompPredicate>(ValueExpr::newColumnExpr("", table, "", SUB_CHUNK_COLUMN),
170+
CompPredicate::EQUALS_OP,
171+
ValueExpr::newSimple(ValueFactor::newConstFactor(SUBCHUNK_TAG)));
172+
}
173+
158174
// ----------------------------------------------------------------
159175
// RelationGraph implementation
160176

@@ -1064,7 +1080,11 @@ void RelationGraph::rewrite(SelectStmtPtrVector& outputs, QueryMapping& mapping)
10641080
for (ListIter i = _vertices.begin(), e = _vertices.end(); i != e; ++i) {
10651081
if (i->info->kind == TableInfo::DIRECTOR) {
10661082
if (i->overlap == 0.0) {
1067-
i->rewriteAsSubChunkTemplate();
1083+
i->rewriteAsChunkTemplate();
1084+
auto predicate = i->makeSubChunkCompPredicate();
1085+
BoolFactor::Ptr bfactor = std::make_shared<BoolFactor>();
1086+
bfactor->_terms.push_back(predicate);
1087+
_query->getWhereClause(true).prependAndTerm(bfactor);
10681088
}
10691089
DbTable dbTable(i->info->database, i->info->table);
10701090
LOGS(_log, LOG_LVL_TRACE, "rewrite db=" << dbTable.db << " table=" << dbTable.table);
@@ -1073,24 +1093,31 @@ void RelationGraph::rewrite(SelectStmtPtrVector& outputs, QueryMapping& mapping)
10731093
}
10741094
unsigned n = static_cast<unsigned>(overlapRefs.size());
10751095
unsigned numPermutations = 1 << n;
1076-
// Each director requiring overlap must be rewritten as both a sub-chunk
1077-
// template and an overlap sub-chunk template. There are 2ⁿ different
1096+
// Each director requiring overlap must be rewritten as both a chunk
1097+
// template and chunk overlap template. There are 2ⁿ different
10781098
// template permutations for n directors requiring overlap; generate them
10791099
// all.
10801100
for (unsigned p = 0; p < numPermutations; ++p) {
1101+
// Given the use of shared_ptr by the IR classes, we could shallow
1102+
// copy everything except the FromList as an optimization. But then
1103+
// code which mutates a particular SelectStmt might in fact mutate
1104+
// many SelectStmt objects. If the IR classes were copy-on-write,
1105+
// this wouldn't be an issue.
1106+
auto query = _query->clone();
10811107
for (unsigned i = 0; i < n; ++i) {
1108+
std::shared_ptr<CompPredicate> predicate;
10821109
if ((p & (1 << i)) != 0) {
10831110
overlapRefs[i]->rewriteAsOverlapTemplate();
1111+
predicate = overlapRefs[i]->makeOverlapCompPredicate();
10841112
} else {
1085-
overlapRefs[i]->rewriteAsSubChunkTemplate();
1113+
overlapRefs[i]->rewriteAsChunkTemplate();
1114+
predicate = overlapRefs[i]->makeSubChunkCompPredicate();
10861115
}
1116+
BoolFactor::Ptr bfactor = std::make_shared<BoolFactor>();
1117+
bfactor->_terms.push_back(predicate);
1118+
query->getWhereClause(true).prependAndTerm(bfactor);
10871119
}
1088-
// Given the use of shared_ptr by the IR classes, we could shallow
1089-
// copy everything except the FromList as an optimization. But then
1090-
// code which mutates a particular SelectStmt might in fact mutate
1091-
// many SelectStmt objects. If the IR classes were copy-on-write,
1092-
// this wouldn't be an issue.
1093-
outputs.push_back(_query->clone());
1120+
outputs.push_back(query);
10941121
}
10951122
}
10961123

‎src/qana/RelationGraph.h‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -494,6 +494,7 @@
494494
namespace lsst::qserv {
495495
namespace query {
496496
class ColumnRef;
497+
class CompPredicate;
497498
class QueryContext;
498499
class SelectStmt;
499500
} // namespace query
@@ -577,19 +578,20 @@ struct Vertex {
577578
tr.setTable(info->getChunkTemplate());
578579
}
579580

580-
/// `rewriteAsSubChunkTemplate` rewrites `tr` to contain a sub-chunk
581-
/// specific name pattern.
582-
void rewriteAsSubChunkTemplate() {
583-
tr.setDb(info->getSubChunkDb());
584-
tr.setTable(info->getSubChunkTemplate());
585-
}
586-
587581
/// `rewriteAsOverlapTemplate` rewrites `tr` to contain an overlap
588582
/// sub-chunk specific name pattern.
589583
void rewriteAsOverlapTemplate() {
590-
tr.setDb(info->getSubChunkDb());
584+
tr.setDb(info->database);
591585
tr.setTable(info->getOverlapTemplate());
592586
}
587+
588+
/// `makeSubChunkCompPredicate` creates a shared pointer to a new
589+
/// comparison predicate for the sub-chunk column.
590+
std::shared_ptr<query::CompPredicate> makeSubChunkCompPredicate() const;
591+
592+
/// `makeOverlapCompPredicate` creates a shared pointer to a new
593+
/// comparison predicate for the sub-chunk column in the overlap table.
594+
std::shared_ptr<query::CompPredicate> makeOverlapCompPredicate() const;
593595
};
594596

595597
std::ostream& operator<<(std::ostream& out, Vertex const& vertex);

‎src/qana/TableInfo.h‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@
6969
#include <vector>
7070

7171
// Qserv headers
72-
#include "global/constants.h" // for SUBCHUNKDB_PREFIX
72+
#include "global/constants.h"
7373

7474
// Forward declarations
7575
namespace lsst::qserv::query {
@@ -134,12 +134,8 @@ struct TableInfo {
134134
return false;
135135
}
136136

137-
std::string const getSubChunkDb() const { return SUBCHUNKDB_PREFIX + database + "_" + CHUNK_TAG; }
138137
std::string const getChunkTemplate() const { return table + "_" + CHUNK_TAG; }
139-
std::string const getSubChunkTemplate() const { return table + "_" + CHUNK_TAG + "_" + SUBCHUNK_TAG; }
140-
std::string const getOverlapTemplate() const {
141-
return table + "FullOverlap_" + CHUNK_TAG + "_" + SUBCHUNK_TAG;
142-
}
138+
std::string const getOverlapTemplate() const { return table + "FullOverlap_" + CHUNK_TAG; }
143139

144140
virtual void dump(std::ostream& os) const;
145141
};

‎src/qproc/testQueryAnaGeneral.cc‎

Lines changed: 35 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -199,16 +199,20 @@ BOOST_AUTO_TEST_CASE(RestrictorNeighborCount) {
199199
"where qserv_areaspec_box(6,6,7,7) AND rFlux_PS<0.005 AND "
200200
"scisql_angSep(o1.ra_Test,o1.decl_Test,o2.ra_Test,o2.decl_Test) < 0.001;";
201201
std::string expected_100_subchunk_core =
202-
"SELECT count(*) AS `QS1_COUNT` FROM `Subchunks_LSST_100`.`Object_100_%S\007S%` AS "
203-
"`o1`,`Subchunks_LSST_100`.`Object_100_%S\007S%` AS `o2` "
204-
"WHERE scisql_s2PtInBox(`o1`.`ra_Test`,`o1`.`decl_Test`,6,6,7,7)=1 AND "
202+
"SELECT count(*) AS `QS1_COUNT` FROM `LSST`.`Object_100` AS "
203+
"`o1`,`LSST`.`Object_100` AS `o2` "
204+
"WHERE `o2`.`subChunkId`=%S\007S% "
205+
"AND `o1`.`subChunkId`=%S\007S% "
206+
"AND scisql_s2PtInBox(`o1`.`ra_Test`,`o1`.`decl_Test`,6,6,7,7)=1 AND "
205207
"scisql_s2PtInBox(`o2`.`ra_Test`,`o2`.`decl_Test`,6,6,7,7)=1 AND "
206208
"`o1`.`rFlux_PS`<0.005 AND "
207209
"scisql_angSep(`o1`.`ra_Test`,`o1`.`decl_Test`,`o2`.`ra_Test`,`o2`.`decl_Test`)<0.001";
208210
std::string expected_100_subchunk_overlap =
209-
"SELECT count(*) AS `QS1_COUNT` FROM `Subchunks_LSST_100`.`Object_100_%S\007S%` AS "
210-
"`o1`,`Subchunks_LSST_100`.`ObjectFullOverlap_100_%S\007S%` AS `o2` "
211-
"WHERE scisql_s2PtInBox(`o1`.`ra_Test`,`o1`.`decl_Test`,6,6,7,7)=1 "
211+
"SELECT count(*) AS `QS1_COUNT` FROM `LSST`.`Object_100` AS "
212+
"`o1`,`LSST`.`ObjectFullOverlap_100` AS `o2` "
213+
"WHERE `o2`.`subChunkId`=%S\007S% "
214+
"AND `o1`.`subChunkId`=%S\007S% "
215+
"AND scisql_s2PtInBox(`o1`.`ra_Test`,`o1`.`decl_Test`,6,6,7,7)=1 "
212216
"AND scisql_s2PtInBox(`o2`.`ra_Test`,`o2`.`decl_Test`,6,6,7,7)=1 "
213217
"AND `o1`.`rFlux_PS`<0.005 AND "
214218
"scisql_angSep(`o1`.`ra_Test`,`o1`.`decl_Test`,`o2`.`ra_Test`,`o2`.`decl_Test`)<0.001";
@@ -253,9 +257,11 @@ BOOST_AUTO_TEST_CASE(Triple) {
253257
"0.024 > scisql_angSep(o1.ra_Test,o1.decl_Test,o2.ra_Test,o2.decl_Test) and "
254258
"Source.objectIdSourceTest=o2.objectIdObjTest;";
255259
std::string expected =
256-
"SELECT * FROM `Subchunks_LSST_100`.`Object_100_%S\007S%` AS "
257-
"`o1`,`Subchunks_LSST_100`.`Object_100_%S\007S%` AS `o2`,`LSST`.`Source_100` AS `LSST.Source` "
258-
"WHERE `o1`.`id`!=`o2`.`id` AND "
260+
"SELECT * FROM `LSST`.`Object_100` AS "
261+
"`o1`,`LSST`.`Object_100` AS `o2`,`LSST`.`Source_100` AS `LSST.Source` "
262+
"WHERE `o1`.`subChunkId`=%S\007S% "
263+
"AND `o2`.`subChunkId`=%S\007S% "
264+
"AND `o1`.`id`!=`o2`.`id` AND "
259265
"0.024>scisql_angSep(`o1`.`ra_Test`,`o1`.`decl_Test`,`o2`.`ra_Test`,`o2`.`decl_Test`) AND "
260266
"`LSST.Source`.`objectIdSourceTest`=`o2`.`objectIdObjTest`";
261267

@@ -509,9 +515,11 @@ BOOST_AUTO_TEST_CASE(ObjectSelfJoinDistance) {
509515
"scisql_angSep(o1.ra_Test,o1.decl_Test,o2.ra_Test,o2.decl_Test) < 0.02";
510516
std::string expected =
511517
"SELECT count(*) AS `QS1_COUNT` "
512-
"FROM `Subchunks_LSST_100`.`Object_100_%S\007S%` AS `o1`,"
513-
"`Subchunks_LSST_100`.`Object_100_%S\007S%` AS `o2` "
514-
"WHERE scisql_s2PtInBox(`o1`.`ra_Test`,`o1`.`decl_Test`,5.5,5.5,6.1,6.1)=1 "
518+
"FROM `LSST`.`Object_100` AS `o1`,"
519+
"`LSST`.`Object_100` AS `o2` "
520+
"WHERE `o2`.`subChunkId`=%S\007S% "
521+
"AND `o1`.`subChunkId`=%S\007S% "
522+
"AND scisql_s2PtInBox(`o1`.`ra_Test`,`o1`.`decl_Test`,5.5,5.5,6.1,6.1)=1 "
515523
"AND scisql_s2PtInBox(`o2`.`ra_Test`,`o2`.`decl_Test`,5.5,5.5,6.1,6.1)=1 "
516524
"AND scisql_angSep(`o1`.`ra_Test`,`o1`.`decl_Test`,`o2`.`ra_Test`,`o2`.`decl_Test`)<0.02";
517525
qsTest.sqlConfig =
@@ -951,9 +959,11 @@ BOOST_AUTO_TEST_CASE(FuncExprPred) {
951959
"(scisql_fluxToAbMag(o2.gFlux_PS)-scisql_fluxToAbMag(o2.rFlux_PS)) ) < 1;";
952960
expected =
953961
"SELECT `o1`.`objectId` AS `o1.objectId`,`o2`.`objectId` AS `objectId2` "
954-
"FROM `Subchunks_LSST_100`.`Object_100_%S\007S%` AS "
955-
"`o1`,`Subchunks_LSST_100`.`Object_100_%S\007S%` AS `o2` "
956-
"WHERE scisql_angSep(`o1`.`ra_Test`,`o1`.`decl_Test`,`o2`.`ra_Test`,`o2`.`decl_Test`)<0.00001 "
962+
"FROM `LSST`.`Object_100` AS "
963+
"`o1`,`LSST`.`Object_100` AS `o2` "
964+
"WHERE `o2`.`subChunkId`=%S\007S% "
965+
"AND `o1`.`subChunkId`=%S\007S% "
966+
"AND scisql_angSep(`o1`.`ra_Test`,`o1`.`decl_Test`,`o2`.`ra_Test`,`o2`.`decl_Test`)<0.00001 "
957967
"AND `o1`.`objectId`<>`o2`.`objectId` AND "
958968
"ABS((scisql_fluxToAbMag(`o1`.`gFlux_PS`)-scisql_fluxToAbMag(`o1`.`rFlux_PS`))-(scisql_"
959969
"fluxToAbMag(`o2`.`gFlux_PS`)-scisql_fluxToAbMag(`o2`.`rFlux_PS`)))<1";
@@ -1294,18 +1304,22 @@ BOOST_AUTO_TEST_CASE(Case01_1081) {
12941304
"WHERE closestToObj = 1 OR closestToObj is NULL;";
12951305
std::string expected_100_subchunk_core =
12961306
"SELECT count(*) AS `QS1_COUNT` "
1297-
"FROM `Subchunks_LSST_100`.`Object_100_%S\007S%` AS `o` "
1307+
"FROM `LSST_100`.`Object_100` AS `o` "
12981308
"INNER JOIN `LSST`.`RefObjMatch_100` AS `o2t` ON `o`.`objectIdObjTest`=`o2t`.`objectId` "
1299-
"INNER JOIN `Subchunks_LSST_100`.`SimRefObject_100_%S\007S%` AS `t` ON "
1309+
"INNER JOIN `LSST_100`.`SimRefObject_100` AS `t` ON "
13001310
"`o2t`.`refObjectId`=`t`.`refObjectId` "
1301-
"WHERE `o`.`closestToObj`=1 OR `o`.`closestToObj` IS NULL";
1311+
"WHERE `t`.`subChunkId`=%S\007S% "
1312+
"AND `o`.`subChunkId`=%S\007S% "
1313+
"AND `o`.`closestToObj`=1 OR `o`.`closestToObj` IS NULL";
13021314
std::string expected_100_subchunk_overlap =
13031315
"SELECT count(*) AS `QS1_COUNT` "
1304-
"FROM `Subchunks_LSST_100`.`Object_100_%S\007S%` AS `o` "
1316+
"FROM `LSST_100`.`Object_100` AS `o` "
13051317
"INNER JOIN `LSST`.`RefObjMatch_100` AS `o2t` ON `o`.`objectIdObjTest`=`o2t`.`objectId` "
1306-
"INNER JOIN `Subchunks_LSST_100`.`SimRefObjectFullOverlap_100_%S\007S%` AS `t` ON "
1318+
"INNER JOIN `LSST_100`.`SimRefObjectFullOverlap_100` AS `t` ON "
13071319
"`o2t`.`refObjectId`=`t`.`refObjectId` "
1308-
"WHERE `o`.`closestToObj`=1 OR `o`.`closestToObj` IS NULL";
1320+
"WHERE `t`.`subChunkId`=%S\007S% "
1321+
"AND `o`.`subChunkId`=%S\007S% "
1322+
"AND `o`.`closestToObj`=1 OR `o`.`closestToObj` IS NULL";
13091323
qsTest.sqlConfig =
13101324
SqlConfig(SqlConfig::MockDbTableColumns({{"LSST",
13111325
{{"Object", {"objectIdObjTest", "closestToObj"}},

‎src/query/SelectStmt.cc‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,11 @@ void SelectStmt::setFromListAsTable(std::string const& t) {
169169
_fromList = std::make_shared<FromList>(tr);
170170
}
171171

172+
WhereClause& SelectStmt::getWhereClause(bool createIfMissing) {
173+
if (createIfMissing && !_whereClause) _whereClause = std::make_shared<WhereClause>();
174+
return *_whereClause;
175+
}
176+
172177
bool SelectStmt::operator==(const SelectStmt& rhs) const {
173178
return (util::ptrCompare<FromList>(_fromList, rhs._fromList) &&
174179
util::ptrCompare<SelectList>(_selectList, rhs._selectList) &&

‎src/query/SelectStmt.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ class SelectStmt {
104104

105105
bool hasWhereClause() const { return static_cast<bool>(_whereClause); }
106106
WhereClause const& getWhereClause() const { return *_whereClause; }
107-
WhereClause& getWhereClause() { return *_whereClause; }
107+
WhereClause& getWhereClause(bool createIfMissing = false);
108108
void setWhereClause(std::shared_ptr<WhereClause> w) { _whereClause = w; }
109109

110110
/**

0 commit comments

Comments
 (0)