SchemaView performance improvements

iModelCore/ECDb/ECDb/ConcurrentQueryManagerImpl.cpp

@@ -1064,8 +1064,11 @@ std::string QueryHelper::FormatQuery(const char* query) {
 // @bsimethod
 //---------------------------------------------------------------------------------------
 void QueryHelper::BindLimits(ECSqlStatement& stmt, QueryLimit const& limit) {
-    const auto idxCount = stmt.GetParameterIndex(LIMIT_VAR_COUNT);
-    const auto idxOffset = stmt.GetParameterIndex(LIMIT_VAR_OFFSET);
+    // A PRAGMA (or any statement FormatQuery left unwrapped) has no LIMIT/OFFSET parameters. Use the
+    // non-logging TryGetParameterIndex so probing for their absence does not emit "No
+    // parameter index found" errors - GetParameterIndex would log one per missing parameter.
+    const auto idxCount = stmt.TryGetParameterIndex(LIMIT_VAR_COUNT);
+    const auto idxOffset = stmt.TryGetParameterIndex(LIMIT_VAR_OFFSET);
     if (idxCount < 0 || idxOffset < 0)
         return; // PRAGMA or other statement without LIMIT/OFFSET parameters
     stmt.BindInt64(idxCount, limit.GetCount());

iModelCore/ECDb/ECDb/ECDbImpl.cpp

@@ -268,6 +268,7 @@ void ECDb::Impl::RegisterECSqlPragmas() const
     GetPragmaManager().Register(PragmaECSqlVersion::Create());
     GetPragmaManager().Register(PragmaSqliteSql::Create());
     GetPragmaManager().Register(PragmaSchemaView::Create());
+    GetPragmaManager().Register(PragmaSchemaViewFragment::Create());
     }
 
 //--------------------------------------------------------------------------------------

iModelCore/ECDb/ECDb/ECSql/ECSqlPragmas.cpp

@@ -32,6 +32,7 @@ DbResult PragmaChecksum::Read(PragmaManager::RowSet& rowSet, ECDbCR ecdb, Pragma
     const Utf8String kECDbSchema = "ecdb_schema";
 	const Utf8String kECDbMap = "ecdb_map";
 	const Utf8String kSQLiteSchema = "sqlite_schema";
+	const Utf8String kSchemaToken = "schema_token";
 
     if (kECDbSchema.EqualsIAscii(val.GetString())) {
         Utf8String sha3;
@@ -87,13 +88,35 @@ DbResult PragmaChecksum::Read(PragmaManager::RowSet& rowSet, ECDbCR ecdb, Pragma
 		rowSet = std::move(result);
 		return BE_SQLITE_OK;
     }
+	// Unlike the other keys, schema_token hashes schema *identity* only (every schema's name and
+	// version, one tiny row each), NOT schema contents. It is the cheap cache-invalidation key for
+	// SchemaView - see PRAGMA checksum docs / SchemaViewBinaryFormat.md. Limitation: a same-version
+	// content change (which ECDb only allows for dynamic schemas) does not change this value.
+    if (kSchemaToken.EqualsIAscii(val.GetString())) {
+        Utf8String sha3;
+		if (SHA3Helper::ComputeHash(sha3, ecdb, SHA3Helper::SourceType::ECDB_SCHEMA_TOKEN, "main", SHA3Helper::HashSize::SHA3_256) != BE_SQLITE_OK) {
+			ecdb.GetImpl().Issues().Report(
+				IssueSeverity::Error,
+				IssueCategory::BusinessProperties,
+				IssueType::ECSQL,
+				ECDbIssueId::ECDb_0593,
+				"Unable to compute schema token.");
+
+			rowSet = std::move(result);
+            return BE_SQLITE_ERROR;
+        }
+		auto row = result->AppendRow();
+        row.appendValue() = sha3;
+		rowSet = std::move(result);
+		return BE_SQLITE_OK;
+    }
 
 	ecdb.GetImpl().Issues().ReportV(
 		IssueSeverity::Error,
 		IssueCategory::BusinessProperties,
 		IssueType::ECSQL,
 		ECDbIssueId::ECDb_0596,
-		"Unable checksum val '%s'. Valid values are ecdb_schema|ecdb_map|sqlite_schema", val.GetString().c_str());
+		"Unable checksum val '%s'. Valid values are ecdb_schema|ecdb_map|sqlite_schema|schema_token", val.GetString().c_str());
 
 	rowSet = std::move(result);
 	return BE_SQLITE_ERROR;
@@ -403,6 +426,8 @@ DbResult SHA3Helper::ComputeHash(Utf8StringR hash, DbCR db, SourceType type, Utf
 		}, dbAlias, hashSize, skipTableThatDoesNotExists);
 	} else if (type == SourceType::SQLITE_SCHEMA) {
         rc = ComputeSQLiteSchemaHash(hash, db, dbAlias, hashSize);
+    } else if (type == SourceType::ECDB_SCHEMA_TOKEN) {
+        rc = ComputeSchemaTokenHash(hash, db, dbAlias, hashSize);
     }
 	if (rc != BE_SQLITE_OK) {
 		return rc;
@@ -412,6 +437,32 @@ DbResult SHA3Helper::ComputeHash(Utf8StringR hash, DbCR db, SourceType type, Utf
     return rc;
 }
 
+//---------------------------------------------------------------------------------------
+// Cheap schema-identity hash: hashes only the name and version of every schema (one row per
+// schema in ec_Schema), NOT their contents. Backs PRAGMA checksum(schema_token) and the schemaToken
+// column of schema_view / schema_view_fragment. Ordered by Name for a session-stable digest.
+// Limitation: a same-version content change (which ECDb only allows for dynamic schemas) does
+// not change this hash. See the PRAGMA checksum(schema_token) docs.
+// @bsimethod
+//+---------------+---------------+---------------+---------------+---------------+------
+DbResult SHA3Helper::ComputeSchemaTokenHash(Utf8String& hash, DbCR db, Utf8CP dbAlias, HashSize hashSize) {
+	Statement stmt;
+	auto rc = stmt.Prepare(db, SqlPrintfString(
+		"select hex(sha3_query(\"select Name,VersionDigit1,VersionDigit2,VersionDigit3 from [%s].ec_Schema order by Name\", %d))", dbAlias, (int)hashSize));
+
+	if (rc != BE_SQLITE_OK) {
+		return rc;
+	}
+
+    rc = stmt.Step();
+    if (rc != BE_SQLITE_ROW) {
+        return rc;
+    }
+
+    hash = stmt.GetValueText(0);
+    return BE_SQLITE_OK;
+}
+
 //---------------------------------------------------------------------------------------
 // @bsimethod
 //+---------------+---------------+---------------+---------------+---------------+------
@@ -1010,9 +1061,10 @@ DbResult PragmaSchemaView::Read(PragmaManager::RowSet& rowSet, ECDbCR ecdb, Prag
 		return writeResult;
 	auto const& output = writer.GetOutput();
 
-	// Compute schema token for cache invalidation
+	// Compute schema token for cache invalidation. This is the cheap schema-identity hash
+	// (ec_Schema names + versions only), NOT the full ecdb_schema checksum - see PRAGMA checksum(schema_token).
 	Utf8String schemaToken;
-	SHA3Helper::ComputeHash(schemaToken, ecdb, SHA3Helper::SourceType::ECDB_SCHEMA, "main", SHA3Helper::HashSize::SHA3_256);
+	SHA3Helper::ComputeHash(schemaToken, ecdb, SHA3Helper::SourceType::ECDB_SCHEMA_TOKEN, "main", SHA3Helper::HashSize::SHA3_256);
 
 	auto row = result->AppendRow();
 	row.appendValue() = "binary";
@@ -1034,6 +1086,116 @@ DbResult PragmaSchemaView::Write(PragmaManager::RowSet& rowSet, ECDbCR ecdb, Pra
 	return BE_SQLITE_READONLY;
 }
 
+//=======================================================================================
+// PragmaSchemaViewFragment
+//=======================================================================================
+namespace {
+// True only for a non-empty run of ASCII decimal digits.
+bool IsAllDigits(Utf8StringCR s) {
+	if (s.empty())
+		return false;
+	for (char c : s) {
+		if (c < '0' || c > '9')
+			return false;
+	}
+	return true;
+}
+}
+
+//---------------------------------------------------------------------------------------
+// @bsimethod
+//---------------------------------------------------------------------------------------
+DbResult PragmaSchemaViewFragment::Read(PragmaManager::RowSet& rowSet, ECDbCR ecdb, PragmaVal const& val, PragmaManager::OptionsMap const& options) {
+	auto reportError = [&](Utf8StringCR message) -> DbResult {
+		ecdb.GetImpl().Issues().ReportV(IssueSeverity::Error, IssueCategory::BusinessProperties, IssueType::ECSQL, ECDbIssueId::ECDb_0601,
+			"PRAGMA %s: %s", GetName().c_str(), message.c_str());
+		rowSet = std::make_unique<StaticPragmaResult>(ecdb);
+		rowSet->FreezeSchemaChanges();
+		return BE_SQLITE_ERROR;
+	};
+
+	if (!val.IsString())
+		return reportError("expects a single string argument: a comma-separated list of decimal ec_Schema ids, optionally prefixed with a 'v<N>;' format-version token.");
+
+	Utf8String arg(val.GetString().c_str());
+
+	// Optional leading 'v<N>;' format-version token. The version is carried inside the one string
+	// argument because the pragma grammar allows only a single value; see the schema_view_fragment
+	// reference doc for the rationale.
+	uint8_t requestedVersion = CURRENT_FORMAT_VERSION;
+	Utf8String idList = arg;
+	size_t const semicolon = arg.find(';');
+	if (semicolon != Utf8String::npos) {
+		Utf8String versionToken = arg.substr(0, semicolon);
+		idList = arg.substr(semicolon + 1);
+		if (versionToken.size() < 2 || (versionToken[0] != 'v' && versionToken[0] != 'V'))
+			return reportError("malformed version prefix; expected 'v<N>;' (for example 'v1;').");
+		Utf8String digits = versionToken.substr(1);
+		if (!IsAllDigits(digits))
+			return reportError("malformed version prefix; expected 'v<N>;' with N a positive integer.");
+		int64_t v = (int64_t)strtoll(digits.c_str(), nullptr, 10);
+		if (v < 1 || v > CURRENT_FORMAT_VERSION)
+			return reportError(Utf8PrintfString("unsupported format version %" PRId64 ". Supported versions: 1-%d.", v, (int)CURRENT_FORMAT_VERSION));
+		requestedVersion = (uint8_t)v;
+	}
+
+	// Parse the comma-separated decimal id list into a de-duplicated set; reject any malformed or
+	// repeated id rather than silently emit a partial fragment.
+	bvector<Utf8String> tokens;
+	BeStringUtilities::Split(idList.c_str(), ",", tokens);
+	std::unordered_set<int64_t> schemaIds;
+	for (Utf8StringR token : tokens) {
+		token.Trim();
+		if (!IsAllDigits(token))
+			return reportError(Utf8PrintfString("invalid schema id '%s'; ids must be positive decimal integers.", token.c_str()));
+		int64_t id = (int64_t)strtoll(token.c_str(), nullptr, 10);
+		if (!schemaIds.insert(id).second)
+			return reportError(Utf8PrintfString("duplicate schema id %" PRId64 " in the request.", id));
+	}
+	if (schemaIds.empty())
+		return reportError("the schema id list is empty.");
+
+	SchemaViewWriter writer;
+	DbResult writeResult = writer.WriteSchemas(ecdb, schemaIds);
+	if (writeResult == BE_SQLITE_NOTFOUND)
+		return reportError("one or more requested schema ids do not exist in this connection.");
+	if (writeResult != BE_SQLITE_OK)
+		return writeResult;
+	auto const& output = writer.GetOutput();
+
+	auto result = std::make_unique<StaticPragmaResult>(ecdb);
+	result->AppendProperty("format", PRIMITIVETYPE_String);
+	result->AppendProperty("formatVersion", PRIMITIVETYPE_Integer);
+	result->AppendProperty("data", PRIMITIVETYPE_String);
+	result->AppendProperty("schemaToken", PRIMITIVETYPE_String);
+	result->FreezeSchemaChanges();
+
+	// schemaToken is the cheap schema-identity hash (ec_Schema names + versions), identical to
+	// schema_view's and PRAGMA checksum(schema_token), so a fragment and the manifest it was planned from share
+	// one cache-invalidation key.
+	Utf8String schemaToken;
+	SHA3Helper::ComputeHash(schemaToken, ecdb, SHA3Helper::SourceType::ECDB_SCHEMA_TOKEN, "main", SHA3Helper::HashSize::SHA3_256);
+
+	auto row = result->AppendRow();
+	row.appendValue() = "binary";
+	row.appendValue() = (int64_t)requestedVersion;
+	row.appendValue().SetBinary(output.data(), output.size());
+	row.appendValue() = schemaToken.c_str();
+
+	rowSet = std::move(result);
+	return BE_SQLITE_OK;
+}
+
+//---------------------------------------------------------------------------------------
+// @bsimethod
+//---------------------------------------------------------------------------------------
+DbResult PragmaSchemaViewFragment::Write(PragmaManager::RowSet& rowSet, ECDbCR ecdb, PragmaVal const&, PragmaManager::OptionsMap const& options) {
+	ecdb.GetImpl().Issues().ReportV(IssueSeverity::Error, IssueCategory::BusinessProperties, IssueType::ECSQL, ECDbIssueId::ECDb_0552, "PRAGMA %s is readonly.", GetName().c_str());
+	rowSet = std::make_unique<StaticPragmaResult>(ecdb);
+	rowSet->FreezeSchemaChanges();
+	return BE_SQLITE_READONLY;
+}
+
 
 END_BENTLEY_SQLITE_EC_NAMESPACE
 

iModelCore/ECDb/ECDb/ECSql/ECSqlPragmas.h

@@ -83,7 +83,7 @@ struct PragmaSqliteSql : PragmaManager::GlobalHandler {
 // @bsiclass PragmaChecksum
 //+===============+===============+===============+===============+===============+======
 struct PragmaChecksum : PragmaManager::GlobalHandler {
-    PragmaChecksum():GlobalHandler("checksum", "checksum([ec_schema|ec_map|db_schema]) return sha1 checksum for data."){}
+    PragmaChecksum():GlobalHandler("checksum", "checksum([ecdb_schema|ecdb_map|sqlite_schema|schema_token]) return sha3 checksum for data. 'schema_token' is a cheap schema-identity hash (names+versions only) for SchemaView cache invalidation."){}
     ~PragmaChecksum(){}
     virtual DbResult Read(PragmaManager::RowSet&, ECDbCR, PragmaVal const&, PragmaManager::OptionsMap const&) override;
     virtual DbResult Write(PragmaManager::RowSet&, ECDbCR, PragmaVal const&, PragmaManager::OptionsMap const&) override;
@@ -172,6 +172,26 @@ struct PragmaSchemaView : PragmaManager::GlobalHandler {
     static std::unique_ptr<PragmaManager::Handler> Create () { return std::make_unique<PragmaSchemaView>(); }
 };
 
+//=======================================================================================
+// @bsiclass PragmaSchemaViewFragment
+// Returns a chosen subset of schemas as a single binary blob, in the same format as
+// schema_view, for incremental SchemaView loading.
+// The single string argument is a comma-separated list of decimal ec_Schema ids, optionally
+// prefixed with a 'v<N>;' format-version token. The id list must be dependency-closed (the
+// caller computes the reference closure; the pragma does not expand references).
+// Usage: PRAGMA schema_view_fragment('131,145,150')      -- latest format version
+//        PRAGMA schema_view_fragment('v1;131,145,150')   -- explicit format version 1
+// Result: single row, same columns as schema_view: format, formatVersion, data, schemaToken.
+//+===============+===============+===============+===============+===============+======
+struct PragmaSchemaViewFragment : PragmaManager::GlobalHandler {
+    static constexpr uint8_t CURRENT_FORMAT_VERSION = 1;
+    PragmaSchemaViewFragment():GlobalHandler("schema_view_fragment", "schema_view_fragment('[v<N>;]id,id,...') - returns the given subset of schemas as a base64-encoded binary blob string for incremental loading. Optional leading 'v<N>;' selects the format version (default: latest)."){}
+    ~PragmaSchemaViewFragment(){}
+    virtual DbResult Read(PragmaManager::RowSet&, ECDbCR, PragmaVal const&, PragmaManager::OptionsMap const&) override;
+    virtual DbResult Write(PragmaManager::RowSet&, ECDbCR, PragmaVal const&, PragmaManager::OptionsMap const&) override;
+    static std::unique_ptr<PragmaManager::Handler> Create () { return std::make_unique<PragmaSchemaViewFragment>(); }
+};
+
 //=======================================================================================
 // @bsiclass
 //+===============+===============+===============+===============+===============+======
@@ -186,13 +206,15 @@ struct SHA3Helper final {
         ECDB_SCHEMA,
         ECDB_MAP,
         SQLITE_SCHEMA,
+        ECDB_SCHEMA_TOKEN, // schema identity only (ec_Schema name + version), cheap; backs PRAGMA checksum(schema_token)
     };
 
 
 private:
     static bool TableExists(DbCR, Utf8CP, Utf8CP);
     static DbResult ComputeHash(Utf8String&, DbCR, std::vector<std::string> const &, Utf8CP, HashSize, bool);
     static DbResult ComputeSQLiteSchemaHash(Utf8String&, DbCR, Utf8CP, HashSize);
+    static DbResult ComputeSchemaTokenHash(Utf8String&, DbCR, Utf8CP, HashSize);
 public:
     static DbResult ComputeHash(Utf8StringR, DbCR, SourceType, Utf8CP dbAlias = "main", HashSize hashSize = HashSize::SHA3_256);
 };