Skip to content

Commit 9933725

Browse files
akoclaude
andcommitted
feat(catalog): index image collections, JS actions, data transformers (schema v4)
These document types had no catalog table, so they were absent from the objects index (and from describe auto-detect / catalog search). Add tables + views for javascript_actions, image_collections, data_transformers and union them into objects (JAVASCRIPT_ACTION / IMAGE_COLLECTION / DATA_TRANSFORMER). A single generic builder (buildSimpleNamedDocs) populates them from the raw-unit surface (ListRawUnitsByType + Name-only decode), so adding these types needed no change to the CatalogReader interface or its backend implementation — keeping clear of the in-flight backend rewrite on the modelsdk branch. Schema bumped to v4 (caches rebuild on next refresh). describe auto-detect now resolves image collections and data transformers by bare name (JS actions are cataloged but have no describe CLI form, so they're left out of auto-detect). Verified on a real project (17 image collections, 72 JS actions in objects; describe of a bare image-collection name works). Agent-editor docs (agent/model/knowledge base/consumed MCP service) remain uncataloged — they are CustomBlobDocuments with the name in a JSON blob, a separate follow-up. (#658) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 292c504 commit 9933725

6 files changed

Lines changed: 171 additions & 2 deletions

File tree

CHANGELOG.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
1414

1515
### Changed
1616

17-
- **Catalog `objects` index includes associations** — the unified `objects` view now unions the `associations` table (`ObjectType = ASSOCIATION`), so it is a complete index for the cataloged document types and consumers no longer need a separate associations query. Catalog schema bumped to v3; cached `.mxcli/catalog.db` files rebuild automatically on the next `refresh catalog`. (Image collections, JavaScript actions, data transformers, and agent-editor docs are not yet cataloged at all, so they remain out of the index — tracked in #658.)
17+
- **Catalog `objects` index includes associations** — the unified `objects` view now unions the `associations` table (`ObjectType = ASSOCIATION`), so it is a complete index for the cataloged document types and consumers no longer need a separate associations query. Catalog schema bumped to v3; cached `.mxcli/catalog.db` files rebuild automatically on the next `refresh catalog`.
18+
- **Catalog indexes image collections, JavaScript actions, and data transformers** — these document types had no catalog table at all; they are now built (via the raw-unit surface, so no `CatalogReader`/backend change) into their own tables and unioned into `objects` (`IMAGE_COLLECTION`, `JAVASCRIPT_ACTION`, `DATA_TRANSFORMER`). `describe` auto-detect resolves image collections and data transformers by bare name. Catalog schema bumped to v4. (Agent-editor docs — agent/model/knowledge base/consumed MCP service — remain uncataloged: they are `CustomBlobDocument`s with the name inside a JSON blob; tracked in #658.)
1819

1920
### Fixed
2021

cmd/mxcli/cmd_describe.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,10 @@ var objectTypeToDescribe = map[string]string{
305305
"REST_CLIENT": "restclient",
306306
"BUSINESS_EVENT_SERVICE": "businesseventservice",
307307
"DATABASE_CONNECTION": "databaseconnection",
308+
"IMAGE_COLLECTION": "imagecollection",
309+
"DATA_TRANSFORMER": "datatransformer",
310+
// JAVASCRIPT_ACTION is cataloged but has no `describe` CLI form, so it is
311+
// intentionally omitted from auto-detect.
308312
}
309313

310314
// unitTypeToDescribe maps a top-level unit's BSON $Type to the `describe`

mdl/catalog/builder.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,18 @@ func (b *Builder) Build(progress ProgressFunc) error {
377377
return fmt.Errorf("failed to build java actions: %w", err)
378378
}
379379

380+
if err := b.buildSimpleNamedDocs("JavaScriptActions$JavaScriptAction", "javascript_actions", "JavaScript Actions"); err != nil {
381+
return fmt.Errorf("failed to build javascript actions: %w", err)
382+
}
383+
384+
if err := b.buildSimpleNamedDocs("Images$ImageCollection", "image_collections", "Image Collections"); err != nil {
385+
return fmt.Errorf("failed to build image collections: %w", err)
386+
}
387+
388+
if err := b.buildSimpleNamedDocs("DataTransformers$DataTransformer", "data_transformers", "Data Transformers"); err != nil {
389+
return fmt.Errorf("failed to build data transformers: %w", err)
390+
}
391+
380392
if err := b.buildMicroflows(); err != nil {
381393
return fmt.Errorf("failed to build microflows: %w", err)
382394
}

mdl/catalog/builder_simpledocs.go

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
3+
package catalog
4+
5+
import "go.mongodb.org/mongo-driver/bson"
6+
7+
// buildSimpleNamedDocs catalogs documents that are plain named units (a top-level
8+
// BSON `Name`, optionally `Documentation`) into a `<table>_data` table with the
9+
// standard Id/Name/QualifiedName/ModuleName/Folder/Description columns.
10+
//
11+
// It reads through the raw-unit surface (ListRawUnitsByType) rather than a typed
12+
// reader method, so adding a new document type to the catalog needs no change to
13+
// the CatalogReader interface or its backend implementation — only a table, this
14+
// call, and an objects-view union. Used for image collections, JavaScript
15+
// actions, and data transformers, which had no catalog table before.
16+
func (b *Builder) buildSimpleNamedDocs(typePrefix, table, reportLabel string) error {
17+
units, err := b.reader.ListRawUnitsByType(typePrefix)
18+
if err != nil {
19+
return err
20+
}
21+
22+
stmt, err := b.tx.Prepare(
23+
"INSERT INTO " + table + "_data " +
24+
"(Id, Name, QualifiedName, ModuleName, Folder, Description, ProjectId, SnapshotId) " +
25+
"VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
26+
)
27+
if err != nil {
28+
return err
29+
}
30+
defer stmt.Close()
31+
32+
projectID, snapshotID := b.snapshotMeta()
33+
34+
count := 0
35+
for _, u := range units {
36+
var doc struct {
37+
Name string `bson:"Name"`
38+
Documentation string `bson:"Documentation"`
39+
}
40+
if err := bson.Unmarshal(u.Contents, &doc); err != nil || doc.Name == "" {
41+
continue
42+
}
43+
moduleID := b.hierarchy.findModuleID(u.ContainerID)
44+
moduleName := b.hierarchy.getModuleName(moduleID)
45+
qualifiedName := moduleName + "." + doc.Name
46+
folderPath := b.hierarchy.buildFolderPath(u.ContainerID)
47+
48+
if _, err := stmt.Exec(
49+
string(u.ID), doc.Name, qualifiedName, moduleName, folderPath,
50+
doc.Documentation, projectID, snapshotID,
51+
); err != nil {
52+
return err
53+
}
54+
count++
55+
}
56+
57+
b.report(reportLabel, count)
58+
return nil
59+
}

mdl/catalog/catalog_test.go

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,48 @@ func TestObjectsView_IncludesAssociations(t *testing.T) {
178178
}
179179
}
180180

181+
// TestObjectsView_IncludesNewDocumentTypes verifies the document types added to
182+
// the catalog in schema v4 (image collections, JavaScript actions, data
183+
// transformers) surface in the unified objects index.
184+
func TestObjectsView_IncludesNewDocumentTypes(t *testing.T) {
185+
cases := []struct {
186+
table string
187+
objectType string
188+
}{
189+
{"javascript_actions", "JAVASCRIPT_ACTION"},
190+
{"image_collections", "IMAGE_COLLECTION"},
191+
{"data_transformers", "DATA_TRANSFORMER"},
192+
}
193+
for _, tc := range cases {
194+
t.Run(tc.objectType, func(t *testing.T) {
195+
cat, err := New()
196+
if err != nil {
197+
t.Fatalf("Failed to create catalog: %v", err)
198+
}
199+
defer cat.Close()
200+
201+
qn := "Mod." + tc.table
202+
if _, err := cat.CatalogDB().Exec(
203+
"INSERT INTO "+tc.table+"_data (Id, Name, QualifiedName, ModuleName) VALUES (?, ?, ?, ?)",
204+
"id-1", tc.table, qn, "Mod",
205+
); err != nil {
206+
t.Fatalf("insert: %v", err)
207+
}
208+
209+
result, err := cat.Query("SELECT ObjectType FROM objects WHERE QualifiedName = '" + qn + "'")
210+
if err != nil {
211+
t.Fatalf("query: %v", err)
212+
}
213+
if result.Count != 1 {
214+
t.Fatalf("expected 1 row in objects view, got %d", result.Count)
215+
}
216+
if got := result.Rows[0][0]; got != tc.objectType {
217+
t.Errorf("ObjectType = %v, want %s", got, tc.objectType)
218+
}
219+
})
220+
}
221+
}
222+
181223
func TestQueryError(t *testing.T) {
182224
cat, err := New()
183225
if err != nil {

mdl/catalog/tables.go

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ package catalog
1212
// SnapshotSource / SourceId / SourceBranch / SourceRevision columns
1313
// from every row (issue #576).
1414
// 1 — initial flat schema with denormalized snapshot columns on every row.
15-
const CatalogSchemaVersion = "3"
15+
const CatalogSchemaVersion = "4"
1616

1717
// MetaSchemaVersion is the catalog_meta key that records the schema version
1818
// the cache was built against.
@@ -245,6 +245,45 @@ func (c *Catalog) createTables() error {
245245
)`,
246246
viewWithFullSnapshot("java_actions"),
247247

248+
// javascript_actions
249+
`CREATE TABLE IF NOT EXISTS javascript_actions_data (
250+
Id TEXT PRIMARY KEY,
251+
Name TEXT,
252+
QualifiedName TEXT,
253+
ModuleName TEXT,
254+
Folder TEXT,
255+
Description TEXT,
256+
ProjectId TEXT,
257+
SnapshotId TEXT
258+
)`,
259+
viewWithFullSnapshot("javascript_actions"),
260+
261+
// image_collections
262+
`CREATE TABLE IF NOT EXISTS image_collections_data (
263+
Id TEXT PRIMARY KEY,
264+
Name TEXT,
265+
QualifiedName TEXT,
266+
ModuleName TEXT,
267+
Folder TEXT,
268+
Description TEXT,
269+
ProjectId TEXT,
270+
SnapshotId TEXT
271+
)`,
272+
viewWithFullSnapshot("image_collections"),
273+
274+
// data_transformers
275+
`CREATE TABLE IF NOT EXISTS data_transformers_data (
276+
Id TEXT PRIMARY KEY,
277+
Name TEXT,
278+
QualifiedName TEXT,
279+
ModuleName TEXT,
280+
Folder TEXT,
281+
Description TEXT,
282+
ProjectId TEXT,
283+
SnapshotId TEXT
284+
)`,
285+
viewWithFullSnapshot("data_transformers"),
286+
248287
// activities
249288
`CREATE TABLE IF NOT EXISTS activities_data (
250289
Id TEXT PRIMARY KEY,
@@ -807,6 +846,18 @@ func (c *Catalog) createTables() error {
807846
ProjectId, ProjectName, SnapshotId, SnapshotDate, SnapshotSource
808847
FROM java_actions
809848
UNION ALL
849+
SELECT Id, 'JAVASCRIPT_ACTION' as ObjectType, Name, QualifiedName, ModuleName, Folder, Description,
850+
ProjectId, ProjectName, SnapshotId, SnapshotDate, SnapshotSource
851+
FROM javascript_actions
852+
UNION ALL
853+
SELECT Id, 'IMAGE_COLLECTION' as ObjectType, Name, QualifiedName, ModuleName, Folder, Description,
854+
ProjectId, ProjectName, SnapshotId, SnapshotDate, SnapshotSource
855+
FROM image_collections
856+
UNION ALL
857+
SELECT Id, 'DATA_TRANSFORMER' as ObjectType, Name, QualifiedName, ModuleName, Folder, Description,
858+
ProjectId, ProjectName, SnapshotId, SnapshotDate, SnapshotSource
859+
FROM data_transformers
860+
UNION ALL
810861
SELECT Id, 'ODATA_CLIENT' as ObjectType, Name, QualifiedName, ModuleName, '' as Folder, '' as Description,
811862
ProjectId, ProjectName, SnapshotId, SnapshotDate, SnapshotSource
812863
FROM odata_clients

0 commit comments

Comments
 (0)