Skip to content

Commit d72c15c

Browse files
shafeeqd959shafeeqd959
committed
fixed am support in export query
1 parent 84c65f8 commit d72c15c

9 files changed

Lines changed: 126 additions & 114 deletions

File tree

‎.talismanrc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,6 @@ fileignoreconfig:
3939
checksum: 9fa87dc7639e821411fba5b4f7871384dc2c4d5c11887a5d3c5a19702e7130be
4040
- filename: packages/contentstack-asset-management/test/unit/import-setup/import-setup-asset-mappers.test.ts
4141
checksum: a3841d92aed9e31aea8f243c2fa51023cacd619eeac7c35048621247145decb4
42+
- filename: packages/contentstack-query-export/src/core/query-executor.ts
43+
checksum: 708f8a9bc837ed15342fe73920588978a97cab9002c401dbc6ad7030e0238f48
4244
version: '1.0'

‎packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import { getAssetItems, writeStreamToFile } from '../utils/export-helpers';
1212
import { runInBatches } from '../utils/concurrent-batch';
1313

1414
const DEFAULT_ASSET_BATCH_SIZE = 100;
15-
const SEARCH_PAGE_LIMIT = 50;
15+
const SEARCH_PAGE_LIMIT = 100;
1616

1717
/**
1818
* Query-based Contentstack Assets exporter.
@@ -182,8 +182,12 @@ class QueryExportWorkspaceAdapter extends CSAssetsExportAdapter {
182182
limit: SEARCH_PAGE_LIMIT,
183183
});
184184
pageItems = getAssetItems(response);
185-
if (pageItems.length === 0 && Array.isArray((response as { assets?: unknown[] }).assets)) {
186-
pageItems = (response as { assets: unknown[] }).assets;
185+
186+
if (pageItems.length === 0 && skip === 0) {
187+
log.warn(
188+
`Search returned 0 assets in space ${spaceRef.space_uid} for UID(s): [${uidBatch.join(', ')}]`,
189+
this.exportContext.context,
190+
);
187191
}
188192

189193
for (const item of pageItems) {

‎packages/contentstack-asset-management/src/types/cs-assets-api.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,8 +136,10 @@ export type SearchAssetsParams = {
136136
/** Response shape from POST /api/search for assets. */
137137
export type SearchAssetsResponse = {
138138
count?: number;
139+
relation?: string;
139140
assets?: unknown[];
140141
items?: unknown[];
142+
results?: unknown[];
141143
folders?: unknown[];
142144
};
143145

‎packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -263,10 +263,16 @@ export class CSAssetsAdapter implements ICSAssetsAdapter {
263263
return { count: 0, assets: [] };
264264
}
265265
const body = {
266-
query: { uid: { $in: assetUIDs } },
266+
query: {
267+
$and: [{ uid: { $in: assetUIDs } }],
268+
},
267269
skip,
268270
limit,
271+
desc: 'updated_at',
272+
search_text: '',
273+
search_field: 'all',
269274
object_type: 'asset',
275+
search_terms_operator: 'or',
270276
fields: [...DEFAULT_SEARCH_ASSET_FIELDS],
271277
spaces,
272278
};

‎packages/contentstack-asset-management/src/utils/export-helpers.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ export function getAssetItems(
1414
): Array<{ uid?: string; _uid?: string; url?: string; filename?: string; file_name?: string }> {
1515
if (Array.isArray(assetsData)) return assetsData;
1616
const data = assetsData as Record<string, unknown>;
17-
const items = data?.items ?? data?.assets;
17+
const items = data?.items ?? data?.assets ?? data?.results;
1818
return Array.isArray(items) ? items : [];
1919
}
2020

‎packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@ describe('CsAssetsQueryExporter', () => {
3838
space: { uid: 'space-1', title: 'Test Space' },
3939
});
4040
searchAssetsStub = sinon.stub(CSAssetsExportAdapter.prototype, 'searchAssets').resolves({
41-
assets: [
41+
count: 2,
42+
relation: 'eq',
43+
results: [
4244
{ uid: 'asset-1', url: 'https://cdn.example.com/a1.png', file_name: 'a1.png', is_dir: false },
4345
{ uid: 'asset-2', url: 'https://cdn.example.com/a2.png', file_name: 'a2.png', is_dir: false },
4446
],
@@ -120,7 +122,7 @@ describe('CSAssetsAdapter.searchAssets', () => {
120122
sinon.restore();
121123
});
122124

123-
it('should POST to /api/search with uid $in query', async () => {
125+
it('should POST to /api/search with $and-wrapped uid $in query and required fields', async () => {
124126
const adapter = new CSAssetsAdapter(baseConfig);
125127
await adapter.searchAssets({
126128
assetUIDs: ['uid-1', 'uid-2'],
@@ -134,8 +136,12 @@ describe('CSAssetsAdapter.searchAssets', () => {
134136
expect(url).to.equal('https://am.example.com/api/search');
135137
expect(init.method).to.equal('POST');
136138
const body = JSON.parse(init.body);
137-
expect(body.query).to.deep.equal({ uid: { $in: ['uid-1', 'uid-2'] } });
139+
expect(body.query).to.deep.equal({ $and: [{ uid: { $in: ['uid-1', 'uid-2'] } }] });
138140
expect(body.object_type).to.equal('asset');
141+
expect(body.desc).to.equal('updated_at');
142+
expect(body.search_text).to.equal('');
143+
expect(body.search_field).to.equal('all');
144+
expect(body.search_terms_operator).to.equal('or');
139145
expect(body.spaces).to.deep.equal([{ space_uid: 'space-1', workspace: 'main' }]);
140146
});
141147

‎packages/contentstack-export/src/export/modules/assets.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,6 @@ export default class ExportAssets extends BaseClass {
123123

124124
this.assetsRootPath = pResolve(
125125
this.exportConfig.exportDir,
126-
this.exportConfig.branchName || '',
127126
(this.assetsRootPath = pResolve(getExportBasePath(this.exportConfig), this.assetConfig.dirName)),
128127
);
129128
log.debug(`Assets root path resolved to: ${this.assetsRootPath}`, this.exportConfig.context);

‎packages/contentstack-export/src/export/modules/publishing-rules.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ export default class ExportPublishingRules extends BaseClass {
2323
async start(): Promise<void> {
2424
this.publishingRulesFolderPath = pResolve(
2525
this.exportConfig.exportDir,
26-
this.exportConfig.branchName || '',
2726
this.publishingRulesConfig.dirName,
2827
);
2928
log.debug(`Publishing rules folder path: ${this.publishingRulesFolderPath}`, this.exportConfig.context);

‎packages/contentstack-query-export/src/core/query-executor.ts‎

Lines changed: 98 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,9 @@ export class QueryExporter {
7373
const branch = await this.stackAPIClient
7474
.branch(branchName)
7575
.fetch({ include_settings: true } as Record<string, unknown>);
76-
const linked = (branch as { settings?: { am_v2?: { linked_workspaces?: QueryExportConfig['linkedWorkspaces'] } } })
77-
?.settings?.am_v2?.linked_workspaces;
76+
const linked = (
77+
branch as { settings?: { am_v2?: { linked_workspaces?: QueryExportConfig['linkedWorkspaces'] } } }
78+
)?.settings?.am_v2?.linked_workspaces;
7879
this.exportQueryConfig.linkedWorkspaces = Array.isArray(linked) ? linked : [];
7980
log.debug(
8081
`Linked workspaces for Contentstack Assets: ${this.exportQueryConfig.linkedWorkspaces?.length ?? 0}`,
@@ -90,10 +91,7 @@ export class QueryExporter {
9091
}
9192

9293
private isCsAssetsExport(): boolean {
93-
return (
94-
(this.exportQueryConfig.linkedWorkspaces?.length ?? 0) > 0 &&
95-
Boolean(this.exportQueryConfig.csAssetsUrl)
96-
);
94+
return (this.exportQueryConfig.linkedWorkspaces?.length ?? 0) > 0 && Boolean(this.exportQueryConfig.csAssetsUrl);
9795
}
9896

9997
/**
@@ -156,14 +154,8 @@ export class QueryExporter {
156154
log.info('Starting export of referenced content types and dependent modules...', this.exportQueryConfig.context);
157155

158156
try {
159-
const ctPath = path.join(
160-
sanitizePath(this.exportQueryConfig.exportDir),
161-
'content_types',
162-
);
163-
const gfPath = path.join(
164-
sanitizePath(this.exportQueryConfig.exportDir),
165-
'global_fields',
166-
);
157+
const ctPath = path.join(sanitizePath(this.exportQueryConfig.exportDir), 'content_types');
158+
const gfPath = path.join(sanitizePath(this.exportQueryConfig.exportDir), 'global_fields');
167159

168160
const referencedHandler = new ReferencedContentTypesHandler(this.exportQueryConfig);
169161
const dependenciesHandler = new ContentTypeDependenciesHandler(this.stackAPIClient, this.exportQueryConfig);
@@ -265,15 +257,21 @@ export class QueryExporter {
265257
}
266258

267259
if (!foundNewCTs && !foundNewGFs) {
268-
log.info('Schema closure complete, no new content types or global fields found', this.exportQueryConfig.context);
260+
log.info(
261+
'Schema closure complete, no new content types or global fields found',
262+
this.exportQueryConfig.context,
263+
);
269264
break;
270265
}
271266
}
272267

273268
// Personalize is a single global module exported once after the closure stabilises.
274269
await this.moduleExporter.exportModule('personalize');
275270

276-
log.success('Referenced content types and dependent modules exported successfully', this.exportQueryConfig.context);
271+
log.success(
272+
'Referenced content types and dependent modules exported successfully',
273+
this.exportQueryConfig.context,
274+
);
277275
} catch (error) {
278276
handleAndLogError(error, this.exportQueryConfig.context, 'Error during schema closure expansion');
279277
throw error;
@@ -374,11 +372,7 @@ export class QueryExporter {
374372
* Export referenced assets into stack assets/ via CMA export module.
375373
*/
376374
private async exportReferencedStackAssets(assetUIDs: string[]): Promise<void> {
377-
const assetsDir = path.join(
378-
sanitizePath(this.exportQueryConfig.exportDir),
379-
sanitizePath(this.exportQueryConfig.branchName || ''),
380-
'assets',
381-
);
375+
const assetsDir = path.join(sanitizePath(this.exportQueryConfig.exportDir), 'assets');
382376

383377
const metadataFilePath = path.join(assetsDir, 'metadata.json');
384378
const assetFilePath = path.join(assetsDir, 'assets.json');
@@ -388,101 +382,101 @@ export class QueryExporter {
388382
const tempAssetFilePath = path.join(assetsDir, 'assets_temp.json');
389383

390384
try {
391-
fs.mkdirSync(assetsDir, { recursive: true });
392-
393-
// Define batch size - can be configurable through exportQueryConfig
394-
const batchSize = this.exportQueryConfig.assetBatchSize || 100;
395-
396-
// if asset size is bigger than batch size, then we need to export in batches
397-
// Calculate number of batches
398-
const totalBatches = Math.ceil(assetUIDs.length / batchSize);
399-
log.info(`Processing assets in ${totalBatches} batches of ${batchSize}`, this.exportQueryConfig.context);
385+
fs.mkdirSync(assetsDir, { recursive: true });
386+
387+
// Define batch size - can be configurable through exportQueryConfig
388+
const batchSize = this.exportQueryConfig.assetBatchSize || 100;
389+
390+
// if asset size is bigger than batch size, then we need to export in batches
391+
// Calculate number of batches
392+
const totalBatches = Math.ceil(assetUIDs.length / batchSize);
393+
log.info(`Processing assets in ${totalBatches} batches of ${batchSize}`, this.exportQueryConfig.context);
394+
395+
// Process assets in batches
396+
for (let i = 0; i < totalBatches; i++) {
397+
const start = i * batchSize;
398+
const end = Math.min(start + batchSize, assetUIDs.length);
399+
const batchAssetUIDs = assetUIDs.slice(start, end);
400+
401+
log.info(
402+
`Exporting batch ${i + 1}/${totalBatches} (${batchAssetUIDs.length} assets)...`,
403+
this.exportQueryConfig.context,
404+
);
405+
406+
const query = {
407+
modules: {
408+
assets: {
409+
uid: { $in: batchAssetUIDs },
410+
},
411+
},
412+
};
413+
414+
await this.moduleExporter.exportModule('assets', { query });
415+
416+
// Read the current batch's metadata.json and assets.json files
417+
const currentMetadata: any = fsUtil.readFile(sanitizePath(metadataFilePath));
418+
const currentAssets: any = fsUtil.readFile(sanitizePath(assetFilePath));
419+
420+
// Check if this is the first batch
421+
if (i === 0) {
422+
// For first batch, initialize temp files with current content
423+
fsUtil.writeFile(sanitizePath(tempMetadataFilePath), currentMetadata);
424+
fsUtil.writeFile(sanitizePath(tempAssetFilePath), currentAssets);
425+
log.info(`Initialized temporary files with first batch data`, this.exportQueryConfig.context);
426+
} else {
427+
// For subsequent batches, append to temp files with incremented keys
428+
429+
// Handle metadata (which contains arrays of asset info)
430+
const tempMetadata: any = fsUtil.readFile(sanitizePath(tempMetadataFilePath)) || {};
431+
432+
// Merge metadata by combining arrays
433+
if (currentMetadata) {
434+
Object.keys(currentMetadata).forEach((key: string) => {
435+
if (!tempMetadata[key]) {
436+
tempMetadata[key] = currentMetadata[key];
437+
}
438+
});
439+
}
400440

401-
// Process assets in batches
402-
for (let i = 0; i < totalBatches; i++) {
403-
const start = i * batchSize;
404-
const end = Math.min(start + batchSize, assetUIDs.length);
405-
const batchAssetUIDs = assetUIDs.slice(start, end);
441+
// Write updated metadata back to temp file
442+
fsUtil.writeFile(sanitizePath(tempMetadataFilePath), tempMetadata);
406443

407-
log.info(
408-
`Exporting batch ${i + 1}/${totalBatches} (${batchAssetUIDs.length} assets)...`,
409-
this.exportQueryConfig.context,
410-
);
444+
// Handle assets (which is an object with numeric keys)
445+
const tempAssets: any = fsUtil.readFile(sanitizePath(tempAssetFilePath)) || {};
446+
let nextIndex = Object.keys(tempAssets).length + 1;
411447

412-
const query = {
413-
modules: {
414-
assets: {
415-
uid: { $in: batchAssetUIDs },
416-
},
417-
},
418-
};
419-
420-
await this.moduleExporter.exportModule('assets', { query });
421-
422-
// Read the current batch's metadata.json and assets.json files
423-
const currentMetadata: any = fsUtil.readFile(sanitizePath(metadataFilePath));
424-
const currentAssets: any = fsUtil.readFile(sanitizePath(assetFilePath));
425-
426-
// Check if this is the first batch
427-
if (i === 0) {
428-
// For first batch, initialize temp files with current content
429-
fsUtil.writeFile(sanitizePath(tempMetadataFilePath), currentMetadata);
430-
fsUtil.writeFile(sanitizePath(tempAssetFilePath), currentAssets);
431-
log.info(`Initialized temporary files with first batch data`, this.exportQueryConfig.context);
432-
} else {
433-
// For subsequent batches, append to temp files with incremented keys
434-
435-
// Handle metadata (which contains arrays of asset info)
436-
const tempMetadata: any = fsUtil.readFile(sanitizePath(tempMetadataFilePath)) || {};
437-
438-
// Merge metadata by combining arrays
439-
if (currentMetadata) {
440-
Object.keys(currentMetadata).forEach((key: string) => {
441-
if (!tempMetadata[key]) {
442-
tempMetadata[key] = currentMetadata[key];
443-
}
444-
});
445-
}
446-
447-
// Write updated metadata back to temp file
448-
fsUtil.writeFile(sanitizePath(tempMetadataFilePath), tempMetadata);
449-
450-
// Handle assets (which is an object with numeric keys)
451-
const tempAssets: any = fsUtil.readFile(sanitizePath(tempAssetFilePath)) || {};
452-
let nextIndex = Object.keys(tempAssets).length + 1;
453-
454-
// Add current assets with incremented keys
455-
Object.values(currentAssets).forEach((value: any) => {
456-
tempAssets[nextIndex.toString()] = value;
457-
nextIndex++;
458-
});
448+
// Add current assets with incremented keys
449+
Object.values(currentAssets).forEach((value: any) => {
450+
tempAssets[nextIndex.toString()] = value;
451+
nextIndex++;
452+
});
459453

460-
fsUtil.writeFile(sanitizePath(tempAssetFilePath), tempAssets);
454+
fsUtil.writeFile(sanitizePath(tempAssetFilePath), tempAssets);
461455

462-
log.info(`Updated temporary files with batch ${i + 1} data`, this.exportQueryConfig.context);
463-
}
456+
log.info(`Updated temporary files with batch ${i + 1} data`, this.exportQueryConfig.context);
457+
}
464458

465-
// Optional: Add delay between batches to avoid rate limiting
466-
if (i < totalBatches - 1 && this.exportQueryConfig.batchDelayMs) {
467-
await new Promise((resolve) => setTimeout(resolve, this.exportQueryConfig.batchDelayMs));
468-
}
459+
// Optional: Add delay between batches to avoid rate limiting
460+
if (i < totalBatches - 1 && this.exportQueryConfig.batchDelayMs) {
461+
await new Promise((resolve) => setTimeout(resolve, this.exportQueryConfig.batchDelayMs));
469462
}
463+
}
470464

471-
// After all batches are processed, copy temp files back to original files
472-
const finalMetadata = fsUtil.readFile(sanitizePath(tempMetadataFilePath));
473-
const finalAssets = fsUtil.readFile(sanitizePath(tempAssetFilePath));
465+
// After all batches are processed, copy temp files back to original files
466+
const finalMetadata = fsUtil.readFile(sanitizePath(tempMetadataFilePath));
467+
const finalAssets = fsUtil.readFile(sanitizePath(tempAssetFilePath));
474468

475-
fsUtil.writeFile(sanitizePath(metadataFilePath), finalMetadata);
476-
fsUtil.writeFile(sanitizePath(assetFilePath), finalAssets);
469+
fsUtil.writeFile(sanitizePath(metadataFilePath), finalMetadata);
470+
fsUtil.writeFile(sanitizePath(assetFilePath), finalAssets);
477471

478-
log.info(`Final data written back to original files`, this.exportQueryConfig.context);
472+
log.info(`Final data written back to original files`, this.exportQueryConfig.context);
479473

480-
// Clean up temp files
481-
fsUtil.removeFile(sanitizePath(tempMetadataFilePath));
482-
fsUtil.removeFile(sanitizePath(tempAssetFilePath));
474+
// Clean up temp files
475+
fsUtil.removeFile(sanitizePath(tempMetadataFilePath));
476+
fsUtil.removeFile(sanitizePath(tempAssetFilePath));
483477

484-
log.info(`Temporary files cleaned up`, this.exportQueryConfig.context);
485-
log.success('Referenced assets exported successfully', this.exportQueryConfig.context);
478+
log.info(`Temporary files cleaned up`, this.exportQueryConfig.context);
479+
log.success('Referenced assets exported successfully', this.exportQueryConfig.context);
486480
} catch (error) {
487481
handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting stack referenced assets');
488482
throw error;

0 commit comments

Comments
 (0)