Merge pull request #307 from contentstack/enhc/DX-9464

feat(global-fields): switch to per-file export/import format

Author: cs-raj

packages/contentstack-audit/src/audit-base-command.ts

@@ -14,9 +14,10 @@ import {
   CLIProgressManager,
   clearProgressModuleSetting,
   readContentTypeSchemas,
+  readGlobalFieldSchemas,
   generateUid
 } from '@contentstack/cli-utilities';
-import { createWriteStream, existsSync, mkdirSync, readFileSync, writeFileSync, rmSync } from 'fs';
+import { createWriteStream, existsSync, mkdirSync, writeFileSync, rmSync } from 'fs';
 import config from './config';
 import { print } from './util/log';
 import { auditMsg } from './messages';
@@ -507,13 +508,9 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
    */
   getCtAndGfSchema() {
     const ctDirPath = join(this.sharedConfig.basePath, this.sharedConfig.moduleConfig['content-types'].dirName);
-    const gfPath = join(
-      this.sharedConfig.basePath,
-      this.sharedConfig.moduleConfig['global-fields'].dirName,
-      this.sharedConfig.moduleConfig['global-fields'].fileName,
-    );
+    const gfDirPath = join(this.sharedConfig.basePath, this.sharedConfig.moduleConfig['global-fields'].dirName);
 
-    const gfSchema = existsSync(gfPath) ? (JSON.parse(readFileSync(gfPath, 'utf8')) as ContentTypeStruct[]) : [];
+    const gfSchema = (readGlobalFieldSchemas(gfDirPath) || []) as ContentTypeStruct[];
     const ctSchema = (readContentTypeSchemas(ctDirPath) || []) as ContentTypeStruct[];
 
     return { ctSchema, gfSchema };

packages/contentstack-audit/src/config/index.ts

@@ -24,7 +24,7 @@ const config = {
     'global-fields': {
       name: 'global field',
       dirName: 'global_fields',
-      fileName: 'globalfields.json',
+      fileName: 'globalfields.json', // Not used - reads from individual files via readGlobalFieldSchemas
     },
     entries: {
       name: 'entries',

packages/contentstack-audit/test/unit/audit-base-command.test.ts

@@ -369,6 +369,30 @@ describe('AuditBaseCommand class', () => {
           gfSchema: [],
         });
       });
+
+    fancy
+      .stdout({ print: process.env.PRINT === 'true' || false })
+      .stub(winston.transports, 'File', () => fsTransport)
+      .stub(winston, 'createLogger', createMockWinstonLogger)
+      .stub(fs, 'createWriteStream', () => new PassThrough())
+      .it('should return per-uid global field schemas and skip globalfields.json', async () => {
+        class CMD extends AuditBaseCommand {
+          async run() {
+            // Point basePath at mock/contents which has global_fields/gf_1.json (per-uid)
+            // and global_fields/globalfields.json (bulk legacy — must be ignored)
+            this.sharedConfig.basePath = resolve(__dirname, 'mock', 'contents');
+            return this.getCtAndGfSchema();
+          }
+        }
+
+        const result = await CMD.run([]);
+        // gf_1.json is read; globalfields.json is excluded by readGlobalFieldSchemas
+        expect(result.gfSchema).to.be.an('array');
+        expect(result.gfSchema.length).to.equal(1);
+        expect((result.gfSchema[0] as any).uid).to.equal('gf_1');
+        // content_types dir absent → ctSchema empty
+        expect(result.ctSchema).to.deep.equal([]);
+      });
   });
 
   describe('Progress Manager Integration', () => {

packages/contentstack-audit/test/unit/mock/contents/global_fields/gf_1.json

@@ -0,0 +1,15 @@
+{
+  "uid": "gf_1",
+  "title": "GF - 1",
+  "schema": [
+    {
+      "data_type": "text",
+      "display_name": "Single Line Textbox",
+      "uid": "single_line",
+      "mandatory": false,
+      "multiple": false,
+      "non_localizable": false,
+      "unique": false
+    }
+  ]
+}

packages/contentstack-export/src/export/modules/global-fields.ts

@@ -17,14 +17,12 @@ export default class GlobalFieldsExport extends BaseClass {
   };
   private globalFieldsConfig: {
     dirName?: string;
-    fileName?: string;
     validKeys?: string[];
     fetchConcurrency?: number;
     writeConcurrency?: number;
     limit?: number;
   };
   private globalFieldsDirPath: string;
-  private globalFields: Record<string, unknown>[];
 
   constructor({ exportConfig, stackAPIClient }: ModuleClassParams) {
     super({ exportConfig, stackAPIClient });
@@ -41,7 +39,6 @@ export default class GlobalFieldsExport extends BaseClass {
       sanitizePath(getExportBasePath(exportConfig)),
       sanitizePath(this.globalFieldsConfig.dirName),
     );
-    this.globalFields = [];
     this.applyQueryFilters(this.qs, 'global-fields');
     this.exportConfig.context.module = MODULE_CONTEXTS.GLOBAL_FIELDS;
     this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.GLOBAL_FIELDS];
@@ -66,20 +63,13 @@ export default class GlobalFieldsExport extends BaseClass {
 
       if (totalCount === 0) {
         log.info(messageHandler.parse('GLOBAL_FIELDS_NOT_FOUND'), this.exportConfig.context);
-        const globalFieldsFilePath = path.join(this.globalFieldsDirPath, this.globalFieldsConfig.fileName);
-        log.debug(`Writing global fields to: ${globalFieldsFilePath}`, this.exportConfig.context);
-        fsUtil.writeFile(globalFieldsFilePath, this.globalFields);
         this.completeProgress(true);
         return;
       }
 
       progress.updateStatus('Fetching global fields...');
       await this.getGlobalFields();
 
-      const globalFieldsFilePath = path.join(this.globalFieldsDirPath, this.globalFieldsConfig.fileName);
-      log.debug(`Writing global fields to: ${globalFieldsFilePath}`, this.exportConfig.context);
-      fsUtil.writeFile(globalFieldsFilePath, this.globalFields);
-
 
 
       this.completeProgressWithMessage();
@@ -133,14 +123,15 @@ export default class GlobalFieldsExport extends BaseClass {
           delete globalField[key];
         }
       }
-      this.globalFields.push(globalField);
+      const filePath = path.join(this.globalFieldsDirPath, `${globalField.uid}.json`);
+      fsUtil.writeFile(filePath, globalField);
 
       // Track progress for each global field
       this.progressManager?.tick(true, `global-field: ${globalField.uid}`);
     });
 
     log.debug(
-      `Sanitization complete. Total global fields processed: ${this.globalFields.length}`,
+      `Sanitization complete. Total global fields processed: ${globalFields.length}`,
       this.exportConfig.context,
     );
   }

packages/contentstack-export/test/unit/export/modules/global-fields.test.ts

@@ -112,9 +112,9 @@ describe('ExportGlobalFields', () => {
       });
     });
 
-    it('should initialize empty globalFields array', () => {
-      expect(exportGlobalFields.globalFields).to.be.an('array');
-      expect(exportGlobalFields.globalFields.length).to.equal(0);
+    it('should initialize global fields dir path', () => {
+      expect(exportGlobalFields.globalFieldsDirPath).to.be.a('string');
+      expect(exportGlobalFields.globalFieldsDirPath).to.include('global_fields');
     });
 
     it('should set correct directory path', () => {
@@ -140,12 +140,12 @@ describe('ExportGlobalFields', () => {
 
       await exportGlobalFields.getGlobalFields();
 
-      // Verify global fields were processed
-      expect(exportGlobalFields.globalFields.length).to.equal(2);
-      // Verify invalid keys were removed
-      expect(exportGlobalFields.globalFields[0].invalidKey).to.be.undefined;
-      expect(exportGlobalFields.globalFields[0].uid).to.equal('gf-1');
-      expect(exportGlobalFields.globalFields[0].title).to.equal('Field 1');
+      // Verify each global field written to its own per-uid file (not globalfields.json)
+      const writeFileStub = FsUtility.prototype.writeFile as sinon.SinonStub;
+      expect(writeFileStub.callCount).to.equal(2);
+      expect(writeFileStub.calledWith(sinon.match(/gf-1\.json/))).to.be.true;
+      expect(writeFileStub.calledWith(sinon.match(/gf-2\.json/))).to.be.true;
+      expect(writeFileStub.neverCalledWith(sinon.match(/globalfields\.json/))).to.be.true;
     });
 
     it('should call getGlobalFields recursively when more fields exist', async () => {
@@ -171,9 +171,10 @@ describe('ExportGlobalFields', () => {
 
       await exportGlobalFields.getGlobalFields();
 
-      // Verify multiple calls were made
+      // Verify multiple calls were made and each item written as per-uid file
       expect(callCount).to.be.greaterThan(1);
-      expect(exportGlobalFields.globalFields.length).to.equal(150);
+      const writeFileStub = FsUtility.prototype.writeFile as sinon.SinonStub;
+      expect(writeFileStub.callCount).to.equal(150);
     });
 
     it('should handle API errors gracefully', async () => {
@@ -201,11 +202,11 @@ describe('ExportGlobalFields', () => {
         }),
       });
 
-      const initialCount = exportGlobalFields.globalFields.length;
+      const writeFileStub = FsUtility.prototype.writeFile as sinon.SinonStub;
       await exportGlobalFields.getGlobalFields();
 
-      // Verify no new global fields were added
-      expect(exportGlobalFields.globalFields.length).to.equal(initialCount);
+      // No items → writeFile never called
+      expect(writeFileStub.called).to.be.false;
     });
 
     it('should handle empty items array', async () => {
@@ -218,11 +219,11 @@ describe('ExportGlobalFields', () => {
         }),
       });
 
-      const initialCount = exportGlobalFields.globalFields.length;
+      const writeFileStub = FsUtility.prototype.writeFile as sinon.SinonStub;
       await exportGlobalFields.getGlobalFields();
 
-      // Verify no processing occurred with null items
-      expect(exportGlobalFields.globalFields.length).to.equal(initialCount);
+      // Null items → writeFile never called
+      expect(writeFileStub.called).to.be.false;
     });
 
     it('should update query params with skip value', async () => {
@@ -251,28 +252,33 @@ describe('ExportGlobalFields', () => {
 
       exportGlobalFields.sanitizeAttribs(globalFields);
 
-      // Verify invalid keys were removed
-      expect(exportGlobalFields.globalFields[0].invalidKey).to.be.undefined;
-      expect(exportGlobalFields.globalFields[0].uid).to.equal('gf-1');
-      expect(exportGlobalFields.globalFields[0].title).to.equal('Field 1');
-      expect(exportGlobalFields.globalFields[0].validKey).to.equal('value1');
+      // sanitizeAttribs modifies the array elements in-place
+      expect(globalFields[0].invalidKey).to.be.undefined;
+      expect(globalFields[0].uid).to.equal('gf-1');
+      expect(globalFields[0].title).to.equal('Field 1');
+      expect(globalFields[0].validKey).to.equal('value1');
+      // Each field written to its own per-uid file (never to globalfields.json)
+      const writeFileStub = FsUtility.prototype.writeFile as sinon.SinonStub;
+      expect(writeFileStub.calledWith(sinon.match(/gf-1\.json/))).to.be.true;
+      expect(writeFileStub.calledWith(sinon.match(/gf-2\.json/))).to.be.true;
+      expect(writeFileStub.neverCalledWith(sinon.match(/globalfields\.json/))).to.be.true;
     });
 
     it('should handle global fields without required keys', () => {
       const globalFields = [{ uid: 'gf-1', invalidKey: 'remove' }];
 
       exportGlobalFields.sanitizeAttribs(globalFields);
 
-      expect(exportGlobalFields.globalFields[0]).to.exist;
-      expect(exportGlobalFields.globalFields[0].invalidKey).to.be.undefined;
+      expect(globalFields[0]).to.exist;
+      expect(globalFields[0].invalidKey).to.be.undefined;
     });
 
     it('should handle empty global fields array', () => {
       const globalFields: any[] = [];
 
       exportGlobalFields.sanitizeAttribs(globalFields);
 
-      expect(exportGlobalFields.globalFields.length).to.equal(0);
+      expect(globalFields.length).to.equal(0);
     });
 
     it('should keep only valid keys from validKeys config', () => {
@@ -289,7 +295,7 @@ describe('ExportGlobalFields', () => {
 
       exportGlobalFields.sanitizeAttribs(globalFields);
 
-      const processedField = exportGlobalFields.globalFields[0];
+      const processedField = globalFields[0];
 
       // Should only keep uid, title, validKey
       expect(processedField.keyToRemove1).to.be.undefined;
@@ -298,6 +304,10 @@ describe('ExportGlobalFields', () => {
       expect(processedField.uid).to.equal('gf-1');
       expect(processedField.title).to.equal('Field 1');
       expect(processedField.validKey).to.equal('value1');
+      // Written to per-uid file path
+      const writeFileStub = FsUtility.prototype.writeFile as sinon.SinonStub;
+      expect(writeFileStub.calledWith(sinon.match(/gf-1\.json/))).to.be.true;
+      expect(writeFileStub.neverCalledWith(sinon.match(/globalfields\.json/))).to.be.true;
     });
   });
 
@@ -322,16 +332,14 @@ describe('ExportGlobalFields', () => {
 
       await exportGlobalFields.start();
 
-      // Verify global fields were processed
-      expect(exportGlobalFields.globalFields.length).to.equal(2);
-      expect(exportGlobalFields.globalFields[0].uid).to.equal('gf-1');
-      expect(exportGlobalFields.globalFields[1].uid).to.equal('gf-2');
-      // Verify file was written
-      expect(writeFileStub.called).to.be.true;
+      // Verify each field written to its own per-uid file (not globalfields.json)
+      expect(writeFileStub.calledWith(sinon.match(/gf-1\.json/))).to.be.true;
+      expect(writeFileStub.calledWith(sinon.match(/gf-2\.json/))).to.be.true;
+      expect(writeFileStub.neverCalledWith(sinon.match(/globalfields\.json/))).to.be.true;
       expect(makeDirectoryStub.called).to.be.true;
     });
 
-    it('should handle empty global fields and still write file', async () => {
+    it('should return early when no global fields found', async () => {
       const writeFileStub = FsUtility.prototype.writeFile as sinon.SinonStub;
 
       mockStackClient.globalField.returns({
@@ -343,12 +351,10 @@ describe('ExportGlobalFields', () => {
         }),
       });
 
-      exportGlobalFields.globalFields = [];
       await exportGlobalFields.start();
 
-      // Verify writeFile was called even with empty array
-      expect(writeFileStub.called).to.be.true;
-      expect(exportGlobalFields.globalFields.length).to.equal(0);
+      // start() exits early on count=0 — no per-uid files written
+      expect(writeFileStub.called).to.be.false;
     });
 
     it('should handle errors during export without throwing', async () => {
@@ -398,9 +404,11 @@ describe('ExportGlobalFields', () => {
 
       await exportGlobalFields.start();
 
-      // Verify all fields were processed
-      expect(exportGlobalFields.globalFields.length).to.equal(150);
+      // Verify all fields processed — one writeFile call per item, no globalfields.json
+      const writeFileStub = FsUtility.prototype.writeFile as sinon.SinonStub;
+      expect(writeFileStub.callCount).to.equal(150);
       expect(callCount).to.be.greaterThan(1);
+      expect(writeFileStub.neverCalledWith(sinon.match(/globalfields\.json/))).to.be.true;
     });
 
     it('should call makeDirectory and writeFile with correct paths', async () => {
@@ -418,9 +426,10 @@ describe('ExportGlobalFields', () => {
 
       await exportGlobalFields.start();
 
-      // Verify directories and files were created
+      // Verify directory created and per-uid file written (not globalfields.json)
       expect(makeDirectoryStub.called).to.be.true;
-      expect(writeFileStub.called).to.be.true;
+      expect(writeFileStub.calledWith(sinon.match(/gf-1\.json/))).to.be.true;
+      expect(writeFileStub.neverCalledWith(sinon.match(/globalfields\.json/))).to.be.true;
     });
   });
 });

packages/contentstack-import-setup/src/config/index.ts

@@ -87,7 +87,7 @@ const config: DefaultConfig = {
     },
     'global-fields': {
       dirName: 'global_fields',
-      fileName: 'globalfields.json',
+      fileName: 'globalfields.json', // Not used - reads individual {uid}.json files
       dependencies: ['extensions', 'marketplace-apps'],
     },
     'marketplace-apps': {

packages/contentstack-import/src/config/index.ts

@@ -177,7 +177,7 @@ const config: DefaultConfig = {
     },
     'global-fields': {
       dirName: 'global_fields',
-      fileName: 'globalfields.json',
+      fileName: 'globalfields.json', // Not used - reads individual {uid}.json files
       validKeys: ['title', 'uid', 'schema', 'options', 'singleton', 'description'],
       limit: 100,
     },

packages/contentstack-import/src/import/modules/content-types.ts

@@ -6,7 +6,7 @@
  */
 import * as path from 'path';
 import { find, cloneDeep, map } from 'lodash';
-import { sanitizePath, log, handleAndLogError, readContentTypeSchemas } from '@contentstack/cli-utilities';
+import { sanitizePath, log, handleAndLogError, readContentTypeSchemas, readGlobalFieldSchemas } from '@contentstack/cli-utilities';
 import { PATH_CONSTANTS } from '../../constants';
 import { ImportConfig, ModuleClassParams } from '../../types';
 import BaseClass, { ApiOptions } from './base-class';
@@ -50,7 +50,6 @@ export default class ContentTypesImport extends BaseClass {
   };
   private gFsConfig: {
     dirName: string;
-    fileName: string;
     validKeys: string[];
     limit: number;
     writeConcurrency?: number;
@@ -105,6 +104,7 @@ export default class ContentTypesImport extends BaseClass {
       ['__master.json', 'true'],
       ['__priority.json', 'true'],
       [PATH_CONSTANTS.FILES.SCHEMA, 'true'],
+      ['globalfields.json', 'true'],
       ['.DS_Store', 'true'],
     ]);
 
@@ -584,7 +584,7 @@ export default class ContentTypesImport extends BaseClass {
       'CONTENT TYPES: Analyzing import data...',
       async () => {
         const cts = readContentTypeSchemas(this.cTsFolderPath);
-        const gfs = fsUtil.readFile(path.resolve(this.gFsFolderPath, this.gFsConfig.fileName));
+        const gfs = readGlobalFieldSchemas(this.gFsFolderPath);
         const pendingGfs = fsUtil.readFile(this.gFsPendingPath);
         const pendingExt = fsUtil.readFile(this.extPendingPath);
         return [cts, gfs, pendingGfs, pendingExt];