fix: handle +100K comps when resolving from org W-17876861 (#1511)

* test: add UT

* chore: bump sfdx-core

* test: increase class qty

* chore: simplify listMember, avoid currying

* fix: array concat to avoid stack overflow

* fix: set default listMetadata batch limit

---------

Co-authored-by: Steve Hetzel <[email protected]>

Author: cristiand391

src/resolve/connectionResolver.ts

@@ -73,8 +73,8 @@ export class ConnectionResolver {
     shouldQueryStandardValueSets = false;
 
     // To limit the number of concurrent requests, batch them per an env var.
-    // By default there is no batching.
-    this.requestBatchSize = env.getNumber('SF_LIST_METADATA_BATCH_SIZE', -1);
+    // Default is 500. From testing we saw jsforce gets stuck on ~1K reqs.
+    this.requestBatchSize = env.getNumber('SF_LIST_METADATA_BATCH_SIZE', 500);
   }
 
   public async resolve(
@@ -171,12 +171,12 @@ export class ConnectionResolver {
 
   // Send batched listMetadata requests based on the SF_LIST_METADATA_BATCH_SIZE env var.
   private async sendBatchedRequests(listMdQueries: string[]): Promise<RelevantFileProperties[]> {
-    const listMetadataResponses: RelevantFileProperties[] = [];
+    let listMetadataResponses: RelevantFileProperties[] = [];
     let listMetadataRequests: Array<Promise<RelevantFileProperties[]>> = [];
 
     const sendIt = async (): Promise<void> => {
       const requestBatch = (await Promise.all(listMetadataRequests)).flat();
-      listMetadataResponses.push(...requestBatch);
+      listMetadataResponses = listMetadataResponses.concat(requestBatch);
     };
 
     // Make batched listMetadata requests
@@ -186,7 +186,7 @@ export class ConnectionResolver {
       if (q[1]) {
         listMdQuery.folder = q[1];
       }
-      listMetadataRequests.push(listMembers(this.registry)(this.connection)(listMdQuery));
+      listMetadataRequests.push(listMembers(this.registry, this.connection, listMdQuery));
       i++;
       if (this.requestBatchSize > 0 && i % this.requestBatchSize === 0) {
         getLogger().debug(`Awaiting listMetadata requests ${i - this.requestBatchSize + 1} - ${i}`);
@@ -212,12 +212,12 @@ export class ConnectionResolver {
   // SF_LIST_METADATA_BATCH_SIZE env var.
   private async sendBatchedQueries(): Promise<RelevantFileProperties[]> {
     const mdType = this.registry.getTypeByName('StandardValueSet');
-    const queryResponses: RelevantFileProperties[] = [];
+    let queryResponses: RelevantFileProperties[] = [];
     let queryRequests: Array<Promise<RelevantFileProperties | undefined>> = [];
 
     const sendIt = async (): Promise<void> => {
       const requestBatch = (await Promise.all(queryRequests)).flat();
-      queryResponses.push(...requestBatch.filter((rb) => !!rb));
+      queryResponses = queryResponses.concat(requestBatch.filter((rb) => !!rb));
     };
 
     // Make batched query requests
@@ -270,57 +270,58 @@ const querySvs =
     }
   };
 
-const listMembers =
-  (registry: RegistryAccess) =>
-  (connection: Connection) =>
-  async (query: ListMetadataQuery): Promise<RelevantFileProperties[]> => {
-    const mdType = registry.getTypeByName(query.type);
-
-    // Workaround because metadata.list({ type: 'StandardValueSet' }) returns [].
-    // Query for a subset of known StandardValueSets after all listMetadata calls.
-    if (mdType.name === registry.getRegistry().types.standardvalueset.name) {
-      shouldQueryStandardValueSets = true;
-      return [];
-    }
-
-    // Workaround because metadata.list({ type: 'BotVersion' }) returns [].
-    if (mdType.name === 'BotVersion') {
-      try {
-        const botDefQuery = 'SELECT Id, DeveloperName FROM BotDefinition';
-        const botVersionQuery = 'SELECT BotDefinitionId, DeveloperName FROM BotVersion';
-        const botDefs = (await connection.query<{ Id: string; DeveloperName: string }>(botDefQuery)).records;
-        const botVersionDefs = (
-          await connection.query<{ BotDefinitionId: string; DeveloperName: string }>(botVersionQuery)
-        ).records;
-        return botVersionDefs
-          .map((bvd) => {
-            const botName = botDefs.find((bd) => bd.Id === bvd.BotDefinitionId)?.DeveloperName;
-            if (botName) {
-              return {
-                fullName: `${botName}.${bvd.DeveloperName}`,
-                fileName: `bots/${bvd.DeveloperName}.botVersion`,
-                type: 'BotVersion',
-              };
-            }
-          })
-          .filter((b) => !!b);
-      } catch (error) {
-        const err = SfError.wrap(error);
-        getLogger().debug(`[${mdType.name}] ${err.message}`);
-        return [];
-      }
-    }
+async function listMembers(
+  registry: RegistryAccess,
+  connection: Connection,
+  query: ListMetadataQuery
+): Promise<RelevantFileProperties[]> {
+  const mdType = registry.getTypeByName(query.type);
+
+  // Workaround because metadata.list({ type: 'StandardValueSet' }) returns [].
+  // Query for a subset of known StandardValueSets after all listMetadata calls.
+  if (mdType.name === registry.getRegistry().types.standardvalueset.name) {
+    shouldQueryStandardValueSets = true;
+    return [];
+  }
 
+  // Workaround because metadata.list({ type: 'BotVersion' }) returns [].
+  if (mdType.name === 'BotVersion') {
     try {
-      requestCount++;
-      getLogger().debug(`listMetadata for ${inspect(query)}`);
-      return (await connection.metadata.list(query)).map(inferFilenamesFromType(mdType));
+      const botDefQuery = 'SELECT Id, DeveloperName FROM BotDefinition';
+      const botVersionQuery = 'SELECT BotDefinitionId, DeveloperName FROM BotVersion';
+      const botDefs = (await connection.query<{ Id: string; DeveloperName: string }>(botDefQuery)).records;
+      const botVersionDefs = (
+        await connection.query<{ BotDefinitionId: string; DeveloperName: string }>(botVersionQuery)
+      ).records;
+      return botVersionDefs
+        .map((bvd) => {
+          const botName = botDefs.find((bd) => bd.Id === bvd.BotDefinitionId)?.DeveloperName;
+          if (botName) {
+            return {
+              fullName: `${botName}.${bvd.DeveloperName}`,
+              fileName: `bots/${bvd.DeveloperName}.botVersion`,
+              type: 'BotVersion',
+            };
+          }
+        })
+        .filter((b) => !!b);
     } catch (error) {
       const err = SfError.wrap(error);
       getLogger().debug(`[${mdType.name}] ${err.message}`);
       return [];
     }
-  };
+  }
+
+  try {
+    requestCount++;
+    getLogger().debug(`listMetadata for ${inspect(query)}`);
+    return (await connection.metadata.list(query)).map(inferFilenamesFromType(mdType));
+  } catch (error) {
+    const err = SfError.wrap(error);
+    getLogger().debug(`[${mdType.name}] ${err.message}`);
+    return [];
+  }
+}
 
 /* if the Metadata Type doesn't return a correct fileName then help it out */
 const inferFilenamesFromType =

test/resolve/connectionResolver.test.ts

@@ -168,6 +168,36 @@ describe('ConnectionResolver', () => {
       expect(metadataQueryStub.calledOnce).to.be.true;
     });
 
+    it('should resolve +100K components', async () => {
+      const promiseAllSpy = $$.SANDBOX.spy(Promise, 'all');
+      // @ts-expect-error spying on private method
+      const resolverSpy = $$.SANDBOX.spy(ConnectionResolver.prototype, 'sendBatchedQueries');
+
+      const metadataListStub = $$.SANDBOX.stub(connection.metadata, 'list');
+
+      const classesQty = 200_000;
+      const tonsOfApexClasses = Array.from({ length: classesQty }, (_value, index) => ({
+        ...StdFileProperty,
+        fileName: `classes/MyApexClass${index}.class`,
+        fullName: `MyApexClass${index}`,
+        type: 'ApexClass',
+      }));
+
+      metadataListStub.withArgs({ type: 'ApexClass' }).resolves(tonsOfApexClasses);
+
+      const mdTypes = ['ApexClass'];
+      const resolver = new ConnectionResolver(connection, undefined, mdTypes);
+      const result = await resolver.resolve();
+      const expected = Array.from({ length: classesQty }, (_value, index) => ({
+        fullName: `MyApexClass${index}`,
+        type: registry.types.apexclass,
+      }));
+
+      expect(result.components).to.deep.equal(expected);
+      expect(promiseAllSpy.callCount, 'Expected Promise.all() to be called 2 times').to.equal(1);
+      expect(promiseAllSpy.firstCall.args[0]).to.be.an('array').with.lengthOf(1);
+      expect(resolverSpy.called).to.be.false;
+    });
     it('should batch requests per SF_LIST_METADATA_BATCH_SIZE', async () => {
       $$.SANDBOX.stub(env, 'getNumber').returns(2);
       const promiseAllSpy = $$.SANDBOX.spy(Promise, 'all');