feat: add retrieve in mdapi format (#519)

* feat: add retrieve in mdapi format

* feat: add unit tests for mdapi retrieve

* chore: adding types for SDR

* fix: syntax improvement

* fix: do not send empty unpackaged manifest

Co-authored-by: Shane McLaughlin <[email protected]>

Author: shetzel

src/client/metadataApiRetrieve.ts

@@ -4,6 +4,9 @@
  * Licensed under the BSD 3-Clause license.
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
+import * as path from 'path';
+import * as fs from 'graceful-fs';
+import * as unzipper from 'unzipper';
 import { asBoolean, isString } from '@salesforce/ts-types';
 import { ConvertOutputConfig, MetadataConverter } from '../convert';
 import { ComponentSet } from '../collections';
@@ -152,6 +155,38 @@ export class MetadataApiRetrieve extends MetadataTransfer<MetadataApiRetrieveSta
     this.canceled = true;
   }
 
+  public async post(result: MetadataApiRetrieveStatus): Promise<RetrieveResult> {
+    let components: ComponentSet;
+    const isMdapiRetrieve = this.options.format === 'metadata';
+
+    if (result.status === RequestStatus.Succeeded) {
+      const zipFileContents = Buffer.from(result.zipFile, 'base64');
+      if (isMdapiRetrieve) {
+        const name = this.options.zipFileName || 'unpackaged.zip';
+        const zipFilePath = path.join(this.options.output, name);
+        fs.writeFileSync(zipFilePath, zipFileContents);
+
+        if (this.options.unzip) {
+          const dir = await unzipper.Open.buffer(zipFileContents);
+          const extractPath = path.join(this.options.output, path.parse(name).name);
+          await dir.extract({ path: extractPath });
+        }
+      } else {
+        components = await this.extract(zipFileContents);
+      }
+    }
+
+    components ??= new ComponentSet(undefined, this.options.registry);
+
+    if (!isMdapiRetrieve) {
+      // This should only be done when retrieving source format since retrieving
+      // mdapi format has no conversion.
+      await this.maybeSaveTempDirectory('source', components);
+    }
+
+    return new RetrieveResult(result, components, this.components);
+  }
+
   protected async pre(): Promise<AsyncResult> {
     const packageNames = this.getPackageNames();
 
@@ -169,26 +204,21 @@ export class MetadataApiRetrieve extends MetadataTransfer<MetadataApiRetrieveSta
     // otherwise don't - it causes errors if undefined or an empty array
     if (packageNames?.length) {
       requestBody.packageNames = packageNames;
+      // delete unpackaged when no components and metadata format to prevent
+      // sending an empty unpackaged manifest.
+      if (this.options.format === 'metadata' && this.components.size === 0) {
+        delete requestBody.unpackaged;
+      }
+    }
+    if (this.options.singlePackage) {
+      requestBody.singlePackage = this.options.singlePackage;
     }
 
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     // @ts-ignore required callback
     return connection.metadata.retrieve(requestBody);
   }
 
-  protected async post(result: MetadataApiRetrieveStatus): Promise<RetrieveResult> {
-    let components: ComponentSet;
-    if (result.status === RequestStatus.Succeeded) {
-      components = await this.extract(Buffer.from(result.zipFile, 'base64'));
-    }
-
-    components = components ?? new ComponentSet(undefined, this.options.registry);
-
-    await this.maybeSaveTempDirectory('source', components);
-
-    return new RetrieveResult(result, components, this.components);
-  }
-
   private getPackageNames(): string[] {
     return this.getPackageOptions()?.map((pkg) => pkg.name);
   }

src/client/types.ts

@@ -8,6 +8,7 @@ import { ComponentSet } from '../collections';
 import { PackageTypeMembers } from '../collections/types';
 import { SourcePath } from '../common/types';
 import { MetadataComponent, SourceComponent } from '../resolve';
+import { SfdxFileFormat } from '../convert';
 
 // ------------------------------------------------
 // API results reformatted for source development
@@ -326,6 +327,23 @@ export interface RetrieveOptions {
    * A list of package names to retrieve, or package names and their retrieval locations.
    */
   packageOptions?: PackageOptions;
+  /**
+   * The file format desired for the retrieved files.
+   */
+  format?: SfdxFileFormat;
+  /**
+   * Specifies whether only a single package is being retrieved (true) or not (false).
+   * If false, then more than one package is being retrieved.
+   */
+  singlePackage?: boolean;
+  /**
+   * The name of the retrieved zip file containing the source from the org. Only applies when `format: metadata`.
+   */
+  zipFileName?: string;
+  /**
+   * Specifies whether to unzip the retrieved zip file. Only applies when `format: metadata`.
+   */
+  unzip?: boolean;
 }
 
 export interface MetadataApiDeployOptions {

test/client/metadataApiRetrieve.test.ts

@@ -7,14 +7,17 @@
 import { fail } from 'assert';
 import { join } from 'path';
 import { expect } from 'chai';
-import { createSandbox, match } from 'sinon';
+import * as unzipper from 'unzipper';
+import { createSandbox, match, SinonStub } from 'sinon';
 import { getString } from '@salesforce/ts-types';
 import * as fs from 'graceful-fs';
 import {
   ComponentSet,
   ComponentStatus,
   FileResponse,
+  MetadataApiRetrieve,
   MetadataApiRetrieveStatus,
+  RequestStatus,
   RetrieveResult,
   SourceComponent,
   VirtualTreeContainer,
@@ -407,6 +410,67 @@ describe('MetadataApiRetrieve', () => {
     });
   });
 
+  describe('post', () => {
+    const output = join('mdapi', 'retrieve', 'dir');
+    const format = 'metadata';
+    const zipFile = 'abcd1234';
+    const zipFileContents = Buffer.from(zipFile, 'base64');
+    const usernameOrConnection = '[email protected]';
+    const fakeResults = { status: RequestStatus.Succeeded, zipFile } as MetadataApiRetrieveStatus;
+    let writeFileStub: SinonStub;
+    let openBufferStub: SinonStub;
+    let extractStub: SinonStub;
+    const mdapiRetrieveExtractStub = env.stub().resolves({});
+
+    beforeEach(() => {
+      writeFileStub = env.stub(fs, 'writeFileSync');
+      extractStub = env.stub().resolves();
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      openBufferStub = env.stub(unzipper.Open, 'buffer').resolves({ extract: extractStub } as any);
+    });
+
+    it('should write the retrieved zip when format=metadata', async () => {
+      const mdapiRetrieve = new MetadataApiRetrieve({ usernameOrConnection, output, format });
+      // @ts-ignore overriding private method
+      mdapiRetrieve.extract = mdapiRetrieveExtractStub;
+      await mdapiRetrieve.post(fakeResults);
+
+      expect(writeFileStub.calledOnce).to.be.true;
+      expect(writeFileStub.firstCall.args[0]).to.equal(join(output, 'unpackaged.zip'));
+      expect(writeFileStub.firstCall.args[1]).to.deep.equal(zipFileContents);
+      expect(openBufferStub.called).to.be.false;
+      expect(mdapiRetrieveExtractStub.called).to.be.false;
+    });
+
+    it('should unzip the retrieved zip when format=metadata and unzip=true', async () => {
+      const mdapiRetrieve = new MetadataApiRetrieve({ usernameOrConnection, output, format, unzip: true });
+      // @ts-ignore overriding private method
+      mdapiRetrieve.extract = mdapiRetrieveExtractStub;
+      await mdapiRetrieve.post(fakeResults);
+
+      expect(writeFileStub.calledOnce).to.be.true;
+      expect(writeFileStub.firstCall.args[0]).to.equal(join(output, 'unpackaged.zip'));
+      expect(writeFileStub.firstCall.args[1]).to.deep.equal(zipFileContents);
+      expect(openBufferStub.called).to.be.true;
+      expect(extractStub.called).to.be.true;
+      expect(mdapiRetrieveExtractStub.called).to.be.false;
+    });
+
+    it('should write the retrieved zip with specified name when format=metadata and zipFileName is set', async () => {
+      const zipFileName = 'retrievedFiles.zip';
+      const mdapiRetrieve = new MetadataApiRetrieve({ usernameOrConnection, output, format, zipFileName });
+      // @ts-ignore overriding private method
+      mdapiRetrieve.extract = mdapiRetrieveExtractStub;
+      await mdapiRetrieve.post(fakeResults);
+
+      expect(writeFileStub.calledOnce).to.be.true;
+      expect(writeFileStub.firstCall.args[0]).to.equal(join(output, zipFileName));
+      expect(writeFileStub.firstCall.args[1]).to.deep.equal(zipFileContents);
+      expect(openBufferStub.called).to.be.false;
+      expect(mdapiRetrieveExtractStub.called).to.be.false;
+    });
+  });
+
   describe('cancel', () => {
     it('should immediately stop polling', async () => {
       const component = COMPONENT;