Merge pull request #26 from amtrack/feat/allowlisting

feat: add allowlisting using --metadata

Author: amtrack

src/cli.ts

@@ -1,19 +1,20 @@
-import getStdin = require('get-stdin');
-import MetadataComponent, {
-  parseMetadataComponentName
-} from './metadata-component';
-
-export async function getMetadataComponentsFromStdinOrString(
+export function parseCommaSeparatedValues(
   commaSeparatedMetadataComponentNames: string
-): Promise<Array<MetadataComponent>> {
-  let rawComponentNames = [];
-  if (commaSeparatedMetadataComponentNames === '-') {
-    rawComponentNames = (await getStdin()).split('\n');
-  } else {
-    rawComponentNames = commaSeparatedMetadataComponentNames.split(',');
+): Array<string> {
+  if (!commaSeparatedMetadataComponentNames) {
+    return [];
+  }
+  return clean(commaSeparatedMetadataComponentNames.split(','));
+}
+export function parseNewLineSeparatedValues(
+  newLineSeparatedValues: string
+): Array<string> {
+  if (!newLineSeparatedValues) {
+    return [];
   }
-  return rawComponentNames
-    .map((x) => x.trim())
-    .filter(Boolean)
-    .map(parseMetadataComponentName);
+  return clean(newLineSeparatedValues.split(/\r?\n/));
+}
+
+function clean(values: Array<string>) {
+  return values.map((x) => x.trim()).filter(Boolean);
 }

src/commands/force/mdapi/listallmetadata.ts

@@ -1,6 +1,11 @@
 import { flags, SfdxCommand } from '@salesforce/command';
 import { promises as fs } from 'fs';
+import {
+  parseCommaSeparatedValues,
+  parseNewLineSeparatedValues
+} from '../../../cli';
 import { listAllMetadata } from '../../../listallmetadata';
+import getStdin = require('get-stdin');
 
 export default class MdapiListAllMetadataCommand extends SfdxCommand {
   public static description = `list all metadata components
@@ -67,10 +72,15 @@ export default class MdapiListAllMetadataCommand extends SfdxCommand {
     children: flags.boolean({
       description: `list metadata components of child types (e.g. 'CustomField' children of 'CustomObject')`
     }),
+    metadata: flags.string({
+      char: 'm',
+      description: `comma-separated list of metadata component name expressions to list
+      Example: 'CustomObject:*,CustomField:Account.*'`
+    }),
     ignore: flags.string({
-      description: `ignore metadata components matching the pattern in the format of <type>:<fullName>
-      Examples: 'InstalledPackage:*', 'Profile:*', 'Report:unfiled$public/*', 'CustomField:Account.*'`,
-      multiple: true
+      char: 'i',
+      description: `comma-separated list of metadata component name expressions to ignore
+      Example: 'InstalledPackage:*,Profile:*,Report:unfiled$public/*,CustomField:Account.*'`
     }),
     names: flags.boolean({
       description: `output only component names (e.g. 'CustomObject:Account',...)`
@@ -79,7 +89,18 @@ export default class MdapiListAllMetadataCommand extends SfdxCommand {
 
   public async run(): Promise<any> {
     const conn = this.org.getConnection();
-    const fileProperties = await listAllMetadata(conn, this.flags);
+    let allowPatterns =
+      this.flags.metadata === '-'
+        ? parseNewLineSeparatedValues(await getStdin())
+        : parseCommaSeparatedValues(this.flags.metadata);
+    allowPatterns = allowPatterns.length ? allowPatterns : ['**/*'];
+    const ignorePatterns = parseCommaSeparatedValues(this.flags.ignore);
+    const fileProperties = await listAllMetadata(
+      conn,
+      this.flags,
+      allowPatterns,
+      ignorePatterns
+    );
     if (this.flags.resultfile) {
       const fileData: string = JSON.stringify(fileProperties, null, 4);
       await fs.writeFile(this.flags.resultfile, fileData);

src/commands/package.xml/create.ts

@@ -1,7 +1,12 @@
 import { flags, SfdxCommand } from '@salesforce/command';
 import { promises as fs } from 'fs';
-import { getMetadataComponentsFromStdinOrString } from '../../cli';
+import {
+  parseCommaSeparatedValues,
+  parseNewLineSeparatedValues
+} from '../../cli';
+import { parseMetadataComponentName } from '../../metadata-component';
 import PackageXml from '../../package-xml';
+import getStdin = require('get-stdin');
 
 export default class PackageXmlCreateCommand extends SfdxCommand {
   public static description = `create a package.xml manifest`;
@@ -43,10 +48,14 @@ export default class PackageXmlCreateCommand extends SfdxCommand {
     if (this.flags.apiversion) {
       meta['version'] = this.flags.apiversion;
     }
-    const metadataComponents = await getMetadataComponentsFromStdinOrString(
-      this.flags.metadata
-    );
-    const packageXml = new PackageXml(metadataComponents, meta).toString();
+    const metadataComponentNames =
+      this.flags.metadata === '-'
+        ? parseNewLineSeparatedValues(await getStdin())
+        : parseCommaSeparatedValues(this.flags.metadata);
+    const packageXml = new PackageXml(
+      metadataComponentNames.map(parseMetadataComponentName),
+      meta
+    ).toString();
     if (this.flags.resultfile) {
       await fs.writeFile(this.flags.resultfile, packageXml);
     } else {

src/commands/package.xml/generate.ts

@@ -1,6 +1,7 @@
 import { flags, SfdxCommand } from '@salesforce/command';
 import { promises as fs } from 'fs';
-import { ignoreMatching } from '../../ignore';
+import { parseCommaSeparatedValues } from '../../cli';
+import { match } from '../../match';
 import { toMetadataComponentName } from '../../metadata-component';
 import PackageXml from '../../package-xml';
 
@@ -53,9 +54,9 @@ export default class PackageXmlGenerateCommand extends SfdxCommand {
       description: 'path to the generated package.xml file'
     }),
     ignore: flags.string({
-      description: `ignore metadata components matching the pattern in the format of <type>:<fullName>
-        Example: --ignore "Profile:*" --ignore "Report:unfiled$public/*" --ignore "CustomField:Account.*"`,
-      multiple: true
+      char: 'i',
+      description: `comma-separated list of metadata component name expressions to ignore
+      Example: 'InstalledPackage:*,Profile:*,Report:unfiled$public/*,CustomField:Account.*'`
     }),
     // ignorefile: flags.filepath({
     //   description: `same as --ignore, just one ignore pattern per line in a file`
@@ -83,12 +84,12 @@ export default class PackageXmlGenerateCommand extends SfdxCommand {
     }
     const ignorePatterns = [];
     if (this.flags.ignore) {
-      ignorePatterns.push(...this.flags.ignore);
+      ignorePatterns.push(...parseCommaSeparatedValues(this.flags.ignore));
     }
     if (this.flags.defaultignore) {
       ignorePatterns.push(...this.flags.defaultignore);
     }
-    const [keep] = ignoreMatching(
+    const [, keep] = match(
       fileProperties,
       ignorePatterns,
       toMetadataComponentName

src/listallmetadata.ts

@@ -1,7 +1,7 @@
 import { OutputFlags } from '@oclif/parser';
 import { Connection } from '@salesforce/core';
 import { FileProperties } from 'jsforce';
-import { ignoreMatching } from './ignore';
+import { match } from './match';
 import { toMetadataComponentName } from './metadata-component';
 import ChildMetadataLister from './metadata-lister/children';
 import FolderBasedMetadataLister from './metadata-lister/folderbased';
@@ -10,7 +10,9 @@ import StandardValueSetMetadataLister from './metadata-lister/standardvaluesets'
 
 export async function listAllMetadata(
   conn: Connection,
-  flags: OutputFlags<any>
+  flags: OutputFlags<any>,
+  allowPatterns?: Array<string>,
+  ignorePatterns?: Array<string>
 ): Promise<Array<FileProperties>> {
   const describeMetadataResult = await conn.metadata.describe();
   const metadataListers = [
@@ -19,28 +21,27 @@ export async function listAllMetadata(
     StandardValueSetMetadataLister,
     ChildMetadataLister
   ];
-  // TODO: filter describeMetadataResult using ignorePatterns
   const result = [];
   for (const MetadataListerImplementation of metadataListers) {
     const listerId = MetadataListerImplementation.id;
-    const instance = new MetadataListerImplementation();
+    const instance = new MetadataListerImplementation(
+      allowPatterns,
+      ignorePatterns
+    );
     if (flags[listerId]) {
       const fileProperties = await instance.run(
         conn,
         describeMetadataResult,
         result
       );
-      const [keep, ignored] = ignoreMatching(
+      // postfilter
+      const [matches] = match(
         fileProperties,
-        flags.ignore ? flags.ignore : [],
-        toMetadataComponentName
+        allowPatterns,
+        toMetadataComponentName,
+        { ignore: ignorePatterns }
       );
-      if (ignored.length) {
-        console.error(
-          `ignored: ${JSON.stringify(ignored.map(toMetadataComponentName))}`
-        );
-      }
-      result.push(...keep);
+      result.push(...matches);
     }
   }
   return result;

src/ignore.ts renamed to src/match.ts

@@ -1,31 +1,32 @@
 import * as picomatch from 'picomatch';
 
-interface ToStringFunction {
+export interface ToStringFunction {
   (item: any): string;
 }
 
 /**
  *
  * @param collection an array of strings or objects
- * @param ignorePatterns a list of ignore patterns similar to .gitignore entries
- * @param toString if the collection is an array of objects: a function to convert an item of the collection to a string to be able to match against the ignorePatterns
+ * @param patterns a list of patterns similar to .gitignore entries
+ * @param toString if the collection is an array of objects: a function to convert an item of the collection to a string to be able to match against the allowPatterns
  * @param picoMatchOptions options for picomatch library
+ * @returns Array with [matched, unmatched]
  */
-export function ignoreMatching(
+export function match(
   collection: Array<any>,
-  ignorePatterns: Array<string>,
+  patterns: Array<string>,
   toString?: ToStringFunction,
   // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
   picoMatchOptions?: any
 ): Array<any> {
-  const ignore = [];
-  const keep = collection.filter((item) => {
+  const unmatched = [];
+  const matched = collection.filter((item) => {
     const str = toString ? toString(item) : item;
-    const isMatch = picomatch.isMatch(str, ignorePatterns, picoMatchOptions);
-    if (isMatch) {
-      ignore.push(item);
+    const isMatch = picomatch.isMatch(str, patterns, picoMatchOptions);
+    if (!isMatch) {
+      unmatched.push(item);
     }
-    return !isMatch;
+    return isMatch;
   });
-  return [keep, ignore];
+  return [matched, unmatched];
 }

src/metadata-lister.ts

@@ -1,3 +1,37 @@
+import { Connection, DescribeMetadataResult, FileProperties } from 'jsforce';
+import { match, ToStringFunction } from './match';
+
+export interface IMetadataLister {
+  id: string;
+  run: (
+    conn: Connection,
+    describeMetadataResult: DescribeMetadataResult,
+    fileProperties: Array<FileProperties>,
+    allowPatterns: Array<string>,
+    ignorePatterns: Array<string>
+  ) => Promise<Array<FileProperties>>;
+}
+
 export default abstract class MetadataLister {
   public static id: string;
+  private allowPatterns: Array<string>;
+  private ignorePatterns: Array<string>;
+
+  constructor(allowPatterns: Array<string>, ignorePatterns: Array<string>) {
+    this.allowPatterns = allowPatterns;
+    this.ignorePatterns = ignorePatterns;
+  }
+
+  abstract run(
+    conn: Connection,
+    describeMetadataResult: DescribeMetadataResult,
+    fileProperties: Array<FileProperties>
+  ): Promise<Array<FileProperties>>;
+
+  public filter(items: Array<any>, toString?: ToStringFunction): Array<any> {
+    const [matched] = match(items, this.allowPatterns, toString, {
+      ignore: this.ignorePatterns
+    });
+    return matched;
+  }
 }

src/metadata-lister/children.ts

@@ -25,7 +25,11 @@ export default class ChildrenMetadata extends MetadataLister {
     const childMetadataQueries = childMetadataTypes.map((childMetadataType) => {
       return { type: childMetadataType };
     });
-    let result = await listMetadataInChunks(conn, childMetadataQueries);
+    const filteredChildMetadataQueries = this.filter(
+      childMetadataQueries,
+      (x) => `${x.type}:`
+    );
+    let result = await listMetadataInChunks(conn, filteredChildMetadataQueries);
     let personAccountRecordTypes = [];
     try {
       personAccountRecordTypes = await queryPersonAccountRecordTypes(conn);

src/metadata-lister/folderbased.ts

@@ -18,29 +18,31 @@ export default class FolderBasedMetadata extends MetadataLister {
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     fileProperties: Array<FileProperties>
   ): Promise<Array<FileProperties>> {
-    return await listFolderBasedMetadata(conn);
+    const folderTypes = Object.keys(FOLDER_BASED_METADATA_MAP);
+    const folderQueries = folderTypes.map((folderType) => {
+      return {
+        type: folderType
+      };
+    });
+    const filteredFolderQueries = this.filter(
+      folderQueries,
+      (x) => `${x.type}:`
+    );
+    const folders = await listMetadataInChunks(conn, filteredFolderQueries);
+    const inFolderQueries = folders.map((folder) => {
+      return {
+        type: FOLDER_BASED_METADATA_MAP[folder.type],
+        folder: folder.fullName
+      };
+    });
+    const filteredInFolderQueries = this.filter(
+      inFolderQueries,
+      (x) => `${x.type}:${x.folder}`
+    );
+    const inFolderFileProperties = await listMetadataInChunks(
+      conn,
+      filteredInFolderQueries
+    );
+    return [...folders, ...inFolderFileProperties];
   }
 }
-
-export async function listFolderBasedMetadata(
-  conn: Connection
-): Promise<Array<FileProperties>> {
-  const folderTypes = Object.keys(FOLDER_BASED_METADATA_MAP);
-  const folderQueries = folderTypes.map((folderType) => {
-    return {
-      type: folderType
-    };
-  });
-  const folders = await listMetadataInChunks(conn, folderQueries);
-  const inFolderQueries = folders.map((folder) => {
-    return {
-      type: FOLDER_BASED_METADATA_MAP[folder.type],
-      folder: folder.fullName
-    };
-  });
-  const inFolderFileProperties = await listMetadataInChunks(
-    conn,
-    inFolderQueries
-  );
-  return [...folders, ...inFolderFileProperties];
-}

src/metadata-lister/regular.ts

@@ -12,15 +12,18 @@ export default class RegularMetadata extends MetadataLister {
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     fileProperties: Array<FileProperties>
   ): Promise<Array<FileProperties>> {
-    const metadataTypes = describeMetadataResult.metadataObjects.map(
-      (metadataType) => metadataType.xmlName
+    const metadataQueries = describeMetadataResult.metadataObjects.map(
+      (cmp) => {
+        return {
+          type: cmp.xmlName
+        };
+      }
+    );
+    const filteredMetadataQueries = this.filter(
+      metadataQueries,
+      (x) => `${x.type}:`
     );
-    const metadataQueries = metadataTypes.map((metadataType) => {
-      return {
-        type: metadataType
-      };
-    });
-    let result = await listMetadataInChunks(conn, metadataQueries);
+    let result = await listMetadataInChunks(conn, filteredMetadataQueries);
     result = fixNilType(result, describeMetadataResult);
     result = fixCustomFeedFilter(result);
     return result;