[AJ-1843] Refactor import requirements (#4861)

Author: nawatts

src/import-data/ImportData.ts

@@ -15,6 +15,7 @@ import * as Utils from 'src/libs/utils';
 import { notifyDataImportProgress } from 'src/workspace-data/import-jobs';
 import { WorkspaceInfo } from 'src/workspaces/utils';
 
+import { getImportSource } from './import-sources';
 import {
   BagItImportRequest,
   CatalogDatasetImportRequest,
@@ -27,7 +28,6 @@ import {
 } from './import-types';
 import { ImportDataDestination } from './ImportDataDestination';
 import { ImportDataOverview } from './ImportDataOverview';
-import { getImportSource } from './protected-data-utils';
 import { useImportRequest } from './useImportRequest';
 
 export interface ImportDataProps {

src/import-data/ImportDataDestination.ts

@@ -21,9 +21,9 @@ import NewWorkspaceModal from 'src/workspaces/NewWorkspaceModal/NewWorkspaceModa
 import { WorkspaceInfo } from 'src/workspaces/utils';
 import { WorkspacePolicies } from 'src/workspaces/WorkspacePolicies/WorkspacePolicies';
 
+import { getRequiredCloudPlatform, requiresSecurityMonitoring } from './import-requirements';
 import { ImportRequest, TemplateWorkspaceInfo } from './import-types';
-import { buildDestinationWorkspaceFilter, getCloudPlatformRequiredForImport } from './import-utils';
-import { isProtectedSource } from './protected-data-utils';
+import { buildDestinationWorkspaceFilter } from './import-utils';
 
 const styles = {
   card: {
@@ -107,7 +107,7 @@ export const ImportDataDestination = (props: ImportDataDestinationProps): ReactN
     onImport,
   } = props;
 
-  const isProtectedData = isProtectedSource(importRequest);
+  const importRequiresSecurityMonitoring = requiresSecurityMonitoring(importRequest);
 
   // Some import types are finished in a single request.
   // For most though, the import request starts a background task that takes time to complete.
@@ -189,13 +189,14 @@ export const ImportDataDestination = (props: ImportDataDestinationProps): ReactN
     buildDestinationWorkspaceFilter(importRequest, { requiredAuthorizationDomain })
   );
 
-  // disable import into existing workspaces if data is marked as protected but no protected workspaces are available
-  const disableExportIntoExisting = isProtectedData && workspacesToImportTo.length === 0;
+  // disable import into existing workspaces if no suitable workspaces are available
+  // TODO: this should apply no matter the exact import requirements
+  const disableExportIntoExisting = importRequiresSecurityMonitoring && workspacesToImportTo.length === 0;
 
   const renderSelectExistingWorkspace = () =>
     h(Fragment, [
       h2({ style: styles.title }, [selectExistingWorkspacePrompt]),
-      isProtectedData &&
+      importRequiresSecurityMonitoring &&
         div({ style: { marginTop: '0.5rem', marginBottom: '0.5rem', lineHeight: '1.5' } }, [
           ' You may only import into workspaces that have additional security monitoring enabled.',
         ]),
@@ -219,7 +220,9 @@ export const ImportDataDestination = (props: ImportDataDestinationProps): ReactN
         h(WorkspacePolicies, {
           workspace: selectedWorkspace,
           noCheckboxes: true,
-          endingNotice: isProtectedData ? div(['Importing this data may add additional access controls']) : undefined,
+          endingNotice: importRequiresSecurityMonitoring
+            ? div(['Importing this data may add additional access controls'])
+            : undefined,
         }),
       div({ style: { display: 'flex', alignItems: 'center', marginTop: '1rem' } }, [
         h(ButtonSecondary, { onClick: () => setMode(undefined), style: { marginLeft: 'auto' } }, ['Back']),
@@ -350,10 +353,10 @@ export const ImportDataDestination = (props: ImportDataDestinationProps): ReactN
             isCreateOpen &&
               h(NewWorkspaceModal, {
                 requiredAuthDomain: requiredAuthorizationDomain,
-                cloudPlatform: getCloudPlatformRequiredForImport(importRequest),
+                cloudPlatform: getRequiredCloudPlatform(importRequest),
                 renderNotice: () => {
                   const children: ReactNode[] = [];
-                  if (isProtectedData) {
+                  if (importRequiresSecurityMonitoring) {
                     children.push(
                       div({ style: { paddingBottom: importMayTakeTime ? '1.0rem' : 0 } }, [
                         'Importing controlled access data will apply any additional access controls associated with the data to this workspace.',
@@ -365,7 +368,7 @@ export const ImportDataDestination = (props: ImportDataDestinationProps): ReactN
                   }
                   return children.length > 0 ? h(Fragment, children) : undefined;
                 },
-                requireEnhancedBucketLogging: isProtectedData,
+                requireEnhancedBucketLogging: importRequiresSecurityMonitoring,
                 waitForServices: {
                   wds: true,
                 },

src/import-data/ImportDataOverview.ts

@@ -3,8 +3,8 @@ import { div, h, h2, h3 } from 'react-hyperscript-helpers';
 import colors from 'src/libs/colors';
 import * as Style from 'src/libs/style';
 
+import { requiresSecurityMonitoring } from './import-requirements';
 import { ImportRequest } from './import-types';
-import { isProtectedSource } from './protected-data-utils';
 
 const styles = {
   container: {
@@ -49,7 +49,7 @@ export interface ImportDataOverviewProps {
 export const ImportDataOverview = (props: ImportDataOverviewProps): ReactNode => {
   const { importRequest } = props;
 
-  const isProtectedData = isProtectedSource(importRequest);
+  const importRequiresSecurityMonitoring = requiresSecurityMonitoring(importRequest);
 
   return div({ style: styles.card }, [
     h2({ style: styles.title }, [getTitleForImportRequest(importRequest)]),
@@ -61,7 +61,7 @@ export const ImportDataOverview = (props: ImportDataOverviewProps): ReactNode =>
     h3({ style: { fontSize: 16 } }, ['Dataset security requirements:']),
     div(
       { style: { marginTop: '1rem' } },
-      isProtectedData
+      importRequiresSecurityMonitoring
         ? ['The data you have selected requires additional security monitoring.']
         : [
             'The data you just chose to import to Terra will be made available to you within a workspace of your choice where you can then perform analysis.',

src/import-data/import-utils.feature-flag.test.ts renamed to src/import-data/import-requirements.feature-flag.test.ts

@@ -1,6 +1,6 @@
 import { genericPfbImportRequest } from './__fixtures__/import-request-fixtures';
+import { getRequiredCloudPlatform } from './import-requirements';
 import { ImportRequest } from './import-types';
-import { getCloudPlatformRequiredForImport } from './import-utils';
 
 type FeaturePreviewExports = typeof import('src/libs/feature-previews');
 
@@ -12,18 +12,18 @@ jest.mock(
   })
 );
 
-// Note that behavior of getRequiredCloudPlatformForImport when the feature flag is set to false is tested
+// Note that behavior of getRequiredCloudPlatform when the feature flag is set to false is tested
 // in import-utils.test.ts. This file only tests behavior when the feature flag is set to true.
 // Since `jest.mock` is global for any given file, the test below is given this new file.
-describe('getRequiredCloudPlatformForImport', () => {
+describe('getRequiredCloudPlatform', () => {
   it('should respect the feature flag for PFB imports', async () => {
     // Arrange
     const importRequest: ImportRequest = genericPfbImportRequest;
 
     // Act
-    const cloudPlatform = getCloudPlatformRequiredForImport(importRequest);
+    const requiredCloudPlatform = getRequiredCloudPlatform(importRequest);
 
     // Assert
-    expect(cloudPlatform).toBeUndefined();
+    expect(requiredCloudPlatform).toBeUndefined();
   });
 });

src/import-data/import-requirements.test.ts

@@ -0,0 +1,82 @@
+import { CloudProvider } from 'src/workspaces/utils';
+
+import {
+  anvilPfbImportRequests,
+  azureTdrSnapshotImportRequest,
+  biodataCatalystPfbImportRequests,
+  gcpTdrSnapshotImportRequest,
+  gcpTdrSnapshotReferenceImportRequest,
+  genericPfbImportRequest,
+  protectedGcpTdrSnapshotImportRequest,
+  protectedGcpTdrSnapshotReferenceImportRequest,
+} from './__fixtures__/import-request-fixtures';
+import { getRequiredCloudPlatform, requiresSecurityMonitoring } from './import-requirements';
+import { ImportRequest } from './import-types';
+
+describe('cloud provider requirements', () => {
+  describe('getRequiredCloudPlatform', () => {
+    it.each([
+      {
+        importRequest: genericPfbImportRequest,
+        expectedCloudPlatform: 'GCP',
+      },
+      {
+        importRequest: azureTdrSnapshotImportRequest,
+        expectedCloudPlatform: 'AZURE',
+      },
+      {
+        importRequest: gcpTdrSnapshotImportRequest,
+        expectedCloudPlatform: 'GCP',
+      },
+    ] as {
+      importRequest: ImportRequest;
+      expectedCloudPlatform: CloudProvider | undefined;
+    }[])('returns cloud provider required for import', async ({ importRequest, expectedCloudPlatform }) => {
+      // Act
+      const requiredCloudPlatform = getRequiredCloudPlatform(importRequest);
+
+      // Assert
+      expect(requiredCloudPlatform).toBe(expectedCloudPlatform);
+    });
+  });
+});
+
+describe('security monitoring requirements', () => {
+  const importsExpectedToRequireSecurityMonitoring: ImportRequest[] = [
+    // AnVIL
+    ...anvilPfbImportRequests,
+    // BioData Catalyst
+    ...biodataCatalystPfbImportRequests,
+    // Protected TDR snapshots
+    protectedGcpTdrSnapshotImportRequest,
+    protectedGcpTdrSnapshotReferenceImportRequest,
+  ];
+
+  const importsExpectedToNotRequireSecurityMonitoring: ImportRequest[] = [
+    genericPfbImportRequest,
+    { type: 'entities', url: new URL('https://example.com/file.json') },
+    gcpTdrSnapshotImportRequest,
+    gcpTdrSnapshotReferenceImportRequest,
+  ];
+
+  describe('isProtectedSource', () => {
+    it.each(importsExpectedToRequireSecurityMonitoring)('$url should require security monitoring', (importRequest) => {
+      // Act
+      const importRequiresSecurityMonitoring = requiresSecurityMonitoring(importRequest);
+
+      // Assert
+      expect(importRequiresSecurityMonitoring).toBe(true);
+    });
+
+    it.each(importsExpectedToNotRequireSecurityMonitoring)(
+      '$url should not require security monitoring',
+      (importRequest) => {
+        // Act
+        const importRequiresSecurityMonitoring = requiresSecurityMonitoring(importRequest);
+
+        // Assert
+        expect(importRequiresSecurityMonitoring).toBe(false);
+      }
+    );
+  });
+});

src/import-data/import-requirements.ts

@@ -0,0 +1,68 @@
+import { Snapshot } from 'src/libs/ajax/DataRepo';
+import { isFeaturePreviewEnabled } from 'src/libs/feature-previews';
+import { ENABLE_AZURE_PFB_IMPORT } from 'src/libs/feature-previews-config';
+import { CloudProvider } from 'src/workspaces/utils';
+
+import { urlMatchesSource, UrlSource } from './import-sources';
+import { ImportRequest } from './import-types';
+
+export const getRequiredCloudPlatform = (importRequest: ImportRequest): CloudProvider | undefined => {
+  switch (importRequest.type) {
+    case 'tdr-snapshot-export':
+    case 'tdr-snapshot-reference':
+      const tdrCloudPlatformToCloudProvider: Record<Snapshot['cloudPlatform'], CloudProvider> = {
+        azure: 'AZURE',
+        gcp: 'GCP',
+      };
+      return tdrCloudPlatformToCloudProvider[importRequest.snapshot.cloudPlatform];
+    case 'pfb':
+      // restrict PFB imports to GCP unless the user has the right feature flag enabled
+      return isFeaturePreviewEnabled(ENABLE_AZURE_PFB_IMPORT) ? undefined : 'GCP';
+    default:
+      return undefined;
+  }
+};
+
+// These must be kept in sync with CWDS' twds.data-import.sources setting.
+// https://github.com/DataBiosphere/terra-workspace-data-service/blob/main/service/src/main/resources/application.yml
+const sourcesRequiringSecurityMonitoring: UrlSource[] = [
+  // AnVIL production
+  { type: 'http', host: 'service.prod.anvil.gi.ucsc.edu' },
+  { type: 's3', bucket: 'edu-ucsc-gi-platform-anvil-prod-storage-anvilprod.us-east-1' },
+  // AnVIL development
+  { type: 'http', host: 'service.anvil.gi.ucsc.edu' },
+  { type: 's3', bucket: 'edu-ucsc-gi-platform-anvil-dev-storage-anvildev.us-east-1' },
+  // BioData Catalyst
+  { type: 'http', host: 'gen3.biodatacatalyst.nhlbi.nih.gov' },
+  { type: 's3', bucket: 'gen3-biodatacatalyst-nhlbi-nih-gov-pfb-export' },
+  { type: 's3', bucket: 'gen3-theanvil-io-pfb-export' },
+];
+
+/**
+ * Determine if a PFB file requires security monitoring.
+ */
+const pfbRequiresSecurityMonitoring = (pfbUrl: URL): boolean => {
+  return sourcesRequiringSecurityMonitoring.some((source) => urlMatchesSource(pfbUrl, source));
+};
+
+/**
+ * Determine if a TDR snapshot requires security monitoring.
+ */
+const snapshotRequiresSecurityMonitoring = (snapshot: Snapshot): boolean => {
+  return snapshot.source.some((source) => source.dataset.secureMonitoringEnabled);
+};
+
+/**
+ * Determine whether an import requires security monitoring.
+ */
+export const requiresSecurityMonitoring = (importRequest: ImportRequest): boolean => {
+  switch (importRequest.type) {
+    case 'pfb':
+      return pfbRequiresSecurityMonitoring(importRequest.url);
+    case 'tdr-snapshot-export':
+    case 'tdr-snapshot-reference':
+      return snapshotRequiresSecurityMonitoring(importRequest.snapshot);
+    default:
+      return false;
+  }
+};

src/import-data/import-sources.test.ts

@@ -0,0 +1,46 @@
+import {
+  anvilPfbImportRequests,
+  biodataCatalystPfbImportRequests,
+  genericPfbImportRequest,
+} from './__fixtures__/import-request-fixtures';
+import { getImportSource, ImportSource, urlMatchesSource, UrlSource } from './import-sources';
+
+describe('urlMatchesSource', () => {
+  it.each([
+    { url: new URL('https://example.com/file.txt'), shouldMatch: true },
+    { url: new URL('https://subdomain.example.com/file.txt'), shouldMatch: true },
+    { url: new URL('https://subdomainexample.com/file.txt'), shouldMatch: false },
+    { url: new URL('https://example-example.com/file.txt'), shouldMatch: false },
+  ] as { url: URL; shouldMatch: boolean }[])(
+    'matches HTTP sources by hostname ($url, $shouldMatch)',
+    ({ url, shouldMatch }) => {
+      // Arrange
+      const source: UrlSource = { type: 'http', host: 'example.com' };
+
+      // Act
+      const matches = urlMatchesSource(url, source);
+
+      // Assert
+      expect(matches).toBe(shouldMatch);
+    }
+  );
+});
+
+describe('getImportSource', () => {
+  it.each([
+    ...anvilPfbImportRequests.map((importRequest) => ({ importUrl: importRequest.url, expectedSource: 'anvil' })),
+    ...[genericPfbImportRequest, ...biodataCatalystPfbImportRequests].map((importRequest) => ({
+      importUrl: importRequest.url,
+      expectedSource: '',
+    })),
+  ] as {
+    importUrl: URL;
+    expectedSource: ImportSource;
+  }[])('identifies import source ($importUrl)', ({ importUrl, expectedSource }) => {
+    // Act
+    const source = getImportSource(importUrl);
+
+    // Assert
+    expect(source).toBe(expectedSource);
+  });
+});

src/import-data/import-sources.ts

@@ -0,0 +1,46 @@
+export type UrlSource = { type: 'http'; host: string } | { type: 's3'; bucket: string };
+
+/**
+ * Determine if a PFB file is considered protected data.
+ */
+export const urlMatchesSource = (url: URL, source: UrlSource): boolean => {
+  switch (source.type) {
+    case 'http':
+      // Match the hostname or subdomains of protected hosts.
+      return url.hostname === source.host || url.hostname.endsWith(`.${source.host}`);
+
+    case 's3':
+      // S3 supports multiple URL formats
+      // https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html
+      return (
+        url.hostname === `${source.bucket}.s3.amazonaws.com` ||
+        (url.hostname === 's3.amazonaws.com' && url.pathname.startsWith(`/${source.bucket}/`))
+      );
+
+    default:
+      // Use TypeScript to verify that all cases have been handled.
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const exhaustiveGuard: never = source;
+      return false;
+  }
+};
+
+export type ImportSource = 'anvil' | '';
+
+/**
+ * This method identifies an import source. Currently it only identifies AnVIL Explorer.
+ */
+export const getImportSource = (url: URL): ImportSource => {
+  const anvilSources = [
+    // AnVIL production
+    'service.prod.anvil.gi.ucsc.edu',
+    'edu-ucsc-gi-platform-anvil-prod-storage-anvilprod.us-east-1',
+    // AnVIL development
+    'service.anvil.gi.ucsc.edu',
+    'edu-ucsc-gi-platform-anvil-dev-storage-anvildev.us-east-1',
+  ];
+  if (anvilSources.some((path) => url.href.includes(path))) {
+    return 'anvil';
+  }
+  return '';
+};