Merge branch 'datahub-project:master' into master

Author: jayacryl

.github/actionlint.yaml

@@ -0,0 +1,7 @@
+self-hosted-runner:
+  labels:
+    - "depot-ubuntu-22.04-small"
+    - "depot-ubuntu-22.04-4"
+    - "depot-ubuntu-22.04"
+    - "depot-ubuntu-24.04"
+    - "depot-ubuntu-24.04-small"

.github/workflows/build-and-test.yml

@@ -158,7 +158,7 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
-  quickstart-compose-validation:
+  docker-codegen-validation:
     runs-on: ubuntu-latest
     needs: setup
     if: ${{ needs.setup.outputs.docker_change == 'true' }}
@@ -170,6 +170,8 @@ jobs:
           python-version: "3.10"
       - name: Quickstart Compose Validation
         run: ./docker/quickstart/generate_and_compare.sh
+      - name: Docker Snippet Validation
+        run: python python-build/generate_ingestion_docker.py --check
 
   event-file:
     runs-on: ubuntu-latest

.github/workflows/docker-unified.yml

@@ -1143,7 +1143,7 @@ jobs:
           docker pull confluentinc/cp-kafka:7.4.0 &
           docker pull mysql:8.2 &
           docker pull opensearchproject/opensearch:2.9.0 &
-          docker pull ${{ env.DATAHUB_INGESTION_IMAGE }}:head  &
+          docker pull ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:head-slim  &
 
           wait
           docker images
@@ -1152,7 +1152,7 @@ jobs:
         env:
           DATAHUB_TELEMETRY_ENABLED: false
           DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
-          DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_IMAGE }}
+          DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
           ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag || 'head-slim' }}
           ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor] acryl-datahub-actions"
           ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml"

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/ingestion/BackfillIngestionSourceInfoIndicesStep.java

@@ -15,7 +15,8 @@
 @Slf4j
 public class BackfillIngestionSourceInfoIndicesStep extends AbstractMCLStep {
 
-  private static final String UPGRADE_ID = BackfillIngestionSourceInfoIndices.class.getSimpleName();
+  private static final String UPGRADE_ID =
+      BackfillIngestionSourceInfoIndices.class.getSimpleName() + "-v2";
   private static final Urn UPGRADE_ID_URN = BootstrapStep.getUpgradeUrn(UPGRADE_ID);
 
   public BackfillIngestionSourceInfoIndicesStep(

datahub-web-react/src/alchemy-components/theme/foundations/colors.ts

@@ -2,28 +2,31 @@ const colors = {
     transparent: 'transparent',
     current: 'currentColor',
     white: '#FFFFFF',
-    black: '#000000',
+    black: '#000000', // do not use black, use grays 600 for primary text and 1700 for secondary
 
     gray: {
-        100: '#EBECF0',
-        200: '#CFD1DA',
-        300: '#A9ADBD',
-        400: '#81879F',
-        500: '#5B6282', // primary value
-        600: '#374066',
-        700: '#2F3657',
-        800: '#272D48',
-        900: '#231A58',
-        1000: '#F1F3FD', // grays 1000-1300 aren't actually gray, should not be used
-        1100: '#F1FBFE',
-        1200: '#FBF3EF',
-        1300: '#F7FBF4',
-        1400: '#E9EAEE',
-        1500: '#F9FAFC',
-        1600: '#F5F6FA',
-        1700: '#5F6685',
-        1800: '#8088A3',
+        0: '#FFFFFF', // white
+        100: '#EBECF0', // border
+        200: '#CFD1DA', // should be deprecated
+        300: '#A9ADBD', // should be deprecated
+        400: '#81879F', // should be deprecated
+        500: '#5B6282', // should be deprecated
+        600: '#374066', // primary text
+        700: '#2F3657', // should be deprecated
+        800: '#272D48', // should be deprecated
+        900: '#231A58', // should be deprecated
+        1000: '#F1F3FD', // grays violet 0
+        1100: '#F1FBFE', // blue 0
+        1200: '#FBF3EF', // red 0
+        1300: '#F7FBF4', // green 0
+        1400: '#E9EAEE', // border
+        1500: '#F9FAFC', // bg surface
+        1600: '#F5F6FA', // bg of new nav
+        1700: '#5F6685', // secondary text
+        1800: '#8088A3', // tertiary text, disabled text, placeholder text, icon
         1900: '#A3A7B9',
+        2000: '#1E2338', // dark bg-suface
+        2100: '#171B2B', // dark bg
     },
 
     violet: {
@@ -32,8 +35,8 @@ const colors = {
         200: '#B0A7EA',
         300: '#8C7EE0',
         400: '#7565DA',
-        500: '#533FD1', // primary value
-        600: '#4C39BE',
+        500: '#533FD1',
+        600: '#4C39BE', // primary value
         700: '#3B2D94',
         800: '#2E2373',
         900: '#231A58',

datahub-web-react/src/app/ingest/source/IngestedAssets.tsx

@@ -1,14 +1,15 @@
 import { Button, Typography } from 'antd';
 import React, { useState } from 'react';
 import styled from 'styled-components';
+import { Maybe, ExecutionRequestResult } from '@src/types.generated';
 import { useGetSearchResultsForMultipleQuery } from '../../../graphql/search.generated';
 import { EmbeddedListSearchModal } from '../../entity/shared/components/styled/search/EmbeddedListSearchModal';
 import { ANTD_GRAY } from '../../entity/shared/constants';
 import { UnionType } from '../../search/utils/constants';
 import { formatNumber } from '../../shared/formatNumber';
 import { Message } from '../../shared/Message';
 import { useEntityRegistry } from '../../useEntityRegistry';
-import { extractEntityTypeCountsFromFacets } from './utils';
+import { extractEntityTypeCountsFromFacets, getEntitiesIngestedByType, getTotalEntitiesIngested } from './utils';
 
 const HeaderContainer = styled.div`
     display: flex;
@@ -51,19 +52,27 @@ const ViewAllButton = styled(Button)`
 
 type Props = {
     id: string;
+    executionResult?: Maybe<Partial<ExecutionRequestResult>>;
 };
 
 const ENTITY_FACET_NAME = 'entity';
 const TYPE_NAMES_FACET_NAME = 'typeNames';
 
-export default function IngestedAssets({ id }: Props) {
+export default function IngestedAssets({ id, executionResult }: Props) {
     const entityRegistry = useEntityRegistry();
 
     // First thing to do is to search for all assets with the id as the run id!
     const [showAssetSearch, setShowAssetSearch] = useState(false);
 
+    // Try getting the counts via the ingestion report.
+    const totalEntitiesIngested = executionResult && getTotalEntitiesIngested(executionResult);
+    const entitiesIngestedByTypeFromReport = executionResult && getEntitiesIngestedByType(executionResult);
+
+    // Fallback to the search across entities.
+    // First thing to do is to search for all assets with the id as the run id!
     // Execute search
     const { data, loading, error } = useGetSearchResultsForMultipleQuery({
+        skip: totalEntitiesIngested === null || entitiesIngestedByTypeFromReport === null,
         variables: {
             input: {
                 query: '*',
@@ -90,11 +99,13 @@ export default function IngestedAssets({ id }: Props) {
     const hasSubTypeFacet = (facets || []).findIndex((facet) => facet.field === TYPE_NAMES_FACET_NAME) >= 0;
     const subTypeFacets =
         (hasSubTypeFacet && facets?.filter((facet) => facet.field === TYPE_NAMES_FACET_NAME)[0]) || undefined;
+
     const countsByEntityType =
-        (entityTypeFacets && extractEntityTypeCountsFromFacets(entityRegistry, entityTypeFacets, subTypeFacets)) || [];
+        entitiesIngestedByTypeFromReport ??
+        (entityTypeFacets ? extractEntityTypeCountsFromFacets(entityRegistry, entityTypeFacets, subTypeFacets) : []);
 
     // The total number of assets ingested
-    const total = data?.searchAcrossEntities?.total || 0;
+    const total = totalEntitiesIngested ?? data?.searchAcrossEntities?.total ?? 0;
 
     return (
         <>

datahub-web-react/src/app/ingest/source/__tests__/tests_utils.test.tsx

@@ -0,0 +1,117 @@
+import { vi, describe, test, expect, beforeEach, afterAll } from 'vitest';
+import { getEntitiesIngestedByType } from '../utils';
+import { ExecutionRequestResult } from '../../../../types.generated';
+
+// Mock the structuredReport property of ExecutionRequestResult
+const mockExecutionRequestResult = (structuredReportData: any): Partial<ExecutionRequestResult> => {
+    return {
+        structuredReport: {
+            serializedValue: JSON.stringify(structuredReportData),
+        },
+    } as Partial<ExecutionRequestResult>;
+};
+
+describe('getEntitiesIngestedByType', () => {
+    // Mock for console.error
+    const originalConsoleError = console.error;
+    console.error = vi.fn();
+
+    beforeEach(() => {
+        vi.clearAllMocks();
+    });
+
+    afterAll(() => {
+        console.error = originalConsoleError;
+    });
+
+    test('returns null when structured report is not available', () => {
+        const result = getEntitiesIngestedByType({} as Partial<ExecutionRequestResult>);
+        expect(result).toBeNull();
+    });
+
+    test('returns null when an exception occurs during processing', () => {
+        // Create a malformed structured report to trigger an exception
+        const malformedReport = {
+            source: {
+                report: {
+                    // Missing aspects property to trigger exception
+                },
+            },
+        };
+
+        const result = getEntitiesIngestedByType(mockExecutionRequestResult(malformedReport));
+        expect(result).toBeNull();
+    });
+
+    test('correctly extracts entity counts from structured report', () => {
+        // Create a structured report based on the example in the comments
+        const structuredReport = {
+            source: {
+                report: {
+                    aspects: {
+                        container: {
+                            containerProperties: 156,
+                            container: 117,
+                        },
+                        dataset: {
+                            status: 1505,
+                            schemaMetadata: 1505,
+                            datasetProperties: 1505,
+                            container: 1505,
+                            operation: 1521,
+                        },
+                    },
+                },
+            },
+        };
+
+        const result = getEntitiesIngestedByType(mockExecutionRequestResult(structuredReport));
+
+        expect(result).toEqual([
+            {
+                count: 156,
+                displayName: 'container',
+            },
+            {
+                count: 1521,
+                displayName: 'dataset',
+            },
+        ]);
+    });
+
+    test('handles empty aspects object', () => {
+        const structuredReport = {
+            source: {
+                report: {
+                    aspects: {},
+                },
+            },
+        };
+
+        const result = getEntitiesIngestedByType(mockExecutionRequestResult(structuredReport));
+        expect(result).toEqual([]);
+    });
+
+    test('handles aspects with non-numeric values', () => {
+        const structuredReport = {
+            source: {
+                report: {
+                    aspects: {
+                        container: {
+                            containerProperties: '156',
+                            container: 117,
+                        },
+                    },
+                },
+            },
+        };
+
+        const result = getEntitiesIngestedByType(mockExecutionRequestResult(structuredReport));
+        expect(result).toEqual([
+            {
+                count: 156,
+                displayName: 'container',
+            },
+        ]);
+    });
+});

datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx

@@ -193,7 +193,9 @@ export const ExecutionDetailsModal = ({ urn, open, onClose }: Props) => {
                 </StatusSection>
                 {(status === SUCCESS || status === SUCCEEDED_WITH_WARNINGS) && (
                     <IngestedAssetsSection>
-                        {data?.executionRequest?.id && <IngestedAssets id={data?.executionRequest?.id} />}
+                        {data?.executionRequest?.id && (
+                            <IngestedAssets executionResult={result} id={data?.executionRequest?.id} />
+                        )}
                     </IngestedAssetsSection>
                 )}
                 <LogsSection>