[ui] Add Integrations Marketplace to app (#29027)

## Summary & Motivation

Behind a (still-hidden) feature flag, add the Integrations Marketplace to the app.

- Retrieve integration data from our new public API.
- Populate a top-level marketplace list.
- Populate pages for individual integrations.

There is still more to add here that will be in followups:

- More metadata, namely for pypi/repo information and "built by" data.
- The individual integration page needs to be updated to match current mocks.
- We have more work to do on the markdown itself, e.g. showing both `pip` and `uv` commands.

<img width="1570" alt="Screenshot 2025-04-09 at 12 36 50" src="https://github.com/user-attachments/assets/e3f882a8-313c-489c-b109-b7e8bd6242c5" />

<img width="1411" alt="Screenshot 2025-04-09 at 12 39 57" src="https://github.com/user-attachments/assets/da9adc3c-5406-43b1-8955-b61972587385" />

## How I Tested These Changes

Add myself to the feature flag, view Integrations Marketplace. Verify correct rendering of list page and individual integration pages.

Outside of the flag, verify that these routes are not accessible, and that the top nav button link does not appear.

Author: hellendag

Diff for: js_modules/dagster-ui/packages/ui-components/src/components/Icon.tsx

@@ -288,6 +288,7 @@ import priority_6 from '../icon-svgs/priority_6.svg';
 import priority_7 from '../icon-svgs/priority_7.svg';
 import priority_8 from '../icon-svgs/priority_8.svg';
 import priority_9 from '../icon-svgs/priority_9.svg';
+import python from '../icon-svgs/python.svg';
 import radio_checked from '../icon-svgs/radio_checked.svg';
 import radio_empty from '../icon-svgs/radio_empty.svg';
 import rainbow from '../icon-svgs/rainbow.svg';
@@ -697,6 +698,7 @@ export const Icons = {
   priority_7,
   priority_8,
   priority_9,
+  python,
   radio_checked,
   radio_empty,
   rainbow,

Diff for: js_modules/dagster-ui/packages/ui-components/src/icon-svgs/python.svg

@@ -18,7 +18,6 @@
     "generate-selection-autocomplete": "ts-node -O '{\"module\": \"commonjs\"}' ./src/scripts/generateSelection.ts && eslint src/selection/generated/ --fix -c .eslintrc.js",
     "generate-run-selection": "ts-node -O '{\"module\": \"commonjs\"}' ./src/scripts/generateRunSelection.ts && eslint src/run-selection/generated/ --fix -c .eslintrc.js",
     "generate-op-selection": "ts-node -O '{\"module\": \"commonjs\"}' ./src/scripts/generateOpSelection.ts && eslint src/op-selection/generated/ --fix -c .eslintrc.js",
-    "generate-integration-docs": "ts-node -O '{\"module\": \"commonjs\"}' ./src/scripts/generateIntegrationDocs.ts",
     "storybook": "storybook dev -p 6006",
     "build-storybook": "storybook build"
   },
@@ -48,6 +47,7 @@
     "chart.js": "^3.4.1",
     "chartjs-adapter-date-fns": "^2.0.0",
     "chartjs-plugin-zoom": "^1.1.1",
+    "clsx": "^2.1.1",
     "codemirror": "^5.65.2",
     "cronstrue": "^2.51.0",
     "dagre": "dagster-io/dagre#0.8.5",
@@ -167,11 +167,9 @@
     "resize-observer-polyfill": "^1.5.1",
     "storybook": "^8.6.0",
     "styled-components": "^5.3.3",
-    "to-vfile": "^8.0.0",
     "ts-node": "10.9.2",
     "ts-prune": "0.10.3",
     "typescript": "5.5.4",
-    "vfile-matter": "^5.0.1",
     "webpack": "^5.94.0"
   },
   "babelMacros": {},

Diff for: js_modules/dagster-ui/packages/ui-core/package.json

@@ -4,6 +4,8 @@ import {NavLink} from 'react-router-dom';
 import {AppTopNavRightOfLogo} from 'shared/app/AppTopNav/AppTopNavRightOfLogo.oss';
 import styled from 'styled-components';
 
+import {MarketplaceTopNavLink} from '../../integrations/MarketplaceTopNavLink';
+import {useFeatureFlags} from '../Flags';
 import {GhostDaggyWithTooltip} from './GhostDaggy';
 import {
   reloadFnForWorkspace,
@@ -20,6 +22,7 @@ interface Props {
 }
 
 export const AppTopNav = ({children, allowGlobalReload = false}: Props) => {
+  const {flagMarketplace} = useFeatureFlags();
   const {reloading, tryReload} = useRepositoryLocationReload({
     scope: 'workspace',
     reloadFn: reloadFnForWorkspace,
@@ -46,6 +49,7 @@ export const AppTopNav = ({children, allowGlobalReload = false}: Props) => {
           </ShortcutHandler>
         ) : null}
         <SearchDialog />
+        {flagMarketplace ? <MarketplaceTopNavLink /> : null}
         {children}
       </Box>
     </AppTopNavContainer>

Diff for: js_modules/dagster-ui/packages/ui-core/src/app/AppTopNav/AppTopNav.tsx

@@ -24,6 +24,7 @@ const SnapshotRoot = lazy(() => import('../snapshots/SnapshotRoot'));
 const GuessJobLocationRoot = lazy(() => import('../workspace/GuessJobLocationRoot'));
 const SettingsRoot = lazy(() => import('../settings/SettingsRoot'));
 const JobsRoot = lazy(() => import('../jobs/JobsRoot'));
+const IntegrationsRoot = lazy(() => import('../integrations/IntegrationsRoot'));
 
 export const ContentRoot = memo(() => {
   const {pathname} = useLocation();
@@ -91,6 +92,9 @@ export const ContentRoot = memo(() => {
           <Route path="/deployment">
             <SettingsRoot />
           </Route>
+          <Route path="/integrations">
+            <IntegrationsRoot />
+          </Route>
           <Route path="*" isNestingRoute>
             <FallthroughRoot />
           </Route>

Diff for: js_modules/dagster-ui/packages/ui-core/src/app/ContentRoot.tsx

@@ -6,6 +6,7 @@ export enum FeatureFlag {
   flagSelectionSyntax = 'flagSelectionSyntax-always-on',
   flagAssetSelectionWorker = 'flagAssetSelectionWorker',
   flagUseNewObserveUIs = 'flagUseNewObserveUIs',
+  flagMarketplace = 'flagMarketplace',
 
   // Flags for tests
   __TestFlagDefaultNone = '__TestFlagDefaultNone',

Diff for: js_modules/dagster-ui/packages/ui-core/src/app/FeatureFlags.oss.tsx

@@ -1,22 +1,24 @@
 import {Box, Icon, IconWrapper} from '@dagster-io/ui-components';
 import styled from 'styled-components';
 
+const INTEGRATIONS_ORIGIN_AND_PATH = 'https://integration-registry.dagster.io/logos';
+
 interface Props {
   name: string;
-  logo: string | {src: string} | null;
+  logoFilename: string | null;
 }
 
-export const IntegrationIcon = ({name, logo}: Props) => {
+export const IntegrationIcon = ({name, logoFilename}: Props) => {
   const icon = () => {
-    if (logo === null) {
+    if (logoFilename === null) {
       return <Icon name="workspace" size={24} />;
     }
 
     return (
       <IntegrationIconWrapper
         role="img"
         $size={32}
-        $img={extractIconSrc(logo)}
+        $img={`${INTEGRATIONS_ORIGIN_AND_PATH}/${logoFilename}`}
         $color={null}
         $rotation={null}
         aria-label={name}
@@ -36,15 +38,6 @@ export const IntegrationIcon = ({name, logo}: Props) => {
   );
 };
 
-function extractIconSrc(iconSvg: string | {src: string}) {
-  // Storybook imports SVGs are string but nextjs imports them as object.
-  // This is a temporary work around until we can get storybook to import them the same way as nextjs
-  if (typeof iconSvg !== 'undefined') {
-    return typeof iconSvg === 'string' ? (iconSvg as any) : iconSvg?.src;
-  }
-  return '';
-}
-
 const IntegrationIconWrapper = styled(IconWrapper)`
   mask-size: contain;
   mask-repeat: no-repeat;

Diff for: js_modules/dagster-ui/packages/ui-core/src/integrations/IntegrationIcon.tsx

@@ -1,8 +1,8 @@
-import {Body, Box, Colors, Heading, PageHeader} from '@dagster-io/ui-components';
+import {Body, Box, Colors} from '@dagster-io/ui-components';
+import clsx from 'clsx';
 import {useLayoutEffect, useRef, useState} from 'react';
 import ReactMarkdown from 'react-markdown';
-import {CodeComponent} from 'react-markdown/lib/ast-to-react';
-import {Link} from 'react-router-dom';
+import {Components} from 'react-markdown/lib/ast-to-react';
 import rehypeHighlight from 'rehype-highlight';
 import remarkGfm from 'remark-gfm';
 
@@ -17,31 +17,15 @@ interface Props {
 
 export const IntegrationPage = ({integration}: Props) => {
   const {
-    frontmatter: {name, title, excerpt},
-    logo,
+    frontmatter: {name, title, excerpt, logoFilename},
     content,
   } = integration;
 
   return (
     <div>
-      <PageHeader
-        title={
-          <Heading>
-            <Box flex={{direction: 'row', gap: 8}}>
-              <Link to="/integrations">Integrations Marketplace</Link>
-              <span> / </span>
-              <div>{title}</div>
-            </Box>
-          </Heading>
-        }
-      />
-      <Box
-        padding={{vertical: 24}}
-        flex={{direction: 'column', gap: 24}}
-        style={{width: '1100px', margin: '0 auto'}}
-      >
+      <Box padding={{vertical: 24}} flex={{direction: 'column', gap: 12}}>
         <Box flex={{direction: 'row', gap: 12, alignItems: 'flex-start'}}>
-          <IntegrationIcon name={name} logo={logo} />
+          <IntegrationIcon name={name} logoFilename={logoFilename} />
           <Box flex={{direction: 'column', gap: 2}} margin={{top: 4}}>
             <div style={{fontSize: 18, fontWeight: 600}}>{title}</div>
             <Body color={Colors.textLight()}>{excerpt}</Body>
@@ -54,6 +38,7 @@ export const IntegrationPage = ({integration}: Props) => {
             rehypePlugins={[[rehypeHighlight, {ignoreMissing: true}]]}
             components={{
               code: Code,
+              a: Anchor,
             }}
           >
             {content}
@@ -64,15 +49,36 @@ export const IntegrationPage = ({integration}: Props) => {
   );
 };
 
-const Code: CodeComponent = (props) => {
-  const {children, className, ...rest} = props;
+const DOCS_ORIGIN = 'https://docs.dagster.io';
+
+const Anchor: Components['a'] = (props) => {
+  const {children, href, ...rest} = props;
+  const finalHref = href?.startsWith('/') ? `${DOCS_ORIGIN}${href}` : href;
+  return (
+    <a href={finalHref} target="_blank" rel="noreferrer" {...rest}>
+      {children}
+    </a>
+  );
+};
+
+const Code: Components['code'] = (props) => {
+  const {children, className, inline, ...rest} = props;
+
   const codeRef = useRef<HTMLElement>(null);
   const [value, setValue] = useState('');
 
   useLayoutEffect(() => {
     setValue(codeRef.current?.textContent?.trim() ?? '');
   }, [children]);
 
+  if (inline) {
+    return (
+      <code className={clsx(className, styles.inlineCode)} {...rest}>
+        {children}
+      </code>
+    );
+  }
+
   return (
     <div className={styles.codeBlock}>
       <code className={className} {...rest} ref={codeRef}>

Diff for: js_modules/dagster-ui/packages/ui-core/src/integrations/IntegrationPage.tsx

@@ -2,36 +2,38 @@ import {IconName} from '@dagster-io/ui-components';
 
 export enum IntegrationTag {
   Alerting = 'alerting',
-  BiTools = 'bi-tools',
-  ComponentReady = 'component-ready',
+  BiTools = 'bi',
+  CommunitySupported = 'community-supported',
   Compute = 'compute',
-  EltTools = 'elt-tools',
+  DagsterSupported = 'dagster-supported',
+  EtlTools = 'etl',
   Metadata = 'metadata',
   Monitoring = 'monitoring',
-  Notifications = 'notifications',
   Storage = 'storage',
 }
 
+export const IntegrationTagKeys: string[] = Object.values(IntegrationTag);
+
 export const IntegrationTagLabel: Record<IntegrationTag, string> = {
   [IntegrationTag.Alerting]: 'Alerting',
   [IntegrationTag.BiTools]: 'BI tools',
-  [IntegrationTag.ComponentReady]: 'Component-ready',
+  [IntegrationTag.CommunitySupported]: 'Community Supported',
   [IntegrationTag.Compute]: 'Compute',
-  [IntegrationTag.EltTools]: 'ELT tools',
+  [IntegrationTag.DagsterSupported]: 'Dagster Supported',
+  [IntegrationTag.EtlTools]: 'ETL tools',
   [IntegrationTag.Metadata]: 'Metadata',
   [IntegrationTag.Monitoring]: 'Monitoring',
-  [IntegrationTag.Notifications]: 'Notifications',
   [IntegrationTag.Storage]: 'Storage',
 };
 
 export const IntegrationTagIcon: Record<IntegrationTag, IconName> = {
   [IntegrationTag.Alerting]: 'alert',
   [IntegrationTag.BiTools]: 'chart_bar',
-  [IntegrationTag.ComponentReady]: 'repo',
+  [IntegrationTag.CommunitySupported]: 'people',
   [IntegrationTag.Compute]: 'speed',
-  [IntegrationTag.EltTools]: 'transform',
+  [IntegrationTag.DagsterSupported]: 'dagster_solid',
+  [IntegrationTag.EtlTools]: 'transform',
   [IntegrationTag.Metadata]: 'metadata',
   [IntegrationTag.Monitoring]: 'visibility',
-  [IntegrationTag.Notifications]: 'notifications',
   [IntegrationTag.Storage]: 'download',
 };

Diff for: js_modules/dagster-ui/packages/ui-core/src/integrations/IntegrationTag.tsx

@@ -0,0 +1,23 @@
+import {Redirect, Route, Switch} from 'react-router';
+
+import {MarketplaceRoot} from './MarketplaceRoot';
+import {SingleIntegrationRoot} from './SingleIntegrationRoot';
+import {useFeatureFlags} from '../app/Flags';
+
+const IntegrationsRoot = () => {
+  const {flagMarketplace} = useFeatureFlags();
+
+  if (!flagMarketplace) {
+    return <Redirect to="/deployment" />;
+  }
+
+  return (
+    <Switch>
+      <Route path="/integrations" component={MarketplaceRoot} exact />
+      <Route path="/integrations/:integrationName" component={SingleIntegrationRoot} />
+    </Switch>
+  );
+};
+
+// eslint-disable-next-line import/no-default-export
+export default IntegrationsRoot;