Merge pull request #13 from datum-cloud/feat/github-catalog-integration

feat: pull catalog data and teams from GitHub

Author: scotwells

app-config.yaml

@@ -59,13 +59,21 @@ backend:
 integrations:
   github:
     - host: github.com
-      # This is a Personal Access Token or PAT from GitHub. You can find out how to generate this token, and more information
-      # about setting up the GitHub integration here: https://backstage.io/docs/integrations/github/locations#configuration
+      # Local development: a Personal Access Token. Resolves to empty when
+      # GITHUB_TOKEN is unset, so the integration is simply inactive locally.
+      # See https://backstage.io/docs/integrations/github/locations#configuration
       token: ${GITHUB_TOKEN}
-    ### Example for how to add your GitHub Enterprise instance using the API:
-    # - host: ghe.example.net
-    #   apiBaseUrl: https://ghe.example.net/api/v3
-    #   token: ${GHE_TOKEN}
+      # Deployed environments authenticate as a GitHub App instead of a PAT
+      # (org-wide install, higher rate limits, no personal token to rotate).
+      # The credentials are already projected into the pod by the
+      # external-secrets component (backstage-github-app -> GITHUB_APP_*).
+      # Enable per-environment by supplying this block from the deploy overlay:
+      # apps:
+      #   - appId: ${GITHUB_APP_ID}
+      #     clientId: ${GITHUB_CLIENT_ID}
+      #     clientSecret: ${GITHUB_CLIENT_SECRET}
+      #     webhookSecret: ${GITHUB_WEBHOOK_SECRET}
+      #     privateKey: ${GITHUB_PRIVATE_KEY}
 
 proxy:
   ### Example for how to add a proxy endpoint for the frontend.
@@ -126,6 +134,29 @@ catalog:
     pullRequestBranchName: backstage-integration
   rules:
     - allow: [Component, System, API, Resource, Location]
+  # GitHub as a source of catalog data. Left unset for local dev / e2e (which
+  # use the file locations below); enable per-environment from the deploy
+  # overlay, where the GitHub App credentials are available. `github` scans the
+  # org for catalog-info.yaml (self-declared component/API/dependency facts);
+  # `githubOrg` imports org members and teams as User/Group entities so
+  # `owner:` refs resolve to real groups.
+  # providers:
+  #   github:
+  #     datumCloud:
+  #       organization: datum-cloud
+  #       catalogPath: /catalog-info.yaml
+  #       filters:
+  #         repository: '.*'
+  #       schedule:
+  #         frequency: { minutes: 30 }
+  #         timeout: { minutes: 3 }
+  #   githubOrg:
+  #     - id: datum-cloud
+  #       githubUrl: https://github.com
+  #       orgs: [datum-cloud]
+  #       schedule:
+  #         frequency: { hours: 1 }
+  #         timeout: { minutes: 5 }
   locations:
     # Local example data, file locations are relative to the backend process, typically `packages/backend`
     - type: file

packages/backend/package.json

@@ -26,6 +26,8 @@
     "@backstage/plugin-auth-backend-module-guest-provider": "^0.2.20",
     "@backstage/plugin-auth-node": "^0.7.2",
     "@backstage/plugin-catalog-backend": "^3.8.0",
+    "@backstage/plugin-catalog-backend-module-github": "^0.13.3",
+    "@backstage/plugin-catalog-backend-module-github-org": "^0.3.23",
     "@backstage/plugin-catalog-backend-module-logs": "^0.1.23",
     "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "^0.2.21",
     "@backstage/plugin-kubernetes-backend": "^0.21.5",

packages/backend/src/index.ts

@@ -36,6 +36,12 @@ backend.add(
   import('@backstage/plugin-catalog-backend-module-scaffolder-entity-model'),
 );
 
+// Discover catalog-info.yaml across the GitHub org and import org users/teams
+// as User/Group entities. Both read config from catalog.providers.* and stay
+// dormant until that config is supplied per-environment.
+backend.add(import('@backstage/plugin-catalog-backend-module-github'));
+backend.add(import('@backstage/plugin-catalog-backend-module-github-org'));
+
 // See https://backstage.io/docs/features/software-catalog/configuration#subscribing-to-catalog-errors
 backend.add(import('@backstage/plugin-catalog-backend-module-logs'));
 

yarn.lock

@@ -2642,6 +2642,47 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@backstage/plugin-catalog-backend-module-github-org@npm:^0.3.23":
+  version: 0.3.23
+  resolution: "@backstage/plugin-catalog-backend-module-github-org@npm:0.3.23"
+  dependencies:
+    "@backstage/backend-plugin-api": "npm:^1.9.2"
+    "@backstage/config": "npm:^1.3.8"
+    "@backstage/plugin-catalog-backend-module-github": "npm:^0.13.3"
+    "@backstage/plugin-catalog-node": "npm:^2.2.2"
+    "@backstage/plugin-events-node": "npm:^0.4.23"
+  checksum: 10c0/435f9340db9419a2ee13d444d63f8fda20ed3ab7da1b864a8c02b902e84d8dff7364bbe245ebc39404f9957990ff77dba6d7619d74df5bfb4e329ed3c660a0e9
+  languageName: node
+  linkType: hard
+
+"@backstage/plugin-catalog-backend-module-github@npm:^0.13.3":
+  version: 0.13.3
+  resolution: "@backstage/plugin-catalog-backend-module-github@npm:0.13.3"
+  dependencies:
+    "@backstage/backend-plugin-api": "npm:^1.9.2"
+    "@backstage/catalog-model": "npm:^1.9.0"
+    "@backstage/config": "npm:^1.3.8"
+    "@backstage/errors": "npm:^1.3.1"
+    "@backstage/integration": "npm:^2.0.3"
+    "@backstage/plugin-catalog-common": "npm:^1.1.10"
+    "@backstage/plugin-catalog-node": "npm:^2.2.2"
+    "@backstage/plugin-events-node": "npm:^0.4.23"
+    "@backstage/types": "npm:^1.2.2"
+    "@octokit/auth-callback": "npm:^5.0.0"
+    "@octokit/core": "npm:^5.2.0"
+    "@octokit/graphql": "npm:^7.0.2"
+    "@octokit/plugin-retry": "npm:^6.0.0"
+    "@octokit/plugin-throttling": "npm:^8.1.3"
+    "@octokit/rest": "npm:^19.0.3"
+    "@octokit/webhooks-types": "npm:^7.6.1"
+    git-url-parse: "npm:^15.0.0"
+    lodash: "npm:^4.17.21"
+    minimatch: "npm:^10.2.1"
+    octokit: "npm:^3.0.0"
+  checksum: 10c0/49d17b678117593ca760a243aebff1b92d0d3afb5c9e5246ca7f479f5b755fd1829a82dd5633c228782d1bb9680802e86e6c8d2b83d72a44a77f8c39889a81c7
+  languageName: node
+  linkType: hard
+
 "@backstage/plugin-catalog-backend-module-logs@npm:^0.1.23":
   version: 0.1.23
   resolution: "@backstage/plugin-catalog-backend-module-logs@npm:0.1.23"
@@ -6987,6 +7028,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@octokit/auth-callback@npm:^5.0.0":
+  version: 5.0.1
+  resolution: "@octokit/auth-callback@npm:5.0.1"
+  checksum: 10c0/851d0930aeecddc626655469ef57727654af30755264d5139d9fdf61ba9fc88c1a8685f5721f853c806c570b8529d25d262f4efe76da90c08c779d9cc1c37edf
+  languageName: node
+  linkType: hard
+
 "@octokit/auth-oauth-app@npm:^5.0.0":
   version: 5.0.6
   resolution: "@octokit/auth-oauth-app@npm:5.0.6"
@@ -7108,7 +7156,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@octokit/core@npm:^5.0.0":
+"@octokit/core@npm:^5.0.0, @octokit/core@npm:^5.2.0":
   version: 5.2.2
   resolution: "@octokit/core@npm:5.2.2"
   dependencies:
@@ -7155,7 +7203,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@octokit/graphql@npm:^7.1.0":
+"@octokit/graphql@npm:^7.0.2, @octokit/graphql@npm:^7.1.0":
   version: 7.1.1
   resolution: "@octokit/graphql@npm:7.1.1"
   dependencies:
@@ -7337,7 +7385,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@octokit/plugin-throttling@npm:^8.0.0":
+"@octokit/plugin-throttling@npm:^8.0.0, @octokit/plugin-throttling@npm:^8.1.3":
   version: 8.2.0
   resolution: "@octokit/plugin-throttling@npm:8.2.0"
   dependencies:
@@ -7482,7 +7530,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@octokit/webhooks-types@npm:7.6.1":
+"@octokit/webhooks-types@npm:7.6.1, @octokit/webhooks-types@npm:^7.6.1":
   version: 7.6.1
   resolution: "@octokit/webhooks-types@npm:7.6.1"
   checksum: 10c0/7c2cb40f9ccd2bd392cf35c23f995ae0719ef35fd3bce0264ced5518cbf0a7087bd069bf5e5963fc33d0232726968db526912df3cb017c1bd1761c8849c31a30
@@ -13055,6 +13103,8 @@ __metadata:
     "@backstage/plugin-auth-backend-module-guest-provider": "npm:^0.2.20"
     "@backstage/plugin-auth-node": "npm:^0.7.2"
     "@backstage/plugin-catalog-backend": "npm:^3.8.0"
+    "@backstage/plugin-catalog-backend-module-github": "npm:^0.13.3"
+    "@backstage/plugin-catalog-backend-module-github-org": "npm:^0.3.23"
     "@backstage/plugin-catalog-backend-module-logs": "npm:^0.1.23"
     "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "npm:^0.2.21"
     "@backstage/plugin-kubernetes-backend": "npm:^0.21.5"