Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
| Parameter | Description | Example | Required |
| --------- | ----------- | ------- | -------- |
| `OCEAN__INTEGRATION__CONFIG__GITHUB_TOKEN` | A GitHub Personal Access Token (PAT) to authenticate with your GitHub Organization. Only required when you're not authenticating as a [Github app](./installation/github-app.mdx) | | ✅ |
| `OCEAN__INTEGRATION__CONFIG__GITHUB_TOKEN` | A GitHub Personal Access Token (PAT) to authenticate with your GitHub Organization. Use classic PAT for multi-org support (v2.0.0-beta+). Only required when you're not authenticating as a [Github app](./installation/github-app.mdx) | | ✅ |
| `OCEAN__INTEGRATION__CONFIG__GTIHUB_HOST` | The API endpoint for your GitHub instance. For GitHub Enterprise Cloud, this will be `https://<org_name.ghe.com>`. Defaults to `https://api.github.com` if not provided. | | ❌ |
| `OCEAN__INTEGRATION__CONFIG__GTIHUB_ORGANIZATION` | The name of your Github organization | | |
| `OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATIONS` | JSON array of GitHub organization names to sync from (e.g., `["org1","org2"]`). From v2.0.0-beta: classic PAT supports multiple orgs, GitHub App supports only 1 org. Leave empty with classic PAT to sync all accessible organizations. | | |
| `OCEAN__PORT__CLIENT_ID` | Your Port client id([How to get the credentials](https://docs.port.io/build-your-software-catalog/custom-integration/api/#find-your-port-credentials)) | | ✅ |
| `OCEAN__PORT__CLIENT_SECRET` | Your Port client secret ([How to get the credentials](https://docs.port.io/build-your-software-catalog/custom-integration/api/#find-your-port-credentials)) | | ✅ |
| `OCEAN__PORT__BASE_URL` | Your Port API URL - `https://api.getport.io` for EU, `https://api.us.getport.io` for US | | ✅ |
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
- [`organization`](https://docs.github.com/en/rest/orgs/orgs?apiVersion=2022-11-28#list-organizations-for-the-authenticated-user)
- [`repository`](https://docs.github.com/en/rest/repos/repos#get-a-repository)
- [`pull-request`](https://docs.github.com/en/rest/pulls/pulls#get-a-pull-request)
- [`file`](/build-your-software-catalog/sync-data-to-catalog/git/github/#ingest-files-from-your-repositories)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ resources:
readme: file://README.md # fetching the README.md file that is within the root folder of the repository and ingesting its contents as a markdown property
url: .html_url
defaultBranch: .default_branch
relations:
organization: .__organization.login
- kind: pull-request
selector:
query: "true" # JQ boolean query. If evaluated to false - skip syncing the object.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,14 @@
},
"mirrorProperties": {},
"calculationProperties": {},
"relations": {}
"relations": {
"organization": {
"title": "Organization",
"target": "githubOrganization",
"required": false,
"many": false
}
}
}
```

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,10 @@ resources:
query: 'true'
files:
- path: 'README.md'
organization: my-org # Required from v2.0.0-beta
repos:
- name: test-repo
branch: main
port:
entity:
mappings:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
query: 'true'
files:
- path: '**/package.json'
organization: my-org # Required from v2.0.0-beta
repos:
- name: vscode
branch: main
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ resources:
query: "true" # JQ boolean query. If evaluated to false - skip syncing the object.
folders: # Specify the repositories and folders to include under this relative path.
- path: apps/* # Relative path to the folders within the repositories.
organization: my-org # Required from v2.0.0-beta
repos: # List of repositories to include folders from.
- name: backend-service
branch: main
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
<details>
<summary><b>Organization blueprint (click to expand)</b></summary>

```json showLineNumbers
{
"identifier": "githubOrganization",
"title": "GitHub Organization",
"icon": "Github",
"schema": {
"properties": {
"url": {
"title": "Organization URL",
"type": "string",
"format": "url"
},
"description": {
"title": "Description",
"type": "string"
},
"repositories": {
"title": "Repositories",
"type": "number"
},
"createdAt": {
"title": "Created At",
"type": "string",
"format": "date-time"
}
},
"required": []
},
"mirrorProperties": {},
"calculationProperties": {},
"relations": {}
}
```

</details>

Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
<details>

<summary><b>Port port-app-config.yml (click to expand)</b></summary>

```yaml showLineNumbers
resources:
- kind: organization
selector:
query: "true" # JQ boolean query. If evaluated to false - skip syncing the object.
port:
entity:
mappings:
identifier: .login
title: .name
blueprint: '"githubOrganization"'
properties:
url: .html_url
description: .description
repositories: .public_repos
createdAt: .created_at
- kind: repository
selector:
query: "true"
port:
entity:
mappings:
identifier: .name
title: .name
blueprint: '"githubRepository"'
properties:
readme: file://README.md
url: .html_url
defaultBranch: .default_branch
relations:
organization: .__organization.login
```

</details>

Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,36 @@ import DependabotAlertBlueprint from './example-repository-alerts/\_github_expor
import CodeScanAlertBlueprint from './example-repository-alerts/\_github_exporter_example_codeScan_alert_blueprint.mdx'
import PortRepositoryDependabotAlertMappingAppConfig from './example-repository-alerts/\_github_exporter_example_repo_dependabot_port_app_config.mdx'

import OrganizationBlueprint from './example-organization/\_github_exporter_example_organization_blueprint.mdx'
import OrganizationAppConfig from './example-organization/\_github_exporter_example_organization_port_app_config.mdx'


# Resource mapping examples

## Map organizations and repositories

:::info Available from v2.0.0-beta
The `organization` kind is available from version `v2.0.0-beta` onwards.
:::

The following example demonstrates how to ingest your GitHub organizations and their repositories to Port.
You can use the following Port blueprint definitions and `port-app-config.yml`:

<OrganizationBlueprint/>

<RepositoryBlueprint/>

<OrganizationAppConfig/>

:::tip learn more

- Port leverages the [JQ JSON processor](https://stedolan.github.io/jq/manual/) to map and transform GitHub objects to Port Entities.
- Click [Here](https://docs.github.com/en/rest/orgs/orgs#get-an-organization) for the GitHub organization object structure.
- Click [Here](https://docs.github.com/en/rest/repos/repos#get-a-repository) for the GitHub repository object structure.

:::

After creating the blueprints and committing the `port-app-config.yml` file, you will see new entities in Port matching your organizations and their repositories. The repositories will have a relation to their parent organization.

## Map repositories and pull requests

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,27 @@ Here's what you can do with the GitHub integration:
- Map and organize your desired GitHub resources and their metadata in Port (see supported resources below).
- Watch for GitHub object changes (create/update/delete) in real-time, and automatically apply the changes to your software catalog.
- Manage Port entities using GitOps.
- **Sync data from multiple GitHub organizations** using a single integration instance.

### Multi-organization support

The GitHub integration supports syncing data from multiple GitHub organizations starting from **version 2.0.0-beta**. You can configure which organizations to sync using the `githubOrganizations` configuration parameter.

:::info Authentication requirements
- **Classic PAT**: Supports multiple organizations. Fine-grained PAT tokens do not support multi-organization authentication.
- **GitHub App**: Only supports a **single organization** (minimum 1, maximum 1). You must specify exactly one organization in the `githubOrganizations` array.
:::

:::caution Performance impact
Syncing multiple organizations will increase the number of API calls to GitHub and may slow down the integration. The more organizations you sync, the longer the resync time and the higher the API rate limit consumption. Consider syncing only the organizations you need.
:::
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same comment about the admonition


**Configuration options:**
- **With classic PAT**: Specify a list of organizations: `githubOrganizations: ["org1", "org2", "org3"]`
- **With classic PAT**: Leave empty to sync all organizations the PAT user is a member of: `githubOrganizations: []`
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DRY

- **With GitHub App**: Specify exactly one organization: `githubOrganizations: ["my-org"]`

When using multi-organization support, the `organization` resource kind is automatically synced, allowing you to model your GitHub organizations in Port.

### Supported resources

Expand Down Expand Up @@ -136,10 +157,39 @@ resources:

Using Port's GitHub integration, you can automatically ingest GitHub resources into Port based on real-time events.

The app allows you to ingest a variety of objects resources provided by the GitHub API, including repositories, pull requests, workflows and more. It also allows you to perform "extract, transform, load (ETL)" on data from the GitHub API into the desired software catalog data model.
The app allows you to ingest a variety of objects resources provided by the GitHub API, including organizations, repositories, pull requests, workflows and more. It also allows you to perform "extract, transform, load (ETL)" on data from the GitHub API into the desired software catalog data model.

The GitHub integration uses a YAML configuration file to describe the ETL process to load data into the developer portal. This approach provides a flexible and powerful way to model your Git data without being overly opinionated or complex.

### Ingest organizations

The GitHub integration can automatically sync organization-level data when using multi-organization support (available from **v2.0.0-beta**).

Here's an example configuration for the `organization` kind:

```yaml showLineNumbers
resources:
- kind: organization
selector:
query: 'true'
port:
entity:
mappings:
identifier: .login
title: .name
blueprint: '"githubOrganization"'
properties:
url: .html_url
description: .description
createdAt: .created_at
blog: .blog
location: .location
```

:::tip Organization as parent entity
Organizations can serve as parent entities for repositories, teams, and other GitHub resources, helping you model your organizational structure in Port.
:::

### Ingest files from your repositories

Port allows you to fetch `JSON` and `YAML` files from your repositories, and create entities from them in your software catalog.
Expand All @@ -149,6 +199,10 @@ For example, say you want to manage your `package.json` files in Port. One optio

The following configuration fetches all `package.json` files from "MyRepo" and "MyOtherRepo", and creates an entity for each of them, based on the `manifest` blueprint:

:::warning Breaking change in v2.0.0-beta
Starting from version 2.0.0-beta, the `file` kind requires an `organization` field to be specified. This is a breaking change from previous versions.
:::

```yaml showLineNumbers
resources:
- kind: file
Expand All @@ -157,6 +211,7 @@ resources:
files:
# Note that glob patterns are supported, so you can use wildcards to match multiple files
- path: '**/package.json'
organization: my-org # Organization name is required from v2.0.0-beta
# The `repos` key can be used to filter the repositories and branch where files should be fetched
repos:
- name: MyRepo
Expand Down Expand Up @@ -679,6 +734,7 @@ resources:
query: 'true'
files:
- path: '**/package.json'
organization: my-org # Organization name is required from v2.0.0-beta
# Note that in this case we are fetching from a specific repository
repos:
- name: MyRepo
Expand Down Expand Up @@ -736,7 +792,11 @@ resources:
query: 'true'
files:
- path: values.yaml
organization: my-org # Organization name is required from v2.0.0-beta
skipParsing: true
repos:
- name: MyRepo
branch: main
port:
entity:
mappings:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,10 @@ To edit a mapping configuration:

For example, say you want to ingest a `package.json` file form your repository. You can add the following to your GitHub integration mapping:

:::warning Breaking change in v2.0.0-beta
Starting from version 2.0.0-beta, the `file` kind requires an `organization` field to be specified for multi-organization support.
:::

```yaml
resources:
...
Expand All @@ -133,13 +137,17 @@ resources:
query: 'true'
files:
- path: package.json
organization: my-org # Required from v2.0.0-beta
repos:
- name: my-repo
branch: main
port:
entity:
mappings:
identifier: .file.name
identifier: .path
blueprint: '"file"'
properties:
content: .file.content
content: .content
```

The `selector.files.path` key also supports glob patterns, so you can ingest multiple files by matching against a pattern and create an entity in Port for each one, for example:
Expand All @@ -150,6 +158,10 @@ The `selector.files.path` key also supports glob patterns, so you can ingest mul
query: 'true'
files:
- path: 'resources/*.yml'
organization: my-org # Required from v2.0.0-beta
repos:
- name: my-repo
branch: main
```

### Advantages
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
config: |
githubHost: ${{ secrets.OCEAN__INTEGRATION__CONFIG__GITHUB_HOST }}
githubToken: ${{ secrets.OCEAN__INTEGRATION__CONFIG__GITHUB_TOKEN }}
githubOrganization: ${{ secrets.OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATION }}
githubOrganizations: ${{ secrets.OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATIONS }} # JSON array format
```

</TabItem>
Expand Down Expand Up @@ -66,7 +66,7 @@ pipeline {
script {
withCredentials([
string(credentialsId: 'OCEAN__INTEGRATION__CONFIG__GITHUB_URL', variable: 'OCEAN__INTEGRATION__CONFIG__GITHUB_URL'),
string(credentialsId: 'OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATION', variable: 'OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATION'),
string(credentialsId: 'OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATIONS', variable: 'OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATIONS'),
string(credentialsId: 'OCEAN__INTEGRATION__CONFIG__GITHUB_TOKEN', variable: 'OCEAN__INTEGRATION__CONFIG__GITHUB_TOKEN'),
string(credentialsId: 'OCEAN__PORT__CLIENT_ID', variable: 'OCEAN__PORT__CLIENT_ID'),
string(credentialsId: 'OCEAN__PORT__CLIENT_SECRET', variable: 'OCEAN__PORT__CLIENT_SECRET'),
Expand All @@ -82,7 +82,7 @@ pipeline {
-e OCEAN__SEND_RAW_DATA_EXAMPLES=true \
-e OCEAN__INTEGRATION__CONFIG__GITHUB_URL=$OCEAN__INTEGRATION__CONFIG__GITHUB_URL \
-e OCEAN__INTEGRATION__CONFIG__GITHUB_TOKEN=$OCEAN__INTEGRATION__CONFIG__GITHUB_TOKEN \
-e OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATION=$OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATION \
-e OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATIONS=$OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATIONS \
-e OCEAN__PORT__CLIENT_ID=$OCEAN__PORT__CLIENT_ID \
-e OCEAN__PORT__CLIENT_SECRET=$OCEAN__PORT__CLIENT_SECRET \
-e OCEAN__PORT__BASE_URL='https://api.getport.io' \
Expand Down Expand Up @@ -129,7 +129,7 @@ steps:
-e OCEAN__SEND_RAW_DATA_EXAMPLES=true \
-e OCEAN__INTEGRATION__CONFIG__GITHUB_HOST=$(OCEAN__INTEGRATION__CONFIG__GITHUB_HOST) \
-e OCEAN__INTEGRATION__CONFIG__GITHUB_TOKEN=$(OCEAN__INTEGRATION__CONFIG__GITHUB_TOKEN) \
-e OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATION=$(OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATION) \
-e OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATIONS=$(OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATIONS) \
-e OCEAN__PORT__CLIENT_ID=$(OCEAN__PORT__CLIENT_ID) \
-e OCEAN__PORT__CLIENT_SECRET=$(OCEAN__PORT__CLIENT_SECRET) \
-e OCEAN__PORT__BASE_URL='https://api.getport.io' \
Expand Down Expand Up @@ -174,7 +174,7 @@ ingest_data:
-e OCEAN__SEND_RAW_DATA_EXAMPLES=true \
-e OCEAN__INTEGRATION__CONFIG__GITHUB_URL=$OCEAN__INTEGRATION__CONFIG__GITHUB_URL \
-e OCEAN__INTEGRATION__CONFIG__GITHUB_TOKEN=$OCEAN__INTEGRATION__CONFIG__GITHUB_TOKEN \
-e OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATION=$OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATION \
-e OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATIONS=$OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATIONS \
-e OCEAN__PORT__CLIENT_ID=$OCEAN__PORT__CLIENT_ID \
-e OCEAN__PORT__CLIENT_SECRET=$OCEAN__PORT__CLIENT_SECRET \
-e OCEAN__PORT__BASE_URL='https://api.getport.io' \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ Run the following command to start the app:
-e OCEAN__BASE_URL="<https.example.com>" \\ #optional, only required if you want to enable live-events
-e OCEAN__EVENT_LISTENER__TYPE="POLLING" \\
-e OCEAN__INTEGRATION__CONFIG__GITHUB_HOST="<GITHUB_HOST>" \\ # e.g https://api.github.com
-e OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATION="<GITHUB_ORGANIZATION>" \\
-e OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATIONS='["<GITHUB_ORGANIZATION>"]' \\ # GitHub App supports only 1 org
-e OCEAN__INTEGRATION__IDENTIFIER="github-ocean" \\
-e OCEAN__INTEGRATION__CONFIG__GITHUB_APP_ID="<GITHUB_APP_ID>" \\ # client id also works
-e OCEAN__INTEGRATION__CONFIG__GITHUB_APP_PRIVATE_KEY="<BASE_64_ENCODED_PRIVATEKEY>" \\
Expand All @@ -38,7 +38,7 @@ Run the following command to start the app:
-e OCEAN__BASE_URL="<https.example.com>" \\ #optional, only required if you want to enable live-events
-e OCEAN__EVENT_LISTENER__TYPE="POLLING" \\
-e OCEAN__INTEGRATION__CONFIG__GITHUB_HOST="<GITHUB_HOST>" \\ # e.g https://api.github.com
-e OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATION="<GITHUB_ORGANIZATION>" \\
-e OCEAN__INTEGRATION__CONFIG__GITHUB_ORGANIZATIONS='["<GITHUB_ORG_1>","<GITHUB_ORG_2>"]' \\ # Classic PAT supports multiple orgs from v2.0.0-beta
-e OCEAN__INTEGRATION__IDENTIFIER="github-ocean" \\
-e OCEAN__INTEGRATION__CONFIG__GITHUB_TOKEN="<GITHUB_TOKEN>" \\
-p 8000:8000 \\
Expand Down
Loading