From 16543d78ab8258d68cb4623452d02692edea05dc Mon Sep 17 00:00:00 2001 From: ghe Date: Mon, 9 Jan 2023 07:29:10 +0000 Subject: [PATCH 1/4] feat: sync command archived repos support --- src/scripts/sync/sync-projects-per-target.ts | 49 +++-- test/scripts/sync/sync-org-projects.test.ts | 178 ++++++++++++++++++- 2 files changed, 209 insertions(+), 18 deletions(-) diff --git a/src/scripts/sync/sync-projects-per-target.ts b/src/scripts/sync/sync-projects-per-target.ts index 9f073e7e..79a1546c 100644 --- a/src/scripts/sync/sync-projects-per-target.ts +++ b/src/scripts/sync/sync-projects-per-target.ts @@ -60,6 +60,7 @@ export async function syncProjectsForTarget( debug(`Syncing projects for target ${target.attributes.displayName}`); let targetMeta: RepoMetaData; + let isDeleted = false; const origin = target.attributes .origin as SupportedIntegrationTypesUpdateProject; const targetData = getTargetConverter(origin)(target); @@ -69,18 +70,44 @@ export async function syncProjectsForTarget( } catch (e) { //TODO: if repo is deleted, deactivate all projects debug(e); - const error = `Getting default branch via ${origin} API failed with error: ${e.message}`; - projects.map((project) => { - failed.add({ - errorMessage: error, - projectPublicId: project.id, - type: ProjectUpdateType.BRANCH, - from: project.branch!, - to: targetMeta.branch, - dryRun: config.dryRun, - target, + if (e.status === 404) { + // TODO: when else could you get a 404? Manually check + isDeleted = true; + } else { + const error = `Getting metadata from ${origin} API failed with error: ${e.message}`; + projects.map((project) => { + failed.add({ + errorMessage: error, + projectPublicId: project.id, + type: ProjectUpdateType.BRANCH, + from: project.branch!, + to: 'unknown', + dryRun: config.dryRun, + target, + }); }); - }); + return { + updated: Array.from(updated).map((t) => ({ ...t, target })), + failed: Array.from(failed).map((t) => ({ ...t, target })), + }; + } + } + + if (isDeleted || targetMeta!.archived) { + const res = await bulkDeactivateProjects( + requestManager, + orgId, + projects, + config.dryRun, + ); + + res.updated.map((t) => ({ ...t, target })).forEach((i) => updated.add(i)); + res.failed.map((t) => ({ ...t, target })).forEach((i) => failed.add(i)); + + return { + updated: Array.from(updated).map((t) => ({ ...t, target })), + failed: Array.from(failed).map((t) => ({ ...t, target })), + }; } const deactivate = []; diff --git a/test/scripts/sync/sync-org-projects.test.ts b/test/scripts/sync/sync-org-projects.test.ts index aac932ca..d38555a6 100644 --- a/test/scripts/sync/sync-org-projects.test.ts +++ b/test/scripts/sync/sync-org-projects.test.ts @@ -236,7 +236,13 @@ describe('updateTargets', () => { ); updateProjectsSpy .mockImplementationOnce(() => - Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + branch: defaultBranch, + })), + }), ) .mockImplementationOnce(() => Promise.reject({ statusCode: '404', message: 'Error' }), @@ -348,7 +354,13 @@ describe('updateTargets', () => { ); listIntegrationsSpy.mockResolvedValue('abc-defg-0123'); updateProjectsSpy.mockImplementation(() => - Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + branch: defaultBranch, + })), + }), ); cloneSpy.mockImplementation(() => Promise.resolve({ @@ -458,7 +470,13 @@ describe('updateTargets', () => { ); listIntegrationsSpy.mockResolvedValue('abc-defg-0123'); updateProjectsSpy.mockImplementation(() => - Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + branch: defaultBranch, + })), + }), ); cloneSpy.mockImplementation(() => Promise.resolve({ @@ -559,7 +577,13 @@ describe('updateTargets', () => { }), ); updateProjectsSpy.mockImplementation(() => - Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + branch: defaultBranch, + })), + }), ); // Act @@ -672,7 +696,13 @@ describe('updateTargets', () => { ); updateProjectsSpy .mockImplementationOnce(() => - Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + branch: defaultBranch, + })), + }), ) .mockImplementationOnce(() => Promise.reject({ statusCode: '404', message: 'Error' }), @@ -788,7 +818,13 @@ describe('updateTargets', () => { }), ); updateProjectsSpy.mockImplementation(() => - Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + branch: defaultBranch, + })), + }), ); // Act const res = await updateTargets( @@ -940,6 +976,126 @@ describe('updateTargets', () => { }, }); }, 5000); + it('Github API errors during getGithubRepoMetaData call', async () => { + // Arrange + const testTargets = [ + { + attributes: { + displayName: 'snyk/monorepo', + isPrivate: false, + origin: 'github', + remoteUrl: null, + }, + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + relationships: { + org: { + data: { + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + type: 'org', + }, + links: {}, + meta: {}, + }, + }, + type: 'target', + }, + ]; + const orgId = 'af137b96-6966-46c1-826b-2e79ac49bbxx'; + const projectId = uuid.v4(); + + const projectsAPIResponse: ProjectsResponse = { + org: { + id: orgId, + }, + projects: [ + { + name: 'snyk/monorepo:build.gradle', + id: projectId, + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'npm', + branch: 'master', + status: 'active', + }, + { + name: 'snyk/monorepo(main):package.json', + id: projectId, + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'maven', + branch: 'master', + status: 'active', + }, + ], + }; + + listProjectsSpy.mockImplementation(() => + Promise.resolve(projectsAPIResponse), + ); + githubSpy.mockImplementation(() => + // TODO: confirm this is correct + Promise.reject({ + status: 404, + message: 'Not found', + }), + ); + updateProjectsSpy.mockImplementation(() => + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + status: 'deactivated', + })), + }), + ); + cloneSpy.mockImplementation(() => + Promise.resolve({ + success: true, + repoPath: path.resolve(fixturesFolderPath, 'monorepo'), + gitResponse: '', + }), + ); + // Act + const res = await updateTargets( + requestManager, + orgId, + testTargets, + integrationId, + undefined, + ); + + // Assert + expect(res).toStrictEqual({ + processedTargets: 1, + failedTargets: 0, + meta: { + projects: { + updated: [ + { + dryRun: false, + from: 'active', + projectPublicId: projectId, + target: testTargets[0], + to: 'deactivated', + type: ProjectUpdateType.DEACTIVATE, + }, + { + dryRun: false, + from: 'active', + projectPublicId: projectId, + target: testTargets[0], + to: 'deactivated', + type: ProjectUpdateType.DEACTIVATE, + }, + ], + failed: [], + }, + }, + }); + }); + // expect(importSingleTargetSpy).not.toHaveBeenCalled(); + // expect(listIntegrationsSpy).not.toHaveBeenCalled(); + // expect(cloneSpy).not.toHaveBeenCalled(); }); describe('Github Enterprise', () => { const integrationId = process.env.GHE_INTEGRATION_ID as string; @@ -1039,7 +1195,15 @@ describe('updateTargets', () => { ); updateProjectsSpy .mockImplementationOnce(() => - Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), + updateProjectsSpy.mockImplementation(() => + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + branch: defaultBranch, + })), + }), + ), ) .mockImplementationOnce(() => Promise.reject({ statusCode: '404', message: 'Error' }), From 35c5bfe543b0d2fc4a815e5e0828593c78c0d007 Mon Sep 17 00:00:00 2001 From: ghe Date: Wed, 11 Jan 2023 09:20:50 +0000 Subject: [PATCH 2/4] feat: sync support deleted repos --- docs/sync.md | 6 +++--- src/scripts/sync/sync-projects-per-target.ts | 2 +- test/scripts/sync/clone-and-analyze.spec.ts | 20 ++++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/sync.md b/docs/sync.md index 7fcefb20..cf8dc65e 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -12,7 +12,7 @@ - [File renamed/moves/deleted](#file-renamedmovesdeleted) - [node\_modules, tests \& fixtures](#node_modules-tests--fixtures) - [Detecting \& importing new files not already monitored in Snyk](#detecting--importing-new-files-not-already-monitored-in-snyk) - - [Repository is archived](#repository-is-archived) + - [Repository is archived or deleted](#repository-is-archived-or-deleted) - [Kick off sync](#kick-off-sync) - [1. Set the env vars](#1-set-the-env-vars) - [2. Download \& run](#2-download--run) @@ -77,9 +77,9 @@ Any projects that were imported but match the default exclusions list (deemed to While analyzing each target known to Snyk any new Snyk supported files found in the repo that do not have a corresponding project in Snyk will be imported in batches. Any files matching the default or user provided `exclusionGlobs` will be ignored. If a file has a corresponding de-activated project in Snyk, it will not be brought in again. Activate manually or via API if it should be active. -## Repository is archived +## Repository is archived or deleted -If the repository is now marked as archived, all relevant Snyk projects will be de-activated. +If the repository is now marked as archived or has been deleted (API returns 404), all relevant Snyk projects will be de-activated. # Kick off sync `sync` command will analyze existing projects & targets (repos) in Snyk organization and determine if any changes are needed. diff --git a/src/scripts/sync/sync-projects-per-target.ts b/src/scripts/sync/sync-projects-per-target.ts index 79a1546c..75470d0b 100644 --- a/src/scripts/sync/sync-projects-per-target.ts +++ b/src/scripts/sync/sync-projects-per-target.ts @@ -69,7 +69,7 @@ export async function syncProjectsForTarget( targetMeta = await getMetaDataGenerator(origin)(targetData, host); } catch (e) { //TODO: if repo is deleted, deactivate all projects - debug(e); + debug(`Failed to get metadata ${JSON.stringify(targetData)}: ` + e); if (e.status === 404) { // TODO: when else could you get a 404? Manually check isDeleted = true; diff --git a/test/scripts/sync/clone-and-analyze.spec.ts b/test/scripts/sync/clone-and-analyze.spec.ts index ea4a0861..d4682e68 100644 --- a/test/scripts/sync/clone-and-analyze.spec.ts +++ b/test/scripts/sync/clone-and-analyze.spec.ts @@ -67,8 +67,8 @@ describe('cloneAndAnalyze', () => { ]; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git', sshUrl: 'git@github.com:snyk-fixtures/monorepo-simple.git', }; @@ -130,8 +130,8 @@ describe('cloneAndAnalyze', () => { ]; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git', sshUrl: 'git@github.com:snyk-fixtures/monorepo-simple.git', }; @@ -205,8 +205,8 @@ describe('cloneAndAnalyze', () => { ]; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git', sshUrl: 'git@github.com:snyk-fixtures/monorepo-simple.git', }; @@ -298,8 +298,8 @@ describe('cloneAndAnalyze', () => { ]; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git', sshUrl: 'git@github.com:snyk-fixtures/monorepo-simple.git', }; @@ -324,8 +324,8 @@ describe('cloneAndAnalyze', () => { const projects: SnykProject[] = []; const repoMeta: RepoMetaData = { - branch: 'main', archived: false, + branch: 'main', cloneUrl: 'https://github.com/snyk-fixtures/no-supported-manifests.git', sshUrl: 'git@github.com:snyk-fixtures/no-supported-manifests.git', }; @@ -350,8 +350,8 @@ describe('cloneAndAnalyze', () => { const projects: SnykProject[] = []; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: 'https://github.com/snyk-fixtures/empty-repo.git', sshUrl: 'git@github.com:snyk-fixtures/empty-repo.git', }; @@ -376,8 +376,8 @@ describe('cloneAndAnalyze', () => { const projects: SnykProject[] = []; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: 'https://github.com/snyk-fixtures/python-requirements-custom-name-inside-folder.git', sshUrl: @@ -414,8 +414,8 @@ describe('cloneAndAnalyze', () => { const projects: SnykProject[] = []; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: `https://${GHE_URL.host}/snyk-fixtures/mono-repo.git`, sshUrl: `git@${GHE_URL.host}/snyk-fixtures/mono-repo.git`, }; @@ -451,8 +451,8 @@ describe('cloneAndAnalyze', () => { const projects: SnykProject[] = []; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: `https://${GHE_URL.host}/snyk-fixtures/docker-goof.git`, sshUrl: `git@${GHE_URL.host}/snyk-fixtures/docker-goof.git`, }; @@ -481,8 +481,8 @@ describe('cloneAndAnalyze', () => { const projects: SnykProject[] = []; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: `https://${GHE_URL.host}/snyk-fixtures/docker-goof.git`, sshUrl: `git@${GHE_URL.host}/snyk-fixtures/docker-goof.git`, }; From a17b4e9b046507c8ab4831f8019d0237b08aacd8 Mon Sep 17 00:00:00 2001 From: ghe Date: Mon, 23 Jan 2023 08:37:53 +0000 Subject: [PATCH 3/4] wip --- docs/sync.md | 4 ++-- test/scripts/sync/sync-org-projects.test.ts | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/sync.md b/docs/sync.md index cf8dc65e..563804b2 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -12,7 +12,7 @@ - [File renamed/moves/deleted](#file-renamedmovesdeleted) - [node\_modules, tests \& fixtures](#node_modules-tests--fixtures) - [Detecting \& importing new files not already monitored in Snyk](#detecting--importing-new-files-not-already-monitored-in-snyk) - - [Repository is archived or deleted](#repository-is-archived-or-deleted) + - [Repository is archived / deleted](#repository-is-archived--deleted) - [Kick off sync](#kick-off-sync) - [1. Set the env vars](#1-set-the-env-vars) - [2. Download \& run](#2-download--run) @@ -77,7 +77,7 @@ Any projects that were imported but match the default exclusions list (deemed to While analyzing each target known to Snyk any new Snyk supported files found in the repo that do not have a corresponding project in Snyk will be imported in batches. Any files matching the default or user provided `exclusionGlobs` will be ignored. If a file has a corresponding de-activated project in Snyk, it will not be brought in again. Activate manually or via API if it should be active. -## Repository is archived or deleted +## Repository is archived / deleted If the repository is now marked as archived or has been deleted (API returns 404), all relevant Snyk projects will be de-activated. # Kick off sync diff --git a/test/scripts/sync/sync-org-projects.test.ts b/test/scripts/sync/sync-org-projects.test.ts index d38555a6..07af09f5 100644 --- a/test/scripts/sync/sync-org-projects.test.ts +++ b/test/scripts/sync/sync-org-projects.test.ts @@ -1093,9 +1093,6 @@ describe('updateTargets', () => { }, }); }); - // expect(importSingleTargetSpy).not.toHaveBeenCalled(); - // expect(listIntegrationsSpy).not.toHaveBeenCalled(); - // expect(cloneSpy).not.toHaveBeenCalled(); }); describe('Github Enterprise', () => { const integrationId = process.env.GHE_INTEGRATION_ID as string; From 791567b2127a3bbbef9648b84faf3f66f5267a53 Mon Sep 17 00:00:00 2001 From: ghe Date: Mon, 23 Jan 2023 20:45:50 +0000 Subject: [PATCH 4/4] feat: validate token access when repo is 404 --- src/lib/source-handlers/github/index.ts | 1 + .../source-handlers/github/validate-token.ts | 28 +++++++++++++++++++ src/scripts/sync/sync-projects-per-target.ts | 23 ++++++++++++--- 3 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 src/lib/source-handlers/github/validate-token.ts diff --git a/src/lib/source-handlers/github/index.ts b/src/lib/source-handlers/github/index.ts index 09f12a54..ff85ac17 100644 --- a/src/lib/source-handlers/github/index.ts +++ b/src/lib/source-handlers/github/index.ts @@ -5,3 +5,4 @@ export * from './get-repo-metadata'; export * from './types'; export * from './is-configured'; export * from './git-clone-url'; +export * from './validate-token'; diff --git a/src/lib/source-handlers/github/validate-token.ts b/src/lib/source-handlers/github/validate-token.ts new file mode 100644 index 00000000..a4bd0654 --- /dev/null +++ b/src/lib/source-handlers/github/validate-token.ts @@ -0,0 +1,28 @@ +import { Octokit } from '@octokit/rest'; +import { retry } from '@octokit/plugin-retry'; +import * as debugLib from 'debug'; +import type { Target } from '../../types'; +import { getGithubToken } from './get-github-token'; +import { getGithubBaseUrl } from './github-base-url'; + +const githubClient = Octokit.plugin(retry); +const debug = debugLib('snyk:get-github-defaultBranch-script'); + +export async function validateToken( + target: Target, + host?: string, +): Promise<{ valid: true }> { + const githubToken = getGithubToken(); + const baseUrl = getGithubBaseUrl(host); + const octokit: Octokit = new githubClient({ + baseUrl, + auth: githubToken, + }); + + debug(`Fetching organization info to validate token access: ${target.owner}`); + + await octokit.orgs.get({ + org: target.owner!, + }); + return { valid: true }; +} diff --git a/src/scripts/sync/sync-projects-per-target.ts b/src/scripts/sync/sync-projects-per-target.ts index 75470d0b..13d8bd9a 100644 --- a/src/scripts/sync/sync-projects-per-target.ts +++ b/src/scripts/sync/sync-projects-per-target.ts @@ -1,7 +1,10 @@ import type { requestsManager } from 'snyk-request-manager'; import * as debugLib from 'debug'; -import { getGithubRepoMetaData } from '../../lib/source-handlers/github'; +import { + getGithubRepoMetaData, + validateToken, +} from '../../lib/source-handlers/github'; import { updateBranch } from '../../lib/project/update-branch'; import type { SnykProject, @@ -28,6 +31,16 @@ export function getMetaDataGenerator( return getDefaultBranchGenerators[origin]; } +export function getTokenValidator( + origin: SupportedIntegrationTypesUpdateProject, +): (target: Target, host?: string | undefined) => Promise<{ valid: true }> { + const generator = { + [SupportedIntegrationTypesUpdateProject.GITHUB]: validateToken, + [SupportedIntegrationTypesUpdateProject.GHE]: validateToken, + }; + return generator[origin]; +} + export function getTargetConverter( origin: SupportedIntegrationTypesUpdateProject, ): (target: SnykTarget) => Target { @@ -64,14 +77,16 @@ export async function syncProjectsForTarget( const origin = target.attributes .origin as SupportedIntegrationTypesUpdateProject; const targetData = getTargetConverter(origin)(target); + const isTokenValid = await getTokenValidator(origin); try { targetMeta = await getMetaDataGenerator(origin)(targetData, host); } catch (e) { - //TODO: if repo is deleted, deactivate all projects debug(`Failed to get metadata ${JSON.stringify(targetData)}: ` + e); - if (e.status === 404) { - // TODO: when else could you get a 404? Manually check + if ( + e.status === 404 && + (await isTokenValid(targetData, host)).valid === true + ) { isDeleted = true; } else { const error = `Getting metadata from ${origin} API failed with error: ${e.message}`;