diff --git a/docs/sync.md b/docs/sync.md index 7fcefb20..563804b2 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -12,7 +12,7 @@ - [File renamed/moves/deleted](#file-renamedmovesdeleted) - [node\_modules, tests \& fixtures](#node_modules-tests--fixtures) - [Detecting \& importing new files not already monitored in Snyk](#detecting--importing-new-files-not-already-monitored-in-snyk) - - [Repository is archived](#repository-is-archived) + - [Repository is archived / deleted](#repository-is-archived--deleted) - [Kick off sync](#kick-off-sync) - [1. Set the env vars](#1-set-the-env-vars) - [2. Download \& run](#2-download--run) @@ -77,9 +77,9 @@ Any projects that were imported but match the default exclusions list (deemed to While analyzing each target known to Snyk any new Snyk supported files found in the repo that do not have a corresponding project in Snyk will be imported in batches. Any files matching the default or user provided `exclusionGlobs` will be ignored. If a file has a corresponding de-activated project in Snyk, it will not be brought in again. Activate manually or via API if it should be active. -## Repository is archived +## Repository is archived / deleted -If the repository is now marked as archived, all relevant Snyk projects will be de-activated. +If the repository is now marked as archived or has been deleted (API returns 404), all relevant Snyk projects will be de-activated. # Kick off sync `sync` command will analyze existing projects & targets (repos) in Snyk organization and determine if any changes are needed. diff --git a/src/lib/source-handlers/github/index.ts b/src/lib/source-handlers/github/index.ts index 09f12a54..ff85ac17 100644 --- a/src/lib/source-handlers/github/index.ts +++ b/src/lib/source-handlers/github/index.ts @@ -5,3 +5,4 @@ export * from './get-repo-metadata'; export * from './types'; export * from './is-configured'; export * from './git-clone-url'; +export * from './validate-token'; diff --git a/src/lib/source-handlers/github/validate-token.ts b/src/lib/source-handlers/github/validate-token.ts new file mode 100644 index 00000000..a4bd0654 --- /dev/null +++ b/src/lib/source-handlers/github/validate-token.ts @@ -0,0 +1,28 @@ +import { Octokit } from '@octokit/rest'; +import { retry } from '@octokit/plugin-retry'; +import * as debugLib from 'debug'; +import type { Target } from '../../types'; +import { getGithubToken } from './get-github-token'; +import { getGithubBaseUrl } from './github-base-url'; + +const githubClient = Octokit.plugin(retry); +const debug = debugLib('snyk:get-github-defaultBranch-script'); + +export async function validateToken( + target: Target, + host?: string, +): Promise<{ valid: true }> { + const githubToken = getGithubToken(); + const baseUrl = getGithubBaseUrl(host); + const octokit: Octokit = new githubClient({ + baseUrl, + auth: githubToken, + }); + + debug(`Fetching organization info to validate token access: ${target.owner}`); + + await octokit.orgs.get({ + org: target.owner!, + }); + return { valid: true }; +} diff --git a/src/scripts/sync/sync-projects-per-target.ts b/src/scripts/sync/sync-projects-per-target.ts index 9f073e7e..13d8bd9a 100644 --- a/src/scripts/sync/sync-projects-per-target.ts +++ b/src/scripts/sync/sync-projects-per-target.ts @@ -1,7 +1,10 @@ import type { requestsManager } from 'snyk-request-manager'; import * as debugLib from 'debug'; -import { getGithubRepoMetaData } from '../../lib/source-handlers/github'; +import { + getGithubRepoMetaData, + validateToken, +} from '../../lib/source-handlers/github'; import { updateBranch } from '../../lib/project/update-branch'; import type { SnykProject, @@ -28,6 +31,16 @@ export function getMetaDataGenerator( return getDefaultBranchGenerators[origin]; } +export function getTokenValidator( + origin: SupportedIntegrationTypesUpdateProject, +): (target: Target, host?: string | undefined) => Promise<{ valid: true }> { + const generator = { + [SupportedIntegrationTypesUpdateProject.GITHUB]: validateToken, + [SupportedIntegrationTypesUpdateProject.GHE]: validateToken, + }; + return generator[origin]; +} + export function getTargetConverter( origin: SupportedIntegrationTypesUpdateProject, ): (target: SnykTarget) => Target { @@ -60,27 +73,56 @@ export async function syncProjectsForTarget( debug(`Syncing projects for target ${target.attributes.displayName}`); let targetMeta: RepoMetaData; + let isDeleted = false; const origin = target.attributes .origin as SupportedIntegrationTypesUpdateProject; const targetData = getTargetConverter(origin)(target); + const isTokenValid = await getTokenValidator(origin); try { targetMeta = await getMetaDataGenerator(origin)(targetData, host); } catch (e) { - //TODO: if repo is deleted, deactivate all projects - debug(e); - const error = `Getting default branch via ${origin} API failed with error: ${e.message}`; - projects.map((project) => { - failed.add({ - errorMessage: error, - projectPublicId: project.id, - type: ProjectUpdateType.BRANCH, - from: project.branch!, - to: targetMeta.branch, - dryRun: config.dryRun, - target, + debug(`Failed to get metadata ${JSON.stringify(targetData)}: ` + e); + if ( + e.status === 404 && + (await isTokenValid(targetData, host)).valid === true + ) { + isDeleted = true; + } else { + const error = `Getting metadata from ${origin} API failed with error: ${e.message}`; + projects.map((project) => { + failed.add({ + errorMessage: error, + projectPublicId: project.id, + type: ProjectUpdateType.BRANCH, + from: project.branch!, + to: 'unknown', + dryRun: config.dryRun, + target, + }); }); - }); + return { + updated: Array.from(updated).map((t) => ({ ...t, target })), + failed: Array.from(failed).map((t) => ({ ...t, target })), + }; + } + } + + if (isDeleted || targetMeta!.archived) { + const res = await bulkDeactivateProjects( + requestManager, + orgId, + projects, + config.dryRun, + ); + + res.updated.map((t) => ({ ...t, target })).forEach((i) => updated.add(i)); + res.failed.map((t) => ({ ...t, target })).forEach((i) => failed.add(i)); + + return { + updated: Array.from(updated).map((t) => ({ ...t, target })), + failed: Array.from(failed).map((t) => ({ ...t, target })), + }; } const deactivate = []; diff --git a/test/scripts/sync/clone-and-analyze.spec.ts b/test/scripts/sync/clone-and-analyze.spec.ts index ea4a0861..d4682e68 100644 --- a/test/scripts/sync/clone-and-analyze.spec.ts +++ b/test/scripts/sync/clone-and-analyze.spec.ts @@ -67,8 +67,8 @@ describe('cloneAndAnalyze', () => { ]; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git', sshUrl: 'git@github.com:snyk-fixtures/monorepo-simple.git', }; @@ -130,8 +130,8 @@ describe('cloneAndAnalyze', () => { ]; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git', sshUrl: 'git@github.com:snyk-fixtures/monorepo-simple.git', }; @@ -205,8 +205,8 @@ describe('cloneAndAnalyze', () => { ]; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git', sshUrl: 'git@github.com:snyk-fixtures/monorepo-simple.git', }; @@ -298,8 +298,8 @@ describe('cloneAndAnalyze', () => { ]; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git', sshUrl: 'git@github.com:snyk-fixtures/monorepo-simple.git', }; @@ -324,8 +324,8 @@ describe('cloneAndAnalyze', () => { const projects: SnykProject[] = []; const repoMeta: RepoMetaData = { - branch: 'main', archived: false, + branch: 'main', cloneUrl: 'https://github.com/snyk-fixtures/no-supported-manifests.git', sshUrl: 'git@github.com:snyk-fixtures/no-supported-manifests.git', }; @@ -350,8 +350,8 @@ describe('cloneAndAnalyze', () => { const projects: SnykProject[] = []; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: 'https://github.com/snyk-fixtures/empty-repo.git', sshUrl: 'git@github.com:snyk-fixtures/empty-repo.git', }; @@ -376,8 +376,8 @@ describe('cloneAndAnalyze', () => { const projects: SnykProject[] = []; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: 'https://github.com/snyk-fixtures/python-requirements-custom-name-inside-folder.git', sshUrl: @@ -414,8 +414,8 @@ describe('cloneAndAnalyze', () => { const projects: SnykProject[] = []; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: `https://${GHE_URL.host}/snyk-fixtures/mono-repo.git`, sshUrl: `git@${GHE_URL.host}/snyk-fixtures/mono-repo.git`, }; @@ -451,8 +451,8 @@ describe('cloneAndAnalyze', () => { const projects: SnykProject[] = []; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: `https://${GHE_URL.host}/snyk-fixtures/docker-goof.git`, sshUrl: `git@${GHE_URL.host}/snyk-fixtures/docker-goof.git`, }; @@ -481,8 +481,8 @@ describe('cloneAndAnalyze', () => { const projects: SnykProject[] = []; const repoMeta: RepoMetaData = { - branch: 'master', archived: false, + branch: 'master', cloneUrl: `https://${GHE_URL.host}/snyk-fixtures/docker-goof.git`, sshUrl: `git@${GHE_URL.host}/snyk-fixtures/docker-goof.git`, }; diff --git a/test/scripts/sync/sync-org-projects.test.ts b/test/scripts/sync/sync-org-projects.test.ts index aac932ca..07af09f5 100644 --- a/test/scripts/sync/sync-org-projects.test.ts +++ b/test/scripts/sync/sync-org-projects.test.ts @@ -236,7 +236,13 @@ describe('updateTargets', () => { ); updateProjectsSpy .mockImplementationOnce(() => - Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + branch: defaultBranch, + })), + }), ) .mockImplementationOnce(() => Promise.reject({ statusCode: '404', message: 'Error' }), @@ -348,7 +354,13 @@ describe('updateTargets', () => { ); listIntegrationsSpy.mockResolvedValue('abc-defg-0123'); updateProjectsSpy.mockImplementation(() => - Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + branch: defaultBranch, + })), + }), ); cloneSpy.mockImplementation(() => Promise.resolve({ @@ -458,7 +470,13 @@ describe('updateTargets', () => { ); listIntegrationsSpy.mockResolvedValue('abc-defg-0123'); updateProjectsSpy.mockImplementation(() => - Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + branch: defaultBranch, + })), + }), ); cloneSpy.mockImplementation(() => Promise.resolve({ @@ -559,7 +577,13 @@ describe('updateTargets', () => { }), ); updateProjectsSpy.mockImplementation(() => - Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + branch: defaultBranch, + })), + }), ); // Act @@ -672,7 +696,13 @@ describe('updateTargets', () => { ); updateProjectsSpy .mockImplementationOnce(() => - Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + branch: defaultBranch, + })), + }), ) .mockImplementationOnce(() => Promise.reject({ statusCode: '404', message: 'Error' }), @@ -788,7 +818,13 @@ describe('updateTargets', () => { }), ); updateProjectsSpy.mockImplementation(() => - Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + branch: defaultBranch, + })), + }), ); // Act const res = await updateTargets( @@ -940,6 +976,123 @@ describe('updateTargets', () => { }, }); }, 5000); + it('Github API errors during getGithubRepoMetaData call', async () => { + // Arrange + const testTargets = [ + { + attributes: { + displayName: 'snyk/monorepo', + isPrivate: false, + origin: 'github', + remoteUrl: null, + }, + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + relationships: { + org: { + data: { + id: 'af137b96-6966-46c1-826b-2e79ac49bbxx', + type: 'org', + }, + links: {}, + meta: {}, + }, + }, + type: 'target', + }, + ]; + const orgId = 'af137b96-6966-46c1-826b-2e79ac49bbxx'; + const projectId = uuid.v4(); + + const projectsAPIResponse: ProjectsResponse = { + org: { + id: orgId, + }, + projects: [ + { + name: 'snyk/monorepo:build.gradle', + id: projectId, + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'npm', + branch: 'master', + status: 'active', + }, + { + name: 'snyk/monorepo(main):package.json', + id: projectId, + created: '2018-10-29T09:50:54.014Z', + origin: 'github', + type: 'maven', + branch: 'master', + status: 'active', + }, + ], + }; + + listProjectsSpy.mockImplementation(() => + Promise.resolve(projectsAPIResponse), + ); + githubSpy.mockImplementation(() => + // TODO: confirm this is correct + Promise.reject({ + status: 404, + message: 'Not found', + }), + ); + updateProjectsSpy.mockImplementation(() => + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + status: 'deactivated', + })), + }), + ); + cloneSpy.mockImplementation(() => + Promise.resolve({ + success: true, + repoPath: path.resolve(fixturesFolderPath, 'monorepo'), + gitResponse: '', + }), + ); + // Act + const res = await updateTargets( + requestManager, + orgId, + testTargets, + integrationId, + undefined, + ); + + // Assert + expect(res).toStrictEqual({ + processedTargets: 1, + failedTargets: 0, + meta: { + projects: { + updated: [ + { + dryRun: false, + from: 'active', + projectPublicId: projectId, + target: testTargets[0], + to: 'deactivated', + type: ProjectUpdateType.DEACTIVATE, + }, + { + dryRun: false, + from: 'active', + projectPublicId: projectId, + target: testTargets[0], + to: 'deactivated', + type: ProjectUpdateType.DEACTIVATE, + }, + ], + failed: [], + }, + }, + }); + }); }); describe('Github Enterprise', () => { const integrationId = process.env.GHE_INTEGRATION_ID as string; @@ -1039,7 +1192,15 @@ describe('updateTargets', () => { ); updateProjectsSpy .mockImplementationOnce(() => - Promise.resolve({ ...projectsAPIResponse, branch: defaultBranch }), + updateProjectsSpy.mockImplementation(() => + Promise.resolve({ + ...projectsAPIResponse, + projects: projectsAPIResponse.projects.map((p) => ({ + ...p, + branch: defaultBranch, + })), + }), + ), ) .mockImplementationOnce(() => Promise.reject({ statusCode: '404', message: 'Error' }),