From 32a3e1d200720e2da0ff2270bf6d2f1e372f0458 Mon Sep 17 00:00:00 2001 From: carlosmonastyrski Date: Tue, 6 May 2025 08:11:50 -0300 Subject: [PATCH 1/4] commit --- backend/src/@types/knex.d.ts | 48 +++++++ .../20250505194916_add-pit-revamp-tables.ts | 130 +++++++++++++++++ .../db/schemas/folder-checkpoint-resources.ts | 23 +++ backend/src/db/schemas/folder-checkpoints.ts | 20 +++ .../src/db/schemas/folder-commit-changes.ts | 22 +++ backend/src/db/schemas/folder-commits.ts | 23 +++ .../folder-tree-checkpoint-resources.ts | 26 ++++ .../src/db/schemas/folder-tree-checkpoints.ts | 20 +++ backend/src/db/schemas/index.ts | 6 + backend/src/db/schemas/models.ts | 8 +- .../src/ee/services/license/license-fns.ts | 54 ++++---- .../folder-checkpoint-resources-dal.ts | 79 +++++++++++ .../folder-checkpoint-dal.ts | 86 ++++++++++++ .../folder-commit-changes-dal.ts | 80 +++++++++++ .../folder-commit/folder-commit-dal.ts | 56 ++++++++ .../folder-tree-checkpoint-resources-dal.ts | 131 ++++++++++++++++++ .../folder-tree-checkpoint-dal.ts | 111 +++++++++++++++ 17 files changed, 895 insertions(+), 28 deletions(-) create mode 100644 backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts create mode 100644 backend/src/db/schemas/folder-checkpoint-resources.ts create mode 100644 backend/src/db/schemas/folder-checkpoints.ts create mode 100644 backend/src/db/schemas/folder-commit-changes.ts create mode 100644 backend/src/db/schemas/folder-commits.ts create mode 100644 backend/src/db/schemas/folder-tree-checkpoint-resources.ts create mode 100644 backend/src/db/schemas/folder-tree-checkpoints.ts create mode 100644 backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts create mode 100644 backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts create mode 100644 backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts create mode 100644 backend/src/services/folder-commit/folder-commit-dal.ts create mode 100644 backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts create mode 100644 backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 13f3bc306a..928fddc360 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -74,6 +74,24 @@ import { TExternalKms, TExternalKmsInsert, TExternalKmsUpdate, + TFolderCheckpointResources, + TFolderCheckpointResourcesInsert, + TFolderCheckpointResourcesUpdate, + TFolderCheckpoints, + TFolderCheckpointsInsert, + TFolderCheckpointsUpdate, + TFolderCommitChanges, + TFolderCommitChangesInsert, + TFolderCommitChangesUpdate, + TFolderCommits, + TFolderCommitsInsert, + TFolderCommitsUpdate, + TFolderTreeCheckpointResources, + TFolderTreeCheckpointResourcesInsert, + TFolderTreeCheckpointResourcesUpdate, + TFolderTreeCheckpoints, + TFolderTreeCheckpointsInsert, + TFolderTreeCheckpointsUpdate, TGateways, TGatewaysInsert, TGatewaysUpdate, @@ -1048,5 +1066,35 @@ declare module "knex/types/tables" { TGithubOrgSyncConfigsInsert, TGithubOrgSyncConfigsUpdate >; + [TableName.FolderCommit]: KnexOriginal.CompositeTableType< + TFolderCommits, + TFolderCommitsInsert, + TFolderCommitsUpdate + >; + [TableName.FolderCommitChanges]: KnexOriginal.CompositeTableType< + TFolderCommitChanges, + TFolderCommitChangesInsert, + TFolderCommitChangesUpdate + >; + [TableName.FolderCheckpoint]: KnexOriginal.CompositeTableType< + TFolderCheckpoints, + TFolderCheckpointsInsert, + TFolderCheckpointsUpdate + >; + [TableName.FolderCheckpointResources]: KnexOriginal.CompositeTableType< + TFolderCheckpointResources, + TFolderCheckpointResourcesInsert, + TFolderCheckpointResourcesUpdate + >; + [TableName.FolderTreeCheckpoint]: KnexOriginal.CompositeTableType< + TFolderTreeCheckpoints, + TFolderTreeCheckpointsInsert, + TFolderTreeCheckpointsUpdate + >; + [TableName.FolderTreeCheckpointResources]: KnexOriginal.CompositeTableType< + TFolderTreeCheckpointResources, + TFolderTreeCheckpointResourcesInsert, + TFolderTreeCheckpointResourcesUpdate + >; } } diff --git a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts new file mode 100644 index 0000000000..f86c800a13 --- /dev/null +++ b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts @@ -0,0 +1,130 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; +import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils"; + +export async function up(knex: Knex): Promise { + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + if (!hasFolderCommitTable) { + await knex.schema.createTable(TableName.FolderCommit, (t) => { + t.bigIncrements("id").primary(); + t.string("actorName").notNullable(); + t.string("actorType").notNullable(); + t.string("message"); + t.timestamp("date").notNullable().defaultTo(knex.fn.now()); + t.uuid("folderId").notNullable(); + t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderId"); + }); + } + + const hasFolderCommitChangesTable = await knex.schema.hasTable(TableName.FolderCommitChanges); + if (!hasFolderCommitChangesTable) { + await knex.schema.createTable(TableName.FolderCommitChanges, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.bigInteger("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.string("changeType").notNullable(); + t.uuid("secretVersionId"); + t.foreign("secretVersionId").references("id").inTable(TableName.SecretVersionV2).onDelete("CASCADE"); + t.uuid("folderVersionId"); + t.foreign("folderVersionId").references("id").inTable(TableName.SecretFolderVersion).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCommitId"); + t.index("secretVersionId"); + t.index("folderVersionId"); + }); + } + + const hasFolderCheckpointTable = await knex.schema.hasTable(TableName.FolderCheckpoint); + if (!hasFolderCheckpointTable) { + await knex.schema.createTable(TableName.FolderCheckpoint, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.bigInteger("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.timestamp("date").notNullable().defaultTo(knex.fn.now()); + t.timestamps(true, true, true); + + t.index("folderCommitId"); + }); + } + + const hasFolderCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderCheckpointResources); + if (!hasFolderCheckpointResourcesTable) { + await knex.schema.createTable(TableName.FolderCheckpointResources, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderCheckpointId").notNullable(); + t.foreign("folderCheckpointId").references("id").inTable(TableName.FolderCheckpoint).onDelete("CASCADE"); + t.uuid("secretVersionId"); + t.foreign("secretVersionId").references("id").inTable(TableName.SecretVersionV2).onDelete("CASCADE"); + t.uuid("folderVersionId"); + t.foreign("folderVersionId").references("id").inTable(TableName.SecretFolderVersion).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCheckpointId"); + t.index("secretVersionId"); + t.index("folderVersionId"); + }); + } + + const hasFolderTreeCheckpointTable = await knex.schema.hasTable(TableName.FolderTreeCheckpoint); + if (!hasFolderTreeCheckpointTable) { + await knex.schema.createTable(TableName.FolderTreeCheckpoint, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.bigInteger("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.timestamp("date").notNullable().defaultTo(knex.fn.now()); + t.timestamps(true, true, true); + + t.index("folderCommitId"); + }); + } + + const hasFolderTreeCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderTreeCheckpointResources); + if (!hasFolderTreeCheckpointResourcesTable) { + await knex.schema.createTable(TableName.FolderTreeCheckpointResources, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderTreeCheckpointId").notNullable(); + t.foreign("folderTreeCheckpointId").references("id").inTable(TableName.FolderTreeCheckpoint).onDelete("CASCADE"); + t.uuid("folderId").notNullable(); + t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("CASCADE"); + t.bigInteger("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderTreeCheckpointId"); + t.index("folderId"); + t.index("folderCommitId"); + }); + } + + await createOnUpdateTrigger(knex, TableName.FolderCommit); + await createOnUpdateTrigger(knex, TableName.FolderCommitChanges); + await createOnUpdateTrigger(knex, TableName.FolderCheckpoint); + await createOnUpdateTrigger(knex, TableName.FolderCheckpointResources); + await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); + await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); +} + +export async function down(knex: Knex): Promise { + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpointResources); + + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpoint); + + await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources); + + await dropOnUpdateTrigger(knex, TableName.FolderCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderCheckpoint); + + await dropOnUpdateTrigger(knex, TableName.FolderCommitChanges); + await knex.schema.dropTableIfExists(TableName.FolderCommitChanges); + + await dropOnUpdateTrigger(knex, TableName.FolderCommit); + await knex.schema.dropTableIfExists(TableName.FolderCommit); +} diff --git a/backend/src/db/schemas/folder-checkpoint-resources.ts b/backend/src/db/schemas/folder-checkpoint-resources.ts new file mode 100644 index 0000000000..5fa8215cf3 --- /dev/null +++ b/backend/src/db/schemas/folder-checkpoint-resources.ts @@ -0,0 +1,23 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCheckpointResourcesSchema = z.object({ + id: z.string().uuid(), + folderCheckpointId: z.string().uuid(), + secretVersionId: z.string().uuid().nullable().optional(), + folderVersionId: z.string().uuid().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCheckpointResources = z.infer; +export type TFolderCheckpointResourcesInsert = Omit, TImmutableDBKeys>; +export type TFolderCheckpointResourcesUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/folder-checkpoints.ts b/backend/src/db/schemas/folder-checkpoints.ts new file mode 100644 index 0000000000..fd7ccae840 --- /dev/null +++ b/backend/src/db/schemas/folder-checkpoints.ts @@ -0,0 +1,20 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCheckpointsSchema = z.object({ + id: z.string().uuid(), + folderCommitId: z.coerce.number(), + date: z.date(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCheckpoints = z.infer; +export type TFolderCheckpointsInsert = Omit, TImmutableDBKeys>; +export type TFolderCheckpointsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/folder-commit-changes.ts b/backend/src/db/schemas/folder-commit-changes.ts new file mode 100644 index 0000000000..9ee2285952 --- /dev/null +++ b/backend/src/db/schemas/folder-commit-changes.ts @@ -0,0 +1,22 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCommitChangesSchema = z.object({ + id: z.string().uuid(), + folderCommitId: z.coerce.number(), + changeType: z.string(), + secretVersionId: z.string().uuid().nullable().optional(), + folderVersionId: z.string().uuid().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCommitChanges = z.infer; +export type TFolderCommitChangesInsert = Omit, TImmutableDBKeys>; +export type TFolderCommitChangesUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/folder-commits.ts b/backend/src/db/schemas/folder-commits.ts new file mode 100644 index 0000000000..4d5f2d2af8 --- /dev/null +++ b/backend/src/db/schemas/folder-commits.ts @@ -0,0 +1,23 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCommitsSchema = z.object({ + id: z.coerce.number(), + actorName: z.string(), + actorType: z.string(), + message: z.string().nullable().optional(), + date: z.date(), + folderId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCommits = z.infer; +export type TFolderCommitsInsert = Omit, TImmutableDBKeys>; +export type TFolderCommitsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/folder-tree-checkpoint-resources.ts b/backend/src/db/schemas/folder-tree-checkpoint-resources.ts new file mode 100644 index 0000000000..1c65e833d0 --- /dev/null +++ b/backend/src/db/schemas/folder-tree-checkpoint-resources.ts @@ -0,0 +1,26 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderTreeCheckpointResourcesSchema = z.object({ + id: z.string().uuid(), + folderTreeCheckpointId: z.string().uuid(), + folderId: z.string().uuid(), + folderCommitId: z.coerce.number(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderTreeCheckpointResources = z.infer; +export type TFolderTreeCheckpointResourcesInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TFolderTreeCheckpointResourcesUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/folder-tree-checkpoints.ts b/backend/src/db/schemas/folder-tree-checkpoints.ts new file mode 100644 index 0000000000..8f637b49ba --- /dev/null +++ b/backend/src/db/schemas/folder-tree-checkpoints.ts @@ -0,0 +1,20 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderTreeCheckpointsSchema = z.object({ + id: z.string().uuid(), + folderCommitId: z.coerce.number(), + date: z.date(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderTreeCheckpoints = z.infer; +export type TFolderTreeCheckpointsInsert = Omit, TImmutableDBKeys>; +export type TFolderTreeCheckpointsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/index.ts b/backend/src/db/schemas/index.ts index b71d519083..5755697cc5 100644 --- a/backend/src/db/schemas/index.ts +++ b/backend/src/db/schemas/index.ts @@ -22,6 +22,12 @@ export * from "./dynamic-secret-leases"; export * from "./dynamic-secrets"; export * from "./external-group-org-role-mappings"; export * from "./external-kms"; +export * from "./folder-checkpoint-resources"; +export * from "./folder-checkpoints"; +export * from "./folder-commit-changes"; +export * from "./folder-commits"; +export * from "./folder-tree-checkpoint-resources"; +export * from "./folder-tree-checkpoints"; export * from "./gateways"; export * from "./git-app-install-sessions"; export * from "./git-app-org"; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 7fd77da6c5..0b80be80f9 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -152,7 +152,13 @@ export enum TableName { MicrosoftTeamsIntegrations = "microsoft_teams_integrations", ProjectMicrosoftTeamsConfigs = "project_microsoft_teams_configs", SecretReminderRecipients = "secret_reminder_recipients", - GithubOrgSyncConfig = "github_org_sync_configs" + GithubOrgSyncConfig = "github_org_sync_configs", + FolderCommit = "folder_commits", + FolderCommitChanges = "folder_commit_changes", + FolderCheckpoint = "folder_checkpoints", + FolderCheckpointResources = "folder_checkpoint_resources", + FolderTreeCheckpoint = "folder_tree_checkpoints", + FolderTreeCheckpointResources = "folder_tree_checkpoint_resources" } export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt"; diff --git a/backend/src/ee/services/license/license-fns.ts b/backend/src/ee/services/license/license-fns.ts index b7ae6f7ee2..cff6ee1810 100644 --- a/backend/src/ee/services/license/license-fns.ts +++ b/backend/src/ee/services/license/license-fns.ts @@ -17,44 +17,44 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({ environmentsUsed: 0, identityLimit: null, identitiesUsed: 0, - dynamicSecret: false, + dynamicSecret: true, secretVersioning: true, - pitRecovery: false, - ipAllowlisting: false, - rbac: false, - githubOrgSync: false, - customRateLimits: false, - customAlerts: false, - secretAccessInsights: false, - auditLogs: false, + pitRecovery: true, + ipAllowlisting: true, + rbac: true, + githubOrgSync: true, + customRateLimits: true, + customAlerts: true, + secretAccessInsights: true, + auditLogs: true, auditLogsRetentionDays: 0, - auditLogStreams: false, + auditLogStreams: true, auditLogStreamLimit: 3, - samlSSO: false, - hsm: false, - oidcSSO: false, - scim: false, - ldap: false, - groups: false, + samlSSO: true, + hsm: true, + oidcSSO: true, + scim: true, + ldap: true, + groups: true, status: null, trial_end: null, has_used_trial: true, - secretApproval: false, - secretRotation: false, - caCrl: false, - instanceUserManagement: false, - externalKms: false, + secretApproval: true, + secretRotation: true, + caCrl: true, + instanceUserManagement: true, + externalKms: true, rateLimits: { readLimit: 60, writeLimit: 200, secretsLimit: 40 }, - pkiEst: false, - enforceMfa: false, - projectTemplates: false, - kmip: false, - gateway: false, - sshHostGroups: false + pkiEst: true, + enforceMfa: true, + projectTemplates: true, + kmip: true, + gateway: true, + sshHostGroups: true }); export const setupLicenseRequestWithStore = (baseURL: string, refreshUrl: string, licenseKey: string) => { diff --git a/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts new file mode 100644 index 0000000000..d0b2200ecf --- /dev/null +++ b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts @@ -0,0 +1,79 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCheckpointResources, TFolderCheckpoints } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCheckpointResourcesDALFactory = ReturnType; + +type ResourceWithCheckpointInfo = TFolderCheckpointResources & { + folderCommitId: string; + date: Date; +}; + +export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { + const folderCheckpointResourcesOrm = ormify(db, TableName.FolderCheckpointResources); + + const findByCheckpointId = async (folderCheckpointId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCheckpointResources) + .where({ folderCheckpointId }) + .select(selectAllTableCols(TableName.FolderCheckpointResources)); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByCheckpointId" }); + } + }; + + const findBySecretVersionId = async (secretVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCheckpointResources & Pick + >(TableName.FolderCheckpointResources) + .where({ secretVersionId }) + .select(selectAllTableCols(TableName.FolderCheckpointResources)) + .join( + TableName.FolderCheckpoint, + `${TableName.FolderCheckpointResources}.folderCheckpointId`, + `${TableName.FolderCheckpoint}.id` + ) + .select( + db.ref("folderCommitId").withSchema(TableName.FolderCheckpoint), + db.ref("date").withSchema(TableName.FolderCheckpoint) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindBySecretVersionId" }); + } + }; + + const findByFolderVersionId = async (folderVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCheckpointResources & Pick + >(TableName.FolderCheckpointResources) + .where({ folderVersionId }) + .select(selectAllTableCols(TableName.FolderCheckpointResources)) + .join( + TableName.FolderCheckpoint, + `${TableName.FolderCheckpointResources}.folderCheckpointId`, + `${TableName.FolderCheckpoint}.id` + ) + .select( + db.ref("folderCommitId").withSchema(TableName.FolderCheckpoint), + db.ref("date").withSchema(TableName.FolderCheckpoint) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderVersionId" }); + } + }; + + return { + ...folderCheckpointResourcesOrm, + findByCheckpointId, + findBySecretVersionId, + findByFolderVersionId + }; +}; diff --git a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts new file mode 100644 index 0000000000..c047c05042 --- /dev/null +++ b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts @@ -0,0 +1,86 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCheckpoints } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCheckpointDALFactory = ReturnType; + +type CheckpointWithCommitInfo = TFolderCheckpoints & { + actorName: string; + actorType: string; + message: string | null; + commitDate: Date; + folderId: string; +}; + +export const folderCheckpointDALFactory = (db: TDbClient) => { + const folderCheckpointOrm = ormify(db, TableName.FolderCheckpoint); + + const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint) + .where({ folderCommitId }) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindByCommitId" }); + } + }; + + const findByFolderId = async (folderId: string, limit?: number, tx?: Knex): Promise => { + try { + let query = (tx || db.replicaNode())(TableName.FolderCheckpoint) + .join(TableName.FolderCommit, `${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id`) + .where({ folderId }) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .select( + db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderCheckpoint}.date`, "desc"); + + if (limit !== undefined) { + query = query.limit(limit); + } + + const docs = await query; + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderId" }); + } + }; + + const findLatestByFolderId = async (folderId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint) + .join(TableName.FolderCommit, `${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id`) + .where({ folderId }) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .select( + db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderCheckpoint}.date`, "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestByFolderId" }); + } + }; + + return { + ...folderCheckpointOrm, + findByCommitId, + findByFolderId, + findLatestByFolderId + }; +}; diff --git a/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts new file mode 100644 index 0000000000..46905bf954 --- /dev/null +++ b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts @@ -0,0 +1,80 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCommitChanges, TFolderCommits } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCommitChangesDALFactory = ReturnType; + +type CommitChangeWithCommitInfo = TFolderCommitChanges & { + actorName: string; + actorType: string; + message: string | null; + date: Date; + folderId: string; +}; + +export const folderCommitChangesDALFactory = (db: TDbClient) => { + const folderCommitChangesOrm = ormify(db, TableName.FolderCommitChanges); + + const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCommitChanges) + .where({ folderCommitId }) + .select(selectAllTableCols(TableName.FolderCommitChanges)); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByCommitId" }); + } + }; + + const findBySecretVersionId = async (secretVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCommitChanges & Pick + >(TableName.FolderCommitChanges) + .where({ secretVersionId }) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) + .select( + db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("date").withSchema(TableName.FolderCommit), + db.ref("folderId").withSchema(TableName.FolderCommit) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindBySecretVersionId" }); + } + }; + + const findByFolderVersionId = async (folderVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCommitChanges & Pick + >(TableName.FolderCommitChanges) + .where({ folderVersionId }) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) + .select( + db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("date").withSchema(TableName.FolderCommit), + db.ref("folderId").withSchema(TableName.FolderCommit) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderVersionId" }); + } + }; + + return { + ...folderCommitChangesOrm, + findByCommitId, + findBySecretVersionId, + findByFolderVersionId + }; +}; diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts new file mode 100644 index 0000000000..2ad973bcd3 --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -0,0 +1,56 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCommits } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCommitDALFactory = ReturnType; + +export const folderCommitDALFactory = (db: TDbClient) => { + const folderCommitOrm = ormify(db, TableName.FolderCommit); + + const findByFolderId = async (folderId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("date", "desc"); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderId" }); + } + }; + + const findById = async (id: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ id }) + .select(selectAllTableCols(TableName.FolderCommit)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindById" }); + } + }; + + const findLatestCommit = async (folderId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("date", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommit" }); + } + }; + + return { + ...folderCommitOrm, + findByFolderId, + findById, + findLatestCommit + }; +}; diff --git a/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts new file mode 100644 index 0000000000..7b4e0bffb5 --- /dev/null +++ b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts @@ -0,0 +1,131 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { + TableName, + TEnvironments, + TFolderTreeCheckpointResources, + TFolderTreeCheckpoints, + TSecretFolders +} from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderTreeCheckpointResourcesDALFactory = ReturnType; + +type ResourceWithCheckpointInfo = TFolderTreeCheckpointResources & { + folderCommitId: string; + date: Date; +}; + +type ResourceWithFolderInfo = TFolderTreeCheckpointResources & { + name: string; + parentId: string | null; + slug: string; + envName: string; +}; + +export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { + const folderTreeCheckpointResourcesOrm = ormify( + db, + TableName.FolderTreeCheckpointResources + ); + + const findByTreeCheckpointId = async ( + folderTreeCheckpointId: string, + tx?: Knex + ): Promise => { + try { + const docs = await (tx || db.replicaNode())( + TableName.FolderTreeCheckpointResources + ) + .where({ folderTreeCheckpointId }) + .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByTreeCheckpointId" }); + } + }; + + const findByFolderId = async (folderId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderTreeCheckpointResources & Pick + >(TableName.FolderTreeCheckpointResources) + .where({ folderId }) + .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) + .join( + TableName.FolderTreeCheckpoint, + `${TableName.FolderTreeCheckpointResources}.folderTreeCheckpointId`, + `${TableName.FolderTreeCheckpoint}.id` + ) + .select( + db.ref("folderCommitId").withSchema(TableName.FolderTreeCheckpoint), + db.ref("date").withSchema(TableName.FolderTreeCheckpoint) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderId" }); + } + }; + + const findByFolderCommitId = async ( + folderCommitId: string, + tx?: Knex + ): Promise<(TFolderTreeCheckpointResources & { date: Date })[]> => { + try { + const docs = await (tx || db.replicaNode())< + TFolderTreeCheckpointResources & Pick + >(TableName.FolderTreeCheckpointResources) + .where({ folderCommitId }) + .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) + .join( + TableName.FolderTreeCheckpoint, + `${TableName.FolderTreeCheckpointResources}.folderTreeCheckpointId`, + `${TableName.FolderTreeCheckpoint}.id` + ) + .select(db.ref("date").withSchema(TableName.FolderTreeCheckpoint)); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderCommitId" }); + } + }; + + const findFoldersInTreeCheckpoint = async ( + folderTreeCheckpointId: string, + tx?: Knex + ): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderTreeCheckpointResources & + Pick & + Pick & { envName: string } + >(TableName.FolderTreeCheckpointResources) + .where({ folderTreeCheckpointId }) + .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) + .join( + TableName.SecretFolder, + `${TableName.FolderTreeCheckpointResources}.folderId`, + `${TableName.SecretFolder}.id` + ) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .select( + db.ref("name").withSchema(TableName.SecretFolder), + db.ref("parentId").withSchema(TableName.SecretFolder), + db.ref("slug").withSchema(TableName.Environment), + db.ref("name").withSchema(TableName.Environment).as("envName") + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindFoldersInTreeCheckpoint" }); + } + }; + + return { + ...folderTreeCheckpointResourcesOrm, + findByTreeCheckpointId, + findByFolderId, + findByFolderCommitId, + findFoldersInTreeCheckpoint + }; +}; diff --git a/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts new file mode 100644 index 0000000000..3377ca6437 --- /dev/null +++ b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts @@ -0,0 +1,111 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCommits, TFolderTreeCheckpoints } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderTreeCheckpointDALFactory = ReturnType; + +type TreeCheckpointWithCommitInfo = TFolderTreeCheckpoints & { + actorName: string; + actorType: string; + message: string | null; + commitDate: Date; + folderId: string; +}; + +export const folderTreeCheckpointDALFactory = (db: TDbClient) => { + const folderTreeCheckpointOrm = ormify(db, TableName.FolderTreeCheckpoint); + + const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) + .where({ folderCommitId }) + .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindByCommitId" }); + } + }; + + const findByProjectId = async ( + projectId: string, + limit?: number, + tx?: Knex + ): Promise => { + try { + const query = (tx || db.replicaNode())< + TFolderTreeCheckpoints & + Pick & { commitDate: Date } + >(TableName.FolderTreeCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderTreeCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .join(TableName.SecretFolder, `${TableName.FolderCommit}.folderId`, `${TableName.SecretFolder}.id`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where({ projectId }) + .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) + .select( + db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderTreeCheckpoint}.date`, "desc"); + + if (limit) { + void query.limit(limit); + } + + const docs = await query; + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByProjectId" }); + } + }; + + const findLatestByProjectId = async ( + projectId: string, + tx?: Knex + ): Promise => { + try { + const doc = await (tx || db.replicaNode())< + TFolderTreeCheckpoints & + Pick & { commitDate: Date } + >(TableName.FolderTreeCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderTreeCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .join(TableName.SecretFolder, `${TableName.FolderCommit}.folderId`, `${TableName.SecretFolder}.id`) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where({ projectId }) + .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) + .select( + db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderTreeCheckpoint}.date`, "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestByProjectId" }); + } + }; + + return { + ...folderTreeCheckpointOrm, + findByCommitId, + findByProjectId, + findLatestByProjectId + }; +}; From f493a617b107f9454ec87938c5be98d8febae90b Mon Sep 17 00:00:00 2001 From: carlosmonastyrski Date: Tue, 6 May 2025 18:57:25 -0300 Subject: [PATCH 2/4] Add new commit logic on every folder/secret operation --- backend/src/@types/fastify.d.ts | 2 + .../20250505194916_add-pit-revamp-tables.ts | 64 ++++--- backend/src/db/schemas/folder-checkpoints.ts | 3 +- .../src/db/schemas/folder-commit-changes.ts | 2 +- backend/src/db/schemas/folder-commits.ts | 6 +- .../folder-tree-checkpoint-resources.ts | 2 +- .../src/db/schemas/folder-tree-checkpoints.ts | 3 +- backend/src/db/schemas/models.ts | 2 +- .../src/ee/services/license/license-fns.ts | 54 +++--- .../secret-approval-request-service.ts | 26 ++- .../secret-replication-service.ts | 5 + .../secret-rotation-v2-service.ts | 23 ++- .../secret-rotation-queue.ts | 20 ++- backend/src/server/routes/index.ts | 39 +++- .../external-migration-fns.ts | 37 ++++ .../external-migration-queue.ts | 8 + .../folder-checkpoint-resources-dal.ts | 11 +- .../folder-checkpoint-dal.ts | 18 +- .../folder-commit-changes-dal.ts | 21 +-- .../folder-commit/folder-commit-dal.ts | 5 +- .../folder-commit/folder-commit-service.ts | 168 ++++++++++++++++++ .../folder-tree-checkpoint-resources-dal.ts | 25 +-- .../folder-tree-checkpoint-dal.ts | 22 +-- .../secret-folder/secret-folder-service.ts | 110 +++++++++++- .../secret-folder-version-dal.ts | 2 +- .../services/secret-sync/secret-sync-queue.ts | 11 +- .../secret-v2-bridge/secret-v2-bridge-fns.ts | 84 ++++++++- .../secret-v2-bridge-service.ts | 16 ++ .../secret-v2-bridge-types.ts | 6 + backend/src/services/secret/secret-fns.ts | 6 +- backend/src/services/secret/secret-queue.ts | 11 +- backend/src/services/secret/secret-types.ts | 3 + 32 files changed, 675 insertions(+), 140 deletions(-) create mode 100644 backend/src/services/folder-commit/folder-commit-service.ts diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index 6ec542c6bc..f83555c508 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -57,6 +57,7 @@ import { TCertificateTemplateServiceFactory } from "@app/services/certificate-te import { TCmekServiceFactory } from "@app/services/cmek/cmek-service"; import { TExternalGroupOrgRoleMappingServiceFactory } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-service"; import { TExternalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TGroupProjectServiceFactory } from "@app/services/group-project/group-project-service"; import { THsmServiceFactory } from "@app/services/hsm/hsm-service"; import { TIdentityServiceFactory } from "@app/services/identity/identity-service"; @@ -252,6 +253,7 @@ declare module "fastify" { microsoftTeams: TMicrosoftTeamsServiceFactory; assumePrivileges: TAssumePrivilegeServiceFactory; githubOrgSync: TGithubOrgSyncServiceFactory; + folderCommit: TFolderCommitServiceFactory; }; // this is exclusive use for middlewares in which we need to inject data // everywhere else access using service layer diff --git a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts index f86c800a13..ea4c8e9bb9 100644 --- a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts +++ b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts @@ -7,11 +7,11 @@ export async function up(knex: Knex): Promise { const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); if (!hasFolderCommitTable) { await knex.schema.createTable(TableName.FolderCommit, (t) => { - t.bigIncrements("id").primary(); - t.string("actorName").notNullable(); + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.bigIncrements("commitId"); + t.jsonb("actorMetadata").notNullable(); t.string("actorType").notNullable(); t.string("message"); - t.timestamp("date").notNullable().defaultTo(knex.fn.now()); t.uuid("folderId").notNullable(); t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("CASCADE"); t.timestamps(true, true, true); @@ -24,7 +24,7 @@ export async function up(knex: Knex): Promise { if (!hasFolderCommitChangesTable) { await knex.schema.createTable(TableName.FolderCommitChanges, (t) => { t.uuid("id").primary().defaultTo(knex.fn.uuid()); - t.bigInteger("folderCommitId").notNullable(); + t.uuid("folderCommitId").notNullable(); t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); t.string("changeType").notNullable(); t.uuid("secretVersionId"); @@ -43,9 +43,8 @@ export async function up(knex: Knex): Promise { if (!hasFolderCheckpointTable) { await knex.schema.createTable(TableName.FolderCheckpoint, (t) => { t.uuid("id").primary().defaultTo(knex.fn.uuid()); - t.bigInteger("folderCommitId").notNullable(); + t.uuid("folderCommitId").notNullable(); t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); - t.timestamp("date").notNullable().defaultTo(knex.fn.now()); t.timestamps(true, true, true); t.index("folderCommitId"); @@ -74,9 +73,8 @@ export async function up(knex: Knex): Promise { if (!hasFolderTreeCheckpointTable) { await knex.schema.createTable(TableName.FolderTreeCheckpoint, (t) => { t.uuid("id").primary().defaultTo(knex.fn.uuid()); - t.bigInteger("folderCommitId").notNullable(); + t.uuid("folderCommitId").notNullable(); t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); - t.timestamp("date").notNullable().defaultTo(knex.fn.now()); t.timestamps(true, true, true); t.index("folderCommitId"); @@ -91,7 +89,7 @@ export async function up(knex: Knex): Promise { t.foreign("folderTreeCheckpointId").references("id").inTable(TableName.FolderTreeCheckpoint).onDelete("CASCADE"); t.uuid("folderId").notNullable(); t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("CASCADE"); - t.bigInteger("folderCommitId").notNullable(); + t.uuid("folderCommitId").notNullable(); t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); t.timestamps(true, true, true); @@ -110,21 +108,45 @@ export async function up(knex: Knex): Promise { } export async function down(knex: Knex): Promise { - await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); - await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpointResources); + const hasFolderCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderCheckpointResources); + const hasFolderTreeCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderTreeCheckpointResources); + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + const hasFolderCommitChangesTable = await knex.schema.hasTable(TableName.FolderCommitChanges); + const hasFolderTreeCheckpointTable = await knex.schema.hasTable(TableName.FolderTreeCheckpoint); + const hasFolderCheckpointTable = await knex.schema.hasTable(TableName.FolderCheckpoint); + + if (hasFolderCheckpointResourcesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources); + } + + if (hasFolderTreeCheckpointResourcesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpointResources); + } - await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); - await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpoint); + if (hasFolderTreeCheckpointTable) { + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpoint); + } - await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources); - await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources); + if (hasFolderCheckpointResourcesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources); + } - await dropOnUpdateTrigger(knex, TableName.FolderCheckpoint); - await knex.schema.dropTableIfExists(TableName.FolderCheckpoint); + if (hasFolderCheckpointTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderCheckpoint); + } - await dropOnUpdateTrigger(knex, TableName.FolderCommitChanges); - await knex.schema.dropTableIfExists(TableName.FolderCommitChanges); + if (hasFolderCommitChangesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCommitChanges); + await knex.schema.dropTableIfExists(TableName.FolderCommitChanges); + } - await dropOnUpdateTrigger(knex, TableName.FolderCommit); - await knex.schema.dropTableIfExists(TableName.FolderCommit); + if (hasFolderCommitTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCommit); + await knex.schema.dropTableIfExists(TableName.FolderCommit); + } } diff --git a/backend/src/db/schemas/folder-checkpoints.ts b/backend/src/db/schemas/folder-checkpoints.ts index fd7ccae840..ba0ce6f71f 100644 --- a/backend/src/db/schemas/folder-checkpoints.ts +++ b/backend/src/db/schemas/folder-checkpoints.ts @@ -9,8 +9,7 @@ import { TImmutableDBKeys } from "./models"; export const FolderCheckpointsSchema = z.object({ id: z.string().uuid(), - folderCommitId: z.coerce.number(), - date: z.date(), + folderCommitId: z.string().uuid(), createdAt: z.date(), updatedAt: z.date() }); diff --git a/backend/src/db/schemas/folder-commit-changes.ts b/backend/src/db/schemas/folder-commit-changes.ts index 9ee2285952..063874f433 100644 --- a/backend/src/db/schemas/folder-commit-changes.ts +++ b/backend/src/db/schemas/folder-commit-changes.ts @@ -9,7 +9,7 @@ import { TImmutableDBKeys } from "./models"; export const FolderCommitChangesSchema = z.object({ id: z.string().uuid(), - folderCommitId: z.coerce.number(), + folderCommitId: z.string().uuid(), changeType: z.string(), secretVersionId: z.string().uuid().nullable().optional(), folderVersionId: z.string().uuid().nullable().optional(), diff --git a/backend/src/db/schemas/folder-commits.ts b/backend/src/db/schemas/folder-commits.ts index 4d5f2d2af8..ec166e7c2d 100644 --- a/backend/src/db/schemas/folder-commits.ts +++ b/backend/src/db/schemas/folder-commits.ts @@ -8,11 +8,11 @@ import { z } from "zod"; import { TImmutableDBKeys } from "./models"; export const FolderCommitsSchema = z.object({ - id: z.coerce.number(), - actorName: z.string(), + id: z.string().uuid(), + commitId: z.coerce.number(), + actorMetadata: z.unknown(), actorType: z.string(), message: z.string().nullable().optional(), - date: z.date(), folderId: z.string().uuid(), createdAt: z.date(), updatedAt: z.date() diff --git a/backend/src/db/schemas/folder-tree-checkpoint-resources.ts b/backend/src/db/schemas/folder-tree-checkpoint-resources.ts index 1c65e833d0..06d5770cc4 100644 --- a/backend/src/db/schemas/folder-tree-checkpoint-resources.ts +++ b/backend/src/db/schemas/folder-tree-checkpoint-resources.ts @@ -11,7 +11,7 @@ export const FolderTreeCheckpointResourcesSchema = z.object({ id: z.string().uuid(), folderTreeCheckpointId: z.string().uuid(), folderId: z.string().uuid(), - folderCommitId: z.coerce.number(), + folderCommitId: z.string().uuid(), createdAt: z.date(), updatedAt: z.date() }); diff --git a/backend/src/db/schemas/folder-tree-checkpoints.ts b/backend/src/db/schemas/folder-tree-checkpoints.ts index 8f637b49ba..ea500af6ba 100644 --- a/backend/src/db/schemas/folder-tree-checkpoints.ts +++ b/backend/src/db/schemas/folder-tree-checkpoints.ts @@ -9,8 +9,7 @@ import { TImmutableDBKeys } from "./models"; export const FolderTreeCheckpointsSchema = z.object({ id: z.string().uuid(), - folderCommitId: z.coerce.number(), - date: z.date(), + folderCommitId: z.string().uuid(), createdAt: z.date(), updatedAt: z.date() }); diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 0b80be80f9..47c5cad540 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -161,7 +161,7 @@ export enum TableName { FolderTreeCheckpointResources = "folder_tree_checkpoint_resources" } -export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt"; +export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt" | "commitId"; export const UserDeviceSchema = z .object({ diff --git a/backend/src/ee/services/license/license-fns.ts b/backend/src/ee/services/license/license-fns.ts index cff6ee1810..b7ae6f7ee2 100644 --- a/backend/src/ee/services/license/license-fns.ts +++ b/backend/src/ee/services/license/license-fns.ts @@ -17,44 +17,44 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({ environmentsUsed: 0, identityLimit: null, identitiesUsed: 0, - dynamicSecret: true, + dynamicSecret: false, secretVersioning: true, - pitRecovery: true, - ipAllowlisting: true, - rbac: true, - githubOrgSync: true, - customRateLimits: true, - customAlerts: true, - secretAccessInsights: true, - auditLogs: true, + pitRecovery: false, + ipAllowlisting: false, + rbac: false, + githubOrgSync: false, + customRateLimits: false, + customAlerts: false, + secretAccessInsights: false, + auditLogs: false, auditLogsRetentionDays: 0, - auditLogStreams: true, + auditLogStreams: false, auditLogStreamLimit: 3, - samlSSO: true, - hsm: true, - oidcSSO: true, - scim: true, - ldap: true, - groups: true, + samlSSO: false, + hsm: false, + oidcSSO: false, + scim: false, + ldap: false, + groups: false, status: null, trial_end: null, has_used_trial: true, - secretApproval: true, - secretRotation: true, - caCrl: true, - instanceUserManagement: true, - externalKms: true, + secretApproval: false, + secretRotation: false, + caCrl: false, + instanceUserManagement: false, + externalKms: false, rateLimits: { readLimit: 60, writeLimit: 200, secretsLimit: 40 }, - pkiEst: true, - enforceMfa: true, - projectTemplates: true, - kmip: true, - gateway: true, - sshHostGroups: true + pkiEst: false, + enforceMfa: false, + projectTemplates: false, + kmip: false, + gateway: false, + sshHostGroups: false }); export const setupLicenseRequestWithStore = (baseURL: string, refreshUrl: string, licenseKey: string) => { diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 262e8e5cfd..054374c7a1 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -20,6 +20,7 @@ import { EnforcementLevel } from "@app/lib/types"; import { triggerWorkflowIntegrationNotification } from "@app/lib/workflow-integrations/trigger-notification"; import { TriggerFeature } from "@app/lib/workflow-integrations/types"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TMicrosoftTeamsServiceFactory } from "@app/services/microsoft-teams/microsoft-teams-service"; @@ -130,6 +131,7 @@ type TSecretApprovalRequestServiceFactoryDep = { licenseService: Pick; projectMicrosoftTeamsConfigDAL: Pick; microsoftTeamsService: Pick; + folderCommitService: Pick; }; export type TSecretApprovalRequestServiceFactory = ReturnType; @@ -161,7 +163,8 @@ export const secretApprovalRequestServiceFactory = ({ projectSlackConfigDAL, resourceMetadataDAL, projectMicrosoftTeamsConfigDAL, - microsoftTeamsService + microsoftTeamsService, + folderCommitService }: TSecretApprovalRequestServiceFactoryDep) => { const requestCount = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod }: TApprovalRequestCountDTO) => { if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" }); @@ -596,6 +599,10 @@ export const secretApprovalRequestServiceFactory = ({ ? await fnSecretV2BridgeBulkInsert({ tx, folderId, + actor: { + actorId, + type: actor + }, orgId: actorOrgId, inputSecrets: secretCreationCommits.map((el) => ({ tagIds: el?.tags.map(({ id }) => id), @@ -618,13 +625,18 @@ export const secretApprovalRequestServiceFactory = ({ secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, - secretVersionTagDAL: secretVersionTagV2BridgeDAL + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService }) : []; const updatedSecrets = secretUpdationCommits.length ? await fnSecretV2BridgeBulkUpdate({ folderId, orgId: actorOrgId, + actor: { + actorId, + type: actor + }, tx, inputSecrets: secretUpdationCommits.map((el) => { const encryptedValue = @@ -658,7 +670,8 @@ export const secretApprovalRequestServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }) : []; const deletedSecret = secretDeletionCommits.length @@ -666,10 +679,13 @@ export const secretApprovalRequestServiceFactory = ({ projectId, folderId, tx, - actorId: "", + actorId, + actorType: actor, secretDAL: secretV2BridgeDAL, secretQueueService, - inputSecrets: secretDeletionCommits.map(({ key }) => ({ secretKey: key, type: SecretType.Shared })) + inputSecrets: secretDeletionCommits.map(({ key }) => ({ secretKey: key, type: SecretType.Shared })), + folderCommitService, + secretVersionDAL: secretVersionV2BridgeDAL }) : []; const updatedSecretApproval = await secretApprovalRequestDAL.updateById( diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index 90fdf561e1..c181268719 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -10,6 +10,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueName, TQueueServiceFactory } from "@app/queue"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -87,6 +88,7 @@ type TSecretReplicationServiceFactoryDep = { projectBotService: Pick; kmsService: Pick; + folderCommitService: Pick; }; export type TSecretReplicationServiceFactory = ReturnType; @@ -132,6 +134,7 @@ export const secretReplicationServiceFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, kmsService, + folderCommitService, resourceMetadataDAL }: TSecretReplicationServiceFactoryDep) => { const $getReplicatedSecrets = ( @@ -446,6 +449,7 @@ export const secretReplicationServiceFactory = ({ tx, secretTagDAL, resourceMetadataDAL, + folderCommitService, secretVersionTagDAL: secretVersionV2TagBridgeDAL, inputSecrets: locallyCreatedSecrets.map((doc) => { return { @@ -466,6 +470,7 @@ export const secretReplicationServiceFactory = ({ orgId, folderId: destinationReplicationFolderId, secretVersionDAL: secretVersionV2BridgeDAL, + folderCommitService, secretDAL: secretV2BridgeDAL, tx, resourceMetadataDAL, diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts index 69743f1336..979cb5b763 100644 --- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts +++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts @@ -61,6 +61,7 @@ import { TAppConnectionDALFactory } from "@app/services/app-connection/app-conne import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns"; import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -96,7 +97,7 @@ export type TSecretRotationV2ServiceFactoryDep = { TSecretV2BridgeDALFactory, "bulkUpdate" | "insertMany" | "deleteMany" | "upsertSecretReferences" | "find" | "invalidateSecretCacheByProjectId" >; - secretVersionV2BridgeDAL: Pick; + secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; secretTagDAL: Pick; @@ -104,6 +105,7 @@ export type TSecretRotationV2ServiceFactoryDep = { snapshotService: Pick; queueService: Pick; appConnectionDAL: Pick; + folderCommitService: Pick; }; export type TSecretRotationV2ServiceFactory = ReturnType; @@ -141,6 +143,7 @@ export const secretRotationV2ServiceFactory = ({ snapshotService, keyStore, queueService, + folderCommitService, appConnectionDAL }: TSecretRotationV2ServiceFactoryDep) => { const $queueSendSecretRotationStatusNotification = async (secretRotation: TSecretRotationV2Raw) => { @@ -533,7 +536,12 @@ export const secretRotationV2ServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, secretTagDAL, - resourceMetadataDAL + folderCommitService, + resourceMetadataDAL, + actor: { + type: actor.type, + actorId: actor.id + } }); await secretRotationV2DAL.insertSecretMappings( @@ -668,7 +676,12 @@ export const secretRotationV2ServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, secretTagDAL, - resourceMetadataDAL + folderCommitService, + resourceMetadataDAL, + actor: { + type: actor.type, + actorId: actor.id + } }); secretsMappingUpdated = true; @@ -786,6 +799,9 @@ export const secretRotationV2ServiceFactory = ({ projectId, folderId, actorId: actor.id, // not actually used since rotated secrets are shared + actorType: actor.type, + folderCommitService, + secretVersionDAL: secretVersionV2BridgeDAL, tx }); } @@ -926,6 +942,7 @@ export const secretRotationV2ServiceFactory = ({ secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService, secretTagDAL, resourceMetadataDAL }); diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts index 2c6124348a..ec92a736fc 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts @@ -14,6 +14,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -53,6 +54,7 @@ type TSecretRotationQueueFactoryDep = { secretVersionV2BridgeDAL: Pick; telemetryService: Pick; kmsService: Pick; + folderCommitService: Pick; }; // These error should stop the repeatable job and ask user to reconfigure rotation @@ -77,6 +79,7 @@ export const secretRotationQueueFactory = ({ telemetryService, secretV2BridgeDAL, secretVersionV2BridgeDAL, + folderCommitService, kmsService }: TSecretRotationQueueFactoryDep) => { const addToQueue = async (rotationId: string, interval: number) => { @@ -330,7 +333,7 @@ export const secretRotationQueueFactory = ({ })), tx ); - await secretVersionV2BridgeDAL.insertMany( + const secretVersions = await secretVersionV2BridgeDAL.insertMany( updatedSecrets.map(({ id, updatedAt, createdAt, ...el }) => ({ ...el, actorType: ActorType.PLATFORM, @@ -338,6 +341,21 @@ export const secretRotationQueueFactory = ({ })), tx ); + + await folderCommitService.createCommit( + { + actor: { + type: ActorType.PLATFORM + }, + message: "Secret rotation", + folderId: secretVersions[0].folderId, + changes: secretVersions.map((sv) => ({ + type: "add", + secretVersionId: sv.id + })) + }, + tx + ); }); await secretV2BridgeDAL.invalidateSecretCacheByProjectId(secretRotation.projectId); diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index d15058bcb7..5ba8954fef 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -140,6 +140,11 @@ import { externalGroupOrgRoleMappingDALFactory } from "@app/services/external-gr import { externalGroupOrgRoleMappingServiceFactory } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-service"; import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue"; import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; +import { folderCheckpointDALFactory } from "@app/services/folder-checkpoint/folder-checkpoint-dal"; +import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal"; +import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; +import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal"; +import { folderTreeCheckpointDALFactory } from "@app/services/folder-tree-checkpoint/folder-tree-checkpoint-dal"; import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal"; import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal"; import { groupProjectServiceFactory } from "@app/services/group-project/group-project-service"; @@ -551,6 +556,19 @@ export const registerRoutes = async ( projectRoleDAL, permissionService }); + + const folderCommitChangesDAL = folderCommitChangesDALFactory(db); + const folderCheckpointDAL = folderCheckpointDALFactory(db); + const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db); + const folderCommitDAL = folderCommitDALFactory(db); + const folderCommitService = folderCommitServiceFactory({ + folderCommitDAL, + folderCommitChangesDAL, + folderCheckpointDAL, + folderTreeCheckpointDAL, + userDAL, + identityDAL + }); const scimService = scimServiceFactory({ licenseService, scimDAL, @@ -973,6 +991,7 @@ export const registerRoutes = async ( projectMembershipDAL, projectBotDAL, secretDAL, + folderCommitService, secretBlindIndexDAL, secretVersionDAL, secretTagDAL, @@ -1019,6 +1038,7 @@ export const registerRoutes = async ( secretReminderRecipientsDAL, orgService, resourceMetadataDAL, + folderCommitService, secretSyncQueue }); @@ -1120,7 +1140,8 @@ export const registerRoutes = async ( folderVersionDAL, projectEnvDAL, snapshotService, - projectDAL + projectDAL, + folderCommitService }); const secretImportService = secretImportServiceFactory({ @@ -1145,6 +1166,7 @@ export const registerRoutes = async ( const secretV2BridgeService = secretV2BridgeServiceFactory({ folderDAL, secretVersionDAL: secretVersionV2BridgeDAL, + folderCommitService, secretQueueService, secretDAL: secretV2BridgeDAL, permissionService, @@ -1188,7 +1210,8 @@ export const registerRoutes = async ( projectSlackConfigDAL, resourceMetadataDAL, projectMicrosoftTeamsConfigDAL, - microsoftTeamsService + microsoftTeamsService, + folderCommitService }); const secretService = secretServiceFactory({ @@ -1273,7 +1296,8 @@ export const registerRoutes = async ( secretV2BridgeDAL, secretVersionV2TagBridgeDAL: secretVersionTagV2BridgeDAL, secretVersionV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const secretRotationQueue = secretRotationQueueFactory({ @@ -1285,6 +1309,7 @@ export const registerRoutes = async ( projectBotService, secretVersionV2BridgeDAL, secretV2BridgeDAL, + folderCommitService, kmsService }); @@ -1557,7 +1582,9 @@ export const registerRoutes = async ( secretDAL: secretV2BridgeDAL, queueService, secretV2BridgeService, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService, + folderVersionDAL }); const migrationService = externalMigrationServiceFactory({ @@ -1619,6 +1646,7 @@ export const registerRoutes = async ( auditLogService, secretV2BridgeDAL, secretTagDAL, + folderCommitService, secretVersionTagV2BridgeDAL, secretVersionV2BridgeDAL, keyStore, @@ -1746,7 +1774,8 @@ export const registerRoutes = async ( secretRotationV2: secretRotationV2Service, microsoftTeams: microsoftTeamsService, assumePrivileges: assumePrivilegeService, - githubOrgSync: githubOrgSyncConfigService + githubOrgSync: githubOrgSyncConfigService, + folderCommit: folderCommitService }); const cronJobs: CronJob[] = []; diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 856b39012f..f3c2326de1 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -10,6 +10,7 @@ import { chunkArray } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { KmsDataKey } from "../kms/kms-types"; import { TProjectDALFactory } from "../project/project-dal"; @@ -18,6 +19,7 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { fnSecretBulkInsert, getAllSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns"; @@ -42,6 +44,8 @@ export type TImportDataIntoInfisicalDTO = { projectService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; + folderCommitService: Pick; + folderVersionDAL: Pick; input: TImportInfisicalDataCreate; }; @@ -507,6 +511,8 @@ export const importDataIntoInfisicalFn = async ({ secretVersionTagDAL, folderDAL, resourceMetadataDAL, + folderVersionDAL, + folderCommitService, input: { data, actor, actorId, actorOrgId, actorAuthMethod } }: TImportDataIntoInfisicalDTO) => { // Import data to infisical @@ -599,6 +605,36 @@ export const importDataIntoInfisicalFn = async ({ tx ); + const newFolderVersion = await folderVersionDAL.create( + { + name: newFolder.name, + envId: newFolder.envId, + version: newFolder.version, + folderId: newFolder.id + }, + tx + ); + + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "External migration", + folderId: parentEnv.rootFolderId, + changes: [ + { + type: "add", + folderVersionId: newFolderVersion.id + } + ] + }, + tx + ); + originalToNewFolderId.set(folder.id, { folderId: newFolder.id, projectId: parentEnv.projectId @@ -772,6 +808,7 @@ export const importDataIntoInfisicalFn = async ({ secretVersionDAL, secretTagDAL, secretVersionTagDAL, + folderCommitService, actor: { type: actor, actorId diff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.ts index 8aa46b94c1..66f2c73e77 100644 --- a/backend/src/services/external-migration/external-migration-queue.ts +++ b/backend/src/services/external-migration/external-migration-queue.ts @@ -3,6 +3,7 @@ import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectServiceFactory } from "../project/project-service"; @@ -10,6 +11,7 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; @@ -36,6 +38,8 @@ export type TExternalMigrationQueueFactoryDep = { projectService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; + folderCommitService: Pick; + folderVersionDAL: Pick; resourceMetadataDAL: Pick; }; @@ -56,6 +60,8 @@ export const externalMigrationQueueFactory = ({ secretTagDAL, secretVersionTagDAL, folderDAL, + folderCommitService, + folderVersionDAL, resourceMetadataDAL }: TExternalMigrationQueueFactoryDep) => { const startImport = async (dto: { @@ -114,6 +120,8 @@ export const externalMigrationQueueFactory = ({ projectService, projectEnvService, secretV2BridgeService, + folderCommitService, + folderVersionDAL, resourceMetadataDAL }); diff --git a/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts index d0b2200ecf..2457abe143 100644 --- a/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts +++ b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts @@ -9,11 +9,10 @@ export type TFolderCheckpointResourcesDALFactory = ReturnType { - const folderCheckpointResourcesOrm = ormify(db, TableName.FolderCheckpointResources); + const folderCheckpointResourcesOrm = ormify(db, TableName.FolderCheckpointResources); const findByCheckpointId = async (folderCheckpointId: string, tx?: Knex): Promise => { try { @@ -29,7 +28,7 @@ export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { const findBySecretVersionId = async (secretVersionId: string, tx?: Knex): Promise => { try { const docs = await (tx || db.replicaNode())< - TFolderCheckpointResources & Pick + TFolderCheckpointResources & Pick >(TableName.FolderCheckpointResources) .where({ secretVersionId }) .select(selectAllTableCols(TableName.FolderCheckpointResources)) @@ -40,7 +39,7 @@ export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { ) .select( db.ref("folderCommitId").withSchema(TableName.FolderCheckpoint), - db.ref("date").withSchema(TableName.FolderCheckpoint) + db.ref("createdAt").withSchema(TableName.FolderCheckpoint) ); return docs; } catch (error) { @@ -51,7 +50,7 @@ export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { const findByFolderVersionId = async (folderVersionId: string, tx?: Knex): Promise => { try { const docs = await (tx || db.replicaNode())< - TFolderCheckpointResources & Pick + TFolderCheckpointResources & Pick >(TableName.FolderCheckpointResources) .where({ folderVersionId }) .select(selectAllTableCols(TableName.FolderCheckpointResources)) @@ -62,7 +61,7 @@ export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { ) .select( db.ref("folderCommitId").withSchema(TableName.FolderCheckpoint), - db.ref("date").withSchema(TableName.FolderCheckpoint) + db.ref("createdAt").withSchema(TableName.FolderCheckpoint) ); return docs; } catch (error) { diff --git a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts index c047c05042..5471f9b610 100644 --- a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts +++ b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts @@ -8,15 +8,15 @@ import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderCheckpointDALFactory = ReturnType; type CheckpointWithCommitInfo = TFolderCheckpoints & { - actorName: string; + actorMetadata: unknown; actorType: string; - message: string | null; + message?: string | null; commitDate: Date; folderId: string; }; export const folderCheckpointDALFactory = (db: TDbClient) => { - const folderCheckpointOrm = ormify(db, TableName.FolderCheckpoint); + const folderCheckpointOrm = ormify(db, TableName.FolderCheckpoint); const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { try { @@ -37,13 +37,13 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { .where({ folderId }) .select(selectAllTableCols(TableName.FolderCheckpoint)) .select( - db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), db.ref("actorType").withSchema(TableName.FolderCommit), db.ref("message").withSchema(TableName.FolderCommit), - db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), db.ref("folderId").withSchema(TableName.FolderCommit) ) - .orderBy(`${TableName.FolderCheckpoint}.date`, "desc"); + .orderBy(`${TableName.FolderCheckpoint}.createdAt`, "desc"); if (limit !== undefined) { query = query.limit(limit); @@ -63,13 +63,13 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { .where({ folderId }) .select(selectAllTableCols(TableName.FolderCheckpoint)) .select( - db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), db.ref("actorType").withSchema(TableName.FolderCommit), db.ref("message").withSchema(TableName.FolderCommit), - db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), db.ref("folderId").withSchema(TableName.FolderCommit) ) - .orderBy(`${TableName.FolderCheckpoint}.date`, "desc") + .orderBy(`${TableName.FolderCheckpoint}.createdAt`, "desc") .first(); return doc; } catch (error) { diff --git a/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts index 46905bf954..0b06f13623 100644 --- a/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts +++ b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts @@ -8,15 +8,14 @@ import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderCommitChangesDALFactory = ReturnType; type CommitChangeWithCommitInfo = TFolderCommitChanges & { - actorName: string; + actorMetadata: unknown; actorType: string; - message: string | null; - date: Date; + message?: string | null; folderId: string; }; export const folderCommitChangesDALFactory = (db: TDbClient) => { - const folderCommitChangesOrm = ormify(db, TableName.FolderCommitChanges); + const folderCommitChangesOrm = ormify(db, TableName.FolderCommitChanges); const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { try { @@ -32,16 +31,17 @@ export const folderCommitChangesDALFactory = (db: TDbClient) => { const findBySecretVersionId = async (secretVersionId: string, tx?: Knex): Promise => { try { const docs = await (tx || db.replicaNode())< - TFolderCommitChanges & Pick + TFolderCommitChanges & + Pick >(TableName.FolderCommitChanges) .where({ secretVersionId }) .select(selectAllTableCols(TableName.FolderCommitChanges)) .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) .select( - db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), db.ref("actorType").withSchema(TableName.FolderCommit), db.ref("message").withSchema(TableName.FolderCommit), - db.ref("date").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit), db.ref("folderId").withSchema(TableName.FolderCommit) ); return docs; @@ -53,16 +53,17 @@ export const folderCommitChangesDALFactory = (db: TDbClient) => { const findByFolderVersionId = async (folderVersionId: string, tx?: Knex): Promise => { try { const docs = await (tx || db.replicaNode())< - TFolderCommitChanges & Pick + TFolderCommitChanges & + Pick >(TableName.FolderCommitChanges) .where({ folderVersionId }) .select(selectAllTableCols(TableName.FolderCommitChanges)) .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) .select( - db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), db.ref("actorType").withSchema(TableName.FolderCommit), db.ref("message").withSchema(TableName.FolderCommit), - db.ref("date").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit), db.ref("folderId").withSchema(TableName.FolderCommit) ); return docs; diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts index 2ad973bcd3..c6888f8cd4 100644 --- a/backend/src/services/folder-commit/folder-commit-dal.ts +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -8,7 +8,8 @@ import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderCommitDALFactory = ReturnType; export const folderCommitDALFactory = (db: TDbClient) => { - const folderCommitOrm = ormify(db, TableName.FolderCommit); + const folderCommitOrm = ormify(db, TableName.FolderCommit); + const { delete: deleteOp, deleteById, ...restOfOrm } = folderCommitOrm; const findByFolderId = async (folderId: string, tx?: Knex): Promise => { try { @@ -48,7 +49,7 @@ export const folderCommitDALFactory = (db: TDbClient) => { }; return { - ...folderCommitOrm, + ...restOfOrm, findByFolderId, findById, findLatestCommit diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts new file mode 100644 index 0000000000..c82de598c3 --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -0,0 +1,168 @@ +import { Knex } from "knex"; + +import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; + +import { ActorType } from "../auth/auth-type"; +import { TFolderCheckpointDALFactory } from "../folder-checkpoint/folder-checkpoint-dal"; +import { TFolderCommitChangesDALFactory } from "../folder-commit-changes/folder-commit-changes-dal"; +import { TFolderTreeCheckpointDALFactory } from "../folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { TIdentityDALFactory } from "../identity/identity-dal"; +import { TUserDALFactory } from "../user/user-dal"; +import { TFolderCommitDALFactory } from "./folder-commit-dal"; + +type TFolderCommitServiceFactoryDep = { + folderCommitDAL: Pick< + TFolderCommitDALFactory, + "create" | "findById" | "findByFolderId" | "findLatestCommit" | "transaction" + >; + folderCommitChangesDAL: Pick; + folderCheckpointDAL: Pick; + folderTreeCheckpointDAL: Pick< + TFolderTreeCheckpointDALFactory, + "create" | "findByProjectId" | "findLatestByProjectId" + >; + userDAL: Pick; + identityDAL: Pick; +}; + +export type TCreateCommitDTO = { + actor: { + type: string; + metadata?: { + name?: string; + id?: string; + }; + }; + message?: string; + folderId: string; + changes: { + type: string; + secretVersionId?: string; + folderVersionId?: string; + }[]; +}; + +export type TCommitChangeDTO = { + folderCommitId: string; + changeType: string; + secretVersionId?: string; + folderVersionId?: string; +}; + +export const folderCommitServiceFactory = ({ + folderCommitDAL, + folderCommitChangesDAL, + folderCheckpointDAL, + folderTreeCheckpointDAL, + userDAL, + identityDAL +}: TFolderCommitServiceFactoryDep) => { + const createCommit = async (data: TCreateCommitDTO, tx?: Knex) => { + const metadata = data.actor.metadata || {}; + try { + if (data.actor.type === ActorType.USER && data.actor.metadata?.id) { + const user = await userDAL.findById(data.actor.metadata?.id, tx); + metadata.name = user?.username; + } + if (data.actor.type === ActorType.IDENTITY && data.actor.metadata?.id) { + const identity = await identityDAL.findById(data.actor.metadata?.id, tx); + metadata.name = identity?.name; + } + const newCommit = await folderCommitDAL.create( + { + actorMetadata: metadata, + actorType: data.actor.type, + message: data.message, + folderId: data.folderId + }, + tx + ); + for (const change of data.changes) { + // eslint-disable-next-line no-await-in-loop + await folderCommitChangesDAL.create( + { + folderCommitId: newCommit.id, + changeType: change.type, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId + }, + tx + ); + } + + return newCommit; + } catch (error) { + throw new DatabaseError({ error, name: "CreateCommit" }); + } + }; + + // Add a change to a commit and trigger checkpoints as needed + const addCommitChange = async (data: TCommitChangeDTO, tx?: Knex) => { + try { + if (!data.secretVersionId && !data.folderVersionId) { + throw new BadRequestError({ message: "Either secretVersionId or folderVersionId must be provided" }); + } + + const commit = await folderCommitDAL.findById(data.folderCommitId, tx); + if (!commit) { + throw new NotFoundError({ message: `Commit with ID ${data.folderCommitId} not found` }); + } + + return await folderCommitChangesDAL.create(data, tx); + } catch (error) { + if (error instanceof NotFoundError || error instanceof BadRequestError) { + throw error; + } + throw new DatabaseError({ error, name: "AddCommitChange" }); + } + }; + + // Retrieve a commit by ID + const getCommitById = async (id: string, tx?: Knex) => { + return folderCommitDAL.findById(id, tx); + }; + + // Get all commits for a folder + const getCommitsByFolderId = async (folderId: string, tx?: Knex) => { + return folderCommitDAL.findByFolderId(folderId, tx); + }; + + // Get changes for a commit + const getCommitChanges = async (commitId: string, tx?: Knex) => { + return folderCommitChangesDAL.findByCommitId(commitId, tx); + }; + + // Get checkpoints for a folder + const getCheckpointsByFolderId = async (folderId: string, limit?: number, tx?: Knex) => { + return folderCheckpointDAL.findByFolderId(folderId, limit, tx); + }; + + // Get the latest checkpoint for a folder + const getLatestCheckpoint = async (folderId: string, tx?: Knex) => { + return folderCheckpointDAL.findLatestByFolderId(folderId, tx); + }; + + // Get tree checkpoints for a project + const getTreeCheckpointsByProjectId = async (projectId: string, limit?: number, tx?: Knex) => { + return folderTreeCheckpointDAL.findByProjectId(projectId, limit, tx); + }; + + // Get the latest tree checkpoint for a project + const getLatestTreeCheckpoint = async (projectId: string, tx?: Knex) => { + return folderTreeCheckpointDAL.findLatestByProjectId(projectId, tx); + }; + + return { + createCommit, + addCommitChange, + getCommitById, + getCommitsByFolderId, + getCommitChanges, + getCheckpointsByFolderId, + getLatestCheckpoint, + getTreeCheckpointsByProjectId, + getLatestTreeCheckpoint + }; +}; + +export type TFolderCommitServiceFactory = ReturnType; diff --git a/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts index 7b4e0bffb5..8813784fda 100644 --- a/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts +++ b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts @@ -3,9 +3,9 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { TableName, - TEnvironments, TFolderTreeCheckpointResources, TFolderTreeCheckpoints, + TProjectEnvironments, TSecretFolders } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; @@ -15,21 +15,17 @@ export type TFolderTreeCheckpointResourcesDALFactory = ReturnType { - const folderTreeCheckpointResourcesOrm = ormify( - db, - TableName.FolderTreeCheckpointResources - ); + const folderTreeCheckpointResourcesOrm = ormify(db, TableName.FolderTreeCheckpointResources); const findByTreeCheckpointId = async ( folderTreeCheckpointId: string, @@ -50,7 +46,7 @@ export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { const findByFolderId = async (folderId: string, tx?: Knex): Promise => { try { const docs = await (tx || db.replicaNode())< - TFolderTreeCheckpointResources & Pick + TFolderTreeCheckpointResources & Pick >(TableName.FolderTreeCheckpointResources) .where({ folderId }) .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) @@ -61,7 +57,7 @@ export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { ) .select( db.ref("folderCommitId").withSchema(TableName.FolderTreeCheckpoint), - db.ref("date").withSchema(TableName.FolderTreeCheckpoint) + db.ref("createdAt").withSchema(TableName.FolderTreeCheckpoint) ); return docs; } catch (error) { @@ -69,13 +65,10 @@ export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { } }; - const findByFolderCommitId = async ( - folderCommitId: string, - tx?: Knex - ): Promise<(TFolderTreeCheckpointResources & { date: Date })[]> => { + const findByFolderCommitId = async (folderCommitId: string, tx?: Knex): Promise => { try { const docs = await (tx || db.replicaNode())< - TFolderTreeCheckpointResources & Pick + TFolderTreeCheckpointResources & Pick >(TableName.FolderTreeCheckpointResources) .where({ folderCommitId }) .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) @@ -84,7 +77,7 @@ export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { `${TableName.FolderTreeCheckpointResources}.folderTreeCheckpointId`, `${TableName.FolderTreeCheckpoint}.id` ) - .select(db.ref("date").withSchema(TableName.FolderTreeCheckpoint)); + .select(db.ref("createdAt").withSchema(TableName.FolderTreeCheckpoint)); return docs; } catch (error) { throw new DatabaseError({ error, name: "FindByFolderCommitId" }); @@ -99,7 +92,7 @@ export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { const docs = await (tx || db.replicaNode())< TFolderTreeCheckpointResources & Pick & - Pick & { envName: string } + Pick & { envName: string } >(TableName.FolderTreeCheckpointResources) .where({ folderTreeCheckpointId }) .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) diff --git a/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts index 3377ca6437..4e9381f9cc 100644 --- a/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts +++ b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts @@ -8,15 +8,15 @@ import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderTreeCheckpointDALFactory = ReturnType; type TreeCheckpointWithCommitInfo = TFolderTreeCheckpoints & { - actorName: string; + actorMetadata: unknown; actorType: string; - message: string | null; + message?: string | null; commitDate: Date; folderId: string; }; export const folderTreeCheckpointDALFactory = (db: TDbClient) => { - const folderTreeCheckpointOrm = ormify(db, TableName.FolderTreeCheckpoint); + const folderTreeCheckpointOrm = ormify(db, TableName.FolderTreeCheckpoint); const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { try { @@ -38,7 +38,7 @@ export const folderTreeCheckpointDALFactory = (db: TDbClient) => { try { const query = (tx || db.replicaNode())< TFolderTreeCheckpoints & - Pick & { commitDate: Date } + Pick & { commitDate: Date } >(TableName.FolderTreeCheckpoint) .join( TableName.FolderCommit, @@ -50,13 +50,13 @@ export const folderTreeCheckpointDALFactory = (db: TDbClient) => { .where({ projectId }) .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) .select( - db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), db.ref("actorType").withSchema(TableName.FolderCommit), db.ref("message").withSchema(TableName.FolderCommit), - db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), db.ref("folderId").withSchema(TableName.FolderCommit) ) - .orderBy(`${TableName.FolderTreeCheckpoint}.date`, "desc"); + .orderBy(`${TableName.FolderTreeCheckpoint}.createdAt`, "desc"); if (limit) { void query.limit(limit); @@ -76,7 +76,7 @@ export const folderTreeCheckpointDALFactory = (db: TDbClient) => { try { const doc = await (tx || db.replicaNode())< TFolderTreeCheckpoints & - Pick & { commitDate: Date } + Pick & { commitDate: Date } >(TableName.FolderTreeCheckpoint) .join( TableName.FolderCommit, @@ -88,13 +88,13 @@ export const folderTreeCheckpointDALFactory = (db: TDbClient) => { .where({ projectId }) .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) .select( - db.ref("actorName").withSchema(TableName.FolderCommit), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), db.ref("actorType").withSchema(TableName.FolderCommit), db.ref("message").withSchema(TableName.FolderCommit), - db.ref("date").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), db.ref("folderId").withSchema(TableName.FolderCommit) ) - .orderBy(`${TableName.FolderTreeCheckpoint}.date`, "desc") + .orderBy(`${TableName.FolderTreeCheckpoint}.createdAt`, "desc") .first(); return doc; } catch (error) { diff --git a/backend/src/services/secret-folder/secret-folder-service.ts b/backend/src/services/secret-folder/secret-folder-service.ts index 842eb2bb7c..fbb40bb5d4 100644 --- a/backend/src/services/secret-folder/secret-folder-service.ts +++ b/backend/src/services/secret-folder/secret-folder-service.ts @@ -10,6 +10,7 @@ import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { OrderByDirection, OrgServiceActor } from "@app/lib/types"; import { buildFolderPath } from "@app/services/secret-folder/secret-folder-fns"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretFolderDALFactory } from "./secret-folder-dal"; @@ -29,7 +30,8 @@ type TSecretFolderServiceFactoryDep = { snapshotService: Pick; folderDAL: TSecretFolderDALFactory; projectEnvDAL: Pick; - folderVersionDAL: TSecretFolderVersionDALFactory; + folderVersionDAL: Pick; + folderCommitService: Pick; projectDAL: Pick; }; @@ -41,6 +43,7 @@ export const secretFolderServiceFactory = ({ permissionService, projectEnvDAL, folderVersionDAL, + folderCommitService, projectDAL }: TSecretFolderServiceFactoryDep) => { const createFolder = async ({ @@ -111,7 +114,7 @@ export const secretFolderServiceFactory = ({ }); parentFolderId = newFolders.at(-1)?.id as string; const docs = await folderDAL.insertMany(newFolders, tx); - await folderVersionDAL.insertMany( + const folderVersions = await folderVersionDAL.insertMany( docs.map((doc) => ({ name: doc.name, envId: doc.envId, @@ -120,6 +123,23 @@ export const secretFolderServiceFactory = ({ })), tx ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder created", + folderId: parentFolderId, + changes: folderVersions.map((fv) => ({ + type: "add", + folderVersionId: fv.id + })) + }, + tx + ); } } @@ -127,7 +147,7 @@ export const secretFolderServiceFactory = ({ { name, envId: env.id, version: 1, parentId: parentFolderId, description }, tx ); - await folderVersionDAL.create( + const folderVersion = await folderVersionDAL.create( { name: doc.name, envId: doc.envId, @@ -136,6 +156,25 @@ export const secretFolderServiceFactory = ({ }, tx ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder created", + folderId: parentFolderId, + changes: [ + { + type: "add", + folderVersionId: folderVersion.id + } + ] + }, + tx + ); return doc; }); @@ -225,7 +264,7 @@ export const secretFolderServiceFactory = ({ { name, description }, tx ); - await folderVersionDAL.create( + const folderVersion = await folderVersionDAL.create( { name: doc.name, envId: doc.envId, @@ -234,6 +273,25 @@ export const secretFolderServiceFactory = ({ }, tx ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder updated", + folderId: parentFolder.id, + changes: [ + { + type: "add", + folderVersionId: folderVersion.id + } + ] + }, + tx + ); if (!doc) { throw new NotFoundError({ message: `Failed to update folder with id '${id}', not found`, @@ -321,7 +379,7 @@ export const secretFolderServiceFactory = ({ { name, description }, tx ); - await folderVersionDAL.create( + const folderVersion = await folderVersionDAL.create( { name: doc.name, envId: doc.envId, @@ -330,6 +388,25 @@ export const secretFolderServiceFactory = ({ }, tx ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder updated", + folderId: parentFolder.id, + changes: [ + { + type: "add", + folderVersionId: folderVersion.id + } + ] + }, + tx + ); if (!doc) throw new NotFoundError({ message: `Failed to update folder with ID '${id}'`, name: "UpdateFolder" }); return doc; }); @@ -381,7 +458,30 @@ export const secretFolderServiceFactory = ({ }, tx ); + if (!doc) throw new NotFoundError({ message: `Failed to delete folder with ID '${idOrName}', not found` }); + + const folderVersions = await folderVersionDAL.findLatestFolderVersions([doc.id]); + + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder deleted", + folderId: parentFolder.id, + changes: [ + { + type: "delete", + folderVersionId: folderVersions[doc.id].id + } + ] + }, + tx + ); return doc; }); diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index 9425c5864c..437b1e1ebd 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -43,7 +43,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { const docs: Array = await (tx || db.replicaNode())( TableName.SecretFolderVersion ) - .whereIn("folderId", folderIds) + .whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds) .join( (tx || db)(TableName.SecretFolderVersion) .groupBy("folderId") diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts index 62b4ba3cc4..6b306cb514 100644 --- a/backend/src/services/secret-sync/secret-sync-queue.ts +++ b/backend/src/services/secret-sync/secret-sync-queue.ts @@ -58,6 +58,7 @@ import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/se import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; import { TAppConnectionDALFactory } from "../app-connection/app-connection-dal"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; export type TSecretSyncQueueFactory = ReturnType; @@ -93,6 +94,7 @@ type TSecretSyncQueueFactoryDep = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; }; type SecretSyncActionJob = Job< @@ -133,7 +135,8 @@ export const secretSyncQueueFactory = ({ secretVersionTagDAL, secretVersionV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }: TSecretSyncQueueFactoryDep) => { const appCfg = getConfig(); @@ -164,7 +167,8 @@ export const secretSyncQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const $updateManySecretsRawFn = updateManySecretsRawFnFactory({ @@ -180,7 +184,8 @@ export const secretSyncQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const $getInfisicalSecrets = async ( diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts index 6fdcadefff..eb2c778787 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts @@ -73,6 +73,7 @@ export const fnSecretBulkInsert = async ({ resourceMetadataDAL, secretTagDAL, secretVersionTagDAL, + folderCommitService, actor, tx }: TFnSecretBulkInsert) => { @@ -131,6 +132,30 @@ export const fnSecretBulkInsert = async ({ tx ); + const commitChanges = secretVersions + .filter(({ type }) => type === SecretType.Shared) + .map((sv) => ({ + type: "add", + secretVersionId: sv.id + })); + + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType || ActorType.PLATFORM, + metadata: { + id: actor?.actorId + } + }, + message: "Secret Creation", + folderId, + changes: commitChanges + }, + tx + ); + } + await secretDAL.upsertSecretReferences( inputSecrets.map(({ references = [], key }) => ({ secretId: newSecretGroupedByKeyName[key][0].id, @@ -185,6 +210,7 @@ export const fnSecretBulkUpdate = async ({ orgId, secretDAL, secretVersionDAL, + folderCommitService, secretTagDAL, secretVersionTagDAL, resourceMetadataDAL, @@ -259,6 +285,30 @@ export const fnSecretBulkUpdate = async ({ ), tx ); + + const commitChanges = secretVersions + .filter(({ type }) => type === SecretType.Shared) + .map((sv) => ({ + type: "add", + secretVersionId: sv.id + })); + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType || ActorType.PLATFORM, + metadata: { + id: actor?.actorId + } + }, + message: "Secret Update", + folderId, + changes: commitChanges + }, + tx + ); + } + await secretDAL.upsertSecretReferences( inputSecrets .filter(({ data: { references } }) => Boolean(references)) @@ -337,8 +387,11 @@ export const fnSecretBulkDelete = async ({ inputSecrets, tx, actorId, + actorType, secretDAL, - secretQueueService + secretQueueService, + folderCommitService, + secretVersionDAL }: TFnSecretBulkDelete) => { const deletedSecrets = await secretDAL.deleteMany( inputSecrets.map(({ type, secretKey }) => ({ @@ -358,6 +411,35 @@ export const fnSecretBulkDelete = async ({ ) ); + const secretVersions = await secretVersionDAL.findLatestVersionMany( + folderId, + deletedSecrets.map(({ id }) => id), + tx + ); + + const commitChanges = deletedSecrets + .filter(({ type }) => type === SecretType.Shared) + .map(({ id }) => ({ + type: "delete", + secretVersionId: secretVersions[id].id + })); + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType || ActorType.PLATFORM, + metadata: { + id: actorId + } + }, + message: "Secret Delete", + folderId, + changes: commitChanges + }, + tx + ); + } + return deletedSecrets; }; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index 1ef4a2d416..33d3cf0f1f 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -34,6 +34,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { ActorType } from "../auth/auth-type"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { KmsDataKey } from "../kms/kms-types"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; @@ -90,6 +91,7 @@ type TSecretV2BridgeServiceFactoryDep = { secretVersionTagDAL: Pick; secretTagDAL: TSecretTagDALFactory; permissionService: Pick; + folderCommitService: Pick; projectEnvDAL: Pick; folderDAL: Pick< TSecretFolderDALFactory, @@ -124,6 +126,7 @@ export const secretV2BridgeServiceFactory = ({ projectEnvDAL, secretTagDAL, secretVersionDAL, + folderCommitService, folderDAL, permissionService, snapshotService, @@ -327,6 +330,7 @@ export const secretV2BridgeServiceFactory = ({ resourceMetadataDAL, secretDAL, secretVersionDAL, + folderCommitService, secretTagDAL, secretVersionTagDAL, actor: { @@ -510,6 +514,7 @@ export const secretV2BridgeServiceFactory = ({ folderId, orgId: actorOrgId, resourceMetadataDAL, + folderCommitService, inputSecrets: [ { filter: { id: secretId }, @@ -650,6 +655,9 @@ export const secretV2BridgeServiceFactory = ({ projectId, folderId, actorId, + actorType: actor, + folderCommitService, + secretVersionDAL, secretDAL, secretQueueService, inputSecrets: [ @@ -1590,6 +1598,7 @@ export const secretV2BridgeServiceFactory = ({ orgId: actorOrgId, secretDAL, resourceMetadataDAL, + folderCommitService, secretVersionDAL, secretTagDAL, secretVersionTagDAL, @@ -1859,6 +1868,7 @@ export const secretV2BridgeServiceFactory = ({ const bulkUpdatedSecrets = await fnSecretBulkUpdate({ folderId, orgId: actorOrgId, + folderCommitService, tx, inputSecrets: secretsToUpdate.map((el) => { const originalSecret = secretsToUpdateInDBGroupedByKey[el.secretKey][0]; @@ -1928,6 +1938,7 @@ export const secretV2BridgeServiceFactory = ({ secretVersionDAL, secretTagDAL, secretVersionTagDAL, + folderCommitService, actor: { type: actor, actorId @@ -2061,6 +2072,8 @@ export const secretV2BridgeServiceFactory = ({ fnSecretBulkDelete({ secretDAL, secretQueueService, + folderCommitService, + secretVersionDAL, inputSecrets: inputSecrets.map(({ type, secretKey }) => ({ secretKey, type: type || SecretType.Shared @@ -2068,6 +2081,7 @@ export const secretV2BridgeServiceFactory = ({ projectId, folderId, actorId, + actorType: actor, tx }) ); @@ -2469,6 +2483,7 @@ export const secretV2BridgeServiceFactory = ({ tx, secretTagDAL, resourceMetadataDAL, + folderCommitService, secretVersionTagDAL, actor: { type: actor, @@ -2495,6 +2510,7 @@ export const secretV2BridgeServiceFactory = ({ folderId: destinationFolder.id, orgId: actorOrgId, resourceMetadataDAL, + folderCommitService, secretVersionDAL, secretDAL, tx, diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts index f4a27d4c56..b9ae742084 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -8,6 +8,7 @@ import { SecretsOrderBy } from "@app/services/secret/secret-types"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal"; @@ -178,6 +179,7 @@ export type TFnSecretBulkInsert = { secretVersionDAL: Pick; secretTagDAL: Pick; secretVersionTagDAL: Pick; + folderCommitService: Pick; actor?: { type: string; actorId: string; @@ -206,6 +208,7 @@ export type TFnSecretBulkUpdate = { secretVersionDAL: Pick; secretTagDAL: Pick; secretVersionTagDAL: Pick; + folderCommitService: Pick; actor?: { type: string; actorId: string; @@ -218,11 +221,14 @@ export type TFnSecretBulkDelete = { projectId: string; inputSecrets: Array<{ type: SecretType; secretKey: string }>; actorId: string; + actorType?: string; tx?: Knex; secretDAL: Pick; secretQueueService: { removeSecretReminder: (data: TRemoveSecretReminderDTO, tx?: Knex) => Promise; }; + folderCommitService: Pick; + secretVersionDAL: Pick; }; export type THandleReminderDTO = { diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index e5f3acdea4..96f89ab5f2 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -778,6 +778,7 @@ export const createManySecretsRawFnFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, + folderCommitService, kmsService, resourceMetadataDAL }: TCreateManySecretsRawFnFactory) => { @@ -850,6 +851,7 @@ export const createManySecretsRawFnFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService, tx }) ); @@ -942,6 +944,7 @@ export const updateManySecretsRawFnFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, resourceMetadataDAL, + folderCommitService, kmsService }: TUpdateManySecretsRawFnFactory) => { const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL); @@ -1032,7 +1035,8 @@ export const updateManySecretsRawFnFactory = ({ secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, - secretVersionTagDAL: secretVersionTagV2BridgeDAL + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService }) ); diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 714df0d3f8..87b2eb467d 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -35,6 +35,7 @@ import { TSecretSyncQueueFactory } from "@app/services/secret-sync/secret-sync-q import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { ActorType } from "../auth/auth-type"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TIntegrationDALFactory } from "../integration/integration-dal"; import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal"; import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service"; @@ -112,6 +113,7 @@ type TSecretQueueFactoryDep = { orgService: Pick; projectUserMembershipRoleDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; secretReminderRecipientsDAL: Pick< TSecretReminderRecipientsDALFactory, "delete" | "findUsersBySecretId" | "insertMany" | "transaction" @@ -178,7 +180,8 @@ export const secretQueueFactory = ({ projectKeyDAL, resourceMetadataDAL, secretReminderRecipientsDAL, - secretSyncQueue + secretSyncQueue, + folderCommitService }: TSecretQueueFactoryDep) => { const integrationMeter = opentelemetry.metrics.getMeter("Integrations"); const errorHistogram = integrationMeter.createHistogram("integration_secret_sync_errors", { @@ -366,7 +369,8 @@ export const secretQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const updateManySecretsRawFn = updateManySecretsRawFnFactory({ @@ -382,7 +386,8 @@ export const secretQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); /** diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 30e3dfafa4..91fc2eb6a3 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -14,6 +14,7 @@ import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-fold import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { ActorType } from "../auth/auth-type"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; @@ -441,6 +442,7 @@ export type TCreateManySecretsRawFnFactory = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; }; export type TCreateManySecretsRawFn = { @@ -478,6 +480,7 @@ export type TUpdateManySecretsRawFnFactory = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; }; export type TUpdateManySecretsRawFn = { From e58dbe853ef8c21f3055b7119a9d0293c2c0dffa Mon Sep 17 00:00:00 2001 From: carlosmonastyrski Date: Wed, 7 May 2025 08:38:19 -0300 Subject: [PATCH 3/4] Minor improvements on commits code quality --- .../20250505194916_add-pit-revamp-tables.ts | 15 ++++-------- .../secret-rotation-v2-service.ts | 3 +++ .../secret-rotation-queue.ts | 2 +- .../external-migration-fns.ts | 2 +- .../folder-checkpoint-dal.ts | 8 ++++--- .../folder-commit/folder-commit-dal.ts | 10 ++++---- .../folder-commit/folder-commit-service.ts | 23 ++++++++----------- .../secret-v2-bridge-types.ts | 4 ++-- 8 files changed, 32 insertions(+), 35 deletions(-) diff --git a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts index ea4c8e9bb9..9855d301e9 100644 --- a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts +++ b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts @@ -115,26 +115,21 @@ export async function down(knex: Knex): Promise { const hasFolderTreeCheckpointTable = await knex.schema.hasTable(TableName.FolderTreeCheckpoint); const hasFolderCheckpointTable = await knex.schema.hasTable(TableName.FolderCheckpoint); - if (hasFolderCheckpointResourcesTable) { - await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources); - await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources); - } - if (hasFolderTreeCheckpointResourcesTable) { await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpointResources); } - if (hasFolderTreeCheckpointTable) { - await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); - await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpoint); - } - if (hasFolderCheckpointResourcesTable) { await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources); await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources); } + if (hasFolderTreeCheckpointTable) { + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpoint); + } + if (hasFolderCheckpointTable) { await dropOnUpdateTrigger(knex, TableName.FolderCheckpoint); await knex.schema.dropTableIfExists(TableName.FolderCheckpoint); diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts index 979cb5b763..0a7a5c12c7 100644 --- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts +++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts @@ -943,6 +943,9 @@ export const secretRotationV2ServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, folderCommitService, + actor: { + type: ActorType.PLATFORM + }, secretTagDAL, resourceMetadataDAL }); diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts index ec92a736fc..95094f7672 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts @@ -347,7 +347,7 @@ export const secretRotationQueueFactory = ({ actor: { type: ActorType.PLATFORM }, - message: "Secret rotation", + message: "Changed by Secret rotation", folderId: secretVersions[0].folderId, changes: secretVersions.map((sv) => ({ type: "add", diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index f3c2326de1..645d58463f 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -623,7 +623,7 @@ export const importDataIntoInfisicalFn = async ({ id: actorId } }, - message: "External migration", + message: "Changed by external migration", folderId: parentEnv.rootFolderId, changes: [ { diff --git a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts index 5471f9b610..c598a8ea0c 100644 --- a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts +++ b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts @@ -3,7 +3,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { TableName, TFolderCheckpoints } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; -import { ormify, selectAllTableCols } from "@app/lib/knex"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; export type TFolderCheckpointDALFactory = ReturnType; @@ -34,7 +34,8 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { try { let query = (tx || db.replicaNode())(TableName.FolderCheckpoint) .join(TableName.FolderCommit, `${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id`) - .where({ folderId }) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderId }, TableName.FolderCommit)) .select(selectAllTableCols(TableName.FolderCheckpoint)) .select( db.ref("actorMetadata").withSchema(TableName.FolderCommit), @@ -60,7 +61,8 @@ export const folderCheckpointDALFactory = (db: TDbClient) => { try { const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint) .join(TableName.FolderCommit, `${TableName.FolderCheckpoint}.folderCommitId`, `${TableName.FolderCommit}.id`) - .where({ folderId }) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderId }, TableName.FolderCommit)) .select(selectAllTableCols(TableName.FolderCheckpoint)) .select( db.ref("actorMetadata").withSchema(TableName.FolderCommit), diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts index c6888f8cd4..1b454be028 100644 --- a/backend/src/services/folder-commit/folder-commit-dal.ts +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -13,10 +13,10 @@ export const folderCommitDALFactory = (db: TDbClient) => { const findByFolderId = async (folderId: string, tx?: Knex): Promise => { try { - const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) .where({ folderId }) .select(selectAllTableCols(TableName.FolderCommit)) - .orderBy("date", "desc"); + .orderBy("createdAt", "desc"); return docs; } catch (error) { throw new DatabaseError({ error, name: "FindByFolderId" }); @@ -25,7 +25,7 @@ export const folderCommitDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex): Promise => { try { - const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) .where({ id }) .select(selectAllTableCols(TableName.FolderCommit)) .first(); @@ -37,10 +37,10 @@ export const folderCommitDALFactory = (db: TDbClient) => { const findLatestCommit = async (folderId: string, tx?: Knex): Promise => { try { - const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) .where({ folderId }) .select(selectAllTableCols(TableName.FolderCommit)) - .orderBy("date", "desc") + .orderBy("createdAt", "desc") .first(); return doc; } catch (error) { diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts index c82de598c3..6a1dd2c89e 100644 --- a/backend/src/services/folder-commit/folder-commit-service.ts +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -77,18 +77,15 @@ export const folderCommitServiceFactory = ({ }, tx ); - for (const change of data.changes) { - // eslint-disable-next-line no-await-in-loop - await folderCommitChangesDAL.create( - { - folderCommitId: newCommit.id, - changeType: change.type, - secretVersionId: change.secretVersionId, - folderVersionId: change.folderVersionId - }, - tx - ); - } + await folderCommitChangesDAL.insertMany( + data.changes.map((change) => ({ + folderCommitId: newCommit.id, + changeType: change.type, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId + })), + tx + ); return newCommit; } catch (error) { @@ -96,7 +93,7 @@ export const folderCommitServiceFactory = ({ } }; - // Add a change to a commit and trigger checkpoints as needed + // Add a change to an existing commit const addCommitChange = async (data: TCommitChangeDTO, tx?: Knex) => { try { if (!data.secretVersionId && !data.folderVersionId) { diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts index b9ae742084..f4171b1a7f 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -182,7 +182,7 @@ export type TFnSecretBulkInsert = { folderCommitService: Pick; actor?: { type: string; - actorId: string; + actorId?: string; }; }; @@ -211,7 +211,7 @@ export type TFnSecretBulkUpdate = { folderCommitService: Pick; actor?: { type: string; - actorId: string; + actorId?: string; }; tx?: Knex; }; From e8d424bbb0961a44b94f45313ffb0e710e1ebead Mon Sep 17 00:00:00 2001 From: carlosmonastyrski Date: Thu, 8 May 2025 09:41:01 -0300 Subject: [PATCH 4/4] PIT: Add initialization and checkpoint logic --- ...010_pit-projects-commits-initialization.ts | 26 +++ backend/src/db/migrations/utils/services.ts | 41 +++++ backend/src/lib/config/env.ts | 3 + backend/src/server/routes/index.ts | 9 +- .../folder-commit/folder-commit-dal.ts | 32 ++-- .../folder-commit/folder-commit-service.ts | 153 +++++++++++++++++- .../secret-folder/secret-folder-dal.ts | 14 +- 7 files changed, 257 insertions(+), 21 deletions(-) create mode 100644 backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts diff --git a/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts b/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts new file mode 100644 index 0000000000..7ad865b10c --- /dev/null +++ b/backend/src/db/migrations/20250507185010_pit-projects-commits-initialization.ts @@ -0,0 +1,26 @@ +import { Knex } from "knex"; + +import { ProjectType, TableName } from "../schemas"; +import { getMigrationPITServices } from "./utils/services"; + +export async function up(knex: Knex): Promise { + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + if (hasFolderCommitTable) { + const { folderCommitService } = await getMigrationPITServices({ db: knex }); + const projects = await knex(TableName.Project).where({ version: 3, type: ProjectType.SecretManager }).select("id"); + await knex.transaction(async (tx) => { + for (const project of projects) { + // eslint-disable-next-line no-await-in-loop + await folderCommitService.initializeProject(project.id, tx); + } + }); + } +} + +export async function down(knex: Knex): Promise { + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + if (hasFolderCommitTable) { + // delete all existing entries + await knex(TableName.FolderCommit).del(); + } +} diff --git a/backend/src/db/migrations/utils/services.ts b/backend/src/db/migrations/utils/services.ts index 731f703e2c..6101126184 100644 --- a/backend/src/db/migrations/utils/services.ts +++ b/backend/src/db/migrations/utils/services.ts @@ -3,12 +3,23 @@ import { Knex } from "knex"; import { initializeHsmModule } from "@app/ee/services/hsm/hsm-fns"; import { hsmServiceFactory } from "@app/ee/services/hsm/hsm-service"; import { TKeyStoreFactory } from "@app/keystore/keystore"; +import { folderCheckpointDALFactory } from "@app/services/folder-checkpoint/folder-checkpoint-dal"; +import { folderCheckpointResourcesDALFactory } from "@app/services/folder-checkpoint-resources/folder-checkpoint-resources-dal"; +import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal"; +import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; +import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal"; +import { folderTreeCheckpointDALFactory } from "@app/services/folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { identityDALFactory } from "@app/services/identity/identity-dal"; import { internalKmsDALFactory } from "@app/services/kms/internal-kms-dal"; import { kmskeyDALFactory } from "@app/services/kms/kms-key-dal"; import { kmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal"; import { kmsServiceFactory } from "@app/services/kms/kms-service"; import { orgDALFactory } from "@app/services/org/org-dal"; import { projectDALFactory } from "@app/services/project/project-dal"; +import { secretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal"; +import { secretVersionV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; +import { userDALFactory } from "@app/services/user/user-dal"; import { TMigrationEnvConfig } from "./env-config"; @@ -50,3 +61,33 @@ export const getMigrationEncryptionServices = async ({ envConfig, db, keyStore } return { kmsService }; }; + +export const getMigrationPITServices = async ({ db }: { db: Knex }) => { + const projectDAL = projectDALFactory(db); + const folderCommitDAL = folderCommitDALFactory(db); + const folderCommitChangesDAL = folderCommitChangesDALFactory(db); + const folderCheckpointDAL = folderCheckpointDALFactory(db); + const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db); + const userDAL = userDALFactory(db); + const identityDAL = identityDALFactory(db); + const folderDAL = secretFolderDALFactory(db); + const folderVersionDAL = secretFolderVersionDALFactory(db); + const secretVersionV2BridgeDAL = secretVersionV2BridgeDALFactory(db); + const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db); + + const folderCommitService = folderCommitServiceFactory({ + folderCommitDAL, + folderCommitChangesDAL, + folderCheckpointDAL, + folderTreeCheckpointDAL, + userDAL, + identityDAL, + folderDAL, + folderVersionDAL, + secretVersionV2BridgeDAL, + projectDAL, + folderCheckpointResourcesDAL + }); + + return { folderCommitService }; +}; diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 907884433a..38e252b2a4 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -228,6 +228,9 @@ const envSchema = z DATADOG_SERVICE: zpStr(z.string().optional().default("infisical-core")), DATADOG_HOSTNAME: zpStr(z.string().optional()), + // PIT + CHECKPOINT_WINDOW: zpStr(z.string().optional().default("10")), + /* CORS ----------------------------------------------------------------------------- */ CORS_ALLOWED_ORIGINS: zpStr( diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 5ba8954fef..4367f69c5a 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -141,6 +141,7 @@ import { externalGroupOrgRoleMappingServiceFactory } from "@app/services/externa import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue"; import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; import { folderCheckpointDALFactory } from "@app/services/folder-checkpoint/folder-checkpoint-dal"; +import { folderCheckpointResourcesDALFactory } from "@app/services/folder-checkpoint-resources/folder-checkpoint-resources-dal"; import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal"; import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal"; @@ -559,6 +560,7 @@ export const registerRoutes = async ( const folderCommitChangesDAL = folderCommitChangesDALFactory(db); const folderCheckpointDAL = folderCheckpointDALFactory(db); + const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db); const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db); const folderCommitDAL = folderCommitDALFactory(db); const folderCommitService = folderCommitServiceFactory({ @@ -567,7 +569,12 @@ export const registerRoutes = async ( folderCheckpointDAL, folderTreeCheckpointDAL, userDAL, - identityDAL + identityDAL, + folderDAL, + folderVersionDAL, + secretVersionV2BridgeDAL, + projectDAL, + folderCheckpointResourcesDAL }); const scimService = scimServiceFactory({ licenseService, diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts index 1b454be028..68394ddb33 100644 --- a/backend/src/services/folder-commit/folder-commit-dal.ts +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -23,35 +23,43 @@ export const folderCommitDALFactory = (db: TDbClient) => { } }; - const findById = async (id: string, tx?: Knex): Promise => { + const findLatestCommit = async (folderId: string, tx?: Knex): Promise => { try { const doc = await (tx || db.replicaNode())(TableName.FolderCommit) - .where({ id }) + .where({ folderId }) .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") .first(); return doc; } catch (error) { - throw new DatabaseError({ error, name: "FindById" }); + throw new DatabaseError({ error, name: "FindLatestCommit" }); } }; - const findLatestCommit = async (folderId: string, tx?: Knex): Promise => { + const getNumberOfCommitsSince = async (folderId: string, folderCommitId: string, tx?: Knex): Promise => { try { - const doc = await (tx || db.replicaNode())(TableName.FolderCommit) - .where({ folderId }) - .select(selectAllTableCols(TableName.FolderCommit)) - .orderBy("createdAt", "desc") + const referencedCommit = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ id: folderCommitId }) + .select("commitId") .first(); - return doc; + + if (referencedCommit?.commitId) { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .where("commitId", ">", referencedCommit.commitId) + .count(); + return Number(doc?.[0].count); + } + return 0; } catch (error) { - throw new DatabaseError({ error, name: "FindLatestCommit" }); + throw new DatabaseError({ error, name: "getNumberOfCommitsSince" }); } }; return { ...restOfOrm, findByFolderId, - findById, - findLatestCommit + findLatestCommit, + getNumberOfCommitsSince }; }; diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts index 6a1dd2c89e..df834b88e7 100644 --- a/backend/src/services/folder-commit/folder-commit-service.ts +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -1,28 +1,37 @@ import { Knex } from "knex"; +import { TSecretFolders } from "@app/db/schemas"; +import { getConfig } from "@app/lib/config/env"; import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; import { ActorType } from "../auth/auth-type"; import { TFolderCheckpointDALFactory } from "../folder-checkpoint/folder-checkpoint-dal"; +import { TFolderCheckpointResourcesDALFactory } from "../folder-checkpoint-resources/folder-checkpoint-resources-dal"; import { TFolderCommitChangesDALFactory } from "../folder-commit-changes/folder-commit-changes-dal"; import { TFolderTreeCheckpointDALFactory } from "../folder-tree-checkpoint/folder-tree-checkpoint-dal"; import { TIdentityDALFactory } from "../identity/identity-dal"; +import { TProjectDALFactory } from "../project/project-dal"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; import { TUserDALFactory } from "../user/user-dal"; import { TFolderCommitDALFactory } from "./folder-commit-dal"; type TFolderCommitServiceFactoryDep = { folderCommitDAL: Pick< TFolderCommitDALFactory, - "create" | "findById" | "findByFolderId" | "findLatestCommit" | "transaction" + "create" | "findById" | "findByFolderId" | "findLatestCommit" | "transaction" | "getNumberOfCommitsSince" >; folderCommitChangesDAL: Pick; folderCheckpointDAL: Pick; - folderTreeCheckpointDAL: Pick< - TFolderTreeCheckpointDALFactory, - "create" | "findByProjectId" | "findLatestByProjectId" - >; + folderCheckpointResourcesDAL: Pick; + folderTreeCheckpointDAL: Pick; userDAL: Pick; identityDAL: Pick; + folderDAL: Pick; + folderVersionDAL: Pick; + secretVersionV2BridgeDAL: Pick; + projectDAL: Pick; }; export type TCreateCommitDTO = { @@ -54,9 +63,72 @@ export const folderCommitServiceFactory = ({ folderCommitChangesDAL, folderCheckpointDAL, folderTreeCheckpointDAL, + folderCheckpointResourcesDAL, userDAL, - identityDAL + identityDAL, + folderDAL, + folderVersionDAL, + secretVersionV2BridgeDAL, + projectDAL }: TFolderCommitServiceFactoryDep) => { + const appCfg = getConfig(); + + const getFolderResources = async (folderId: string, tx?: Knex) => { + const resources = []; + const subFolders = await folderDAL.findByParentId(folderId, tx); + if (subFolders.length > 0) { + const subFolderIds = subFolders.map((folder) => folder.id); + const folderVersions = await folderVersionDAL.findLatestFolderVersions(subFolderIds, tx); + resources.push(...Object.values(folderVersions).map((folderVersion) => ({ folderVersionId: folderVersion.id }))); + } + const secretVersions = await secretVersionV2BridgeDAL.findLatestVersionByFolderId(folderId, tx); + if (secretVersions.length > 0) { + resources.push(...secretVersions.map((secretVersion) => ({ secretVersionId: secretVersion.id }))); + } + return resources; + }; + + const createFolderCheckpoint = async ({ + folderId, + folderCommitId, + force = false, + tx + }: { + folderId: string; + folderCommitId?: string; + force?: boolean; + tx?: Knex; + }) => { + let latestCommitId = folderCommitId; + if (!latestCommitId) { + latestCommitId = (await folderCheckpointDAL.findLatestByFolderId(folderId, tx))?.folderCommitId; + } + if (!latestCommitId) { + throw new BadRequestError({ message: "Latest commit ID not found" }); + return; + } + if (!force) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getNumberOfCommitsSince(folderId, latestCommitId, tx); + if (commitsSinceLastCheckpoint < Number(appCfg.CHECKPOINT_WINDOW)) { + return; + } + } + const checkpointResources = await getFolderResources(folderId, tx); + + if (checkpointResources.length > 0) { + const newCheckpoint = await folderCheckpointDAL.create( + { + folderCommitId: latestCommitId + }, + tx + ); + await folderCheckpointResourcesDAL.insertMany( + checkpointResources.map((resource) => ({ folderCheckpointId: newCheckpoint.id, ...resource })), + tx + ); + } + }; + const createCommit = async (data: TCreateCommitDTO, tx?: Knex) => { const metadata = data.actor.metadata || {}; try { @@ -87,6 +159,7 @@ export const folderCommitServiceFactory = ({ tx ); + await createFolderCheckpoint({ folderId: data.folderId, folderCommitId: newCommit.id, tx }); return newCommit; } catch (error) { throw new DatabaseError({ error, name: "CreateCommit" }); @@ -149,6 +222,69 @@ export const folderCommitServiceFactory = ({ return folderTreeCheckpointDAL.findLatestByProjectId(projectId, tx); }; + const initializeFolder = async (folderId: string, tx?: Knex) => { + const folderResources = await getFolderResources(folderId, tx); + const changes = folderResources.map((resource) => ({ type: "add", ...resource })); + if (changes.length > 0) { + const newCommit = await createCommit( + { + actor: { + type: ActorType.PLATFORM + }, + message: "Initialized folder", + folderId, + changes + }, + tx + ); + await createFolderCheckpoint({ folderId, folderCommitId: newCommit.id, force: true, tx }); + } + }; + + function sortFoldersByHierarchy(folders: TSecretFolders[]) { + // Create a map for quick lookup of children by parent ID + const childrenMap: Map = new Map(); + folders.forEach((folder) => { + const { parentId } = folder; + if (!childrenMap.has(parentId || null)) { + childrenMap.set(parentId || null, []); + } + childrenMap.get(parentId || null)?.push(folder); + }); + + // Start with root folders (null parentId) + const result = []; + const rootFolders = childrenMap.get(null) || []; + + // Process each level of the hierarchy + let currentLevel = rootFolders; + result.push(...currentLevel); + + while (currentLevel.length > 0) { + const nextLevel = []; + + for (const folder of currentLevel) { + const children = childrenMap.get(folder.id) || []; + nextLevel.push(...children); + } + + result.push(...nextLevel); + currentLevel = nextLevel; + } + + return result; + } + + const initializeProject = async (projectId: string, tx?: Knex) => { + const project = await projectDAL.findById(projectId, tx); + if (!project) { + throw new NotFoundError({ message: `Project with ID ${projectId} not found` }); + } + const folders = await folderDAL.findByProjectId(projectId, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + await Promise.all(sortedFolders.map((folder) => initializeFolder(folder.id, tx))); + }; + return { createCommit, addCommitChange, @@ -158,7 +294,10 @@ export const folderCommitServiceFactory = ({ getCheckpointsByFolderId, getLatestCheckpoint, getTreeCheckpointsByProjectId, - getLatestTreeCheckpoint + getLatestTreeCheckpoint, + initializeFolder, + initializeProject, + createFolderCheckpoint }; }; diff --git a/backend/src/services/secret-folder/secret-folder-dal.ts b/backend/src/services/secret-folder/secret-folder-dal.ts index e136c5a50f..d6d5c4aad0 100644 --- a/backend/src/services/secret-folder/secret-folder-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-dal.ts @@ -488,6 +488,17 @@ export const secretFolderDALFactory = (db: TDbClient) => { } }; + const findByParentId = async (parentId: string, tx?: Knex) => { + try { + const folders = await (tx || db.replicaNode())(TableName.SecretFolder) + .where({ parentId }) + .select(selectAllTableCols(TableName.SecretFolder)); + return folders; + } catch (error) { + throw new DatabaseError({ error, name: "findByParentId" }); + } + }; + return { ...secretFolderOrm, update, @@ -499,6 +510,7 @@ export const secretFolderDALFactory = (db: TDbClient) => { findClosestFolder, findByProjectId, findByMultiEnv, - findByEnvsDeep + findByEnvsDeep, + findByParentId }; };