From e573b0e0a1de0fb88a08738784331d82f47d70aa Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Wed, 23 Apr 2025 15:59:49 +1000 Subject: [PATCH 1/2] Options to allow not pushing keys such as for production DB migrations where keys are already managed natively Signed-off-by: Peter Baker --- api/README.md | 15 +++++++++++--- api/src/couchdb/index.ts | 42 ++++++++++++++++++++++++++------------ api/src/scripts/migrate.ts | 20 +++++++++++++++++- 3 files changed, 60 insertions(+), 17 deletions(-) diff --git a/api/README.md b/api/README.md index f4913b651..c8ab2f665 100644 --- a/api/README.md +++ b/api/README.md @@ -66,10 +66,19 @@ npm run watch instead, which will monitor for changes with `nodemon`. -## Initialisation +## Initialisation and migration Once the services are up and running we need to initialise the CouchDB -database. This is done with the migrate script: +database. + +If you want to include pushing the keys (only recommended for local development): + +```bash +npm run migrate --keys +``` + +If you just want to migrate databases without applying any public key +configuration, then exclude this flag ```bash npm run migrate @@ -82,7 +91,7 @@ run in the container rather than from your local environment: docker compose exec conductor npm run migrate ``` -This ensures that the correct CouchDB URL is used to access the database. The same +This ensures that the correct CouchDB URL is used to access the database. The same applies for the commands below. For development, there is also a script that will populate the database with projects (notebooks diff --git a/api/src/couchdb/index.ts b/api/src/couchdb/index.ts index 317503f80..db3422c6b 100644 --- a/api/src/couchdb/index.ts +++ b/api/src/couchdb/index.ts @@ -449,22 +449,31 @@ export const initialiseDataDb = async ({ }; /** - * Critical method which initialises all databases, including remotely on the configured couch instance. + * Critical method which initialises all databases, including remotely on the + * configured couch instance. * - * This systematically generates a set of initialisation content from the data model, then applies this initialisation using a helper method in the data model. + * This systematically generates a set of initialisation content from the data + * model, then applies this initialisation using a helper method in the data + * model. * - * Some local information is injected as part of the config generation step - e.g. conductor name/description. + * Some local information is injected as part of the config generation step - + * e.g. conductor name/description. * * Also initialises keys based on the configured key service. * * If force = true, documents will always be written, even if it already exists. * + * If pushKeys = true, will update the public keys + * * @param force Write on clash */ export const initialiseDbAndKeys = async ({ force = false, + pushKeys = true, }: { force?: boolean; + // Should we push the key configuration? + pushKeys?: boolean; }) => { // Are we in a testing environment? const isTesting = process.env.NODE_ENV === 'test'; @@ -622,15 +631,19 @@ export const initialiseDbAndKeys = async ({ await initialiseDataDb({projectId, force}); } - // Setup keys - try { - await initialiseJWTKey(); - } catch (error) { - console.log( - 'something wrong PUTing jwt_keys into the db configuration...', - error - ); - throw error; + if (pushKeys) { + // Setup keys + try { + await initialiseJWTKey(); + } catch (error) { + console.log( + 'something wrong PUTing jwt_keys into the db configuration...', + error + ); + throw error; + } + } else { + console.log('Not pushing key configuration.'); } }; @@ -639,10 +652,13 @@ export const initialiseDbAndKeys = async ({ */ export const initialiseAndMigrateDBs = async ({ force = false, + pushKeys = true, }: { force?: boolean; + // Should we push the key configuration? + pushKeys?: boolean; }) => { - await initialiseDbAndKeys({force}); + await initialiseDbAndKeys({force, pushKeys}); let dbs: {dbType: DATABASE_TYPE; dbName: string; db: PouchDB.Database}[] = [ {db: getAuthDB(), dbType: DatabaseType.AUTH, dbName: AUTH_DB_NAME}, diff --git a/api/src/scripts/migrate.ts b/api/src/scripts/migrate.ts index 823bde670..f96e6eb3d 100644 --- a/api/src/scripts/migrate.ts +++ b/api/src/scripts/migrate.ts @@ -1,9 +1,26 @@ /* eslint-disable n/no-process-exit */ import {initialiseAndMigrateDBs} from '../couchdb'; +/** + * Main function to run database initialization and migration + * Accepts optional --keys flag to control whether public keys should be pushed + */ const main = async () => { try { - await initialiseAndMigrateDBs({force: true}); + // Check if --keys flag is present in command line arguments + const pushKeys = process.argv.includes('--keys'); + + // Log whether keys will be configured + console.log( + `Public keys will ${pushKeys ? '' : 'not '}be configured during migration` + ); + + // Run database initialization and migration with force and pushKeys parameters + await initialiseAndMigrateDBs({ + force: true, + pushKeys: pushKeys, + }); + console.log('Migration completed successfully'); process.exit(0); } catch (error) { @@ -12,4 +29,5 @@ const main = async () => { } }; +// Execute the main function main(); From 8ad564e49652fed449d2758ed17afec194e6682f Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Wed, 23 Apr 2025 16:03:47 +1000 Subject: [PATCH 2/2] Adding to local dev with keys since it's the first creation Signed-off-by: Peter Baker --- localdev.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/localdev.sh b/localdev.sh index 34e141932..20cebdb31 100755 --- a/localdev.sh +++ b/localdev.sh @@ -103,8 +103,8 @@ else fi echo "Initialising database using API container" -echo ">docker compose exec api sh -c \"cd api && npm run migrate\"" -docker compose exec api sh -c "cd api && npm run migrate" +echo ">docker compose exec api sh -c \"cd api && npm run migrate --keys\"" +docker compose exec api sh -c "cd api && npm run migrate --keys" echo "Service is setup, to load notebooks and templates follow the below steps"