Skip to content

Commit fae8a87

Browse files
authored
feat(storage): change mechanics of public/private containers to be aligned with expectations (#1414)
feat(storage): enable CDN-aware configuration
1 parent 368a369 commit fae8a87

File tree

15 files changed

+510
-45
lines changed

15 files changed

+510
-45
lines changed

modules/storage/src/Storage.ts

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,12 @@ import path from 'path';
1616
import { status } from '@grpc/grpc-js';
1717
import { isEmpty, isNil } from 'lodash-es';
1818
import { runMigrations } from './migrations/index.js';
19+
import { migratePublicContainers } from './migrations/publicContainerMigration.js';
20+
import {
21+
CdnConfiguration,
22+
cdnConfigsAreEqual,
23+
migrateCdnConfigChanges,
24+
} from './migrations/cdnConfigMigration.js';
1925
import {
2026
CreateFileByUrlRequest,
2127
CreateFileRequest,
@@ -71,6 +77,8 @@ export default class Storage extends ManagedModule<Config> {
7177
private _adminFileHandlers: AdminFileHandlers;
7278
private enableAuthRoutes: boolean = false;
7379
private storageParamAdapter: StorageParamAdapter;
80+
private publicContainerMigrationRan: boolean = false;
81+
private previousCdnConfig: CdnConfiguration = {};
7482

7583
constructor() {
7684
super('storage');
@@ -129,6 +137,19 @@ export default class Storage extends ManagedModule<Config> {
129137
});
130138
this._fileHandlers.updateProvider(this.storageProvider);
131139
this._adminFileHandlers.updateProvider(this.storageProvider);
140+
// Run the public container migration once after provider is configured
141+
if (!this.publicContainerMigrationRan && provider !== 'local') {
142+
this.publicContainerMigrationRan = true;
143+
await migratePublicContainers(this.grpcSdk, this.storageProvider);
144+
}
145+
// Detect CDN config changes and run migration if needed (order-independent comparison)
146+
const currentCdnConfig = (ConfigController.getInstance().config.cdnConfiguration ??
147+
{}) as CdnConfiguration;
148+
if (!cdnConfigsAreEqual(this.previousCdnConfig, currentCdnConfig)) {
149+
await migrateCdnConfigChanges(this.grpcSdk, this.previousCdnConfig);
150+
// Deep copy to store as previous
151+
this.previousCdnConfig = { ...currentCdnConfig };
152+
}
132153
this.adminRouter = new AdminRoutes(
133154
this.grpcServer,
134155
this.grpcSdk,

modules/storage/src/admin/adminFile.ts

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import {
1414
_createFileUploadUrl,
1515
_updateFile,
1616
_updateFileUploadUrl,
17+
applyCdnHost,
1718
deepPathHandler,
1819
normalizeFolderPath,
1920
storeNewFile,
@@ -208,13 +209,14 @@ export class AdminFileHandlers {
208209
}
209210
return { redirect: found.url };
210211
}
211-
let options: UrlOptions = {
212+
const options: UrlOptions = {
212213
download: call.request.params.download ?? false,
213214
fileName: found.alias ?? found.name,
214215
};
215-
const url = await this.storageProvider
216+
const rawUrl = await this.storageProvider
216217
.container(found.container)
217218
.getSignedUrl((found.folder === '/' ? '' : found.folder) + found.name, options);
219+
const url = applyCdnHost(rawUrl, found.container);
218220

219221
if (!call.request.params.redirect) {
220222
return { result: url };
@@ -275,7 +277,7 @@ export class AdminFileHandlers {
275277
}
276278
const exists = await this.storageProvider.containerExists(container);
277279
if (!exists) {
278-
await this.storageProvider.createContainer(container);
280+
await this.storageProvider.createContainer(container, isPublic);
279281
}
280282
await _StorageContainer.getInstance().create({
281283
name: container,

modules/storage/src/admin/index.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -452,7 +452,7 @@ export class AdminRoutes {
452452
const exists = await this.fileHandlers.storage.containerExists(name);
453453

454454
if (!exists) {
455-
await this.fileHandlers.storage.createContainer(name);
455+
await this.fileHandlers.storage.createContainer(name, isPublic);
456456
}
457457
container = await _StorageContainer.getInstance().create({
458458
name,

modules/storage/src/config/config.ts

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,4 +83,15 @@ export default {
8383
doc: 'Defines if a suffix should be appended to the name of a file, upon creation, when name already exists',
8484
default: false,
8585
},
86+
/**
87+
* cdnConfiguration: {
88+
* my_container: 'cdn.myhost.com'
89+
* }
90+
* Maps container names to their CDN host (single CDN per container)
91+
*/
92+
cdnConfiguration: {
93+
format: 'Object',
94+
doc: 'Defines CDN host for each container (single CDN URL per container)',
95+
default: {},
96+
},
8697
};

modules/storage/src/handlers/file.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
_createFileUploadUrl,
1616
_updateFile,
1717
_updateFileUploadUrl,
18+
applyCdnHost,
1819
deepPathHandler,
1920
normalizeFolderPath,
2021
storeNewFile,
@@ -279,9 +280,10 @@ export class FileHandlers {
279280
download: call.request.params.download ?? false,
280281
fileName: found.alias ?? found.name,
281282
};
282-
const url = await this.storageProvider
283+
const rawUrl = await this.storageProvider
283284
.container(found.container)
284285
.getSignedUrl((found.folder === '/' ? '' : found.folder) + found.name, options);
286+
const url = applyCdnHost(rawUrl, found.container);
285287

286288
if (!call.request.params.redirect) {
287289
return { result: url };
@@ -343,7 +345,7 @@ export class FileHandlers {
343345
}
344346
const exists = await this.storageProvider.containerExists(container);
345347
if (!exists) {
346-
await this.storageProvider.createContainer(container);
348+
await this.storageProvider.createContainer(container, isPublic);
347349
}
348350
await _StorageContainer.getInstance().create({
349351
name: container,

modules/storage/src/interfaces/IStorageProvider.ts

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,16 @@ export interface IStorageProvider {
1717
/**
1818
* Used to create a new container
1919
* @param name For the container
20+
* @param isPublic Whether the container should be publicly accessible
2021
*/
21-
createContainer(name: string): Promise<boolean | Error>;
22+
createContainer(name: string, isPublic?: boolean): Promise<boolean | Error>;
23+
24+
/**
25+
* Used to set the public access level of an existing container
26+
* @param name For the container
27+
* @param isPublic Whether the container should be publicly accessible
28+
*/
29+
setContainerPublicAccess(name: string, isPublic: boolean): Promise<boolean | Error>;
2230

2331
/**
2432
* Used to switch the current container.
@@ -41,7 +49,13 @@ export interface IStorageProvider {
4149

4250
getSignedUrl(fileName: string, options?: UrlOptions): Promise<any | Error>;
4351

44-
getPublicUrl(fileName: string): Promise<any | Error>;
52+
/**
53+
* Gets a publicly accessible URL for a file.
54+
* @param fileName The file name/path
55+
* @param containerIsPublic Whether the container is publicly accessible.
56+
* If true, returns a plain URL. If false, returns a long-lived signed URL.
57+
*/
58+
getPublicUrl(fileName: string, containerIsPublic?: boolean): Promise<string | Error>;
4559

4660
getUploadUrl(fileName: string): Promise<string | Error>;
4761
}
Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk';
2+
import { ConfigController } from '@conduitplatform/module-tools';
3+
import { File } from '../models/index.js';
4+
import { applyCdnHost } from '../utils/index.js';
5+
6+
export type CdnConfiguration = Record<string, string>;
7+
8+
/**
9+
* Compares two CDN configurations and returns the list of containers
10+
* that have actually changed (added, removed, or value modified).
11+
* Order-independent comparison.
12+
*/
13+
export function getChangedContainers(
14+
previousConfig: CdnConfiguration,
15+
currentConfig: CdnConfiguration,
16+
): string[] {
17+
const changedContainers: string[] = [];
18+
const allContainers = new Set([
19+
...Object.keys(previousConfig),
20+
...Object.keys(currentConfig),
21+
]);
22+
23+
for (const container of allContainers) {
24+
const prevValue = previousConfig[container];
25+
const currValue = currentConfig[container];
26+
27+
// Changed if: value differs, was added, or was removed
28+
if (prevValue !== currValue) {
29+
changedContainers.push(container);
30+
}
31+
}
32+
33+
return changedContainers;
34+
}
35+
36+
/**
37+
* Checks if two CDN configurations are equivalent (order-independent).
38+
*/
39+
export function cdnConfigsAreEqual(
40+
configA: CdnConfiguration | undefined,
41+
configB: CdnConfiguration | undefined,
42+
): boolean {
43+
const a = configA ?? {};
44+
const b = configB ?? {};
45+
46+
const keysA = Object.keys(a);
47+
const keysB = Object.keys(b);
48+
49+
// Different number of keys = not equal
50+
if (keysA.length !== keysB.length) {
51+
return false;
52+
}
53+
54+
// Check each key has the same value
55+
for (const key of keysA) {
56+
if (a[key] !== b[key]) {
57+
return false;
58+
}
59+
}
60+
61+
return true;
62+
}
63+
64+
/**
65+
* Migration to update file URLs when CDN configuration changes.
66+
* Compares the new CDN config with the previous one and updates affected files.
67+
*/
68+
export async function migrateCdnConfigChanges(
69+
grpcSdk: ConduitGrpcSdk,
70+
previousCdnConfig: CdnConfiguration,
71+
): Promise<void> {
72+
const logger = ConduitGrpcSdk.Logger;
73+
const currentConfig = ConfigController.getInstance().config;
74+
const currentCdnConfig = (currentConfig.cdnConfiguration ?? {}) as CdnConfiguration;
75+
76+
// Find containers with changed CDN configuration (order-independent)
77+
const changedContainers = getChangedContainers(previousCdnConfig, currentCdnConfig);
78+
79+
if (changedContainers.length === 0) {
80+
return;
81+
}
82+
83+
logger.log(
84+
`CDN configuration changed for ${
85+
changedContainers.length
86+
} container(s): ${changedContainers.join(', ')}`,
87+
);
88+
89+
// Update URLs for files in changed containers
90+
for (const container of changedContainers) {
91+
try {
92+
// Find all public files in this container
93+
const allFiles = await File.getInstance().findMany({
94+
container,
95+
isPublic: true,
96+
});
97+
98+
// Filter to only files with a sourceUrl
99+
const files = allFiles.filter(f => f.sourceUrl);
100+
101+
if (files.length === 0) {
102+
logger.log(`No public files to update in container "${container}"`);
103+
continue;
104+
}
105+
106+
logger.log(`Updating ${files.length} file(s) in container "${container}"`);
107+
108+
// Update each file's URL with the new CDN host
109+
for (const file of files) {
110+
const newUrl = applyCdnHost(file.sourceUrl, container);
111+
await File.getInstance().findByIdAndUpdate(file._id, {
112+
url: newUrl,
113+
});
114+
}
115+
116+
logger.log(`Successfully updated files in container "${container}"`);
117+
} catch (error) {
118+
logger.error(
119+
`Failed to update files in container "${container}": ${(error as Error).message}`,
120+
);
121+
}
122+
}
123+
124+
logger.log('CDN configuration migration completed');
125+
}
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import { ConduitGrpcSdk } from '@conduitplatform/grpc-sdk';
2+
import { IStorageProvider } from '../interfaces/index.js';
3+
import { _StorageContainer } from '../models/index.js';
4+
5+
/**
6+
* Migration to update existing public containers to actually be public on the cloud provider.
7+
* This is needed because previously, containers were only marked as public in the database
8+
* but not actually configured as public on the cloud storage provider.
9+
*/
10+
export async function migratePublicContainers(
11+
grpcSdk: ConduitGrpcSdk,
12+
storageProvider: IStorageProvider,
13+
): Promise<void> {
14+
const logger = ConduitGrpcSdk.Logger;
15+
16+
try {
17+
// Find all containers marked as public in the database
18+
const publicContainers = await _StorageContainer.getInstance().findMany({
19+
isPublic: true,
20+
});
21+
22+
if (publicContainers.length === 0) {
23+
logger.log('No public containers found to migrate');
24+
return;
25+
}
26+
27+
logger.log(
28+
`Found ${publicContainers.length} public container(s) to migrate to cloud provider`,
29+
);
30+
31+
for (const container of publicContainers) {
32+
try {
33+
// Check if container exists on the provider
34+
const exists = await storageProvider.containerExists(container.name);
35+
if (!exists) {
36+
logger.warn(
37+
`Container "${container.name}" does not exist on cloud provider, skipping`,
38+
);
39+
continue;
40+
}
41+
42+
// Set the container to public on the cloud provider
43+
await storageProvider.setContainerPublicAccess(container.name, true);
44+
logger.log(
45+
`Successfully set public access for container "${container.name}" on cloud provider`,
46+
);
47+
} catch (error) {
48+
logger.error(
49+
`Failed to set public access for container "${container.name}": ${
50+
(error as Error).message
51+
}`,
52+
);
53+
}
54+
}
55+
56+
logger.log('Public container migration completed');
57+
} catch (error) {
58+
logger.error(`Public container migration failed: ${(error as Error).message}`);
59+
}
60+
}

modules/storage/src/models/File.schema.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ const schema: ConduitModel = {
2828
default: false,
2929
},
3030
url: TYPE.String,
31+
sourceUrl: TYPE.String,
3132
mimeType: TYPE.String,
3233
createdAt: TYPE.Date,
3334
updatedAt: TYPE.Date,
@@ -56,6 +57,7 @@ export class File extends ConduitActiveSchema<File> {
5657
size!: number;
5758
isPublic?: boolean;
5859
url!: string;
60+
sourceUrl?: string;
5961
mimeType!: string;
6062
createdAt!: Date;
6163
updatedAt!: Date;

0 commit comments

Comments
 (0)