Skip to content

chore: add remote config data layer classes and migration #1872

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 30 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 28 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
6a4898f
first implementation exercise
jeromy-cannon Apr 2, 2025
ba26fcc
refactor: update LayeredModelConfigSource to implement ModelConfigSou…
jeromy-cannon Apr 2, 2025
7c465f0
fix: update yaml parsing and stringifying methods to use the yaml pac…
jeromy-cannon Apr 17, 2025
a584c79
feat: implement YamlConfigMapStorageBackend for YAML object storage s…
jeromy-cannon Apr 17, 2025
2366ed0
docs: add JSDoc comment for YamlConfigMapStorageBackend class
jeromy-cannon Apr 17, 2025
b07c7ea
refactor: rename CTObjectMapper to ClassToObjectMapper and update ref…
jeromy-cannon Apr 17, 2025
0602da8
refactor: update name getter to return constructor name dynamically
jeromy-cannon Apr 17, 2025
16bca60
feat: add RemoteConfigSource, RemoteConfigSchema, and migration for r…
jeromy-cannon Apr 17, 2025
90e482d
feat: implement migration logic for RemoteConfigV1, updating metadata…
jeromy-cannon Apr 17, 2025
a847c99
feat: add TODO for completing remote config components after deployment
jeromy-cannon Apr 17, 2025
3063d6d
save
JeffreyDallas Apr 20, 2025
34040a6
first implementation exercise
jeromy-cannon Apr 2, 2025
4c86981
refactor: update LayeredModelConfigSource to implement ModelConfigSou…
jeromy-cannon Apr 2, 2025
4e5e3a1
fix: update yaml parsing and stringifying methods to use the yaml pac…
jeromy-cannon Apr 17, 2025
3d7e1f5
feat: implement YamlConfigMapStorageBackend for YAML object storage s…
jeromy-cannon Apr 17, 2025
22b12c2
docs: add JSDoc comment for YamlConfigMapStorageBackend class
jeromy-cannon Apr 17, 2025
df13d38
refactor: rename CTObjectMapper to ClassToObjectMapper and update ref…
jeromy-cannon Apr 17, 2025
40bb8ce
refactor: update name getter to return constructor name dynamically
jeromy-cannon Apr 17, 2025
50d38e5
feat: add RemoteConfigSource, RemoteConfigSchema, and migration for r…
jeromy-cannon Apr 17, 2025
1c4e54b
feat: implement migration logic for RemoteConfigV1, updating metadata…
jeromy-cannon Apr 17, 2025
c48989e
feat: add TODO for completing remote config components after deployment
jeromy-cannon Apr 17, 2025
db6e122
feat: update remote configuration with new component structure and ve…
jeromy-cannon Apr 21, 2025
4f46a81
Merge branch 'data-layer-remote-config-and-config-map' of https://git…
JeffreyDallas Apr 22, 2025
907c09a
fix bug in function readBytes
JeffreyDallas Apr 22, 2025
1761583
save
JeffreyDallas Apr 23, 2025
ea3afcd
fix remote config test yaml
JeffreyDallas Apr 23, 2025
f10f6ca
take out common expect as functions
JeffreyDallas Apr 23, 2025
2abde34
Merge commit '6785846eeb29efe77831626791dc2e1060ae01c2' into data-lay…
JeffreyDallas Apr 23, 2025
4e8e74c
Merge remote-tracking branch 'origin/main' into data-layer-remote-con…
instamenta Apr 30, 2025
72b0853
task format
instamenta Apr 30, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/core/dependency-injection/container-init.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import {NodeCommandTasks} from '../../commands/node/tasks.js';
import {ClusterCommandConfigs} from '../../commands/cluster/configs.js';
import {NodeCommandConfigs} from '../../commands/node/configs.js';
import {ErrorHandler} from '../error-handler.js';
import {CTObjectMapper} from '../../data/mapper/impl/ct-object-mapper.js';
import {ClassToObjectMapper} from '../../data/mapper/impl/class-to-object-mapper.js';
import {HelmExecutionBuilder} from '../../integration/helm/execution/helm-execution-builder.js';
import {DefaultHelmClient} from '../../integration/helm/impl/default-helm-client.js';
import {HelpRenderer} from '../help-renderer.js';
Expand Down Expand Up @@ -92,7 +92,7 @@ export class Container {
}

// Data Layer ObjectMapper
container.register(InjectTokens.ObjectMapper, {useClass: CTObjectMapper}, {lifecycle: Lifecycle.Singleton});
container.register(InjectTokens.ObjectMapper, {useClass: ClassToObjectMapper}, {lifecycle: Lifecycle.Singleton});
container.register(InjectTokens.KeyFormatter, {useValue: ConfigKeyFormatter.instance()});

// Data Layer Config
Expand Down
82 changes: 82 additions & 0 deletions src/data/backend/impl/config-map-storage-backend.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
// SPDX-License-Identifier: Apache-2.0

import {type StorageBackend} from '../api/storage-backend.js';
import {StorageOperation} from '../api/storage-operation.js';
import {MissingArgumentError} from '../../../core/errors/missing-argument-error.js';
import {type ConfigMap} from '../../../integration/kube/resources/config-map/config-map.js';
import {StorageBackendError} from '../api/storage-backend-error.js';

/**
* ConfigMapStorageBackend is a storage backend that uses a {@link ConfigMap} to store data.
* The key will be the name of the property within the data object within the ConfigMap.
*/
export class ConfigMapStorageBackend implements StorageBackend {
public constructor(private readonly configMap: ConfigMap) {
if (!this.configMap) {
throw new MissingArgumentError('ConfigMapStorageBackend is missing the configMap argument');
}
}

public async delete(key: string): Promise<void> {
try {
const data: Record<string, string> = this.configMap.data;

if (data && Object.keys(data).length > 0 && data.hasOwnProperty(key)) {
delete data[key];
} else {
throw new StorageBackendError(`key: ${key} not found in config map`);
}
} catch (error) {
throw error instanceof StorageBackendError
? error
: new StorageBackendError(`error deleting config map data key: ${key}`, error);
}
}

public isSupported(op: StorageOperation): boolean {
switch (op) {
case StorageOperation.List:
case StorageOperation.ReadBytes:
case StorageOperation.WriteBytes:
case StorageOperation.Delete: {
return true;
}
default: {
return false;
}
}
}

public async list(): Promise<string[]> {
const data: Record<string, string> = this.configMap.data;

return data ? Object.keys(data) : [];
}

public async readBytes(key: string): Promise<Buffer> {
try {
const data: Record<string, string> = this.configMap.data;

if (data && Object.keys(data).length > 0) {
const value: string = data[key];
return Buffer.from(value, 'utf8');
} else {
throw new StorageBackendError(`config map is empty: ${key}`);
}
} catch (error) {
throw error instanceof StorageBackendError
? error
: new StorageBackendError(`error reading config map: ${key}`, error);
}
}

public async writeBytes(key: string, data: Buffer): Promise<void> {
try {
this.configMap.data[key] = data.toString('utf8');
} catch (error) {
throw error instanceof StorageBackendError
? error
: new StorageBackendError(`error writing config map: ${key}`, error);
}
}
}
43 changes: 43 additions & 0 deletions src/data/backend/impl/yaml-config-map-storage-backend.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
// SPDX-License-Identifier: Apache-2.0

import yaml from 'yaml';
import {ConfigMapStorageBackend} from './config-map-storage-backend.js';
import {type ObjectStorageBackend} from '../api/object-storage-backend.js';
import {StorageBackendError} from '../api/storage-backend-error.js';
import {IllegalArgumentError} from '../../../core/errors/illegal-argument-error.js';

/**
* YamlConfigMapStorageBackend is a storage backend that uses a {@link ConfigMap} to store data.
* The key will be the name of the property within the data object within the ConfigMap.
*/
export class YamlConfigMapStorageBackend extends ConfigMapStorageBackend implements ObjectStorageBackend {
public async readObject(key: string): Promise<object> {
const data: Buffer = await this.readBytes(key);
if (!data) {
throw new StorageBackendError(`failed to read key: ${key} from config map`);
}

if (data.length === 0) {
throw new StorageBackendError(`data is empty for key: ${key}`);
}

try {
return yaml.parse(data.toString('utf8'));
} catch (error) {
throw new StorageBackendError(`error parsing yaml from key: ${key}`, error);
}
}

public async writeObject(key: string, data: object): Promise<void> {
if (!data) {
throw new IllegalArgumentError('data must not be null or undefined');
}

try {
const yamlData: string = yaml.stringify(data, {sortMapEntries: true});
await this.writeBytes(key, Buffer.from(yamlData, 'utf8'));
} catch (error) {
throw new StorageBackendError(`error writing yaml for key: ${key} to config map`, error);
}
}
}
8 changes: 4 additions & 4 deletions src/data/backend/impl/yaml-file-storage-backend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import {type ObjectStorageBackend} from '../api/object-storage-backend.js';
import {FileStorageBackend} from './file-storage-backend.js';
import {StorageBackendError} from '../api/storage-backend-error.js';
import {parse, stringify} from 'yaml';
import yaml from 'yaml';
import {IllegalArgumentError} from '../../../core/errors/illegal-argument-error.js';
import {PathEx} from '../../../business/utils/path-ex.js';

Expand All @@ -25,7 +25,7 @@ export class YamlFileStorageBackend extends FileStorageBackend implements Object
}

try {
return parse(data.toString('utf-8'));
return yaml.parse(data.toString('utf8'));
} catch (error) {
throw new StorageBackendError(`error parsing yaml file: ${filePath}`, error);
}
Expand All @@ -38,8 +38,8 @@ export class YamlFileStorageBackend extends FileStorageBackend implements Object

const filePath: string = PathEx.join(this.basePath, key);
try {
const yamlData: string = stringify(data, {sortMapEntries: true});
await this.writeBytes(key, Buffer.from(yamlData, 'utf-8'));
const yamlData: string = yaml.stringify(data, {sortMapEntries: true});
await this.writeBytes(key, Buffer.from(yamlData, 'utf8'));
} catch (error) {
throw new StorageBackendError(`error writing yaml file: ${filePath}`, error);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@ import {ConfigurationError} from '../api/configuration-error.js';
import {IllegalArgumentError} from '../../../business/errors/illegal-argument-error.js';
import {Forest} from '../../key/lexer/forest.js';
import {type ObjectStorageBackend} from '../../backend/api/object-storage-backend.js';
import {type ModelConfigSource} from '../spi/model-config-source.js';

export abstract class LayeredModelConfigSource<T extends object>
extends LayeredConfigSource
implements LayeredModelConfigSource<T>
implements ModelConfigSource<T>
{
private _modelData: T;

Expand Down
2 changes: 1 addition & 1 deletion src/data/configuration/impl/local-config-source.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ export class LocalConfigSource extends MutableModelConfigSource<LocalConfig> imp
}

public get name(): string {
return 'LocalConfigSource';
return this.constructor.name;
}

public get ordinal(): number {
Expand Down
26 changes: 26 additions & 0 deletions src/data/configuration/impl/remote-config-source.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// SPDX-License-Identifier: Apache-2.0

import {MutableModelConfigSource} from './mutable-model-config-source.js';
import {type RemoteConfig} from '../../schema/model/remote/remote-config.js';
import {type Refreshable} from '../spi/refreshable.js';
import {type ObjectMapper} from '../../mapper/api/object-mapper.js';
import {type ObjectStorageBackend} from '../../backend/api/object-storage-backend.js';
import {type RemoteConfigSchema} from '../../schema/migration/impl/remote/remote-config-schema.js';

export class RemoteConfigSource extends MutableModelConfigSource<RemoteConfig> implements Refreshable {
public constructor(schema: RemoteConfigSchema, mapper: ObjectMapper, backend: ObjectStorageBackend) {
super('remote-config-data', schema, backend, mapper);
}

public get name(): string {
return this.constructor.name;
}

public get ordinal(): number {
return 300;
}

public async refresh(): Promise<void> {
await this.load();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@ import {type Primitive} from '../../../business/utils/primitive.js';
import {type PrimitiveArray} from '../../../business/utils/primitive-array.js';

@injectable()
export class CTObjectMapper implements ObjectMapper {
export class ClassToObjectMapper implements ObjectMapper {
private readonly flatMapper: FlatKeyMapper;

public constructor(@inject(InjectTokens.KeyFormatter) private readonly formatter: KeyFormatter) {
this.flatMapper = new FlatKeyMapper(patchInject(formatter, InjectTokens.KeyFormatter, CTObjectMapper.name));
this.flatMapper = new FlatKeyMapper(patchInject(formatter, InjectTokens.KeyFormatter, ClassToObjectMapper.name));
}

public fromArray<T extends R, R>(cls: ClassConstructor<T>, array: object[]): R[] {
Expand Down
35 changes: 35 additions & 0 deletions src/data/schema/migration/impl/remote/remote-config-schema.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
// SPDX-License-Identifier: Apache-2.0

import {SchemaBase} from '../../api/schema-base.js';
import {type Schema} from '../../api/schema.js';
import {RemoteConfig} from '../../../model/remote/remote-config.js';
import {type ClassConstructor} from '../../../../../business/utils/class-constructor.type.js';
import {type SchemaMigration} from '../../api/schema-migration.js';
import {type Version} from '../../../../../business/utils/version.js';
import {InjectTokens} from '../../../../../core/dependency-injection/inject-tokens.js';
import {type ObjectMapper} from '../../../../mapper/api/object-mapper.js';
import {RemoteConfigV1Migration} from './remote-config-v1-migration.js';
import {inject, injectable} from 'tsyringe-neo';

@injectable()
export class RemoteConfigSchema extends SchemaBase<RemoteConfig> implements Schema<RemoteConfig> {
public constructor(@inject(InjectTokens.ObjectMapper) mapper: ObjectMapper) {
super(mapper);
}

public get name(): string {
return RemoteConfig.name;
}

public get version(): Version<number> {
return RemoteConfig.SCHEMA_VERSION;
}

public get classCtor(): ClassConstructor<RemoteConfig> {
return RemoteConfig;
}

public get migrations(): SchemaMigration[] {
return [new RemoteConfigV1Migration()];
}
}
Loading
Loading