Skip to content

Commit 5d0f070

Browse files
terion-labsterion-labs
committed
Use configured S3 credentials for CLI backups
1 parent 2db09a2 commit 5d0f070

4 files changed

Lines changed: 52 additions & 9 deletions

File tree

‎scripts/lib/common.ts‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,16 @@ export function yamlStringify(value: unknown): string {
5151
return stringify(value);
5252
}
5353

54-
type CommandOptions = { cwd?: string; stdin?: string | Uint8Array };
54+
type CommandOptions = { cwd?: string; stdin?: string | Uint8Array; env?: Record<string, string> };
5555

5656
async function shellCommand(cmd: string[], options: CommandOptions = {}) {
5757
const proc = $`${cmd}`.quiet().nothrow();
5858
if (options.cwd) {
5959
proc.cwd(options.cwd);
6060
}
61+
if (options.env) {
62+
proc.env(options.env);
63+
}
6164
if (options.stdin !== undefined) {
6265
const writer = proc.stdin.getWriter();
6366
await writer.write(typeof options.stdin === "string" ? new TextEncoder().encode(options.stdin) : options.stdin);
@@ -76,7 +79,7 @@ export async function runText(cmd: string[], prefix: string, options: CommandOpt
7679
return proc.stdout.toString();
7780
}
7881

79-
export async function runJson<T>(cmd: string[], prefix: string, options: { cwd?: string } = {}): Promise<T> {
82+
export async function runJson<T>(cmd: string[], prefix: string, options: { cwd?: string; env?: Record<string, string> } = {}): Promise<T> {
8083
const stdout = await runText(cmd, prefix, options);
8184
return JSON.parse(stdout || "null") as T;
8285
}
@@ -93,8 +96,8 @@ export async function runAllowFailure(
9396
};
9497
}
9598

96-
export async function runShell(command: string, prefix: string, cwd?: string): Promise<void> {
97-
await runText(["bash", "-lc", command], prefix, { cwd });
99+
export async function runShell(command: string, prefix: string, options: { cwd?: string; env?: Record<string, string> } = {}): Promise<void> {
100+
await runText(["bash", "-lc", command], prefix, options);
98101
}
99102

100103
export async function runInteractive(cmd: string[], prefix: string, options: { cwd?: string } = {}): Promise<void> {

‎scripts/terrarium-s3-export.ts‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,18 @@ type LxcInstance = {
1111
type?: string;
1212
};
1313

14+
function s3Env(config: Record<string, unknown>): Record<string, string> {
15+
const env: Record<string, string> = {};
16+
const accessKey = configString(config, "terrarium_s3_access_key");
17+
const secretKey = configString(config, "terrarium_s3_secret_key");
18+
const region = configString(config, "terrarium_s3_region", "us-east-1");
19+
if (accessKey) env.AWS_ACCESS_KEY_ID = accessKey;
20+
if (secretKey) env.AWS_SECRET_ACCESS_KEY = secretKey;
21+
if (region) env.AWS_DEFAULT_REGION = region;
22+
env.AWS_EC2_METADATA_DISABLED = "true";
23+
return env;
24+
}
25+
1426
async function latestSnapshot(dataset: string): Promise<string> {
1527
const stdout = await runText(["zfs", "list", "-H", "-t", "snapshot", "-o", "name", "-s", "creation"], PREFIX);
1628
let latest = "";
@@ -36,6 +48,7 @@ export async function backupExportCmd(configPath = DEFAULT_CONFIG_PATH): Promise
3648
const endpoint = configString(config, "terrarium_s3_endpoint");
3749
const prefix = configString(config, "terrarium_s3_prefix", "terrarium");
3850
const pool = configString(config, "terrarium_lxd_pool_name", "terrarium");
51+
const awsEnv = s3Env(config);
3952
const awsBase = ["aws"];
4053
if (endpoint) {
4154
awsBase.push("--endpoint-url", endpoint);
@@ -76,7 +89,8 @@ export async function backupExportCmd(configPath = DEFAULT_CONFIG_PATH): Promise
7689

7790
await runShell(
7891
`${streamSource} | zstd -T0 | ${awsBase.map(shellEscape).join(" ")} s3 cp - ${shellEscape(`s3://${bucket}/${objectKey}`)}`,
79-
PREFIX
92+
PREFIX,
93+
{ env: awsEnv }
8094
);
8195

8296
const manifest = {
@@ -89,7 +103,7 @@ export async function backupExportCmd(configPath = DEFAULT_CONFIG_PATH): Promise
89103
created_at: new Date().toISOString()
90104
};
91105
writeJsonFile(manifestPath, manifest);
92-
await runText([...awsBase, "s3", "cp", manifestPath, `s3://${bucket}/${manifestKey}`], PREFIX);
106+
await runText([...awsBase, "s3", "cp", manifestPath, `s3://${bucket}/${manifestKey}`], PREFIX, { env: awsEnv });
93107
writeFileSync(stateFile, `${latest}\n`, "utf8");
94108
}
95109
}

‎scripts/terrarium-zfs-reconstruct.ts‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,18 @@ type Manifest = {
1212
created_at: string;
1313
};
1414

15+
function s3Env(config: Record<string, unknown>): Record<string, string> {
16+
const env: Record<string, string> = {};
17+
const accessKey = configString(config, "terrarium_s3_access_key");
18+
const secretKey = configString(config, "terrarium_s3_secret_key");
19+
const region = configString(config, "terrarium_s3_region", "us-east-1");
20+
if (accessKey) env.AWS_ACCESS_KEY_ID = accessKey;
21+
if (secretKey) env.AWS_SECRET_ACCESS_KEY = secretKey;
22+
if (region) env.AWS_DEFAULT_REGION = region;
23+
env.AWS_EC2_METADATA_DISABLED = "true";
24+
return env;
25+
}
26+
1527
function selectChain(directory: string, match = ""): Manifest[] {
1628
const manifests: Manifest[] = [];
1729
for (const entry of new Bun.Glob("*.json").scanSync(directory)) {
@@ -45,14 +57,17 @@ export async function reconstructFromS3(instance: string, at: string, targetData
4557
const bucket = configString(config, "terrarium_s3_bucket");
4658
const endpoint = configString(config, "terrarium_s3_endpoint");
4759
const prefix = configString(config, "terrarium_s3_prefix", "terrarium");
60+
const awsEnv = s3Env(config);
4861
const awsBase = ["aws"];
4962
if (endpoint) {
5063
awsBase.push("--endpoint-url", endpoint);
5164
}
5265

5366
const tempDir = makeTempDir("terrarium-restore.");
5467
try {
55-
await runText([...awsBase, "s3", "cp", `s3://${bucket}/${prefix}/manifests/${instance}/`, `${tempDir}/`, "--recursive"], PREFIX);
68+
await runText([...awsBase, "s3", "cp", `s3://${bucket}/${prefix}/manifests/${instance}/`, `${tempDir}/`, "--recursive"], PREFIX, {
69+
env: awsEnv
70+
});
5671
const chain = selectChain(tempDir, at);
5772

5873
const datasetCheck = await runAllowFailure(["zfs", "list", "-H", targetDataset]);
@@ -63,7 +78,8 @@ export async function reconstructFromS3(instance: string, at: string, targetData
6378
for (const manifest of chain) {
6479
await runShell(
6580
`${awsBase.map(shellEscape).join(" ")} s3 cp ${shellEscape(`s3://${bucket}/${manifest.object_key}`)} - | zstd -d | zfs receive -F ${shellEscape(targetDataset)}`,
66-
PREFIX
81+
PREFIX,
82+
{ env: awsEnv }
6783
);
6884
}
6985
} finally {

‎scripts/terrariumctl.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,14 @@ async function backupListCmd(): Promise<void> {
153153
const bucket = configString(config, "terrarium_s3_bucket");
154154
const prefix = configString(config, "terrarium_s3_prefix", "terrarium");
155155
const endpoint = configString(config, "terrarium_s3_endpoint");
156+
const awsEnv: Record<string, string> = {};
157+
const accessKey = configString(config, "terrarium_s3_access_key");
158+
const secretKey = configString(config, "terrarium_s3_secret_key");
159+
const region = configString(config, "terrarium_s3_region", "us-east-1");
160+
if (accessKey) awsEnv.AWS_ACCESS_KEY_ID = accessKey;
161+
if (secretKey) awsEnv.AWS_SECRET_ACCESS_KEY = secretKey;
162+
if (region) awsEnv.AWS_DEFAULT_REGION = region;
163+
awsEnv.AWS_EC2_METADATA_DISABLED = "true";
156164
const awsBase = ["aws"];
157165
if (endpoint) {
158166
awsBase.push("--endpoint-url", endpoint);
@@ -170,7 +178,9 @@ async function backupListCmd(): Promise<void> {
170178

171179
if (configBoolean(config, "terrarium_enable_s3") && bucket) {
172180
console.log(`\n${heading("S3 manifests")}`);
173-
const output = (await runAllowFailure([...awsBase, "s3", "ls", `s3://${bucket}/${prefix}/manifests/`, "--recursive"])).stdout.trim();
181+
const output = (
182+
await runAllowFailure([...awsBase, "s3", "ls", `s3://${bucket}/${prefix}/manifests/`, "--recursive"], { env: awsEnv })
183+
).stdout.trim();
174184
if (output) {
175185
console.log(output);
176186
}

0 commit comments

Comments
 (0)