fix(restore): bound kopia's content cache and rotate CLI logs

Kopia's content cache grows without an explicit cap and its CLI log
files accumulate indefinitely. The per-replica cache PVC eventually
fills up, after which kopia can't even write its config and every
restore Job pod exits in 1-2 minutes with "no space left on device".
Observed in production across multiple replicas: the only path back
to working was manually wiping the PVC contents.

Pass content/metadata cache size limits on `kopia repository connect`
in the restore Job so kopia evicts its own data once it reaches the
cap. Content cache is sized to the cache PVC minus a small reserve
(for metadata cache, logs, and slop); metadata cache is a fixed small
value. Both limits are persisted to the local kopia config on
connect, so subsequent operations inherit them.

Also apply global log-rotation flags to every kopia invocation in
both the restore script and the snapshot-list script:
`--log-dir-max-files=20 --log-dir-max-age=24h`. Without this the CLI
logs alone reached hundreds of MB across hundreds of files (one new
file per snapshot-list reconcile).

These together turn the cache PVC's role into kopia's own LRU eviction
rather than a one-shot fill-then-fail.

Author: passcod

src/controllers/replica/resources.rs

@@ -84,7 +84,11 @@ if [ "$KOPIA_DISABLE_TLS" = "true" ]; then
   ENDPOINT_ARGS="$ENDPOINT_ARGS --disable-tls --disable-tls-verification"
 fi
 
-kopia repository connect s3 \
+# Global kopia flags: rotate CLI logs so they don't accumulate over
+# many snapshot-list invocations.
+KOPIA_GLOBAL_FLAGS="--log-dir-max-files=20 --log-dir-max-age=24h"
+
+kopia $KOPIA_GLOBAL_FLAGS repository connect s3 \
   --bucket="$KOPIA_BUCKET" \
   --region="$KOPIA_REGION" \
   --access-key="$AWS_ACCESS_KEY_ID" \
@@ -95,7 +99,7 @@ kopia repository connect s3 \
 
 SNAP_FILE=$(mktemp)
 trap 'rm -f "$SNAP_FILE"' EXIT
-kopia snapshot list --json --all > "$SNAP_FILE" || echo "[]" > "$SNAP_FILE"
+kopia $KOPIA_GLOBAL_FLAGS snapshot list --json --all > "$SNAP_FILE" || echo "[]" > "$SNAP_FILE"
 cat "$SNAP_FILE"
 if [ -n "$SNAPSHOT_CALLBACK_URL" ]; then
   SNAP_SIZE=$(wc -c < "$SNAP_FILE")

src/controllers/replica/tests.rs

@@ -1,6 +1,9 @@
-use crate::kopia::Snapshot;
+use k8s_openapi::api::core::v1::SecretReference;
+use kube::api::ObjectMeta;
 
-use super::generate_password;
+use crate::{kopia::Snapshot, types::*, util::TimeSpan};
+
+use super::{generate_password, resources::build_snapshot_list_job};
 
 #[test]
 fn generate_password_length_and_charset() {
@@ -74,3 +77,63 @@ fn parse_kopia_snapshot_with_backslash_paths() {
 	let snaps: Vec<Snapshot> = serde_json::from_str(raw).unwrap();
 	assert_eq!(snaps[0].source.path, r"C:\Users\backup\data");
 }
+
+#[test]
+fn snapshot_list_job_rotates_kopia_logs() {
+	// Snapshot-list jobs run on every scheduled reconcile (many times
+	// per day per replica). Without log rotation kopia's CLI logs
+	// accumulate in the pod's writable layer / cache PVC over time and
+	// eventually contribute to filling it. Confirm the script applies
+	// the global log-rotation flags to every kopia invocation.
+	let replica = PostgresPhysicalReplica {
+		metadata: ObjectMeta {
+			name: Some("test".into()),
+			namespace: Some("default".into()),
+			uid: Some("uid".into()),
+			..Default::default()
+		},
+		spec: PostgresPhysicalReplicaSpec {
+			kopia_secret_ref: SecretReference {
+				name: Some("creds".into()),
+				namespace: None,
+			},
+			snapshot_filter: None,
+			schedule: "0 * * * *".into(),
+			schedule_jitter: TimeSpan(jiff::Span::new()),
+			minimum_ttl: None,
+			switchover_grace_period: TimeSpan(jiff::Span::new()),
+			analytics_username: "analytics".into(),
+			storage_class: None,
+			storage_size_override: None,
+			resources: None,
+			service_annotations: None,
+			pod_annotations: None,
+			affinity: None,
+			tolerations: vec![],
+			read_only: true,
+			postgres_extra_config: None,
+			notifications: vec![],
+			persistent_schemas: None,
+			storage_size_maximum: k8s_openapi::apimachinery::pkg::api::resource::Quantity(
+				"2Ti".to_string(),
+			),
+		},
+		status: None,
+	};
+
+	let job = build_snapshot_list_job(&replica, "test-snap", "default", "kopia:latest", "http://x")
+		.expect("job builds");
+	let script = job.spec.unwrap().template.spec.unwrap().containers[0]
+		.args
+		.as_ref()
+		.unwrap()[0]
+		.clone();
+	assert!(
+		script.contains("--log-dir-max-files=20"),
+		"snapshot-list script must rotate kopia logs by file count"
+	);
+	assert!(
+		script.contains("--log-dir-max-age=24h"),
+		"snapshot-list script must rotate kopia logs by age"
+	);
+}

src/controllers/restore/builders.rs

@@ -387,6 +387,37 @@ pub fn kopia_cache_pvc_size(snapshot_size: &Quantity) -> Quantity {
 	chosen.into()
 }
 
+/// MB to leave free on the cache PVC for kopia's metadata cache, its
+/// rolling CLI logs, and slop. Subtracted from the PVC capacity to
+/// derive the content-cache hard cap.
+pub const KOPIA_CACHE_RESERVE_MB: u64 = 2048;
+/// Hardcoded metadata-cache cap. Kopia's metadata cache is small
+/// (indices and manifest data) so a fixed allocation is fine.
+pub const KOPIA_METADATA_CACHE_MB: u64 = 512;
+/// Floor for the content-cache cap in MB, so degenerate-small PVCs
+/// still get a useful cache.
+pub const KOPIA_CONTENT_CACHE_FLOOR_MB: u64 = 1024;
+
+/// Compute the content-cache cap (MB) passed to `kopia repository
+/// connect --content-cache-size-mb`. Sized to the cache PVC minus a
+/// fixed reserve for metadata cache + logs + slop.
+///
+/// Without a cap kopia's content cache grows unbounded and eventually
+/// fills the PVC, after which kopia can't even write its config and
+/// every restore Job pod exits in 1–2 minutes. The cap turns that into
+/// kopia's own LRU eviction, which is what we want.
+pub fn kopia_content_cache_mb(snapshot_size: &Quantity) -> u64 {
+	let pvc_size = kopia_cache_pvc_size(snapshot_size);
+	let pvc_bytes = ParsedQuantity::try_from(pvc_size)
+		.ok()
+		.and_then(|q| q.to_bytes_f64())
+		.unwrap_or(0.0);
+	let pvc_mb = (pvc_bytes / 1024.0 / 1024.0) as u64;
+	pvc_mb
+		.saturating_sub(KOPIA_CACHE_RESERVE_MB)
+		.max(KOPIA_CONTENT_CACHE_FLOOR_MB)
+}
+
 pub fn build_kopia_cache_pvc(
 	replica: &PostgresPhysicalReplica,
 	snapshot_size: &Quantity,
@@ -446,17 +477,29 @@ if [ "$KOPIA_DISABLE_TLS" = "true" ]; then
   ENDPOINT_ARGS="$ENDPOINT_ARGS --disable-tls --disable-tls-verification"
 fi
 
+# Global kopia flags applied to every invocation: rotate CLI logs so
+# they don't fill the cache PVC. Cap at 20 most-recent files and
+# 24 hours, plenty for debugging a current restore without growing
+# without bound.
+KOPIA_GLOBAL_FLAGS="--log-dir-max-files=20 --log-dir-max-age=24h"
+
 echo "Connecting to kopia repository..."
-kopia repository connect s3 \
+# --content-cache-size-mb / --metadata-cache-size-mb are persisted to
+# the local config on connect, so subsequent operations inherit the
+# bound. Without them kopia's content cache grows unbounded and
+# eventually fills the cache PVC (observed across multiple replicas).
+kopia $KOPIA_GLOBAL_FLAGS repository connect s3 \
   --bucket="$KOPIA_BUCKET" \
   --region="$KOPIA_REGION" \
   --access-key="$AWS_ACCESS_KEY_ID" \
   --secret-access-key="$AWS_SECRET_ACCESS_KEY" \
   --password="$KOPIA_PASSWORD" \
+  --content-cache-size-mb="$KOPIA_CONTENT_CACHE_MB" \
+  --metadata-cache-size-mb="$KOPIA_METADATA_CACHE_MB" \
   $ENDPOINT_ARGS
 
 echo "Starting restore..."
-kopia snapshot restore "$SNAPSHOT_ID" /pgdata/postgres
+kopia $KOPIA_GLOBAL_FLAGS snapshot restore "$SNAPSHOT_ID" /pgdata/postgres
 
 echo "Restore complete"
 ls -la /pgdata/
@@ -544,11 +587,26 @@ echo -n "$VERSION" > /dev/termination-log
 						args: Some(vec![restore_script.to_string()]),
 						env: Some(
 							[
-								vec![EnvVar {
-									name: "SNAPSHOT_ID".to_string(),
-									value: Some(restore.spec.snapshot.clone()),
-									..Default::default()
-								}],
+								vec![
+									EnvVar {
+										name: "SNAPSHOT_ID".to_string(),
+										value: Some(restore.spec.snapshot.clone()),
+										..Default::default()
+									},
+									EnvVar {
+										name: "KOPIA_CONTENT_CACHE_MB".to_string(),
+										value: Some(
+											kopia_content_cache_mb(&restore.spec.snapshot_size)
+												.to_string(),
+										),
+										..Default::default()
+									},
+									EnvVar {
+										name: "KOPIA_METADATA_CACHE_MB".to_string(),
+										value: Some(KOPIA_METADATA_CACHE_MB.to_string()),
+										..Default::default()
+									},
+								],
 								kopia_writable_env(),
 								vec![
 									env_from_secret("KOPIA_BUCKET", kopia_secret, "bucket"),

src/controllers/restore/tests.rs

@@ -236,6 +236,77 @@ fn kopia_cache_pvc_owned_by_replica() {
 	assert_eq!(access_modes, &vec!["ReadWriteOnce".to_string()]);
 }
 
+#[test]
+fn kopia_content_cache_mb_floor_for_small_pvc() {
+	// 10Gi PVC (the floor for small snapshots) minus the 2Gi reserve =
+	// 8 Gi content cache, expressed in MiB.
+	let small = Quantity("1Gi".to_string());
+	let mb = super::builders::kopia_content_cache_mb(&small);
+	let expected = 10 * 1024 - super::builders::KOPIA_CACHE_RESERVE_MB;
+	assert_eq!(mb, expected);
+	assert!(
+		mb >= super::builders::KOPIA_CONTENT_CACHE_FLOOR_MB,
+		"content cache must always be at least the floor"
+	);
+}
+
+#[test]
+fn kopia_content_cache_mb_scales_with_snapshot() {
+	// 100Gi snapshot → 20Gi PVC → 20Gi - 2Gi reserve = 18Gi cache.
+	let big = Quantity("100Gi".to_string());
+	let mb = super::builders::kopia_content_cache_mb(&big);
+	let expected = 20 * 1024 - super::builders::KOPIA_CACHE_RESERVE_MB;
+	assert_eq!(mb, expected);
+}
+
+#[test]
+fn restore_job_passes_cache_caps_and_log_rotation() {
+	// The restore Job's pod spec must set KOPIA_CONTENT_CACHE_MB and
+	// KOPIA_METADATA_CACHE_MB so the embedded script can cap kopia's
+	// caches, and the script must rotate CLI logs. Without these the
+	// cache PVC fills up and every subsequent restore Job pod exits
+	// in 1–2 minutes ("no space left on device").
+	let (restore, replica) = test_restore_and_replica();
+	let job = build_restore_job(
+		&restore,
+		"test-restore-restore",
+		"default",
+		&replica,
+		"kopia:latest",
+	)
+	.unwrap();
+	let pod_spec = job.spec.unwrap().template.spec.unwrap();
+	let container = &pod_spec.containers[0];
+	let env = container.env.as_ref().expect("container must declare env");
+	let names: Vec<&str> = env.iter().map(|e| e.name.as_str()).collect();
+	assert!(
+		names.contains(&"KOPIA_CONTENT_CACHE_MB"),
+		"restore Job env must include KOPIA_CONTENT_CACHE_MB"
+	);
+	assert!(
+		names.contains(&"KOPIA_METADATA_CACHE_MB"),
+		"restore Job env must include KOPIA_METADATA_CACHE_MB"
+	);
+
+	let script = &container.args.as_ref().unwrap()[0];
+	assert!(
+		script.contains("--content-cache-size-mb=\"$KOPIA_CONTENT_CACHE_MB\""),
+		"connect command must cap content cache via env var"
+	);
+	assert!(
+		script.contains("--metadata-cache-size-mb=\"$KOPIA_METADATA_CACHE_MB\""),
+		"connect command must cap metadata cache"
+	);
+	assert!(
+		script.contains("--log-dir-max-files=20"),
+		"kopia invocations must rotate CLI logs by file count"
+	);
+	assert!(
+		script.contains("--log-dir-max-age=24h"),
+		"kopia invocations must rotate CLI logs by age"
+	);
+}
+
 #[test]
 fn cache_size_needs_grow_ratchet() {
 	let small = Quantity("10Gi".to_string());