Skip to content

Commit 702f6a5

Browse files
stubbiclaude
andcommitted
fix: add K8s API egress and sandbox scheduling env vars
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 165cfe8 commit 702f6a5

10 files changed

Lines changed: 195 additions & 21 deletions

File tree

api/v1alpha1/paperclipinstance_types.go

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -357,12 +357,18 @@ type AdaptersSpec struct {
357357
// +optional
358358
CloudSandbox *CloudSandboxSpec `json:"cloudSandbox,omitempty"`
359359

360-
// ManagedInferenceSecretRef references a Secret containing the platform LLM API key.
361-
// The Secret must contain a key "PAPERCLIP_MANAGED_INFERENCE_API_KEY".
360+
// ManagedInferenceSecretRef references a Secret containing platform LLM API keys.
361+
// The Secret should contain one or more of these keys:
362+
// PAPERCLIP_MANAGED_ANTHROPIC_API_KEY
363+
// PAPERCLIP_MANAGED_OPENAI_API_KEY
364+
// PAPERCLIP_MANAGED_GEMINI_API_KEY
365+
// PAPERCLIP_MANAGED_OPENROUTER_API_KEY
366+
// For backward compatibility, PAPERCLIP_MANAGED_INFERENCE_API_KEY is also supported.
362367
// +optional
363368
ManagedInferenceSecretRef *corev1.LocalObjectReference `json:"managedInferenceSecretRef,omitempty"`
364369

365-
// ManagedInferenceProvider is the LLM provider for managed inference (e.g. "anthropic", "openrouter").
370+
// ManagedInferenceProvider is the LLM provider for the legacy single-key mode.
371+
// Ignored when per-provider keys are used.
366372
// +kubebuilder:default="anthropic"
367373
// +optional
368374
ManagedInferenceProvider string `json:"managedInferenceProvider,omitempty"`

charts/paperclip-operator/templates/crds/paperclip.inc_instances.yaml

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -273,13 +273,19 @@ spec:
273273
type: string
274274
managedInferenceProvider:
275275
default: anthropic
276-
description: ManagedInferenceProvider is the LLM provider for
277-
managed inference (e.g. "anthropic", "openrouter").
276+
description: |-
277+
ManagedInferenceProvider is the LLM provider for the legacy single-key mode.
278+
Ignored when per-provider keys are used.
278279
type: string
279280
managedInferenceSecretRef:
280281
description: |-
281-
ManagedInferenceSecretRef references a Secret containing the platform LLM API key.
282-
The Secret must contain a key "PAPERCLIP_MANAGED_INFERENCE_API_KEY".
282+
ManagedInferenceSecretRef references a Secret containing platform LLM API keys.
283+
The Secret should contain one or more of these keys:
284+
PAPERCLIP_MANAGED_ANTHROPIC_API_KEY
285+
PAPERCLIP_MANAGED_OPENAI_API_KEY
286+
PAPERCLIP_MANAGED_GEMINI_API_KEY
287+
PAPERCLIP_MANAGED_OPENROUTER_API_KEY
288+
For backward compatibility, PAPERCLIP_MANAGED_INFERENCE_API_KEY is also supported.
283289
properties:
284290
name:
285291
default: ""

charts/paperclip-operator/templates/rbac.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ rules:
2727
verbs: ["create", "delete", "get", "list", "patch", "watch"]
2828
- apiGroups: [""]
2929
resources: ["pods/exec"]
30-
verbs: ["create"]
30+
verbs: ["create", "get"]
3131
- apiGroups: [""]
3232
resources: ["pods/log"]
3333
verbs: ["get"]

config/crd/bases/paperclip.inc_instances.yaml

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -267,13 +267,19 @@ spec:
267267
type: string
268268
managedInferenceProvider:
269269
default: anthropic
270-
description: ManagedInferenceProvider is the LLM provider for
271-
managed inference (e.g. "anthropic", "openrouter").
270+
description: |-
271+
ManagedInferenceProvider is the LLM provider for the legacy single-key mode.
272+
Ignored when per-provider keys are used.
272273
type: string
273274
managedInferenceSecretRef:
274275
description: |-
275-
ManagedInferenceSecretRef references a Secret containing the platform LLM API key.
276-
The Secret must contain a key "PAPERCLIP_MANAGED_INFERENCE_API_KEY".
276+
ManagedInferenceSecretRef references a Secret containing platform LLM API keys.
277+
The Secret should contain one or more of these keys:
278+
PAPERCLIP_MANAGED_ANTHROPIC_API_KEY
279+
PAPERCLIP_MANAGED_OPENAI_API_KEY
280+
PAPERCLIP_MANAGED_GEMINI_API_KEY
281+
PAPERCLIP_MANAGED_OPENROUTER_API_KEY
282+
For backward compatibility, PAPERCLIP_MANAGED_INFERENCE_API_KEY is also supported.
277283
properties:
278284
name:
279285
default: ""

config/rbac/role.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ rules:
4343
- pods/exec
4444
verbs:
4545
- create
46+
- get
4647
- apiGroups:
4748
- ""
4849
resources:

internal/controller/instance_controller.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,9 @@ type InstanceReconciler struct {
7979
// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete
8080
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch
8181
// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete
82-
// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch
82+
// +kubebuilder:rbac:groups="",resources=pods,verbs=create;delete;get;list;patch;watch
83+
// +kubebuilder:rbac:groups="",resources=pods/exec,verbs=create;get
84+
// +kubebuilder:rbac:groups="",resources=pods/log,verbs=get
8385
// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
8486
// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
8587
// +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch;delete

internal/resources/networkpolicy.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,21 @@ func BuildNetworkPolicy(instance *paperclipv1alpha1.Instance) *networkingv1.Netw
6161
},
6262
}
6363

64+
// Allow egress to K8s API server when cloud sandbox is enabled.
65+
// The server needs to create/manage sandbox pods via the K8s API.
66+
// An explicit rule is needed because some CNIs (k3s Flannel, Calico)
67+
// do not match host-network destinations with portOnly egress rules.
68+
if instance.Spec.Adapters.CloudSandbox != nil && instance.Spec.Adapters.CloudSandbox.Enabled {
69+
np.Spec.Egress = append(np.Spec.Egress, networkingv1.NetworkPolicyEgressRule{
70+
Ports: []networkingv1.NetworkPolicyPort{
71+
{
72+
Port: Ptr(intstr.FromInt32(6443)),
73+
Protocol: Ptr(corev1.ProtocolTCP),
74+
},
75+
},
76+
})
77+
}
78+
6479
// Allow egress to managed database if applicable
6580
if instance.Spec.Database.Mode == "managed" || instance.Spec.Database.Mode == "" {
6681
np.Spec.Egress = append(np.Spec.Egress, networkingv1.NetworkPolicyEgressRule{

internal/resources/resources_test.go

Lines changed: 95 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import (
1010
paperclipv1alpha1 "github.com/paperclipinc/paperclip-operator/api/v1alpha1"
1111
)
1212

13-
//nolint:unparam // test helper kept flexible for future test cases
1413
func newTestInstance(name string) *paperclipv1alpha1.Instance {
1514
return &paperclipv1alpha1.Instance{
1615
ObjectMeta: metav1.ObjectMeta{
@@ -277,6 +276,39 @@ func TestBuildNetworkPolicy(t *testing.T) {
277276
}
278277
}
279278

279+
func TestBuildNetworkPolicyCloudSandboxK8sAPIEgress(t *testing.T) {
280+
instance := newTestInstance("my-paperclip")
281+
instance.Spec.Adapters.CloudSandbox = &paperclipv1alpha1.CloudSandboxSpec{
282+
Enabled: true,
283+
}
284+
np := BuildNetworkPolicy(instance)
285+
286+
found := false
287+
for _, rule := range np.Spec.Egress {
288+
for _, port := range rule.Ports {
289+
if port.Port != nil && port.Port.IntValue() == 6443 {
290+
found = true
291+
}
292+
}
293+
}
294+
if !found {
295+
t.Error("expected egress rule for K8s API port 6443 when cloud sandbox enabled")
296+
}
297+
}
298+
299+
func TestBuildNetworkPolicyNoK8sAPIEgressWithoutSandbox(t *testing.T) {
300+
instance := newTestInstance("my-paperclip")
301+
np := BuildNetworkPolicy(instance)
302+
303+
for _, rule := range np.Spec.Egress {
304+
for _, port := range rule.Ports {
305+
if port.Port != nil && port.Port.IntValue() == 6443 {
306+
t.Error("should not have K8s API egress rule when cloud sandbox is not enabled")
307+
}
308+
}
309+
}
310+
}
311+
280312
func TestBuildIngress(t *testing.T) {
281313
instance := newTestInstance("my-paperclip")
282314
instance.Spec.Networking.Ingress = &paperclipv1alpha1.IngressSpec{
@@ -666,6 +698,68 @@ func TestBuildStatefulSetNoCloudSandbox(t *testing.T) {
666698
}
667699
}
668700

701+
func TestBuildStatefulSetCloudSandboxSchedulingEnvVars(t *testing.T) {
702+
instance := newTestInstance("my-paperclip")
703+
instance.Spec.Adapters.CloudSandbox = &paperclipv1alpha1.CloudSandboxSpec{
704+
Enabled: true,
705+
}
706+
instance.Spec.Availability.NodeSelector = map[string]string{
707+
"cloud.google.com/gke-nodepool": "sandbox",
708+
}
709+
instance.Spec.Availability.Tolerations = []corev1.Toleration{
710+
{
711+
Key: "sandbox",
712+
Operator: corev1.TolerationOpEqual,
713+
Value: "true",
714+
Effect: corev1.TaintEffectNoSchedule,
715+
},
716+
}
717+
718+
sts := BuildStatefulSet(instance, nil)
719+
container := sts.Spec.Template.Spec.Containers[0]
720+
721+
envMap := make(map[string]string)
722+
for _, env := range container.Env {
723+
if env.Value != "" {
724+
envMap[env.Name] = env.Value
725+
}
726+
}
727+
728+
// Verify nodeSelector env var
729+
nsVal, ok := envMap["PAPERCLIP_CLOUD_SANDBOX_NODE_SELECTOR"]
730+
if !ok {
731+
t.Fatal("expected PAPERCLIP_CLOUD_SANDBOX_NODE_SELECTOR to be set")
732+
}
733+
if nsVal != `{"cloud.google.com/gke-nodepool":"sandbox"}` {
734+
t.Errorf("unexpected nodeSelector JSON: %s", nsVal)
735+
}
736+
737+
// Verify tolerations env var
738+
tolVal, ok := envMap["PAPERCLIP_CLOUD_SANDBOX_TOLERATIONS"]
739+
if !ok {
740+
t.Fatal("expected PAPERCLIP_CLOUD_SANDBOX_TOLERATIONS to be set")
741+
}
742+
if tolVal != `[{"key":"sandbox","operator":"Equal","value":"true","effect":"NoSchedule"}]` {
743+
t.Errorf("unexpected tolerations JSON: %s", tolVal)
744+
}
745+
746+
// Verify these are NOT set when availability scheduling is empty
747+
instance2 := newTestInstance("my-paperclip-2")
748+
instance2.Spec.Adapters.CloudSandbox = &paperclipv1alpha1.CloudSandboxSpec{
749+
Enabled: true,
750+
}
751+
sts2 := BuildStatefulSet(instance2, nil)
752+
container2 := sts2.Spec.Template.Spec.Containers[0]
753+
for _, env := range container2.Env {
754+
if env.Name == "PAPERCLIP_CLOUD_SANDBOX_NODE_SELECTOR" {
755+
t.Error("unexpected PAPERCLIP_CLOUD_SANDBOX_NODE_SELECTOR when nodeSelector is empty")
756+
}
757+
if env.Name == "PAPERCLIP_CLOUD_SANDBOX_TOLERATIONS" {
758+
t.Error("unexpected PAPERCLIP_CLOUD_SANDBOX_TOLERATIONS when tolerations is empty")
759+
}
760+
}
761+
}
762+
669763
func TestBuildSandboxRole(t *testing.T) {
670764
instance := newTestInstance("my-paperclip")
671765
role := BuildSandboxRole(instance, "test-ns")

internal/resources/sandbox_rbac.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ func sandboxBaseRules() []rbacv1.PolicyRule {
2828
{
2929
APIGroups: []string{""},
3030
Resources: []string{"pods/exec"},
31-
Verbs: []string{"create"},
31+
Verbs: []string{"create", "get"},
3232
},
3333
{
3434
APIGroups: []string{""},

internal/resources/statefulset.go

Lines changed: 50 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package resources
22

33
import (
4+
"encoding/json"
45
"fmt"
56

67
appsv1 "k8s.io/api/apps/v1"
@@ -371,18 +372,42 @@ func buildManagedInferenceEnvVars(instance *paperclipv1alpha1.Instance) []corev1
371372
return nil
372373
}
373374

374-
vars := []corev1.EnvVar{
375-
{
376-
Name: "PAPERCLIP_MANAGED_INFERENCE_API_KEY",
375+
secretRef := *instance.Spec.Adapters.ManagedInferenceSecretRef
376+
377+
// Per-provider keys - each is optional in the Secret
378+
providerKeys := []string{
379+
"PAPERCLIP_MANAGED_ANTHROPIC_API_KEY",
380+
"PAPERCLIP_MANAGED_OPENAI_API_KEY",
381+
"PAPERCLIP_MANAGED_GEMINI_API_KEY",
382+
"PAPERCLIP_MANAGED_OPENROUTER_API_KEY",
383+
}
384+
385+
vars := make([]corev1.EnvVar, 0, len(providerKeys)+3)
386+
for _, key := range providerKeys {
387+
vars = append(vars, corev1.EnvVar{
388+
Name: key,
377389
ValueFrom: &corev1.EnvVarSource{
378390
SecretKeyRef: &corev1.SecretKeySelector{
379-
LocalObjectReference: *instance.Spec.Adapters.ManagedInferenceSecretRef,
380-
Key: "PAPERCLIP_MANAGED_INFERENCE_API_KEY",
391+
LocalObjectReference: secretRef,
392+
Key: key,
393+
Optional: Ptr(true),
381394
},
382395
},
383-
},
396+
})
384397
}
385398

399+
// Legacy single-key for backward compatibility
400+
vars = append(vars, corev1.EnvVar{
401+
Name: "PAPERCLIP_MANAGED_INFERENCE_API_KEY",
402+
ValueFrom: &corev1.EnvVarSource{
403+
SecretKeyRef: &corev1.SecretKeySelector{
404+
LocalObjectReference: secretRef,
405+
Key: "PAPERCLIP_MANAGED_INFERENCE_API_KEY",
406+
Optional: Ptr(true),
407+
},
408+
},
409+
})
410+
386411
if instance.Spec.Adapters.ManagedInferenceProvider != "" {
387412
vars = append(vars, corev1.EnvVar{
388413
Name: "PAPERCLIP_MANAGED_INFERENCE_PROVIDER",
@@ -447,6 +472,25 @@ func buildCloudSandboxEnvVars(instance *paperclipv1alpha1.Instance) []corev1.Env
447472
vars = append(vars, corev1.EnvVar{Name: "PAPERCLIP_CLOUD_SANDBOX_MULTI_NAMESPACE", Value: "true"})
448473
}
449474

475+
// Node scheduling: pass the instance's scheduling constraints so the
476+
// Paperclip server can apply them to sandbox pods it creates.
477+
if len(instance.Spec.Availability.NodeSelector) > 0 {
478+
if b, err := json.Marshal(instance.Spec.Availability.NodeSelector); err == nil {
479+
vars = append(vars, corev1.EnvVar{
480+
Name: "PAPERCLIP_CLOUD_SANDBOX_NODE_SELECTOR",
481+
Value: string(b),
482+
})
483+
}
484+
}
485+
if len(instance.Spec.Availability.Tolerations) > 0 {
486+
if b, err := json.Marshal(instance.Spec.Availability.Tolerations); err == nil {
487+
vars = append(vars, corev1.EnvVar{
488+
Name: "PAPERCLIP_CLOUD_SANDBOX_TOLERATIONS",
489+
Value: string(b),
490+
})
491+
}
492+
}
493+
450494
return vars
451495
}
452496

0 commit comments

Comments
 (0)