spark_k8s/charts/spark-4.1/values-scenario-airflow-connect-k8s.yaml at main · fall-out-bug/spark_k8s · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
# Preset: Airflow + Spark Connect + K8s backend
# Source: scripts/test-e2e-airflow-connect.sh (Spark 4.1, BACKEND_MODE=k8s)
# Components: Airflow + Connect + MinIO
# Note: Airflow deployed separately, this preset only for Connect

global:
  s3:
    endpoint: "http://minio:9000"
    accessKey: "minioadmin"
    secretKey: "minioadmin"
    pathStyleAccess: true
    sslEnabled: false

spark-base:
  enabled: true
  minio:
    enabled: false
  postgresql:
    enabled: false

rbac:
  create: false  # Shared with Airflow
  serviceAccountName: "spark"

connect:
  enabled: true
  replicas: 1
  backendMode: k8s
  image:
    repository: ghcr.io/fall-out-bug/spark-k8s-spark-custom
    tag: "4.1.0"
    pullPolicy: IfNotPresent
  resources:
    requests:
      cpu: "200m"
      memory: "512Mi"
    limits:
      cpu: "1"
      memory: "2Gi"
  driver:
    host: ""  # Set to "<release>-spark-41-connect.<namespace>.svc.cluster.local"
    port: 7078
    memory: "512m"
  executor:
    cores: "0.1"
    coresLimit: "0.5"
    memory: "512Mi"
    memoryLimit: "1Gi"
  standalone:
    masterService: ""
    masterPort: 7077
  eventLog:
    enabled: true
    dir: "s3a://spark-logs/events"
  dynamicAllocation:
    enabled: false
    minExecutors: 1
    maxExecutors: 1
  sparkConf:
    spark.driver.memory: "512m"
    spark.executor.memory: "512m"
    spark.executor.instances: "1"
    spark.sql.shuffle.partitions: "2"
    spark.kubernetes.executor.request.cores: "0.1"
    spark.kubernetes.executor.limit.cores: "0.5"

jupyter:
  enabled: false

hiveMetastore:
  enabled: false

historyServer:
  enabled: false

ingress:
  enabled: false

security:
  podSecurityStandards: false

# Disable optional components
sparkOperator:
  enabled: false

celeborn:
  enabled: false