spark_k8s/charts/spark-4.1/values-scenario-airflow-connect-standalone.yaml at main · fall-out-bug/spark_k8s · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
# Preset: Airflow + Spark Connect + Standalone backend
# Source: scripts/test-e2e-airflow-connect.sh (Spark 4.1, BACKEND_MODE=standalone)
# Components: Airflow + Connect + MinIO + Standalone
# Note: Airflow + Standalone deployed separately, this preset only for Connect

global:
  s3:
    endpoint: "http://minio:9000"
    accessKey: "minioadmin"
    secretKey: "minioadmin"
    pathStyleAccess: true
    sslEnabled: false

spark-base:
  enabled: true
  minio:
    enabled: false
  postgresql:
    enabled: false

rbac:
  create: false  # Shared with Airflow
  serviceAccountName: "spark"

connect:
  enabled: true
  replicas: 1
  backendMode: standalone
  image:
    repository: ghcr.io/fall-out-bug/spark-k8s-spark-custom
    tag: "4.1.0"
    pullPolicy: IfNotPresent
  resources:
    requests:
      cpu: "200m"
      memory: "512Mi"
    limits:
      cpu: "1"
      memory: "2Gi"
  driver:
    host: "${POD_IP}"  # Use POD_IP for standalone backend
    port: 7078
    memory: "512m"
  executor:
    cores: "1"
    coresLimit: "1"
    memory: "512Mi"
    memoryLimit: "640Mi"
  standalone:
    masterService: "spark-sa-spark-standalone-master"
    masterPort: 7077
  eventLog:
    enabled: true
    dir: "s3a://spark-logs/events"
  dynamicAllocation:
    enabled: false
  sparkConf:
    spark.driver.memory: "512m"
    spark.executor.memory: "512m"
    spark.executor.memoryOverhead: "128m"
    spark.executor.cores: "1"
    spark.executor.instances: "1"
    spark.cores.max: "1"
    spark.scheduler.minRegisteredResourcesRatio: "0.0"
    spark.scheduler.maxRegisteredResourcesWaitingTime: "30s"
    spark.sql.shuffle.partitions: "2"

jupyter:
  enabled: false

hiveMetastore:
  enabled: false

historyServer:
  enabled: false

ingress:
  enabled: false

security:
  podSecurityStandards: false

# Disable optional components
sparkOperator:
  enabled: false

celeborn:
  enabled: false