Tiltfile

# Tiltfile for developing provider-kserve.
#
# Installs the latest released OpenEverest v2 core, then builds and deploys this
# provider with live-reload. You do NOT need a local checkout of the core.
#
# Usage:
#   make dev-up            # create the k3d cluster and run `tilt up`
#   tilt up -f dev/Tiltfile
#
# See dev/README.md for configuration and for developing against a core built
# from source (INSTALL_OPENEVEREST=false).

# ===========================
# Utilities
# ===========================
def str2bool(v):
    return str(v).lower() in ("yes", "true", "t", "1")

# ===========================
# Settings
# ===========================
# The OpenEverest core chart installs CRDs and multiple workloads; 30s (the
# default) is not enough if container images are not cached on cold start.
update_settings(k8s_upsert_timeout_secs=600)

# ===========================
# Configuration (.env)
# ===========================
load('ext://dotenv', 'dotenv')
local_env_file = '{0}/.env'.format(config.main_dir)
if os.path.exists(local_env_file):
    print('Loading local .env file={0}'.format(local_env_file))
    dotenv(fn=local_env_file)

# Whether to install the released OpenEverest core. Set to false when you run
# the core from source in a separate Tilt instance.
install_openeverest = str2bool(os.getenv('INSTALL_OPENEVEREST', 'true'))

# Released OpenEverest core version to install. Defaults to the OpenEverest v2
# dev release (published as a prerelease chart). Empty means latest prerelease.
openeverest_version = os.getenv('OPENEVEREST_VERSION', '2.0.0-dev.2')

# Helm repository that hosts the OpenEverest core chart.
openeverest_helm_repo = os.getenv(
    'OPENEVEREST_HELM_REPO', 'https://openeverest.github.io/helm-charts/')

# OpenEverest git ref for CRDs not yet bundled in the published core chart.
# release-2.0 includes the unified parametersSchema API
# (https://github.com/openeverest/openeverest/pull/2583).
openeverest_crd_branch = os.getenv('OPENEVEREST_BRANCH', 'release-2.0')

# Namespaces.
everest_namespace = os.getenv('EVEREST_NAMESPACE', 'everest-system')
# The provider bundles the KServe controllers and the KServe runtime configs
# (LLMInferenceServiceConfig presets + ClusterServingRuntimes). The presets are
# rendered into the release namespace by the provider chart, and the llmisvc
# controller resolves them from its own POD_NAMESPACE, so the stack works in any
# namespace. Override PROVIDER_NAMESPACE to install elsewhere.
provider_namespace = os.getenv('PROVIDER_NAMESPACE', 'default')

# Whether to deploy MinIO + a BackupStorage CR for testing backups/restores.
enable_minio = str2bool(os.getenv('ENABLE_MINIO', 'false'))

# Whether to install Envoy AI Gateway and deploy the small CPU test Instance
# from dev/resources/ai-gateway-llm.yaml.
enable_ai_gateway = str2bool(os.getenv('ENABLE_AI_GATEWAY', 'false'))

# Whether to install kube-prometheus-stack (Prometheus + Grafana) and provision
# the vLLM Grafana dashboard so model metrics are visible end-to-end.
enable_metrics = str2bool(os.getenv('ENABLE_METRICS', 'false'))

# Whether to install cert-manager. The KServe controllers require it for their
# admission-webhook certificates. Installed as its own release (rather than the
# bundled subchart) so the dev loop is reliable on fresh clusters. Set to false
# when the cluster already runs cert-manager.
install_cert_manager = str2bool(os.getenv('INSTALL_CERT_MANAGER', 'true'))
cert_manager_version = os.getenv('CERT_MANAGER_VERSION', 'v1.16.2')

# Go build environment for the provider binary.
provider_build_env = os.getenv('PROVIDER_BUILD_ENV', 'CGO_ENABLED=0 GOOS=linux')

# Initial admin password for the Everest UI/API.
ui_admin_password = os.getenv('UI_ADMIN_PASSWORD', 'admin')

# ===========================
# Paths
# ===========================
provider_dir = os.path.realpath('{0}/..'.format(config.main_dir))
chart_dir = '{0}/charts/provider-kserve'.format(provider_dir)
print('Using provider dir: {0}'.format(provider_dir))
print('Using provider chart dir: {0}'.format(chart_dir))

# ===========================
# External cluster / registry
# ===========================
k8s_context = os.getenv('K8S_CONTEXT', '')
if k8s_context:
    print('Using Kubernetes context: {0}'.format(k8s_context))
    allow_k8s_contexts(k8s_context)

docker_registry_url = os.getenv('DOCKER_REGISTRY_URL', '')
if docker_registry_url:
    print('Using Docker registry URL: {0}'.format(docker_registry_url))
    default_registry(docker_registry_url)

# ===========================
# Extensions
# ===========================
load('ext://helm_resource', 'helm_resource', 'helm_repo')
load('ext://restart_process', 'docker_build_with_restart')

# ===========================
# OpenEverest core (released)
# ===========================
# Installed with a real `helm upgrade --install` (via the helm_resource
# extension) so that the core CRDs bundled in the chart are applied correctly.
if install_openeverest:
    helm_repo(
        'openeverest-repo',
        openeverest_helm_repo,
        labels=['openeverest'],
    )

    core_flags = [
        '--create-namespace',
        # OpenEverest v2 charts are published as prereleases; --devel lets helm
        # resolve them.
        '--devel',
        '--set', 'server.initialAdminPassword={0}'.format(ui_admin_password),
        '--set', 'telemetry=false',
        '--set', 'upgrade.preflightChecks=false',
        # NOTE: do not disable the monitoring subchart. The core controller
        # unconditionally watches the VictoriaMetrics VMAgent CRD, which is
        # installed by the bundled victoria-metrics-operator subchart; disabling
        # it makes the controller crash on startup ("no matches for kind
        # VMAgent"). kube-state-metrics is the only monitoring piece we can
        # safely skip for a lean dev install.
        '--set', 'kube-state-metrics.enabled=false',
    ]
    if openeverest_version:
        core_flags = core_flags + ['--version', openeverest_version]

    helm_resource(
        'openeverest',
        'openeverest-repo/openeverest',
        release_name='everest',
        namespace=everest_namespace,
        flags=core_flags,
        resource_deps=['openeverest-repo'],
        labels=['openeverest'],
    )

    # Keep the UI reachable at http://localhost:8080 while Tilt is running.
    # helm_resource port_forwards often miss the everest Service; kubectl PF is reliable.
    local_resource(
        'everest-ui',
        serve_cmd='kubectl port-forward svc/everest -n {0} 8080:8080'.format(
            everest_namespace),
        resource_deps=['openeverest'],
        links=[link('http://localhost:8080', 'Everest UI (admin / admin)')],
        labels=['openeverest'],
    )

    # Upgrade OpenEverest CRDs after core install. The published 2.0.0-dev.2
    # chart still ships pre-#2583 schemas (customSpecSchema/configSchema); the
    # provider emits parametersSchema and needs the release-2.0 CRDs.
    openeverest_crd_base = 'https://raw.githubusercontent.com/openeverest/openeverest/{0}/config/crd/bases'.format(
        openeverest_crd_branch)
    openeverest_crd_files = [
        'core.openeverest.io_providers.yaml',
        'core.openeverest.io_instances.yaml',
        'core.openeverest.io_instancepresets.yaml',
        'monitoring.openeverest.io_monitoringconfigs.yaml',
        'backup.openeverest.io_backupclasses.yaml',
        'backup.openeverest.io_backups.yaml',
        'backup.openeverest.io_restores.yaml',
        'backup.openeverest.io_backupstorages.yaml',
    ]
    openeverest_crd_cmds = [
        'kubectl apply --server-side --force-conflicts -f {0}/{1}'.format(
            openeverest_crd_base, crd_file)
        for crd_file in openeverest_crd_files
    ]
    local_resource(
        'openeverest-crds',
        ' && '.join(openeverest_crd_cmds),
        resource_deps=['openeverest'],
        labels=['openeverest'],
    )

# ===========================
# Provider: generate + build
# ===========================
local_resource(
    'provider-generate',
    'make -C {0} manifests helm-sync-rbac'.format(provider_dir),
    deps=['{0}/definition'.format(provider_dir)],
    labels=['provider'],
)

local_resource(
    'provider-build',
    'cd {0} && {1} go build -o bin/provider ./cmd/provider/'.format(
        provider_dir, provider_build_env),
    deps=[
        '{0}/go.mod'.format(provider_dir),
        '{0}/go.sum'.format(provider_dir),
        '{0}/cmd'.format(provider_dir),
        '{0}/internal'.format(provider_dir),
        '{0}/definition'.format(provider_dir),
    ],
    ignore=['{0}/**/*_test.go'.format(provider_dir)],
    resource_deps=['provider-generate'],
    labels=['provider'],
)

# ===========================
# Provider: image (live update)
# ===========================
provider_image = 'ghcr.io/openeverest/provider-kserve:tilt-dev'
docker_build_with_restart(
    provider_image,
    context=provider_dir,
    dockerfile='{0}/dev/provider.Dockerfile'.format(provider_dir),
    target='dev',
    entrypoint=['/home/provider/provider'],
    only=['bin/provider'],
    live_update=[
        sync('{0}/bin/provider'.format(provider_dir), '/home/provider/provider'),
    ],
)

# ===========================
# cert-manager (webhook certs for the KServe controllers)
# ===========================
# Both bundled KServe controllers run admission webhooks whose certificates are
# issued by cert-manager. The provider chart can bundle cert-manager, but
# installing it in the same Helm release as the resources that consume it races
# on a fresh cluster (the cert-manager webhook isn't ready when the Issuer /
# Certificate objects are admitted). So in dev we install cert-manager as its
# own release and disable the bundled copy below — this also mirrors the common
# production case where cert-manager already exists in the cluster.
if install_cert_manager:
    helm_repo(
        'jetstack',
        'https://charts.jetstack.io',
        labels=['cert-manager'],
    )
    helm_resource(
        'cert-manager',
        'jetstack/cert-manager',
        release_name='cert-manager',
        namespace='cert-manager',
        flags=[
            '--create-namespace',
            '--version', cert_manager_version,
            '--set', 'crds.enabled=true',
        ],
        resource_deps=['jetstack'],
        labels=['cert-manager'],
    )

# ===========================
# Provider chart: Helm dependencies
# ===========================
# The provider chart bundles the KServe controllers (kserve-resources,
# kserve-llmisvc-resources) and cert-manager as subchart dependencies. Vendor
# them into charts/ before the chart is installed.
local_resource(
    'provider-helm-deps',
    'make -C {0} helm-deps'.format(provider_dir),
    deps=['{0}/charts/provider-kserve/Chart.yaml'.format(provider_dir)],
    labels=['provider'],
)

# ===========================
# Provider: Helm chart
# ===========================
# Deployed with a real `helm upgrade --install` (via the helm_resource
# extension). Tilt injects the locally built image into the chart's
# image.repository/image.tag values and live-updates the running pod. Using
# helm_resource also guarantees the Provider CR is applied only after the core
# CRDs exist.
#
# First install: KServe CRDs (vendored into crds/, applied before templates) +
# llmisvc controller. kserve-resources is off — its ClusterServingRuntimes and
# LLMInferenceServiceConfigs go through admission webhooks that are not ready
# until the controllers are. Post-install turns kserve-resources on, then the
# webhook-gated resources. cert-manager is its own release, bundled copy off.

provider_resource_deps = ['provider-build', 'provider-helm-deps']
if install_openeverest:
    provider_resource_deps = provider_resource_deps + ['openeverest', 'openeverest-crds']
if install_cert_manager:
    provider_resource_deps = provider_resource_deps + ['cert-manager']

provider_flags = [
    '--create-namespace',
    '--set', 'image.pullPolicy=IfNotPresent',
    '--set', 'securityContext.readOnlyRootFilesystem=false',
    # cert-manager is installed as its own release above.
    '--set', 'cert-manager.enabled=false',
    '--set', 'kserveResources.enabled=false',
    # Disable resources that trigger webhooks (ClusterServingRuntimes,
    # LLMInferenceServiceConfigs). They will be applied after controllers are
    # ready via the post-install resource below.
    '--set', 'kserveRuntimeConfigs.enabled=false',
    '--set', 'llmPresets.enabled=false',
    '--set', 'cpuProfile.enabled=false',
]
if enable_ai_gateway:
    provider_flags = provider_flags + [
        '--set', 'aiGateway.enabled=true',
        '--set', 'envoy-ai-gateway.envoyGateway.namespace={0}'.format(
            provider_namespace),
    ]
if enable_metrics:
    # Labeled ConfigMaps for kube-prometheus-stack Grafana sidecar (see
    # charts/provider-kserve/templates/grafana-dashboards.yaml).
    provider_flags = provider_flags + [
        '--set', 'metrics.grafanaDashboards.enabled=true',
        '--set', 'metrics.grafanaDashboards.namespace=monitoring',
    ]

helm_resource(
    'provider-kserve',
    chart_dir,
    namespace=provider_namespace,
    flags=provider_flags,
    image_deps=[provider_image],
    image_keys=[('image.repository', 'image.tag')],
    resource_deps=provider_resource_deps,
    labels=['provider'],
)

# ===========================
# Post-install: enable runtime configs after controllers are ready
# ===========================
# Enable kserve-resources now that the CRDs exist, wait for both controllers,
# then enable resources their webhooks validate (ClusterServingRuntimes,
# LLMInferenceServiceConfigs).
post_install_cmd = (
    'helm upgrade provider-kserve {chart} --namespace {ns} --reuse-values '
    + '--set image.pullPolicy=IfNotPresent '
    + '--set securityContext.readOnlyRootFilesystem=false '
    + '--set cert-manager.enabled=false '
    + '--set kserveResources.enabled=true '
    + '--set kserveRuntimeConfigs.enabled=false '
    + '--set llmPresets.enabled=false '
    + '--set cpuProfile.enabled=false && '
    + 'echo "Waiting for KServe controllers to exist..." && '
    + 'for i in $(seq 1 90); do kubectl get deployment/kserve-controller-manager -n {ns} >/dev/null 2>&1 && break; echo "Attempt $i: not found yet, sleeping 5s..."; sleep 5; done && '
    + 'kubectl get deployment/kserve-controller-manager -n {ns} || (echo "ERROR: kserve-controller-manager never appeared" && exit 1) && '
    + 'kubectl wait --for=condition=available deployment/kserve-controller-manager -n {ns} --timeout=300s && '
    + 'kubectl wait --for=condition=available deployment/llmisvc-controller-manager -n {ns} --timeout=300s && '
    + 'sleep 10 && '
    + 'helm upgrade provider-kserve {chart} --namespace {ns} --reuse-values '
    + '--set image.pullPolicy=IfNotPresent '
    + '--set securityContext.readOnlyRootFilesystem=false '
    + '--set cert-manager.enabled=false '
    + '--set kserveResources.enabled=true '
    + '--set kserveRuntimeConfigs.enabled=true '
    + '--set llmPresets.enabled=true '
    + '--set cpuProfile.enabled=true'
).format(ns=provider_namespace, chart=chart_dir)

if enable_ai_gateway:
    post_install_cmd = post_install_cmd + ' --set aiGateway.enabled=true --set envoy-ai-gateway.envoyGateway.namespace={0}'.format(provider_namespace)

if enable_metrics:
    post_install_cmd = post_install_cmd + ' --set metrics.grafanaDashboards.enabled=true --set metrics.grafanaDashboards.namespace=monitoring'

local_resource(
    'provider-kserve-post-install',
    post_install_cmd,
    resource_deps=['provider-kserve'],
    labels=['provider'],
)

# ===========================
# Envoy AI Gateway test model — optional
# ===========================
if enable_ai_gateway:
    ai_gateway_llm_yaml = str(read_file(
        '{0}/resources/ai-gateway-llm.yaml'.format(config.main_dir)))
    ai_gateway_llm_yaml = ai_gateway_llm_yaml.replace(
        '__NAMESPACE__', provider_namespace)
    k8s_yaml(blob(ai_gateway_llm_yaml))
    k8s_resource(
        objects=[
            'ai-gateway-smollm:instance:{0}'.format(provider_namespace),
        ],
        new_name='ai-gateway-test-model',
        resource_deps=['provider-kserve'],
        labels=['ai-gateway'],
    )

# ===========================
# Backup storage (MinIO) — optional
# ===========================
if enable_minio:
    k8s_yaml('{0}/resources/minio.yaml'.format(config.main_dir))
    k8s_resource(
        workload='minio',
        objects=['minio:namespace'],
        port_forwards=[port_forward(9090, 9090, 'MinIO Console')],
        labels=['backup-storage'],
    )

    # Render the BackupStorage CR (+ credentials Secret) into the provider
    # namespace so the provider can be tested against it.
    backupstorage_yaml = str(read_file('{0}/resources/backupstorage.yaml'.format(
        config.main_dir)))
    backupstorage_yaml = backupstorage_yaml.replace(
        '__NAMESPACE__', provider_namespace)
    k8s_yaml(blob(backupstorage_yaml))
    k8s_resource(
        objects=[
            'minio-credentials:secret:{0}'.format(provider_namespace),
            'minio-s3:backupstorage:{0}'.format(provider_namespace),
        ],
        new_name='backup-storage',
        resource_deps=['provider-kserve'],
        labels=['backup-storage'],
    )

# ===========================
# Metrics: Prometheus + Grafana + dashboard ConfigMaps — optional
# ===========================
# Installs kube-prometheus-stack (Prometheus Operator + Prometheus + Grafana).
# Dashboard JSON is provisioned via labeled ConfigMaps on the provider Helm
# chart (metrics.grafanaDashboards.enabled=true when ENABLE_METRICS is set).
# The provider emits PodMonitors; kube-prom Prometheus scrapes them.
if enable_metrics:
    helm_repo(
        'prometheus-community',
        'https://prometheus-community.github.io/helm-charts',
        labels=['metrics'],
    )
    helm_resource(
        'kube-prometheus-stack',
        'prometheus-community/kube-prometheus-stack',
        release_name='monitoring',
        namespace='monitoring',
        flags=[
            '--create-namespace',
            # Read ALL PodMonitors, not only those labeled release=monitoring.
            # The provider's PodMonitor has no such label, so without this
            # Prometheus would silently ignore it (the classic empty-dashboard
            # gotcha). See docs/observability.md.
            '--set', 'prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false',
            '--set', 'prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false',
        ],
        resource_deps=['prometheus-community'],
        labels=['metrics'],
    )

    # Grafana is on Service port 80; forward it explicitly (reliable, unlike
    # attaching to one of the stack's several pods). Default creds: admin /
    # prom-operator. Dashboard ConfigMaps come from provider-kserve Helm.
    local_resource(
        'grafana',
        serve_cmd='kubectl port-forward -n monitoring svc/monitoring-grafana 3000:80',
        resource_deps=['kube-prometheus-stack', 'provider-kserve'],
        links=[link('http://localhost:3000', 'Grafana (admin / prom-operator)')],
        labels=['metrics'],
    )