Taskfile.yml

# Copyright 2025 Snowflake Inc.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

version: "3"

vars:
  # SKILL_DIR: where the polaris-local-forge source lives (follows symlinks)
  # Uses ROOT_DIR (absolute path to Taskfile dir) so it works from any cwd.
  # readlink -f follows symlinks for isolated test folders where Taskfile.yml is a symlink.
  SKILL_DIR:
    sh: dirname "$(readlink -f "{{.ROOT_DIR}}/Taskfile.yml" 2>/dev/null || echo "{{.ROOT_DIR}}/Taskfile.yml")"
  # WORK_DIR: explicit work directory override (empty = use pwd)
  # Usage: task setup:all WORK_DIR=/path/to/project
  # Supports: ~/path, $HOME/path, $ENV_VAR/path
  # Read from env so plforge's PLF_WORK_DIR export is honored
  WORK_DIR:
    sh: 'echo "${WORK_DIR:-}"'
  # PROJECT_HOME: WORK_DIR if specified, otherwise current working directory
  # Uses eval to expand ~ and $ENV_VARS that Go-Task doesn't expand
  PROJECT_HOME:
    sh: |
      if [ -n "{{.WORK_DIR}}" ]; then
        eval echo "{{.WORK_DIR}}"
      else
        echo "$PWD"
      fi
  KUBECONFIG: "{{.PROJECT_HOME}}/.kube/config"
  K3D_CLUSTER_NAME:
    sh: basename $(pwd)
  K3S_VERSION: v1.35.1-k3s1
  FEATURES_DIR: "{{.PROJECT_HOME}}/k8s"
  PODMAN_MACHINE: k3d
  # PLF command - uses --project to find source, --work-dir for output
  # VIRTUAL_ENV= inline is the primary suppression; env block below is a fallback
  PLF: VIRTUAL_ENV= uv run --project "{{.SKILL_DIR}}" polaris-local-forge --work-dir "{{.PROJECT_HOME}}"
  # Detect if running in source repo (has SKILL.md + src/polaris_local_forge)
  # Used to prevent accidental writes to the read-only source directory
  IS_SOURCE_REPO:
    sh: '[ -f "{{.PROJECT_HOME}}/SKILL.md" ] && [ -d "{{.PROJECT_HOME}}/src/polaris_local_forge" ] && echo "true" || echo "false"'
  # Files/directories to clean up on 'teardown -- all'
  # .snow-utils is ALWAYS preserved for replay/audit
  CLEANUP_PATHS: ".kube work k8s scripts bin notebooks datasets .env .aws .envrc .gitignore .venv pyproject.toml uv.lock"

silent: true

env:
  KUBECONFIG: "{{.KUBECONFIG}}"
  # Fallback: Go-Task env block doesn't reliably override parent shell values
  # before sibling sh: evaluations. Primary suppression is VIRTUAL_ENV= inline in PLF var.
  VIRTUAL_ENV: ""
  # DOCKER_HOST via CLI - single source of truth for Podman SSH socket
  DOCKER_HOST:
    sh: VIRTUAL_ENV= uv run --project "{{.SKILL_DIR}}" polaris-local-forge --work-dir "{{.PROJECT_HOME}}" runtime docker-host 2>/dev/null || true

includes:
  podman: ./taskfiles/podman.yml
  cluster: ./taskfiles/cluster.yml
  polaris: ./taskfiles/polaris.yml
  catalog: ./taskfiles/catalog.yml
  ops: ./taskfiles/ops.yml
  l2c: ./taskfiles/l2c.yml

tasks:
  # ============================================
  # Installation
  # ============================================

  install:
    desc: Install plforge command and shell completions
    summary: |
      Symlinks plforge to ~/.local/bin and generates Zsh completions.
      Ensure ~/.local/bin is in PATH and completions dir is in fpath.
      
      Options:
        --bash    Also append Bash completions to ~/.bashrc
      
      Example:
        plforge install
        plforge install --bash
    cmds:
      - |
        mkdir -p ~/.local/bin
        ln -sf "{{.SKILL_DIR}}/plforge" ~/.local/bin/plforge
        echo "Installed: ~/.local/bin/plforge -> {{.SKILL_DIR}}/plforge"
      - |
        mkdir -p ~/.config/zsh/completions
        "{{.SKILL_DIR}}/plforge" --completion zsh > ~/.config/zsh/completions/_plforge
        echo "Installed: ~/.config/zsh/completions/_plforge"
      - |
        if [ "{{.CLI_ARGS}}" = "--bash" ]; then
          "{{.SKILL_DIR}}/plforge" --completion bash >> ~/.bashrc
          echo "Installed: Bash completions appended to ~/.bashrc"
        fi
      - |
        echo ""
        echo "Done. Restart your shell or run: exec zsh"

  install:remove:
    desc: Remove plforge command and shell completions
    summary: |
      Removes ~/.local/bin/plforge symlink and Zsh completions file.
      Bash completions in ~/.bashrc must be removed manually if added.
      
      Example:
        plforge install:remove
    cmds:
      - rm -f ~/.local/bin/plforge
      - rm -f ~/.config/zsh/completions/_plforge
      - |
        echo "Removed plforge and completions."
        echo "Note: Bash completions in ~/.bashrc must be removed manually if added."

  install:uv:
    desc: Install uv Python package manager
    summary: |
      Installs uv if not already present. Skipped if uv is on PATH.
      
      Example:
        plforge install:uv
    cmds:
      - |
        if ! command -v uv &> /dev/null; then
          echo "Installing uv..."
          pip install uv || curl -LsSf https://astral.sh/uv/install.sh | sh
        else
          echo "uv is already installed ($(uv --version))"
        fi
    status:
      - command -v uv

  # ============================================
  # Manifest Management
  # ============================================

  manifest:init:
    desc: Initialize manifest with PENDING status
    summary: |
      Creates .snow-utils/snow-utils-manifest.md with all resources set to PENDING.
      
      Variables:
        WORK_DIR    Target directory (or set PLF_WORK_DIR env var)
      
      Example:
        plforge manifest:init
    cmds:
      - "{{.PLF}} setup manifest init"

  manifest:start:
    desc: Set manifest status to IN_PROGRESS
    summary: |
      Updates manifest status from PENDING to IN_PROGRESS.
      Called at the start of the setup workflow.
      
      Example:
        plforge manifest:start
    cmds:
      - "{{.PLF}} setup manifest start"

  manifest:update:
    desc: Update a resource row to DONE
    summary: |
      Updates a specific resource row status to DONE.
      
      Arguments:
        Resource number (1-7)
          1 = k3d cluster    2 = RustFS     3 = PostgreSQL
          4 = Polaris        5 = Catalog    6 = Principal
          7 = Demo data
      
      Example:
        plforge manifest:update 1   # Mark k3d cluster as DONE
        plforge manifest:update 4   # Mark Polaris as DONE
    cmds:
      - "{{.PLF}} setup manifest update {{.CLI_ARGS}}"

  manifest:complete:
    desc: Set manifest status to COMPLETE
    summary: |
      Updates manifest status to COMPLETE.
      Called after all resources are successfully created.
      
      Example:
        plforge manifest:complete
    cmds:
      - "{{.PLF}} setup manifest complete"

  manifest:remove:
    desc: Set manifest status to REMOVED
    summary: |
      Updates manifest status and all resource statuses to REMOVED.
      Called automatically by teardown.
      
      Example:
        plforge manifest:remove
    cmds:
      - "{{.PLF}} setup manifest remove"

  # ============================================
  # Setup and Initialization
  # ============================================

  setup:python:
    desc: Set up Python environment using uv
    summary: |
      Creates a Python 3.12 virtual environment and installs dependencies.
      
      Example:
        plforge setup:python
    deps:
      - install:uv
    cmds:
      - uv python pin 3.12
      - uv venv
      - uv sync
      - |
        echo ""
        echo "Python environment setup complete."
        echo ""
        echo "Next step: Activate the virtual environment in your shell:"
        echo "  source .venv/bin/activate"
        echo ""

  prepare:
    desc: Generate required sensitive files from templates using Ansible
    summary: |
      Generates sensitive files from Ansible templates (RSA keys, K8s manifests, credentials).
      
      Variables:
        WORK_DIR    Target directory (or set PLF_WORK_DIR env var)
        TAGS        Comma-separated Ansible tags (default: all)
                    Available: rsa, setup, bootstrap, k8s
      
      Example:
        plforge prepare
        plforge prepare TAGS=rsa,k8s
    preconditions:
      - sh: '[ "{{.IS_SOURCE_REPO}}" != "true" ]'
        msg: |
          ERROR: Cannot run prepare in the source repository.
          
          Use WORK_DIR to specify the target directory:
            task prepare WORK_DIR=~/polaris-dev
      - sh: test -f .venv/bin/activate || test -f .venv/Scripts/activate
        msg: "Python venv not found. Run 'task setup:python' first."
    cmds:
      - "{{.PLF}} prepare {{if .TAGS}}--tags={{.TAGS}}{{end}}"

  runtime:setup:
    desc: Ensure container runtime is ready (Podman machine setup if needed)
    summary: |
      Detects and starts the container runtime (Podman or Docker).
      On macOS, starts the Podman machine if not running.
      
      Example:
        plforge runtime:setup
    cmds:
      - "{{.PLF}} setup runtime ensure"

  runtime:teardown:
    desc: Stop container runtime after teardown (Podman machine stop if needed)
    summary: |
      Stops the Podman machine (macOS). No-op on Linux with native Podman.
      
      Example:
        plforge runtime:teardown
    cmds:
      - "{{.PLF}} setup runtime stop"

  setup:
    desc: "Complete setup (alias for setup:all)"
    summary: |
      Alias for setup:all. Creates a complete Apache Polaris environment.

      Variables:
        WORK_DIR    Isolated project directory (or set PLF_WORK_DIR env var)

      Example:
        plforge setup
        plforge setup WORK_DIR=~/polaris-dev
    cmds:
      - task: setup:all

  setup:all:
    desc: "Complete setup with manifest tracking. Use WORK_DIR=/path for isolated setup"
    summary: |
      Creates a complete Apache Polaris environment with manifest tracking.
      
      Steps: init -> doctor -> prepare -> cluster -> Polaris -> catalog -> load -> verify
      Manifest: .snow-utils/snow-utils-manifest.md
      
      Variables:
        WORK_DIR    Isolated project directory (or set PLF_WORK_DIR env var)
      
      Example:
        plforge setup:all
        plforge setup:all WORK_DIR=~/polaris-dev
    preconditions:
      - sh: '[ "{{.IS_SOURCE_REPO}}" != "true" ]'
        msg: |
          ERROR: Cannot run setup in the source repository.
          
          The skill directory is READ-ONLY. Use WORK_DIR to specify a separate directory:
            task setup:all WORK_DIR=~/polaris-dev
    cmds:
      # Initialize manifest with PENDING status
      - task: manifest:init
      # Set status to IN_PROGRESS
      - task: manifest:start
      # Same sequence as SKILL.md agentic workflow
      - "{{.PLF}} init"
      - "{{.PLF}} doctor --fix"
      - "{{.PLF}} prepare"
      - "{{.PLF}} cluster create"
      # Mark k3d cluster as DONE
      - task: manifest:update
        vars: { CLI_ARGS: "1" }
      - task: cluster:bootstrap-check
      # Mark RustFS and PostgreSQL as DONE (they come up with cluster)
      - task: manifest:update
        vars: { CLI_ARGS: "2" }
      - task: manifest:update
        vars: { CLI_ARGS: "3" }
      - "{{.PLF}} polaris deploy"
      - "{{.PLF}} polaris bootstrap"
      - task: cluster:polaris-check
      # Mark Polaris and Principal as DONE
      - task: manifest:update
        vars: { CLI_ARGS: "4" }
      - task: manifest:update
        vars: { CLI_ARGS: "6" }
      - "{{.PLF}} catalog setup"
      # Mark Catalog as DONE
      - task: manifest:update
        vars: { CLI_ARGS: "5" }
      # Load demo data and verify
      - task: catalog:load
        vars: { DATASET: "wildlife" }
      - task: catalog:verify
      # Mark Demo data as DONE
      - task: manifest:update
        vars: { CLI_ARGS: "7" }
      # Set status to COMPLETE
      - task: manifest:complete
      - 'echo ""'
      - 'echo "Setup complete! Test data verified."'
      - 'echo "Manifest: .snow-utils/snow-utils-manifest.md"'
      - 'echo "Explore further with: task catalog:generate-notebook"'

  setup:replay:
    desc: "Replay or resume setup from manifest"
    summary: |
      Checks manifest status and continues appropriately:
        REMOVED     -> Full replay (re-run setup:all)
        IN_PROGRESS -> Resume from first PENDING resource
        COMPLETE    -> Already done, offer reset option
        No manifest -> Error, run setup:all instead
      
      Variables:
        WORK_DIR    Target directory with manifest (or set PLF_WORK_DIR env var)
      
      Example:
        plforge setup:replay WORK_DIR=~/polaris-dev
    preconditions:
      - sh: '[ "{{.IS_SOURCE_REPO}}" != "true" ]'
        msg: |
          ERROR: Cannot run setup:replay in the source repository.
          
          Use WORK_DIR to specify the target directory:
            task setup:replay WORK_DIR=~/polaris-dev
    cmds:
      - |
        {{.PLF}} setup replay
        EXIT_CODE=$?
        case $EXIT_CODE in
          0)  ;; # COMPLETE and healthy - nothing to do
          10) task setup:all WORK_DIR="{{.PROJECT_HOME}}" ;;
          11) task setup:resume WORK_DIR="{{.PROJECT_HOME}}" ;;
          *)  exit $EXIT_CODE ;;
        esac

  setup:resume:
    desc: "Resume setup from IN_PROGRESS manifest (internal)"
    summary: |
      Resumes setup from an IN_PROGRESS manifest by checking which resources
      are still PENDING and running only those steps.
      
      This is typically called by setup:replay, not directly.
    cmds:
      - "{{.PLF}} setup resume"

  teardown:
    desc: "Teardown cluster with confirmation prompt"
    summary: |
      Deletes the k3d cluster and all Polaris resources.
      Prompts for confirmation before proceeding.
      
      Options:
        all     Also clean generated files (preserves .snow-utils for replay/audit)
      
      Variables:
        WORK_DIR    Target directory (or set PLF_WORK_DIR env var)
      
      Example:
        plforge teardown            # Cluster only (keeps local files)
        plforge teardown all        # Also clean generated files
    preconditions:
      - sh: '[ "{{.IS_SOURCE_REPO}}" != "true" ]'
        msg: |
          ERROR: Cannot run teardown in the source repository.
          
          Use WORK_DIR to specify the target directory:
            task teardown WORK_DIR=~/polaris-dev
    prompt: |
      This will delete the k3d cluster and all resources.
      
      Cleanup scope:
        - Default:      Cluster only (keeps all local files)
        - With 'all':   Clean generated files (preserves .snow-utils)
      
      Continue?
    cmds:
      - "{{.PLF}} teardown --yes"
      - task: manifest:remove
      - |
        if [ "{{.CLI_ARGS}}" = "all" ]; then
          echo "Cleaning local directory..."
          for item in {{.CLEANUP_PATHS}}; do
            rm -rf "{{.PROJECT_HOME}}/$item"
          done
          echo ""
          echo "Local directory cleaned."
          echo "Note: .snow-utils/ preserved for audit and replay."
        fi
      - 'echo ""'
      - 'echo "Teardown complete."'

  teardown:full:
    desc: Complete teardown - cluster + Podman stop + clean generated files
    summary: |
      Deletes cluster, stops Podman machine, and cleans generated files.
      Preserves .snow-utils for replay/audit. Prompts for confirmation.
      
      Variables:
        WORK_DIR    Target directory (or set PLF_WORK_DIR env var)
      
      Example:
        plforge teardown:full
        plforge teardown:full WORK_DIR=~/polaris-dev
    preconditions:
      - sh: '[ "{{.IS_SOURCE_REPO}}" != "true" ]'
        msg: |
          ERROR: Cannot run teardown:full in the source repository.
          
          Use WORK_DIR to specify the target directory:
            task teardown:full WORK_DIR=~/polaris-dev
    prompt: |
      This will delete the k3d cluster, all resources, stop the Podman machine,
      AND clean generated files (preserves .snow-utils for replay/audit).
      
      Continue?
    cmds:
      - "{{.PLF}} teardown --yes --stop-podman"
      - task: manifest:remove
      - |
        echo "Cleaning local directory..."
        for item in {{.CLEANUP_PATHS}}; do
          rm -rf "{{.PROJECT_HOME}}/$item"
        done
        echo ""
        echo "Local directory cleaned."
        echo "Note: .snow-utils/ preserved for audit and replay."
      - 'echo ""'
      - 'echo "Teardown complete."'

  reset:all:
    desc: Complete reset - delete cluster, recreate everything with fresh catalog
    summary: |
      Runs teardown then setup:all. Destroys and recreates everything from scratch.
      
      Example:
        plforge reset:all
    cmds:
      - task: teardown
      - task: setup:all

  clean:all:
    desc: Delete cluster and all resources (use teardown for full cleanup)
    summary: |
      Deletes cluster without stopping Podman or updating manifest.
      For full cleanup with prompts, use teardown instead.
      
      Example:
        plforge clean:all
    cmds:
      - "{{.PLF}} teardown --yes --no-stop-podman"

  cluster:reset:
    desc: Delete and recreate cluster with fresh catalog
    summary: |
      Deletes cluster, recreates it, deploys Polaris, and sets up catalog.
      Faster than reset:all since it skips init/doctor/prepare.
      
      Example:
        plforge cluster:reset
    cmds:
      - task: cluster:delete
      - task: cluster:create
      - task: cluster:bootstrap-check
      - task: polaris:deploy
      - task: cluster:polaris-check
      - task: catalog:setup
      - 'echo ""'
      - 'echo "Cluster reset complete."'
      - 'echo "All services are running with a fresh catalog."'

  catalog:reset:full:
    desc: Full reset - purge Polaris database and recreate catalog
    summary: |
      Purges the entire Polaris database and recreates the catalog from scratch.
      Keeps the cluster running. More drastic than catalog:reset.
      
      Example:
        plforge catalog:reset:full
    cmds:
      - echo "WARNING -- This will purge the entire Polaris database and recreate from scratch."
      - task: polaris:reset
      - task: catalog:setup
      - 'echo ""'
      - 'echo "Full catalog reset complete."'
      - 'echo "Polaris database was purged and catalog recreated."'
      - 'echo ""'
      - 'echo "Verification options:"'
      - 'echo "  - Run DuckDB verification: task verify"'
      - 'echo "  - Generate notebook: task catalog:generate-notebook"'

  # ============================================
  # Quick Reference / Aliases
  # ============================================

  doctor:
    desc: Check system prerequisites and health (uses CLI for consistent detection)
    summary: |
      Checks system prerequisites and attempts to fix issues.
      
      Options:
        --fix       Auto-fix issues (e.g., start Podman machine)
        --verbose   Show detailed output
      
      Checks:
        - Required tools: podman/docker, k3d, kubectl, uv
        - Podman machine status and SSH config (macOS)
        - Port availability: 19000, 19001, 18181
      
      Example:
        plforge doctor
        plforge doctor --fix
    cmds:
      - "{{.PLF}} doctor {{.CLI_ARGS}}"

  config:
    desc: Show current configuration from .env file
    summary: |
      Displays non-commented, non-empty .env values sorted alphabetically.
      
      Variables:
        WORK_DIR    Target directory (or set PLF_WORK_DIR env var)
      
      Example:
        plforge config
    cmds:
      - |
        if [ -f "{{.PROJECT_HOME}}/.env" ]; then
          echo "=== Configuration ({{.PROJECT_HOME}}/.env) ==="
          grep -v '^#' "{{.PROJECT_HOME}}/.env" | grep -v '^$' | sort
        else
          echo ".env file not found. Run 'task prepare' first."
        fi

  status:
    desc: Show status of cluster and Polaris
    summary: |
      Shows k3d cluster list and Polaris namespace deployments/pods.
      
      Example:
        plforge status
    cmds:
      - |
        echo "=== Cluster Status ==="
        k3d cluster list
        echo ""
        echo "=== Polaris Status ==="
        kubectl get deployments,pods -n polaris 2>/dev/null || echo "Cluster not running or polaris namespace not found"

  verify:
    desc: Verify Polaris catalog data with DuckDB read-only analysis
    summary: |
      Runs DuckDB read-only analysis to verify catalog data.
      Alias for catalog:verify.

      Example:
        plforge verify
    cmds:
      - task: catalog:verify


  # ============================================
  # Testing
  # ============================================

  test:isolated:
    desc: Create isolated test environment (protects source tree)
    summary: |
      Creates a test environment in /tmp/plf-test-<pid> with symlinked Taskfile.
      Protects the source tree from accidental writes.
      
      Example:
        plforge test:isolated
    cmds:
      - ./test/isolated-test.sh setup

  test:isolated:clean:
    desc: Remove all isolated test folders
    summary: |
      Removes all /tmp/plf-test-* directories.
      
      Example:
        plforge test:isolated:clean
    cmds:
      - ./test/isolated-test.sh clean

  test:isolated:list:
    desc: List existing isolated test folders
    summary: |
      Lists all existing /tmp/plf-test-* directories.
      
      Example:
        plforge test:isolated:list
    cmds:
      - ./test/isolated-test.sh list

  help:
    desc: Show available tasks with descriptions
    summary: |
      Lists all available tasks with descriptions.
      
      Common variables (pass as VAR=value, or set PLF_WORK_DIR env var):
        WORK_DIR    Isolated project directory (or set PLF_WORK_DIR)
        TAGS        Filter Ansible tags for prepare/catalog commands
      
      Example:
        plforge help
        plforge <task> --help       # Detailed help for a task
    cmds:
      - task --list