install_gcs_git_tools

#!/usr/bin/env bash
set -euo pipefail

INSTALL_DIR="${HOME}/.local/bin"

mkdir -p "$INSTALL_DIR"

cat > "$INSTALL_DIR/git_gcs_artifacts" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail

LOCATION_DEFAULT="US"
STORAGE_CLASS_DEFAULT="STANDARD"

usage() {
  cat <<'USAGE'
Usage:
  git_gcs_artifacts init [--bucket BUCKET] [--location LOCATION] [--project PROJECT]

  git_gcs_artifacts add --local PATH \
    [--bucket BUCKET] [--gcs-prefix PREFIX] [--pointer POINTER.gcs] \
    [--location LOCATION] [--project PROJECT] [--storage-class CLASS] \
    [--commit] [--push]

  git_gcs_artifacts pull \
    [--pointer POINTER.gcs | --dir DIRECTORY | --all] \
    [--bucket BUCKET] [--location LOCATION] [--project PROJECT]

  git_gcs_artifacts status [--dir DIRECTORY]

Notes:
  - Default bucket is git-<org>-<repo>, inferred from origin remote.
  - If origin cannot be parsed, fallback bucket is git-local-<repo>.
  - Files and directories are supported.
  - Pointer files are created next to the target artifact by default.
USAGE
}

die() { echo "ERROR: $*" >&2; exit 1; }

need_cmd() {
  command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1"
}

ensure_git_repo() {
  git rev-parse --is-inside-work-tree >/dev/null 2>&1 || die "Not inside a git repo."
}

repo_root() {
  git rev-parse --show-toplevel
}

sanitize_name() {
  local s="$1"
  s="$(printf '%s' "$s" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-+//; s/-+$//; s/-+/-/g')"
  [[ -n "$s" ]] || die "Could not sanitize name"
  printf '%s' "$s"
}

git_remote_url() {
  git remote get-url origin 2>/dev/null || true
}

infer_org_and_repo() {
  local url path org repo root
  url="$(git_remote_url)"

  if [[ -n "$url" ]]; then
    case "$url" in
      git@*:*/*)
        path="${url#*:}"
        ;;
      ssh://git@*/*)
        path="$(printf '%s' "$url" | sed -E 's#^ssh://git@[^/]+/##')"
        ;;
      https://*/*/*|http://*/*/*)
        path="$(printf '%s' "$url" | sed -E 's#^https?://[^/]+/##')"
        ;;
      *)
        path=""
        ;;
    esac

    if [[ -n "$path" ]]; then
      path="${path%.git}"
      org="$(printf '%s' "$path" | cut -d/ -f1)"
      repo="$(basename "$path")"
      if [[ -n "$org" && -n "$repo" ]]; then
        printf '%s\t%s\n' "$(sanitize_name "$org")" "$(sanitize_name "$repo")"
        return 0
      fi
    fi
  fi

  root="$(repo_root)"
  repo="$(basename "$root")"
  printf 'local\t%s\n' "$(sanitize_name "$repo")"
}

default_bucket() {
  local org repo
  IFS=$'\t' read -r org repo < <(infer_org_and_repo)
  printf 'git-%s-%s' "$org" "$repo"
}

sha256_file() {
  sha256sum "$1" | awk '{print $1}'
}

file_size_bytes() {
  stat -c '%s' "$1"
}

rel_to_repo() {
  local p="$1"
  local root abs
  root="$(repo_root)"
  abs="$(realpath "$p")"
  case "$abs" in
    "$root"/*) printf '%s' "${abs#$root/}" ;;
    "$root") printf '.' ;;
    *) die "Path is outside repo: $p" ;;
  esac
}

ensure_bucket() {
  local bucket="$1" location="$2" project="$3" storage_class="$4"
  local uri="gs://$bucket"

  if gcloud storage buckets describe "$uri" >/dev/null 2>&1; then
    return 0
  fi

  echo "Bucket not found. Creating: $uri"
  if [[ -n "$project" ]]; then
    gcloud storage buckets create "$uri" \
      --project="$project" \
      --location="$location" \
      --default-storage-class="$storage_class"
  else
    gcloud storage buckets create "$uri" \
      --location="$location" \
      --default-storage-class="$storage_class"
  fi
}

append_gitignore_exact() {
  local entry="$1"
  local gi
  gi="$(repo_root)/.gitignore"
  touch "$gi"
  if ! grep -Fxq "$entry" "$gi"; then
    printf '%s\n' "$entry" >> "$gi"
  fi
}

ensure_pointer_tracked() {
  local ptr_rel="$1"
  append_gitignore_exact "!/$ptr_rel"
}

read_kv() {
  local key="$1" ptr="$2"
  awk -F': ' -v k="$key" '$1 == k {sub($1 FS, ""); print; exit}' "$ptr"
}

build_dir_manifest() {
  local dir="$1" out="$2"
  : > "$out"
  while IFS= read -r -d '' f; do
    local rel sha size
    rel="$(realpath --relative-to="$dir" "$f")"
    sha="$(sha256_file "$f")"
    size="$(file_size_bytes "$f")"
    printf '%s\t%s\t%s\n' "$rel" "$sha" "$size" >> "$out"
  done < <(find "$dir" -type f -print0 | sort -z)
}

cmd_init() {
  local bucket="" location="$LOCATION_DEFAULT" project=""
  while [[ $# -gt 0 ]]; do
    case "$1" in
      --bucket) bucket="$2"; shift 2 ;;
      --location) location="$2"; shift 2 ;;
      --project) project="$2"; shift 2 ;;
      -h|--help) usage; exit 0 ;;
      *) die "Unknown arg: $1" ;;
    esac
  done

  ensure_git_repo
  need_cmd gcloud
  need_cmd git
  need_cmd realpath
  need_cmd sha256sum
  need_cmd stat

  [[ -n "$bucket" ]] || bucket="$(default_bucket)"
  ensure_bucket "$bucket" "$location" "$project" "$STORAGE_CLASS_DEFAULT"

  touch .gitignore
  if ! grep -Fq '# --- git-gcs managed artifacts ---' .gitignore; then
    cat >> .gitignore <<'GITEOF'

# --- git-gcs managed artifacts ---
# Exact managed file/directory paths are appended below.
GITEOF
  fi

  echo "Init complete."
  echo "Repo root: $(repo_root)"
  echo "Bucket:    gs://$bucket"
}

cmd_add() {
  local local_path="" pointer="" bucket="" gcs_prefix="" project="" location="$LOCATION_DEFAULT"
  local storage_class="$STORAGE_CLASS_DEFAULT" do_commit=0 do_push=0

  while [[ $# -gt 0 ]]; do
    case "$1" in
      --local) local_path="$2"; shift 2 ;;
      --pointer) pointer="$2"; shift 2 ;;
      --bucket) bucket="$2"; shift 2 ;;
      --gcs-prefix) gcs_prefix="$2"; shift 2 ;;
      --project) project="$2"; shift 2 ;;
      --location) location="$2"; shift 2 ;;
      --storage-class) storage_class="$2"; shift 2 ;;
      --commit) do_commit=1; shift ;;
      --push) do_push=1; shift ;;
      -h|--help) usage; exit 0 ;;
      *) die "Unknown arg: $1" ;;
    esac
  done

  ensure_git_repo
  need_cmd gcloud
  need_cmd git
  need_cmd sha256sum
  need_cmd realpath
  need_cmd stat

  [[ -n "$local_path" ]] || die "--local is required"
  [[ -e "$local_path" ]] || die "Local path not found: $local_path"

  [[ -n "$bucket" ]] || bucket="$(default_bucket)"
  ensure_bucket "$bucket" "$location" "$project" "$storage_class"

  local root rel base pointer_abs pointer_rel gcs_base
  root="$(repo_root)"
  rel="$(rel_to_repo "$local_path")"
  base="$(basename "$rel")"

  if [[ -z "$pointer" ]]; then
    pointer="$rel.gcs"
  fi

  pointer_abs="$(realpath -m "$root/$pointer")"
  case "$pointer_abs" in
    "$root"/*) : ;;
    *) die "Pointer path must be inside repo: $pointer" ;;
  esac
  pointer_rel="${pointer_abs#$root/}"

  if [[ -n "$gcs_prefix" ]]; then
    gcs_base="${gcs_prefix%/}/$base"
  else
    gcs_base="$rel"
  fi

  mkdir -p "$(dirname "$pointer_abs")"

  if [[ -f "$local_path" ]]; then
    local sha size uri
    sha="$(sha256_file "$local_path")"
    size="$(file_size_bytes "$local_path")"
    uri="gs://$bucket/$gcs_base"

    echo "Uploading file: $local_path"
    echo "To:            $uri"
    gcloud storage cp "$local_path" "$uri"

    cat > "$pointer_abs" <<POINTEREOF
version: 1
type: file
uri: $uri
sha256: $sha
size_bytes: $size
source_repo_relpath: $rel
POINTEREOF

    append_gitignore_exact "/$rel"
    ensure_pointer_tracked "$pointer_rel"

  elif [[ -d "$local_path" ]]; then
    local uri manifest_tmp manifest_uri manifest_sha file_count total_bytes
    uri="gs://$bucket/$gcs_base"
    manifest_tmp="$(mktemp)"
    build_dir_manifest "$local_path" "$manifest_tmp"
    manifest_sha="$(sha256_file "$manifest_tmp")"
    file_count="$(wc -l < "$manifest_tmp" | awk '{print $1}')"
    total_bytes="$(awk -F'\t' '{s+=$3} END {print s+0}' "$manifest_tmp")"
    manifest_uri="gs://$bucket/$gcs_base.__manifest__.tsv"

    echo "Uploading directory: $local_path"
    echo "To:                 $uri"
    gcloud storage cp --recursive "$local_path" "$uri"
    gcloud storage cp "$manifest_tmp" "$manifest_uri"
    rm -f "$manifest_tmp"

    cat > "$pointer_abs" <<POINTEREOF
version: 1
type: dir
uri: $uri
manifest_uri: $manifest_uri
manifest_sha256: $manifest_sha
file_count: $file_count
total_bytes: $total_bytes
source_repo_relpath: $rel
POINTEREOF

    append_gitignore_exact "/$rel"
    ensure_pointer_tracked "$pointer_rel"

  else
    die "Unsupported path type: $local_path"
  fi

  echo "Wrote pointer: $pointer_rel"

  if [[ $do_commit -eq 1 ]]; then
    git add "$pointer_rel" .gitignore
    if ! git diff --cached --quiet -- "$pointer_rel" .gitignore 2>/dev/null; then
      git commit -m "Add GCS pointer for $rel" || true
      echo "Committed pointer."
    else
      echo "No staged changes to commit for $pointer_rel"
    fi
  fi

  if [[ $do_push -eq 1 ]]; then
    git push
    echo "Pushed git changes."
  fi

  echo "Done."
}

pull_from_pointer() {
  local ptr="$1"
  [[ -f "$ptr" ]] || die "Pointer not found: $ptr"

  local type uri out manifest_uri manifest_sha tmp_manifest
  type="$(read_kv type "$ptr")"
  uri="$(read_kv uri "$ptr")"
  out="${ptr%.gcs}"

  [[ -n "$type" ]] || die "Pointer missing type: $ptr"
  [[ -n "$uri" ]] || die "Pointer missing uri: $ptr"

  mkdir -p "$(dirname "$out")"

  if [[ "$type" == "file" ]]; then
    local sha got
    sha="$(read_kv sha256 "$ptr")"
    echo "Downloading file: $uri"
    echo "To:               $out"
    gcloud storage cp "$uri" "$out"
    if [[ -n "$sha" ]]; then
      got="$(sha256_file "$out")"
      [[ "$got" == "$sha" ]] || die "SHA256 mismatch for $out"
    fi
    echo "OK: $out"

  elif [[ "$type" == "dir" ]]; then
    manifest_uri="$(read_kv manifest_uri "$ptr")"
    manifest_sha="$(read_kv manifest_sha256 "$ptr")"
    [[ -n "$manifest_uri" ]] || die "Directory pointer missing manifest_uri: $ptr"

    tmp_manifest="$(mktemp)"
    gcloud storage cp "$manifest_uri" "$tmp_manifest"

    if [[ -n "$manifest_sha" ]]; then
      local got_manifest_sha
      got_manifest_sha="$(sha256_file "$tmp_manifest")"
      [[ "$got_manifest_sha" == "$manifest_sha" ]] || die "Manifest SHA256 mismatch for $ptr"
    fi

    mkdir -p "$out"
    echo "Restoring directory: $uri"
    echo "To:                 $out"

    while IFS=$'\t' read -r rel sha _size; do
      [[ -n "$rel" ]] || continue
      mkdir -p "$out/$(dirname "$rel")"
      gcloud storage cp "$uri/$rel" "$out/$rel"
      local got
      got="$(sha256_file "$out/$rel")"
      [[ "$got" == "$sha" ]] || die "SHA256 mismatch for $out/$rel"
    done < "$tmp_manifest"

    rm -f "$tmp_manifest"
    echo "OK: $out"

  else
    die "Unknown pointer type '$type' in $ptr"
  fi
}

collect_pointers_under() {
  local dir="$1"
  [[ -d "$dir" ]] || die "Directory not found: $dir"
  find "$dir" -type f -name '*.gcs' | sort
}

cmd_pull() {
  local pointer="" dir="" pull_all=0 bucket="" location="$LOCATION_DEFAULT" project=""

  while [[ $# -gt 0 ]]; do
    case "$1" in
      --pointer) pointer="$2"; shift 2 ;;
      --dir) dir="$2"; shift 2 ;;
      --all) pull_all=1; shift ;;
      --bucket) bucket="$2"; shift 2 ;;
      --location) location="$2"; shift 2 ;;
      --project) project="$2"; shift 2 ;;
      -h|--help) usage; exit 0 ;;
      *) die "Unknown arg: $1" ;;
    esac
  done

  ensure_git_repo
  need_cmd gcloud
  need_cmd sha256sum

  [[ -n "$bucket" ]] || bucket="$(default_bucket)"
  ensure_bucket "$bucket" "$location" "$project" "$STORAGE_CLASS_DEFAULT"

  local root ptrs=()
  root="$(repo_root)"

  if [[ -n "$pointer" ]]; then
    ptrs+=("$pointer")
  elif [[ -n "$dir" ]]; then
    while IFS= read -r f; do ptrs+=("$f"); done < <(collect_pointers_under "$dir")
  elif [[ $pull_all -eq 1 ]]; then
    while IFS= read -r f; do ptrs+=("$f"); done < <(find "$root" -type f -name '*.gcs' | sort)
  else
    die "Specify one of: --pointer POINTER.gcs, --dir DIRECTORY, or --all"
  fi

  [[ ${#ptrs[@]} -gt 0 ]] || die "No pointer files found to pull."

  local p
  for p in "${ptrs[@]}"; do
    pull_from_pointer "$p"
  done

  echo "Pull complete."
}

cmd_status() {
  local dir="."
  while [[ $# -gt 0 ]]; do
    case "$1" in
      --dir) dir="$2"; shift 2 ;;
      -h|--help) usage; exit 0 ;;
      *) die "Unknown arg: $1" ;;
    esac
  done

  ensure_git_repo
  need_cmd git

  echo "Git status:"
  git status --porcelain
  echo
  echo "Pointers under $dir:"
  find "$dir" -type f -name '*.gcs' | sort || true
  echo
  echo "Default bucket: gs://$(default_bucket)"
}

main() {
  [[ $# -ge 1 ]] || { usage; exit 1; }
  local cmd="$1"
  shift
  case "$cmd" in
    init) cmd_init "$@" ;;
    add) cmd_add "$@" ;;
    pull) cmd_pull "$@" ;;
    status) cmd_status "$@" ;;
    -h|--help) usage ;;
    *) die "Unknown command: $cmd" ;;
  esac
}

main "$@"
EOF

cat > "$INSTALL_DIR/gcsinit" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
backend="$(command -v git_gcs_artifacts)"
[[ -n "$backend" ]] || { echo "ERROR: git_gcs_artifacts not found on PATH"; exit 1; }
exec "$backend" init "$@"
EOF

cat > "$INSTALL_DIR/gcspush" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail

backend="$(command -v git_gcs_artifacts)"
[[ -n "$backend" ]] || { echo "ERROR: git_gcs_artifacts not found on PATH"; exit 1; }

usage() {
  cat <<'EOH'
Usage:
  gcspush PATH
  gcspush PATH --no-commit
  gcspush PATH --no-push
  gcspush PATH --no-commit --no-push
  gcspush -r DIRECTORY
  gcspush -r DIRECTORY -t 25
  gcspush -r DIRECTORY --threshold-mb 25
  gcspush -r DIRECTORY --no-push

Behavior:
  - Default mode pushes one file or one directory.
  - Recursive mode scans a directory and pushes each file whose size is >= threshold.
  - Threshold default is 10 MB.
  - Default behavior is upload + commit + push.
  - Recursive mode does a single batch commit and single push at the end.
EOH
}

recursive=0
threshold_mb=10
target=""
do_commit=1
do_push=1
pass_args=()

while [[ $# -gt 0 ]]; do
  case "$1" in
    -r|--recursive)
      recursive=1
      shift
      ;;
    -t|--threshold-mb)
      [[ $# -ge 2 ]] || { echo "ERROR: missing value for $1"; exit 1; }
      threshold_mb="$2"
      shift 2
      ;;
    --no-commit)
      do_commit=0
      shift
      ;;
    --no-push)
      do_push=0
      shift
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      if [[ -z "$target" ]]; then
        target="$1"
      else
        pass_args+=("$1")
      fi
      shift
      ;;
  esac
done

[[ -n "$target" ]] || { usage; exit 1; }

threshold_re='^[0-9]+([.][0-9]+)?$'
[[ "$threshold_mb" =~ $threshold_re ]] || { echo "ERROR: threshold must be numeric"; exit 1; }

if [[ "$recursive" -eq 0 ]]; then
  backend_flags=()
  [[ "$do_commit" -eq 1 ]] && backend_flags+=(--commit)
  [[ "$do_push" -eq 1 ]] && backend_flags+=(--push)
  exec "$backend" add --local "$target" "${backend_flags[@]}" "${pass_args[@]}"
fi

[[ -d "$target" ]] || { echo "ERROR: recursive mode requires a directory"; exit 1; }
git rev-parse --is-inside-work-tree >/dev/null 2>&1 || { echo "ERROR: run inside a git repo"; exit 1; }

repo_root="$(git rev-parse --show-toplevel)"
old_pwd="$(pwd)"

threshold_bytes="$(python3 - <<PY
print(int(float("$threshold_mb") * 1024 * 1024))
PY
)"

pushed_any=0
pointer_paths=()
gitignore_changed=0

while IFS= read -r -d '' f; do
  [[ "$f" == *.gcs ]] && continue

  size="$(stat -c '%s' "$f")"
  if (( size >= threshold_bytes )); then
    pushed_any=1
    echo "Pushing: $f"
    "$backend" add --local "$f" --no-commit --no-push "${pass_args[@]}"

    rel="$(realpath "$f")"
    rel="${rel#$repo_root/}"
    pointer_paths+=("$rel.gcs")
  fi
done < <(find "$target" -type f -print0)

if [[ "$pushed_any" -eq 0 ]]; then
  echo "No files at or above ${threshold_mb} MB found under $target"
  exit 0
fi

if [[ -f "$repo_root/.gitignore" ]]; then
  if ! git diff --quiet -- "$repo_root/.gitignore" 2>/dev/null; then
    gitignore_changed=1
  fi
fi

if [[ "$do_commit" -eq 1 ]]; then
  cd "$repo_root"
  add_args=()
  for p in "${pointer_paths[@]}"; do
    [[ -f "$p" ]] && add_args+=("$p")
  done
  [[ "$gitignore_changed" -eq 1 ]] && add_args+=(".gitignore")

  if [[ "${#add_args[@]}" -gt 0 ]]; then
    git add "${add_args[@]}"
    if ! git diff --cached --quiet -- "${add_args[@]}" 2>/dev/null; then
      git commit -m "Add GCS pointers under ${target}"
      echo "Committed pointers."
    else
      echo "No staged changes to commit."
    fi
  fi
  cd "$old_pwd"
fi

if [[ "$do_push" -eq 1 ]]; then
  git -C "$repo_root" push
  echo "Pushed git changes."
fi
EOF

cat > "$INSTALL_DIR/gcspull" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
backend="$(command -v git_gcs_artifacts)"
[[ -n "$backend" ]] || { echo "ERROR: git_gcs_artifacts not found on PATH"; exit 1; }

[[ $# -ge 1 ]] || {
  echo "Usage: gcspull POINTER.gcs | DIRECTORY | --all"
  exit 1
}

if [[ "$1" == "--all" ]]; then
  shift
  exec "$backend" pull --all "$@"
fi

target="$1"
shift

if [[ -f "$target" && "$target" == *.gcs ]]; then
  exec "$backend" pull --pointer "$target" "$@"
elif [[ -d "$target" ]]; then
  exec "$backend" pull --dir "$target" "$@"
else
  echo "ERROR: argument must be a .gcs pointer file, a directory, or --all"
  exit 1
fi
EOF

cat > "$INSTALL_DIR/gcsstatus" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
backend="$(command -v git_gcs_artifacts)"
[[ -n "$backend" ]] || { echo "ERROR: git_gcs_artifacts not found on PATH"; exit 1; }
exec "$backend" status "$@"
EOF

chmod +x "$INSTALL_DIR/git_gcs_artifacts"
chmod +x "$INSTALL_DIR/gcsinit"
chmod +x "$INSTALL_DIR/gcspush"
chmod +x "$INSTALL_DIR/gcspull"
chmod +x "$INSTALL_DIR/gcsstatus"

SHELL_RC=""
if [[ -n "${BASH_VERSION:-}" ]]; then
  SHELL_RC="$HOME/.bashrc"
elif [[ -n "${ZSH_VERSION:-}" ]]; then
  SHELL_RC="$HOME/.zshrc"
elif [[ -f "$HOME/.bashrc" ]]; then
  SHELL_RC="$HOME/.bashrc"
elif [[ -f "$HOME/.zshrc" ]]; then
  SHELL_RC="$HOME/.zshrc"
fi

if ! printf '%s' "$PATH" | tr ':' '\n' | grep -Fxq "$INSTALL_DIR"; then
  if [[ -n "$SHELL_RC" ]]; then
    grep -qxF 'export PATH="$HOME/.local/bin:$PATH"' "$SHELL_RC" || \
      echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$SHELL_RC"
    echo "Added $INSTALL_DIR to PATH in $SHELL_RC"
  else
    echo "WARNING: could not determine shell rc file"
    echo "Add this manually:"
    echo 'export PATH="$HOME/.local/bin:$PATH"'
  fi
fi

echo
echo "Installed:"
echo "  $INSTALL_DIR/git_gcs_artifacts"
echo "  $INSTALL_DIR/gcsinit"
echo "  $INSTALL_DIR/gcspush"
echo "  $INSTALL_DIR/gcspull"
echo "  $INSTALL_DIR/gcsstatus"
echo
echo "Pointer behavior:"
echo "  gcspush xx/yy/zz.csv   -> xx/yy/zz.csv.gcs"
echo "  gcspush models/run_01  -> models/run_01.gcs"
echo
echo "Recursive behavior:"
echo "  gcspush -r DIR                 # threshold 10 MB, single batch commit+push"
echo "  gcspush -r DIR -t 25           # threshold 25 MB"
echo "  gcspush -r DIR --no-push       # commit only"
echo "  gcspush -r DIR --no-commit     # upload only"
echo
echo "Default behavior:"
echo "  gcspush PATH                   # upload + commit + push"
echo
echo "Open a new shell or run:"
echo "  export PATH=\"\$HOME/.local/bin:\$PATH\""