Skip to content

Commit 0a0ec63

Browse files
MioYuuIHMioYuuIH
authored
Merge pull request #158 from AMDResearch/feature/git-clone-persistence-option
feat(hub): add git clone persistence option
2 parents a2dc970 + f47ae80 commit 0a0ec63

15 files changed

Lines changed: 702 additions & 47 deletions

File tree

‎runtime/chart/values.schema.json‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎runtime/chart/values.schema.yaml‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3649,8 +3649,10 @@ properties:
36493649
additionalProperties: false
36503650
description: |
36513651
Git repository cloning configuration.
3652-
Users can optionally provide a Git URL on the spawn form; the repo is
3652+
Users can optionally provide a Git URL on the spawn form. The repo is
36533653
cloned into their home directory via an init container at startup.
3654+
By default, cloned repositories are kept after the user's server stops.
3655+
Existing unmanaged same-name directories are preserved and not overwritten.
36543656
properties:
36553657
initContainerImage:
36563658
type: string
@@ -3683,6 +3685,19 @@ properties:
36833685
Used as fallback when user provides no PAT and no OAuth token is available.
36843686
Priority: user PAT > OAuth token > defaultAccessToken.
36853687
Helm auto-creates a K8s Secret (jupyterhub-git-default-token) from this value.
3688+
defaultPersistence:
3689+
type: boolean
3690+
description: |
3691+
Whether cloned repositories are kept by default after the user's server stops.
3692+
Defaults to true, so cloned repos persist unless an admin sets this
3693+
to false or allows users to choose ephemeral cleanup. Persistent mode
3694+
does not auto-pull, reset, or sync repository contents after clone.
3695+
allowPersistenceChoice:
3696+
type: boolean
3697+
description: |
3698+
Allow users to choose whether cloned repositories persist in the spawn form.
3699+
When false, the admin default is enforced and no user toggle is shown.
3700+
When true, the spawn form choice defaults to `defaultPersistence`.
36863701
36873702
hub:
36883703
type: object

‎runtime/hub/core/config.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,8 @@ class GitCloneSettings(BaseModel):
138138
maxCloneTimeout: int = 300
139139
githubAppName: str = ""
140140
defaultAccessToken: str = ""
141+
allowPersistenceChoice: bool = False
142+
defaultPersistence: bool = True
141143

142144
model_config = {"extra": "allow"}
143145

‎runtime/hub/core/handlers.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -998,6 +998,8 @@ def group_sort_key(group_name: str) -> tuple[int, int, bool, str]:
998998
"acceleratorKeys": list(config.accelerators.keys()),
999999
"allowedGitProviders": list(config.git_clone.allowedProviders),
10001000
"githubAppName": config.git_clone.githubAppName,
1001+
"allowPersistenceChoice": config.git_clone.allowPersistenceChoice,
1002+
"defaultPersistence": config.git_clone.defaultPersistence,
10011003
}
10021004
)
10031005
)

‎runtime/hub/core/scripts/git-clone.sh‎

Lines changed: 160 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@
2020

2121
# Git repository clone script for JupyterHub init container.
2222
#
23-
# Clones a repository onto the user's home PVC. The clone directory is
24-
# cleaned up by a preStop lifecycle hook when the session ends, so the
25-
# repository does not persist between sessions.
23+
# Clones a repository onto the user's home PVC. Existing clone directories are
24+
# reused or replaced only when repo-external AUPLC metadata proves they are
25+
# managed by this script.
2626
#
2727
# Environment variables (required):
2828
# REPO_URL - HTTPS URL of the git repository to clone
@@ -31,6 +31,8 @@
3131
#
3232
# Environment variables (optional):
3333
# BRANCH - Branch or tag to check out (default: repository's default branch)
34+
# PERSIST_CLONED_REPO - true to reuse a compatible managed clone (default: true)
35+
# AUPLC_GIT_METADATA_DIR - Directory for repo-external clone metadata
3436

3537
export HOME=/tmp
3638

@@ -43,38 +45,173 @@ git config --global http.sslVerify true
4345
git config --global user.email jupyterhub@local
4446
git config --global user.name JupyterHub
4547

48+
case "${PERSIST_CLONED_REPO:-true}" in
49+
true|True|TRUE|1|yes|Yes|YES)
50+
_persistence_mode="persistent"
51+
;;
52+
false|False|FALSE|0|no|No|NO)
53+
_persistence_mode="ephemeral"
54+
;;
55+
*)
56+
echo "Invalid PERSIST_CLONED_REPO value. Use true or false."
57+
exit 1
58+
;;
59+
esac
60+
61+
_metadata_dir=${AUPLC_GIT_METADATA_DIR:-"$(dirname "$CLONE_DIR")/.auplc/git-clones"}
62+
_clone_base=$(basename "$CLONE_DIR" | tr -c 'A-Za-z0-9._-' '_' | sed 's/^_*//; s/_*$//')
63+
if [ -z "$_clone_base" ]; then
64+
_clone_base="clone"
65+
fi
66+
_clone_hash=$(printf '%s' "$CLONE_DIR" | cksum | cut -d ' ' -f 1)
67+
_metadata_file="$_metadata_dir/${_clone_base}-${_clone_hash}.metadata"
68+
_metadata_version="1"
69+
_branch_value=${BRANCH:-}
70+
71+
sanitize_repo_url() {
72+
_url=$1
73+
case "$_url" in
74+
https://*)
75+
_sanitize_scheme="https://"
76+
_sanitize_rest=${_url#https://}
77+
;;
78+
http://*)
79+
_sanitize_scheme="http://"
80+
_sanitize_rest=${_url#http://}
81+
;;
82+
*)
83+
printf '%s' "$_url"
84+
return 0
85+
;;
86+
esac
87+
88+
_sanitize_authority=${_sanitize_rest%%/*}
89+
case "$_sanitize_authority" in
90+
*@*)
91+
_sanitize_authority=${_sanitize_authority##*@}
92+
;;
93+
*)
94+
;;
95+
esac
96+
97+
case "$_sanitize_rest" in
98+
*/*)
99+
printf '%s%s/%s' "$_sanitize_scheme" "$_sanitize_authority" "${_sanitize_rest#*/}"
100+
;;
101+
*)
102+
printf '%s%s' "$_sanitize_scheme" "$_sanitize_authority"
103+
;;
104+
esac
105+
}
106+
107+
metadata_value() {
108+
_key=$1
109+
_file=$2
110+
if [ ! -f "$_file" ]; then
111+
return 1
112+
fi
113+
sed -n "s/^${_key}=//p" "$_file" | sed -n '1p'
114+
}
115+
116+
is_metadata_compatible() {
117+
_file=$1
118+
[ "$(metadata_value state "$_file")" = "complete" ] || return 1
119+
[ "$(metadata_value version "$_file")" = "$_metadata_version" ] || return 1
120+
[ "$(metadata_value clone_dir "$_file")" = "$CLONE_DIR" ] || return 1
121+
[ "$(metadata_value repo_url "$_file")" = "$_sanitized_repo_url" ] || return 1
122+
[ "$(metadata_value branch "$_file")" = "$_branch_value" ] || return 1
123+
return 0
124+
}
125+
126+
write_metadata() {
127+
mkdir -p "$_metadata_dir"
128+
_tmp_metadata="$_metadata_file.tmp.$$"
129+
{
130+
printf 'version=%s\n' "$_metadata_version"
131+
printf 'state=complete\n'
132+
printf 'clone_dir=%s\n' "$CLONE_DIR"
133+
printf 'repo_url=%s\n' "$_sanitized_repo_url"
134+
printf 'branch=%s\n' "$_branch_value"
135+
printf 'persistence_mode=%s\n' "$_persistence_mode"
136+
date -u '+timestamp=%Y-%m-%dT%H:%M:%SZ'
137+
} > "$_tmp_metadata"
138+
mv "$_tmp_metadata" "$_metadata_file"
139+
}
140+
141+
clone_repo() {
142+
rm -f "$_metadata_file" "$_metadata_file.tmp.$$"
143+
mkdir -p "$(dirname "$CLONE_DIR")"
144+
if [ -n "${BRANCH:-}" ]; then
145+
echo "Cloning $_sanitized_repo_url (branch: $BRANCH) into $CLONE_DIR"
146+
if ! timeout "$MAX_CLONE_TIMEOUT" git clone --depth 1 --branch "$BRANCH" "$REPO_URL" "$CLONE_DIR"; then
147+
rm -f "$_metadata_file.tmp.$$"
148+
echo "Clone failed - check URL, branch name, and network access"
149+
exit 1
150+
fi
151+
else
152+
echo "Cloning $_sanitized_repo_url into $CLONE_DIR"
153+
if ! timeout "$MAX_CLONE_TIMEOUT" git clone --depth 1 "$REPO_URL" "$CLONE_DIR"; then
154+
rm -f "$_metadata_file.tmp.$$"
155+
echo "Clone failed - check URL and network access"
156+
exit 1
157+
fi
158+
fi
159+
write_metadata
160+
}
161+
162+
_sanitized_repo_url=$(sanitize_repo_url "$REPO_URL")
163+
46164
# Inject token into HTTPS URLs via git URL rewriting.
47165
# This approach works for all token types (classic PAT, fine-grained PAT, OAuth)
48166
# because the token is embedded directly in the URL rather than going through
49167
# the credential challenge/response flow.
50168
# The rewrite targets only the actual host in REPO_URL, so it works for any provider.
51169
if [ -n "${GIT_ACCESS_TOKEN:-}" ]; then
52-
_repo_host=$(echo "$REPO_URL" | sed 's|https://||' | cut -d/ -f1)
53-
git config --global \
54-
url."https://x-access-token:${GIT_ACCESS_TOKEN}@${_repo_host}/".insteadOf \
55-
"https://${_repo_host}/"
170+
case "$_sanitized_repo_url" in
171+
https://*)
172+
_repo_host=$(printf '%s' "$_sanitized_repo_url" | sed 's|https://||' | cut -d/ -f1)
173+
git config --global \
174+
url."https://x-access-token:${GIT_ACCESS_TOKEN}@${_repo_host}/".insteadOf \
175+
"https://${_repo_host}/"
176+
;;
177+
*)
178+
;;
179+
esac
56180
unset _repo_host
57181
fi
58182

59-
# Remove leftover clone directory from a previous session (e.g. preStop hook
60-
# was skipped due to force-delete or node failure).
61183
if [ -d "$CLONE_DIR" ]; then
62-
echo "Removing leftover directory $CLONE_DIR"
63-
rm -rf "$CLONE_DIR"
64-
fi
65-
66-
if [ -n "${BRANCH:-}" ]; then
67-
echo "Cloning $REPO_URL (branch: $BRANCH) into $CLONE_DIR"
68-
if ! timeout "$MAX_CLONE_TIMEOUT" git clone --depth 1 --branch "$BRANCH" "$REPO_URL" "$CLONE_DIR"; then
69-
echo "Clone failed - check URL, branch name, and network access"
70-
exit 1
71-
fi
72-
else
73-
echo "Cloning $REPO_URL into $CLONE_DIR"
74-
if ! timeout "$MAX_CLONE_TIMEOUT" git clone --depth 1 "$REPO_URL" "$CLONE_DIR"; then
75-
echo "Clone failed - check URL and network access"
184+
if ! is_metadata_compatible "$_metadata_file"; then
185+
echo "Refusing to modify existing directory $CLONE_DIR without compatible AUPLC clone metadata"
76186
exit 1
77187
fi
188+
189+
_metadata_persistence_mode=$(metadata_value persistence_mode "$_metadata_file")
190+
case "$_persistence_mode:$_metadata_persistence_mode" in
191+
persistent:persistent)
192+
echo "Reusing existing managed clone $CLONE_DIR"
193+
exit 0
194+
;;
195+
persistent:ephemeral)
196+
write_metadata
197+
echo "Reusing existing managed clone $CLONE_DIR"
198+
exit 0
199+
;;
200+
ephemeral:ephemeral)
201+
echo "Replacing existing managed ephemeral clone $CLONE_DIR"
202+
rm -rf "$CLONE_DIR"
203+
;;
204+
ephemeral:persistent)
205+
echo "Refusing to replace persistent managed clone $CLONE_DIR for an ephemeral request"
206+
exit 1
207+
;;
208+
*)
209+
echo "Refusing to modify existing directory $CLONE_DIR with unknown metadata persistence mode"
210+
exit 1
211+
;;
212+
esac
78213
fi
79214

215+
clone_repo
216+
80217
echo "Done"

‎runtime/hub/core/spawner/kubernetes.py‎

Lines changed: 49 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -312,11 +312,34 @@ def options_from_form(self, formdata) -> dict[str, Any]:
312312

313313
if repo_url:
314314
options["repo_url"] = repo_url
315+
options["repo_persist"] = self._resolve_repo_persist_option(formdata)
315316
if repo_branch:
316317
options["repo_branch"] = repo_branch
317318

318319
return options
319320

321+
def _resolve_repo_persist_option(self, formdata) -> bool:
322+
git_config = self._hub_config.git_clone if self._hub_config else None
323+
allow_choice = bool(getattr(git_config, "allowPersistenceChoice", False))
324+
default_persistence = bool(getattr(git_config, "defaultPersistence", True))
325+
326+
if not allow_choice:
327+
return default_persistence
328+
329+
try:
330+
raw_value = formdata.get("repo_persist", [None])[0]
331+
except Exception:
332+
raw_value = None
333+
334+
if isinstance(raw_value, str):
335+
normalized_value = raw_value.strip().lower()
336+
if normalized_value == "true":
337+
return True
338+
if normalized_value == "false":
339+
return False
340+
341+
return default_persistence
342+
320343
def _get_public_hub_home_url(self) -> str:
321344
"""Return the public Hub home URL for links opened from code-server."""
322345
hub_path = "/hub/home"
@@ -476,15 +499,16 @@ async def _build_git_init_container(
476499
repo_name: str,
477500
home_volume_name: str,
478501
home_mount_path: str,
502+
repo_persist: bool,
479503
repo_branch: str = "",
480504
access_token: str = "",
481505
) -> dict:
482506
"""
483507
Build an init container spec that clones a repository into the home mount path.
484508
485509
The init container mounts the same home PVC as the main container and clones
486-
into <home_mount_path>/<repo_name>. A preStop lifecycle hook on the main container
487-
removes the directory when the session ends so it does not persist.
510+
into <home_mount_path>/<repo_name>. In ephemeral mode, a preStop lifecycle hook
511+
on the main container removes the directory when the session ends.
488512
489513
The script is read from core/scripts/git-clone.sh, base64-encoded and decoded
490514
at runtime to prevent KubeSpawner's _expand_all from treating shell braces as
@@ -495,10 +519,13 @@ async def _build_git_init_container(
495519
encoded = base64.b64encode(f.read()).decode()
496520

497521
clone_dir = f"{home_mount_path}/{repo_name}"
522+
metadata_dir = f"{home_mount_path}/.auplc/git-clones"
498523

499524
env: list[dict[str, Any]] = [
500525
{"name": "REPO_URL", "value": repo_url},
501526
{"name": "CLONE_DIR", "value": clone_dir},
527+
{"name": "PERSIST_CLONED_REPO", "value": "true" if repo_persist else "false"},
528+
{"name": "AUPLC_GIT_METADATA_DIR", "value": metadata_dir},
502529
{"name": "MAX_CLONE_TIMEOUT", "value": str(self.MAX_CLONE_TIMEOUT)},
503530
]
504531
if repo_branch:
@@ -609,6 +636,13 @@ def _build_runtime_metadata_env(
609636

610637
return env
611638

639+
@staticmethod
640+
def _with_ephemeral_clone_cleanup(extra_container_config: dict | None, clone_dir: str) -> dict:
641+
extra = copy.deepcopy(extra_container_config or {})
642+
lifecycle = extra.setdefault("lifecycle", {})
643+
lifecycle["preStop"] = {"exec": {"command": ["rm", "-rf", clone_dir]}}
644+
return extra
645+
612646
def _get_launch_mode(self, resource_type: str) -> str:
613647
"""Return the configured launch mode for a resource."""
614648
resource_metadata = self._hub_config.get_resource_metadata(resource_type) if self._hub_config else None
@@ -917,16 +951,24 @@ async def start(self):
917951
safe_username = self._expand_user_properties("{username}")
918952
home_volume_name = f"volume-{safe_username}"
919953
home_mount_path = self._get_home_mount_path(home_volume_name)
954+
repo_persist = bool(self.user_options.get("repo_persist", True))
920955
init_container = await self._build_git_init_container(
921-
sanitized_url, repo_name, home_volume_name, home_mount_path, repo_branch, access_token
956+
sanitized_url,
957+
repo_name,
958+
home_volume_name,
959+
home_mount_path,
960+
repo_persist,
961+
repo_branch,
962+
access_token,
922963
)
923964
self.init_containers = [init_container] + list(self.init_containers or [])
924965

925-
# preStop lifecycle hook: remove the cloned directory on session end
926-
extra = dict(self.extra_container_config or {})
927966
clone_dir = f"{home_mount_path}/{repo_name}"
928-
extra["lifecycle"] = {"preStop": {"exec": {"command": ["rm", "-rf", clone_dir]}}}
929-
self.extra_container_config = extra
967+
if not repo_persist:
968+
self.extra_container_config = self._with_ephemeral_clone_cleanup(
969+
self.extra_container_config,
970+
clone_dir,
971+
)
930972

931973
self.notebook_dir = home_mount_path
932974
if self._launches_code_server(resource_type):

0 commit comments

Comments
 (0)