Skip to content

Commit ad42c10

Browse files
forsyth2forsyth2
committed
Fix auths and add test
1 parent cb00833 commit ad42c10

File tree

4 files changed

+430
-170
lines changed

4 files changed

+430
-170
lines changed

‎conda/dev.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@ dependencies:
88
- pip=22.2.2
99
- python=3.9.13
1010
- six=1.16.0
11-
- globus-sdk=3.2.1
12-
- fair-research-login=0.2.6
11+
- globus-sdk=3.15.0
1312
# Developer Tools
1413
# =================
1514
# If versions are updated, also update 'rev' in `.pre-commit.config.yaml`

‎tests/scripts/globus_auth.bash‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
setup()
2+
{
3+
echo "##########################################################################################################"
4+
local case_name="${1}"
5+
local src_dir="${2}"
6+
echo "Testing: ${case_name}"
7+
full_dir="${src_dir}/${case_name}"
8+
rm -rf ${full_dir}
9+
mkdir -p ${full_dir}
10+
cd ${full_dir}
11+
12+
mkdir zstash_demo
13+
mkdir zstash_demo/empty_dir
14+
mkdir zstash_demo/dir
15+
echo -n '' > zstash_demo/file_empty.txt
16+
echo 'file0 stuff' > zstash_demo/dir/file0.txt
17+
}
18+
19+
check_log_has()
20+
{
21+
local expected_grep="${1}"
22+
local log_file="${2}"
23+
grep "${expected_grep}" ${log_file}
24+
if [ $? != 0 ]; then
25+
echo "Expected grep '${expected_grep}' not found in ${log_file}. Test failed."
26+
exit 2
27+
fi
28+
}
29+
30+
check_log_does_not_have()
31+
{
32+
local not_expected_grep="${1}"
33+
local log_file="${2}"
34+
grep "${not_expected_grep}" ${log_file}
35+
if [ $? == 0 ]; then
36+
echo "Not-expected grep '${expected_grep}' was found in ${log_file}. Test failed."
37+
exit 2
38+
fi
39+
}
40+
41+
run_test_cases()
42+
{
43+
# This script requires user input and thus cannot be run automatically as part of a test suite.
44+
45+
# To start fresh with Globus:
46+
# 1. Log into endpoints (LCRC Improv DTN, NERSC Perlmutter) at globus.org: File Manager > Add the endpoints in the "Collection" fields
47+
# 2. To start fresh, with no consents: https://auth.globus.org/v2/web/consents > Manage Your Consents > Globus Endpoint Performance Monitoring > rescind all"
48+
49+
# Before each run:
50+
# Perlmutter:
51+
# cd /global/homes/f/forsyth/zstash/tests/
52+
# rm -rf test_globus_auth_try1 # Or just change $DST_DIR to a new directory
53+
#
54+
# Chrysalis:
55+
# cd ~/ez/zstash/
56+
# conda activate <env-name>
57+
# pre-commit run --all-files
58+
# python -m pip install .
59+
# cd tests/scripts
60+
# ./globus_auth.bash
61+
62+
PERLMUTTER_ENDPOINT=6bdc7956-fc0f-4ad2-989c-7aa5ee643a79
63+
64+
SRC_DIR=/lcrc/group/e3sm/ac.forsyth2/zstash_testing/test_globus_auth # Chrysalis
65+
DST_DIR=globus://${PERLMUTTER_ENDPOINT}/global/homes/f/forsyth/zstash/tests/test_globus_auth_try5
66+
67+
GLOBUS_CFG=/home/ac.forsyth2/.globus-native-apps.cfg
68+
INI_PATH=/home/ac.forsyth2/.zstash.ini
69+
TOKEN_FILE=/home/ac.forsyth2/.zstash_globus_tokens.json
70+
71+
# Start fresh
72+
rm -rf ${GLOBUS_CFG}
73+
rm -rf ${INI_PATH}
74+
rm -rf ${TOKEN_FILE}
75+
76+
echo "Running globus_auth test"
77+
echo "Exit codes: 0 -- success, 1 -- zstash failed, 2 -- grep check failed"
78+
79+
case_name="run1"
80+
setup ${case_name} "${SRC_DIR}"
81+
# Expecting to see exactly 1 authentication prompt
82+
zstash create --hpss=${DST_DIR}/${case_name} zstash_demo 2>&1 | tee ${case_name}.log
83+
if [ $? != 0 ]; then
84+
echo "${case_name} failed. Check ${case_name}_create.log for details. Cannot continue."
85+
exit 1
86+
fi
87+
echo "${case_name} completed successfully. Checking ${case_name}.log now."
88+
# From check_state_files
89+
check_log_does_not_have "WARNING: Globus CFG ${GLOBUS_CFG} exists. This may be left over from earlier versions of zstash, and may cause issues. Consider deleting." ${case_name}.log
90+
check_log_has "INFO: ${INI_PATH} does NOT exist. This means we won't be able to read the local endpoint ID from it." ${case_name}.log
91+
check_log_has "INFO: Token file ${TOKEN_FILE} does NOT exist. This means we won't be able to load tokens from it." ${case_name}.log
92+
# From get_local_endpoint_id
93+
check_log_has "INFO: Writing to empty ${INI_PATH}" ${case_name}.log
94+
check_log_has "INFO: Setting local_endpoint_id based on FQDN chrlogin2.lcrc.anl.gov:" ${case_name}.log
95+
# From get_transfer_client_with_auth
96+
check_log_has "INFO: No stored tokens found - starting authentication" ${case_name}.log
97+
check_log_has "Please go to this URL and login:" ${case_name}.log # Our one expected authentication prompt
98+
# From save_tokens
99+
check_log_has "INFO: Tokens saved successfully" ${case_name}.log
100+
101+
102+
case_name="run2"
103+
setup ${case_name} "${SRC_DIR}"
104+
# Expecting to see exactly 0 authentication prompts
105+
zstash create --hpss=${DST_DIR}/${case_name} zstash_demo 2>&1 | tee ${case_name}.log
106+
if [ $? != 0 ]; then
107+
echo "${case_name} failed. Check ${case_name}_create.log for details. Cannot continue."
108+
exit 1
109+
fi
110+
echo "${case_name} completed successfully. Checking ${case_name}.log now."
111+
# From check_state_files
112+
check_log_does_not_have "WARNING: Globus CFG ${GLOBUS_CFG} exists. This may be left over from earlier versions of zstash, and may cause issues. Consider deleting." ${case_name}.log
113+
check_log_has "INFO: ${INI_PATH} exists. We can try to read the local endpoint ID from it." ${case_name}.log # Differs from run1
114+
check_log_has "INFO: Token file ${TOKEN_FILE} exists. We can try to load tokens from it." ${case_name}.log # Differs from run1
115+
# From get_local_endpoint_id
116+
check_log_has "INFO: Setting local_endpoint_id based on ${INI_PATH}" ${case_name}.log # Differs from run1
117+
check_log_has "INFO: Setting local_endpoint_id based on FQDN chrlogin2.lcrc.anl.gov:" ${case_name}.log
118+
# From get_transfer_client_with_auth
119+
check_log_has "INFO: Found stored refresh token - using it" ${case_name}.log # Differs from run1
120+
check_log_does_not_have "Please go to this URL and login:" ${case_name}.log # There should be no login prompts for run2!
121+
# From save_tokens
122+
check_log_does_not_have "INFO: Tokens saved successfully" ${case_name}.log # Differs from run1
123+
}
124+
125+
run_test_cases

‎zstash/globus.py‎

Lines changed: 31 additions & 168 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,24 @@
11
from __future__ import absolute_import, print_function
22

3-
import configparser
4-
import os
5-
import os.path
6-
import re
7-
import socket
83
import sys
9-
from typing import Dict, List
4+
from typing import List, Optional
105

11-
from fair_research_login.client import NativeClient
126
from globus_sdk import TransferAPIError, TransferClient, TransferData
7+
from globus_sdk.response import GlobusHTTPResponse
138
from globus_sdk.services.transfer.response.iterable import IterableTransferResponse
149
from six.moves.urllib.parse import urlparse
1510

11+
from .globus_utils import (
12+
HPSS_ENDPOINT_MAP,
13+
check_state_files,
14+
get_local_endpoint_id,
15+
get_transfer_client_with_auth,
16+
set_up_TransferData,
17+
submit_transfer_with_checks,
18+
)
1619
from .settings import logger
1720
from .utils import ts_utc
1821

19-
hpss_endpoint_map: Dict[str, str] = {
20-
"ALCF": "de463ec4-6d04-11e5-ba46-22000b92c6ec",
21-
"NERSC": "9cd89cfd-6d04-11e5-ba46-22000b92c6ec",
22-
}
23-
24-
# This is used if the `globus_endpoint_uuid` is not set in `~/.zstash.ini`
25-
regex_endpoint_map: Dict[str, str] = {
26-
r"theta.*\.alcf\.anl\.gov": "08925f04-569f-11e7-bef8-22000b9a448b",
27-
r"blueslogin.*\.lcrc\.anl\.gov": "15288284-7006-4041-ba1a-6b52501e49f1",
28-
r"chrlogin.*\.lcrc\.anl\.gov": "15288284-7006-4041-ba1a-6b52501e49f1",
29-
r"b\d+\.lcrc\.anl\.gov": "15288284-7006-4041-ba1a-6b52501e49f1",
30-
r"chr.*\.lcrc\.anl\.gov": "15288284-7006-4041-ba1a-6b52501e49f1",
31-
r"compy.*\.pnl\.gov": "68fbd2fa-83d7-11e9-8e63-029d279f7e24",
32-
r"perlmutter.*\.nersc\.gov": "6bdc7956-fc0f-4ad2-989c-7aa5ee643a79",
33-
}
34-
3522
remote_endpoint = None
3623
local_endpoint = None
3724
transfer_client: TransferClient = None
@@ -40,50 +27,6 @@
4027
archive_directory_listing: IterableTransferResponse = None
4128

4229

43-
def get_all_endpoint_scopes(endpoints: List[str]) -> str:
44-
inner = " ".join(
45-
[f"*https://auth.globus.org/scopes/{ep}/data_access" for ep in endpoints]
46-
)
47-
return f"urn:globus:auth:scope:transfer.api.globus.org:all[{inner}]"
48-
49-
50-
# Used exclusively by submit_transfer_with_checks, exclusively when there is a TransferAPIError
51-
# This function is really to diagnose an error: are the endpoints ok?
52-
# That is, we don't *need* to check endpoint versions if everything worked out fine.
53-
def check_endpoint_version_5(ep_id):
54-
output = transfer_client.get_endpoint(ep_id)
55-
version = output.get("gcs_version", "0.0")
56-
if output["gcs_version"] is None:
57-
return False
58-
elif int(version.split(".")[0]) >= 5:
59-
return True
60-
return False
61-
62-
63-
def submit_transfer_with_checks(transfer_data):
64-
try:
65-
task = transfer_client.submit_transfer(transfer_data)
66-
except TransferAPIError as err:
67-
if err.info.consent_required:
68-
scopes = "urn:globus:auth:scope:transfer.api.globus.org:all["
69-
for ep_id in [remote_endpoint, local_endpoint]:
70-
if ep_id and check_endpoint_version_5(ep_id):
71-
scopes += f" *https://auth.globus.org/scopes/{ep_id}/data_access"
72-
scopes += " ]"
73-
native_client = NativeClient(
74-
client_id="6c1629cf-446c-49e7-af95-323c6412397f", app_name="Zstash"
75-
)
76-
native_client.login(requested_scopes=scopes)
77-
# Quit here and tell user to re-try
78-
print(
79-
"Consents added, please re-run the previous command to start transfer"
80-
)
81-
sys.exit(0)
82-
else:
83-
raise err
84-
return task
85-
86-
8730
def globus_activate(hpss: str):
8831
"""
8932
Read the local globus endpoint UUID from ~/.zstash.ini.
@@ -97,81 +40,18 @@ def globus_activate(hpss: str):
9740
url = urlparse(hpss)
9841
if url.scheme != "globus":
9942
return
100-
globus_cfg: str = os.path.expanduser("~/.globus-native-apps.cfg")
101-
logger.info(f"Checking if {globus_cfg} exists")
102-
if os.path.exists(globus_cfg):
103-
logger.info(
104-
f"{globus_cfg} exists. If this file does not have the proper settings, it may cause a TransferAPIError (e.g., 'Token is not active', 'No credentials supplied')"
105-
)
106-
else:
107-
logger.info(
108-
f"{globus_cfg} does not exist. zstash will need to prompt for authentications twice, and then you will need to re-run."
109-
)
43+
check_state_files()
11044
remote_endpoint = url.netloc
111-
112-
ini_path = os.path.expanduser("~/.zstash.ini")
113-
ini = configparser.ConfigParser()
114-
if ini.read(ini_path):
115-
if "local" in ini.sections():
116-
local_endpoint = ini["local"].get("globus_endpoint_uuid")
117-
else:
118-
ini["local"] = {"globus_endpoint_uuid": ""}
119-
try:
120-
with open(ini_path, "w") as f:
121-
ini.write(f)
122-
except Exception as e:
123-
logger.error(e)
124-
sys.exit(1)
125-
if not local_endpoint:
126-
fqdn = socket.getfqdn()
127-
if re.fullmatch(r"n.*\.local", fqdn) and os.getenv("HOSTNAME", "NA").startswith(
128-
"compy"
129-
):
130-
fqdn = "compy.pnl.gov"
131-
for pattern in regex_endpoint_map.keys():
132-
if re.fullmatch(pattern, fqdn):
133-
local_endpoint = regex_endpoint_map.get(pattern)
134-
break
135-
# FQDN is not set on Perlmutter at NERSC
136-
if not local_endpoint:
137-
nersc_hostname = os.environ.get("NERSC_HOST")
138-
if nersc_hostname and (
139-
nersc_hostname == "perlmutter" or nersc_hostname == "unknown"
140-
):
141-
local_endpoint = regex_endpoint_map.get(r"perlmutter.*\.nersc\.gov")
142-
if not local_endpoint:
143-
logger.error(
144-
"{} does not have the local Globus endpoint set nor could one be found in regex_endpoint_map.".format(
145-
ini_path
146-
)
147-
)
148-
sys.exit(1)
149-
150-
if remote_endpoint.upper() in hpss_endpoint_map.keys():
151-
remote_endpoint = hpss_endpoint_map.get(remote_endpoint.upper())
152-
153-
native_client = NativeClient(
154-
client_id="6c1629cf-446c-49e7-af95-323c6412397f",
155-
app_name="Zstash",
156-
default_scopes="openid urn:globus:auth:scope:transfer.api.globus.org:all",
157-
)
158-
if local_endpoint and remote_endpoint:
159-
all_scopes: str = get_all_endpoint_scopes([local_endpoint, remote_endpoint])
160-
native_client.login(
161-
requested_scopes=all_scopes, no_local_server=True, refresh_tokens=True
162-
)
163-
else:
164-
native_client.login(no_local_server=True, refresh_tokens=True)
165-
transfer_authorizer = native_client.get_authorizers().get("transfer.api.globus.org")
166-
transfer_client = TransferClient(authorizer=transfer_authorizer)
167-
168-
for ep_id in [local_endpoint, remote_endpoint]:
45+
local_endpoint = get_local_endpoint_id(local_endpoint)
46+
if remote_endpoint.upper() in HPSS_ENDPOINT_MAP.keys():
47+
remote_endpoint = HPSS_ENDPOINT_MAP.get(remote_endpoint.upper())
48+
both_endpoints: List[Optional[str]] = [local_endpoint, remote_endpoint]
49+
transfer_client = get_transfer_client_with_auth(both_endpoints)
50+
for ep_id in both_endpoints:
16951
r = transfer_client.endpoint_autoactivate(ep_id, if_expires_in=600)
17052
if r.get("code") == "AutoActivationFailed":
17153
logger.error(
172-
"The {} endpoint is not activated or the current activation expires soon. Please go to https://app.globus.org/file-manager/collections/{} and (re)activate the endpoint.".format(
173-
ep_id, ep_id
174-
)
54+
f"The {ep_id} endpoint is not activated or the current activation expires soon. Please go to https://app.globus.org/file-manager/collections/{ep_id} and (re)activate the endpoint."
17555
)
17656
sys.exit(1)
17757

@@ -220,34 +100,17 @@ def globus_transfer( # noqa: C901
220100
)
221101
sys.exit(1)
222102

223-
if transfer_type == "get":
224-
src_ep = remote_endpoint
225-
src_path = os.path.join(remote_path, name)
226-
dst_ep = local_endpoint
227-
dst_path = os.path.join(os.getcwd(), name)
228-
else:
229-
src_ep = local_endpoint
230-
src_path = os.path.join(os.getcwd(), name)
231-
dst_ep = remote_endpoint
232-
dst_path = os.path.join(remote_path, name)
233-
234-
subdir = os.path.basename(os.path.normpath(remote_path))
235-
subdir_label = re.sub("[^A-Za-z0-9_ -]", "", subdir)
236-
filename = name.split(".")[0]
237-
label = subdir_label + " " + filename
238-
239-
if not transfer_data:
240-
transfer_data = TransferData(
241-
transfer_client,
242-
src_ep,
243-
dst_ep,
244-
label=label,
245-
verify_checksum=True,
246-
preserve_timestamp=True,
247-
fail_on_quota_errors=True,
248-
)
249-
transfer_data.add_item(src_path, dst_path)
250-
transfer_data["label"] = label
103+
transfer_data = set_up_TransferData(
104+
transfer_type,
105+
local_endpoint, # Global
106+
remote_endpoint, # Global
107+
remote_path,
108+
name,
109+
transfer_client, # Global
110+
transfer_data, # Global
111+
)
112+
113+
task: GlobusHTTPResponse
251114
try:
252115
if task_id:
253116
task = transfer_client.get_task(task_id)
@@ -291,7 +154,7 @@ def globus_transfer( # noqa: C901
291154

292155
# SUBMIT new transfer here
293156
logger.info(f"{ts_utc()}: DIVING: Submit Transfer for {transfer_data['label']}")
294-
task = submit_transfer_with_checks(transfer_data)
157+
task = submit_transfer_with_checks(transfer_client, transfer_data)
295158
task_id = task.get("task_id")
296159
# NOTE: This log message is misleading. If we have accumulated multiple tar files for transfer,
297160
# the "lable" given here refers only to the LAST tarfile in the TransferData list.
@@ -447,7 +310,7 @@ def globus_finalize(non_blocking: bool = False):
447310
# SUBMIT new transfer here
448311
logger.info(f"{ts_utc()}: DIVING: Submit Transfer for {transfer_data['label']}")
449312
try:
450-
last_task = submit_transfer_with_checks(transfer_data)
313+
last_task = submit_transfer_with_checks(transfer_client, transfer_data)
451314
last_task_id = last_task.get("task_id")
452315
except TransferAPIError as e:
453316
if e.code == "NoCredException":

0 commit comments

Comments
 (0)