Skip to content

Commit 1078500

Browse files
forsyth2forsyth2
committed
Script with correct auths
1 parent 572210c commit 1078500

File tree

3 files changed

+186
-73
lines changed

3 files changed

+186
-73
lines changed

‎conda/dev.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@ dependencies:
88
- pip=22.2.2
99
- python=3.9.13
1010
- six=1.16.0
11-
- globus-sdk=3.2.1
12-
- fair-research-login=0.2.6
11+
- globus-sdk=3.15.0
1312
# Developer Tools
1413
# =================
1514
# If versions are updated, also update 'rev' in `.pre-commit.config.yaml`

‎examples/simple_globus.py‎

Lines changed: 153 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,18 @@
11
import configparser
2+
import json
23
import os
34
import re
45
import shutil
56
from typing import List, Optional, Tuple
67
from urllib.parse import ParseResult, urlparse
78

8-
from fair_research_login.client import NativeClient
9-
from globus_sdk import TransferAPIError, TransferClient, TransferData
9+
from globus_sdk import (
10+
NativeAppAuthClient,
11+
RefreshTokenAuthorizer,
12+
TransferAPIError,
13+
TransferClient,
14+
TransferData,
15+
)
1016
from globus_sdk.response import GlobusHTTPResponse
1117

1218
"""
@@ -43,14 +49,15 @@
4349

4450
# Settings ####################################################################
4551
REQUEST_SCOPES_EARLY: bool = True
46-
REMOTE_DIR_PREFIX: str = "zstash_simple_globus_try2"
52+
REMOTE_DIR_PREFIX: str = "zstash_simple_globus_try6"
4753

4854
LOCAL_ENDPOINT: str = "LCRC Improv DTN"
4955
REMOTE_ENDPOINT: str = "NERSC Perlmutter"
5056

5157
# Constants ###################################################################
5258
GLOBUS_CFG: str = os.path.expanduser("~/.globus-native-apps.cfg")
5359
INI_PATH: str = os.path.expanduser("~/.zstash.ini")
60+
TOKEN_FILE = os.path.expanduser("~/.zstash_globus_tokens.json")
5461
ZSTASH_CLIENT_ID: str = "6c1629cf-446c-49e7-af95-323c6412397f"
5562
NAME_TO_ENDPOINT_MAP = {
5663
"NERSC HPSS": "9cd89cfd-6d04-11e5-ba46-22000b92c6ec",
@@ -69,6 +76,7 @@ def main():
6976
skipped_second_auth = simple_transfer("toy_run")
7077
except RuntimeError:
7178
print("Now that we have the authentications, let's re-run.")
79+
skipped_second_auth = simple_transfer("toy_run")
7280
print(f"For toy_run, skipped_second_auth={skipped_second_auth}")
7381
if skipped_second_auth:
7482
print("We didn't need to authenticate a second time on the toy run!")
@@ -80,10 +88,11 @@ def main():
8088

8189

8290
def reset_state_files():
83-
if os.path.exists(INI_PATH):
84-
os.remove(INI_PATH)
85-
if os.path.exists(GLOBUS_CFG):
86-
os.remove(GLOBUS_CFG)
91+
files_to_remove = [INI_PATH, GLOBUS_CFG, TOKEN_FILE]
92+
for file_path in files_to_remove:
93+
if os.path.exists(file_path):
94+
os.remove(file_path)
95+
print(f"Removed {file_path}")
8796

8897

8998
def simple_transfer(run_dir: str) -> bool:
@@ -98,50 +107,52 @@ def simple_transfer(run_dir: str) -> bool:
98107
print(f"url.scheme={url.scheme}, url.netloc={url.netloc}")
99108
local_endpoint: str = get_local_endpoint_id()
100109
both_endpoints: List[str] = [local_endpoint, remote_endpoint]
101-
native_client = NativeClient(
102-
client_id=ZSTASH_CLIENT_ID,
103-
app_name="Zstash",
104-
default_scopes="openid urn:globus:auth:scope:transfer.api.globus.org:all",
105-
)
106-
# May print 'Please Paste your Auth Code Below:'
107-
# This is the 1st authentication prompt!
108-
print("Might ask for 1st authentication prompt:")
109-
if REQUEST_SCOPES_EARLY:
110-
all_scopes: str = get_all_endpoint_scopes(both_endpoints)
111-
native_client.login(
112-
requested_scopes=all_scopes, no_local_server=True, refresh_tokens=True
113-
)
114-
else:
115-
native_client.login(no_local_server=True, refresh_tokens=True)
116-
print("Authenticated for the 1st time!")
117-
transfer_authorizer = native_client.get_authorizers().get("transfer.api.globus.org")
118-
transfer_client: TransferClient = TransferClient(authorizer=transfer_authorizer)
110+
111+
# Get the transfer client with proper authentication
112+
transfer_client, needed_fresh_auth = get_transfer_client_with_auth(both_endpoints)
113+
119114
for ep_id in both_endpoints:
120115
r = transfer_client.endpoint_autoactivate(ep_id, if_expires_in=600)
121116
assert r.get("code") != "AutoActivationFailed"
117+
122118
os.chdir(config_path)
123119
print(f"Now in {os.getcwd()}")
124120
transfer_data: TransferData = construct_TransferData(
125121
url, txt_file, transfer_client, local_endpoint, remote_endpoint
126122
)
123+
127124
task: GlobusHTTPResponse
128-
skipped_second_auth: bool = False
125+
skipped_second_auth: bool = not needed_fresh_auth
126+
129127
try:
130128
task = transfer_client.submit_transfer(transfer_data)
131-
print("Bypassed 2nd authentication.")
129+
if not needed_fresh_auth:
130+
print("Bypassed authentication entirely - used stored tokens!")
131+
else:
132+
print("Used fresh authentication - tokens now stored for next time")
132133
skipped_second_auth = True
133134
except TransferAPIError as err:
134135
if err.info.consent_required:
136+
# This should be much less likely now with proper scope handling
137+
print("Consent required - this suggests scope issues")
138+
skipped_second_auth = False
139+
135140
scopes = "urn:globus:auth:scope:transfer.api.globus.org:all["
136141
for ep_id in both_endpoints:
137142
scopes += f" *https://auth.globus.org/scopes/{ep_id}/data_access"
138143
scopes += " ]"
139-
native_client = NativeClient(client_id=ZSTASH_CLIENT_ID, app_name="Zstash")
140-
# May print 'Please Paste your Auth Code Below:'
141-
# This is the 2nd authentication prompt!
142-
print("Might ask for 2nd authentication prompt:")
143-
native_client.login(requested_scopes=scopes)
144-
print("Authenticated for the 2nd time!")
144+
145+
print("Getting additional consents...")
146+
client = NativeAppAuthClient(ZSTASH_CLIENT_ID)
147+
client.oauth2_start_flow(requested_scopes=scopes, refresh_tokens=True)
148+
authorize_url = client.oauth2_get_authorize_url()
149+
print("Please go to this URL and login: {0}".format(authorize_url))
150+
auth_code = input(
151+
"Please enter the code you get after login here: "
152+
).strip()
153+
token_response = client.oauth2_exchange_code_for_tokens(auth_code)
154+
save_tokens(token_response)
155+
145156
print(
146157
"Consents added, please re-run the previous command to start transfer"
147158
)
@@ -150,6 +161,7 @@ def simple_transfer(run_dir: str) -> bool:
150161
if err.info.authorization_parameters:
151162
print("Error is in authorization parameters")
152163
raise err
164+
153165
task_id = task.get("task_id")
154166
wait_timeout = 300 # 300 sec = 5 min
155167
print(f"Wait for task to complete, wait_timeout={wait_timeout}")
@@ -159,15 +171,33 @@ def simple_transfer(run_dir: str) -> bool:
159171
return skipped_second_auth
160172

161173

174+
def get_dir_and_file_to_archive(run_dir: str) -> Tuple[str, str]:
175+
if os.path.exists(run_dir):
176+
shutil.rmtree(run_dir)
177+
os.mkdir(run_dir)
178+
os.chdir(run_dir)
179+
print(f"Now in {os.getcwd()}")
180+
dir_to_archive: str = "dir_to_archive"
181+
txt_file: str = "file0.txt"
182+
os.mkdir(dir_to_archive)
183+
with open(f"{dir_to_archive}/{txt_file}", "w") as f:
184+
f.write("file contents")
185+
config_path: str = os.path.abspath(dir_to_archive)
186+
assert os.path.isdir(config_path)
187+
return config_path, txt_file
188+
189+
162190
def check_state_files():
163-
if os.path.exists(INI_PATH):
164-
print(f"{INI_PATH} exists.")
165-
else:
166-
print(f"{INI_PATH} does NOT exist.")
167-
if os.path.exists(GLOBUS_CFG):
168-
print(f"{GLOBUS_CFG} exists.")
169-
else:
170-
print(f"{GLOBUS_CFG} does NOT exist.")
191+
files_to_check = [
192+
(INI_PATH, "INI_PATH"),
193+
(GLOBUS_CFG, "GLOBUS_CFG"),
194+
(TOKEN_FILE, "TOKEN_FILE"),
195+
]
196+
for file_path, name in files_to_check:
197+
if os.path.exists(file_path):
198+
print(f"{name}: {file_path} exists.")
199+
else:
200+
print(f"{name}: {file_path} does NOT exist.")
171201

172202

173203
def get_local_endpoint_id() -> str:
@@ -188,20 +218,62 @@ def get_local_endpoint_id() -> str:
188218
return local_endpoint
189219

190220

191-
def get_dir_and_file_to_archive(run_dir: str) -> Tuple[str, str]:
192-
if os.path.exists(run_dir):
193-
shutil.rmtree(run_dir)
194-
os.mkdir(run_dir)
195-
os.chdir(run_dir)
196-
print(f"Now in {os.getcwd()}")
197-
dir_to_archive: str = "dir_to_archive"
198-
txt_file: str = "file0.txt"
199-
os.mkdir(dir_to_archive)
200-
with open(f"{dir_to_archive}/{txt_file}", "w") as f:
201-
f.write("file contents")
202-
config_path: str = os.path.abspath(dir_to_archive)
203-
assert os.path.isdir(config_path)
204-
return config_path, txt_file
221+
def get_transfer_client_with_auth(
222+
both_endpoints: List[str],
223+
) -> Tuple[TransferClient, bool]:
224+
"""
225+
Get a TransferClient, handling authentication properly.
226+
Returns (transfer_client, needed_fresh_auth)
227+
"""
228+
tokens = load_tokens()
229+
230+
# Check if we have stored refresh tokens
231+
if "transfer.api.globus.org" in tokens:
232+
token_data = tokens["transfer.api.globus.org"]
233+
if "refresh_token" in token_data:
234+
print("Found stored refresh token - using it")
235+
# Create a simple auth client for the RefreshTokenAuthorizer
236+
auth_client = NativeAppAuthClient(ZSTASH_CLIENT_ID)
237+
transfer_authorizer = RefreshTokenAuthorizer(
238+
refresh_token=token_data["refresh_token"], auth_client=auth_client
239+
)
240+
transfer_client = TransferClient(authorizer=transfer_authorizer)
241+
return transfer_client, False # No fresh auth needed
242+
243+
# No stored tokens, need to authenticate
244+
print("No stored tokens found - starting authentication")
245+
246+
# Get the required scopes
247+
if REQUEST_SCOPES_EARLY:
248+
all_scopes = get_all_endpoint_scopes(both_endpoints)
249+
print(f"Requesting scopes early: {all_scopes}")
250+
else:
251+
all_scopes = "urn:globus:auth:scope:transfer.api.globus.org:all"
252+
253+
# Use the NativeAppAuthClient pattern from the documentation
254+
client = NativeAppAuthClient(ZSTASH_CLIENT_ID)
255+
client.oauth2_start_flow(
256+
requested_scopes=all_scopes,
257+
refresh_tokens=True, # This is the key to persistent auth!
258+
)
259+
260+
authorize_url = client.oauth2_get_authorize_url()
261+
print("Please go to this URL and login: {0}".format(authorize_url))
262+
263+
auth_code = input("Please enter the code you get after login here: ").strip()
264+
token_response = client.oauth2_exchange_code_for_tokens(auth_code)
265+
266+
# Save tokens for next time
267+
save_tokens(token_response)
268+
269+
# Get the transfer token and create authorizer
270+
globus_transfer_data = token_response.by_resource_server["transfer.api.globus.org"]
271+
transfer_authorizer = RefreshTokenAuthorizer(
272+
refresh_token=globus_transfer_data["refresh_token"], auth_client=client
273+
)
274+
275+
transfer_client = TransferClient(authorizer=transfer_authorizer)
276+
return transfer_client, True # Fresh auth was needed
205277

206278

207279
def get_all_endpoint_scopes(endpoints: List[str]) -> str:
@@ -211,6 +283,21 @@ def get_all_endpoint_scopes(endpoints: List[str]) -> str:
211283
return f"urn:globus:auth:scope:transfer.api.globus.org:all[{inner}]"
212284

213285

286+
def save_tokens(token_response):
287+
"""Save tokens from a token response."""
288+
tokens_to_save = {}
289+
for resource_server, token_data in token_response.by_resource_server.items():
290+
tokens_to_save[resource_server] = {
291+
"access_token": token_data["access_token"],
292+
"refresh_token": token_data.get("refresh_token"),
293+
"expires_at": token_data.get("expires_at_seconds"),
294+
}
295+
296+
with open(TOKEN_FILE, "w") as f:
297+
json.dump(tokens_to_save, f, indent=2)
298+
print("Tokens saved successfully")
299+
300+
214301
def construct_TransferData(
215302
url: ParseResult,
216303
txt_file: str,
@@ -239,6 +326,17 @@ def construct_TransferData(
239326
return transfer_data
240327

241328

329+
def load_tokens():
330+
"""Load stored tokens if they exist."""
331+
if os.path.exists(TOKEN_FILE):
332+
try:
333+
with open(TOKEN_FILE, "r") as f:
334+
return json.load(f)
335+
except (json.JSONDecodeError, IOError):
336+
return {}
337+
return {}
338+
339+
242340
# Run #########################################################################
243341
if __name__ == "__main__":
244342
main()

‎examples/simple_globus_output.md‎

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,33 @@
1-
- Case 1: `REQUEST_SCOPES_EARLY: bool = False` => authenticate 2x on toy run, 0x on real run
2-
- Case 2: `REQUEST_SCOPES_EARLY: bool = True` => authenticate 1x on toy run, 1x on real run
1+
Case 2: `REQUEST_SCOPES_EARLY: bool = True` => authenticate 1x on toy run, 0x on real run
32

4-
| Code block | Case 1 | Case 2 |
5-
| --- | --- | --- |
6-
| TOY: `check_state_files` `INI_PATH` | `/home/ac.forsyth2/.zstash.ini does NOT exist.` | Same |
7-
| TOY: `check_state_files` `GLOBUS_CFG` | `/home/ac.forsyth2/.globus-native-apps.cfg does NOT exist.` | Same |
8-
| TOY: `get_local_endpoint_id` | `Added local_endpoint to ini file`, `Got local endpoint from NAME_TO_ENDPOINT_MAP` | Same |
9-
| TOY: `native_client_login`| Pasting URL brings us to "Allow" screen immediately, paste auth code at command line | Prompt (login NOT required) for Argonne, prompt (login NOT required) for NERSC, "Allow" screen, paste auth code at command line |
10-
| TOY: `transfer_client.submit_transfer` try/except | Prompt (login required) for Argonne, prompt (login required) for NERSC, "Allow" screen, paste auth code at command line, `Consents added, please re-run the previous command to start transfer` | `Bypassed 2nd authentication.` |
11-
| `For toy_run, skipped_second_auth=` | `False` | `True` |
12-
| REAL: `check_state_files` `INI_PATH` | `/home/ac.forsyth2/.zstash.ini exists.` | Same |
13-
| REAL: `check_state_files` `GLOBUS_CFG` | `/home/ac.forsyth2/.globus-native-apps.cfg exists.` | Same |
14-
| REAL: `get_local_endpoint_id` | `Got local_endpoint from ini file`, `Got local endpoint from NAME_TO_ENDPOINT_MAP` (implies the value retreived was `None`...) | Same |
15-
| REAL: `native_client_login`| No logins or prompts | Pasting URL brings us to "Allow" screen immediately, paste auth code at command line |
16-
| REAL: `transfer_client.submit_transfer` try/except | `Bypassed 2nd authentication.` | Same |
17-
| `For real_run, skipped_second_auth=` | `True` | Same |
3+
| Code block | Run |
4+
| --- | --- |
5+
| TOY: `check_state_files` | `INI_PATH: /home/ac.forsyth2/.zstash.ini does NOT exist.`, `GLOBUS_CFG: /home/ac.forsyth2/.globus-native-apps.cfg does NOT exist.`, `TOKEN_FILE: /home/ac.forsyth2/.zstash_globus_tokens.json does NOT exist.` |
6+
| TOY: `get_local_endpoint_id` | `Added local_endpoint to ini file`, `Got local endpoint from NAME_TO_ENDPOINT_MAP` |
7+
| TOY: `get_transfer_client_with_auth` | `No stored tokens found - starting authentication`, paste URL to web browser, Argonne prompt (no login), NERSC prompt (no login), Must add label (no default), "Allow", paste auth code to command line |
8+
| TOY: `transfer_client.submit_transfer` try/except block | `Used fresh authentication - tokens now stored for next time` |
9+
| `For toy_run, skipped_second_auth=` | `True` |
10+
| REAL: `check_state_files` | `INI_PATH: /home/ac.forsyth2/.zstash.ini exists.`, `GLOBUS_CFG: /home/ac.forsyth2/.globus-native-apps.cfg does NOT exist.`, `TOKEN_FILE: /home/ac.forsyth2/.zstash_globus_tokens.json exists.` |
11+
| REAL: `get_local_endpoint_id` | `Got local_endpoint from ini file`, `Got local endpoint from NAME_TO_ENDPOINT_MAP` (implies the value retreived was `None`...) |
12+
| REAL: `get_transfer_client_with_auth` | `Found stored refresh token - using it` |
13+
| REAL: `transfer_client.submit_transfer` try/except block | `Bypassed authentication entirely - used stored tokens!` |
14+
| `For real_run, skipped_second_auth` | `True` |
15+
16+
After run:
17+
```bash
18+
cat /home/ac.forsyth2/.zstash.ini
19+
# [local]
20+
# globus_endpoint_uuid =
21+
22+
cat /home/ac.forsyth2/.globus-native-apps.cfg
23+
# cat: /home/ac.forsyth2/.globus-native-apps.cfg: No such file or directory
24+
25+
cat /home/ac.forsyth2/.zstash_globus_tokens.json
26+
# {
27+
# "transfer.api.globus.org": {
28+
# "access_token": "alphanumeric token here>",
29+
# "refresh_token": "<alphanumeric token here>",
30+
# "expires_at": <number here>
31+
# }
32+
# }
33+
```

0 commit comments

Comments
 (0)