Skip to content

Commit 67ac848

Browse files
nellhnellh
authored
Merge pull request #3273 from OpenNeuroOrg/cli-feat-annex-remote
Deno CLI special remote implementation
2 parents ec2c84b + 64d8fb9 commit 67ac848

File tree

11 files changed

+910
-674
lines changed

11 files changed

+910
-674
lines changed

‎.github/workflows/deno.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ jobs:
2727
- uses: actions/checkout@v4
2828
- uses: denoland/setup-deno@v2
2929
with:
30-
deno-version: v1.x
30+
deno-version: v2.x
3131
- name: Collect coverage
3232
run: deno task coverage
3333
if: ${{ always() }}

‎bin/git-annex-remote-openneuro‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/sh
2+
# Use this script as your openneuro special remote when using datalad or git-annex to access annexed objects
3+
deno run -A jsr:@openneuro/cli special-remote

‎cli/deno.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
"coverage": "deno test --allow-read --allow-write --allow-net --allow-env --coverage ./ && deno coverage ./coverage --lcov > ./coverage.lcov"
1818
},
1919
"imports": {
20-
"@bids/validator": "jsr:@bids/validator@^1.14.12",
20+
"@bids/validator": "jsr:@bids/validator@^2.0.1",
2121
"@cliffy/command": "jsr:@cliffy/command@^1.0.0-rc.5",
2222
"@cliffy/prompt": "jsr:@cliffy/prompt@^1.0.0-rc.5",
2323
"@deno-library/progress": "jsr:@deno-library/progress@^1.4.9",

‎cli/deno.lock‎

Lines changed: 725 additions & 631 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎cli/mod.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,17 @@ Sentry.init({
99
release: `openneuro-cli@${denoJson.version}`,
1010
})
1111
import { commandLine } from "./src/options.ts"
12+
import { annexSpecialRemote } from "./src/commands/special-remote.ts"
1213

1314
/**
1415
* Entrypoint for running OpenNeuro command line tools
1516
*/
1617
export async function main() {
17-
await commandLine(Deno.args)
18+
if (Deno.execPath().endsWith("git-annex-remote-openneuro")) {
19+
await annexSpecialRemote()
20+
} else {
21+
await commandLine(Deno.args)
22+
}
1823
}
1924

2025
await main()

‎cli/src/commands/git-credential.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,10 @@ export const gitCredential = new Command()
8888
.description(
8989
"A git credentials helper for easier datalad or git-annex access to datasets.",
9090
)
91-
.command("fill")
91+
// Credentials here are short lived so store is not useful
92+
.command("store")
93+
.action(() => {})
94+
.command("get")
9295
.action(async () => {
9396
console.log(await gitCredentialAction())
9497
})

‎cli/src/commands/special-remote.ts‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
import { Command } from "@cliffy/command"
2+
import * as readline from "node:readline"
3+
import { once } from "node:events"
4+
import {
5+
checkKey,
6+
removeKey,
7+
retrieveKey,
8+
storeKey,
9+
type TransferKeyState,
10+
} from "../worker/transferKey.ts"
11+
import process from "node:process"
12+
import { getRepoAccess } from "./git-credential.ts"
13+
14+
const GIT_ANNEX_VERSION = "VERSION 1"
15+
16+
export async function handleGitAnnexMessage(
17+
line: string,
18+
state: TransferKeyState,
19+
) {
20+
if (line.startsWith("EXTENSIONS")) {
21+
return "EXTENSIONS"
22+
} else if (line.startsWith("PREPARE")) {
23+
// Ask for configuration to validate
24+
return "GETCONFIG url"
25+
} else if (line.startsWith("VALUE ")) {
26+
// Check if VALUE is configured already
27+
if (state.url) {
28+
return "PREPARE-SUCCESS"
29+
} else {
30+
return "PREPARE-FAILURE url must be configured when running initremote or enableremote"
31+
}
32+
} else if (line.startsWith("TRANSFER STORE")) {
33+
const [, , key, file] = line.split(" ", 4)
34+
if (await storeKey(state, key, file)) {
35+
return `TRANSFER-SUCCESS STORE ${key}`
36+
} else {
37+
return `TRANSFER-FAILURE STORE ${key}`
38+
}
39+
} else if (line.startsWith("TRANSFER RETRIEVE")) {
40+
const [, , key, file] = line.split(" ", 4)
41+
if (await retrieveKey(state, key, file)) {
42+
return `TRANSFER-SUCCESS RETRIEVE ${key}`
43+
} else {
44+
return `TRANSFER-FAILURE RETRIEVE ${key}`
45+
}
46+
} else if (line.startsWith("CHECKPRESENT")) {
47+
const key = line.split("CHECKPRESENT ", 2)[1]
48+
if (await checkKey(state, key)) {
49+
return `CHECKPRESENT-SUCCESS ${key}`
50+
} else {
51+
return `CHECKPRESENT-FAILURE ${key}`
52+
}
53+
} else if (line.startsWith("INITREMOTE")) {
54+
// No init steps are required - always succeed
55+
return "INITREMOTE-SUCCESS"
56+
} else if (line.startsWith("GETAVAILABILITY")) {
57+
return "AVAILABILITY GLOBAL"
58+
} else if (line.startsWith("REMOVE")) {
59+
const key = line.split("REMOVE ", 2)[1]
60+
if (await removeKey(state, key)) {
61+
return `REMOVE-SUCCESS ${key}`
62+
} else {
63+
return `REMOVE-FAILURE ${key}`
64+
}
65+
} else {
66+
return "UNSUPPORTED-REQUEST"
67+
}
68+
}
69+
70+
/**
71+
* Stateful response handling for git annex protocol
72+
* @returns {() => void}
73+
*/
74+
export const response = () => {
75+
const state: TransferKeyState = {
76+
url: "",
77+
token: "",
78+
}
79+
return async (line: string) => {
80+
if (line.startsWith("VALUE ")) {
81+
try {
82+
const url = line.split("VALUE ")[1]
83+
// Obtain the filename (no extensions) in url value
84+
const datasetId = url.substring(url.lastIndexOf("/") + 1, url.length)
85+
state.url = url
86+
const { token } = await getRepoAccess(datasetId)
87+
state.token = token
88+
} catch (_err) {
89+
state.url = ""
90+
state.token = ""
91+
}
92+
}
93+
console.log(await handleGitAnnexMessage(line, state))
94+
}
95+
}
96+
97+
/**
98+
* Git annex special remote
99+
*/
100+
export async function annexSpecialRemote() {
101+
try {
102+
const rl = readline.createInterface({
103+
input: process.stdin,
104+
})
105+
console.log(GIT_ANNEX_VERSION)
106+
rl.on("line", response())
107+
await once(rl, "close")
108+
} catch (err) {
109+
console.error(err)
110+
}
111+
}
112+
113+
export const specialRemote = new Command()
114+
.name("special-remote")
115+
.description(
116+
"git-annex special remote for uploading or downloading from OpenNeuro",
117+
)
118+
.action(async () => {
119+
await annexSpecialRemote()
120+
})

‎cli/src/options.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import { login } from "./commands/login.ts"
77
import { upload } from "./commands/upload.ts"
88
import { download } from "./commands/download.ts"
99
import { gitCredential } from "./commands/git-credential.ts"
10+
import { specialRemote } from "./commands/special-remote.ts"
1011

1112
export type OpenNeuroOptions = {
1213
datasetPath: string
@@ -29,8 +30,10 @@ const openneuroCommand = new Command()
2930
.globalEnv("OPENNEURO_API_KEY=<key:string>", "Specify an OpenNeuro API key.")
3031
.command("login", login)
3132
.command("download", download)
33+
// @ts-expect-error This is typed correctly but not loaded from the dependency as expected
3234
.command("upload", upload)
3335
.command("git-credential", gitCredential)
36+
.command("special-remote", specialRemote)
3437

3538
/**
3639
* Parse command line options and return a OpenNeuroOptions config

‎cli/src/worker/git.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@ async function commitAnnexBranch(annexKeys: Record<string, string>) {
192192
try {
193193
uuidLog = await readAnnexPath("uuid.log", context)
194194
} catch (err) {
195-
if (err.name !== "NotFoundError") {
195+
if (err instanceof Error && err.name !== "NotFoundError") {
196196
throw err
197197
}
198198
}
@@ -204,7 +204,7 @@ async function commitAnnexBranch(annexKeys: Record<string, string>) {
204204
})
205205
} catch (err) {
206206
// Create the branch if it doesn't exist
207-
if (err.name === "NotFoundError") {
207+
if (err instanceof Error && err.name === "NotFoundError") {
208208
await createAnnexBranch()
209209
}
210210
}
@@ -225,7 +225,7 @@ async function commitAnnexBranch(annexKeys: Record<string, string>) {
225225
{ encoding: "utf8" },
226226
)
227227
} catch (_err) {
228-
if (_err.name !== "NotFound") {
228+
if (_err instanceof Error && _err.name !== "NotFound") {
229229
throw _err
230230
}
231231
} finally {
@@ -247,7 +247,7 @@ async function commitAnnexBranch(annexKeys: Record<string, string>) {
247247
try {
248248
log = await readAnnexPath(annexBranchPath, context)
249249
} catch (err) {
250-
if (err.name === "NotFoundError") {
250+
if (err instanceof Error && err.name === "NotFoundError") {
251251
logger.debug(`Annex branch object "${annexBranchPath}" not found`)
252252
} else {
253253
throw err
@@ -283,7 +283,7 @@ async function commitAnnexBranch(annexKeys: Record<string, string>) {
283283
ref: "main",
284284
})
285285
} catch (err) {
286-
if (err.name === "NotFoundError") {
286+
if (err instanceof Error && err.name === "NotFoundError") {
287287
// Fallback to master and error if neither exists
288288
await git.checkout({
289289
...context.config(),

‎cli/src/worker/transferKey.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ export async function storeKey(
7575
try {
7676
fileHandle?.close()
7777
} catch (err) {
78-
if (err.name !== "BadResource") {
78+
if (err instanceof Error && err.name !== "BadResource") {
7979
logger.error(err)
8080
}
8181
}

0 commit comments

Comments
 (0)