Merge pull request #3392 from OpenNeuroOrg/feat/readme-md

nellh · web-flow · commit 9de4e1bc2236 · 2025-03-24T09:26:57.000-07:00
Handle README.md and README.rst
diff --git a/packages/openneuro-app/src/scripts/dataset/files/file-viewer-type.jsx b/packages/openneuro-app/src/scripts/dataset/files/file-viewer-type.jsx
@@ -8,6 +8,7 @@ import FileViewerCsv from "./viewers/file-viewer-csv.jsx"
 import FileViewerHtml from "./viewers/file-viewer-html.jsx"
 import { FileViewerNeurosift } from "./viewers/file-viewer-neurosift"
 import { isNifti } from "./file-types"
+import FileViewerMarkdown from "./viewers/file-viewer-markdown"
 
 /**
  * Choose the right viewer for each file type
@@ -18,13 +19,16 @@ const FileViewerType = ({ path, url, data }) => {
     path.endsWith("CHANGES") ||
     path.endsWith(".bidsignore") ||
     path.endsWith(".gitignore") ||
-    path.endsWith(".txt")
+    path.endsWith(".txt") ||
+    path.endsWith(".rst")
   ) {
     return <FileViewerText data={data} />
   } else if (
     isNifti(path)
   ) {
     return <FileViewerNifti imageUrl={url} />
+  } else if (path.endsWith(".md")) {
+    return <FileViewerMarkdown data={data} />
   } else if (path.endsWith(".json")) {
     return <FileViewerJson data={data} />
   } else if (path.endsWith(".tsv")) {
diff --git a/packages/openneuro-app/src/scripts/dataset/files/viewers/file-viewer-markdown.tsx b/packages/openneuro-app/src/scripts/dataset/files/viewers/file-viewer-markdown.tsx
@@ -0,0 +1,13 @@
+import React from "react"
+import { Markdown } from "../../../utils/markdown"
+
+interface FileViewerMarkdownProps {
+  data: ArrayBuffer
+}
+
+const FileViewerMarkdown = ({ data }: FileViewerMarkdownProps) => {
+  const decoder = new TextDecoder()
+  return <Markdown>{decoder.decode(data)}</Markdown>
+}
+
+export default FileViewerMarkdown
diff --git a/packages/openneuro-server/src/datalad/readme.ts b/packages/openneuro-server/src/datalad/readme.ts
@@ -34,8 +34,8 @@ export const readme = (obj) => {
   })
 }
 
-export const setReadme = (datasetId, readme, user) => {
-  return addFileString(datasetId, "README", "text/plain", readme).then(() =>
+export const setReadme = (datasetId, readme, filename, user) => {
+  return addFileString(datasetId, filename, "text/plain", readme).then(() =>
     commitFiles(datasetId, user)
   )
 }
diff --git a/packages/openneuro-server/src/graphql/resolvers/draft.ts b/packages/openneuro-server/src/graphql/resolvers/draft.ts
@@ -10,7 +10,7 @@ import { filterRemovedAnnexObjects } from "../utils/file.js"
 import { validation } from "./validation"
 
 // A draft must have a dataset parent
-const draftFiles = async (dataset, args, { userInfo }) => {
+export const draftFiles = async (dataset, args, { userInfo }) => {
   const hexsha = await getDraftRevision(dataset.id)
   const files = await getFiles(dataset.id, args.tree || hexsha)
   return filterRemovedAnnexObjects(dataset.id, userInfo)(files)
diff --git a/packages/openneuro-server/src/graphql/resolvers/readme.ts b/packages/openneuro-server/src/graphql/resolvers/readme.ts
@@ -5,10 +5,30 @@
 import { setReadme } from "../../datalad/readme"
 import { checkDatasetWrite } from "../permissions"
 export { readme } from "../../datalad/readme"
+import { draftFiles } from "./draft"
 
-export const updateReadme = (obj, { datasetId, value }, { user, userInfo }) => {
-  return checkDatasetWrite(datasetId, user, userInfo).then(() => {
-    // Save to backend
-    return setReadme(datasetId, value, userInfo).then(() => true)
+export async function updateReadme(
+  obj,
+  { datasetId, value },
+  { user, userInfo },
+) {
+  await checkDatasetWrite(datasetId, user, userInfo)
+  const files = await draftFiles({ id: datasetId }, { tree: "HEAD" }, {
+    userInfo,
   })
+  // Default to README.md if none exists
+  let filename = "README.md"
+  for (const file of files) {
+    if (
+      file.filename === "README.md" || file.filename === "README.rst" ||
+      file.filename === "README.txt" ||
+      file.filename === "README"
+    ) {
+      filename = file.filename
+      break
+    }
+  }
+  // Save to backend
+  await setReadme(datasetId, value, filename, userInfo)
+  return true
 }
diff --git a/services/datalad/datalad_service/common/bids.py b/services/datalad/datalad_service/common/bids.py
@@ -1,12 +1,15 @@
 import json
 
+import pygit2
+
 from datalad_service.common.git import git_show
 
 
 def read_dataset_description(dataset_path, commit):
     try:
+        repo = pygit2.Repository(dataset_path)
         raw_description = git_show(
-            dataset_path, commit, 'dataset_description.json')
+            repo, commit, 'dataset_description.json')
         return json.loads(raw_description)
     except json.decoder.JSONDecodeError:
         return None
diff --git a/services/datalad/datalad_service/common/git.py b/services/datalad/datalad_service/common/git.py
@@ -1,3 +1,4 @@
+import pathlib
 import re
 import subprocess
 
@@ -14,23 +15,32 @@ class OpenNeuroGitError(Exception):
     """OpenNeuro git repo states that should not arise under normal use but may be a valid git operation in other contexts."""
 
 
-def git_show(path, committish, obj):
-    repo = pygit2.Repository(path)
+def git_show(repo, committish, obj):
+    """Equivalent to `git show <committish>:<obj>` on `repo` repository."""
     commit, _ = repo.resolve_refish(committish)
     data_bytes = (commit.tree / obj).read_raw()
     result = from_bytes(data_bytes).best()
     return str(result)
 
 
-def git_show_object(path, obj):
-    repo = pygit2.Repository(path)
+def git_show_object(repo, obj):
     git_obj = repo.get(obj)
     if git_obj:
         return git_obj.read_raw().decode()
     else:
         raise KeyError('object not found in repository')
 
 
+def git_tree(repo, committish, filepath):
+    """Retrieve the tree parent for a given commit and filename."""
+    path = pathlib.Path(filepath)
+    commit, _ = repo.resolve_refish(committish)
+    tree = commit.tree
+    for part in path.parts[:-1]:
+        tree = tree / part
+    return tree
+
+
 def delete_tag(path, tag):
     repo = pygit2.Repository(path)
     repo.references.delete(f'refs/tags/{tag}')
diff --git a/services/datalad/datalad_service/datalad.py b/services/datalad/datalad_service/datalad.py
@@ -1,10 +1,13 @@
 from os import path
 
+import pygit2
+
 class DataladStore:
     """Store for Datalad state accessed by resource handlers."""
 
     def __init__(self, annex_path):
         self.annex_path = annex_path
+        self.repos = {}
 
     def get_dataset_path(self, name):
         return path.join(self.annex_path, name)
@@ -13,3 +16,8 @@ def get_upload_path(self, dataset, upload):
         prefix_a = upload[0:2]
         prefix_b = upload[2:4]
         return path.join(self.annex_path, 'uploads', dataset, prefix_a, prefix_b, upload)
+
+    def get_dataset_repo(self, dataset):
+        if dataset not in self.repos:
+            self.repos[dataset] = pygit2.Repository(self.get_dataset_path(dataset))
+        return self.repos[dataset]
diff --git a/services/datalad/datalad_service/handlers/files.py b/services/datalad/datalad_service/handlers/files.py
@@ -1,11 +1,12 @@
 import logging
 import os
+from pathlib import Path
 
 import falcon
 import pygit2
 import aiofiles
 
-from datalad_service.common.git import git_show
+from datalad_service.common.git import git_show, git_tree
 from datalad_service.common.user import get_user_info
 from datalad_service.common.stream import update_file
 from datalad_service.tasks.files import remove_files
@@ -21,7 +22,16 @@ async def on_get(self, req, resp, dataset, filename, snapshot='HEAD'):
         ds_path = self.store.get_dataset_path(dataset)
         try:
             try:
-                file_content = git_show(ds_path, snapshot, filename)
+                repo = self.store.get_dataset_repo(dataset)
+                tree = git_tree(repo, snapshot, filename)
+                # Look for any files that only differ by extension
+                path = Path(filename)
+                if path.name not in tree:
+                    for obj in tree:
+                        if Path(obj.name).stem == path.name:
+                            filename = obj.name
+                            break
+                file_content = git_show(repo, snapshot, filename)
                 # If the file begins with an annex path, return that path
                 if file_content[0:4096].find('.git/annex') != -1:
                     # Resolve absolute path for annex target
diff --git a/services/datalad/datalad_service/handlers/objects.py b/services/datalad/datalad_service/handlers/objects.py
@@ -12,10 +12,10 @@ def __init__(self, store):
         self.logger = logging.getLogger('datalad_service.' + __name__)
 
     async def on_get(self, req, resp, dataset, obj):
-        ds_path = self.store.get_dataset_path(dataset)
+        repo = self.store.get_dataset_repo(dataset)
         try:
             if len(obj) == 40:
-                resp.text = git_show_object(ds_path, obj)
+                resp.text = git_show_object(repo, obj)
                 resp.status = falcon.HTTP_OK
             else:
                 resp.media = {
diff --git a/services/datalad/datalad_service/tasks/description.py b/services/datalad/datalad_service/tasks/description.py
@@ -12,9 +12,9 @@ def edit_description(description, new_fields):
 
 
 def update_description(store, dataset, description_fields, name=None, email=None):
-    dataset_path = store.get_dataset_path(dataset)
+    repo = store.get_dataset_repo(dataset)
     description = git_show(
-        dataset_path, 'HEAD', 'dataset_description.json')
+        repo, 'HEAD', 'dataset_description.json')
     description_json = json.loads(description)
     if description_json.get('License') != 'CC0':
         description_fields = edit_description(
diff --git a/services/datalad/datalad_service/tasks/publish.py b/services/datalad/datalad_service/tasks/publish.py
@@ -95,7 +95,8 @@ def check_remote_has_version(dataset_path, remote, tag):
 
         # extract remote uuid and associated git tree id from `git show git-annex:export.log`
         # this command only logs the latest export. previously exported tags will not show
-        export_log = git_show(dataset_path, 'git-annex', 'export.log')
+        repo = pygit2.Repository(dataset_path)
+        export_log = git_show(repo, 'git-annex', 'export.log')
         log_remote_id_pattern = re.compile(':(.+) .+$')
         match = log_remote_id_pattern.search(export_log)
         remote_id_B = match.group(1)
diff --git a/services/datalad/datalad_service/tasks/snapshots.py b/services/datalad/datalad_service/tasks/snapshots.py
@@ -87,7 +87,8 @@ def edit_changes(changes, new_changes, tag, date):
 
 def get_head_changes(dataset_path):
     try:
-        return git_show(dataset_path, 'HEAD', 'CHANGES')
+        repo = pygit2.Repository(dataset_path)
+        return git_show(repo, 'HEAD', 'CHANGES')
     except KeyError:
         return None
 
@@ -137,8 +138,9 @@ def validate_snapshot_name(store, dataset, snapshot):
 def validate_datalad_config(store, dataset):
     """Add a .datalad/config file if one does not exist."""
     dataset_path = store.get_dataset_path(dataset)
+    repo = store.get_dataset_repo(dataset)
     try:
-        git_show(dataset_path, 'HEAD', '.datalad/config')
+        git_show(repo, 'HEAD', '.datalad/config')
     except KeyError:
         create_datalad_config(dataset_path)
         commit_files(store, dataset, ['.datalad/config'])
diff --git a/services/datalad/tests/test_files.py b/services/datalad/tests/test_files.py
@@ -29,6 +29,16 @@ def test_get_file(client):
     assert json.loads(result.content)['BIDSVersion'] == '1.0.2'
 
 
+def test_get_file_extensionless(client):
+    ds_id = 'ds000001'
+    result = client.simulate_get(
+        f'/datasets/{ds_id}/files/dataset_description', file_wrapper=FileWrapper)
+    content_len = int(result.headers['content-length'])
+    assert content_len == len(result.content)
+    print(result.content)
+    assert json.loads(result.content)['BIDSVersion'] == '1.0.2'
+
+
 def test_get_annexed_file(client):
     ds_id = 'ds000001'
     file_data = 'test image, please ignore'
diff --git a/services/datalad/tests/test_git.py b/services/datalad/tests/test_git.py
@@ -17,7 +17,8 @@
 
 
 def test_git_show(new_dataset):
-    assert git.git_show(new_dataset.path, 'HEAD',
+    repo = pygit2.Repository(new_dataset.path)
+    assert git.git_show(repo, 'HEAD',
                         'dataset_description.json') == '{"BIDSVersion": "1.0.2", "License": "This is not a real dataset", "Name": "Test fixture new dataset"}'
 
 
@@ -37,7 +38,8 @@ def test_git_show_non_iso_test(new_dataset):
         f.write(non_utf8_events)
     ds.save(events_path)
     ds.close()
-    assert git.git_show(new_dataset.path, 'HEAD',
+    repo = pygit2.Repository(new_dataset.path)
+    assert git.git_show(repo, 'HEAD',
                         'events.tsv') == non_utf8_events.decode('cp852')
 
 dataset_description_4096 = """{
@@ -69,7 +71,8 @@ def test_git_show_unicode_after_4096(new_dataset):
         f.write(dataset_description_4096)
     ds.save(desc_path)
     ds.close()
-    assert git.git_show(new_dataset.path, 'HEAD',
+    repo = pygit2.Repository(new_dataset.path)
+    assert git.git_show(repo, 'HEAD',
                         'dataset_description.json') == dataset_description_4096.decode('utf-8')
 
 
@@ -160,3 +163,9 @@ def test_git_tag_tree(new_dataset):
     # Create a tag
     repo.references.create(f'refs/tags/{tag}', str(repo.head.target))
     assert git.git_tag_tree(repo, tag) == repo.get(repo.head.target).tree_id
+
+
+def test_git_tree(new_dataset):
+    repo = pygit2.Repository(new_dataset.path)
+    tree = git.git_tree(repo, str(repo.head.target), "dataset_description.json")
+    assert tree.id == repo.get(repo.head.target).tree_id
diff --git a/services/datalad/tests/test_snapshots.py b/services/datalad/tests/test_snapshots.py
@@ -167,7 +167,8 @@ def test_write_new_changes(datalad_store, new_dataset):
     # Get a fresh dataset object and verify correct CHANGES
     dataset = Dataset(os.path.join(datalad_store.annex_path, ds_id))
     assert not dataset.repo.dirty
-    assert git_show(dataset.path, 'HEAD', 'CHANGES') == '''1.0.1 2019-01-01
+    repo = datalad_store.get_dataset_repo(ds_id)
+    assert git_show(repo, 'HEAD', 'CHANGES') == '''1.0.1 2019-01-01
   - Some changes
 1.0.0 2018-01-01
   - Initial version
@@ -184,6 +185,7 @@ def test_write_with_empty_changes(datalad_store, new_dataset):
     # Get a fresh dataset object and verify correct CHANGES
     dataset = Dataset(os.path.join(datalad_store.annex_path, ds_id))
     assert not dataset.repo.dirty
-    assert git_show(dataset.path, 'HEAD', 'CHANGES') == '''1.0.1 2019-01-01
+    repo = datalad_store.get_dataset_repo(ds_id)
+    assert git_show(repo, 'HEAD', 'CHANGES') == '''1.0.1 2019-01-01
   - Some changes
 '''

Original file line number	Diff line number	Diff line change
`@@ -34,8 +34,8 @@ export const readme = (obj) => {`
`34`	`34`	`})`
`35`	`35`	`}`
`36`	`36`
`37`		`-export const setReadme = (datasetId, readme, user) => {`
`38`		`- return addFileString(datasetId, "README", "text/plain", readme).then(() =>`
	`37`	`+export const setReadme = (datasetId, readme, filename, user) => {`
	`38`	`+ return addFileString(datasetId, filename, "text/plain", readme).then(() =>`
`39`	`39`	`commitFiles(datasetId, user)`
`40`	`40`	`)`
`41`	`41`	`}`