Merge pull request #3535 from OpenNeuroOrg/feat/update-file-check-mutation

Add FileCheck model and updateFileCheck mutation

Author: nellh

packages/openneuro-server/src/graphql/resolvers/fileCheck.ts

@@ -0,0 +1,17 @@
+import FileCheck from "../../models/fileCheck"
+import { checkDatasetAdmin } from "../permissions"
+
+export const updateFileCheck = async (
+  obj,
+  { datasetId, hexsha, refs, remote, annexFsck },
+  { user, userInfo },
+) => {
+  await checkDatasetAdmin(datasetId, user, userInfo)
+  return await FileCheck.findOneAndUpdate(
+    { datasetId, hexsha },
+    { datasetId, hexsha, remote, refs, annexFsck },
+    { upsert: true, new: true },
+  )
+    .lean()
+    .exec()
+}

packages/openneuro-server/src/graphql/resolvers/mutation.ts

@@ -45,6 +45,7 @@ import {
 } from "./importRemoteDataset"
 import { saveAdminNote } from "./datasetEvents"
 import { createGitEvent } from "./gitEvents"
+import { updateFileCheck } from "./fileCheck"
 
 const Mutation = {
   createDataset,
@@ -93,6 +94,7 @@ const Mutation = {
   updateUser,
   saveAdminNote,
   createGitEvent,
+  updateFileCheck,
 }
 
 export default Mutation

packages/openneuro-server/src/graphql/schema.ts

@@ -205,6 +205,14 @@ export const typeDefs = `
     saveAdminNote(id: ID, datasetId: ID!, note: String!): DatasetEvent
     # Create a git event log for dataset changes
     createGitEvent(datasetId: ID!, commit: String!, reference: String!): DatasetEvent
+    # Create or update a fileCheck document
+    updateFileCheck(
+      datasetId: ID!
+      hexsha: String!
+      refs: [String!]!
+      annexFsck: [AnnexFsckInput!]!
+      remote: String
+    ): FileCheck
   }
 
   # Anonymous dataset reviewer
@@ -900,6 +908,33 @@ export const typeDefs = `
     # Notes associated with the event
     note: String
   }
+
+  type FileCheck {
+    datasetId: String!
+    hexsha: String!
+    refs: [String!]!
+    annexFsck: [AnnexFsck!]
+    remote: String
+  }
+
+  type AnnexFsck {
+    command: String
+    errorMessages: [String]
+    file: String
+    key: String
+    note: String
+    success: Boolean
+  }
+
+  input AnnexFsckInput {
+    command: String
+    errorMessages: [String]
+    file: String
+    key: String
+    note: String
+    success: Boolean
+  }
+
 `
 
 schemaComposer.addTypeDefs(typeDefs)

packages/openneuro-server/src/models/fileCheck.ts

@@ -0,0 +1,37 @@
+import mongoose from "mongoose"
+import type { Document } from "mongoose"
+const { Schema, model } = mongoose
+
+export interface FileCheckDocument extends Document {
+  datasetId: string
+  hexsha: string
+  refs: string[]
+  remote: string
+  annexFsck: {
+    command: string
+    "error-messages": string[]
+    file: string
+    key: string
+    note: string
+    success: boolean
+  }[]
+}
+
+const fileCheckSchema = new Schema({
+  datasetId: { type: String, required: true },
+  hexsha: { type: String, required: true },
+  refs: { type: [String], required: true },
+  remote: { type: String, default: "local", required: true },
+  annexFsck: [{
+    command: String,
+    "error-messages": [String],
+    file: String,
+    key: String,
+    note: String,
+    success: Boolean,
+  }],
+})
+
+const FileCheck = model<FileCheckDocument>("FileCheck", fileCheckSchema)
+
+export default FileCheck

services/datalad/datalad_service/common/openneuro.py

@@ -1,6 +1,26 @@
 import requests
+from pathlib import Path
+import jwt
+import logging
+from datetime import datetime, timedelta, timezone
 
-from datalad_service.config import GRAPHQL_ENDPOINT
+from datalad_service.config import GRAPHQL_ENDPOINT, JWT_SECRET
+
+
+def generate_service_token(dataset_id):
+    utc_now = datetime.now(timezone.utc)
+    one_day_ahead = utc_now + timedelta(hours=24)
+    return jwt.encode(
+        {
+            'sub': 'dataset-worker',
+            'iat': int(utc_now.timestamp()),
+            'exp': int(one_day_ahead.timestamp()),
+            'scopes': ['dataset:worker'],
+            'dataset': dataset_id,
+        },
+        JWT_SECRET,
+        algorithm='HS256',
+    )
 
 
 def cache_clear_mutation(dataset_id, tag):
@@ -18,3 +38,30 @@ def clear_dataset_cache(dataset, tag, cookies={}):
     )
     if r.status_code != 200:
         raise Exception(r.text)
+
+
+def update_file_check(dataset_path, commit, references, bad_files, remote=None):
+    """Post results of git-annex fsck to graphql endpoint."""
+    dataset_id = Path(dataset_path).name
+    try:
+        post_body = {
+            'query': 'mutation updateFileCheck($datasetId: ID!, $hexsha: String!, $refs: [String!]!, $annexFsck: [AnnexFsckInput!]!) { updateFileCheck(datasetId: $datasetId, hexsha: $hexsha, refs: $refs, annexFsck: $annexFsck) { datasetId, hexsha } }',
+            'variables': {
+                'datasetId': dataset_id,
+                'hexsha': str(commit.id),
+                'refs': references,
+                'annexFsck': bad_files,
+            },
+            'operationName': 'updateFileCheck',
+        }
+        if remote:
+            post_body['variables']['remote'] = remote
+        req = requests.post(
+            url=GRAPHQL_ENDPOINT,
+            json=post_body,
+            headers={'authorization': f'Bearer {generate_service_token(dataset_id)}'},
+        )
+        req.raise_for_status()
+    except requests.exceptions.HTTPError as e:
+        logging.error(e)
+        logging.error(req.text)