diff --git a/packages/openneuro-server/src/graphql/resolvers/fileCheck.ts b/packages/openneuro-server/src/graphql/resolvers/fileCheck.ts new file mode 100644 index 000000000..e0f3ca834 --- /dev/null +++ b/packages/openneuro-server/src/graphql/resolvers/fileCheck.ts @@ -0,0 +1,17 @@ +import FileCheck from "../../models/fileCheck" +import { checkDatasetAdmin } from "../permissions" + +export const updateFileCheck = async ( + obj, + { datasetId, hexsha, refs, remote, annexFsck }, + { user, userInfo }, +) => { + await checkDatasetAdmin(datasetId, user, userInfo) + return await FileCheck.findOneAndUpdate( + { datasetId, hexsha }, + { datasetId, hexsha, remote, refs, annexFsck }, + { upsert: true, new: true }, + ) + .lean() + .exec() +} diff --git a/packages/openneuro-server/src/graphql/resolvers/mutation.ts b/packages/openneuro-server/src/graphql/resolvers/mutation.ts index cec8f3139..f3860795c 100644 --- a/packages/openneuro-server/src/graphql/resolvers/mutation.ts +++ b/packages/openneuro-server/src/graphql/resolvers/mutation.ts @@ -45,6 +45,7 @@ import { } from "./importRemoteDataset" import { saveAdminNote } from "./datasetEvents" import { createGitEvent } from "./gitEvents" +import { updateFileCheck } from "./fileCheck" const Mutation = { createDataset, @@ -93,6 +94,7 @@ const Mutation = { updateUser, saveAdminNote, createGitEvent, + updateFileCheck, } export default Mutation diff --git a/packages/openneuro-server/src/graphql/schema.ts b/packages/openneuro-server/src/graphql/schema.ts index 86052f1cb..c9eaafa32 100644 --- a/packages/openneuro-server/src/graphql/schema.ts +++ b/packages/openneuro-server/src/graphql/schema.ts @@ -205,6 +205,14 @@ export const typeDefs = ` saveAdminNote(id: ID, datasetId: ID!, note: String!): DatasetEvent # Create a git event log for dataset changes createGitEvent(datasetId: ID!, commit: String!, reference: String!): DatasetEvent + # Create or update a fileCheck document + updateFileCheck( + datasetId: ID! + hexsha: String! + refs: [String!]! + annexFsck: [AnnexFsckInput!]! + remote: String + ): FileCheck } # Anonymous dataset reviewer @@ -900,6 +908,33 @@ export const typeDefs = ` # Notes associated with the event note: String } + + type FileCheck { + datasetId: String! + hexsha: String! + refs: [String!]! + annexFsck: [AnnexFsck!] + remote: String + } + + type AnnexFsck { + command: String + errorMessages: [String] + file: String + key: String + note: String + success: Boolean + } + + input AnnexFsckInput { + command: String + errorMessages: [String] + file: String + key: String + note: String + success: Boolean + } + ` schemaComposer.addTypeDefs(typeDefs) diff --git a/packages/openneuro-server/src/models/fileCheck.ts b/packages/openneuro-server/src/models/fileCheck.ts new file mode 100644 index 000000000..1f05278e8 --- /dev/null +++ b/packages/openneuro-server/src/models/fileCheck.ts @@ -0,0 +1,37 @@ +import mongoose from "mongoose" +import type { Document } from "mongoose" +const { Schema, model } = mongoose + +export interface FileCheckDocument extends Document { + datasetId: string + hexsha: string + refs: string[] + remote: string + annexFsck: { + command: string + "error-messages": string[] + file: string + key: string + note: string + success: boolean + }[] +} + +const fileCheckSchema = new Schema({ + datasetId: { type: String, required: true }, + hexsha: { type: String, required: true }, + refs: { type: [String], required: true }, + remote: { type: String, default: "local", required: true }, + annexFsck: [{ + command: String, + "error-messages": [String], + file: String, + key: String, + note: String, + success: Boolean, + }], +}) + +const FileCheck = model("FileCheck", fileCheckSchema) + +export default FileCheck diff --git a/services/datalad/datalad_service/common/openneuro.py b/services/datalad/datalad_service/common/openneuro.py index 26e4240d8..97c9ac61e 100644 --- a/services/datalad/datalad_service/common/openneuro.py +++ b/services/datalad/datalad_service/common/openneuro.py @@ -1,6 +1,26 @@ import requests +from pathlib import Path +import jwt +import logging +from datetime import datetime, timedelta, timezone -from datalad_service.config import GRAPHQL_ENDPOINT +from datalad_service.config import GRAPHQL_ENDPOINT, JWT_SECRET + + +def generate_service_token(dataset_id): + utc_now = datetime.now(timezone.utc) + one_day_ahead = utc_now + timedelta(hours=24) + return jwt.encode( + { + 'sub': 'dataset-worker', + 'iat': int(utc_now.timestamp()), + 'exp': int(one_day_ahead.timestamp()), + 'scopes': ['dataset:worker'], + 'dataset': dataset_id, + }, + JWT_SECRET, + algorithm='HS256', + ) def cache_clear_mutation(dataset_id, tag): @@ -18,3 +38,30 @@ def clear_dataset_cache(dataset, tag, cookies={}): ) if r.status_code != 200: raise Exception(r.text) + + +def update_file_check(dataset_path, commit, references, bad_files, remote=None): + """Post results of git-annex fsck to graphql endpoint.""" + dataset_id = Path(dataset_path).name + try: + post_body = { + 'query': 'mutation updateFileCheck($datasetId: ID!, $hexsha: String!, $refs: [String!]!, $annexFsck: [AnnexFsckInput!]!) { updateFileCheck(datasetId: $datasetId, hexsha: $hexsha, refs: $refs, annexFsck: $annexFsck) { datasetId, hexsha } }', + 'variables': { + 'datasetId': dataset_id, + 'hexsha': str(commit.id), + 'refs': references, + 'annexFsck': bad_files, + }, + 'operationName': 'updateFileCheck', + } + if remote: + post_body['variables']['remote'] = remote + req = requests.post( + url=GRAPHQL_ENDPOINT, + json=post_body, + headers={'authorization': f'Bearer {generate_service_token(dataset_id)}'}, + ) + req.raise_for_status() + except requests.exceptions.HTTPError as e: + logging.error(e) + logging.error(req.text)