fix(worker): Implement workaround for git-annex initremote on GCP object storage with fileprefix

Author: nellh

services/datalad/datalad_service/common/annex.py

@@ -5,6 +5,8 @@
 import subprocess
 import os
 import urllib.parse
+import time
+import uuid
 
 import aiofiles
 import pygit2
@@ -22,6 +24,12 @@
 ]
 
 
+class InitRemoteException(Exception):
+    """Raised when git-annex initremote fails."""
+
+    pass
+
+
 class EditAnnexedFileException(Exception):
     """Snapshot conflicts with existing name."""
 
@@ -380,3 +388,52 @@ def test_key_remote(dataset_path, key, remote_name='s3-PUBLIC'):
                 if remote['uuid'] in line:
                     return parse_rmet_line(remote, line)
     return None
+
+
+def annex_initremote(dataset_path, remote_name, remote_options):
+    """Initialize a git-annex remote manually."""
+    remote_uuid = str(uuid.uuid4())
+    repo = pygit2.Repository(dataset_path)
+    branch = repo.branches.get('git-annex')
+    if not branch:
+        raise InitRemoteException('git-annex branch not found')
+    commit = branch.peel()
+    tree = commit.tree
+    # Read remote.log
+    file_content = ''
+    log_entry = None
+    try:
+        # Check if 'remote.log' exists in the tree
+        log_entry = tree['remote.log']
+        blob = repo.get(log_entry.id)
+
+        # Ensure file is not binary
+        if not blob.is_binary:
+            file_content = blob.data.decode('utf-8')
+        else:
+            raise InitRemoteException("'remote.log' is a binary file, cannot read.")
+    except KeyError:
+        # 'remote.log' doesn't exist yet, we'll create it.
+        file_content = ''  # Ensure it's an empty string
+
+    new_content = (
+        file_content
+        + f'{remote_uuid} type=S3 name={remote_name} timestamp={int(time.time())}s '
+        + ' '.join(remote_options)
+    )
+    new_blob_oid = repo.create_blob(new_content.encode('utf-8'))
+    builder = repo.TreeBuilder(tree)
+    builder.insert('remote.log', new_blob_oid, pygit2.GIT_FILEMODE_BLOB)
+    new_tree_oid = builder.write()
+    author = pygit2.Signature(SERVICE_USER, SERVICE_EMAIL)
+    committer = pygit2.Signature(SERVICE_USER, SERVICE_EMAIL)
+    commit_message = f'[OpenNeuro] Initialize git-annex remote {remote_name}'
+    repo.create_commit(
+        branch.name,  # The ref to update (our branch)
+        author,  # The commit author
+        committer,  # The commit committer
+        commit_message,  # The commit message
+        new_tree_oid,  # The OID of the new tree
+        [commit.id],  # List of parent commit OIDs
+    )
+    return remote_uuid

services/datalad/datalad_service/common/s3.py

@@ -1,7 +1,11 @@
 import os
 import subprocess
 
+import boto3
+from botocore.config import Config
+
 import datalad_service.config
+from datalad_service.common.annex import annex_initremote
 
 
 class S3ConfigException(Exception):
@@ -59,6 +63,13 @@ def backup_remote_env():
     backup_remote_env['AWS_SECRET_ACCESS_KEY'] = backup_remote_env[
         'GCP_SECRET_ACCESS_KEY'
     ]
+    # Overwrite the AWS keys with the GCP keys from config
+    backup_remote_env['AWS_ACCESS_KEY_ID'] = getattr(
+        datalad_service.config, 'GCP_ACCESS_KEY_ID'
+    )
+    backup_remote_env['AWS_SECRET_ACCESS_KEY'] = getattr(
+        datalad_service.config, 'GCP_SECRET_ACCESS_KEY'
+    )
     return backup_remote_env
 
 
@@ -83,6 +94,43 @@ def setup_s3_backup_sibling(dataset_path):
     )
 
 
+def setup_s3_backup_sibling_workaround(dataset_path):
+    """setup_s3_backup_sibling with workaround for git-annex bug."""
+    dataset_id = os.path.basename(dataset_path)
+    uuid = annex_initremote(
+        dataset_path,
+        get_s3_backup_remote(),
+        generate_s3_annex_options(dataset_path, backup=True),
+    )
+    # Manually upload the remote uuid file to the bucket
+    aws_access_key_id = getattr(datalad_service.config, 'GCP_ACCESS_KEY_ID')
+    aws_secret_access_key = getattr(datalad_service.config, 'GCP_SECRET_ACCESS_KEY')
+    gcp_config = Config(
+        region_name='auto',
+        signature_version='s3v4',
+        # This is required for GCP compatibility with boto3
+        request_checksum_calculation='when_required',
+    )
+    s3 = boto3.client(
+        's3',
+        aws_access_key_id=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key,
+        endpoint_url='https://storage.googleapis.com',
+        config=gcp_config,
+    )
+    s3.put_object(
+        Bucket=get_s3_backup_bucket(),
+        Key=f'{dataset_id}/annex-uuid',
+        Body=uuid.encode('utf-8'),
+    )
+    # Enableremote after
+    subprocess.run(
+        ['git-annex', 'enableremote', get_s3_backup_remote()],
+        cwd=dataset_path,
+        env=backup_remote_env(),
+    )
+
+
 def update_s3_sibling(dataset_path):
     """Update S3 remote with latest config."""
     # note: enableremote command will only upsert config options, none are deleted

services/datalad/datalad_service/tasks/publish.py

@@ -57,7 +57,7 @@ def s3_sibling(dataset_path):
     if not is_git_annex_remote(dataset_path, get_s3_remote()):
         datalad_service.common.s3.setup_s3_sibling(dataset_path)
     if not is_git_annex_remote(dataset_path, get_s3_backup_remote()):
-        datalad_service.common.s3.setup_s3_backup_sibling(dataset_path)
+        datalad_service.common.s3.setup_s3_backup_sibling_workaround(dataset_path)
 
 
 @broker.task