merged in main

Author: pgm

arch.md

@@ -104,3 +104,85 @@ Taiga is deployed as a containerized application:
 1. **Authentication**: Token-based authentication
 2. **Authorization**: Group-based access control (This is not really fully implemented. If there is code about group-based access, I am willing to bet it is half-implement if at all)
 3. **Secure Storage**: Taiga will sign URLs for fetching data from S3 and give them to the client, so that the transfer of large data happens directly between client and S3.
+
+## Deployment Diagram
+
+```mermaid
+graph TD
+    subgraph "Client"
+        Browser[Web Browser]
+    end
+
+    subgraph "Application Server"
+        Flask[Flask API]
+        Celery[Celery Workers]
+        Redis[Redis Queue]
+    end
+
+    subgraph "Storage"
+        Postgres[PostgreSQL]
+        S3[AWS S3]
+        GCS[Google Cloud Storage]
+    end
+
+    subgraph "External Services"
+        Figshare[Figshare]
+        Email[Email Service]
+    end
+
+    Browser -->|HTTP/HTTPS| Flask
+    Flask -->|Query/Update| Postgres
+    Flask -->|Enqueue Tasks| Redis
+    Redis -->|Process Tasks| Celery
+    Celery -->|File Operations| S3
+    Celery -->|File Operations| GCS
+    Celery -->|Publish| Figshare
+    Celery -->|Send Notifications| Email
+    Flask -->|Generate Signed URLs| S3
+    Flask -->|Generate Signed URLs| GCS
+```
+
+## Service Architecture
+
+```mermaid
+flowchart LR
+    subgraph "Frontend"
+        React[React App]
+        TypeScript[TypeScript Models]
+    end
+
+    subgraph "Backend API"
+        Endpoints[API Endpoints]
+        Controllers[Controllers]
+        Models[Data Models]
+        Auth[Authentication]
+    end
+
+    subgraph "Background Processing"
+        TaskQueue[Celery Queue]
+        Workers[Celery Workers]
+        Conversion[File Conversion]
+        Import[Data Import]
+        Export[Data Export]
+    end
+
+    subgraph "Storage"
+        MetadataDB[PostgreSQL]
+        FileStorage[S3/GCS]
+    end
+
+    React -->|API Calls| Endpoints
+    TypeScript -->|Data Types| React
+    Endpoints -->|Process Requests| Controllers
+    Controllers -->|CRUD Operations| Models
+    Models -->|Persist| MetadataDB
+    Controllers -->|Enqueue Tasks| TaskQueue
+    TaskQueue -->|Process| Workers
+    Workers -->|Execute| Conversion
+    Workers -->|Execute| Import
+    Workers -->|Execute| Export
+    Conversion -->|Read/Write| FileStorage
+    Import -->|Write| FileStorage
+    Export -->|Read| FileStorage
+    Auth -->|Secure| Endpoints
+```

docker-base/requirements.txt

@@ -5,6 +5,7 @@ celery==4.1.1
 certifi==2019.6.16
 Click==7.0
 clickclick==1.2.2
+openapi-spec-validator==0.2.9
 connexion[swagger-ui]==2.3.0
 docutils==0.14
 Flask==1.0.3
@@ -16,7 +17,7 @@ google-cloud-storage==1.35.0
 h5py==2.6.0
 humanize==0.5.1
 jsonpointer==2.0
-jsonschema==2.6.0
+jsonschema==2.7.0
 kombu==4.2.1
 mailjet_rest==1.3.3
 marshmallow==2.15.1

migrations/versions/c1be0bfabe2e_.py

@@ -0,0 +1,50 @@
+"""empty message
+
+Revision ID: c1be0bfabe2e
+Revises: a286182fdf59
+Create Date: 2023-03-06 11:16:22.434754
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "c1be0bfabe2e"
+down_revision = "a286182fdf59"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("tmp_datafiles_cleanup")
+    op.add_column("datafiles", sa.Column("custom_metadata", sa.JSON(), nullable=True))
+    op.add_column(
+        "upload_session_files", sa.Column("custom_metadata", sa.JSON(), nullable=True)
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("upload_session_files", "custom_metadata")
+    op.drop_column("datafiles", "custom_metadata")
+    op.create_table(
+        "tmp_datafiles_cleanup",
+        sa.Column("id", sa.VARCHAR(length=80), autoincrement=False, nullable=True),
+        sa.Column(
+            "orig_underlying_data_file_id",
+            sa.VARCHAR(length=80),
+            autoincrement=False,
+            nullable=True,
+        ),
+        sa.Column(
+            "new_underlying_data_file_id",
+            sa.VARCHAR(length=80),
+            autoincrement=False,
+            nullable=True,
+        ),
+        sa.Column("update_count", sa.INTEGER(), autoincrement=False, nullable=True),
+    )
+    # ### end Alembic commands ###

taiga2/controllers/endpoint.py

@@ -479,6 +479,11 @@ def de_delete_dataset_version(datasetVersionId):
 @validate
 def create_upload_session_file(uploadMetadata, sid):
     filename = uploadMetadata["filename"]
+
+    # Optional per file metadata, stored as a json encoded string.
+    # Not queryable independently from files.
+    custom_metadata = uploadMetadata.get("custom_metadata")
+
     if uploadMetadata["filetype"] == "s3":
         S3UploadedFileMetadata = uploadMetadata["s3Upload"]
         s3_bucket = S3UploadedFileMetadata["bucket"]
@@ -492,6 +497,7 @@ def create_upload_session_file(uploadMetadata, sid):
         upload_session_file = models_controller.add_upload_session_s3_file(
             session_id=sid,
             filename=filename,
+            custom_metadata=custom_metadata,
             initial_file_type=initial_file_type,
             initial_s3_key=initial_s3_key,
             s3_bucket=s3_bucket,
@@ -514,11 +520,12 @@ def create_upload_session_file(uploadMetadata, sid):
         return flask.jsonify(task.id)
     elif uploadMetadata["filetype"] == "virtual":
         existing_taiga_id = uploadMetadata["existingTaigaId"]
-
         try:
-            data_file = models_controller.get_datafile_by_taiga_id(
+            data_file_by_taiga_id = models_controller.get_datafile_by_taiga_id(
                 existing_taiga_id, one_or_none=True
             )
+            data_file = models_controller.get_underlying_file(data_file_by_taiga_id)
+
         except InvalidTaigaIdFormat as ex:
             api_error(
                 "The following was not formatted like a valid taiga ID: {}".format(
@@ -529,8 +536,20 @@ def create_upload_session_file(uploadMetadata, sid):
         if data_file is None:
             api_error("Unknown taiga ID: " + existing_taiga_id)
 
+        if custom_metadata == None:
+            custom_metadata = data_file_by_taiga_id.custom_metadata
+        else:
+            custom_metadata = (
+                custom_metadata
+                if data_file_by_taiga_id.custom_metadata == None
+                else {**data_file_by_taiga_id.custom_metadata, **custom_metadata}
+            )
+
         models_controller.add_upload_session_virtual_file(
-            session_id=sid, filename=filename, data_file_id=data_file.id
+            session_id=sid,
+            filename=filename,
+            custom_metadata=custom_metadata,
+            data_file_id=data_file.id,
         )
 
         return flask.jsonify("done")
@@ -554,6 +573,7 @@ def create_upload_session_file(uploadMetadata, sid):
         models_controller.add_upload_session_gcs_file(
             session_id=sid,
             filename=filename,
+            custom_metadata=custom_metadata,
             gcs_path=gcs_path,
             generation_id=str(generation_id),
         )
@@ -1078,7 +1098,7 @@ def copy_datafile_to_google_bucket(datafileGCSCopy):
     datafile_id = datafileGCSCopy["datafile_id"]
     gcs_path = datafileGCSCopy["gcs_path"]
 
-    datafile = models_controller.get_datafile_by_taiga_id(datafile_id)
+    datafile = models_controller.get_underlying_datafile_by_taiga_id(datafile_id)
     if datafile is None:
         raise flask.abort(404)