real_multi_modality

.gitignore

@@ -14,3 +14,15 @@ temp_data
 *.egg*
 __pycache__
 build/
+*.log
+
+#ignore example result
+examples/**/*.h5ad
+examples/tuning/**/*.yaml
+examples/tuning/**/*.csv
+examples/tuning/**/*.sh
+examples/tuning/**/*.h5
+examples/tuning/**/*.tar.gz
+examples/tuning/**/*.tif
+examples/tuning/**/*.txt
+examples/atlas/config/run_config.csv

.pre-commit-config.yaml

@@ -1,50 +1,50 @@
+---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
         exclude: ^docs/
-
+  # - repo: https://github.com/lyz-code/yamlfix  # The tool that actually fixes
+  #   rev: 1.16.0  # <---- UPDATE THIS to the latest stable version
+  #   hooks:
+  #     - id: yamlfix
+  #       name: yamlfix (Auto-Fix)  # Optional: Rename for clarity
+  #       additional_dependencies:
+  #         - maison==1.4.3  # <--- 例子：指定 maison 版本为 0.2.3
   - repo: https://github.com/adrienverge/yamllint.git
     rev: v1.35.1
     hooks:
       - id: yamllint
         args: [-c=.yamllint.yml]
-
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.16.0
     hooks:
       - id: pyupgrade
         args: [--py3-plus]
-
   - repo: https://github.com/google/yapf
     rev: v0.40.2
     hooks:
       - id: yapf
         name: Format code
         additional_dependencies: [toml]
-
   - repo: https://github.com/pycqa/isort
     rev: 5.13.2
     hooks:
       - id: isort
         name: Sort imports
-        args: [--line-width, "120", --profile, black]
-
+        args: [--line-width, '120', --profile, black]
   - repo: https://github.com/PyCQA/docformatter
-    rev: v1.7.5
+    rev: eb1df34
     hooks:
       - id: docformatter
         name: Format docstring
         additional_dependencies: [tomli]
         args: [--config, pyproject.toml]
-
   - repo: https://github.com/executablebooks/mdformat
     rev: 0.7.17
     hooks:
       - id: mdformat
         name: Format markdown
-        additional_dependencies:
-          - mdformat-gfm
-          - mdformat-tables
+        additional_dependencies: [mdformat-gfm, mdformat-tables]

dance/atlas/data_dropbox_upload.py

@@ -0,0 +1,155 @@
+import json
+import os
+import pathlib
+
+import dropbox
+import numpy as np
+import pandas as pd
+import scanpy as sc
+from dropbox.exceptions import ApiError, AuthError
+
+from dance.utils import logger
+
+
+def upload_file_to_dropbox(dropbox_path, access_token, local_path):
+    """Upload a local file to Dropbox.
+
+    Parameters
+    ----------
+    dropbox_path : str
+        Destination path in Dropbox
+    access_token : str
+        Dropbox API access token
+    local_path : str or pathlib.Path
+        Path to local file to upload
+
+    Returns
+    -------
+    None
+        Returns None if upload fails
+
+    """
+    dbx = dropbox.Dropbox(access_token)
+
+    # Verify access token
+    try:
+        dbx.users_get_current_account()
+    except AuthError as err:
+        print("ERROR: Invalid access token; please check your access token.")
+        return None
+    try:
+        file_upload(dbx=dbx, local_path=local_path, remote_path=dropbox_path)
+        print("Upload successful.")
+    except ApiError as err:
+        print(f"API error: {err}")
+        return None
+
+
+def file_upload(dbx: dropbox.Dropbox, local_path: pathlib.Path, remote_path: str):
+    """Upload large files to Dropbox using chunked upload.
+
+    Parameters
+    ----------
+    dbx : dropbox.Dropbox
+        Authenticated Dropbox client
+    local_path : pathlib.Path
+        Path to local file
+    remote_path : str
+        Destination path in Dropbox
+
+    """
+    CHUNKSIZE = 100 * 1024 * 1024
+    upload_session_start_result = dbx.files_upload_session_start(b'')
+    cursor = dropbox.files.UploadSessionCursor(session_id=upload_session_start_result.session_id, offset=0)
+    with local_path.open("rb") as f:
+        while True:
+            data = f.read(CHUNKSIZE)
+            if data == b"":
+                break
+            logger.debug("Pushing %d bytes", len(data))
+            dbx.files_upload_session_append_v2(data, cursor)
+            cursor.offset += len(data)
+    commit = dropbox.files.CommitInfo(path=remote_path)
+    dbx.files_upload_session_finish(b'', cursor, commit)
+
+
+def create_shared_link(dbx, dropbox_path):
+    """Create or get existing shared link.
+
+    :param dbx: Dropbox object
+    :param dropbox_path: File path on Dropbox
+    :return: Shared link URL
+
+    """
+    try:
+        links = dbx.sharing_list_shared_links(path=dropbox_path, direct_only=True).links
+        if links:
+            # If shared link already exists, return the first one
+            return links[0].url
+        else:
+            # Create a new shared link
+            link = dbx.sharing_create_shared_link_with_settings(dropbox_path)
+            return link.url
+    except ApiError as err:
+        print(f"Error creating shared link: {err}")
+        return None
+
+
+def get_link(data_fname, local_path, ACCESS_TOKEN, DROPBOX_DEST_PATH):
+    DROPBOX_DEST_PATH = DROPBOX_DEST_PATH + "/" + data_fname
+
+    upload_file_to_dropbox(dropbox_path=DROPBOX_DEST_PATH, access_token=ACCESS_TOKEN, local_path=local_path)
+
+    # Create Dropbox object to get shared link
+    dbx = dropbox.Dropbox(ACCESS_TOKEN)
+    # Get shared link
+    shared_link = create_shared_link(dbx, DROPBOX_DEST_PATH)
+    if shared_link:
+        # Dropbox shared link defaults to `dl=0` at the end, which means preview in browser.
+        # change it to `dl=1`.
+        download_link = shared_link.replace('&dl=0', '&dl=1')
+        print(f"Download link: {download_link}")
+        return download_link
+    else:
+        print("Unable to get shared link.")
+
+
+def get_ans(data: sc.AnnData, tissue: str, dataset_id: str, local_path, ACCESS_TOKEN, DROPBOX_DEST_PATH):
+    """Generate metadata dictionary for dataset and upload to Dropbox.
+
+    Parameters
+    ----------
+    data : sc.AnnData
+        Annotated data matrix
+    tissue : str
+        Tissue type
+    dataset_id : str
+        Unique identifier for dataset
+    local_path : str or pathlib.Path
+        Path to local data file
+    ACCESS_TOKEN : str
+        Dropbox API access token
+    DROPBOX_DEST_PATH : str
+        Base path in Dropbox for uploads
+
+    Returns
+    -------
+    dict
+        Metadata dictionary containing dataset information and Dropbox URLs
+
+    """
+    # keys=["species","tissue","dataset","split","celltype_fname","celltype_url","data_fname","data_url"]
+    # Create metadata dictionary with dataset info
+    ans = {}
+    ans["species"] = "human"
+    ans["tissue"] = tissue.capitalize()
+    # Store number of observations (cells) in dataset
+    ans["dataset"] = data.n_obs
+    ans["split"] = "train"
+    ans["celltype_fname"] = ""
+    ans["celltype_url"] = ""
+    ans["data_fname"] = f"train_human_{tissue.capitalize()}{dataset_id}_data.h5ad"
+    ans["data_url"] = get_link(data_fname=ans["data_fname"].split("_", 1)[1], local_path=local_path,
+                               ACCESS_TOKEN=ACCESS_TOKEN, DROPBOX_DEST_PATH=DROPBOX_DEST_PATH)
+    ans["is_ALL_Integer"] = np.all(np.equal(data.X.data, data.X.data.astype(int)))
+    return ans