upload data to kaggle and hugging face

Author: Dpbm

airflow-setup-test.sh

@@ -19,6 +19,10 @@ export DB_PASSWORD="airflowpass"
 
 export KAGGLE_USERNAME=$(clear_quotation_marks $(cat $KAGGLE_KEY_FILE | jq .username))
 export KAGGLE_KEY=$(clear_quotation_marks $(cat $KAGGLE_KEY_FILE | jq .key))
+export KAGGLE_DATASET="dpbmanalysis/quantum-circuit-images"
+
+# HF_TOKEN comes from system
+export HF_DATASET="Dpbm/quantum-circuits"
 
 # fix permission for volume
 sudo rm -rf ./data

airflow.Dockerfile

@@ -8,9 +8,7 @@ RUN ${PIPENV} install -r requirements.txt
 
 
 FROM debian:bookworm-slim AS entry
-
 RUN apt update && apt install zip make -y
-
 WORKDIR /
 COPY airflow-entrypoint.sh entrypoint.sh
 RUN chmod +x entrypoint.sh
@@ -24,7 +22,6 @@ COPY . .
 
 WORKDIR /home/airflow/.local/bin
 COPY --from=entry --chown=airflow:root /usr/bin/zip zip
-COPY --from=entry --chown=airflow:root /usr/bin/make make
 
 WORKDIR /
 COPY --from=entry /entrypoint.sh .

airflow.Dockerfile.dockerignore

@@ -4,6 +4,5 @@
 !generate/
 !utils/
 !args/
-!Makefile
 !requirements.txt
 !airflow-entrypoint.sh

compose.yml

@@ -38,6 +38,11 @@ services:
       - USER=${AIRFLOW_USERNAME}
       - PASSWORD=${AIRFLOW_PASSWORD}
       - EMAIL=${AIRFLOW_EMAIL}
+      - KAGGLE_USERNAME=${KAGGLE_USERNAME}
+      - KAGGLE_KEY=${KAGGLE_KEY}
+      - KAGGLE_DATASET=${KAGGLE_DATASET}
+      - HF_TOKEN=${HF_TOKEN}
+      - HF_DATASET=${HF_DATASET}
       - PYTHONPATH=/home/airflow/project
       - TARGET_FOLDER=/home/airflow/data
       - TZ=America/Sao_Paulo

dags/dataset.py

@@ -24,6 +24,8 @@
     DEFAULT_THREADS,
 )
 from generate.ghz import gen_circuit
+from export.kaggle import upload_dataset as upload_dataset_kaggle
+from export.huggingface import upload_dataset as upload_dataset_huggingface
 
 default_args = {
     "depends_on_past": True,
@@ -92,7 +94,7 @@
     with resized and normalized images.
     """
 
-    command = f"zip -r {folder}/dataset-images.zip {folder}/dataset/"
+    command = f"cd {folder} && zip -r dataset-images.zip dataset/"
     pack_img = BashOperator(task_id="pack_images", bash_command=command)
 
     pack_img.doc_md = """
@@ -118,10 +120,31 @@
     Run training after finishing all processes.
     """
 
+    kaggle_dataset = os.environ.get("KAGGLE_DATASET")
+    send_kaggle = PythonOperator(
+        task_id="send_kaggle",
+        python_callable=upload_dataset_kaggle,
+        op_args=[kaggle_dataset,folder]
+    )
+    
+    hf_dataset = os.environ.get("HF_DATASET")
+    send_hf = PythonOperator(
+        task_id="send_huggingface",
+        python_callable=upload_dataset_huggingface,
+        op_args=[hf_dataset,folder]
+    )
+
+    send_hf.doc_md = """
+    Send dataset files to huggingface
+    """
+
     create_folder >> [gen_ghz, gen_df]
     gen_df >> gen_images
     gen_images >> remove_duplicates
     remove_duplicates >> transform_img
     transform_img >> pack_img
 
     [gen_ghz, pack_img] >> trigger_dag_train
+    [gen_ghz, pack_img] >> send_kaggle
+    [gen_ghz, pack_img] >> send_hf
+

export/huggingface.py

@@ -0,0 +1,14 @@
+"""Export dataset and model to huggingface"""
+
+import os
+
+from huggingface_hub import HfApi
+
+def upload_dataset(dataset_name:str, folder:str):
+    api = HfApi(token=os.getenv("HF_TOKEN"))
+    api.upload_folder(
+        folder_path=folder,
+        repo_id=dataset_name,
+        repo_type="dataset",
+        ignore_patterns=["dataset/"]
+    )

export/kaggle.py

@@ -1,3 +1,18 @@
 """Export dataset and model to kaggle"""
 
-# import kagglehub as kh
+from time import ctime
+
+import kagglehub as kh
+
+
+def upload_dataset(dataset_name:str, folder:str):
+    """
+    Upload dataset files to kaggle
+    """
+
+    version = ctime()
+    kh.dataset_upload(
+        dataset_name, 
+        folder, 
+        version_notes=version,
+        ignore_patterns=["dataset/"])

generate/dataset.py

@@ -175,7 +175,7 @@ def generate_images(
                     df.vstack(tmp_df, in_place=True)
 
             progress.update(total_threads)
-    save_df(df, dataset_file_path)
+            save_df(df, dataset_file_path)
 
 
 def remove_duplicated_files(target_folder: FilePath):