Implement ability to download multiple metadata files in a zip

codytodonnell · codytodonnell · commit d02728a1a210 · 2025-12-19T12:07:22.000-05:00
diff --git a/nmdc_server/api.py b/nmdc_server/api.py
@@ -1,4 +1,5 @@
 import csv
+import io
 import json
 import logging
 import time
@@ -7,6 +8,7 @@
 from io import BytesIO, StringIO
 from typing import Any, Dict, List, Optional, Union, cast
 from uuid import UUID, uuid4
+import zipfile
 
 import httpx
 import requests
@@ -375,17 +377,51 @@ async def get_biosample_source(biosample_id: str):
     tags=["biosample"],
 )
 async def search_biosample_source(
-    biosample_query: query.BiosampleQuerySchema = query.BiosampleQuerySchema(),
+    q: query.SearchQuery = query.SearchQuery(),
     db: Session = Depends(get_db),
 ):
     biosample_search = BiosampleSearch()
-    biosamples = biosample_query.query(db).all()
-    results = biosample_search.get_records_by_id([biosample.id for biosample in biosamples])
+    biosample_ids = crud.search_biosample(db, q.conditions, []).with_entities(models.Biosample.id).all()
+    results = biosample_search.get_records_by_id([id for (id,) in biosample_ids])
     if not results:
         raise HTTPException(status_code=404, detail="Could not retrieve source data for biosamples")
     return results
 
 
+# Download multiple metadata lists as a zip file given a list of endpoint labels.
+# Endpoint labels are mapped to functions that retrieve JSON
+@router.post(
+    "/download_metadata", 
+    tags=["bulk_download"]
+)
+async def download_metadata(
+    q: query.MultiSearchQuery,
+    db: Session = Depends(get_db)
+):
+
+    endpoint_map = {
+        "biosamples": search_biosample_source,
+        "studies": search_study_source,
+    }
+    
+    zip_buffer = io.BytesIO()
+    
+    with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zip_file:
+        for endpoint_name in q.endpoints:
+            if endpoint_name in endpoint_map:
+                data = await endpoint_map[endpoint_name](q, db)
+                json_str = json.dumps(data, indent=2, ensure_ascii=False)
+                zip_file.writestr(f"{endpoint_name}.json", json_str.encode('utf-8'))
+    
+    zip_buffer.seek(0)
+    
+    return Response(
+        content=zip_buffer.getvalue(),
+        media_type="application/zip",
+        headers={"Content-Disposition": "attachment; filename=metadata.zip"}
+    )
+
+
 @router.get(
     "/envo/tree",
     response_model=schemas.EnvoTreeResponse,
@@ -592,6 +628,24 @@ async def get_study_source(study_id: str):
     return source_study
 
 
+# Get a list of study source data via the Runtime API
+# based on supplied conditions
+@router.post(
+    "/study/search/source",
+    tags=["study"],
+)
+async def search_study_source(
+    q: query.SearchQuery = query.SearchQuery(),
+    db: Session = Depends(get_db),
+):
+    study_search = StudySearch()
+    study_ids = crud.search_study(db, q.conditions).with_entities(models.Study.id).all()
+    results = study_search.get_records_by_id([id for (id,) in study_ids])
+    if not results:
+        raise HTTPException(status_code=404, detail="Could not retrieve source data for studies")
+    return results
+
+
 # data_generation
 # Note the intermingling of the terms "data generation" and "omics processing."
 # The Berkeley schema (NMDC schema v11) did away with the phrase "omics processing."
diff --git a/nmdc_server/query.py b/nmdc_server/query.py
@@ -821,6 +821,7 @@ def omics_processing_for_biosample_ids(self, db: Session, biosample_ids):
 
 class BiosampleQuerySchema(BaseQuerySchema):
     data_object_filter: List[DataObjectFilter] = []
+    endpoints: List[str] = []
 
     @property
     def table(self) -> Table:
@@ -1039,6 +1040,10 @@ class SearchQuery(BaseModel):
     conditions: List[ConditionSchema] = []
 
 
+class MultiSearchQuery(SearchQuery):
+    endpoints: List[str] = []
+
+
 class ConditionResultSchema(SimpleConditionSchema):
     model_config = ConfigDict(from_attributes=True)
 
diff --git a/web/src/components/BulkDownload.vue b/web/src/components/BulkDownload.vue
@@ -7,6 +7,8 @@ import DownloadDialog from '@/components/DownloadDialog.vue';
 import useBulkDownload from '@/use/useBulkDownload';
 import { humanFileSize } from '@/data/utils';
 import { api } from '@/data/api';
+import { downloadBlob } from '@/utils';
+// import { downloadJson } from '@/utils';
 
 export default defineComponent({
 
@@ -30,6 +32,7 @@ export default defineComponent({
     const downloadMenuOpen = ref(false);
     const treeMenuOpen = ref(false);
     const tab = ref('one');
+    const metadataDownloadSelected = ref<string[]>([]);
 
     const {
       loading,
@@ -46,7 +49,7 @@ export default defineComponent({
       return `${labelString})`;
     }
 
-    const options = computed(() => {
+    const dataProductOptions = computed(() => {
       if (!downloadOptions.value || typeof downloadOptions.value !== 'object') {
         return [];
       }
@@ -61,6 +64,17 @@ export default defineComponent({
         }));
     });
 
+    const metadataOptions = computed(() => [
+      {
+        id: 'biosamples',
+        label: 'Biosamples',
+      },
+      {
+        id: 'studies',
+        label: 'Studies',
+      },
+    ]);
+
     async function createAndDownload() {
       const val = await download();
       termsDialog.value = false;
@@ -72,19 +86,9 @@ export default defineComponent({
     }
 
     async function downloadSamplesMetadata() {
-      const samples = await api.searchBiosampleSource(stateRefs.conditions.value)
-      console.log(samples);
-      // .then((data) => {
-        //   const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' });
-        //   const url = window.URL.createObjectURL(blob);
-        //   const link = document.createElement('a');
-        //   link.href = url;
-        //   link.setAttribute('download', 'samples_metadata.json');
-        //   document.body.appendChild(link);
-        //   link.click();
-        //   document.body.removeChild(link);
-        //   window.URL.revokeObjectURL(url);
-        // });
+      const endpoints = metadataDownloadSelected.value;
+      const blob = await api.getMetadataZip(stateRefs.conditions.value, endpoints);
+      downloadBlob(blob, 'metadata.zip');
     }
 
     watch(
@@ -98,7 +102,8 @@ export default defineComponent({
 
     return {
       bulkDownloadSelected: stateRefs.bulkDownloadSelected,
-      options,
+      dataProductOptions,
+      metadataOptions,
       loading,
       downloadSummary,
       termsDialog,
@@ -109,6 +114,7 @@ export default defineComponent({
       downloadMenuOpen,
       treeMenuOpen,
       downloadSamplesMetadata,
+      metadataDownloadSelected,
     };
   },
 });
@@ -150,9 +156,46 @@ export default defineComponent({
       >
         <v-tab value="data-products">
           Data Products
+          <v-tooltip
+            location="top"
+            min-width="300px"
+            max-width="300px"
+          >
+            <template #activator="{ props }">
+              <v-icon
+                class="ml-2"
+                size="small"
+                v-bind="props"
+              >
+                mdi-help-circle
+              </v-icon>
+            </template>
+            <span>
+              Choose a group of files to download based on file type
+              from the currently filtered search results.
+            </span>
+          </v-tooltip>
         </v-tab>
         <v-tab value="metadata">
           Metadata
+          <v-tooltip
+            location="top"
+            min-width="300px"
+            max-width="300px"
+          >
+            <template #activator="{ props }">
+              <v-icon
+                class="ml-2"
+                size="small"
+                v-bind="props"
+              >
+                mdi-help-circle
+              </v-icon>
+            </template>
+            <span>
+              Download metadata as JSON for the currently filtered search results.
+            </span>
+          </v-tooltip>
         </v-tab>
       </v-tabs>
       <v-divider />
@@ -165,44 +208,18 @@ export default defineComponent({
             variant="flat"
             class="pa-3 d-flex flex-column"
           >
-            <div class="d-flex flex-row align-center justify-space-between mb-2">
-              <div class="d-flex flex-row align-center mr-3">
-                <div class="pr-2 text-caption font-weight-bold">
-                  Bulk Download
-                </div>
-                <v-tooltip
-                  left
-                  nudge-bottom="20px"
-                  min-width="300px"
-                  max-width="300px"
-                >
-                  <template #activator="{ props }">
-                    <v-icon
-                      size="small"
-                      v-bind="props"
-                    >
-                      mdi-help-circle
-                    </v-icon>
-                  </template>
-                  <span>
-                    Choose a group of files to download based on file type
-                    from the currently filtered search results.
-                  </span>
-                </v-tooltip>
-              </div>
-              <span
-                class="text-caption font-weight-bold white--text"
-              >
-                <template v-if="downloadSummary.count === 0">
-                  No files selected
-                </template>
-                <template v-else>
-                  Download {{ downloadSummary.count }} files
-                  from {{ searchResultCount }} sample search results.
-                  (Download archive size {{ humanFileSize(downloadSummary.size) }})
-                </template>
-              </span>
-            </div>
+            <span
+              class="text-caption font-weight-bold"
+            >
+              <template v-if="downloadSummary.count === 0">
+                No files selected
+              </template>
+              <template v-else>
+                Download {{ downloadSummary.count }} files
+                from {{ searchResultCount }} sample search results.
+                (Download archive size {{ humanFileSize(downloadSummary.size) }})
+              </template>
+            </span>
             <div>
               <div @click.stop>
                 <Treeselect
@@ -212,8 +229,8 @@ export default defineComponent({
                   multiple
                   value-consists-of="LEAF_PRIORITY"
                   open-direction="below"
-                  :options="options"
-                  placeholder="Select file type"
+                  :options="dataProductOptions"
+                  placeholder="Select file types"
                   z-index="2001"
                   @open="treeMenuOpen = true"
                   @close="treeMenuOpen = false"
@@ -233,7 +250,7 @@ export default defineComponent({
                     <v-icon class="pr-3">
                       mdi-download
                     </v-icon>
-                    Download ZIP
+                    Download Data Products ZIP
                   </v-btn>
                 </template>
                 <DownloadDialog
@@ -246,13 +263,29 @@ export default defineComponent({
         </v-tabs-window-item>
         <v-tabs-window-item value="metadata">
           <v-sheet class="pa-5">
+            <div @click.stop>
+              <Treeselect
+                v-model="metadataDownloadSelected"
+                append-to-body
+                class="flex-1-1-0"
+                multiple
+                value-consists-of="LEAF_PRIORITY"
+                open-direction="below"
+                :options="metadataOptions"
+                placeholder="Select metadata types"
+                z-index="2001"
+                @open="treeMenuOpen = true"
+                @close="treeMenuOpen = false"
+              />
+            </div>
             <v-btn
+              class="mt-3"
               @click="downloadSamplesMetadata"
             >
               <v-icon class="pr-3">
                 mdi-download
               </v-icon>
-              Samples JSON
+              Download Metadata Zip
             </v-btn>
           </v-sheet>
         </v-tabs-window-item>
@@ -263,7 +296,7 @@ export default defineComponent({
 
 <style scoped>
 .download-menu {
-  width: 500px;
+  width: 550px;
 }
 
 .vue3-treeselect {
diff --git a/web/src/data/api.ts b/web/src/data/api.ts
@@ -582,6 +582,16 @@ async function searchBiosampleSource(conditions: Condition[]) {
   return data;
 }
 
+async function getMetadataZip(conditions: Condition[], endpoints: string[]) {
+  const { data } = await client.post<any>(
+    `download_metadata`,
+    { conditions, endpoints },
+    { responseType: 'blob' }
+
+  );
+  return data;
+}
+
 async function getStudySource(id: string): Promise<StudyResultFromSource> {
   const { data } = await client.get<StudyResultFromSource>(`study/${id}/source`);
   return data;
@@ -1013,6 +1023,7 @@ const api = {
   getEnvironmentSankeyAggregation,
   getEnvoTrees,
   getFacetSummary,
+  getMetadataZip,
   getStudy,
   getStudySource,
   getSubmissionSchema,
diff --git a/web/src/utils.ts b/web/src/utils.ts
