Quick implementation of DIS features (#247)

* Implemented new paths, mocks, etc. for the DIS data
Fixed mocks not working with newest models.

* Applied most suggestions

* Use a map to merge the file info

---------

Co-authored-by: Christoph Zwerschke <[email protected]>

Author: ac-jorellanaf

.devcontainer/.data_portal_ui.yaml

@@ -3,6 +3,7 @@ host: 0.0.0.0
 port: 8080
 mass_url: "api/mass"
 metldata_url: "api/metldata"
+dis_url: "api/dins"
 ars_url: "api/ars"
 auth_url: "api/auth"
 wps_url: "api/wps"

configure_build_serve/run.py

@@ -52,6 +52,7 @@ class Config(BaseSettings):
     ribbon_text: str = "Development v$v"
     mass_url: str = "/api/mass"
     metldata_url: str = "/api/metldata"
+    dis_url: str = "/api/dins"
     ars_url: str = "/api/ars"
     auth_url: str = "/api/auth"
     wps_url: str = "/api/wps"

src/api/browse.ts

@@ -20,8 +20,9 @@ import {
   MetadataSummaryModel,
 } from "../models/dataset";
 import { FacetFilterModel } from "../models/facets";
-import { MASS_URL, METLDATA_URL, fetchJson } from "../utils/utils";
+import { DIS_URL, MASS_URL, METLDATA_URL, fetchJson } from "../utils/utils";
 import { showMessage } from "../components/messages/usage";
+import { DatasetInformationFileSummaryModel } from "../models/files";
 
 const showFetchDataError = () => {
   showMessage({
@@ -30,15 +31,6 @@ const showFetchDataError = () => {
   });
 };
 
-type getDatasetsSearchRespType = (
-  callbackFunc: (hits: SearchResponseModel) => void,
-  filterQuery: FacetFilterModel[],
-  searchKeyword: string,
-  skip: number,
-  limit: number,
-  documentType: string
-) => void;
-
 /**
  * Async function to retrieve the search results from API, calls the callbackFunc
  * function with search results, returns nothing.
@@ -50,7 +42,7 @@ type getDatasetsSearchRespType = (
  * @param documentType - String representing the document type to search. Default: "Dataset".
  * @returns Nothing
  */
-export const querySearchService: getDatasetsSearchRespType = async (
+export const querySearchService = async (
   callbackFunc: (hits: SearchResponseModel) => void,
   filterQuery: FacetFilterModel[],
   searchKeyword = "",
@@ -87,21 +79,16 @@ export const querySearchService: getDatasetsSearchRespType = async (
   }
 };
 
-type getDatasetDetailsType = (
-  datasetAccession: string,
-  callbackFunc: (dataset: DatasetEmbeddedModel) => void
-) => void;
-
 /**
  * Async function to retrieve the details of specified dataset from API,
  * calls the callbackFunc function with the response data, returns nothing.
  * @param datasetAccession - ID of the dataset of interest.
  * @param callbackFunc - Function used to process response data.
  * @returns Nothing
  */
-export const getDatasetDetails: getDatasetDetailsType = async (
-  datasetAccession,
-  callbackFunc
+export const getDatasetDetails = async (
+  datasetAccession: string,
+  callbackFunc: (dataset: DatasetEmbeddedModel) => void
 ) => {
   let url = new URL(
     `artifacts/embedded_public/classes/EmbeddedDataset/resources/${datasetAccession}`,
@@ -117,21 +104,16 @@ export const getDatasetDetails: getDatasetDetailsType = async (
   }
 };
 
-type getDatasetSummaryType = (
-  datasetAccession: string,
-  callbackFunc: (dataset: DatasetDetailsSummaryModel) => void
-) => void;
-
 /**
  * Async function to retrieve the summary of specified dataset from API,
  * calls the callbackFunc function with the response data, returns nothing.
  * @param datasetAccession - ID of the dataset of interest.
  * @param callbackFunc - Function used to process response data.
  * @returns Nothing
  */
-export const getDatasetSummary: getDatasetSummaryType = async (
-  datasetAccession,
-  callbackFunc
+export const getDatasetSummary = async (
+  datasetAccession: string,
+  callbackFunc: (dataset: DatasetDetailsSummaryModel) => void
 ) => {
   const url = new URL(
     `artifacts/stats_public/classes/DatasetStats/resources/${datasetAccession}`,
@@ -147,6 +129,28 @@ export const getDatasetSummary: getDatasetSummaryType = async (
   }
 };
 
+/**
+ * Async function to retrieve the summary of specified dataset's files from API,
+ * calls the callbackFunc function with the response data, returns nothing.
+ * @param datasetAccession - ID of the dataset of interest.
+ * @param callbackFunc - Function used to process response data.
+ * @returns Nothing
+ */
+export const getDatasetFiles = async (
+  datasetAccession: string,
+  callbackFunc: (dataset: DatasetInformationFileSummaryModel) => void
+) => {
+  const url = new URL(`dataset_information/${datasetAccession}`, DIS_URL);
+  try {
+    const response = await fetchJson(url);
+    const data = await response.json();
+    callbackFunc(data);
+  } catch (error) {
+    showFetchDataError();
+    console.error(error);
+  }
+};
+
 type getMetadataSummaryType = (
   callbackFunc: (summary: MetadataSummaryModel) => void
 ) => void;

src/components/browse/singleDatasetView/singleDatasetView.tsx

@@ -21,13 +21,14 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { useEffect, useState } from "react";
 import { Button, Col, Row, Spinner } from "react-bootstrap";
 import { useLocation, useNavigate, useParams } from "react-router-dom";
-import { getDatasetDetails } from "../../../api/browse";
+import { getDatasetDetails, getDatasetFiles } from "../../../api/browse";
 import { DatasetEmbeddedModel } from "../../../models/dataset";
 import DataRequestFormModal from "../../../utils/dataRequestFormModal";
 import SingleDatasetViewAccordion from "./singleDatasetViewAccordion/singleDatasetViewAccordion";
 import SingleDatasetViewSummary from "./singleDatasetViewSummary/singleDatasetViewSummary";
 import SingleDatasetViewTabs from "./singleDatasetViewTabs/singleDatasetViewTabs";
 import RequestAccessButton from "../../../utils/requestAccessButton";
+import { DatasetInformationFileSummaryModel } from "../../../models/files";
 
 /** Single dataset details page */
 const SingleDatasetView = () => {
@@ -44,12 +45,16 @@ const SingleDatasetView = () => {
   const [details, setDetails] = useState<
     DatasetEmbeddedModel | null | undefined
   >(undefined);
+  const [datasetFiles, setDatasetFiles] = useState<
+    DatasetInformationFileSummaryModel | null | undefined
+  >(undefined);
 
   useEffect(() => {
     const getDetails = (datasetAccession: string) => {
       if (!queried) {
         setQueried(true);
         getDatasetDetails(datasetAccession, setDetails);
+        getDatasetFiles(datasetAccession, setDatasetFiles);
       }
     };
     if (!queried) {
@@ -119,7 +124,17 @@ const SingleDatasetView = () => {
           />
           <SingleDatasetViewSummary details={details} />
           <SingleDatasetViewTabs details={details} />
-          <SingleDatasetViewAccordion details={details} />
+          <SingleDatasetViewAccordion
+            details={details}
+            files={
+              datasetFiles
+                ? datasetFiles
+                : {
+                    accession: "",
+                    file_information: [],
+                  }
+            }
+          />
         </>
       )}
     </div>

src/components/browse/singleDatasetView/singleDatasetViewAccordion/singleDatasetViewAccordion.tsx

@@ -20,9 +20,11 @@ import { FilesTable } from "./tables/filesTable";
 import { SamplesTable } from "./tables/samplesTable";
 import SortButton, { SDSVTableDefinition } from "../../../../utils/sortButton";
 import { parseBytes } from "../../../../utils/utils";
+import { DatasetInformationFileSummaryModel } from "../../../../models/files";
 
 interface SingleDatasetViewAccordionProps {
   details: DatasetEmbeddedModel;
+  files: DatasetInformationFileSummaryModel;
 }
 
 /** Section at the end of dataset details page consisting of three collapsible summary tables. */
@@ -33,7 +35,18 @@ const SingleDatasetViewAccordion = (props: SingleDatasetViewAccordionProps) => {
       const files = (props.details as any)[key];
       const file_category =
         key.charAt(0).toUpperCase() + key.slice(1, -1).replaceAll("_", " ");
-      return files.map((file: any) => ({ ...file, file_category }));
+      const fileInfoMap = new Map();
+      props.files.file_information.forEach((fileInfo) =>
+        fileInfoMap.set(fileInfo.accession, fileInfo)
+      );
+      return files.map((file: any) => {
+        const fileInfo = fileInfoMap.get(file.accession) || {
+          size: 0,
+          sha256_hash: "",
+          storage_alias: "",
+        };
+        return { ...file, ...fileInfo, file_category };
+      });
     });
 
   let Tables: SDSVTableDefinition[] = [

src/components/browse/singleDatasetView/singleDatasetViewAccordion/tables/filesTable.ts

@@ -14,7 +14,7 @@
 // limitations under the License.
 
 import { useState } from "react";
-import { transposeTableForHTML } from "../../../../../utils/utils";
+import { parseBytes, transposeTableForHTML } from "../../../../../utils/utils";
 import {
   SDSVTableDefinition,
   TableFields,
@@ -67,16 +67,37 @@ export const FilesTable = (props: FilesTableProps) => {
       data: allFiles.map((x) => x.file_category),
       cssClasses: "",
     },
+    {
+      header: "File Size",
+      data: allFiles.map((x) => parseBytes(x.size)),
+      cssClasses: "",
+    },
+    {
+      header: "File Location",
+      data: allFiles.map((x) => x.storage_alias),
+      cssClasses: "",
+    },
+    {
+      header: "File Hash",
+      data: allFiles.map((x) => x.sha256_hash),
+      cssClasses: "",
+    },
   ];
 
   const [sortedData, setSortedData] = useState<any>(
     transposeTableForHTML(filesTable.map((x) => x.data))
   );
 
+  var totalSize = allFiles.reduce((a, x) => a + x.size, 0);
+
   const filesTableDef: SDSVTableDefinition = {
     table: filesTable,
     buttonText: allFiles
-      ? "File Summary (" + allFiles.length + " files)"
+      ? "File Summary (" +
+        allFiles.length +
+        " files: " +
+        parseBytes(totalSize) +
+        ")"
       : "File Summary (0 files)",
     sortDefinition: sortDefinition,
     setSortDefinition: setSortDefinition,

src/mocks/data.js

@@ -376,6 +376,7 @@ export const searchResults = {
   count: 25, // just to test the paginator
   hits: [
     {
+      id_: "GHGAD588887987",
       content: {
         accession: "GHGAD588887987",
         ega_accession: "EGAD588887987",
@@ -386,6 +387,7 @@ export const searchResults = {
       },
     },
     {
+      id_: "GHGAD588887988",
       content: {
         accession: "GHGAD588887988",
         ega_accession: "EGAD588887988",
@@ -396,6 +398,7 @@ export const searchResults = {
       },
     },
     {
+      id_: "GHGAD588887989",
       content: {
         accession: "GHGAD588887989",
         ega_accession: "EGAD588887989",
@@ -407,3 +410,13 @@ export const searchResults = {
     },
   ],
 };
+
+export const filesSummary = {
+  accession: "GHGAD588887988",
+  file_information: [{
+    accession: "GHGASF956121331",
+    sha256_hash: "58b2e5a09936322db4ab1b921847d0f3b83027e0255cd24d7e58c420845286dc",
+    size: 123456789,
+    storage_alias: "Tübingen"
+  }]
+}

src/mocks/responses.js

@@ -22,6 +22,7 @@ import {
   workPackageToken,
   searchResults,
   allIVAs,
+  filesSummary,
 } from "./data";
 
 // Responses to be returned for various endpoints.
@@ -119,6 +120,9 @@ export const responses = {
   // Get summary data from entire metadata database
   "GET /api/metldata/stats": metadataSummary,
 
+  // Get summary data from all files in a dataset
+  "GET /api/dins/dataset_information/*": filesSummary,
+
   // Metadata Search Service
   "GET /api/mass/search*": {
     facets: searchResults.facets,

src/models/files.ts

@@ -0,0 +1,23 @@
+// Copyright 2021 - 2024 Universität Tübingen, DKFZ and EMBL
+// for the German Human Genome-Phenome Archive (GHGA)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+export interface DatasetInformationFileSummaryModel {
+  accession: string;
+  file_information: {
+    accession: string;
+    sha256_hash: string;
+    size: number;
+    storage_alias: string;
+  }[];
+}

src/utils/utils.ts

@@ -39,6 +39,7 @@ export const AUTH_URL = getConfiguredUrl("AUTH_URL");
 export const ARS_URL = getConfiguredUrl("ARS_URL");
 export const MASS_URL = getConfiguredUrl("MASS_URL");
 export const METLDATA_URL = getConfiguredUrl("METLDATA_URL");
+export const DIS_URL = getConfiguredUrl("DIS_URL");
 export const WPS_URL = getConfiguredUrl("WPS_URL");
 
 export const OIDC_AUTHORITY_URL = getConfiguredUrl("OIDC_AUTHORITY_URL");