merge zarr features into sebastien

Author: gtramonte

.github/workflows/build_on_release.yaml

@@ -39,7 +39,6 @@ jobs:
         push: true
         build-args: |
           REGISTRY=${{ vars.GEOKUBE_REGISTRY  }}
-          TAG=v0.2a6
         cache-from: type=gha
         cache-to: type=gha,mode=max
         tags: | 

.gitignore

@@ -113,6 +113,9 @@ _catalogs/
 _old/
 
 .DS_Store
-db_init
 *.zarr
-*.nc
+*.nc
+db_init
+catalog
+docker-compose.yaml
+nginx.conf

README.md

@@ -1,5 +1,3 @@
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10598417.svg)](https://doi.org/10.5281/zenodo.10598417)
-
 # geolake
 
 ## Description
@@ -12,13 +10,14 @@ The system has been designed using a cloud-native architecture, based on contain
 
 It uses [geokube](https://github.com/CMCC-Foundation/geokube) as an Analytics Engine to perform geospatial operations.
 
-## Authors
+## Developer Team
+
+- [Valentina Scardigno](https://github.com/vale95-eng)
+- [Gabriele Tramonte](https://github.com/gtramonte)
 
-**Project Lead**:
-[Marco Mancini](https://github.com/km4rcus)
+### Former Developers
 
-**Main Developers**
+- [Marco Mancini](https://github.com/km4rcus)
 - [Jakub Walczak](https://github.com/jamesWalczak)
 - [Mirko Stojiljkovic](https://github.com/MMStojiljkovic)
-- [Valentina Scardigno](https://github.com/vale95-eng)
 

api/Dockerfile

@@ -6,7 +6,7 @@ RUN apt update && apt install -y cron curl
 
 WORKDIR /app
 COPY requirements.txt /code/requirements.txt
-RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt --break-system-packages
 COPY app /app
 EXPOSE 80
 

api/app/endpoint_handlers/file.py

@@ -1,18 +1,26 @@
 """Module with functions to handle file related endpoints"""
+import io
 import os
+import zipfile
+from pathlib import Path
+from zipfile import ZipFile
 
 from fastapi.responses import FileResponse
+
+
 from dbmanager.dbmanager import DBManager, RequestStatus
+from starlette.requests import Request
+from starlette.responses import HTMLResponse, RedirectResponse, StreamingResponse
+from starlette.staticfiles import StaticFiles
 
 from utils.api_logging import get_dds_logger
 from utils.metrics import log_execution_time
 import exceptions as exc
 
 log = get_dds_logger(__name__)
 
-
 @log_execution_time(log)
-def download_request_result(request_id: int):
+def download_request_result(request_id: int, filename: str = None):
     """Realize the logic for the endpoint:
 
     `GET /download/{request_id}`
@@ -60,7 +68,15 @@ def download_request_result(request_id: int):
             download_details.location_path,
         )
         raise FileNotFoundError
-    return FileResponse(
-        path=download_details.location_path,
-        filename=download_details.location_path.split(os.sep)[-1],
-    )
+
+    if download_details.location_path.endswith(".zarr"):
+        log.info("Zarr detected")
+        return FileResponse(
+            path=f'{download_details.location_path}/{filename}',
+            filename=filename,
+        )
+    else:
+        return FileResponse(
+            path=download_details.location_path,
+            filename=download_details.location_path.split(os.sep)[-1],
+        )

api/app/main.py

@@ -679,4 +679,52 @@ async def dcatapit(request: Request):
     # response = dcatapit_graph.serialize(format='pretty-xml')
     response = serialize_and_concatenate_graphs(catalog_graph, datasets_graph, distributions_graph, vcard_graph)
 
-    return Response(content=response, media_type="application/rdf+xml")
+    return Response(content=response, media_type="application/rdf+xml")
+@app.get("/download/{request_id}/{filename}", tags=[tags.REQUEST])
+@timer(
+    app.state.api_request_duration_seconds,
+    labels={"route": "GET /download/{request_id}/{filename}"},
+)
+# @requires([scopes.AUTHENTICATED]) # TODO: mange download auth in the web component
+async def download_request_result(
+    request: Request,
+    request_id: int,
+    filename: str,
+):
+    """Download result of the request"""
+    app.state.api_http_requests_total.inc(
+        {"route": "GET /download/{request_id}/{filename}"}
+    )
+    try:
+        return file_handler.download_request_result(request_id=request_id, filename=filename)
+    except exc.BaseDDSException as err:
+        raise err.wrap_around_http_exception() from err
+    except FileNotFoundError as err:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail="File was not found!"
+        ) from err
+
+@app.get("/download/{request_id}/{filename}/{subfile}", tags=[tags.REQUEST])
+@timer(
+    app.state.api_request_duration_seconds,
+    labels={"route": "GET /download/{request_id}/{filename}/{subfile}"},
+)
+# @requires([scopes.AUTHENTICATED])
+async def download_request_result(
+    request: Request,
+    request_id: int,
+    filename: str,
+    subfile: str,
+):
+    """Download result of the request"""
+    app.state.api_http_requests_total.inc(
+        {"route": "GET /download/{request_id}/{filename}/{subfile}"}
+    )
+    try:
+        return file_handler.download_request_result(request_id=request_id, filename=f'{filename}/{subfile}')
+    except exc.BaseDDSException as err:
+        raise err.wrap_around_http_exception() from err
+    except FileNotFoundError as err:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail="File was not found!"
+        ) from err