Skip to content

Commit 5eef173

Browse files
gtramontegtramonte
authored
Merge pull request #35 from CMCC-Foundation/feature_34
 Feature 34 Hotfix
2 parents be914c5 + 9049c51 commit 5eef173

File tree

6 files changed

+43
-24
lines changed

6 files changed

+43
-24
lines changed

‎.github/workflows/build_on_pull_request.yaml‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,58 +20,68 @@ jobs:
2020
build
2121
--user
2222
- name: Build a binary wheel and a source for drivers
23-
run: python3 -m build ./drivers
23+
run: python3 -m build ./drivers
2424
- name: Set Docker image tag name
2525
run: echo "TAG=$(date +'%Y.%m.%d.%H.%M')" >> $GITHUB_ENV
26+
- name: TAG ECHO
27+
run: echo ${{ env.TAG }}
2628
- name: Login to Scaleway Container Registry
2729
uses: docker/login-action@v2
2830
with:
2931
username: nologin
3032
password: ${{ secrets.DOCKER_PASSWORD }}
3133
registry: ${{ vars.DOCKER_REGISTRY }}
3234
- name: Set up Docker Buildx
33-
uses: docker/setup-buildx-action@v2
35+
uses: docker/setup-buildx-action@v2
3436
- name: Build and push drivers
35-
uses: docker/build-push-action@v4
37+
uses: docker/build-push-action@v5
3638
with:
3739
context: ./drivers
3840
file: ./drivers/Dockerfile
3941
push: true
42+
cache-from: type=gha
43+
cache-to: type=gha,mode=max
4044
build-args: |
4145
REGISTRY=${{ vars.GEOKUBE_REGISTRY }}
4246
tags: |
4347
${{ vars.DOCKER_REGISTRY }}/geolake-drivers:${{ env.TAG }}
44-
${{ vars.DOCKER_REGISTRY }}/geolake-drivers:latest
48+
${{ vars.DOCKER_REGISTRY }}/geolake-drivers:latest
4549
- name: Build and push datastore component
46-
uses: docker/build-push-action@v4
50+
uses: docker/build-push-action@v5
4751
with:
4852
context: ./datastore
4953
file: ./datastore/Dockerfile
5054
push: true
5155
build-args: |
5256
REGISTRY=${{ vars.DOCKER_REGISTRY }}
57+
cache-from: type=gha
58+
cache-to: type=gha,mode=max
5359
tags: |
5460
${{ vars.DOCKER_REGISTRY }}/geolake-datastore:${{ env.TAG }}
55-
${{ vars.DOCKER_REGISTRY }}/geolake-datastore:latest
61+
${{ vars.DOCKER_REGISTRY }}/geolake-datastore:latest
5662
- name: Build and push api component
57-
uses: docker/build-push-action@v4
63+
uses: docker/build-push-action@v5
5864
with:
5965
context: ./api
6066
file: ./api/Dockerfile
6167
push: true
6268
build-args: |
6369
REGISTRY=${{ vars.DOCKER_REGISTRY }}
70+
cache-from: type=gha
71+
cache-to: type=gha,mode=max
6472
tags: |
6573
${{ vars.DOCKER_REGISTRY }}/geolake-api:${{ env.TAG }}
66-
${{ vars.DOCKER_REGISTRY }}/geolake-api:latest
74+
${{ vars.DOCKER_REGISTRY }}/geolake-api:latest
6775
- name: Build and push executor component
68-
uses: docker/build-push-action@v4
76+
uses: docker/build-push-action@v5
6977
with:
7078
context: ./executor
7179
file: ./executor/Dockerfile
7280
push: true
7381
build-args: |
7482
REGISTRY=${{ vars.DOCKER_REGISTRY }}
83+
cache-from: type=gha
84+
cache-to: type=gha,mode=max
7585
tags: |
7686
${{ vars.DOCKER_REGISTRY }}/geolake-executor:${{ env.TAG }}
77-
${{ vars.DOCKER_REGISTRY }}/geolake-executor:latest
87+
${{ vars.DOCKER_REGISTRY }}/geolake-executor:latest

‎.github/workflows/build_on_release.yaml‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,46 +32,54 @@ jobs:
3232
- name: Set up Docker Buildx
3333
uses: docker/setup-buildx-action@v2
3434
- name: Build and push drivers
35-
uses: docker/build-push-action@v4
35+
uses: docker/build-push-action@v5
3636
with:
3737
context: ./drivers
3838
file: ./drivers/Dockerfile
3939
push: true
4040
build-args: |
4141
REGISTRY=${{ vars.GEOKUBE_REGISTRY }}
4242
TAG=v0.2a6
43+
cache-from: type=gha
44+
cache-to: type=gha,mode=max
4345
tags: |
4446
${{ vars.GEOLAKE_REGISTRY }}/geolake-drivers:${{ env.RELEASE_TAG }}
4547
- name: Build and push datastore component
46-
uses: docker/build-push-action@v4
48+
uses: docker/build-push-action@v5
4749
with:
4850
context: ./datastore
4951
file: ./datastore/Dockerfile
5052
push: true
5153
build-args: |
5254
REGISTRY=${{ vars.GEOLAKE_REGISTRY }}
5355
TAG=${{ env.RELEASE_TAG }}
56+
cache-from: type=gha
57+
cache-to: type=gha,mode=max
5458
tags: |
5559
${{ vars.GEOLAKE_REGISTRY }}/geolake-datastore:${{ env.RELEASE_TAG }}
5660
- name: Build and push api component
57-
uses: docker/build-push-action@v4
61+
uses: docker/build-push-action@v5
5862
with:
5963
context: ./api
6064
file: ./api/Dockerfile
6165
push: true
6266
build-args: |
6367
REGISTRY=${{ vars.GEOLAKE_REGISTRY }}
6468
TAG=${{ env.RELEASE_TAG }}
69+
cache-from: type=gha
70+
cache-to: type=gha,mode=max
6571
tags: |
6672
${{ vars.GEOLAKE_REGISTRY }}/geolake-api:${{ env.RELEASE_TAG }}
6773
- name: Build and push executor component
68-
uses: docker/build-push-action@v4
74+
uses: docker/build-push-action@v5
6975
with:
7076
context: ./executor
7177
file: ./executor/Dockerfile
7278
push: true
7379
build-args: |
7480
REGISTRY=${{ vars.GEOLAKE_REGISTRY }}
7581
TAG=${{ env.RELEASE_TAG }}
82+
cache-from: type=gha
83+
cache-to: type=gha,mode=max
7684
tags: |
7785
${{ vars.GEOLAKE_REGISTRY }}/geolake-executor:${{ env.RELEASE_TAG }}

‎datastore/Dockerfile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
ARG REGISTRY=rg.fr-par.scw.cloud/geolake
22
ARG TAG=latest
33
FROM $REGISTRY/geolake-drivers:$TAG
4-
RUN conda install -c conda-forge --yes --freeze-installed psycopg2 \
5-
&& conda clean -afy
4+
65
COPY requirements.txt /app/requirements.txt
7-
RUN pip install --no-cache-dir -r /app/requirements.txt
6+
RUN pip install --no-cache-dir -r /app/requirements.txt
87
COPY ./datastore /app/datastore
98
COPY ./workflow /app/workflow
109
COPY ./dbmanager /app/dbmanager
@@ -13,3 +12,4 @@ COPY ./utils /app/utils
1312
COPY ./tests /app/tests
1413
COPY ./wait-for-it.sh /
1514

15+
WORKDIR /

‎datastore/datastore/datastore.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ def __init__(self) -> None:
5050

5151
@log_execution_time(_LOG)
5252
def get_cached_product_or_read(
53-
self, dataset_id: str, product_id: str
53+
self, dataset_id: str, product_id: str, query: GeoQuery | None = None
5454
) -> DataCube | Dataset:
5555
"""Get product from the cache instead of loading files indicated in
5656
the catalog if `metadata_caching` set to `True`.
@@ -81,7 +81,7 @@ def get_cached_product_or_read(
8181
)
8282
return self.catalog(CACHE_DIR=self.cache_dir)[dataset_id][
8383
product_id
84-
].read_chunked()
84+
].get(geoquery=query, compute=False).read_chunked()
8585
return self.cache[dataset_id][product_id]
8686

8787
@log_execution_time(_LOG)
@@ -389,7 +389,7 @@ def estimate(
389389
# NOTE: we always use catalog directly and single product cache
390390
self._LOG.debug("loading product...")
391391
# NOTE: for estimation we use cached products
392-
kube = self.get_cached_product_or_read(dataset_id, product_id)
392+
kube = self.get_cached_product_or_read(dataset_id, product_id, query=query)
393393
self._LOG.debug("original kube len: %s", len(kube))
394394
return Datastore._process_query(kube, geoquery, False).nbytes
395395

‎datastore/requirements.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
networkx
2-
pydantic<2.0.0
2+
pydantic<2.0.0
3+
psycopg2-binary

‎drivers/Dockerfile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
ARG REGISTRY=rg.fr-par.scw.cloud/geokube
2+
# ARG TAG=v0.2.6b1
23
ARG TAG=latest
34
FROM $REGISTRY/geokube:$TAG
4-
RUN conda install -c conda-forge --yes --freeze-installed intake=0.6.6
5-
RUN conda clean -afy
5+
66
COPY dist/intake_geokube-0.1a0-py3-none-any.whl /
77
RUN pip install /intake_geokube-0.1a0-py3-none-any.whl
8-
RUN rm /intake_geokube-0.1a0-py3-none-any.whl
8+
RUN rm /intake_geokube-0.1a0-py3-none-any.whl

0 commit comments

Comments
 (0)