Skip to content

Commit aa1c99b

Browse files
atakeigoratakeigor
authored
Merge pull request #1 from CMCC-Foundation/test_marketplace
Test marketplace
2 parents f8d0f2c + a219d13 commit aa1c99b

File tree

101 files changed

+21886
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

101 files changed

+21886
-0
lines changed

‎.cdsapirc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
url: https://cds-beta.climate.copernicus.eu/api
2+
key: None

‎.github/workflows/ci.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
name: ci
2+
on:
3+
push:
4+
branches:
5+
- main
6+
permissions:
7+
contents: write
8+
jobs:
9+
deploy:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- uses: actions/checkout@v4
13+
- name: Configure Git Credentials
14+
run: |
15+
git config user.name atakeigor
16+
git config user.email [email protected]
17+
- uses: actions/setup-python@v5
18+
with:
19+
python-version: 3.x
20+
- run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
21+
- uses: actions/cache@v4
22+
with:
23+
key: mkdocs-material-${{ env.cache_id }}
24+
path: .cache
25+
restore-keys: |
26+
mkdocs-material-
27+
- run: pip install mkdocs-material
28+
- working-directory: ./documentation
29+
run: mkdocs gh-deploy --force

‎.gitignore‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
# system files
2+
.DS_Store
3+
src/preprocessing/__pycache__/
4+
src/utils/__pycache__/
5+
src/postprocessing/__pycache__/
6+
src/preprocessing/extract/bin/
7+
src/__pycache__/
8+
src/plot/__pycache__/
9+
__pycache__/main.cpython-311.pyc
10+
src/download/__pycache__/__init__.cpython-311.pyc
11+
src/download/__pycache__/download_copernicus_parser.cpython-311.pyc
12+
src/download/__pycache__/download_era5_parser.cpython-311.pyc
13+
# log files
14+
compile.log
15+
medslik_run.log
16+
# folders and files used in other branches
17+
src/model/bin/
18+
src/model/mod/
19+
src/model/test/*.o
20+
src/model/test/*.exe
21+
src/model/test/*.mod
22+
src/model/obj/
23+
src/utils/jday/bin/
24+
src/model/test/data/test_envdata_20170423.nc
25+
src/model/test/data/test_envdata_20170424.nc
26+
src/model/test/data/test_envdata_20170425.nc
27+
fort.91
28+
output
29+
# data not uploaded
30+
data/gebco/GEBCO_2023.nc
31+
data/gshhs/GSHHS_shp/c/*
32+
data/gshhs/GSHHS_shp/f/*
33+
data/gshhs/GSHHS_shp/h/*
34+
data/gshhs/GSHHS_shp/i/*
35+
data/gshhs/GSHHS_shp/l/*
36+
data/gshhs/f/GSHHS_f_L1.shp
37+
data/gshhs/f/GSHHS_f_L1.shx
38+
# algeria test case
39+
cases/algeria/out_files/
40+
cases/algeria/bnc_files/dtmcst1.d
41+
cases/algeria/xp_files/
42+
cases/algeria/bnc_files/
43+
cases/algeria/oce_files/
44+
cases/algeria/met_files/
45+
cases/algeria/*.log
46+
cases/algeria/detections
47+
# paria test case
48+
cases/paria/out_files/oil_concentration.nc
49+
cases/parameters/*
50+
cases/paria/out_files
51+
cases/paria/xp_files
52+
cases/paria/*.log
53+
cases/paria/bnc_files/dtmcst1.d
54+
cases/paria/detections
55+
# lebanon test case
56+
cases/lebanon/out_files/
57+
cases/lebanon/bnc_files/dtmcst1.d
58+
cases/lebanon/xp_files/
59+
cases/lebanon/*.log
60+
cases/lebanon/detections
61+
# Syria test case
62+
cases/syria/out_files/
63+
cases/syria/bnc_files/dtmcst1.d
64+
cases/syria/xp_files/
65+
cases/syria/bnc_files/
66+
cases/syria/oce_files/
67+
cases/syria/met_files/
68+
cases/syria/*.log
69+
cases/syria/detections

‎Dockerfile‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
# THIS FILE CONTAINS THE NECESSARY ELEMENTS TO RUN MEDSLIK-II in a container
2+
FROM continuumio/miniconda3
3+
4+
# Set the working directory
5+
WORKDIR /app
6+
7+
COPY requirements_conda.txt .
8+
9+
COPY requirements.txt .
10+
11+
# Create a Conda environment with a specific Python version
12+
RUN conda create -n medslik --file ./requirements_conda.txt
13+
14+
# Make RUN commands use the new environment
15+
SHELL ["conda", "run", "-n", "medslik", "/bin/bash", "-c"]
16+
17+
RUN conda install -n medslik pip
18+
19+
# Install pip packages from the requirements.txt file
20+
RUN pip install -r ./requirements.txt
21+
22+
#Set up the Home dor as the Medslik Directory
23+
WORKDIR /Medslik-II
24+
25+
# Create a non-privileged user that the app will run under.
26+
# See https://docs.docker.com/go/dockerfile-user-best-practices/
27+
ARG UID=10001
28+
RUN adduser \
29+
--disabled-password \
30+
--gecos "" \
31+
--home "/Medslik-II" \
32+
--uid "${UID}" \
33+
appuser
34+
35+
#Allow appuser to navigate and read scripts
36+
RUN chown -R appuser:appuser /Medslik-II
37+
38+
# Download dependencies as a separate step to take advantage of Docker's caching.
39+
# Leverage a cache mount to /root/.cache/pip to speed up subsequent builds.
40+
# Leverage a bind mount to requirements.txt to avoid having to copy them into
41+
# into this layer.
42+
# RUN --mount=type=cache,target=/root/.cache/pip \
43+
# --mount=type=bind,source=requirements.txt,target=requirements.txt \
44+
# python -m pip install -r requirements.txt
45+
46+
RUN apt-get update \
47+
&& apt-get install -yq --no-install-recommends \
48+
build-essential \
49+
curl \
50+
fuse \
51+
gfortran \
52+
g++ \
53+
git \
54+
gnupg \
55+
gnupg2 \
56+
keychain \
57+
libcurl4-openssl-dev \
58+
libfuse-dev \
59+
liblapack-dev \
60+
libssl-dev \
61+
locate \
62+
lsb-release \
63+
make \
64+
m4 \
65+
nano \
66+
rsync \
67+
tzdata \
68+
tini \
69+
unzip \
70+
vim \
71+
wget \
72+
zip
73+
74+
# build netcdf with gcc and g-fortran
75+
ENV CC=gcc
76+
ENV FC=gfortran
77+
78+
# set library location
79+
ENV PREFIXDIR=/usr/local
80+
81+
WORKDIR /
82+
83+
## get zlib
84+
ENV ZLIB_VERSION=zlib-1.3.1
85+
RUN wget https://zlib.net/${ZLIB_VERSION}.tar.gz && tar -xvzf ${ZLIB_VERSION}.tar.gz
86+
RUN cd ${ZLIB_VERSION} \
87+
&& ./configure --prefix=${PREFIXDIR} \
88+
&& make install
89+
WORKDIR /
90+
RUN rm -rf ${ZLIB_VERSION}.tar.gz ${ZLIB_VERSION}
91+
92+
## get hdf5-1.8
93+
ENV HDF518_VERSION=hdf5-1.8.21
94+
RUN wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8/${HDF518_VERSION}/src/${HDF518_VERSION}.tar.gz && tar -xvzf ${HDF518_VERSION}.tar.gz
95+
RUN cd ${HDF518_VERSION} \
96+
&& ./configure --with-zlib=${PREFIXDIR} --prefix=${PREFIXDIR} --enable-hl --enable-shared\
97+
&& make \
98+
&& make install
99+
WORKDIR /
100+
RUN rm -rf ${HDF518_VERSION}.tar.gz ${HDF518_VERSION}
101+
102+
## get netcdf-c
103+
ENV NETCDFC_VERSION=4.8.0
104+
RUN wget https://github.com/Unidata/netcdf-c/archive/v${NETCDFC_VERSION}.tar.gz && tar -xvzf v${NETCDFC_VERSION}.tar.gz
105+
RUN cd netcdf-c-${NETCDFC_VERSION} \
106+
&& CPPFLAGS=-I${PREFIXDIR}/include LDFLAGS=-L${PREFIXDIR}/lib ./configure --prefix=${PREFIXDIR} --enable-netcdf-4 --enable-shared --enable-dap \
107+
&& make install
108+
WORKDIR /
109+
RUN rm -rf v${NETCDFC_VERSION}.tar.gz netcdf-fortran-${NETCDFC_VERSION}
110+
111+
# Set these flags because some NETCDF libraries do not allow some numerical warnings. As they are not error, these f;ags do not interfer with the results
112+
ENV FCFLAGS="-w -fallow-argument-mismatch -O2"
113+
ENV FFLAGS="-w -fallow-argument-mismatch -O2"
114+
115+
## get netcdf-fortran
116+
ENV NETCDFFORTRAN_VERSION=4.6.0
117+
RUN wget https://github.com/Unidata/netcdf-fortran/archive/v${NETCDFFORTRAN_VERSION}.tar.gz && tar -xvzf v${NETCDFFORTRAN_VERSION}.tar.gz
118+
RUN cd netcdf-fortran-${NETCDFFORTRAN_VERSION} \
119+
&& CPPFLAGS=-I${PREFIXDIR}/include LDFLAGS=-L${PREFIXDIR}/lib ./configure --prefix=${PREFIXDIR} \
120+
# && make check \
121+
&& make install
122+
WORKDIR /
123+
RUN rm -rf v${NETCDFFORTRAN_VERSION}.tar.gz netcdf-fortran-${NETCDFFORTRAN_VERSION}
124+
125+
#Create a env variable to indicate where the netcdf libaries are saved. Otherwise, executables are not able to run
126+
ENV LD_LIBRARY_PATH=/usr/local/lib
127+
128+
# Switch to the non-privileged user to run the application.
129+
USER appuser
130+
131+
#Change to home directory again
132+
WORKDIR /Medslik-II
133+
134+
RUN echo "source activate medslik" > ~/.bashrc
135+
136+
ENV PATH /opt/conda/envs/medslik/bin:$PATH
137+
138+
# Copy the source code into the container.
139+
COPY --chown=appuser:appuser . .
140+
141+
EXPOSE 8501
142+
143+
HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health
144+
145+
# Run the application.
146+
ENTRYPOINT ["streamlit", "run", "interface/WITOIL_on_Cloud.py","--server.port=8501", "--server.address=0.0.0.0"]

‎cases/syria/config_syria.toml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
[simulation]
2+
name = "syria" # name of the simulation.
3+
experiment_path = "cases/" # path where experiment folder should be placed
4+
start_datetime = 2021-08-22T13:43:00 # start date of the simulation
5+
sim_length = 24.0 # length of the simulation in HOURS
6+
spill_lat = [35.25] # lat of oil spill
7+
spill_lon = [35.90] # lon of oil spill
8+
spill_duration = [0.0] # duration of oil spill HOURS. = 0 for instantaneous release
9+
spill_rate = [27.78] # spill rate TONS/HOUR
10+
slick_age = [0.0] # age of oil slick in HOURS
11+
oil = [28] # either api (number) of the oil or name (string), e.g. ["Ragusa"]. considering that name must be exact
12+
area_spill = true
13+
area_vertex = false #comprehends thre levels of lists. 1st: all slicks. 2nd: individual slick. 3rd: Coordinates of each vertex in each individual slick
14+
multiple_slick = false
15+
advanced_parameters = false # user must provide parameters.toml file
16+
advanced_parameters_path = "" # this path shuld be provided only if "advanced_parameters" is = true
17+
[download]
18+
download_data = true
19+
download_curr = true
20+
download_wind = true
21+
copernicus_user = "my_username"
22+
copernicus_password = "my_password"
23+
[input_files]
24+
set_domain = false
25+
delta = [0.75] # default distance in degrees to download or crop data in case lat and lon areas are not provided
26+
lat = [9, 12]
27+
lon = [-62, -65]
28+
[input_files.dtm]
29+
bathymetry_path = "data/gebco/GEBCO_2023.nc" # GEBCO 2023 bathymetry file
30+
coastline_path = "data/gshhs/f/GSHHS_f_L1.shp" # coastline shapefile gshhs
31+
[input_files.metoce]
32+
oce_data_path = "path/to/oce/data"
33+
met_data_path = "path/to/met/data"
34+
[input_files.shapefile]
35+
shape_path = "cases/syria/result.shp" # add "path/to/shapefile" in case you want to start from shapefile
36+
[run_options]
37+
preprocessing = true
38+
preprocessing_metoce = true
39+
preprocessing_dtm = true
40+
run_model = true
41+
postprocessing = true # conversion from particles to concentration
42+
[plot_options]
43+
plotting = true
44+
# Domain for plotting
45+
plot_lon = [35.75,36] # [min lon, max lon]
46+
plot_lat = [35.1,35.4] # [min lat, max lat]]

‎compile.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# set folders
2+
DIR_EXE="$PWD/src/model/bin/"
3+
DIR_SRC="$PWD/src/model/"
4+
NETCDF="$CONDA_PREFIX/"
5+
# print message
6+
echo "============================="
7+
echo " MEDSLIK-II SOFTWARE ... "
8+
echo "============================="
9+
echo "MODEL FOLDER $DIR_SRC"
10+
echo "EXEC FOLDER $DIR_EXE"
11+
echo "NETCDF LIBRARIES TAKEN FROM $NETCDF"
12+
echo "============================="
13+
# compile model
14+
echo " -- Compiling model ..."
15+
mkdir -p "$DIR_EXE"
16+
gfortran -I"$NETCDF/include" -L"$NETCDF/lib" "$DIR_SRC/simulation.for" -lnetcdf -lnetcdff -o "$DIR_EXE/simulation.exe"
17+
# print message
18+
echo "============================="
19+
echo "END SUCCESSFULLY"

‎config.toml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# NOTE: this config is provided as a TEMPLATE, it should be changed by the user before using it.
2+
[simulation]
3+
name = "my_experiment" # name of the simulation.
4+
experiment_path = "path/to/user/output/folder" # path where experiment folder should be placed
5+
start_datetime = 2021-08-21T03:43:00 # start date of the simulation
6+
sim_length = 48.0 # length of the simulation in HOURS
7+
spill_lat = [35.25] # lat of oil spill (deg N)
8+
spill_lon = [35.90] # lon of oil spill (deg E)
9+
spill_duration = [0.0] # duration of oil spill HOURS. = 0.0 for instantaneous release
10+
spill_rate = [27.78] # spill rate TONS/HOUR
11+
slick_age = [0.0] # age of oil slick in HOURS
12+
oil = [28] # either oil api (number) or name (string), e.g. ["Ragusa"]. Please, consider that name must be exact.
13+
area_spill = true
14+
area_vertex = false # comprehends thre levels of lists. 1st: all slicks. 2nd: individual slick. 3rd: Coordinates of each vertex in each individual slick
15+
multiple_slick = false
16+
advanced_parameters = false # if = true, user must provide parameters.toml file
17+
advanced_parameters_path = "src/parameters.toml" # this path shuld be provided only if "advanced_parameters" is = true
18+
[download]
19+
download_data = false # = true if data are not provided by the user
20+
download_curr = false # = true : OCE data are downloaded from Copernicus Marine Service
21+
download_wind = false # = true : MET data are downloaded from ECMWF ERA5 product
22+
copernicus_user = "my_username"
23+
copernicus_password = "my_password"
24+
[input_files]
25+
set_domain = true # If the user wants to set the domain for cropping/preprocessing input data
26+
lat = [31, 38]
27+
lon = [32, 37]
28+
delta = [0.75] # default domain length in degrees (applied to both lon/lat), to download or crop data
29+
# note: delta is used only if set_domain = false
30+
[input_files.dtm]
31+
bathymetry_path = "data/gebco/GEBCO_2023.nc" # GEBCO 2023 bathymetry file
32+
coastline_path = "data/gshhs/f/GSHHS_f_L1.shp" # coastline shapefile gshhs
33+
[input_files.metoce]
34+
oce_data_path = "path/to/oce/data" # to provide if dowload_curr = false
35+
met_data_path = "path/to/met/data" # to provide if dowload_wind = false
36+
[input_files.shapefile]
37+
shape_path = "path/to/shapefile" # add "path/to/shapefile" in case you want to start from shapefile.
38+
# set shape_path = false or "none" if you do not want to start from a shapefile
39+
[run_options]
40+
preprocessing = true # = false if no preprocessing at all should be performed
41+
preprocessing_metoce = true # MET/OCE data preprocessing
42+
preprocessing_dtm = true # bathymetry and coastline preprocessing
43+
run_model = true # run oil spill model
44+
postprocessing = true # conversion from particles to concentration
45+
[plot_options]
46+
plotting = true # = true if results should be plotted (it should be used only if postprocessing = true)
47+
# Domain for plotting
48+
plot_lon = [35.5,36.5] # [min lon, max lon]
49+
plot_lat = [35,36] # [min lat, max lat]]

‎data/COPERNICUS/.gitkeep‎

Whitespace-only changes.

‎data/ERA5/.gitkeep‎

Whitespace-only changes.

‎data/gebco/.gitkeep‎

Whitespace-only changes.

0 commit comments

Comments
 (0)