Skip to content

Commit b5b8793

Browse files
caviricaviri
committed
feat: Password Protected Links
1 parent 9be39e1 commit b5b8793

8 files changed

Lines changed: 188 additions & 7 deletions

File tree

‎.env.dist‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
LINK=
2+
PASSWORD=

‎.github/workflows/multiplatform_docker_build.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: Multi-Platform Docker Build
2+
3+
on:
4+
workflow_dispatch:
5+
6+
jobs:
7+
build-and-publish:
8+
runs-on: ubuntu-latest
9+
10+
steps:
11+
# Step 1: Check out the repository and submodules
12+
- name: Check out code
13+
uses: actions/checkout@v3
14+
with:
15+
submodules: true # Fetch submodules
16+
fetch-depth: 0 # Ensure the full history is fetched
17+
18+
# Step 2: Set up Docker Buildx
19+
- name: Set up Docker Buildx
20+
uses: docker/setup-buildx-action@v2
21+
22+
# Step 3: Install yq
23+
- name: Install yq
24+
run: |
25+
sudo apt-get update && sudo apt-get install -y wget
26+
sudo wget https://github.com/mikefarah/yq/releases/download/v4.35.1/yq_linux_amd64 -O /usr/bin/yq
27+
sudo chmod +x /usr/bin/yq
28+
29+
# Step 4: Extract component-version and component-name from odtp.yml
30+
- name: Extract component-version and component-name
31+
id: extract_info
32+
run: |
33+
VERSION=$(yq e '.component-version' odtp.yml)
34+
NAME=$(yq e '.component-name' odtp.yml)
35+
echo "VERSION=${VERSION}"
36+
echo "NAME=${NAME}"
37+
echo "COMPONENT_VERSION=${VERSION}" >> $GITHUB_ENV
38+
echo "COMPONENT_NAME=${NAME}" >> $GITHUB_ENV
39+
40+
# Step 5: Log in to GitHub Container Registry
41+
- name: Log in to GitHub Container Registry
42+
uses: docker/login-action@v2
43+
with:
44+
registry: ghcr.io
45+
username: ${{ github.actor }}
46+
password: ${{ secrets.GITHUB_TOKEN }}
47+
48+
# Step 6: Build and push Docker image for multiple platforms
49+
- name: Build and push Docker image
50+
run: |
51+
IMAGE_NAME=ghcr.io/${{ github.repository }}/${{ env.COMPONENT_NAME }}
52+
docker buildx build \
53+
--platform linux/amd64,linux/arm64 \
54+
--build-arg COMPONENT_VERSION=${{ env.COMPONENT_VERSION }} \
55+
-t $IMAGE_NAME:${{ env.COMPONENT_VERSION }} \
56+
-t $IMAGE_NAME:latest \
57+
--push .

‎.github/workflows/multiplatform_docker_build_dockerhub.yaml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
name: Multi-Platform Docker Build for Dockerhub
2+
3+
on:
4+
workflow_dispatch:
5+
6+
jobs:
7+
build-and-publish:
8+
runs-on: ubuntu-latest
9+
10+
steps:
11+
# Step 1: Check out the repository and submodules
12+
- name: Check out code
13+
uses: actions/checkout@v3
14+
with:
15+
submodules: true # Fetch submodules
16+
fetch-depth: 0 # Ensure the full history is fetched
17+
18+
# Step 2: Set up Docker Buildx
19+
- name: Set up Docker Buildx
20+
uses: docker/setup-buildx-action@v2
21+
22+
# Step 3: Install yq
23+
- name: Install yq
24+
run: |
25+
sudo apt-get update && sudo apt-get install -y wget
26+
sudo wget https://github.com/mikefarah/yq/releases/download/v4.35.1/yq_linux_amd64 -O /usr/bin/yq
27+
sudo chmod +x /usr/bin/yq
28+
29+
# Step 4: Extract component-version and component-name from odtp.yml
30+
- name: Extract component-version and component-name
31+
id: extract_info
32+
run: |
33+
VERSION=$(yq e '.component-version' odtp.yml)
34+
NAME=$(yq e '.component-name' odtp.yml)
35+
echo "VERSION=${VERSION}"
36+
echo "NAME=${NAME}"
37+
echo "COMPONENT_VERSION=${VERSION}" >> $GITHUB_ENV
38+
echo "COMPONENT_NAME=${NAME}" >> $GITHUB_ENV
39+
40+
# Step 5: Log in to Docker Hub
41+
- name: Log in to Docker Hub
42+
uses: docker/login-action@v2
43+
with:
44+
username: ${{ secrets.DOCKER_USERNAME }}
45+
password: ${{ secrets.DOCKER_PASSWORD }}
46+
47+
# Step 6: Build and push Docker image for multiple platforms
48+
- name: Build and push Docker image
49+
run: |
50+
IMAGE_NAME=${{ secrets.DOCKER_USERNAME }}/${{ env.COMPONENT_NAME }}
51+
docker buildx build \
52+
--platform linux/amd64,linux/arm64 \
53+
--build-arg COMPONENT_VERSION=${{ env.COMPONENT_VERSION }} \
54+
-t $IMAGE_NAME:${{ env.COMPONENT_VERSION }} \
55+
-t $IMAGE_NAME:latest \
56+
--push .

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
1+
# ODTP Testing
2+
odtp-input/
3+
odtp-output/
4+
5+
# Mac crap
6+
.DS_Store
7+
18
# Byte-compiled / optimized / DLL files
29
__pycache__/
310
*.py[cod]

‎Dockerfile‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,4 +66,12 @@ COPY ./odtp-component-client /odtp/odtp-component-client
6666
COPY ./app /odtp/odtp-app
6767
WORKDIR /odtp
6868

69+
# Avoid error when building on windows
70+
RUN sed -i 's/\r$//' /odtp/odtp-component-client/src/shell/log.sh
71+
RUN sed -i 's/\r$//' /odtp/odtp-component-client/src/shell/traceback.sh
72+
RUN sed -i 's/\r$//' /odtp/odtp-component-client/scripts/component-update.sh
73+
RUN sed -i 's/\r$//' /odtp/odtp-component-client/odtp-app.sh
74+
RUN sed -i 's/\r$//' /odtp/odtp-component-client/startup.sh
75+
RUN sed -i 's/\r$//' /odtp/odtp-app/app.sh
76+
6977
ENTRYPOINT ["bash", "/odtp/odtp-component-client/startup.sh"]

‎README.md‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,44 @@ This is the component that will download and arrange files to make it compatible
66
```
77
odtp new odtp-component-entry \
88
--name odtp-eqasim-dataloader \
9-
--component-version 0.3.2 \
9+
--component-version 0.4.0 \
1010
--repository https://github.com/odtp-org/odtp-eqasim-dataloader
1111
```
1212

1313
## Tutorial
1414

1515
This component is a temporal solution until we have dataloaders defined for each source of data. This component accept one link to switchdrive and the download and uncompress it content to an output folder.
1616

17-
Therefore, the only ENV variable available is:
17+
- LINK: Link to the switch drive.
18+
- PASSWORD: Optional: If the link to the switch drive contains a password it should be provided.
19+
20+
## Tutorial
21+
22+
### How to run this component as docker for development
23+
24+
Build the dockerfile
25+
26+
```
27+
docker build -t odtp-eqasim-dataloader .
28+
```
29+
30+
Run the following command. Mount the correct volumes for input/output folders.
31+
32+
```
33+
docker run -it --rm -v ${pwd}/odtp-input:/odtp/odtp-input -v ${pwd}/odtp-output:/odtp/odtp-output --env-file .env --entrypoint bash odtp-eqasim-dataloader
34+
```
1835

19-
- LINK:
2036

2137
## Changelog
2238

39+
- v0.4.0
40+
41+
- v0.3.5
42+
43+
- v0.3.4
44+
45+
- v0.3.3
46+
2347
- v0.3.2
2448
- Ubuntu fixed at 22.04
2549
- Python fixed at 3.10

‎app/app.sh‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,26 @@
33
# Downloading Link provided
44
# The option --progress=bar:force:noscroll makes the log output less verbose
55
# While the progressbar is displayed only one line is added for it in the logs
6-
wget --progress=bar:force:noscroll $LINK/download
7-
mv download download.zip
6+
wget --progress=bar:force:noscroll -O download.zip $LINK/download
7+
echo "Downloaded file with user id and password. Now procedding with unzip"
88
unzip -q download.zip -d /odtp/odtp-output
9+
echo "Uncompressed file."
10+
11+
if [ -n "$PASSWORD" ]; then
12+
# Check if the required environment variables are set
13+
if [ -z "$LINK" ] || [ -z "$PASSWORD" ]; then
14+
echo "Error: LINK and PASSWORD environment variables must be set."
15+
exit 1
16+
fi
17+
18+
# Extract the user id from the LINK
19+
USER_ID=$(echo $LINK | sed 's#.*/s/\(.*\)#\1#')
20+
21+
# Download the file using wget with the provided user id and password
22+
wget --user=$USER_ID --password=$PASSWORD --progress=bar:force:noscroll --header='X-Requested-With: XMLHttpRequest' -O download.zip https://drive.switch.ch/public.php/webdav/
23+
24+
echo "Downloaded file with user id and password. Now procedding with unzip"
25+
unzip -qo download.zip -d /odtp/odtp-output
26+
27+
echo "Uncompressed file."
28+
fi

‎odtp.yml‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# This file should contain basic component information for your component.
22
component-name: odtp-eqasim-dataloader
33
component-author: Carlos Vivar Rios
4-
component-version: 0.3.1
4+
component-version: v0.4.0
55
component-repository: https://github.com/odtp-org/odtp-eqasim-dataloader
66
component-license: AGPL-3.0
77
component-type: ephemeral
@@ -15,7 +15,13 @@ tools: null
1515

1616
# If your tool require some secrets token to be passed as ENV to the component
1717
# This won't be traced in MongoDB
18-
secrets: null
18+
secrets:
19+
- name: LINK
20+
default-value: null
21+
datatype: string
22+
description: Password to the switch drive link to download
23+
options: null
24+
allow-custom-value: true
1925

2026
# If the tool requires some building arguments such as Matlab license
2127
build-args: null
@@ -34,6 +40,7 @@ parameters:
3440
options: null
3541
allow-custom-value: true
3642

43+
3744
# If applicable, data-input list required by the component
3845
data-inputs: null
3946

0 commit comments

Comments
 (0)