Skip to content

Commit 87bb9df

Browse files
ImadYIdrissiImadYIdrissi
authored
Merge pull request #1 from ImadYIdrissi/config/Create_pip_compile_ecosystem
Config/create pip compile ecosystem & create first CI for the project
2 parents e919d7b + a8b7c3e commit 87bb9df

File tree

6 files changed

+447
-76
lines changed

6 files changed

+447
-76
lines changed

‎.github/workflows/code_ci.yml‎

Lines changed: 77 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,6 @@ on:
44
push:
55
branches:
66
- "**"
7-
pull_request:
8-
branches:
9-
- master
107

118
env:
129
ENV_FILE: .env
@@ -16,49 +13,78 @@ jobs:
1613
name: Get Environment Variables
1714
runs-on: ubuntu-latest
1815
outputs:
19-
python-version: ${{ steps.load-env.outputs.python-version }}
16+
python-version: ${{ steps.load-internal-env.outputs.python-version }}
17+
github-actions: ${{steps.load-external-env.outputs.github-actions}}
2018
steps:
2119
- name: Checkout Code
2220
uses: actions/checkout@v3
2321

24-
- name: Load Environment Variables
25-
id: load-env
22+
- name: Load Project's Internal Environment Variables
23+
id: load-internal-env
2624
run: |
27-
set -a
25+
set -x
2826
[ -f ${{ env.ENV_FILE }} ] && source ${{ env.ENV_FILE }}
29-
echo "::set-output name=python-version::$PYTHON_VERSION"
27+
echo "python-version=$PYTHON_VERSION" >> $GITHUB_OUTPUT
28+
- name: Load External Environment Variables
29+
id: load-external-env
30+
run: |
31+
set -x
32+
if [ "$GITHUB_ACTIONS" = "true" ]; then
33+
echo "github-actions=$GITHUB_ACTIONS" >> $GITHUB_OUTPUT
34+
else
35+
echo "github-actions=false" >> $GITHUB_OUTPUT
36+
fi
3037
3138
env-setup:
3239
name: Setup Python and Install Dependencies
3340
runs-on: ubuntu-latest
3441
needs: env-vars
42+
3543
steps:
3644
- name: Checkout Code
3745
uses: actions/checkout@v3
38-
3946
- name: Set up Python
40-
id: setup-python
4147
uses: actions/setup-python@v4
4248
with:
4349
python-version: ${{ needs.env-vars.outputs.python-version }}
44-
45-
- name: Create Python Dev-tools Dependencies Cache
46-
uses: actions/cache@v3
47-
with:
48-
path: ~/.cache/pip
49-
key: ${{ runner.os }}-pip-dev-tools-dev-tools-${{ hashFiles('requirements.dev.txt') }}
50-
51-
- name: Install Development Dependencies
52-
run: pip install -r requirements.dev.txt
53-
54-
- name: Create Python prod-libs Dependencies Cache
50+
- name: Create Non-root User & Install dependencies # This step does not run on github actions
51+
if: ${{ needs.env-vars.outputs.github-actions == 'false' }}
52+
run: |
53+
useradd -ms /bin/bash pyrunner
54+
mkdir -p /home/pyrunner/.cache/pip
55+
chown -R pyrunner:pyrunner /home/pyrunner
56+
chmod -R 700 /home/pyrunner/.cache/pip
57+
58+
sudo -u pyrunner bash -c "
59+
python -m venv ~/venv &&
60+
source ~/venv/bin/activate &&
61+
pip install --upgrade pip &&
62+
pip install -r requirements.txt
63+
"
64+
- name: Install Python Dependencies
65+
if: ${{ needs.env-vars.outputs.github-actions == 'true' }}
66+
run: |
67+
python -m venv ~/venv &&
68+
source ~/venv/bin/activate &&
69+
pip install --upgrade pip &&
70+
pip install -r requirements.txt
71+
- name: Verify Python Version
72+
run: |
73+
source ~/venv/bin/activate
74+
ACTUAL_VERSION=$(python --version 2>&1 | sed 's/Python //')
75+
EXPECTED_VERSION="${{ needs.env-vars.outputs.python-version }}"
76+
echo "Actual: $ACTUAL_VERSION"
77+
echo "Expected: $EXPECTED_VERSION"
78+
79+
if [ "$ACTUAL_VERSION" != "$EXPECTED_VERSION" ]; then
80+
echo "ERROR: Python version mismatch: expected $EXPECTED_VERSION, got $ACTUAL_VERSION"
81+
exit 1
82+
fi
83+
- name: Save Python Environment Cache
5584
uses: actions/cache@v3
5685
with:
57-
path: ~/.cache/pip
58-
key: ${{ runner.os }}-pip-prod-libs-${{ hashFiles('requirements.txt') }}
59-
60-
- name: Install Prod Dependencies
61-
run: pip install -r requirements.txt
86+
path: ~/venv
87+
key: ${{ runner.os }}-python-venv-${{ hashFiles('requirements.txt') }}
6288

6389
lint:
6490
name: Flake8 linting
@@ -68,18 +94,16 @@ jobs:
6894
- name: Checkout Code
6995
uses: actions/checkout@v3
7096

71-
- name: Restore Python Dev-tools Dependencies Cache
97+
- name: Restore Python Environment Cache
7298
uses: actions/cache@v3
7399
with:
74-
path: ~/.cache/pip
75-
key: ${{ runner.os }}-pip-dev-tools-${{ hashFiles('requirements.dev.txt') }}
76-
restore-keys: |
77-
${{ runner.os }}-pip-dev-tools-
78-
- name: Install Flake8
79-
run: pip install flake8 flake8-pyproject
100+
path: ~/venv
101+
key: ${{ runner.os }}-python-venv-${{ hashFiles('requirements.txt') }}
80102

81103
- name: Run Flake8
82-
run: flake8 .
104+
run: |
105+
source ~/venv/bin/activate
106+
flake8 .
83107
84108
security:
85109
name: Security Check
@@ -89,18 +113,16 @@ jobs:
89113
- name: Checkout Code
90114
uses: actions/checkout@v3
91115

92-
- name: Restore Python Dev-tools Dependencies Cache
116+
- name: Restore Python Environment Cache
93117
uses: actions/cache@v3
94118
with:
95-
path: ~/.cache/pip
96-
key: ${{ runner.os }}-pip-dev-tools-${{ hashFiles('requirements.dev.txt') }}
97-
restore-keys: |
98-
${{ runner.os }}-pip-dev-tools-
99-
- name: Install Bandit
100-
run: pip install bandit
119+
path: ~/venv
120+
key: ${{ runner.os }}-python-venv-${{ hashFiles('requirements.txt') }}
101121

102122
- name: Run Bandit Security Check
103-
run: bandit -r .
123+
run: |
124+
source ~/venv/bin/activate
125+
bandit -r .
104126
105127
quality:
106128
name: Code Quality
@@ -110,34 +132,24 @@ jobs:
110132
- name: Checkout Code
111133
uses: actions/checkout@v3
112134

113-
- name: Restore Python Dev-tools Dependencies Cache
114-
uses: actions/cache@v3
115-
with:
116-
path: ~/.cache/pip
117-
key: ${{ runner.os }}-pip-dev-tools-${{ hashFiles('requirements.dev.txt') }}
118-
restore-keys: |
119-
${{ runner.os }}-pip-dev-tools-
120-
121-
- name: Restore Python Prod Dependencies Cache
135+
- name: Restore Python Environment Cache
122136
uses: actions/cache@v3
123137
with:
124-
path: ~/.cache/pip
125-
key: ${{ runner.os }}-pip-prod-${{ hashFiles('requirements.txt') }}
126-
restore-keys: |
127-
${{ runner.os }}-pip-prod-
138+
path: ~/venv
139+
key: ${{ runner.os }}-python-venv-${{ hashFiles('requirements.txt') }}
128140

129-
- name: Install Pylint, Black & Pydocstyle
130-
run: pip install pylint black pydocstyle
131-
132-
- name: Install Prod libs
133-
run: pip install -r requirements.txt
134-
135-
- name: Pylint linting
136-
run: pylint .
141+
- name: Run Pylint
142+
run: |
143+
source ~/venv/bin/activate
144+
pylint .
137145
138146
- name: Check Black Formatting
139-
run: black --check .
147+
run: |
148+
source ~/venv/bin/activate
149+
black --check .
140150
141151
- name: Check Pydocstyle
142-
run: pydocstyle .
152+
run: |
153+
source ~/venv/bin/activate
154+
pydocstyle .
143155
continue-on-error: true # Non-blocking

‎README.md‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,25 @@ DataOps-Forge/ # Root of the repository containing all proje
3737
- Docker / Docker-desktop
3838
- act : To test CICD jobs
3939

40+
## Setup the environment
41+
42+
1. Create virtual env via pyenv, for convention call it dataops-forge
43+
2. Activate virtual env
44+
45+
```bash
46+
pyenv activate dataops-forge
47+
pip install -U pip pip-tools
48+
```
49+
50+
NB: It is recommended to set up this environment as your default IDE interpreter. 3. Install and update python tools
51+
52+
3. [Optional] if need to update python requirements, then recompile them this way :
53+
54+
```bash
55+
pip-compile requirements.in
56+
pip-compile requirements.dev.in
57+
```
58+
4059
## Run CI locally
4160

4261
Move to the root of the project.
@@ -58,3 +77,26 @@ Or if you want to test just a particular area
5877
```bash
5978
act -j quality
6079
```
80+
81+
You can also visualize the dependency graph with
82+
83+
```bash
84+
act --graph
85+
```
86+
87+
Example output
88+
89+
```plaintext
90+
INFO[0000] Using docker host 'unix:///var/run/docker.sock', and daemon socket 'unix:///var/run/docker.sock'
91+
╭───────────────────────────╮
92+
│ Get Environment Variables │
93+
╰───────────────────────────╯
94+
⬇
95+
╭───────────────────────────────────────╮
96+
│ Setup Python and Install Dependencies │
97+
╰───────────────────────────────────────╯
98+
⬇
99+
╭────────────────╮ ╭────────────────╮ ╭──────────────╮
100+
│ Flake8 linting │ │ Security Check │ │ Code Quality │
101+
╰────────────────╯ ╰────────────────╯ ╰──────────────╯
102+
```

‎requirements.dev.in‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
black
2+
bandit
3+
flake8
4+
flake8-pyproject
5+
pylint
6+
pydocstyle

‎requirements.dev.txt‎

Lines changed: 82 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,82 @@
1-
black
2-
bandit
3-
flake8
4-
flake8-pyproject
5-
pylint
6-
pydocstyle
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.12
3+
# by the following command:
4+
#
5+
# pip-compile requirements.dev.in
6+
#
7+
astroid==3.3.8
8+
# via pylint
9+
bandit==1.8.0
10+
# via -r requirements.dev.in
11+
black==24.10.0
12+
# via -r requirements.dev.in
13+
build==1.2.2.post1
14+
# via pip-tools
15+
click==8.1.8
16+
# via
17+
# black
18+
# pip-tools
19+
dill==0.3.9
20+
# via pylint
21+
flake8==7.1.1
22+
# via
23+
# -r requirements.dev.in
24+
# flake8-pyproject
25+
flake8-pyproject==1.2.3
26+
# via -r requirements.dev.in
27+
isort==5.13.2
28+
# via pylint
29+
markdown-it-py==3.0.0
30+
# via rich
31+
mccabe==0.7.0
32+
# via
33+
# flake8
34+
# pylint
35+
mdurl==0.1.2
36+
# via markdown-it-py
37+
mypy-extensions==1.0.0
38+
# via black
39+
packaging==24.2
40+
# via
41+
# black
42+
# build
43+
pathspec==0.12.1
44+
# via black
45+
pbr==6.1.0
46+
# via stevedore
47+
pip-tools==7.4.1
48+
# via -r requirements.dev.in
49+
platformdirs==4.3.6
50+
# via
51+
# black
52+
# pylint
53+
pycodestyle==2.12.1
54+
# via flake8
55+
pydocstyle==6.3.0
56+
# via -r requirements.dev.in
57+
pyflakes==3.2.0
58+
# via flake8
59+
pygments==2.18.0
60+
# via rich
61+
pylint==3.3.3
62+
# via -r requirements.dev.in
63+
pyproject-hooks==1.2.0
64+
# via
65+
# build
66+
# pip-tools
67+
pyyaml==6.0.2
68+
# via bandit
69+
rich==13.9.4
70+
# via bandit
71+
snowballstemmer==2.2.0
72+
# via pydocstyle
73+
stevedore==5.4.0
74+
# via bandit
75+
tomlkit==0.13.2
76+
# via pylint
77+
wheel==0.45.1
78+
# via pip-tools
79+
80+
# The following packages are considered to be unsafe in a requirements file:
81+
# pip
82+
# setuptools

‎requirements.in‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
-r requirements.dev.in
2+
pandas
3+
db-dtypes
4+
fastapi[standard]
5+
google-cloud-bigquery
6+
google-cloud-bigquery-storage

0 commit comments

Comments
 (0)