Skip to content

Commit 7b8449d

Browse files
stroblmestroblme
committed
Merge branch 'doc'
2 parents 327baf4 + 594fb8d commit 7b8449d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+7894
-187
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
venv
22
mlruns
3+
.mlruns_bckp
34
*.pdf
45
.ipython
56
*.png

‎.vscode/launch.json

+12-1
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,16 @@
8585
// Any other arguments should be passed as a comma-seperated-list
8686
// e.g "args": ["run", "--pipeline", "pipeline_name"]
8787
},
88+
{
89+
"name": "KR: - validation",
90+
"type": "python",
91+
"request": "launch",
92+
"console": "integratedTerminal",
93+
"module": "kedro",
94+
"args": ["run", "--pipeline", "validation_qgnn_pipeline"]
95+
// Any other arguments should be passed as a comma-seperated-list
96+
// e.g "args": ["run", "--pipeline", "pipeline_name"]
97+
},
8898
{
8999
"name": "KR: - training Optuna",
90100
"type": "python",
@@ -114,7 +124,8 @@
114124
"request": "launch",
115125
"console": "integratedTerminal",
116126
"module": "kedro",
117-
"args": ["run"]
127+
"args": ["run"],
128+
// "justMyCode": false
118129
// Any other arguments should be passed as a comma-seperated-list
119130
// e.g "args": ["run", "--pipeline", "pipeline_name"]
120131
}

‎.vscode/mlflow_cleanup.py

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import glob
2+
import os
3+
import yaml
4+
import shutil
5+
6+
mlflow_path = "./mlruns/1"
7+
backup_dir = "./.mlruns_bckp"
8+
9+
cut_after = 1 # 1670237437014
10+
11+
runs = glob.glob(os.path.join(mlflow_path, "*"))
12+
13+
for r in runs:
14+
mark_for_deletion = False
15+
mark_for_deprecation = False
16+
if not os.path.isdir(r):
17+
continue
18+
with open(os.path.join(r, "meta.yaml"), "r") as f:
19+
try:
20+
content = yaml.safe_load(f)
21+
except yaml.YAMLError as exc:
22+
print(exc)
23+
24+
if content["status"] != 3:
25+
mark_for_deletion = True
26+
elif int(content["end_time"]) < cut_after:
27+
mark_for_deprecation = True
28+
content["lifecycle_stage"] = "deleted"
29+
30+
if mark_for_deletion:
31+
shutil.move(r, os.path.join(backup_dir, os.path.basename(r)))
32+
elif mark_for_deprecation:
33+
with open(os.path.join(r, "meta.yaml"), "w") as f:
34+
yaml.safe_dump(content, f)

‎.vscode/setup.sh

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
poetry install --without dev
2+
poetry run kedro mlflow init

‎.vscode/setup_dev.sh

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
poetry install
2+
poetry run pre-commit autoupdate
3+
poetry run pre-commit install
4+
poetry run kedro mlflow init
5+
poetry run pytest
6+
poetry run mkdocs build

‎.vscode/tasks.json

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
{
2+
"version": "2.0.0",
3+
"tasks": [
4+
5+
{
6+
"label": "Setup",
7+
"type": "shell",
8+
"command": ".vscode/setup.sh",
9+
"group": "none",
10+
"presentation": {
11+
"reveal": "never",
12+
"panel": "new"
13+
},
14+
"problemMatcher": []
15+
},
16+
{
17+
"label": "Kedro Test",
18+
"type": "shell",
19+
"command": "poetry",
20+
"args": ["run", "pytest"],
21+
"group": "none",
22+
"presentation": {
23+
"reveal": "never",
24+
"panel": "dedicated"
25+
},
26+
"problemMatcher": []
27+
},
28+
{
29+
"label": "Kedro Viz",
30+
"type": "shell",
31+
"command": "poetry",
32+
"args": ["run", "kedro", "viz", "--autoreload"],
33+
"group": "none",
34+
"presentation": {
35+
"reveal": "never",
36+
"panel": "new"
37+
},
38+
"runOptions": {
39+
"runOn": "folderOpen"
40+
},
41+
"problemMatcher": []
42+
},
43+
{
44+
"label": "MLFlow Dashboard",
45+
"type": "shell",
46+
"command": "poetry",
47+
"args": ["run", "mlflow", "ui"],
48+
"group": "none",
49+
"presentation": {
50+
"reveal": "never",
51+
"panel": "new"
52+
},
53+
"runOptions": {
54+
"runOn": "folderOpen"
55+
},
56+
"problemMatcher": []
57+
},
58+
{
59+
"label": "MkDocs Build",
60+
"type": "shell",
61+
"command": "poetry",
62+
"args": ["run", "mkdocs", "build"],
63+
"group": "none",
64+
"presentation": {
65+
"reveal": "never",
66+
"panel": "dedicated"
67+
},
68+
"problemMatcher": []
69+
},
70+
{
71+
"label": "MkDocs Serve",
72+
"type": "shell",
73+
"command": "poetry",
74+
"args": ["run", "mkdocs", "serve"],
75+
"group": "none",
76+
"presentation": {
77+
"reveal": "never",
78+
"panel": "dedicated"
79+
},
80+
"problemMatcher": []
81+
},
82+
{
83+
"label": "MkDocs Deploy",
84+
"type": "shell",
85+
"command": "poetry",
86+
"args": ["run", "mkdocs", "gh-deploy"],
87+
"group": "none",
88+
"presentation": {
89+
"reveal": "never",
90+
"panel": "dedicated"
91+
},
92+
"problemMatcher": []
93+
},
94+
]
95+
}

‎README.md

+12-122
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,18 @@ pip compile
1818

1919
torch_scatter needs gcc-c++ and python3-devel packages to build successfully.
2020

21+
with poetry you need to release the tensorflow-probability dependency from phasespace such that the line in ```.venv/lib/python3.10/site-packages/phasespace-1.8.0.dist-info/METADATA``` becomes
22+
```
23+
Requires-Dist: tensorflow-probability (>=0.15)
24+
```
25+
To achieve this, you may want to run
26+
```
27+
poetry add phasespace
28+
```
29+
prior to the installation of the other packages using
30+
```
31+
poe
32+
2133
changed module dependencies in phasespace:
2234
2335
before:
@@ -44,125 +56,3 @@ run kedro mlflow init
4456
4557
4658
47-
48-
49-
## Overview
50-
51-
This is your new Kedro project, which was generated using `Kedro 0.17.7`.
52-
53-
Take a look at the [Kedro documentation](https://kedro.readthedocs.io) to get started.
54-
55-
## Rules and guidelines
56-
57-
In order to get the best out of the template:
58-
59-
* Don't remove any lines from the `.gitignore` file we provide
60-
* Make sure your results can be reproduced by following a [data engineering convention](https://kedro.readthedocs.io/en/stable/12_faq/01_faq.html#what-is-data-engineering-convention)
61-
* Don't commit data to your repository
62-
* Don't commit any credentials or your local configuration to your repository. Keep all your credentials and local configuration in `conf/local/`
63-
64-
## How to install dependencies
65-
66-
Declare any dependencies in `src/requirements.txt` for `pip` installation and `src/environment.yml` for `conda` installation.
67-
68-
To install them, run:
69-
70-
```
71-
kedro install
72-
```
73-
74-
## How to run your Kedro pipeline
75-
76-
You can run your Kedro project with:
77-
78-
```
79-
kedro run
80-
```
81-
82-
## How to test your Kedro project
83-
84-
Have a look at the file `src/tests/test_run.py` for instructions on how to write your tests. You can run your tests as follows:
85-
86-
```
87-
kedro test
88-
```
89-
90-
To configure the coverage threshold, go to the `.coveragerc` file.
91-
92-
## Project dependencies
93-
94-
To generate or update the dependency requirements for your project:
95-
96-
```
97-
kedro build-reqs
98-
```
99-
100-
This will copy the contents of `src/requirements.txt` into a new file `src/requirements.in` which will be used as the source for `pip-compile`. You can see the output of the resolution by opening `src/requirements.txt`.
101-
102-
After this, if you'd like to update your project requirements, please update `src/requirements.in` and re-run `kedro build-reqs`.
103-
104-
[Further information about project dependencies](https://kedro.readthedocs.io/en/stable/04_kedro_project_setup/01_dependencies.html#project-specific-dependencies)
105-
106-
## How to work with Kedro and notebooks
107-
108-
> Note: Using `kedro jupyter` or `kedro ipython` to run your notebook provides these variables in scope: `context`, `catalog`, and `startup_error`.
109-
>
110-
> Jupyter, JupyterLab, and IPython are already included in the project requirements by default, so once you have run `kedro install` you will not need to take any extra steps before you use them.
111-
112-
### Jupyter
113-
To use Jupyter notebooks in your Kedro project, you need to install Jupyter:
114-
115-
```
116-
pip install jupyter
117-
```
118-
119-
After installing Jupyter, you can start a local notebook server:
120-
121-
```
122-
kedro jupyter notebook
123-
```
124-
125-
### JupyterLab
126-
To use JupyterLab, you need to install it:
127-
128-
```
129-
pip install jupyterlab
130-
```
131-
132-
You can also start JupyterLab:
133-
134-
```
135-
kedro jupyter lab
136-
```
137-
138-
### IPython
139-
And if you want to run an IPython session:
140-
141-
```
142-
kedro ipython
143-
```
144-
145-
### How to convert notebook cells to nodes in a Kedro project
146-
You can move notebook code over into a Kedro project structure using a mixture of [cell tagging](https://jupyter-notebook.readthedocs.io/en/stable/changelog.html#release-5-0-0) and Kedro CLI commands.
147-
148-
By adding the `node` tag to a cell and running the command below, the cell's source code will be copied over to a Python file within `src/<package_name>/nodes/`:
149-
150-
```
151-
kedro jupyter convert <filepath_to_my_notebook>
152-
```
153-
> *Note:* The name of the Python file matches the name of the original notebook.
154-
155-
Alternatively, you may want to transform all your notebooks in one go. Run the following command to convert all notebook files found in the project root directory and under any of its sub-folders:
156-
157-
```
158-
kedro jupyter convert --all
159-
```
160-
161-
### How to ignore notebook output cells in `git`
162-
To automatically strip out all output cell contents before committing to `git`, you can run `kedro activate-nbstripout`. This will add a hook in `.git/config` which will run `nbstripout` before anything is committed to `git`.
163-
164-
> *Note:* Your output cells will be retained locally.
165-
166-
## Package your Kedro project
167-
168-
[Further information about building project documentation and packaging your project](https://kedro.readthedocs.io/en/stable/03_tutorial/08_package_a_project.html)

‎conf/base/parameters/data_generation.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ max_depth: 3 # this will control the dimensionality of the classes
77
max_children: 3 # careful, this only controls how many children a single particle can have
88
min_children: 2
99
isp_weight: 1.0
10-
train_events_per_top: 500
11-
val_events_per_top: 100
10+
train_events_per_top: 100
11+
val_events_per_top: 20
1212
test_events_per_top: 10
1313
seed: 1111
1414
iso_retries: 0

0 commit comments

Comments
 (0)