Skip to content

Commit 70fe1de

Browse files
stevebachmeierstevebachmeier
authored
update ci (#188)
1 parent 23e697a commit 70fe1de

3 files changed

Lines changed: 142 additions & 6 deletions

File tree

‎Jenkinsfile‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,39 @@
1-
@Library("vivarium_build_utils") _
1+
/* This Jenkinsfile simply loads the `reusable_pipeline` pipeline from the
2+
vivariu_build_utils repository (https://github.com/ihmeuw/vivarium_build_utils).
3+
4+
vivarium_build_utils is loaded as a Jenkins shared library
5+
(https://www.jenkins.io/doc/book/pipeline/shared-libraries/).
6+
Jenkins shared library convention dictates that importable modules must be stored
7+
in the 'vars' folder.
8+
9+
Jenkins shared libraries can be configured in the Jenkins UI:
10+
* Manage Jenkins
11+
* Configure System
12+
* Global Pipeline Libraries section
13+
* Library subsection
14+
* Name: The Name for the lib
15+
* Version: The branch you want to use. Throws an error
16+
for nonexistent branches.
17+
* Project Repository: Url to the shared lib
18+
* Credentials: SSH key to access the repo
19+
20+
Note that updating the shared repo will take affect on the next pipeline invocation.
21+
*/
22+
23+
// Load the get_vbu_version function from vivarium_build_utils/bootstrap/
24+
// (the directory to load from is defined in the Jenkins shared library configuration)
25+
@Library("get_vbu_version@main") _
26+
27+
// Load the full vivarium_build_utils library at the expected version
28+
library("vivarium_build_utils@${get_vbu_version()}")
29+
230
reusable_pipeline(
3-
test_types: ["all-tests"],
31+
scheduled_branches: [
32+
// Add additional branches for cron jobs below.
33+
// Note: Newly added branches will require a manual trigger of the first build
34+
// for subsequent scheduled builds to run.
35+
"main"
36+
],
437
skip_doc_build: true,
538
upstream_repos: ["vivarium", "vivarium_inputs", "vivarium_public_health", "vivarium_cluster_tools", "gbd_mapping", "layered_config_tree"],
639
run_mypy: false,

‎Makefile‎

Lines changed: 106 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,116 @@
11
# Check if we're running in Jenkins
22
ifdef JENKINS_URL
3-
# Files are already in workspace from shared library
3+
# Files are already in workspace from shared library
44
MAKE_INCLUDES := .
55
else
6-
# For local dev, search in parent directory
7-
MAKE_INCLUDES := ../vivarium_build_utils/resources/makefiles
6+
# For local dev, use the installed vivarium_build_utils package if it exists
7+
# First, check if we can import vivarium_build_utils and assign 'yes' or 'no'.
8+
# We do this by importing the package in python and redirecting stderr to the null device.
9+
# If the import is successful (&&), it will print 'yes', otherwise (||) it will print 'no'.
10+
VIVARIUM_BUILD_UTILS_AVAILABLE := $(shell python -c "import vivarium_build_utils" 2>/dev/null && echo "yes" || echo "no")
11+
# If vivarium_build_utils is available, get the makefiles path or else set it to empty
12+
ifeq ($(VIVARIUM_BUILD_UTILS_AVAILABLE),yes)
13+
MAKE_INCLUDES := $(shell python -c "from vivarium_build_utils.resources import get_makefiles_path; print(get_makefiles_path())")
14+
else
15+
MAKE_INCLUDES :=
16+
endif
817
endif
918

10-
PACKAGE_NAME = vivarium_gates_nutrition_optimization_child
19+
# Set the package name as the last part of this file's parent directory path
20+
PACKAGE_NAME = $(notdir $(CURDIR))
1121

22+
# Helper function for validating enum arguments
23+
validate_arg = $(if $(filter-out $(2),$(1)),$(error Error: '$(3)' must be one of: $(2), got '$(1)'))
24+
25+
ifneq ($(MAKE_INCLUDES),) # not empty
1226
# Include makefiles from vivarium_build_utils
1327
include $(MAKE_INCLUDES)/base.mk
1428
include $(MAKE_INCLUDES)/test.mk
29+
else # empty
30+
# Use this help message (since the vivarium_build_utils version is not available)
31+
help:
32+
@echo
33+
@echo "For Make's standard help, run 'make --help'."
34+
@echo
35+
@echo "Most of our Makefile targets are provided by the vivarium_build_utils"
36+
@echo "package. To access them, you need to create a development environment first."
37+
@echo
38+
@echo "make build-env"
39+
@echo
40+
@echo "USAGE:"
41+
@echo " make build-env [type=<environment type>] [name=<environment name>] [py=<python version>] [include_timestamp=<yes|no>] [lfs=<yes|no>]"
42+
@echo
43+
@echo "ARGUMENTS:"
44+
@echo " type [optional]"
45+
@echo " Type of conda environment. Either 'simulation' (default) or 'artifact'"
46+
@echo " name [optional]"
47+
@echo " Name of the conda environment to create (defaults to <PACKAGE_NAME>_<TYPE>)"
48+
@echo " include_timestamp [optional]"
49+
@echo " Whether to append a timestamp to the environment name. Either 'yes' or 'no' (default)"
50+
@echo " lfs [optional]"
51+
@echo " Whether to install git-lfs in the environment. Either 'yes' or 'no' (default)"
52+
@echo " py [optional]"
53+
@echo " Python version (defaults to latest supported)"
54+
@echo
55+
@echo "After creating the environment:"
56+
@echo " 1. Activate it: 'conda activate <environment_name>'"
57+
@echo " 2. Run 'make help' again to see all newly available targets"
58+
@echo
59+
endif
60+
61+
build-env: # Create a new environment with installed packages
62+
# Validate arguments - exit if unsupported arguments are passed
63+
@allowed="type name lfs py include_timestamp"; \
64+
for arg in $(filter-out build-env,$(MAKECMDGOALS)) $(MAKEFLAGS); do \
65+
case $$arg in \
66+
*=*) \
67+
arg_name=$${arg%%=*}; \
68+
if ! echo " $$allowed " | grep -q " $$arg_name "; then \
69+
allowed_list=$$(echo $$allowed | sed 's/ /, /g'); \
70+
echo "Error: Invalid argument '$$arg_name'. Allowed arguments are: $$allowed_list" >&2; \
71+
exit 1; \
72+
fi \
73+
;; \
74+
esac; \
75+
done
76+
77+
# Handle arguments and set defaults
78+
# type
79+
@$(eval type ?= simulation)
80+
@$(call validate_arg,$(type),simulation artifact,type)
81+
# name
82+
@$(eval name ?= $(PACKAGE_NAME)_$(type))
83+
# timestamp
84+
@$(eval include_timestamp ?= no)
85+
@$(call validate_arg,$(include_timestamp),yes no,include_timestamp)
86+
@$(if $(filter yes,$(include_timestamp)),$(eval override name := $(name)_$(shell date +%Y%m%d_%H%M%S)),)
87+
# lfs
88+
@$(eval lfs ?= no)
89+
@$(call validate_arg,$(lfs),yes no,lfs)
90+
# python version
91+
@$(eval py ?= $(shell python -c "import json; versions = json.load(open('python_versions.json')); print(max(versions, key=lambda x: tuple(map(int, x.split('.')))))"))
92+
93+
conda create -n $(name) python=$(py) --yes
94+
# Bootstrap vivarium_build_utils into the new environment
95+
conda run -n $(name) pip install vivarium_build_utils
96+
# Install packages based on type
97+
@if [ "$(type)" = "simulation" ]; then \
98+
conda run -n $(name) make install ENV_REQS=dev; \
99+
conda install -n $(name) redis -c anaconda -y; \
100+
elif [ "$(type)" = "artifact" ]; then \
101+
conda run -n $(name) make install ENV_REQS=data; \
102+
fi
103+
@if [ "$(lfs)" = "yes" ]; then \
104+
conda run -n $(name) conda install -c conda-forge git-lfs --yes; \
105+
conda run -n $(name) git lfs install; \
106+
fi
107+
108+
@echo
109+
@echo "Finished building environment"
110+
@echo " name: $(name)"
111+
@echo " type: $(type)"
112+
@echo " git-lfs installed: $(lfs)"
113+
@echo " python version: $(py)"
114+
@echo
115+
@echo "Don't forget to activate it with: 'conda activate $(name)'"
116+
@echo

‎setup.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
long_description = f.read()
4545

4646
install_requirements = [
47+
"vivarium_build_utils>2.0.3,<3.0.0",
4748
"gbd_mapping>=4.0.0",
4849
"vivarium>=3.0.0, <4.0.0",
4950
"vivarium_public_health>=3.0.0, <4.0.0",

0 commit comments

Comments
 (0)