Skip to content

Commit dbfc8cc

Browse files
nkovela1sampathwebsup3rgiu
authored
Fetch Kokoro changes from master to kaggle (#2227)
* Adds Kokoro GPU Tests (#2224) * Adds Kokoro tests * Add Kokoro Tests * Update random_cutout.py to be a subclass of VectorizedBaseImageAugmentationLayer (#2123) * Update random_cutout.py Make RandomCutout a subclass of VectorizedBaseImageAugmentationLayer * Fix RandomCutout for ragged inputs * Fix typo and style * Vectorized implementation of RandomColorDegeneration and Equalization preprocessing layers (#2214) * Vectorize RandomColorDegeneration * Vectorize Equalization * Vectorize Equalization * Fix Equalization for ragged input --------- Co-authored-by: Ramesh Sampath <[email protected]> Co-authored-by: SUPERGIU <[email protected]>
1 parent cc084f8 commit dbfc8cc

File tree

14 files changed

+490
-102
lines changed

14 files changed

+490
-102
lines changed

Diff for: ‎.kokoro/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
CI to run on PR and merge to Master.

Diff for: ‎.kokoro/github/ubuntu/gpu/build.sh

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
set -e
2+
set -x
3+
4+
cd "${KOKORO_ROOT}/"
5+
6+
sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1
7+
8+
PYTHON_BINARY="/usr/bin/python3.9"
9+
10+
"${PYTHON_BINARY}" -m venv venv
11+
source venv/bin/activate
12+
# Check the python version
13+
python --version
14+
python3 --version
15+
16+
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:"
17+
# Check cuda
18+
nvidia-smi
19+
nvcc --version
20+
21+
cd "src/github/keras-cv"
22+
pip install -U pip setuptools
23+
24+
if [ "${KERAS2:-0}" == "1" ]
25+
then
26+
echo "Keras2 detected."
27+
pip install -r requirements-common.txt --progress-bar off
28+
pip install tensorflow~=2.14
29+
pip install --extra-index-url https://download.pytorch.org/whl/cpu torch==2.1.0+cpu
30+
pip install torchvision~=0.16.0
31+
pip install "jax[cpu]"
32+
33+
elif [ "$KERAS_BACKEND" == "tensorflow" ]
34+
then
35+
echo "TensorFlow backend detected."
36+
pip install -r requirements-tensorflow-cuda.txt --progress-bar off
37+
38+
elif [ "$KERAS_BACKEND" == "jax" ]
39+
then
40+
echo "JAX backend detected."
41+
pip install -r requirements-jax-cuda.txt --progress-bar off
42+
43+
elif [ "$KERAS_BACKEND" == "torch" ]
44+
then
45+
echo "PyTorch backend detected."
46+
pip install -r requirements-torch-cuda.txt --progress-bar off
47+
fi
48+
49+
pip install --no-deps -e "." --progress-bar off
50+
51+
# Run Extra Large Tests for Continuous builds
52+
if [ "${RUN_XLARGE:-0}" == "1" ]
53+
then
54+
pytest --check_gpu --run_large --run_extra_large --durations 0 \
55+
keras_cv/bounding_box \
56+
keras_cv/callbacks \
57+
keras_cv/losses \
58+
keras_cv/layers/object_detection \
59+
keras_cv/layers/preprocessing \
60+
keras_cv/models/backbones \
61+
keras_cv/models/classification \
62+
keras_cv/models/object_detection/retinanet \
63+
keras_cv/models/object_detection/yolo_v8 \
64+
keras_cv/models/object_detection_3d \
65+
keras_cv/models/segmentation \
66+
keras_cv/models/stable_diffusion \
67+
--cov=keras-cv
68+
else
69+
pytest --check_gpu --run_large --durations 0 \
70+
keras_cv/bounding_box \
71+
keras_cv/callbacks \
72+
keras_cv/losses \
73+
keras_cv/layers/object_detection \
74+
keras_cv/layers/preprocessing \
75+
keras_cv/models/backbones \
76+
keras_cv/models/classification \
77+
keras_cv/models/object_detection/retinanet \
78+
keras_cv/models/object_detection/yolo_v8 \
79+
keras_cv/models/object_detection_3d \
80+
keras_cv/models/segmentation \
81+
keras_cv/models/stable_diffusion \
82+
--cov=keras-cv
83+
fi

Diff for: ‎.kokoro/github/ubuntu/gpu/jax/continuous.cfg

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
build_file: "keras-cv/.kokoro/github/ubuntu/gpu/build.sh"
2+
3+
action {
4+
define_artifacts {
5+
regex: "**/sponge_log.log"
6+
regex: "**/sponge_log.xml"
7+
}
8+
}
9+
10+
env_vars: {
11+
key: "KERAS_BACKEND"
12+
value: "jax"
13+
}
14+
15+
env_vars: {
16+
key: "RUN_XLARGE"
17+
value: "1"
18+
}

Diff for: ‎.kokoro/github/ubuntu/gpu/jax/presubmit.cfg

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
build_file: "keras-cv/.kokoro/github/ubuntu/gpu/build.sh"
2+
3+
action {
4+
define_artifacts {
5+
regex: "**/sponge_log.log"
6+
regex: "**/sponge_log.xml"
7+
}
8+
}
9+
10+
env_vars: {
11+
key: "KERAS_BACKEND"
12+
value: "jax"
13+
}
14+
15+
# Set timeout to 60 mins from default 180 mins
16+
timeout_mins: 60

Diff for: ‎.kokoro/github/ubuntu/gpu/keras2/continuous.cfg

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
build_file: "keras-cv/.kokoro/github/ubuntu/gpu/build.sh"
2+
3+
action {
4+
define_artifacts {
5+
regex: "**/sponge_log.log"
6+
regex: "**/sponge_log.xml"
7+
}
8+
}
9+
10+
env_vars: {
11+
key: "KERAS2"
12+
value: "1"
13+
}
14+
15+
env_vars: {
16+
key: "RUN_XLARGE"
17+
value: "1"
18+
}

Diff for: ‎.kokoro/github/ubuntu/gpu/keras2/presubmit.cfg

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
build_file: "keras-cv/.kokoro/github/ubuntu/gpu/build.sh"
2+
3+
action {
4+
define_artifacts {
5+
regex: "**/sponge_log.log"
6+
regex: "**/sponge_log.xml"
7+
}
8+
}
9+
10+
env_vars: {
11+
key: "KERAS2"
12+
value: "1"
13+
}
14+
15+
# Set timeout to 60 mins from default 180 mins
16+
timeout_mins: 60

Diff for: ‎.kokoro/github/ubuntu/gpu/tensorflow/continuous.cfg

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
build_file: "keras-cv/.kokoro/github/ubuntu/gpu/build.sh"
2+
3+
action {
4+
define_artifacts {
5+
regex: "**/sponge_log.log"
6+
regex: "**/sponge_log.xml"
7+
}
8+
}
9+
10+
env_vars: {
11+
key: "KERAS_BACKEND"
12+
value: "tensorflow"
13+
}
14+
15+
env_vars: {
16+
key: "RUN_XLARGE"
17+
value: "1"
18+
}

Diff for: ‎.kokoro/github/ubuntu/gpu/tensorflow/presubmit.cfg

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
build_file: "keras-cv/.kokoro/github/ubuntu/gpu/build.sh"
2+
3+
action {
4+
define_artifacts {
5+
regex: "**/sponge_log.log"
6+
regex: "**/sponge_log.xml"
7+
}
8+
}
9+
10+
env_vars: {
11+
key: "KERAS_BACKEND"
12+
value: "tensorflow"
13+
}
14+
15+
# Set timeout to 60 mins from default 180 mins
16+
timeout_mins: 60

Diff for: ‎.kokoro/github/ubuntu/gpu/torch/continuous.cfg

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
build_file: "keras-cv/.kokoro/github/ubuntu/gpu/build.sh"
2+
3+
action {
4+
define_artifacts {
5+
regex: "**/sponge_log.log"
6+
regex: "**/sponge_log.xml"
7+
}
8+
}
9+
10+
env_vars: {
11+
key: "KERAS_BACKEND"
12+
value: "torch"
13+
}
14+
15+
env_vars: {
16+
key: "RUN_XLARGE"
17+
value: "1"
18+
}

Diff for: ‎.kokoro/github/ubuntu/gpu/torch/presubmit.cfg

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
build_file: "keras-cv/.kokoro/github/ubuntu/gpu/build.sh"
2+
3+
action {
4+
define_artifacts {
5+
regex: "**/sponge_log.log"
6+
regex: "**/sponge_log.xml"
7+
}
8+
}
9+
10+
env_vars: {
11+
key: "KERAS_BACKEND"
12+
value: "torch"
13+
}
14+
15+
# Set timeout to 60 mins from default 180 mins
16+
timeout_mins: 60

Diff for: ‎keras_cv/conftest.py

+27
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import tensorflow as tf
1818
from packaging import version
1919

20+
from keras_cv.backend import config as backend_config
2021
from keras_cv.backend.config import keras_3
2122

2223

@@ -33,9 +34,35 @@ def pytest_addoption(parser):
3334
default=False,
3435
help="run extra_large tests",
3536
)
37+
parser.addoption(
38+
"--check_gpu",
39+
action="store_true",
40+
default=False,
41+
help="fail if a gpu is not present",
42+
)
3643

3744

3845
def pytest_configure(config):
46+
# Verify that device has GPU and detected by backend
47+
if config.getoption("--check_gpu"):
48+
found_gpu = False
49+
backend = backend_config.backend()
50+
if backend == "jax":
51+
import jax
52+
53+
try:
54+
found_gpu = bool(jax.devices("gpu"))
55+
except RuntimeError:
56+
found_gpu = False
57+
elif backend == "tensorflow":
58+
found_gpu = bool(tf.config.list_logical_devices("GPU"))
59+
elif backend == "torch":
60+
import torch
61+
62+
found_gpu = bool(torch.cuda.device_count())
63+
if not found_gpu:
64+
pytest.fail(f"No GPUs discovered on the {backend} backend.")
65+
3966
config.addinivalue_line(
4067
"markers", "large: mark test as being slow or requiring a network"
4168
)

0 commit comments

Comments
 (0)