diff --git a/.github/workflows/run-unittests-py310-py311.yml b/.github/workflows/run-unittests-py310-py311.yml index 1434439a0..42110cb59 100644 --- a/.github/workflows/run-unittests-py310-py311.yml +++ b/.github/workflows/run-unittests-py310-py311.yml @@ -89,7 +89,7 @@ jobs: timeout-minutes: 60 shell: bash env: - CONDA_PREFIX: /usr/share/miniconda + CONDA_PREFIX: /usr/share/miniforge run: | set -x # print commands that are executed diff --git a/.github/workflows/run-unittests-py39-cov-report.yml b/.github/workflows/run-unittests-py39-cov-report.yml index 7aa73f411..149aeda2e 100644 --- a/.github/workflows/run-unittests-py39-cov-report.yml +++ b/.github/workflows/run-unittests-py39-cov-report.yml @@ -83,7 +83,7 @@ jobs: timeout-minutes: 60 shell: bash env: - CONDA_PREFIX: /usr/share/miniconda + CONDA_PREFIX: /usr/share/miniforge run: | set -x # print commands that are executed diff --git a/README-development.md b/README-development.md index dd8b5d339..19343b541 100644 --- a/README-development.md +++ b/README-development.md @@ -3,7 +3,7 @@ The Oracle Accelerated Data Science (ADS) SDK used by data scientists and analysts for data exploration and experimental machine learning to democratize machine learning and -analytics by providing easy-to-use, +analytics by providing easy-to-use, performant, and user-friendly tools that brings together the best of data science practices. @@ -53,16 +53,16 @@ for development and testing purposes. ### Step 1: Create a conda environment -Install Anaconda from `https://repo.continuum.io/miniconda/` for the operating system you are using. +Install Miniforge from `https://github.com/conda-forge/miniforge` for the operating system you are using. In the terminal client, enter the following where is the name you want to call your environment, and set the Python version you want to use. ADS SDK requires Python >=3.8. ```bash - conda create -n python=3.8 anaconda + conda create -n -c conda-forge python=3.8 ``` -This installs the Python version and all the associated anaconda packaged libraries at `path_to_your_anaconda_location/anaconda/envs/` +This installs the Python version and associated packages from conda-forge at `path_to_your_conda_location/envs/` ### Step 2: Activate your environment @@ -168,14 +168,14 @@ These are the steps to run the AQUA (AI Quick Actions) API Server for developmen ### Step 1: Requirements + Complete the [Getting Started](#getting-started) Section above, create a conda environment with python >3.9 or 3.10 -+ install any Rest API Client in your IDE (Thunder Client on [vscode](https://marketplace.visualstudio.com/items?itemName=rangav.vscode-thunder-client) or Postman) ++ install any Rest API Client in your IDE (Thunder Client on [vscode](https://marketplace.visualstudio.com/items?itemName=rangav.vscode-thunder-client) or Postman) + Activate the conda environment from the Getting Started Section and run ``` pip install -r test-requirements.txt ``` -### Step 2: Create local .env files +### Step 2: Create local .env files Running the local JupyterLab server requires setting OCI authentication, proxy, and OCI namespace parameters. Adapt this .env file with your specific OCI profile and OCIDs to set these variables. ``` @@ -186,17 +186,17 @@ HTTP_PROXY="" HTTPS_PROXY="" OCI_ODSC_SERVICE_ENDPOINT="your_service_endpoint" AQUA_SERVICE_MODELS_BUCKET="service-managed-models" -AQUA_TELEMETRY_BUCKET_NS="" -PROJECT_COMPARTMENT_OCID="ocid1.compartment.oc1." +AQUA_TELEMETRY_BUCKET_NS="" +PROJECT_COMPARTMENT_OCID="ocid1.compartment.oc1." OCI_CONFIG_PROFILE="your_oci_profile_name" OCI_IAM_TYPE="security_token" # no modification needed if using token-based auth TENANCY_OCID="ocid1.tenancy.oc1." AQUA_JOB_SUBNET_ID="ocid1.subnet.oc1." -ODSC_MODEL_COMPARTMENT_OCID="ocid1.compartment.oc1." +ODSC_MODEL_COMPARTMENT_OCID="ocid1.compartment.oc1." PROJECT_OCID="ocid1.datascienceproject.oc1." ``` -### Step 3: Add the run_ads.sh script in the ADS Repository +### Step 3: Add the run_ads.sh script in the ADS Repository + add the shell script below and .env file from step 2 to your local directory of the cloned ADS Repository + Run ```chmox +x run_ads.sh``` after you create this script. ``` @@ -220,10 +220,10 @@ fi "$@" ``` -### Step 4: Run the JupyterLab Server +### Step 4: Run the JupyterLab Server We can start the JupyterLab server using the following command -``` +``` ./run_ads.sh jupyter lab --no-browser --ServerApp.disable_check_xsrf=True ``` + run ```pkill jupyter-lab``` to kill the JupyterLab server and re-run server to reflect changes made locally to the AQUA API @@ -244,7 +244,7 @@ GET http://localhost:8888/aqua/deployments # calling the deployment_handler.py Handlers can be found [here](https://github.com/oracle/accelerated-data-science/tree/21ba00b95aef8581991fee6c7d558e2f2b1680ac/ads/aqua/extension). ### Step 5: Run the unit tests for the AQUA API -All the unit tests can be found [here](https://github.com/oracle/accelerated-data-science/tree/main/tests/unitary/with_extras/aqua). +All the unit tests can be found [here](https://github.com/oracle/accelerated-data-science/tree/main/tests/unitary/with_extras/aqua). The following commands detail how the unit tests can be run. ``` # Run all tests in AQUA project diff --git a/ads/opctl/distributed/cmds.py b/ads/opctl/distributed/cmds.py index 0f9ef1b7e..6ed745a24 100644 --- a/ads/opctl/distributed/cmds.py +++ b/ads/opctl/distributed/cmds.py @@ -1,7 +1,6 @@ #!/usr/bin/env python -# -*- coding: utf-8; -*- -# Copyright (c) 2022, 2023 Oracle and/or its affiliates. +# Copyright (c) 2022, 2026 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ import json @@ -198,23 +197,16 @@ def update_ini(tag, registry, dockerfile, source_folder, config, nobuild): config.get("spec", {}).get("cluster", {}).get("spec", {}).get("image") ) - if tag is not None: + if tag is not None or img_name.startswith("@"): tag1 = tag + elif len(img_name.rsplit(":", 1)) == 1: + tag1 = "latest" else: - if img_name.startswith("@"): - tag1 = tag - else: - if len(img_name.rsplit(":", 1)) == 1: - tag1 = "latest" - else: - tag1 = img_name.rsplit(":", 1)[1] - if registry is not None: + tag1 = img_name.rsplit(":", 1)[1] + if registry is not None or img_name.startswith("@"): registry1 = registry else: - if img_name.startswith("@"): - registry1 = registry - else: - registry1 = img_name.rsplit(":", 1)[0] + registry1 = img_name.rsplit(":", 1)[0] if os.path.isfile(ini_file): ini.read(ini_file) @@ -238,11 +230,10 @@ def update_ini(tag, registry, dockerfile, source_folder, config, nobuild): raise ValueError("registry arg is missing") if dockerfile is not None: ini.set("main", "dockerfile", dockerfile) + elif nobuild: + ini.set("main", "dockerfile", "DUMMY_PATH") else: - if nobuild: - ini.set("main", "dockerfile", "DUMMY_PATH") - else: - raise ValueError("dockerfile arg is missing") + raise ValueError("dockerfile arg is missing") if source_folder is not None: ini.set("main", "source_folder", source_folder) else: @@ -411,7 +402,7 @@ def run_cmd(cmd): """ proc = run_command(cmd) if proc.returncode != 0: - raise RuntimeError(f"Docker build failed.") + raise RuntimeError("Docker build failed.") return 1 @@ -444,14 +435,14 @@ def horovod_cmd(code_mount, oci_key_mount, config): "OCI_IAM_TYPE=api_key", "--rm", "--entrypoint", - "/miniconda/envs/env/bin/horovodrun", + "/miniforge/envs/env/bin/horovodrun", config["spec"]["cluster"]["spec"]["image"], "--gloo", "-np", "2", "-H", "localhost:2", - "/miniconda/envs/env/bin/python", + "/miniforge/envs/env/bin/python", config["spec"]["runtime"]["spec"]["entryPoint"], ] return command @@ -534,7 +525,7 @@ def dask_cmd(code_mount, oci_key_mount, config): config["spec"]["cluster"]["spec"]["image"], "-c", "(nohup dask-scheduler >scheduler.log &) && (nohup dask-worker localhost:8786 >worker.log &) && " - "/miniconda/envs/daskenv/bin/python " + "/miniforge/envs/daskenv/bin/python " + config["spec"]["runtime"]["spec"]["entryPoint"], ] return command @@ -592,7 +583,7 @@ def tensorflow_cmd(code_mount, oci_key_mount, config): 'TF_CONFIG={"cluster": {"worker": ["localhost:12345"]}, "task": {"type": "worker", "index": 0}}', "--rm", "--entrypoint", - "/miniconda/bin/python", + "/miniforge/bin/python", config["spec"]["cluster"]["spec"]["image"], config["spec"]["runtime"]["spec"]["entryPoint"], ] @@ -632,7 +623,7 @@ def local_run(config, ini): elif config["spec"]["cluster"]["kind"].lower() == "tensorflow": command = tensorflow_cmd(code_mount, oci_key_mount, config) else: - raise RuntimeError(f"Framework not supported") + raise RuntimeError("Framework not supported") try: command += [str(arg) for arg in config["spec"]["runtime"]["spec"]["args"]] except KeyError: @@ -640,7 +631,7 @@ def local_run(config, ini): print("Running: ", " ".join(command)) proc = run_command(command) if proc.returncode != 0: - raise RuntimeError(f"Failed to run local") + raise RuntimeError("Failed to run local") return 1 @@ -683,22 +674,21 @@ def verify_and_publish_image(nopush, config): """ if not nopush: publish_image_cmd(config["spec"]["cluster"]["spec"]["image"]) - else: - if not verify_image(config["spec"]["cluster"]["spec"]["image"]): - print( - "\u26A0 Image: " - + config["spec"]["cluster"]["spec"]["image"] - + " does not exist in registry" + elif not verify_image(config["spec"]["cluster"]["spec"]["image"]): + print( + "\u26a0 Image: " + + config["spec"]["cluster"]["spec"]["image"] + + " does not exist in registry" + ) + print("In order to push the image to registry enter Y else N ") + inp = input("[Y/N]\n") + if inp == "Y": + print("\u2705 pushing image to registry") + publish_image_cmd(config["spec"]["cluster"]["spec"]["image"]) + else: + raise RuntimeError( + "Stopping the execution as image doesn't exist in OCI registry" ) - print("In order to push the image to registry enter Y else N ") - inp = input("[Y/N]\n") - if inp == "Y": - print("\u2705 pushing image to registry") - publish_image_cmd(config["spec"]["cluster"]["spec"]["image"]) - else: - raise RuntimeError( - "Stopping the execution as image doesn't exist in OCI registry" - ) return 1 diff --git a/ads/opctl/docker/Dockerfile.job b/ads/opctl/docker/Dockerfile.job index 55eb60baf..69bbe0065 100644 --- a/ads/opctl/docker/Dockerfile.job +++ b/ads/opctl/docker/Dockerfile.job @@ -62,10 +62,10 @@ RUN chown -R $DATASCIENCE_USER /opt USER $DATASCIENCE_USER WORKDIR /home/datascience -ARG MINICONDA_VER=23.5.2-0 -RUN wget -nv https://repo.anaconda.com/miniconda/Miniconda3-py38_${MINICONDA_VER}-Linux-x86_64.sh -O /home/datascience/Miniconda3.sh \ - && /bin/bash /home/datascience/Miniconda3.sh -f -b -p /opt/conda \ - && rm /home/datascience/Miniconda3.sh \ +ARG MINIFORGE_URL=https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh +RUN wget -nv ${MINIFORGE_URL} -O /home/datascience/Miniforge3.sh \ + && /bin/bash /home/datascience/Miniforge3.sh -f -b -p /opt/conda \ + && rm /home/datascience/Miniforge3.sh \ && /opt/conda/bin/conda clean -yaf WORKDIR / @@ -100,4 +100,4 @@ RUN rm -rf /tmp/* RUN mkdir -p /etc/datascience/operators -USER datascience \ No newline at end of file +USER datascience diff --git a/ads/opctl/docker/Dockerfile.job.arm b/ads/opctl/docker/Dockerfile.job.arm index f859d32a2..8e0f6adbd 100644 --- a/ads/opctl/docker/Dockerfile.job.arm +++ b/ads/opctl/docker/Dockerfile.job.arm @@ -1,7 +1,7 @@ # Copyright (c) 2023 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ -# Used OL8 because miniconda required a higher version of glibc that was unavoidable +# Used OL8 because miniforge required a higher version of glibc that was unavoidable FROM ghcr.io/oracle/oraclelinux:8-slim # Configure environment @@ -66,10 +66,10 @@ RUN chown -R $DATASCIENCE_USER /opt USER $DATASCIENCE_USER WORKDIR /home/datascience # Note in order to run sudo commands as a non root user, you must specify --credential yes if using qemu static to build the image -ARG MINICONDA_VER=23.5.2-0 -RUN wget -nv https://repo.anaconda.com/miniconda/Miniconda3-py38_${MINICONDA_VER}-Linux-aarch64.sh -O /home/datascience/Miniconda3.sh \ - && /bin/bash /home/datascience/Miniconda3.sh -f -b -p /opt/conda \ - && rm /home/datascience/Miniconda3.sh \ +ARG MINIFORGE_URL=https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh +RUN wget -nv ${MINIFORGE_URL} -O /home/datascience/Miniforge3.sh \ + && /bin/bash /home/datascience/Miniforge3.sh -f -b -p /opt/conda \ + && rm /home/datascience/Miniforge3.sh \ && /opt/conda/bin/conda clean -yaf WORKDIR / diff --git a/ads/opctl/docker/Dockerfile.job.gpu b/ads/opctl/docker/Dockerfile.job.gpu index 3514b9efb..170966049 100644 --- a/ads/opctl/docker/Dockerfile.job.gpu +++ b/ads/opctl/docker/Dockerfile.job.gpu @@ -107,10 +107,10 @@ RUN chown -R $DATASCIENCE_USER /opt USER $DATASCIENCE_USER WORKDIR /home/datascience -ARG MINICONDA_VER=23.5.2-0 -RUN wget -nv https://repo.anaconda.com/miniconda/Miniconda3-py38_${MINICONDA_VER}-Linux-x86_64.sh -O /home/datascience/Miniconda3.sh \ - && /bin/bash /home/datascience/Miniconda3.sh -f -b -p /opt/conda \ - && rm /home/datascience/Miniconda3.sh \ +ARG MINIFORGE_URL=https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh +RUN wget -nv ${MINIFORGE_URL} -O /home/datascience/Miniforge3.sh \ + && /bin/bash /home/datascience/Miniforge3.sh -f -b -p /opt/conda \ + && rm /home/datascience/Miniforge3.sh \ && /opt/conda/bin/conda clean -yaf WORKDIR / diff --git a/ads/opctl/docker/operator/Dockerfile b/ads/opctl/docker/operator/Dockerfile index 905b101a0..2045cc094 100644 --- a/ads/opctl/docker/operator/Dockerfile +++ b/ads/opctl/docker/operator/Dockerfile @@ -10,9 +10,9 @@ RUN \ rm -rf /var/cache/yum/* ########################### CONDA INSTALLATION ######################################## -RUN curl -L https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh >> miniconda.sh -RUN bash ./miniconda.sh -b -p /miniconda; rm ./miniconda.sh; -ENV PATH="/miniconda/bin:$PATH" +RUN curl -L https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh >> miniforge.sh +RUN bash ./miniforge.sh -b -p /miniforge; rm ./miniforge.sh; +ENV PATH="/miniforge/bin:$PATH" USER root @@ -23,7 +23,7 @@ ARG RND COPY ./${CONDA_ENV_FILE} /opt/env.yaml RUN conda install conda-forge::mamba && mamba env create -f /opt/env.yaml --name ${CONDA_ENV_NAME} && conda clean -afy -ENV PATH="/miniconda/envs/${CONDA_ENV_NAME}}/bin:$PATH" +ENV PATH="/miniforge/envs/${CONDA_ENV_NAME}}/bin:$PATH" RUN conda init bash && source ~/.bashrc && conda activate ${CONDA_ENV_NAME} diff --git a/ads/opctl/docker/operator/Dockerfile.gpu b/ads/opctl/docker/operator/Dockerfile.gpu index f74f924dd..7d2ae0b35 100644 --- a/ads/opctl/docker/operator/Dockerfile.gpu +++ b/ads/opctl/docker/operator/Dockerfile.gpu @@ -54,9 +54,9 @@ RUN CUDNN_DOWNLOAD_SUM=7eaec8039a2c30ab0bc758d303588767693def6bf49b22485a2c00bf2 ldconfig ########################### CONDA INSTALLATION ######################################## -RUN curl -L https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh >> miniconda.sh -RUN bash ./miniconda.sh -b -p /miniconda; rm ./miniconda.sh; -ENV PATH="/miniconda/bin:$PATH" +RUN curl -L https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh >> miniforge.sh +RUN bash ./miniforge.sh -b -p /miniforge; rm ./miniforge.sh; +ENV PATH="/miniforge/bin:$PATH" USER root @@ -67,7 +67,7 @@ ARG RND COPY ./${CONDA_ENV_FILE} /opt/env.yaml RUN conda install conda-forge::mamba && mamba env create -f /opt/env.yaml --name ${CONDA_ENV_NAME} && conda clean -afy -ENV PATH="/miniconda/envs/${CONDA_ENV_NAME}}/bin:$PATH" +ENV PATH="/miniforge/envs/${CONDA_ENV_NAME}}/bin:$PATH" RUN conda init bash && source ~/.bashrc && conda activate ${CONDA_ENV_NAME} diff --git a/docs/README.md b/docs/README.md index cff6eb1ac..c4fb33040 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,13 +4,13 @@ Setup Conda. -Download the latest Miniconda installer for your machine from here: https://docs.conda.io/en/latest/miniconda.html#latest-miniconda-installer-links +Download the latest Miniforge installer for your machine from here: https://github.com/conda-forge/miniforge For example for linux you can run the script below: ```bash -curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o Miniconda3-latest-Linux-x86_64.sh -bash Miniconda3-latest-Linux-x86_64.sh +curl https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh -o Miniforge3-Linux-x86_64.sh +bash Miniforge3-Linux-x86_64.sh ``` Create conda environment. @@ -57,4 +57,4 @@ zip -r ads-latest.zip docs_html/. ## Contribute -Now you can make updates to the docs and contributed via PRs. \ No newline at end of file +Now you can make updates to the docs and contributed via PRs. diff --git a/tests/unitary/with_extras/opctl/test_opctl_distributed_training.py b/tests/unitary/with_extras/opctl/test_opctl_distributed_training.py index a3934daef..5bb98da17 100644 --- a/tests/unitary/with_extras/opctl/test_opctl_distributed_training.py +++ b/tests/unitary/with_extras/opctl/test_opctl_distributed_training.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright (c) 2022, 2023 Oracle and/or its affiliates. +# Copyright (c) 2022, 2026 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ import builtins @@ -10,8 +10,8 @@ from io import StringIO from unittest import mock -from ads.opctl.distributed.common.abstract_cluster_provider import ClusterProvider from ads.opctl.cmds import _save_yaml +from ads.opctl.distributed.common.abstract_cluster_provider import ClusterProvider def yaml_content(): @@ -322,23 +322,23 @@ def yaml_content_one_node(): return content -from ads.opctl.config.yaml_parsers import YamlSpecParser -from ads.opctl.distributed.common.cluster_config_helper import ( - ClusterConfigToJobSpecConverter, -) +import yaml +from ads.opctl.config.yaml_parsers import YamlSpecParser from ads.opctl.distributed.cmds import ( - update_image, - increment_tag, + dask_cmd, get_cmd, - verify_image, horovod_cmd, + increment_tag, pytorch_cmd, - dask_cmd, tensorflow_cmd, update_config_image, + update_image, + verify_image, +) +from ads.opctl.distributed.common.cluster_config_helper import ( + ClusterConfigToJobSpecConverter, ) -import yaml def test_yaml_parsing(): @@ -623,8 +623,8 @@ def test_horovod_cmd_test(): assert ( " ".join(cmd) == "docker run -v /target/:/code/ -v /oci/:/home/keys/ --env OCI_IAM_TYPE=api_key --rm " - "--entrypoint /miniconda/envs/env/bin/horovodrun @default --gloo -np 2 -H localhost:2 " - "/miniconda/envs/env/bin/python printhello.py" + "--entrypoint /miniforge/envs/env/bin/horovodrun @default --gloo -np 2 -H localhost:2 " + "/miniforge/envs/env/bin/python printhello.py" ) @@ -639,7 +639,7 @@ def test_dask_cmd_test(): == "docker run -v /target/:/code/ -v /oci/:/home/keys/ --env OCI_IAM_TYPE=api_key --env " "SCHEDULER_IP=tcp://127.0.0.1 --rm --entrypoint /bin/sh @default -c (nohup dask-scheduler " ">scheduler.log &) && (nohup dask-worker localhost:8786 >worker.log &) && " - "/miniconda/envs/daskenv/bin/python printhello.py" + "/miniforge/envs/daskenv/bin/python printhello.py" ) @@ -676,7 +676,7 @@ def test_tensorflow_cmd_2_test(): assert ( " ".join(cmd) - == """docker run -v /target/:/code/ -v /oci/:/home/keys/ --env OCI_IAM_TYPE=api_key --env TF_CONFIG={"cluster": {"worker": ["localhost:12345"]}, "task": {"type": "worker", "index": 0}} --rm --entrypoint /miniconda/bin/python @default printhello.py""" + == """docker run -v /target/:/code/ -v /oci/:/home/keys/ --env OCI_IAM_TYPE=api_key --env TF_CONFIG={"cluster": {"worker": ["localhost:12345"]}, "task": {"type": "worker", "index": 0}} --rm --entrypoint /miniforge/bin/python @default printhello.py""" )