New env for workstations and workbench

.gitignore

@@ -149,3 +149,6 @@ dmypy.json
 cython_debug/
 
 *_kernel/
+
+# Dataset
+bert_dataset/

.pre-commit-config.yaml

@@ -26,24 +26,20 @@ repos:
 -   repo: https://github.com/nbQA-dev/nbQA
     rev: 1.9.1
     hooks:
-    -   id: nbqa-black
-        args: [-l80]
     -   id: nbqa-pyupgrade
         args: [--py36-plus]
-    -   id: nbqa-isort
-        args: [--profile=black, -l80]
 -   repo: https://github.com/PyCQA/isort
     rev: 7.0.0
     hooks:
     -   id: isort
         args: ["--profile", "black", "-l80"]
--   repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 25.12.0
-    hooks:
-    -   id: black
-        args: [-l 80]
-    -   id: black-jupyter
-        args: [-l 80]
+        types_or: [python, pyi, jupyter]
+- repo: https://github.com/psf/black
+  rev: 25.12.0
+  hooks:
+    - id: black
+      args: [-l 80]
+      types_or: [python, pyi, jupyter]
 -   repo: https://github.com/asottile/pyupgrade
     rev: v3.21.2
     hooks:

Makefile

@@ -1,4 +1,4 @@
-# Copyright 2021 Google LLC. All Rights Reserved.
+# Copyright 2025 Google LLC. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,51 +12,62 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-all: clean install
 
-kernels: \
- object_detection_kernel \
- pytorch_kfp_kernel \
- tf_privacy_kernel
+SHELL := /bin/bash
+export PATH := $(HOME)/.local/bin:$(PATH)
+SETUP_SCRIPT = ./scripts/setup_kernel.sh
+PYTHON_VERSION = 3.12
+
+PROJECTS = \
+	asl_core:asl_core:"ASL Core" \
+	asl_genai:asl_genai:"ASL Gen AI" \
+	asl_mlops:asl_mlops:"ASL MLOps"
+
+.PHONY: all install clean setup-apt setup-ide setup build-kernels
+
+all: setup build-kernels install-pre-commit
+
+install: all
 
-.PHONY: clean
 clean:
 	@find . -name '*.pyc' -delete
-	@find . -name '*.pytest_cache' -delete
 	@find . -name '__pycache__' -delete
-	@find . -name '*egg-info' -delete
-
-.PHONY: install
-install:
-	@pip install --user -U pip
-	@pip install --user "Cython<3"
-	@pip install --user -e .
-	@pip install --user --no-deps -r requirements-without-deps.txt
-	@./scripts/setup_on_jupyterlab.sh
-	@pre-commit install
-	@sudo apt-get update
-	@sudo apt-get -y install graphviz
-
-.PHONY: precommit
-precommit:
-	@pre-commit run --all-files
-
-.PHONY: object_detection_kernel
-object_detection_kernel:
-	./kernels/object_detection.sh
-
-.PHONY: pytorch_kfp_kernel
-pytorch_kfp_kernel:
-	./kernels/pytorch_kfp.sh
-
-.PHONY: tf_privacy_kernel
-tf_privacy_kernel:
-	./kernels/tf_privacy.sh
-
-.PHONY: keras_cv_kernel
-keras_cv_kernel:
-	./kernels/keras_cv.sh
-
-.PHONY: tests
-tests:
-	pytest tests/unit
+	@find . -type d -name '*.egg-info' -exec rm -rf {} +
+
+	@for config in $(PROJECTS); do \
+		IFS=: read -r dir name disp <<< "$$config"; \
+		bash $(SETUP_SCRIPT) $$dir $$name "$$disp" remove; \
+	done
+
+setup-apt:
+	$(eval TOKEN=$(shell curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token | sed 's/.*"access_token":"\([^"]*\)".*/\1/'))
+	@export CLOUDSDK_AUTH_ACCESS_TOKEN=$(TOKEN); \
+	export GOOGLE_APPLICATION_CREDENTIALS=""; \
+	sudo rm -f /etc/apt/sources.list.d/yarn.list /usr/share/keyrings/yarn.gpg; \
+	curl -fsSL https://dl.yarnpkg.com/debian/pubkey.gpg | sudo gpg --dearmor -o /usr/share/keyrings/yarn.gpg; \
+	echo "deb [signed-by=/usr/share/keyrings/yarn.gpg] https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list; \
+	sudo apt-get update
+
+setup-ide:
+	@if command -v code-oss-cloud-workstations > /dev/null; then \
+		echo "Installing Workstation extensions..."; \
+		code-oss-cloud-workstations --install-extension ms-python.python --force; \
+		code-oss-cloud-workstations --install-extension ms-toolsai.jupyter --force; \
+	fi
+
+setup: setup-apt setup-ide
+	sudo apt-get -y install graphviz
+	@command -v uv >/dev/null 2>&1 || curl -LsSf https://astral.sh/uv/install.sh | sh
+	uv python install $(PYTHON_VERSION)
+	uv tool install jupyter-core --with jupyter-client
+	@grep -q "local/bin" ~/.bashrc || echo 'export PATH="$$HOME/.local/bin:$$PATH"' >> ~/.bashrc
+
+build-kernels:
+	@for config in $(PROJECTS); do \
+		IFS=: read -r dir name disp <<< "$$config"; \
+		bash $(SETUP_SCRIPT) $$dir $$name "$$disp"; \
+	done
+
+install-pre-commit:
+	uv tool install pre-commit
+	uv tool run pre-commit install

README.md

@@ -1,60 +1,93 @@
 # Advanced Solutions Lab
 
 ## Overview
-This repository contains Jupyter notebooks meant to be run on Vertex AI. This is maintained by Google Cloud’s [Advanced Solutions Lab (ASL)](https://cloud.google.com/asl) team. [Vertex AI](https://cloud.google.com/vertex-ai) is the next generation AI Platform on the Google Cloud Platform.
-The material covered in this repo will take a software engineer with no exposure to machine learning to an advanced level.
-
-In particular, the notebooks in this repository cover
-- A wide range of model architectures (DNN, CNN, RNN, transformers, SNGP, etc.) targeting many data modalities (tabular, image, text, time-series) implemented mainly in Tensorflow and Keras.
-- Tools on Google Cloud’s Vertex AI for operationalizing Tensorflow, Scikit-learn and PyTorch models at scale (e.g. Vertex training, tuning, and serving, TFX and Kubeflow pipelines).
-
-If you are new to machine learning or Vertex AI start here:  [Introduction to TensorFlow](https://github.com/GoogleCloudPlatform/asl-ml-immersion/tree/master/notebooks/introduction_to_tensorflow)
+This repository contains AI and Machine Learning contents meant to be run on Google Cloud. This is maintained by Google Cloud’s [Advanced Solutions Lab (ASL)](https://cloud.google.com/asl) team.
 
+This repository contains 3 main modules to covers various AI/ML toipcs:
+- `asl_core`: A wide range of model architectures (DNN, CNN, RNN, transformers, SNGP, etc.) targeting many data modalities (tabular, image, text, time-series) implemented mainly in Tensorflow and Keras.
+- `asl_mlops`: Tools on Google Cloud’s Vertex AI for operationalizing Tensorflow, Scikit-learn and PyTorch models at scale (e.g. Vertex training, tuning, and serving, TFX and Kubeflow pipelines).
+- `asl_genai`: Generative AI and Agent System using Gemini and Agentic Frameworks like Google ADK.
 
 ## Repository Structure
-All notebooks are in the notebooks folder. This folder is organized by different ML topics. Each folder contains a `labs` and a `solutions` folder. Use the `labs` notebooks to test your coding skills by filling in TODOs and refer to the notebooks in the `solutions` folder to verify your code.
+Each module (`asl_core`, `asl_mlops`, `asl_genai`) has separate environment and materials, which are organized in each directory.
+
+All learning materials are in the contets folder. This folder is organized by different topics. Each folder contains a `labs` and a `solutions` folder. Use the `labs` notebooks to test your coding skills by filling in TODOs and refer to the notebooks in the `solutions` folder to verify your code.
 
 We have three main folders described below:
 
 ```
-├── kernels - contains kernel scripts needed for certain notebooks in lab folder
-├── notebooks - contains labs and solutions notebook organized by topic
-│   ├── bigquery
-│   ├── building_production_ml_systems
-│   ├── docker_and_kubernetes
-│   ├── . . .
-├── scripts - contains setup scripts for enabling and setting up services on Vertex AI
+├── asl_core
+│   ├── notebooks - contains learning materials organized by topic
+│   │   ├── building_production_ml_systems
+│   │   │   ├── labs
+│   │   │   └── solutions
+│   │   ├── end-to-end-structured
+│   │   ├── image_models
+│   │   ├── ...
+│   ├── kernels - contains kernel scripts needed for certain notebooks
+│   ├── scaffolds - contains sample code to accelerate AI/ML projects
+│   ├── requirements.txt - dependencies for this module
+├── asl_mlops
+│   ├── ...
+├── asl_genai
+│   ├── ...
+├── ...
 ```
 
-For a more detailed breakdown of the notebooks in this repo, please refer to this [readme](https://github.com/GoogleCloudPlatform/asl-ml-immersion/blob/master/notebooks/README.md).
-
-
-## Environment Setup (Vertex AI)
-
-First, open [CloudShell](https://cloud.google.com/shell) and run the following instructions:
+## Environment Setup
+### Step 1. Run the Setup Script on Cloud Shell
+This repository is tested on Vertex AI Workbench and Cloud Workstations. To begin, run the setup script in [Cloud Shell](https://shell.cloud.google.com) to configure essential project infrastructure (APIs, IAM, Buckets).
 
+Run the setup script in [Cloud Shell](https://shell.cloud.google.com) to provision your environment.
 ```bash
 git clone https://github.com/GoogleCloudPlatform/asl-ml-immersion.git
 cd asl-ml-immersion
-./scripts/setup_on_cloudshell.sh
+bash scripts/setup_env.sh
 ```
 
-Second, follow the instruction of [the official documentation](https://cloud.google.com/vertex-ai/docs/workbench/instances/create-console-quickstart) to set up a JupyterLab instance on [Vertex AI Workbench Instance](https://cloud.google.com/vertex-ai/docs/workbench/instances/introduction).
+You will be prompted to select the environment to set up:
+*   **1) Vertex AI Workbench:** Setup Vertex AI Workbench.
+*   **2) Cloud Workstations:** Setup Cloud Workstations.
+*   **3) Setup both:** Setup both environments.
+*   **4) Skip:** Setup project infrastructure (APIs, IAM, Buckets) only.
+
+By selecting the option 1-3, you can automatically setup the environment, or you can select 4 and manually set up the environment following the official documentation:
+*   **Vertex AI Workbench:** [Create a user-managed notebook instance](https://cloud.google.com/vertex-ai/docs/workbench/instances/create-console-quickstart)
+*   **Cloud Workstations:** [Create a workstation](https://docs.cloud.google.com/workstations/docs/create-workstation)
 
-**Note:** Accelerators (GPU/TPU) are not required in most of the labs, but some notebooks recommend using them.
+Next, you will be asked if you want to attach a GPU (Nvidia T4) to the environment. Select `y` or `n` depending on your preference.
 
-After creating a Vertex Workbench Instance, open the terminal *in your JupyterLab instance* and run the following commands:
+**Note:** Accelerators (GPU/TPU) are not required in most of the notebooks, but some notebooks recommend using them.
+
+### Step 2. Build the Environemnt
+Once your environment is running, open it. Then, run the commands below in Terminal **inside the environment** to clone this repository, and build the environemnt (venvs and jupyter kernels).
 
 ```bash
 git clone https://github.com/GoogleCloudPlatform/asl-ml-immersion.git
 cd asl-ml-immersion
-export PATH=$PATH:~/.local/bin
-make install
+make
 ```
 
+On Cloud Workstations, click `Open Folder` -> `asl-ml-immersion` to open the repository window. If the folder is already opend, `Command + Shift + P` and type `Developer: Reload Window` to reflect the changes.
+
+## Using the Environment
+### Running a notebook
+After the setup above, you can open a Jupyter notebook file, and execute on a module kernel (`ASL Core`, `ASL MLOps`, or `ASL Agent`). <br>
+If a correct kernel is not pre-selected, click `Select Kernel` and select a correct one.
+
+**On Cloud Workstations**, you can find a kernel under `Select Kernel` -> `Jupyter Kernels`. <br>
+If you can't find `Jupyter Kernels`, click `Python Environment` -> `<- (Left Arrow)` to reload the environment.
+
 **Note**: Some notebooks might require additional setup, please refer to the instructions in specific notebooks.
 
-After running these commands, you can open and execute a notebook on the base "Python 3" kernel.
+### Running a command on Terminal
+When running a command from the terminal, make sure to activate a venv for a specific environment.
+
+E.g. (under asl-ml-immersion directory),
+```bash
+source ./asl_genai/.venv/bin/activate
+adk web ./asl_genai/notebooks/vertex_genai/solutions/adk_agents
+```
 
 ## Contributions
 Currently, only Googlers can contribute to this repo. See [CONTRIBUTING.md](https://github.com/GoogleCloudPlatform/asl-ml-immersion/blob/master/CONTRIBUTING.md) for more details on the contribution workflow.

notebooks/building_production_ml_systems/labs/1_training_at_scale_vertex.ipynb → asl_core/notebooks/building_production_ml_systems/labs/1_training_at_scale_vertex.ipynb

@@ -46,6 +46,7 @@
     "%env PATH=/home/jupyter/.local/bin:{PATH}\n",
     "\n",
     "%load_ext tensorboard\n",
+    "%load_ext google.cloud.bigquery\n",
     "warnings.filterwarnings(\"ignore\")\n",
     "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"2\""
    ]
@@ -1240,9 +1241,9 @@
    "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m132"
   },
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel) (Local)",
+   "display_name": "ASL Core",
    "language": "python",
-   "name": "conda-base-py"
+   "name": "asl_core"
   },
   "language_info": {
    "codemirror_mode": {

notebooks/building_production_ml_systems/labs/2_hyperparameter_tuning_vertex.ipynb → asl_core/notebooks/building_production_ml_systems/labs/2_hyperparameter_tuning_vertex.ipynb

@@ -884,9 +884,9 @@
    "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m131"
   },
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel) (Local)",
+   "display_name": "ASL Core",
    "language": "python",
-   "name": "conda-base-py"
+   "name": "asl_core"
   },
   "language_info": {
    "codemirror_mode": {

notebooks/building_production_ml_systems/solutions/1_training_at_scale_vertex.ipynb → asl_core/notebooks/building_production_ml_systems/solutions/1_training_at_scale_vertex.ipynb

@@ -48,6 +48,7 @@
     "%env PATH=/home/jupyter/.local/bin:{PATH}\n",
     "\n",
     "%load_ext tensorboard\n",
+    "%load_ext google.cloud.bigquery\n",
     "warnings.filterwarnings(\"ignore\")\n",
     "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"2\""
    ]
@@ -1244,9 +1245,9 @@
    "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m132"
   },
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel) (Local)",
+   "display_name": "ASL Core",
    "language": "python",
-   "name": "conda-base-py"
+   "name": "asl_core"
   },
   "language_info": {
    "codemirror_mode": {

notebooks/building_production_ml_systems/solutions/2_hyperparameter_tuning_vertex.ipynb → asl_core/notebooks/building_production_ml_systems/solutions/2_hyperparameter_tuning_vertex.ipynb

@@ -873,9 +873,9 @@
    "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m131"
   },
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel) (Local)",
+   "display_name": "ASL Core",
    "language": "python",
-   "name": "conda-base-py"
+   "name": "asl_core"
   },
   "language_info": {
    "codemirror_mode": {

notebooks/end-to-end-structured/labs/1a_explore_data_babyweight.ipynb → asl_core/notebooks/end-to-end-structured/labs/1a_explore_data_babyweight.ipynb

@@ -422,9 +422,9 @@
    "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m121"
   },
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel) (Local)",
+   "display_name": "ASL Core",
    "language": "python",
-   "name": "conda-base-py"
+   "name": "asl_core"
   },
   "language_info": {
    "codemirror_mode": {

notebooks/end-to-end-structured/labs/1b_prepare_data_babyweight.ipynb → asl_core/notebooks/end-to-end-structured/labs/1b_prepare_data_babyweight.ipynb

@@ -51,7 +51,8 @@
    "source": [
     "import os\n",
     "\n",
-    "from google.cloud import bigquery"
+    "from google.cloud import bigquery\n",
+    "%load_ext google.cloud.bigquery"
    ]
   },
   {
@@ -67,7 +68,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "PROJECT = !gcloud config list --format 'value(core.project)'\n",
+    "PROJECT = !gcloud config get-value project\n",
     "PROJECT = PROJECT[0]\n",
     "BUCKET = PROJECT\n",
     "REGION = \"us-central1\"\n",
@@ -471,9 +472,9 @@
    "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m121"
   },
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel) (Local)",
+   "display_name": "ASL Core",
    "language": "python",
-   "name": "conda-base-py"
+   "name": "asl_core"
   },
   "language_info": {
    "codemirror_mode": {