Skip to content

Commit ad8d32e

Browse files
authored
feat(genai/embeddings): migrate code retrieval and model tuning examples to genai (#14475)
* Migrate code retrieval example for embeddings to genai. * - Migrated model_tuning_example sample to modern aiplatform implementation of embedded model tuning. - Added missing python doc. - Updated requirements files. * - Migrated model_tuning_example to genai/embeddings. - created tests for migrated samples. * - Applied Gemini review code suggestions. * Removed unused attribute. * Testing changes. * Fixed docstring. * addressed comments on blank lines.
1 parent b380db7 commit ad8d32e

5 files changed

Lines changed: 170 additions & 3 deletions

File tree

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START aiplatform_genai_embedding_code_retrieval]
16+
import os
17+
18+
from google import genai
19+
20+
# TODO (Developer) set the following environment variables.
21+
PROJECT_ID = os.getenv("PROJECT_ID")
22+
LOCATION_ID = os.getenv("LOCATION_ID", "us-central1")
23+
MODEL_NAME = os.getenv("MODEL_NAME", "gemini-embedding-001")
24+
25+
QUERY_LINES = ["Retrieve a function that adds two numbers"]
26+
CODE_RETRIEVAL_QUERY = "CODE_RETRIEVAL_QUERY"
27+
RETRIEVAL_DOCUMENT = "RETRIEVAL_DOCUMENT"
28+
SOURCE_CODE = [
29+
"def func(a, b): return a + b",
30+
"def func(a, b): return a - b",
31+
"def func(a, b): return (a ** 2 + b ** 2) ** 0.5",
32+
]
33+
34+
35+
def embed_test() -> (
36+
tuple[genai.types.EmbedContentResponse, genai.types.EmbedContentResponse]
37+
):
38+
"""Generates embeddings for source code indexing and code search queries using the Gemini API.
39+
40+
Returns:
41+
tuple[genai.types.EmbedContentResponse, genai.types.EmbedContentResponse]: A tuple containing
42+
the final source code indexing response and search query embedding response.
43+
"""
44+
client = genai.Client(enterprise=True, project=PROJECT_ID, location=LOCATION_ID)
45+
46+
# Index Source Code
47+
for line in SOURCE_CODE:
48+
config = genai.types.EmbedContentConfig(task_type=RETRIEVAL_DOCUMENT)
49+
50+
index_response = client.models.embed_content(
51+
model=MODEL_NAME, contents=line, config=config
52+
)
53+
54+
print(
55+
f"Task: {RETRIEVAL_DOCUMENT} | "
56+
f"Vector length: {len(index_response.embeddings)} | "
57+
f"Preview: {index_response.embeddings[:3]}..."
58+
)
59+
60+
# Embed Search Prompts
61+
for line in QUERY_LINES:
62+
config = genai.types.EmbedContentConfig(task_type=CODE_RETRIEVAL_QUERY)
63+
64+
query_response = client.models.embed_content(
65+
model=MODEL_NAME, contents=line, config=config
66+
)
67+
68+
print(
69+
f"Task: {CODE_RETRIEVAL_QUERY} | "
70+
f"Vector length: {len(query_response.embeddings)} | "
71+
f"Preview: {query_response.embeddings[:3]}..."
72+
)
73+
74+
return index_response, query_response
75+
76+
77+
# [END aiplatform_genai_embedding_code_retrieval]
Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START aiplatform_genai_embedding_model_tuning]
16+
import os
17+
18+
from google.cloud import aiplatform
19+
20+
# TODO (Developer) set the following environment variables.
21+
PROJECT_ID = os.getenv("PROJECT_ID")
22+
LOCATION_ID = os.getenv("LOCATION_ID", "us-central1")
23+
MODEL_NAME = os.getenv("MODEL_NAME", "text-embedding-004")
24+
# A storage bucket: gs://your-bucket-name/embedding-tuning-output
25+
OUTPUT_URI = os.getenv("OUTPUT_DIR")
26+
27+
TRAIN_LABEL_PATH = (
28+
"gs://cloud-samples-data/ai-platform/embedding/goog-10k-2024/r11/train.tsv"
29+
)
30+
TEST_LABEL_PATH = (
31+
"gs://cloud-samples-data/ai-platform/embedding/goog-10k-2024/r11/test.tsv"
32+
)
33+
CORPUS_PATH = (
34+
"gs://cloud-samples-data/ai-platform/embedding/goog-10k-2024/r11/corpus.jsonl"
35+
)
36+
QUERIES_PATH = (
37+
"gs://cloud-samples-data/ai-platform/embedding/goog-10k-2024/r11/queries.jsonl"
38+
)
39+
40+
ACCELERATOR_TYPE = "NVIDIA_L4"
41+
42+
# Official Google Cloud KFP pipeline template URI for text embedding model tuning
43+
EMBEDDING_TUNING_PIPELINE_URI = "https://us-kfp.pkg.dev/ml-pipeline/llm-text-embedding/tune-text-embedding-model/v1.1.3"
44+
45+
46+
def tune_embedding_model() -> aiplatform.PipelineJob:
47+
"""Tune an embedding model using the specified parameters."""
48+
49+
aiplatform.init(project=PROJECT_ID, location=LOCATION_ID)
50+
51+
# Configure parameters expected by the embedding tuning pipeline template
52+
pipeline_parameters = {
53+
"base_model_version_id": MODEL_NAME,
54+
"corpus_path": CORPUS_PATH,
55+
"queries_path": QUERIES_PATH,
56+
"train_label_path": TRAIN_LABEL_PATH,
57+
"test_label_path": TEST_LABEL_PATH,
58+
"accelerator_type": ACCELERATOR_TYPE,
59+
}
60+
61+
# Instantiate the Vertex AI Pipeline job
62+
pipeline_job = aiplatform.PipelineJob(
63+
display_name="tune-text-embedding-model-job",
64+
template_path=EMBEDDING_TUNING_PIPELINE_URI,
65+
pipeline_root=OUTPUT_URI,
66+
parameter_values=pipeline_parameters,
67+
project=PROJECT_ID,
68+
location=LOCATION_ID,
69+
)
70+
71+
pipeline_job.submit()
72+
73+
print(f"Pipeline submitted successfully: {pipeline_job.resource_name}")
74+
75+
return pipeline_job
76+
77+
78+
# [END aiplatform_genai_embedding_model_tuning]
Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
google-api-core==2.24.0
2-
pytest==9.0.3; python_version >= "3.10"
1+
google-api-core==2.33.0
2+
pytest==9.1.1

genai/embeddings/requirements.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1-
google-genai==1.42.0
1+
google-genai==2.16.0
2+
google-cloud-aiplatform[pipelines]==1.163.0

genai/embeddings/test_embeddings_examples.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@
1818

1919
import os
2020

21+
import code_retrieval_example
2122
import embeddings_docretrieval_with_txt
23+
import model_tuning_example
2224

2325
os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True"
2426
os.environ["GOOGLE_CLOUD_LOCATION"] = "us-central1"
@@ -29,3 +31,12 @@
2931
def test_embeddings_docretrieval_with_txt() -> None:
3032
response = embeddings_docretrieval_with_txt.embed_content()
3133
assert response
34+
35+
36+
def test_code_retrieval_example() -> None:
37+
response = code_retrieval_example.embed_test()
38+
assert response
39+
40+
def test_model_tuning_example() -> None:
41+
response = model_tuning_example.tune_embedding_model()
42+
assert response

0 commit comments

Comments
 (0)