Skip to content

Commit 6dd1960

Browse files
authored
chore(bigframes): add specific build script for doctest to restrict execution (#16711)
> [!Warning] > This is a work in progress. Do not review until invited. Creates a build script: `packages/bigframes/scripts/run_doctest.sh`. This script checks if `bigframes` has any changes (or if it's a `continuous` build) before runnin' the `cleanup` `doctest` sessions. This will prevent it from running when other packages are modified and failing the build. Updates the Kokoro config: `.kokoro/presubmit/presubmit-doctest-bigframes.cfg` to set `TRAMPOLINE_BUILD_FILE` to point to the new script.
1 parent 81b7fad commit 6dd1960

7 files changed

Lines changed: 71 additions & 11 deletions

File tree

.kokoro/presubmit/presubmit-doctest-bigframes.cfg

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,10 @@ env_vars: {
99
env_vars: {
1010
key: "GOOGLE_CLOUD_PROJECT"
1111
value: "bigframes-testing"
12+
}
13+
14+
# Override the build file to only run for bigframes.
15+
env_vars: {
16+
key: "TRAMPOLINE_BUILD_FILE"
17+
value: "github/google-cloud-python/packages/bigframes/scripts/run_doctest.sh"
1218
}

packages/bigframes/bigframes/bigquery/_operations/struct.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ def struct(value: dataframe.DataFrame) -> series.Series:
4141
4242
>>> srs = series.Series([{"version": 1, "project": "pandas"}, {"version": 2, "project": "numpy"},])
4343
>>> df = srs.struct.explode()
44+
>>> df = df[["project", "version"]] # set the column order to ensure stable output for doctest
4445
>>> bbq.struct(df)
4546
0 {'project': 'pandas', 'version': 1}
4647
1 {'project': 'numpy', 'version': 2}

packages/bigframes/noxfile.py

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -401,9 +401,6 @@ def system_noextras(session: nox.sessions.Session):
401401
@nox.session(python="3.12")
402402
def doctest(session: nox.sessions.Session):
403403
"""Run the system test suite."""
404-
session.skip(
405-
"Temporary skip to enable a PR merge. Remove skip as part of closing https://github.com/googleapis/google-cloud-python/issues/16489"
406-
)
407404

408405
run_system(
409406
session=session,

packages/bigframes/scripts/manage_cloud_functions.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ def get_bigframes_functions(project, region):
6767
functions = GCF_CLIENT.list_functions(
6868
functions_v2.ListFunctionsRequest(parent=parent)
6969
)
70+
7071
# Filter bigframes created functions
7172
functions = [
7273
function
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#!/bin/bash
2+
set -eo pipefail
3+
4+
# Disable buffering, so that the logs stream through.
5+
export PYTHONUNBUFFERED=1
6+
7+
# Assume we are running from the repo root or we need to find it.
8+
# If this script is in packages/bigframes/scripts/run_doctest.sh,
9+
# then repo root is 3 levels up.
10+
export PROJECT_ROOT=$(realpath "$(dirname "${BASH_SOURCE[0]}")/../../..")
11+
cd "$PROJECT_ROOT"
12+
13+
git config --global --add safe.directory "$(realpath .)"
14+
15+
package_name="bigframes"
16+
package_path="packages/${package_name}"
17+
files_to_check="${package_path}"
18+
19+
# Use the IF block to handle the case where KOKORO vars are missing
20+
# (e.g. local testing)
21+
if [[ -n "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}" && -n "${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" ]]; then
22+
echo "checking changes with 'git diff ${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT} -- ${files_to_check}'"
23+
24+
package_modified=$(git diff "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" -- "${files_to_check}" | wc -l)
25+
else
26+
# If not a PR (like a local run or a different CI trigger),
27+
# we treat it as 0 so it falls through to the "continuous" check.
28+
package_modified=0
29+
fi
30+
31+
# Check if modified OR if it's a continuous build
32+
if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then
33+
echo "------------------------------------------------------------"
34+
echo "Running doctest for: ${package_name}"
35+
echo "------------------------------------------------------------"
36+
37+
# Ensure credentials are set for system tests in Kokoro
38+
if [[ -z "${GOOGLE_APPLICATION_CREDENTIALS}" && -f "${KOKORO_GFILE_DIR}/service-account.json" ]]; then
39+
export GOOGLE_APPLICATION_CREDENTIALS="${KOKORO_GFILE_DIR}/service-account.json"
40+
fi
41+
42+
export GOOGLE_CLOUD_PROJECT="bigframes-testing"
43+
NOX_SESSION=("cleanup" "doctest")
44+
45+
cd "${package_path}"
46+
python3 -m nox -s "${NOX_SESSION[@]}"
47+
else
48+
echo "No changes in ${package_name} and not a continuous build, skipping."
49+
fi

packages/bigframes/tests/system/small/bigquery/test_json.py

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15+
import json
16+
1517
import geopandas as gpd # type: ignore
1618
import pandas as pd
1719
import pyarrow as pa
@@ -396,8 +398,8 @@ def test_to_json_from_int():
396398
def test_to_json_from_struct():
397399
s = bpd.Series(
398400
[
399-
{"version": 1, "project": "pandas"},
400-
{"version": 2, "project": "numpy"},
401+
{"project": "pandas", "version": 1},
402+
{"project": "numpy", "version": 2},
401403
]
402404
)
403405
assert dtypes.is_struct_like(s.dtype)
@@ -408,7 +410,9 @@ def test_to_json_from_struct():
408410
dtype=dtypes.JSON_DTYPE,
409411
)
410412

411-
pd.testing.assert_series_equal(actual.to_pandas(), expected.to_pandas())
413+
actual_json = [json.loads(x) for x in actual.to_pandas()]
414+
expected_json = [json.loads(x) for x in expected.to_pandas()]
415+
assert actual_json == expected_json
412416

413417

414418
def test_to_json_string_from_int():
@@ -421,8 +425,8 @@ def test_to_json_string_from_int():
421425
def test_to_json_string_from_struct():
422426
s = bpd.Series(
423427
[
424-
{"version": 1, "project": "pandas"},
425-
{"version": 2, "project": "numpy"},
428+
{"project": "pandas", "version": 1},
429+
{"project": "numpy", "version": 2},
426430
]
427431
)
428432
assert dtypes.is_struct_like(s.dtype)
@@ -433,7 +437,9 @@ def test_to_json_string_from_struct():
433437
dtype=dtypes.STRING_DTYPE,
434438
)
435439

436-
pd.testing.assert_series_equal(actual.to_pandas(), expected.to_pandas())
440+
actual_json = [json.loads(x) for x in actual.to_pandas()]
441+
expected_json = [json.loads(x) for x in expected.to_pandas()]
442+
assert actual_json == expected_json
437443

438444

439445
def test_json_keys():

packages/bigframes/tests/system/small/test_series.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1234,7 +1234,7 @@ def test_divmods_series(scalars_dfs, col_x, col_y, method):
12341234
# BigQuery's mod functions return NUMERIC values for non-INT64 inputs.
12351235
if bf_div_result.dtype == pd.Int64Dtype():
12361236
bigframes.testing.utils.assert_series_equal(
1237-
pd_div_result, bf_div_result.to_pandas()
1237+
pd_div_result, bf_div_result.to_pandas(), check_dtype=False
12381238
)
12391239
else:
12401240
bigframes.testing.utils.assert_series_equal(
@@ -1279,7 +1279,7 @@ def test_divmods_scalars(scalars_dfs, col_x, other, method):
12791279
# BigQuery's mod functions return NUMERIC values for non-INT64 inputs.
12801280
if bf_div_result.dtype == pd.Int64Dtype():
12811281
bigframes.testing.utils.assert_series_equal(
1282-
pd_div_result, bf_div_result.to_pandas()
1282+
pd_div_result, bf_div_result.to_pandas(), check_dtype=False
12831283
)
12841284
else:
12851285
bigframes.testing.utils.assert_series_equal(

0 commit comments

Comments
 (0)