diff --git a/notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb b/notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb index 582a7ea5f..dfccf184d 100644 --- a/notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb +++ b/notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb @@ -416,7 +416,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_NAME" + "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ] }, { @@ -436,7 +436,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_NAME" + "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] }, { @@ -474,7 +474,6 @@ "\n", "from google.cloud.aiplatform import gapic as aip\n", "from google.protobuf import json_format\n", - "from google.protobuf.json_format import MessageToJson, ParseDict\n", "from google.protobuf.struct_pb2 import Struct, Value" ] }, @@ -784,11 +783,11 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gsutil cat $FILE | wc -l\n", + "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $FILE | head" + "! gcloud storage cat $FILE | head" ] }, { @@ -1215,7 +1214,6 @@ " }\n", " response = clients[\"model\"].export_model(name=name, output_config=output_config)\n", " print(\"Long running operation:\", response.operation.name)\n", - " result = response.result(timeout=1800)\n", " metadata = response.operation.metadata\n", " artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n", " print(\"Artifact Uri\", artifact_uri)\n", @@ -1244,9 +1242,9 @@ }, "outputs": [], "source": [ - "! gsutil ls $model_package\n", + "! gcloud storage ls $model_package\n", "# Download the model artifacts\n", - "! gsutil cp -r $model_package tflite\n", + "! gcloud storage cp --recursive $model_package tflite\n", "\n", "tflite_path = \"tflite/model.tflite\"" ] @@ -1305,7 +1303,7 @@ }, "outputs": [], "source": [ - "test_items = ! gsutil cat $IMPORT_FILE | head -n1\n", + "test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "test_item = test_items[0].split(\",\")[0]\n", "\n", "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n", @@ -1449,7 +1447,7 @@ " print(e)\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r $BUCKET_NAME" + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], diff --git a/notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb b/notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb index 2c3a7e11d..4a06d69ea 100644 --- a/notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb +++ b/notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb @@ -416,7 +416,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_NAME" + "! gcloud storage buckets create --location $REGION $BUCKET_NAME" ] }, { @@ -436,7 +436,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_NAME" + "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] }, { @@ -473,9 +473,7 @@ "import time\n", "\n", "from google.cloud.aiplatform import gapic as aip\n", - "from google.protobuf import json_format\n", - "from google.protobuf.json_format import MessageToJson, ParseDict\n", - "from google.protobuf.struct_pb2 import Struct, Value" + "from google.protobuf import json_format\n" ] }, { @@ -785,11 +783,11 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gsutil cat $FILE | wc -l\n", + "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $FILE | head" + "! gcloud storage cat $FILE | head" ] }, { @@ -1218,7 +1216,6 @@ " }\n", " response = clients[\"model\"].export_model(name=name, output_config=output_config)\n", " print(\"Long running operation:\", response.operation.name)\n", - " result = response.result(timeout=1800)\n", " metadata = response.operation.metadata\n", " artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n", " print(\"Artifact Uri\", artifact_uri)\n", @@ -1247,9 +1244,9 @@ }, "outputs": [], "source": [ - "! gsutil ls $model_package\n", + "! gcloud storage ls $model_package\n", "# Download the model artifacts\n", - "! gsutil cp -r $model_package tflite\n", + "! gcloud storage cp --recursive $model_package tflite\n", "\n", "tflite_path = \"tflite/model.tflite\"" ] @@ -1308,7 +1305,7 @@ }, "outputs": [], "source": [ - "test_items = ! gsutil cat $IMPORT_FILE | head -n1\n", + "test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "test_item = test_items[0].split(\",\")[0]\n", "\n", "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n", @@ -1452,7 +1449,7 @@ " print(e)\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r $BUCKET_NAME" + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], diff --git a/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb index 244a59995..51f640e58 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb @@ -430,7 +430,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { @@ -450,7 +450,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { @@ -520,9 +520,8 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n","\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { @@ -807,13 +806,13 @@ " + \"/evaluation_metrics\"\n", " )\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", - " ! gsutil cat $EXECUTE_OUTPUT\n", + " ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n", - " ! gsutil cat $GCP_RESOURCES\n", + " ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n", - " ! gsutil cat $EVAL_METRICS\n", + " ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n", "\n", " return None\n", @@ -1167,9 +1166,7 @@ "\n", "\n", "@component()\n", - "def multi_output(\n", - " text1: str, text2: str\n", - ") -> NamedTuple(\n", + "def multi_output(text1: str, text2: str) -> NamedTuple(\n", " \"Outputs\",\n", " [\n", " (\"output_1\", str), # Return parameters\n", @@ -1587,7 +1584,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], diff --git a/notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb index 3869f7d86..9e234689d 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb @@ -464,7 +464,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { @@ -484,7 +484,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { @@ -892,7 +892,7 @@ " f.write(json.dumps(instance) + \"\\n\")\n", "\n", "# Upload to Cloud Storage bucket\n", - "! gsutil cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", + "! gcloud storage cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", "\n", "print(\"Uploaded instances to: \", BATCH_PREDICTION_GCS_SOURCE)" ] @@ -1108,7 +1108,7 @@ "outputs": [], "source": [ "data_file = \"gs://cloud-samples-data/ai-platform/iris/iris_data.csv\"\n", - "! gsutil cat $data_file | head -n 10 > test.csv\n", + "! gcloud storage cat $data_file | head -n 10 > test.csv\n", "\n", "! cat test.csv\n", "\n", @@ -1118,7 +1118,7 @@ " BUCKET_URI + \"/batch_prediction_instances/\" + BATCH_PREDICTION_INSTANCES_FILE\n", ")\n", "\n", - "! gsutil cp test.csv $BATCH_PREDICTION_GCS_SOURCE" + "! gcloud storage cp test.csv $BATCH_PREDICTION_GCS_SOURCE" ] }, { @@ -1497,7 +1497,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf {BUCKET_URI}" + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], diff --git a/notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb index 377b45369..94c499514 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb @@ -504,7 +504,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { @@ -524,7 +524,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { @@ -1734,7 +1734,7 @@ "! rm -rf app\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf {BUCKET_URI}" + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], diff --git a/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb b/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb index 7f70415bc..b527c843f 100644 --- a/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb +++ b/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb @@ -537,7 +537,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI" + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] }, { @@ -557,7 +557,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { @@ -627,9 +627,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { @@ -1383,7 +1383,7 @@ "outputs": [], "source": [ "# Load the results\n", - "attributions = !gsutil cat $feat_attrs_gcs_uri\n", + "attributions = !gcloud storage cat $feat_attrs_gcs_uri\n", "\n", "# Print the results obtained\n", "attributions = json.loads(attributions[0])\n", @@ -1463,7 +1463,7 @@ "# Delete Cloud Storage objects\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil -m rm -r $BUCKET_URI" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], diff --git a/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb b/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb index a85300e3f..6ad34ad51 100644 --- a/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb +++ b/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb @@ -288,7 +288,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { @@ -707,7 +707,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" ] }, { @@ -1260,7 +1260,7 @@ "\n", "gcs_input_uri = BUCKET_URI + \"/test.csv\"\n", "\n", - "! gsutil cp batch.csv $gcs_input_uri" + "! gcloud storage cp batch.csv $gcs_input_uri" ] }, { @@ -1398,7 +1398,7 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", "# Remove the locally generated files\n", "! rm -rf batch.csv custom.tar.gz custom" diff --git a/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb b/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb index a14da893f..d4ebc1769 100644 --- a/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb +++ b/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb @@ -293,7 +293,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { @@ -706,7 +706,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" ] }, { @@ -1444,7 +1444,7 @@ "\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], diff --git a/notebooks/official/training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb b/notebooks/official/training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb index 4a894fa66..ea328a9cb 100644 --- a/notebooks/official/training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb +++ b/notebooks/official/training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb @@ -279,7 +279,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { @@ -1264,7 +1264,7 @@ }, "outputs": [], "source": [ - "!gsutil ls $GCS_MODEL_PATH" + "!gcloud storage ls $GCS_MODEL_PATH" ] }, { @@ -1517,7 +1517,7 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil -m rm -r $BUCKET_URI" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ],