diff --git a/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb b/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb index 4742266e0..98f188ee2 100644 --- a/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb +++ b/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb @@ -467,8 +467,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -487,8 +486,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -711,8 +709,7 @@ "IMPORT_FILE = \"petfinder-tabular-classification-tabnet-with-header.csv\"\n", "TRAINING_DATA_PATH = f\"{BUCKET_URI}/data/petfinder/train.csv\"\n", "\n", - "! gsutil cp gs://cloud-samples-data/ai-platform-unified/datasets/tabular/{IMPORT_FILE} {TRAINING_DATA_PATH}" - ] + "! gcloud storage cp gs://cloud-samples-data/ai-platform-unified/datasets/tabular/{IMPORT_FILE} {TRAINING_DATA_PATH}" ] }, { "cell_type": "markdown", @@ -1422,8 +1419,7 @@ " print(e)\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb index 34df1ad4e..3e28b3b40 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb @@ -441,8 +441,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create $BUCKET_URI --location=$REGION" ] }, { "cell_type": "markdown", @@ -461,8 +460,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -531,10 +529,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -1006,8 +1003,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_horses_or_humans.tar.gz" - ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_horses_or_humans.tar.gz" ] }, { "cell_type": "markdown", @@ -1433,14 +1429,11 @@ " + \"/evaluation_metrics\"\n", " )\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", - " ! gsutil cat $EXECUTE_OUTPUT\n", - " return EXECUTE_OUTPUT\n", + " ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n", - " ! gsutil cat $GCP_RESOURCES\n", - " return GCP_RESOURCES\n", + " ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n", - " ! gsutil cat $EVAL_METRICS\n", - " return EVAL_METRICS\n", + " ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n", "\n", " return None\n", "\n", @@ -1454,8 +1447,7 @@ "print(\"getbesttrialop\")\n", "artifacts = print_pipeline_output(pipeline, \"getbesttrialop\")\n", "print(\"\\n\\n\")\n", - "output = !gsutil cat $artifacts\n", - "output = json.loads(output[0])\n", + "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", "best_trial = json.loads(output[\"parameters\"][\"Output\"][\"stringValue\"])\n", "model_id = best_trial[\"id\"]\n", "print(\"BEST MODEL\", model_id)\n", @@ -1529,8 +1521,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb b/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb index b2677c56d..badfa31c5 100644 --- a/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb +++ b/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb @@ -449,8 +449,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -469,8 +468,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -1374,7 +1372,7 @@ }, "outputs": [], "source": [ - "! gsutil cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg\n", + "! gcloud storage cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg\n", "\n", "import base64\n", "\n", @@ -1574,8 +1572,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI\n", - "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", "!rm -f test.jpg" ] } diff --git a/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb index f4e4b849e..4c1e0ff95 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb @@ -455,8 +455,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -475,8 +474,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -927,8 +925,7 @@ " f.write(json.dumps(x.tolist()) + \"\\n\")\n", "\n", "# Upload to Cloud Storage bucket\n", - "! gsutil cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", - "\n", + "! gcloud storage cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", "\n", "print(\"Uploaded instances to: \", BATCH_PREDICTION_GCS_SOURCE)" ] }, @@ -1445,8 +1442,7 @@ " batch_prediction_job.delete()\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb b/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb index baee931d6..6433ec335 100644 --- a/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb +++ b/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb @@ -496,8 +496,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -516,8 +515,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -687,9 +685,7 @@ "outputs": [], "source": [ "%cd ..\n", - "!gsutil cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", - "!gsutil ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" - ] + "!gcloud storage cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" ] }, { "cell_type": "markdown", @@ -1792,8 +1788,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb b/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb index 54e6135fc..ced7eb02a 100644 --- a/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb +++ b/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb @@ -451,8 +451,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "code", @@ -462,8 +461,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -493,8 +491,7 @@ }, "outputs": [], "source": [ - "! gsutil cp -r gs://featurestore_integration/model/* $ARTIFACT_URI" - ] + "! gcloud storage cp --recursive gs://featurestore_integration/model/* $ARTIFACT_URI" ] }, { "cell_type": "markdown", @@ -855,8 +852,7 @@ }, "outputs": [], "source": [ - "! gsutil cp gs://featurestore_integration/data/mobile_gaming_dataset.csv $SOURCE_URI" - ] + "! gcloud storage cp gs://featurestore_integration/data/mobile_gaming_dataset.csv $SOURCE_URI" ] }, { "cell_type": "code", @@ -1198,8 +1194,7 @@ }, "outputs": [], "source": [ - "!gsutil cp prediction_featurestore_fetch_config.yaml $ARTIFACT_URI" - ] + "!gcloud storage cp prediction_featurestore_fetch_config.yaml $ARTIFACT_URI" ] }, { "cell_type": "markdown", @@ -1731,8 +1726,7 @@ "outputs": [], "source": [ "# Delete bucket\n", - "!gsutil -m rm -r $BUCKET_URI" - ] + "!gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/workbench/spark/spark_ml.ipynb b/notebooks/official/workbench/spark/spark_ml.ipynb index de300857b..4af188288 100644 --- a/notebooks/official/workbench/spark/spark_ml.ipynb +++ b/notebooks/official/workbench/spark/spark_ml.ipynb @@ -383,8 +383,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -1424,8 +1423,7 @@ " ! rm -rf $BUCKET_FOLDER\n", " ! mkdir $BUCKET_FOLDER\n", " bike_gbt_model.write().overwrite().save(f\"{BUCKET_FOLDER}\")\n", - " ! gsutil cp -r $BUCKET_FOLDER $BUCKET_URI\n", - "else:\n", + " ! gcloud storage cp --recursive $BUCKET_FOLDER $BUCKET_URI\n", "else:\n", " bike_gbt_model.write().overwrite().save(f\"{BUCKET_URI}/\")" ] }, @@ -1460,8 +1458,7 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] }, { "cell_type": "markdown",