diff --git a/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb b/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb index c5008ce38..dc0609f56 100644 --- a/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb +++ b/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb @@ -109,27 +109,27 @@ "id": "Af0jTPSgl9yh" }, "source": [ - "## Before you begin\r\n", - "\r\n", - "### GPU run-time\r\n", - "\r\n", - "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\r\n", - "\r\n", - "### Set up your GCP project\r\n", - "\r\n", - "**The following steps are required, regardless of your notebook environment.**\r\n", - "\r\n", - "1. [Select or create a GCP project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\r\n", - "\r\n", - "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\r\n", - "\r\n", - "3. [Enable the AutoML APIs and Compute Engine APIs.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component)\r\n", - "\r\n", - "4. [Google Cloud SDK](https://cloud.google.com/sdk) is already installed in AutoML Notebooks.\r\n", - "\r\n", - "5. Enter your project ID in the cell below. Then run the cell to make sure the\r\n", - "Cloud SDK uses the right project for all the commands in this notebook.\r\n", - "\r\n", + "## Before you begin\n", + "\n", + "### GPU run-time\n", + "\n", + "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n", + "\n", + "### Set up your GCP project\n", + "\n", + "**The following steps are required, regardless of your notebook environment.**\n", + "\n", + "1. [Select or create a GCP project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n", + "\n", + "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n", + "\n", + "3. [Enable the AutoML APIs and Compute Engine APIs.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component)\n", + "\n", + "4. [Google Cloud SDK](https://cloud.google.com/sdk) is already installed in AutoML Notebooks.\n", + "\n", + "5. Enter your project ID in the cell below. Then run the cell to make sure the\n", + "Cloud SDK uses the right project for all the commands in this notebook.\n", + "\n", "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands." ] }, @@ -240,11 +240,11 @@ "id": "h_L3MRsOmYED" }, "source": [ - "### Authenticate your GCP account\r\n", - "\r\n", - "**If you are using AutoML Notebooks**, your environment is already\r\n", - "authenticated. Skip this step.\r\n", - "\r\n", + "### Authenticate your GCP account\n", + "\n", + "**If you are using AutoML Notebooks**, your environment is already\n", + "authenticated. Skip this step.\n", + "\n", "*Note: If you are on an AutoML notebook and run the cell, the cell knows to skip executing the authentication steps.*" ] }, @@ -336,7 +336,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION gs://$BUCKET_NAME" + "! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME" ] }, { @@ -356,7 +356,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al gs://$BUCKET_NAME" + "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ] }, { @@ -391,7 +391,6 @@ "outputs": [], "source": [ "import json\n", - "import time\n", "\n", "from google.cloud import automl\n", "from google.protobuf.json_format import MessageToJson" @@ -483,7 +482,7 @@ }, "outputs": [], "source": [ - "! gsutil cat $IMPORT_FILE | head -n 10" + "! gcloud storage cat $IMPORT_FILE | head -n 10" ] }, { @@ -1140,7 +1139,7 @@ }, "outputs": [], "source": [ - "test_item = ! gsutil cat $IMPORT_FILE | head -n1\n", + "test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "test_item, test_label = str(test_item[0]).split(\",\")\n", "\n", "print(test_item, test_label)" @@ -1175,8 +1174,8 @@ }, "outputs": [], "source": [ - "! gsutil cat $gcs_input_uri\n", - "! gsutil cat $test_item_uri" + "! gcloud storage cat $gcs_input_uri\n", + "! gcloud storage cat $test_item_uri" ] }, { @@ -1309,9 +1308,9 @@ "id": "771dDuKzg8Mk" }, "source": [ - "*Example output*:\r\n", - "```\r\n", - "{}\r\n", + "*Example output*:\n", + "```\n", + "{}\n", "```" ] }, @@ -1325,8 +1324,8 @@ "source": [ "destination_uri = output_config[\"gcs_destination\"][\"output_uri_prefix\"][:-1]\n", "\n", - "! gsutil ls $destination_uri/*\n", - "! gsutil cat $destination_uri/prediction*/*.jsonl" + "! gcloud storage ls $destination_uri/*\n", + "! gcloud storage cat $destination_uri/prediction*/*.jsonl" ] }, { @@ -1409,9 +1408,9 @@ "id": "CZ-62obNmBNc" }, "source": [ - "*Example output*:\r\n", - "```\r\n", - "{}\r\n", + "*Example output*:\n", + "```\n", + "{}\n", "```" ] }, @@ -1441,7 +1440,7 @@ }, "outputs": [], "source": [ - "test_item = ! gsutil cat $IMPORT_FILE | head -n1\n", + "test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "test_item, test_label = str(test_item[0]).split(\",\")" ] }, @@ -1599,11 +1598,11 @@ "id": "bQ-VVaSxJjkd" }, "source": [ - "# Cleaning up\r\n", - "\r\n", - "To clean up all GCP resources used in this project, you can [delete the GCP\r\n", - "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\r\n", - "\r\n", + "# Cleaning up\n", + "\n", + "To clean up all GCP resources used in this project, you can [delete the GCP\n", + "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n", + "\n", "Otherwise, you can delete the individual resources you created in this tutorial." ] }, @@ -1635,7 +1634,7 @@ "\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r gs://$BUCKET_NAME" + " ! gcloud storage rm --recursive gs://$BUCKET_NAME" ] } ], diff --git a/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb b/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb index de87018c9..018c25f21 100644 --- a/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb +++ b/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb @@ -435,7 +435,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { @@ -455,7 +455,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { @@ -850,7 +850,7 @@ "! bq extract -m $param\n", "\n", "MODEL_DIR = f\"{BUCKET_URI}/{BQ_DATASET_NAME}\"\n", - "! gsutil ls $MODEL_DIR" + "! gcloud storage ls $MODEL_DIR" ] }, { @@ -1398,7 +1398,7 @@ "delete_storage = False\n", "if delete_storage or os.getenv(\"IS_TESTING\"):\n", " # Delete the created GCS bucket\n", - " ! gsutil rm -r $BUCKET_URI" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], diff --git a/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb index 30b23e420..34b9ad5b9 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb @@ -462,7 +462,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { @@ -482,7 +482,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { @@ -552,9 +552,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { @@ -801,9 +801,9 @@ "outputs": [], "source": [ "GCS_WC_PY = BUCKET_URI + \"/wc.py\"\n", - "! gsutil cp wc.py $GCS_WC_PY\n", + "! gcloud storage cp wc.py $GCS_WC_PY\n", "GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n", - "! gsutil cp requirements.txt $GCS_REQUIREMENTS_TXT\n", + "! gcloud storage cp requirements.txt $GCS_REQUIREMENTS_TXT\n", "\n", "GCS_WC_OUT = BUCKET_URI + \"/wc_out.txt\"" ] @@ -872,7 +872,7 @@ "\n", "pipeline.run()\n", "\n", - "! gsutil cat {GCS_WC_OUT}* | head -n10\n", + "! gcloud storage cat {GCS_WC_OUT}* | head -n10\n", "\n", "! rm -f dataflow_wc.json wc.py requirements.txt" ] @@ -1097,11 +1097,11 @@ "outputs": [], "source": [ "GCS_SPLIT_PY = BUCKET_URI + \"/split.py\"\n", - "! gsutil cp split.py $GCS_SPLIT_PY\n", + "! gcloud storage cp split.py $GCS_SPLIT_PY\n", "GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n", - "! gsutil cp requirements.txt $GCS_REQUIREMENTS_TXT\n", + "! gcloud storage cp requirements.txt $GCS_REQUIREMENTS_TXT\n", "GCS_SETUP_PY = BUCKET_URI + \"/setup.py\"\n", - "! gsutil cp setup.py $GCS_SETUP_PY" + "! gcloud storage cp setup.py $GCS_SETUP_PY" ] }, { @@ -1202,7 +1202,7 @@ "\n", "pipeline.run()\n", "\n", - "! gsutil ls {BUCKET_URI}/exported_data\n", + "! gcloud storage ls {BUCKET_URI}/exported_data\n", "\n", "! rm -f dataflow_split.json split.py requirements.txt" ] @@ -1258,7 +1258,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], diff --git a/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb b/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb index ac00416dd..21b4f5cdd 100644 --- a/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb +++ b/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb @@ -357,7 +357,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { @@ -427,9 +427,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { @@ -1386,7 +1386,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], diff --git a/notebooks/official/migration/sdk-automl-image-classification-batch-online.ipynb b/notebooks/official/migration/sdk-automl-image-classification-batch-online.ipynb index 584e45f43..919222914 100644 --- a/notebooks/official/migration/sdk-automl-image-classification-batch-online.ipynb +++ b/notebooks/official/migration/sdk-automl-image-classification-batch-online.ipynb @@ -279,7 +279,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} {BUCKET_URI}" + "! gcloud storage buckets create --location {LOCATION} {BUCKET_URI}" ] }, { @@ -375,11 +375,11 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gsutil cat $FILE | wc -l\n", + "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $FILE | head" + "! gcloud storage cat $FILE | head" ] }, { @@ -730,7 +730,7 @@ }, "outputs": [], "source": [ - "test_items = !gsutil cat $IMPORT_FILE | head -n2\n", + "test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n", "if len(str(test_items[0]).split(\",\")) == 3:\n", " _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n", " _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n", @@ -764,8 +764,8 @@ "file_1 = test_item_1.split(\"/\")[-1]\n", "file_2 = test_item_2.split(\"/\")[-1]\n", "\n", - "! gsutil cp $test_item_1 $BUCKET_URI/$file_1\n", - "! gsutil cp $test_item_2 $BUCKET_URI/$file_2\n", + "! gcloud storage cp $test_item_1 $BUCKET_URI/$file_1\n", + "! gcloud storage cp $test_item_2 $BUCKET_URI/$file_2\n", "\n", "test_item_1 = BUCKET_URI + \"/\" + file_1\n", "test_item_2 = BUCKET_URI + \"/\" + file_2" @@ -809,7 +809,7 @@ " f.write(json.dumps(data) + \"\\n\")\n", "\n", "print(gcs_input_uri)\n", - "! gsutil cat $gcs_input_uri" + "! gcloud storage cat $gcs_input_uri" ] }, { @@ -1066,7 +1066,7 @@ }, "outputs": [], "source": [ - "test_item = !gsutil cat $IMPORT_FILE | head -n1\n", + "test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(str(test_item[0]).split(\",\")) == 3:\n", " _, test_item, test_label = str(test_item[0]).split(\",\")\n", "else:\n", @@ -1201,7 +1201,7 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = True # Set True for deletion\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], diff --git a/notebooks/official/model_monitoring/model_monitoring.ipynb b/notebooks/official/model_monitoring/model_monitoring.ipynb index 183a91170..4cbd5851a 100644 --- a/notebooks/official/model_monitoring/model_monitoring.ipynb +++ b/notebooks/official/model_monitoring/model_monitoring.ipynb @@ -311,7 +311,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { @@ -1029,7 +1029,7 @@ }, "outputs": [], "source": [ - "!gsutil ls gs://cloud-ai-platform-fdfb4810-148b-4c86-903c-dbdff879f6e1/*/*" + "!gcloud storage ls gs://cloud-ai-platform-fdfb4810-148b-4c86-903c-dbdff879f6e1/*/*" ] }, { @@ -1172,7 +1172,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gsutil rm -rf {BUCKET_URI}" + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] }, { diff --git a/notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb b/notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb index a2feec79d..7e4f24207 100644 --- a/notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb +++ b/notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb @@ -303,7 +303,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { @@ -386,9 +386,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { @@ -1088,7 +1088,7 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], diff --git a/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb b/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb index f1a299908..0dd3be4f4 100644 --- a/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb +++ b/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb @@ -340,7 +340,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { @@ -1474,7 +1474,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ],