Merge pull request #102 from bhandarivijay-png/ai-gsutil-migration-ae277dae317e4c0f8a6aa44815ef7ded

removed changes for model garden PR 4316

Author: gurusai-voleti

notebooks/community/model_garden/model_garden_tfvision_image_classification.ipynb

@@ -169,10 +169,10 @@
         "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
         "    BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
         "    BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
-        "    ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n",
+        "    ! gsutil mb -l {REGION} {BUCKET_URI}\n",
         "else:\n",
         "    assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
-        "    shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n",
+        "    shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
         "    bucket_region = shell_output[0].strip().lower()\n",
         "    if bucket_region != REGION:\n",
         "        raise ValueError(\n",
@@ -197,7 +197,7 @@
         "\n",
         "\n",
         "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
-        "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
         "\n",
         "! gcloud config set project $PROJECT_ID\n",
         "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
@@ -220,7 +220,7 @@
         "    destination = os.path.join(CONFIG_DIR, filename)\n",
         "    print(\"Copy\", url, \"to\", destination)\n",
         "    ! wget \"$url\" -O \"$filename\"\n",
-        "    ! gcloud storage cp \"$filename\" \"$destination\"\n",
+        "    ! gsutil cp \"$filename\" \"$destination\"\n",
         "\n",
         "\n",
         "upload_config_to_gcs(\n",
@@ -457,7 +457,7 @@
         "                checkpoint_path = os.path.relpath(checkpoint_path, checkpoint_name)\n",
         "                break\n",
         "\n",
-        "    ! gcloud storage cp --recursive $checkpoint_name $CHECKPOINT_BUCKET/\n",
+        "    ! gsutil cp -r $checkpoint_name $CHECKPOINT_BUCKET/\n",
         "    checkpoint_uri = os.path.join(CHECKPOINT_BUCKET, checkpoint_name, checkpoint_path)\n",
         "    print(\"Checkpoint uploaded to\", checkpoint_uri)\n",
         "    return checkpoint_uri\n",
@@ -588,8 +588,8 @@
         "        current_trial_best_ckpt_evaluation_filepath = os.path.join(\n",
         "            current_trial_best_ckpt_dir, \"info.json\"\n",
         "        )\n",
-        "        ! gcloud storage cp $current_trial_best_ckpt_evaluation_filepath .\n",
-        "        with open(\"info.json\") as f:\n",
+        "        ! gsutil cp $current_trial_best_ckpt_evaluation_filepath .\n",
+        "        with open(\"info.json\", \"r\") as f:\n",
         "            eval_metric_results = json.load(f)\n",
         "            current_performance = eval_metric_results[evaluation_metric]\n",
         "            if current_performance > best_performance:\n",
@@ -696,7 +696,9 @@
         "    serving_container_image_uri=PREDICTION_CONTAINER_URI,\n",
         "    serving_container_args=SERVING_CONTAINER_ARGS,\n",
         "    serving_container_environment_variables=serving_env,\n",
-        "    model_garden_source_model_name=(f\"publishers/google/models/{publisher_model_id}\"),\n",
+        "    model_garden_source_model_name=(\n",
+        "        f\"publishers/google/models/{publisher_model_id}\"\n",
+        "    ),\n",
         ")\n",
         "\n",
         "models[\"model_icn\"].wait()\n",
@@ -724,7 +726,9 @@
         "    accelerator_count=1,\n",
         "    min_replica_count=1,\n",
         "    max_replica_count=1,\n",
-        "    system_labels={\"NOTEBOOK_NAME\": \"model_garden_tfvision_image_classification.ipynb\"},\n",
+        "    system_labels={\n",
+        "        \"NOTEBOOK_NAME\": \"model_garden_tfvision_image_classification.ipynb\"\n",
+        "    },\n",
         ")\n",
         "\n",
         "endpoint_id = endpoints[\"endpoint_icn\"].name\n",
@@ -772,7 +776,7 @@
         "    \"\"\"\n",
         "    label_map_filename = os.path.basename(label_map_yaml_filepath)\n",
         "    subprocess.check_output(\n",
-        "        [\"gcloud\", \"storage\", \"cp\", label_map_yaml_filepath, label_map_filename],\n",
+        "        [\"gsutil\", \"cp\", label_map_yaml_filepath, label_map_filename],\n",
         "        stderr=subprocess.STDOUT,\n",
         "    )\n",
         "    with open(label_map_filename, \"rb\") as input_file:\n",
@@ -793,7 +797,7 @@
         "    if new_width <= 0:\n",
         "        test_file = os.path.basename(test_filepath)\n",
         "        subprocess.check_output(\n",
-        "            [\"gcloud\", \"storage\", \"cp\", test_filepath, test_file],\n",
+        "            [\"gsutil\", \"cp\", test_filepath, test_file], stderr=subprocess.STDOUT\n",
         "            stderr=subprocess.STDOUT,\n",
         "        )\n",
         "        with open(test_file, \"rb\") as input_file:\n",
@@ -905,7 +909,7 @@
         "\n",
         "delete_bucket = False  # @param {type:\"boolean\"}\n",
         "if delete_bucket:\n",
-        "    ! gcloud storage rm --recursive $BUCKET_NAME"
+        "    ! gsutil -m rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/community/model_garden/model_garden_timesfm_2_0_deployment_on_vertex.ipynb

@@ -162,10 +162,10 @@
         "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
         "    BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
         "    BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
-        "    ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n",
+        "    ! gsutil mb -l {REGION} {BUCKET_URI}\n",
         "else:\n",
         "    assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
-        "    shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n",
+        "    shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
         "    bucket_region = shell_output[0].strip().lower()\n",
         "    if bucket_region != REGION:\n",
         "        raise ValueError(\n",
@@ -190,7 +190,7 @@
         "\n",
         "\n",
         "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
-        "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
         "\n",
         "! gcloud config set project $PROJECT_ID\n",
         "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
@@ -911,7 +911,7 @@
         "\n",
         "delete_bucket = False  # @param {type:\"boolean\"}\n",
         "if delete_bucket:\n",
-        "    ! gcloud storage rm --recursive $BUCKET_NAME"
+        "    ! gsutil -m rm -r $BUCKET_NAME"
       ]
     }
   ],