Fix for data download notebook

Author: sheridana

lsd/tutorial/notebooks/lsd_data_download.ipynb

@@ -4,8 +4,7 @@
   "metadata": {
     "colab": {
       "name": "lsd_data_download.ipynb",
-      "provenance": [],
-      "collapsed_sections": []
+      "provenance": []
     },
     "kernelspec": {
       "name": "python3",
@@ -30,14 +29,14 @@
         "  * the s3 bucket contains about 1.7 tb of data, so bear in mind before downloading the whole bucket\n",
         "  * boto3 is a python api useful for accessing s3 data, you can use it to find sizes of folders before committing to downloading a directory (here is an example https://stackoverflow.com/questions/49759940/how-to-find-size-of-a-folder-inside-an-s3-bucket), and there are examples for downloading some data below (links at bottom of cell)\n",
         "  * s3 bucket contains a mixture of zarr, n5, nml, json, bson\n",
-        "  * google bucket volume metadata is stored in info files. zarr/n5 metadata is stored in attributes files. To see offsets/shapes of cropped data (i.e hemi roi 1) check these files. \n",
+        "  * google bucket volume metadata is stored in info files. zarr/n5 metadata is stored in attributes files. To see offsets/shapes of cropped data (i.e hemi roi 1) check these files.\n",
         "  * google bucket volumes are in xyz voxel space.\n",
         "  * zarr volumes are in zyx world space (i.e nanometers)\n",
         "  * n5 volumes are in xyz world space\n",
         "\n",
         "---\n",
         "\n",
-        "###Key: \n",
+        "###Key:\n",
         "  * s3 = amazon s3 bucket\n",
         "  * gb = google bucket\n",
         "\n",
@@ -133,17 +132,11 @@
     {
       "cell_type": "code",
       "metadata": {
-        "id": "n2O8isY0fFAJ"
+        "id": "n2O8isY0fFAJ",
+        "collapsed": true
       },
       "source": [
-        "!pip install boto3\n",
-        "!pip install cloud-volume\n",
-        "!pip install matplotlib\n",
-        "!pip install pandas\n",
-        "!pip install plotly\n",
-        "!pip install requests\n",
-        "!pip install scikit-image\n",
-        "!pip install zarr\n",
+        "!pip install boto3 cloud-volume matplotlib pandas plotly pymongo requests scikit-image zarr\n",
         "\n",
         "# since colab has its own versions pre-installed but we\n",
         "# need some other versions, the runtime needs to be restarted\n",
@@ -170,9 +163,11 @@
         "import pandas as pd\n",
         "import plotly.express as px\n",
         "import zarr\n",
+        "from botocore import UNSIGNED\n",
+        "from botocore.client import Config\n",
         "from cloudvolume import CloudVolume"
       ],
-      "execution_count": null,
+      "execution_count": 44,
       "outputs": []
     },
     {
@@ -240,7 +235,7 @@
         "  # ensure there is a voxel offset in the metadata so cloudvolume is happy\n",
         "  for scale in vol.info['scales']:\n",
         "        scale['voxel_offset'] = [0, 0, 0]\n",
-        "  \n",
+        "\n",
         "  # slice data\n",
         "  data = vol[x0:x1, y0:y1, z0:z1]\n",
         "\n",
@@ -259,7 +254,7 @@
         "id": "IztUw3U1-56_"
       },
       "source": [
-        "# get a random 1000x1000 voxel patch \n",
+        "# get a random 1000x1000 voxel patch\n",
         "raw_data = cloud_to_np(raw_vol,1000,2000,2000,3000,300,301)\n",
         "seg_data = cloud_to_np(seg_vol,1000,2000,2000,3000,300,301)"
       ],
@@ -327,10 +322,7 @@
         "id": "pDL38VnxhI6T"
       },
       "source": [
-        "# set bucket credentials\n",
-        "access_key = ''\n",
-        "secret_key = ''\n",
-        "bucket = ''"
+        "bucket = 'open-neurodata'"
       ],
       "execution_count": null,
       "outputs": []
@@ -341,16 +333,16 @@
         "id": "a2gTfGEvgLr0"
       },
       "source": [
-        "# connect to client\n",
-        "client = boto3.client('s3', aws_access_key_id=access_key, aws_secret_access_key=secret_key)"
+        "client = boto3.client('s3', config=Config(signature_version=UNSIGNED))"
       ],
       "execution_count": null,
       "outputs": []
     },
     {
       "cell_type": "code",
       "metadata": {
-        "id": "bVGfxKT2i3MX"
+        "id": "bVGfxKT2i3MX",
+        "collapsed": true
       },
       "source": [
         "# list data\n",
@@ -415,21 +407,19 @@
         "# function to download all files nested in a bucket path\n",
         "def downloadDirectory(\n",
         "    bucket_name,\n",
-        "    path,\n",
-        "    access_key,\n",
-        "    secret_key):\n",
-        "  \n",
+        "    path):\n",
+        "\n",
         "    resource = boto3.resource(\n",
         "        's3',\n",
-        "        aws_access_key_id=access_key,\n",
-        "        aws_secret_access_key=secret_key)\n",
-        "    \n",
+        "        config=Config(signature_version=UNSIGNED)\n",
+        "    )\n",
+        "\n",
         "    bucket = resource.Bucket(bucket_name)\n",
         "\n",
         "    for obj in bucket.objects.filter(Prefix=path):\n",
         "        if not os.path.exists(os.path.dirname(obj.key)):\n",
         "            os.makedirs(os.path.dirname(obj.key))\n",
-        "        \n",
+        "\n",
         "        key = obj.key\n",
         "\n",
         "        print(f'Downloading {key}')\n",
@@ -447,9 +437,8 @@
         "# download example fib25 training data\n",
         "downloadDirectory(\n",
         "    bucket,\n",
-        "    'funke/fib25/training/trvol-250-1.zarr',\n",
-        "    access_key,\n",
-        "    secret_key)"
+        "    'funke/fib25/training/trvol-250-1.zarr'\n",
+        ")"
       ],
       "execution_count": null,
       "outputs": []
@@ -583,9 +572,8 @@
         "# download example hemi region adjacency graph\n",
         "downloadDirectory(\n",
         "    bucket,\n",
-        "    'funke/hemi/testing/segmentations/data.zarr/rags/ACLSD/hemi_affs_from_lsd_200k_roi_1',\n",
-        "    access_key,\n",
-        "    secret_key)"
+        "    'funke/hemi/testing/segmentations/data.zarr/rags/ACLSD/hemi_affs_from_lsd_200k_roi_1'\n",
+        ")"
       ],
       "execution_count": null,
       "outputs": []
@@ -726,9 +714,8 @@
         "# download example zebrafinch validation skeletons\n",
         "downloadDirectory(\n",
         "    bucket,\n",
-        "    'funke/zebrafinch/testing/ground_truth/validation/consolidated/zebrafinch_gt_skeletons_new_gt_9_9_20_validation',\n",
-        "    access_key,\n",
-        "    secret_key)"
+        "    'funke/zebrafinch/testing/ground_truth/validation/consolidated/zebrafinch_gt_skeletons_new_gt_9_9_20_validation'\n",
+        ")"
       ],
       "execution_count": null,
       "outputs": []
@@ -814,4 +801,4 @@
       "outputs": []
     }
   ]
-}
+}