Pipeline_AWS.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5wyy-APEHaAO"
      },
      "outputs": [],
      "source": [
        "!mkdir -p data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_m4yeO_GNdHA"
      },
      "outputs": [],
      "source": [
        "import urllib.request\n",
        "\n",
        "def extract_data(url, filename):\n",
        "  try:\n",
        "    urllib.request.urlretrieve(url, filename)\n",
        "\n",
        "  except Exception as e:\n",
        "    print(e)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3TA5yDtbo4Xu"
      },
      "outputs": [],
      "source": [
        "extract_data(\"https://data.boston.gov/dataset/8048697b-ad64-4bfc-b090-ee00169f2323/resource/c9509ab4-6f6d-4b97-979a-0cf2a10c922b/download/311_service_requests_2015.csv\", \"data/dados_2015.csv\")\n",
        "extract_data(\"https://data.boston.gov/dataset/8048697b-ad64-4bfc-b090-ee00169f2323/resource/b7ea6b1b-3ca4-4c5b-9713-6dc1db52379a/download/311_service_requests_2016.csv\", \"data/dados_2016.csv\")\n",
        "extract_data(\"https://data.boston.gov/dataset/8048697b-ad64-4bfc-b090-ee00169f2323/resource/30022137-709d-465e-baae-ca155b51927d/download/311_service_requests_2017.csv\", \"data/dados_2017.csv\")\n",
        "extract_data(\"https://data.boston.gov/dataset/8048697b-ad64-4bfc-b090-ee00169f2323/resource/2be28d90-3a90-4af1-a3f6-f28c1e25880a/download/311_service_requests_2018.csv\", \"data/dados_2018.csv\")\n",
        "extract_data(\"https://data.boston.gov/dataset/8048697b-ad64-4bfc-b090-ee00169f2323/resource/ea2e4696-4a2d-429c-9807-d02eb92e0222/download/311_service_requests_2019.csv\", \"data/dados_2019.csv\")\n",
        "extract_data(\"https://data.boston.gov/dataset/8048697b-ad64-4bfc-b090-ee00169f2323/resource/6ff6a6fd-3141-4440-a880-6f60a37fe789/download/script_105774672_20210108153400_combine.csv\", \"data/dados_2020.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HyJTcrCYsgyH"
      },
      "outputs": [],
      "source": [
        "arquivos = [\n",
        "    \"data/dados_2015.csv\",\n",
        "    \"data/dados_2016.csv\",\n",
        "    \"data/dados_2017.csv\",\n",
        "    \"data/dados_2018.csv\",\n",
        "    \"data/dados_2019.csv\",\n",
        "    \"data/dados_2020.csv\",\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hwKYMiVxs6B3"
      },
      "outputs": [],
      "source": [
        "dfs = {}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PMyyOoc7sydQ"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "\n",
        "for arquivo in arquivos:\n",
        "    ano = arquivo.split(\"_\")[-1].split(\".\")[0]\n",
        "    dfs[ano] = pd.read_csv(arquivo)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 516
        },
        "id": "XObasg-atkTn",
        "outputId": "547220bd-2563-446e-b7c5-be43b69a5257"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe"
            },
            "text/html": [
              "\n",
              "  <div id=\"df-28741581-0110-4ee8-b186-f02874cef00a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>case_enquiry_id</th>\n",
              "      <th>open_dt</th>\n",
              "      <th>target_dt</th>\n",
              "      <th>closed_dt</th>\n",
              "      <th>ontime</th>\n",
              "      <th>case_status</th>\n",
              "      <th>closure_reason</th>\n",
              "      <th>case_title</th>\n",
              "      <th>subject</th>\n",
              "      <th>reason</th>\n",
              "      <th>...</th>\n",
              "      <th>police_district</th>\n",
              "      <th>neighborhood</th>\n",
              "      <th>neighborhood_services_district</th>\n",
              "      <th>ward</th>\n",
              "      <th>precinct</th>\n",
              "      <th>location_street_name</th>\n",
              "      <th>location_zipcode</th>\n",
              "      <th>latitude</th>\n",
              "      <th>longitude</th>\n",
              "      <th>source</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>101002296861</td>\n",
              "      <td>2018-01-01 00:08:00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2018-03-01 15:18:12</td>\n",
              "      <td>ONTIME</td>\n",
              "      <td>Closed</td>\n",
              "      <td>Case Closed. Closed date : 2018-03-01 15:18:12...</td>\n",
              "      <td>Fire Hydrant</td>\n",
              "      <td>Boston Water &amp; Sewer Commission</td>\n",
              "      <td>Fire Hydrant</td>\n",
              "      <td>...</td>\n",
              "      <td>E13</td>\n",
              "      <td>Jamaica Plain</td>\n",
              "      <td>11</td>\n",
              "      <td>Ward 19</td>\n",
              "      <td>1903</td>\n",
              "      <td>413 Centre St</td>\n",
              "      <td>2130.0</td>\n",
              "      <td>42.3214</td>\n",
              "      <td>-71.1109</td>\n",
              "      <td>Constituent Call</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>101002296862</td>\n",
              "      <td>2018-01-01 00:09:46</td>\n",
              "      <td>2018-01-02 08:30:00</td>\n",
              "      <td>2018-01-02 21:15:22</td>\n",
              "      <td>OVERDUE</td>\n",
              "      <td>Closed</td>\n",
              "      <td>Case Closed. Closed date : 2018-01-02 21:15:22...</td>\n",
              "      <td>Parking Enforcement</td>\n",
              "      <td>Transportation - Traffic Division</td>\n",
              "      <td>Enforcement &amp; Abandoned Vehicles</td>\n",
              "      <td>...</td>\n",
              "      <td>C6</td>\n",
              "      <td>South Boston / South Boston Waterfront</td>\n",
              "      <td>5</td>\n",
              "      <td>Ward 6</td>\n",
              "      <td>0604</td>\n",
              "      <td>428 W Second St</td>\n",
              "      <td>2127.0</td>\n",
              "      <td>42.3373</td>\n",
              "      <td>-71.0445</td>\n",
              "      <td>Citizens Connect App</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>101002296864</td>\n",
              "      <td>2018-01-01 00:11:24</td>\n",
              "      <td>2018-01-02 08:30:00</td>\n",
              "      <td>2018-01-02 21:15:11</td>\n",
              "      <td>OVERDUE</td>\n",
              "      <td>Closed</td>\n",
              "      <td>Case Closed. Closed date : 2018-01-02 21:15:11...</td>\n",
              "      <td>Parking Enforcement</td>\n",
              "      <td>Transportation - Traffic Division</td>\n",
              "      <td>Enforcement &amp; Abandoned Vehicles</td>\n",
              "      <td>...</td>\n",
              "      <td>C6</td>\n",
              "      <td>South Boston / South Boston Waterfront</td>\n",
              "      <td>5</td>\n",
              "      <td>Ward 6</td>\n",
              "      <td>0604</td>\n",
              "      <td>401-403 W Second St</td>\n",
              "      <td>2127.0</td>\n",
              "      <td>42.3371</td>\n",
              "      <td>-71.0442</td>\n",
              "      <td>Citizens Connect App</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>101002296865</td>\n",
              "      <td>2018-01-01 00:19:06</td>\n",
              "      <td>2018-01-02 08:30:00</td>\n",
              "      <td>2018-01-02 21:14:48</td>\n",
              "      <td>OVERDUE</td>\n",
              "      <td>Closed</td>\n",
              "      <td>Case Closed. Closed date : 2018-01-02 21:14:48...</td>\n",
              "      <td>Parking Enforcement</td>\n",
              "      <td>Transportation - Traffic Division</td>\n",
              "      <td>Enforcement &amp; Abandoned Vehicles</td>\n",
              "      <td>...</td>\n",
              "      <td>B3</td>\n",
              "      <td>Dorchester</td>\n",
              "      <td>9</td>\n",
              "      <td>17</td>\n",
              "      <td>1701</td>\n",
              "      <td>INTERSECTION School St &amp; Washington St</td>\n",
              "      <td>NaN</td>\n",
              "      <td>42.3594</td>\n",
              "      <td>-71.0587</td>\n",
              "      <td>Citizens Connect App</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>101002296866</td>\n",
              "      <td>2018-01-01 00:21:25</td>\n",
              "      <td>2018-01-02 08:30:00</td>\n",
              "      <td>2018-01-02 21:14:32</td>\n",
              "      <td>OVERDUE</td>\n",
              "      <td>Closed</td>\n",
              "      <td>Case Closed. Closed date : 2018-01-02 21:14:32...</td>\n",
              "      <td>Parking Enforcement</td>\n",
              "      <td>Transportation - Traffic Division</td>\n",
              "      <td>Enforcement &amp; Abandoned Vehicles</td>\n",
              "      <td>...</td>\n",
              "      <td>B3</td>\n",
              "      <td>Dorchester</td>\n",
              "      <td>9</td>\n",
              "      <td>Ward 17</td>\n",
              "      <td>1701</td>\n",
              "      <td>6-8 School St</td>\n",
              "      <td>2124.0</td>\n",
              "      <td>42.2965</td>\n",
              "      <td>-71.0729</td>\n",
              "      <td>Citizens Connect App</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 29 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-28741581-0110-4ee8-b186-f02874cef00a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-28741581-0110-4ee8-b186-f02874cef00a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-28741581-0110-4ee8-b186-f02874cef00a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-93bb97d3-bcbf-4b45-ade3-2884b98c8311\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-93bb97d3-bcbf-4b45-ade3-2884b98c8311')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-93bb97d3-bcbf-4b45-ade3-2884b98c8311 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "   case_enquiry_id              open_dt            target_dt  \\\n",
              "0     101002296861  2018-01-01 00:08:00                  NaN   \n",
              "1     101002296862  2018-01-01 00:09:46  2018-01-02 08:30:00   \n",
              "2     101002296864  2018-01-01 00:11:24  2018-01-02 08:30:00   \n",
              "3     101002296865  2018-01-01 00:19:06  2018-01-02 08:30:00   \n",
              "4     101002296866  2018-01-01 00:21:25  2018-01-02 08:30:00   \n",
              "\n",
              "             closed_dt   ontime case_status  \\\n",
              "0  2018-03-01 15:18:12   ONTIME      Closed   \n",
              "1  2018-01-02 21:15:22  OVERDUE      Closed   \n",
              "2  2018-01-02 21:15:11  OVERDUE      Closed   \n",
              "3  2018-01-02 21:14:48  OVERDUE      Closed   \n",
              "4  2018-01-02 21:14:32  OVERDUE      Closed   \n",
              "\n",
              "                                      closure_reason           case_title  \\\n",
              "0  Case Closed. Closed date : 2018-03-01 15:18:12...         Fire Hydrant   \n",
              "1  Case Closed. Closed date : 2018-01-02 21:15:22...  Parking Enforcement   \n",
              "2  Case Closed. Closed date : 2018-01-02 21:15:11...  Parking Enforcement   \n",
              "3  Case Closed. Closed date : 2018-01-02 21:14:48...  Parking Enforcement   \n",
              "4  Case Closed. Closed date : 2018-01-02 21:14:32...  Parking Enforcement   \n",
              "\n",
              "                             subject                            reason  ...  \\\n",
              "0    Boston Water & Sewer Commission                      Fire Hydrant  ...   \n",
              "1  Transportation - Traffic Division  Enforcement & Abandoned Vehicles  ...   \n",
              "2  Transportation - Traffic Division  Enforcement & Abandoned Vehicles  ...   \n",
              "3  Transportation - Traffic Division  Enforcement & Abandoned Vehicles  ...   \n",
              "4  Transportation - Traffic Division  Enforcement & Abandoned Vehicles  ...   \n",
              "\n",
              "  police_district                            neighborhood  \\\n",
              "0             E13                           Jamaica Plain   \n",
              "1              C6  South Boston / South Boston Waterfront   \n",
              "2              C6  South Boston / South Boston Waterfront   \n",
              "3              B3                              Dorchester   \n",
              "4              B3                              Dorchester   \n",
              "\n",
              "  neighborhood_services_district     ward precinct  \\\n",
              "0                             11  Ward 19     1903   \n",
              "1                              5   Ward 6     0604   \n",
              "2                              5   Ward 6     0604   \n",
              "3                              9       17     1701   \n",
              "4                              9  Ward 17     1701   \n",
              "\n",
              "                     location_street_name location_zipcode latitude longitude  \\\n",
              "0                           413 Centre St           2130.0  42.3214  -71.1109   \n",
              "1                         428 W Second St           2127.0  42.3373  -71.0445   \n",
              "2                     401-403 W Second St           2127.0  42.3371  -71.0442   \n",
              "3  INTERSECTION School St & Washington St              NaN  42.3594  -71.0587   \n",
              "4                           6-8 School St           2124.0  42.2965  -71.0729   \n",
              "\n",
              "                 source  \n",
              "0      Constituent Call  \n",
              "1  Citizens Connect App  \n",
              "2  Citizens Connect App  \n",
              "3  Citizens Connect App  \n",
              "4  Citizens Connect App  \n",
              "\n",
              "[5 rows x 29 columns]"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dfs[\"2018\"].head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QTaX8pkZ0OjR",
        "outputId": "c957c2ac-c318-4ce1-a133-8a84970d0dff"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting boto3\n",
            "  Downloading boto3-1.34.144-py3-none-any.whl (139 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.2/139.2 kB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting botocore<1.35.0,>=1.34.144 (from boto3)\n",
            "  Downloading botocore-1.34.144-py3-none-any.whl (12.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.4/12.4 MB\u001b[0m \u001b[31m49.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1 (from boto3)\n",
            "  Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n",
            "Collecting s3transfer<0.11.0,>=0.10.0 (from boto3)\n",
            "  Downloading s3transfer-0.10.2-py3-none-any.whl (82 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.7/82.7 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.144->boto3) (2.8.2)\n",
            "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.144->boto3) (2.0.7)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.144->boto3) (1.16.0)\n",
            "Installing collected packages: jmespath, botocore, s3transfer, boto3\n",
            "Successfully installed boto3-1.34.144 botocore-1.34.144 jmespath-1.0.1 s3transfer-0.10.2\n"
          ]
        }
      ],
      "source": [
        "pip install boto3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VEPewpDz2x45"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import boto3\n",
        "\n",
        "aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID')\n",
        "aws_secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY')\n",
        "region_name = \"us-east-2\"\n",
        "\n",
        "boto3.setup_default_session(\n",
        "    aws_access_key_id=aws_access_key_id,\n",
        "    aws_secret_access_key=aws_secret_access_key,\n",
        "    region_name=region_name\n",
        ")\n",
        "\n",
        "s3 = boto3.client(\"s3\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5pCr-jmdPM5c"
      },
      "source": [
        "Salvando arquivo em Parquet\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kWXSEHz-PQ1z"
      },
      "outputs": [],
      "source": [
        "from io import BytesIO\n",
        "\n",
        "for ano, df in dfs.items():\n",
        "  parquet_buffer = BytesIO()\n",
        "  df.to_parquet(parquet_buffer)\n",
        "\n",
        "  s3.put_object(\n",
        "      Bucket=\"datalake-aws1\",\n",
        "      Key=f\"Bronze/dados_{ano}.parquet\",\n",
        "      Body=parquet_buffer.getvalue(),\n",
        "  )\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y-8mpiuXRHe1"
      },
      "outputs": [],
      "source": [
        "response = s3.list_objects(Bucket=\"datalake-aws1\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GR7LujWxU52L",
        "outputId": "44ef9969-4788-44a3-e823-08f5a2495eb8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['Bronze/', 'Bronze/dados_2015.parquet', 'Bronze/dados_2016.parquet', 'Bronze/dados_2017.parquet', 'Bronze/dados_2018.parquet', 'Bronze/dados_2019.parquet', 'Bronze/dados_2020.parquet', 'Bronze/hello-s3', 'Gold/', 'Silver/']\n"
          ]
        }
      ],
      "source": [
        "keys = [obj[\"Key\"] for obj in response[\"Contents\"]]\n",
        "print(keys)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}