Merge pull request #23 from carbonplan/s3_links

updates azure to s3

Author: norlandrhagen

notebooks/Figure-1.ipynb

@@ -16,20 +16,21 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from collections import Counter\n",
-    "import fsspec\n",
     "import json\n",
     "import os\n",
+    "from collections import Counter\n",
     "\n",
+    "import fsspec\n",
     "import matplotlib.pyplot as plt\n",
     "import numpy as np\n",
     "import pandas as pd\n",
     "import seaborn as sns\n",
-    "from sklearn.metrics import mean_squared_error\n",
-    "\n",
-    "from carbonplan_forest_offsets.analysis.project_crediting_error import get_slag_to_total_scalar\n",
+    "from carbonplan_forest_offsets.analysis.project_crediting_error import (\n",
+    "    get_slag_to_total_scalar,\n",
+    ")\n",
     "from carbonplan_forest_offsets.data import cat\n",
-    "from carbonplan_forest_offsets.load.issuance import load_issuance_table, ifm_opr_ids"
+    "from carbonplan_forest_offsets.load.issuance import ifm_opr_ids, load_issuance_table\n",
+    "from sklearn.metrics import mean_squared_error"
    ]
   },
   {
@@ -133,7 +134,8 @@
     "subsets = {\n",
     "    \"all\": np.tile(True, len(df)),\n",
     "    \"all_forest\": df[\"project_type\"] == \"forest\",\n",
-    "    \"compliance_ifm\": (df[\"opr_id\"].isin(ifm_opr_ids)) & (df[\"Early Action/ Compliance\"] == \"COP\"),\n",
+    "    \"compliance_ifm\": (df[\"opr_id\"].isin(ifm_opr_ids))\n",
+    "    & (df[\"Early Action/ Compliance\"] == \"COP\"),\n",
     "    \"non_graduated_compliance_ifms\": (df[\"opr_id\"].isin(compliance_opr_ids))\n",
     "    & (df[\"Early Action/ Compliance\"] == \"COP\"),\n",
     "    \"upfront_ifm\": (df[\"opr_id\"].isin(upfront_opr_ids)) & (df[\"arb_rp_id\"].isin([\"A\"])),\n",
@@ -202,7 +204,9 @@
    "source": [
     "sc_data = cat.rfia_all(assessment_area_id=297).read()\n",
     "\n",
-    "sc_data = sc_data[sc_data[\"YEAR\"] == 2010].copy()  # use 2010 because comporable to CP data\n",
+    "sc_data = sc_data[\n",
+    "    sc_data[\"YEAR\"] == 2010\n",
+    "].copy()  # use 2010 because comporable to CP data\n",
     "\n",
     "\n",
     "sc_data[\"CARB_ACRE\"] = sc_data[\"CARB_ACRE\"] * 44 / 12 * 0.907185"
@@ -237,7 +241,9 @@
    "source": [
     "standing_carbon = {}\n",
     "for k, v in fortyps_of_interest.items():\n",
-    "    standing_carbon[k] = round(sc_data.loc[sc_data[\"FORTYPCD\"] == v, \"CARB_ACRE\"].item(), 1)\n",
+    "    standing_carbon[k] = round(\n",
+    "        sc_data.loc[sc_data[\"FORTYPCD\"] == v, \"CARB_ACRE\"].item(), 1\n",
+    "    )\n",
     "display(standing_carbon)"
    ]
   },
@@ -258,7 +264,9 @@
    "source": [
     "# ak has three assessment areas but lets summarize across all to report inline value\n",
     "ak_assessment_areas = [285, 286, 287]\n",
-    "ak_all = pd.concat([cat.rfia_all(assessment_area_id=aa_id).read() for aa_id in ak_assessment_areas])\n",
+    "ak_all = pd.concat(\n",
+    "    [cat.rfia_all(assessment_area_id=aa_id).read() for aa_id in ak_assessment_areas]\n",
+    ")\n",
     "\n",
     "ak_all = ak_all[ak_all[\"YEAR\"] == 2013].copy()  # 2013 to match what used in CP\n",
     "\n",
@@ -321,7 +329,7 @@
    "outputs": [],
    "source": [
     "with fsspec.open(\n",
-    "    \"https://carbonplan.blob.core.windows.net/carbonplan-forests/offsets/archive/results/reclassification-crediting-error.json\",\n",
+    "    \"https://carbonplan-forests.s3.us-west-2.amazonaws.com/offsets/archive/results/reclassification-crediting-error.json\",\n",
     "    mode=\"r\",\n",
     ") as f:\n",
     "    reclass_credit = json.load(f)"
@@ -340,7 +348,9 @@
     "    ).round(1)\n",
     "\n",
     "    as_frac = crediting_error / project[\"arbocs\"][\"calculated\"]\n",
-    "    print(f\"{project['opr_id']} has a {crediting_error[1]} crediting error ({as_frac[1].round(3)})\")\n",
+    "    print(\n",
+    "        f\"{project['opr_id']} has a {crediting_error[1]} crediting error ({as_frac[1].round(3)})\"\n",
+    "    )\n",
     "    print(f\"CI: {crediting_error[0]}, {crediting_error[2]}\")\n",
     "    print(f\"% CI: {as_frac[0].round(3)}, {as_frac[2].round(3)}\")"
    ]
@@ -429,7 +439,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "fn = \"https://carbonplan.blob.core.windows.net/carbonplan-forests/offsets/archive/intermediate/rfia/297_by_ecosection.csv\"\n",
+    "fn = \"https://carbonplan-forests.s3.us-west-2.amazonaws.com/offsets/archive/intermediate/rfia/297_by_ecosection.csv\"\n",
     "with fsspec.open(fn, mode=\"r\") as f:\n",
     "    data = pd.read_csv(f)\n",
     "\n",
@@ -669,7 +679,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "fn = \"https://carbonplan.blob.core.windows.net/carbonplan-forests/offsets/archive/results/classifier_fscores.json\"\n",
+    "fn = \"https://carbonplan-forests.s3.us-west-2.amazonaws.com/offsets/archive/results/classifier_fscores.json\"\n",
     "with fsspec.open(fn, mode=\"r\") as f:\n",
     "    fscores = json.load(f)"
    ]
@@ -886,7 +896,8 @@
     "projects = [\n",
     "    x\n",
     "    for x in db\n",
-    "    if x[\"carbon\"][\"initial_carbon_stock\"][\"value\"] > x[\"carbon\"][\"common_practice\"][\"value\"]\n",
+    "    if x[\"carbon\"][\"initial_carbon_stock\"][\"value\"]\n",
+    "    > x[\"carbon\"][\"common_practice\"][\"value\"]\n",
     "]"
    ]
   },
@@ -937,7 +948,7 @@
     }
    ],
    "source": [
-    "sum((cp_df[\"baseline\"] <= cp_df[\"cp\"] * 1.05)) / len(cp_df.dropna())"
+    "sum(cp_df[\"baseline\"] <= cp_df[\"cp\"] * 1.05) / len(cp_df.dropna())"
    ]
   },
   {
@@ -955,7 +966,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "fn = f\"https://carbonplan.blob.core.windows.net/carbonplan-forests/offsets/archive/results/common-practice-verification.json\"\n",
+    "fn = f\"https://carbonplan-forests.s3.us-west-2.amazonaws.com/offsets/archive/results/common-practice-verification.json\"\n",
     "with fsspec.open(fn, mode=\"r\") as f:\n",
     "    cp_verification = json.load(f)"
    ]
@@ -1000,7 +1011,9 @@
     }
    ],
    "source": [
-    "data = pd.DataFrame(cp_verification[\"projects\"])[[\"opr_id\", \"recalculated\", \"project_reported\"]]\n",
+    "data = pd.DataFrame(cp_verification[\"projects\"])[\n",
+    "    [\"opr_id\", \"recalculated\", \"project_reported\"]\n",
+    "]\n",
     "mean_squared_error(data[\"recalculated\"], data[\"project_reported\"]) ** 0.5"
    ]
   },
@@ -1011,7 +1024,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "data[\"diff\"] = (data[\"recalculated\"] - data[\"project_reported\"]) / (data[\"project_reported\"])\n",
+    "data[\"diff\"] = (data[\"recalculated\"] - data[\"project_reported\"]) / (\n",
+    "    data[\"project_reported\"]\n",
+    ")\n",
     "data = data[np.isfinite(data[\"diff\"])]  # CAR1186 = infite bc original CP = 0."
    ]
   },
@@ -1182,13 +1197,15 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "fn = \"https://carbonplan.blob.core.windows.net/carbonplan-forests/offsets/archive/results/reclassification-crediting-error.json\"\n",
+    "fn = \"https://carbonplan-forests.s3.us-west-2.amazonaws.com/offsets/archive/results/reclassification-crediting-error.json\"\n",
     "with fsspec.open(fn, mode=\"r\") as f:\n",
     "    crediting_error = json.load(f)\n",
     "\n",
     "crediting_df = pd.DataFrame({k: v[\"delta_arbocs\"] for k, v in crediting_error.items()})\n",
     "\n",
-    "median_crediting_error = {k: np.median(v[\"delta_arbocs\"]) for k, v in crediting_error.items()}"
+    "median_crediting_error = {\n",
+    "    k: np.median(v[\"delta_arbocs\"]) for k, v in crediting_error.items()\n",
+    "}"
    ]
   },
   {
@@ -1199,7 +1216,10 @@
    "outputs": [],
    "source": [
     "tp = pd.concat(\n",
-    "    [pd.Series(median_crediting_error).rename(\"crediting_error\"), error_cp0.rename(\"cp\")],\n",
+    "    [\n",
+    "        pd.Series(median_crediting_error).rename(\"crediting_error\"),\n",
+    "        error_cp0.rename(\"cp\"),\n",
+    "    ],\n",
     "    axis=1,\n",
     ")"
    ]
@@ -1332,7 +1352,9 @@
    "outputs": [],
    "source": [
     "ak_assessment_areas = [285, 286, 287]\n",
-    "ak_all = pd.concat([cat.rfia_all(assessment_area_id=aa_id).read() for aa_id in ak_assessment_areas])"
+    "ak_all = pd.concat(\n",
+    "    [cat.rfia_all(assessment_area_id=aa_id).read() for aa_id in ak_assessment_areas]\n",
+    ")"
    ]
   },
   {
@@ -1388,7 +1410,13 @@
     }
    ],
    "source": [
-    "sum([project[\"arbocs\"][\"issuance\"] for project in db if 287 in project[\"supersection_ids\"]])"
+    "sum(\n",
+    "    [\n",
+    "        project[\"arbocs\"][\"issuance\"]\n",
+    "        for project in db\n",
+    "        if 287 in project[\"supersection_ids\"]\n",
+    "    ]\n",
+    ")"
    ]
   },
   {

notebooks/Figure-3.ipynb

@@ -7,11 +7,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import json\n",
     "import os\n",
+    "\n",
     "import fsspec\n",
-    "import json\n",
-    "import pandas as pd\n",
-    "import numpy as np"
+    "import numpy as np\n",
+    "import pandas as pd"
    ]
   },
   {
@@ -38,13 +39,13 @@
    "outputs": [],
    "source": [
     "with fsspec.open(\n",
-    "    \"https://carbonplan.blob.core.windows.net/carbonplan-forests/offsets/archive/results/reclassification-crediting-error.json\",\n",
+    "    \"https://carbonplan-forests.s3.us-west-2.amazonaws.com/offsets/archive/results/reclassification-crediting-error.json\",\n",
     "    \"r\",\n",
     ") as f:\n",
     "    data = json.load(f)\n",
     "\n",
     "with fsspec.open(\n",
-    "    \"https://carbonplan.blob.core.windows.net/carbonplan-forests/offsets/database/forest-offsets-database-v1.0.json\",\n",
+    "    \"https://carbonplan-forests.s3.us-west-2.amazonaws.com/offsets/database/forest-offsets-database-v1.0.json\",\n",
     "    \"r\",\n",
     ") as f:\n",
     "    db = json.load(f)\n",
@@ -112,8 +113,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "def format_si(num, precision=1, suffixes=[\"\", \"K\", \"M\", \"G\", \"T\", \"P\"], hide_suffix=False):\n",
-    "    m = sum([abs(num / 1000.0 ** x) >= 1 for x in range(1, len(suffixes))])\n",
+    "def format_si(\n",
+    "    num, precision=1, suffixes=[\"\", \"K\", \"M\", \"G\", \"T\", \"P\"], hide_suffix=False\n",
+    "):\n",
+    "    m = sum([abs(num / 1000.0**x) >= 1 for x in range(1, len(suffixes))])\n",
     "    if hide_suffix:\n",
     "        return f\"{num/1000.0**m:.{precision}f}\"\n",
     "    else:\n",
@@ -252,7 +255,9 @@
     }
    ],
    "source": [
-    "condition = lambda x: \"Finite Carbon\" in x[\"developers\"] or \"Finite Carbon\" in x[\"owners\"]\n",
+    "condition = (\n",
+    "    lambda x: \"Finite Carbon\" in x[\"developers\"] or \"Finite Carbon\" in x[\"owners\"]\n",
+    ")\n",
     "get_overcrediting(condition=condition, percentage=True, display=True)"
    ]
   },
@@ -315,7 +320,9 @@
     "df = pd.DataFrame()\n",
     "df[\"Developer\"] = developers\n",
     "results = [\n",
-    "    get_overcrediting(condition=lambda x: d in x[\"developers\"] or d in x[\"owners\"], percentage=True)\n",
+    "    get_overcrediting(\n",
+    "        condition=lambda x: d in x[\"developers\"] or d in x[\"owners\"], percentage=True\n",
+    "    )\n",
     "    for d in developers\n",
     "]\n",
     "df[\"Count\"] = [d[\"count\"] for d in results]\n",

notebooks/Figure-5.ipynb

@@ -15,13 +15,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import json\n",
     "import os\n",
+    "import random\n",
+    "\n",
     "import fsspec\n",
-    "import json\n",
-    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
     "import numpy as np\n",
-    "import random\n",
-    "import matplotlib.pyplot as plt"
+    "import pandas as pd"
    ]
   },
   {
@@ -34,13 +35,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "700c2753",
    "metadata": {},
    "outputs": [],
    "source": [
     "with fsspec.open(\n",
-    "    \"https://carbonplan.blob.core.windows.net/carbonplan-forests/offsets/archive/results/classifier_fscores.json\",\n",
+    "    \"https://carbonplan-forests.s3.us-west-2.amazonaws.com/offsets/archive/results/classifier_fscores.json\",\n",
     "    \"r\",\n",
     ") as f:\n",
     "    data = json.load(f)"

notebooks/Figure-6.ipynb

@@ -15,13 +15,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import json\n",
     "import os\n",
+    "import random\n",
+    "\n",
     "import fsspec\n",
-    "import json\n",
-    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
     "import numpy as np\n",
-    "import random\n",
-    "import matplotlib.pyplot as plt"
+    "import pandas as pd"
    ]
   },
   {
@@ -34,27 +35,27 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "d4950531",
    "metadata": {},
    "outputs": [],
    "source": [
     "with fsspec.open(\n",
-    "    \"https://carbonplan.blob.core.windows.net/carbonplan-forests/offsets/archive/results/reclassification-labels.json\",\n",
+    "    \"https://carbonplan-forests.s3.us-west-2.amazonaws.com/offsets/archive/results/reclassification-labels.json\",\n",
     "    \"r\",\n",
     ") as f:\n",
     "    data = json.load(f)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "id": "d85ce3ac",
    "metadata": {},
    "outputs": [],
    "source": [
     "with fsspec.open(\n",
-    "    \"https://carbonplan.blob.core.windows.net/carbonplan-forests/offsets/archive/results/reclassification-crediting-error.json\",\n",
+    "    \"https://carbonplan-forests.s3.us-west-2.amazonaws.com/offsets/archive/results/reclassification-crediting-error.json\",\n",
     "    \"r\",\n",
     ") as f:\n",
     "    analysis = json.load(f)"
@@ -90,7 +91,10 @@
     "]\n",
     "df[\"Classification\"] = [\n",
     "    \"\\n\".join(\n",
-    "        [str(s[0]).capitalize() + \" : \" + \"%.1f\" % (s[1] * 100) + \"%\" for s in d[\"classification\"]]\n",
+    "        [\n",
+    "            str(s[0]).capitalize() + \" : \" + \"%.1f\" % (s[1] * 100) + \"%\"\n",
+    "            for s in d[\"classification\"]\n",
+    "        ]\n",
     "    )\n",
     "    for d in data\n",
     "]"