docs(notebooks): automate data access in tutorials/examples (#2392)

First step towards #1872. Use pooch for data access. This is ugly, but it makes notebooks runnable (provided exes and python environment) out of the box. Local files will be used if detected, otherwise downloaded, following the pattern in the mf6 example models.

An eventual models API could hide all the details of model access.

Also mention the optional dependencies requirement on the tutorials and examples gallery pages.

Author: wpbonelli

.docs/Notebooks/array_output_tutorial.py

@@ -29,12 +29,15 @@
 # + pycharm={"name": "#%%\n"}
 import os
 import sys
+from pathlib import Path
 from pprint import pformat
 from tempfile import TemporaryDirectory
 
+import git
 import matplotlib as mpl
 import matplotlib.pyplot as plt
 import numpy as np
+import pooch
 
 import flopy
 
@@ -44,8 +47,40 @@
 exe_name = "mf2005"
 mfexe = exe_name
 
+# Check if we are in the repository and define the data path.
+
+try:
+    root = Path(git.Repo(".", search_parent_directories=True).working_dir)
+except:
+    root = None
+
+data_path = root / "examples" / "data" if root else Path.cwd()
+
+sim_name = "freyberg"
+
+file_names = {
+    "freyberg.bas": "63266024019fef07306b8b639c6c67d5e4b22f73e42dcaa9db18b5e0f692c097",
+    "freyberg.dis": "62d0163bf36c7ee9f7ee3683263e08a0abcdedf267beedce6dd181600380b0a2",
+    "freyberg.githds": "abe92497b55e6f6c73306e81399209e1cada34cf794a7867d776cfd18303673b",
+    "freyberg.gitlist": "aef02c664344a288264d5f21e08a748150e43bb721a16b0e3f423e6e3e293056",
+    "freyberg.lpf": "06500bff979424f58e5e4fbd07a7bdeb0c78f31bd08640196044b6ccefa7a1fe",
+    "freyberg.nam": "e66321007bb603ef55ed2ba41f4035ba6891da704a4cbd3967f0c66ef1532c8f",
+    "freyberg.oc": "532905839ccbfce01184980c230b6305812610b537520bf5a4abbcd3bd703ef4",
+    "freyberg.pcg": "0d1686fac4680219fffdb56909296c5031029974171e25d4304e70fa96ebfc38",
+    "freyberg.rch": "37a1e113a7ec16b61417d1fa9710dd111a595de738a367bd34fd4a359c480906",
+    "freyberg.riv": "7492a1d5eb23d6812ec7c8227d0ad4d1e1b35631a765c71182b71e3bd6a6d31d",
+    "freyberg.wel": "00aa55f59797c02f0be5318a523b36b168fc6651f238f34e8b0938c04292d3e7",
+}
+for fname, fhash in file_names.items():
+    pooch.retrieve(
+        url=f"https://github.com/modflowpy/flopy/raw/develop/examples/data/{sim_name}/{fname}",
+        fname=fname,
+        path=data_path / sim_name,
+        known_hash=fhash,
+    )
+
 # Set the paths
-loadpth = os.path.join("..", "..", "examples", "data", "freyberg")
+loadpth = data_path / sim_name
 temp_dir = TemporaryDirectory()
 modelpth = temp_dir.name
 

.docs/Notebooks/export_tutorial.py

@@ -20,8 +20,12 @@
 # +
 import os
 import sys
+from pathlib import Path
 from tempfile import TemporaryDirectory
 
+import git
+import pooch
+
 import flopy
 
 print(sys.version)
@@ -30,8 +34,42 @@
 
 # Load our old friend...the Freyberg model
 
+sim_name = "freyberg_multilayer_transient"
+
+# Check if we are in the repository and define the data path.
+
+try:
+    root = Path(git.Repo(".", search_parent_directories=True).working_dir)
+except:
+    root = None
+
+data_path = root / "examples" / "data" if root else Path.cwd()
+
+file_names = {
+    "freyberg.bas": None,
+    "freyberg.cbc": None,
+    "freyberg.ddn": None,
+    "freyberg.dis": None,
+    "freyberg.drn": None,
+    "freyberg.hds": None,
+    "freyberg.list": None,
+    "freyberg.nam": None,
+    "freyberg.nwt": None,
+    "freyberg.oc": None,
+    "freyberg.rch": None,
+    "freyberg.upw": None,
+    "freyberg.wel": None,
+}
+for fname, fhash in file_names.items():
+    pooch.retrieve(
+        url=f"https://github.com/modflowpy/flopy/raw/develop/examples/data/{sim_name}/{fname}",
+        fname=fname,
+        path=data_path / sim_name,
+        known_hash=fhash,
+    )
+
 nam_file = "freyberg.nam"
-model_ws = os.path.join("..", "..", "examples", "data", "freyberg_multilayer_transient")
+model_ws = data_path / sim_name
 ml = flopy.modflow.Modflow.load(nam_file, model_ws=model_ws, check=False)
 
 # We can see the ``Modelgrid`` instance has generic entries, as does ``start_datetime``

.docs/Notebooks/export_vtk_tutorial.py

@@ -33,7 +33,9 @@
 from pprint import pformat
 from tempfile import TemporaryDirectory
 
+import git
 import numpy as np
+import pooch
 
 import flopy
 from flopy.export import vtk
@@ -42,11 +44,39 @@
 print(f"flopy version: {flopy.__version__}")
 # -
 
+try:
+    root = Path(git.Repo(".", search_parent_directories=True).working_dir)
+except:
+    root = None
+
+data_path = root / "examples" / "data" if root else Path.cwd()
+sim_name = "freyberg_multilayer_transient"
+file_names = {
+    "freyberg.bas": None,
+    "freyberg.cbc": None,
+    "freyberg.ddn": None,
+    "freyberg.dis": None,
+    "freyberg.drn": None,
+    "freyberg.hds": None,
+    "freyberg.list": None,
+    "freyberg.nam": None,
+    "freyberg.nwt": None,
+    "freyberg.oc": None,
+    "freyberg.rch": None,
+    "freyberg.upw": None,
+    "freyberg.wel": None,
+}
+for fname, fhash in file_names.items():
+    pooch.retrieve(
+        url=f"https://github.com/modflowpy/flopy/raw/develop/examples/data/{sim_name}/{fname}",
+        fname=fname,
+        path=data_path / sim_name,
+        known_hash=fhash,
+    )
+
 # load model for examples
 nam_file = "freyberg.nam"
-model_ws = Path(
-    os.path.join("..", "..", "examples", "data", "freyberg_multilayer_transient")
-)
+model_ws = data_path / sim_name
 ml = flopy.modflow.Modflow.load(nam_file, model_ws=model_ws, check=False)
 
 # Create a temporary workspace.

.docs/Notebooks/feat_working_stack_examples.py

@@ -23,33 +23,68 @@
 from pprint import pformat
 from tempfile import TemporaryDirectory
 
+import git
 import matplotlib as mpl
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
-
-# +
+import pooch
 from IPython.display import clear_output, display
 
-proj_root = Path.cwd().parent.parent
-
-# run installed version of flopy or add local path
 import flopy
 
 print(sys.version)
 print(f"numpy version: {np.__version__}")
 print(f"matplotlib version: {mpl.__version__}")
 print(f"pandas version: {pd.__version__}")
 print(f"flopy version: {flopy.__version__}")
-# -
 
+# First create a temporary workspace.
+
+sim_name = "freyberg_multilayer_transient"
+temp_dir = TemporaryDirectory()
+workspace = Path(temp_dir.name)
+
+# Check if we are in the repository and define the data path.
+
+try:
+    root = Path(git.Repo(".", search_parent_directories=True).working_dir)
+except:
+    root = None
+
+data_path = root / "examples" / "data" if root else Path.cwd()
+
+# Download files if needed.
+
+file_names = {
+    "freyberg.bas": "781585c140d40a27bce9369baee262c621bcf969de82361ad8d6b4d8c253ee02",
+    "freyberg.cbc": "d4e18e968cabde8470fcb7cb8a1c4cc57fcd643bd63b23e7751460bfdb651ea4",
+    "freyberg.ddn": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "freyberg.dis": "1ef61a467a219c036e58902ce11297e06b4eeb5f2f9d2ea40245b421a248a471",
+    "freyberg.drn": "93c22ab27d599938a8c2fc5b420ec03e5251b11b050d6ae1cb23ce2aa1b77997",
+    "freyberg.hds": "0b3e911ef35f625d2d046e05a20bc1300341b41028220c5b25ace6f5a267ceef",
+    "freyberg.list": "14ec36c22b48d253d6b82c44f36c5bad4f0785b3a3384b386f6b69c4ee2e31bf",
+    "freyberg.nam": "9e3747ce6d6229caec55a9357285a96cb4608dae11d90dd165a23e0bb394a2bd",
+    "freyberg.nwt": "d66c5cc255d050a0f871639af4af0cef8d48fa59c1c64217de65fc6e7fd78cb1",
+    "freyberg.oc": "faefd462d11b9a21c4579420b2156fb616ca642bc1e66fc5eb5e1b9046449e43",
+    "freyberg.rch": "93a12742a2d37961d53df0405e39cbecf0e6f14d45b5ca8cbba84a2d90828258",
+    "freyberg.upw": "80838be7af2f97c92965bad1d121c252b69d9c66e4885c5f3f49a6e99582deac",
+    "freyberg.wel": "dd322655eadff3f618f0835c9277af30720197bd48328aae2d6772f26eef2686",
+}
+for fname, fhash in file_names.items():
+    pooch.retrieve(
+        url=f"https://github.com/modflowpy/flopy/raw/develop/examples/data/{sim_name}/{fname}",
+        fname=fname,
+        path=data_path / sim_name,
+        known_hash=fhash,
+    )
+
+# -
 # ### Model Inputs
 
-# first lets load an existing model
-model_ws = proj_root / "examples" / "data" / "freyberg_multilayer_transient"
 ml = flopy.modflow.Modflow.load(
     "freyberg.nam",
-    model_ws=model_ws,
+    model_ws=data_path / sim_name,
     verbose=False,
     check=False,
     exe_name="mfnwt",
@@ -66,11 +101,6 @@
 ml.drn.plot(key="cond")
 ml.drn.plot(key="elev")
 
-# First create a temporary workspace.
-
-# create a temporary workspace
-temp_dir = TemporaryDirectory()
-workspace = Path(temp_dir.name)
 
 # Write a shapefile of the DIS package.
 
@@ -96,7 +126,7 @@
 #
 # First, let's look at the list file. The list file summarizes the model's results.
 
-mfl = flopy.utils.MfListBudget(model_ws / "freyberg.list")
+mfl = flopy.utils.MfListBudget(workspace / "freyberg.list")
 df_flux, df_vol = mfl.get_dataframes(start_datetime="10-21-2015")
 df_flux
 
@@ -116,7 +146,7 @@
 # Now let's look at the simulated head.
 
 # if you pass the model instance, then the plots will be offset and rotated
-h = flopy.utils.HeadFile(model_ws / "freyberg.hds", model=ml)
+h = flopy.utils.HeadFile(workspace / "freyberg.hds", model=ml)
 h.times
 
 h.plot(totim=900, contour=True, grid=True, colorbar=True, figsize=(10, 10))

.docs/Notebooks/groundwater2023_watershed_example.py

@@ -25,10 +25,12 @@
 import pathlib as pl
 import sys
 
+import git
 import matplotlib as mpl
 import matplotlib.gridspec as gridspec
 import matplotlib.pyplot as plt
 import numpy as np
+import pooch
 import shapely
 import yaml
 from shapely.geometry import LineString, Polygon
@@ -106,10 +108,25 @@ def set_idomain(grid, boundary):
     grid.idomain = idomain
 
 
-geometries = yaml.safe_load(
-    open(pl.Path("../../examples/data/groundwater2023/geometries.yml"))
+# Check if we are in the repository and define the data path.
+
+try:
+    root = pl.Path(git.Repo(".", search_parent_directories=True).working_dir)
+except:
+    root = None
+
+data_path = root / "examples" / "data" if root else pl.Path.cwd()
+folder_name = "groundwater2023"
+fname = "geometries.yml"
+pooch.retrieve(
+    url=f"https://github.com/modflowpy/flopy/raw/develop/examples/data/{folder_name}/{fname}",
+    fname=fname,
+    path=data_path / folder_name,
+    known_hash=None,
 )
 
+geometries = yaml.safe_load(open(data_path / folder_name / fname))
+
 # basic figure size
 figwidth = 180  # 90 # mm
 figwidth = figwidth / 10 / 2.54  # inches
@@ -161,7 +178,13 @@ def set_idomain(grid, boundary):
     os.mkdir(temp_path)
 
 # Load the fine topography that will be sampled
-ascii_file = pl.Path("../../examples/data/geospatial/fine_topo.asc")
+fname = "fine_topo.asc"
+ascii_file = pooch.retrieve(
+    url=f"https://github.com/modflowpy/flopy/raw/develop/examples/data/geospatial/{fname}",
+    fname=fname,
+    path=data_path / "geospatial",
+    known_hash=None,
+)
 fine_topo = flopy.utils.Raster.load(ascii_file)
 
 # Define the problem size and extents

.docs/Notebooks/groundwater_paper_uspb_example.py

@@ -22,11 +22,14 @@
 # +
 import os
 import sys
+from pathlib import Path
 from pprint import pformat
 
+import git
 import matplotlib as mpl
 import matplotlib.pyplot as plt
 import numpy as np
+import pooch
 import scipy.ndimage
 
 import flopy
@@ -41,9 +44,23 @@
 if not os.path.exists(ws):
     os.makedirs(ws)
 
-fn = os.path.join(
-    "..", "groundwater_paper", "uspb", "results", "USPB_capture_fraction_04_01.dat"
+# Check if we are in the repository and define the data path.
+
+try:
+    root = Path(git.Repo(".", search_parent_directories=True).working_dir)
+except:
+    root = None
+
+data_path = root / ".docs" / "groundwater_paper" if root else Path.cwd()
+
+fname = "USPB_capture_fraction_04_01.dat"
+pooch.retrieve(
+    url=f"https://github.com/modflowpy/flopy/raw/develop/.docs/groundwater_paper/uspb/results/{fname}",
+    fname=fname,
+    path=data_path / "uspb" / "results",
+    known_hash=None,
 )
+fn = data_path / "uspb" / "results" / fname
 cf = np.loadtxt(fn)
 print(cf.shape)
 
@@ -53,7 +70,7 @@
 c = plt.imshow(cf2, cmap="jet")
 plt.colorbar(c)
 
-wsl = os.path.join("..", "groundwater_paper", "uspb", "flopy")
+wsl = data_path / "uspb" / "flopy"
 ml = flopy.modflow.Modflow.load("DG.nam", model_ws=wsl, verbose=False)
 
 nlay, nrow, ncol = ml.nlay, ml.dis.nrow, ml.dis.ncol
@@ -191,9 +208,14 @@
 plt.savefig(os.path.join(ws, "uspb_heads.png"), dpi=300)
 # -
 
-fn = os.path.join(
-    "..", "groundwater_paper", "uspb", "results", "USPB_capture_fraction_04_10.dat"
+fname = "USPB_capture_fraction_04_10.dat"
+pooch.retrieve(
+    url=f"https://github.com/modflowpy/flopy/raw/develop/.docs/groundwater_paper/uspb/results/{fname}",
+    fname=fname,
+    path=data_path / "uspb" / "results",
+    known_hash=None,
 )
+fn = data_path / "uspb" / "results" / fname
 cf = np.loadtxt(fn)
 cf2 = scipy.ndimage.zoom(cf, 4, order=0)