diff --git a/conda_requirements_test.txt b/conda_requirements_test.txt index 926ab40..099eef0 100644 --- a/conda_requirements_test.txt +++ b/conda_requirements_test.txt @@ -12,5 +12,6 @@ pytest pytest-cov progressbar +pooch diff --git a/examples/non_conforming_files/load_arbitrary_ugrid.py b/examples/non_conforming_files/load_arbitrary_ugrid.py index 3cf3201..2883f5f 100644 --- a/examples/non_conforming_files/load_arbitrary_ugrid.py +++ b/examples/non_conforming_files/load_arbitrary_ugrid.py @@ -5,7 +5,7 @@ an arbitrary text file. You should be able to work with it once loaded. -But ASaving it back out as a conforming file is broken: +But Saving it back out as a conforming file is broken: We need a "proper" way to save a full dataset. Currently the code can save a UGRId, but the rest of teh dataset info is lost. i.e. it can't find use the time variable. or it's coordinates. @@ -68,7 +68,6 @@ location='node', attributes=None, ) - # global attributes attrs = {key: nc.getncattr(key) for key in nc.ncattrs()} @@ -79,8 +78,6 @@ attributes=attrs ) - ## now learn a bit about it: - # What is its grid type? print("The dataset Grid is:", type(ds.grid)) diff --git a/gridded/gridded.py b/gridded/gridded.py index 28b4928..1f129b9 100644 --- a/gridded/gridded.py +++ b/gridded/gridded.py @@ -7,10 +7,7 @@ The core class that encapsulates the gridded data model """ - - - -# py2/3 compatibility +import warnings from gridded.grids import Grid from gridded.variable import Variable @@ -21,12 +18,8 @@ ) from . import VALID_LOCATIONS -""" -The main gridded.Dataset code -""" - -class Dataset(): +class Dataset: """ An object that represent an entire complete dataset -- a collection of Variables and the Grid that they are stored on. @@ -71,6 +64,12 @@ def __init__(self, If a filename is passed in, the attributes will be pulled from the file, and the input ones ignored. """ + if ncfile is not None: + # raise ValueError("don't create from a file") + warnings.warn("Creating a Dataset from a netcdfile directly is deprecated. " + "Please use Dataset.from_netCDF() instead. " + "Or one of the utilities in gridded.io", DeprecationWarning) + if ncfile is not None: if (grid is not None or variables is not None or @@ -90,12 +89,64 @@ def __init__(self, self.variables = {} if variables is None else variables self.attributes = {} if attributes is None else attributes + @classmethod + def from_netCDF(cls, + filename=None, + grid_file=None, + variable_files=None, + grid_topology=None): + """ + + NOTE: only loading from a single file is currently implemented. + you can create a DATaset by hand, by loading the grid and + variables separately, and then adding them + + load a gridded.Dataset from a netCDF file + + :param filename: filename or netCDF4 compatible OpeNDAP url. + It is assumed to contain the grid and variables. + :param grid_file: filename of the file that contains the grid, if separate. + + :param variable_files: filename of filenames that contain the variables. + + NOTE: Either the filename or the grid_file and variable_files should be specified. + not all three. + + :param grid_topology: mapping of grid topology components to netcdf variable names. + used to load non-confirming files. + :type grid_topology: mapping with keys of topology components and values are + variable names. + """ + if (grid_file is not None) or (variable_files is not None): + raise NotImplementedError("Loading from separate netcdf files is not yet supported") + + # create an empty DAtaset: + ds = cls() + + ds.nc_dataset = get_dataset(filename) + ds.filename = ds.nc_dataset.filepath() + ds.grid = Grid.from_netCDF(filename=ds.filename, + dataset=ds.nc_dataset, + grid_topology=grid_topology) + ds.variables = ds._load_variables(ds.nc_dataset) + ds.attributes = get_dataset_attrs(ds.nc_dataset) + + return ds + def __getitem__(self, key): """ shortcut to getting a variable object """ return self.variables[key] + def __str__(self): + descp = (f"gridded.Dataset with:\n" + f"grid: {type(self.grid)}\n" + f"variables: {list(self.variables.keys())}" + ) + return descp + + def _load_variables(self, ds): """ load up the variables in the nc file diff --git a/gridded/io/__init__.py b/gridded/io/__init__.py new file mode 100644 index 0000000..7206cd1 --- /dev/null +++ b/gridded/io/__init__.py @@ -0,0 +1,18 @@ +""" +subpackge for custom input / ouput code + + +At some point, it would be nice to have gridded have a +registry of file readers, so you could point Dataset() +at any file, and it would loop through them and try to +read them, and so find whicherver one worked.one + +But for now, special functions for loading custom file +types can go here. + + +""" + +from .verdat import (load_verdat, + save_verdat + ) diff --git a/gridded/io/verdat.py b/gridded/io/verdat.py new file mode 100644 index 0000000..c0b1b6d --- /dev/null +++ b/gridded/io/verdat.py @@ -0,0 +1,237 @@ +""" +functions for reading/writting "verdat" files + +verdat ("vertex data") is a text file format for bathymetry grids for +triangular mesh models. + +It is used by NOAA's Emergegency Response Division for its CATS model + +It is limited to storing points with associated depths, and grid boudnaries +(including islands), but that's about it. +""" +import numpy as np +import gridded + +# verdat only supports FEET or METERS +FEET = ("foot", "ft", "feet") +METER = ("meter", "m", "meters", "metre") +UNITS_MAP = {u: "FEET" for u in FEET} +UNITS_MAP.update({u: "METERS" for u in METER}) + + + +def load_verdat(filename): + + grid = gridded.grids.Grid_U() + + # read the file + with open(filename) as infile: + header = infile.readline().strip() + try: + units = header.split()[1].lower() + except IndexError: + units = "" + + # read the points: + lons, lats, depths = [], [], [] + for line in infile: + ind, lon, lat, depth = [float(x) for x in line.split(",")] + if ind == 0.0 and lon == 0.0 and lat == 0.0 and depth == 0.0: + break + lons.append(lon) + lats.append(lat) + depths.append(depth) + # read the boundaries: + line = infile.readline().strip() + try: + num_bounds = int(line) + except ValueError: + if line == "": + num_bounds = 0 + else: + raise ValueError("something wrong with file after the end of the points\n" + "(The line after the line with all zeros should be the\n" + "number of boundaries)") + bounds = [] + start_point = 0 + for _ in range(num_bounds): + end_point = int(infile.readline().strip()) + bound = [] + for i in range(start_point, end_point-1): + bound.append((i, i + 1)) + bound.append(((i + 1), start_point)) + start_point = end_point + bounds.extend(bound) + + + nodes = np.c_[lons, lats] + + grid = gridded.grids.Grid_U(nodes=nodes, + boundaries=bounds) + + depth_var = gridded.variable.Variable(name="depth", + units=units.lower(), + data=depths, + location='node', + ) + ds = gridded.Dataset(grid=grid, + variables={'depth': depth_var}, + ) + + return ds + + + + +def save_verdat(ds, filename, depth_var="depth"): + """ + Saves an appropriate dataset as a verdat file + + :param ds: The gridded.Dataset you want to save + + :param filename: name (full or relative path) of the file to save + + :param depth_var="depth": name of the variable with the depths in it. + if depth is None, all depths will be set to 1 + + The dataset must: Have a UGrid grid + + If it has boundaries, they will be used. Otherwise, + it will create them from the grid. + """ + nodes = ds.grid.nodes + if depth_var is None: + depth = np.ones((nodes.shape[0],), dtype=np.float32) + depth_units = "" + else: + depth = ds[depth_var] + depth_units = UNITS_MAP[depth.units.strip().lower()] + f_string = "{0:4d}, {1:10.6f}, {2:10.6f}, {3:8.3f}\n" + + with open(filename, 'w') as outfile: + outfile.write("DOGS") + outfile.write(f" {depth_units}\n") + + depth = depth.data + # write out the boundaries first + if ds.grid.boundaries is None: + ds.grid.build_boundaries() + bounds, open_bounds = order_boundary_segments(ds.grid.boundaries) + points_written = [] + i = 1 + for bound in bounds: + for p in bound: + lon = nodes[p, 0] + lat = nodes[p, 1] + d = depth[p] + outfile.write(f_string.format(i, + lon, + lat, + d)) + points_written.append(p) + i += 1 + # write the field points. + points_written.sort() + for j in range(len(nodes)): + if j not in points_written: + outfile.write(f_string.format(i, + nodes[j, 0], + nodes[j, 1], + depth[j])) + i += 1 + outfile.write(f_string.format(0, 0, 0, 0)) + outfile.write("{:d}\n".format(len(bounds))) + i = 0 + for bound in bounds: + i += len(bound) + outfile.write("{:d}\n".format(i)) + + +def order_boundary_segments(bound_segs): + """ + verdat requires that the boundary segments all be in order + + This code re-orders the segments as required + """ + # make a list so they can be removed as processed + bound_segs = bound_segs.tolist() + # sort just in case the point numbers are close + # to each other and reverse so that we can work from the + # back + bound_segs.sort(reverse=True) + + # There can be zero or more boundaries + closed_bounds = [] + open_bounds = [] + # start with the first boundary segment: + while bound_segs: + seg = bound_segs.pop() + first_p, second_p = seg + bound = [first_p, second_p] + # find a connecting segment + done = False + while not done: + for i in range(len(bound_segs) - 1, -1, -1): + p0, p1 = bound_segs[i] + if p0 == bound[-1]: + bound.append(p1) + bound_segs.pop(i) + elif p1 == bound[-1]: + bound.append(p0) + bound_segs.pop(i) + elif p0 == bound[0]: + bound.insert(p1) + bound_segs.pop(i) + elif p1 == bound[0]: + bound.insert(0, p0) + bound_segs.pop(i) + else: + continue + if bound[0] == bound[-1]: # closed the bound + bound.pop() # take the duplicate point off + closed_bounds.append(bound) + done = True + break + else: + done = False + break + else: # didn't find any more -- not closed + # didn't get closed + open_bounds.append(bound) + done = True + return closed_bounds, open_bounds + + +def make_outer_first(bounds, nodes): + """ + figures out which boundary is the outer boundary, + and puts it first in the list + """ + # note: code for this is in the NOAA ERD ood_utils package. + raise NotImplementedError("this code needs to be written") + try: + import geometry_utils + except ImportError: + print("writing verdat requires the geometry_utils module:\n" + "github.com/NOAA-ORR-ERD/geometry_utils") + + #Assume the first bound is the outer one to start + outer = bounds[0] + for bound in bounds[1:]: + pass + + +def set_winding_order(bounds, nodes, order="clockwise"): + raise NotImplementedError("This code needs to be written") + + + + + + + + + + + + diff --git a/gridded/pyugrid/ugrid.py b/gridded/pyugrid/ugrid.py index 758aa1e..9f0b129 100644 --- a/gridded/pyugrid/ugrid.py +++ b/gridded/pyugrid/ugrid.py @@ -16,7 +16,6 @@ """ - import hashlib from collections import OrderedDict import warnings @@ -323,7 +322,7 @@ def faces(self, faces_indexes): #faces index minimum less than -1 raise ValueError("faces index minimum out of range. min: {0}".format(faces_indexes.min())) self._faces = faces_indexes - + else: self._faces = None # Other things are no longer valid. @@ -655,7 +654,7 @@ def locate_faces(self, points, method='celltree', _memo=True, _copy=False, _hash return indices[0] else: return indices - + def index_of(self, points, method='celltree', diff --git a/gridded/tests/.gitignore b/gridded/tests/.gitignore index 3eab20a..6551f0f 100644 --- a/gridded/tests/.gitignore +++ b/gridded/tests/.gitignore @@ -1,2 +1,4 @@ temp_data/ test_data/cdl/*.cdl.nc +test_io/output +example_data/*.nc diff --git a/gridded/tests/example_data/.gitkeep b/gridded/tests/example_data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gridded/tests/get_remote_data.py b/gridded/tests/get_remote_data.py deleted file mode 100644 index 4858d98..0000000 --- a/gridded/tests/get_remote_data.py +++ /dev/null @@ -1,100 +0,0 @@ -''' -Download data from remote server - -Some test files are too big to put in git - -This code will download test files from an ORR server: - -http://gnome.orr.noaa.gov/py_gnome_testdata/ - -''' - -import os -import urllib.request as urllib_request # for python 3 - -from urllib.parse import urljoin - -# maybe want to add this back in -# import progressbar as pb - -DATA_SERVER = 'https://gnome.orr.noaa.gov/py_gnome_testdata/' - - -CHUNKSIZE = 1024 * 1024 - - -def get_datafile(file_): - """ - Function looks to see if file_ exists in local directory. If it exists, - then it simply returns the 'file_' back as a string. - If 'file_' does not exist in local filesystem, then it tries to download it - from the gnome server: - - http://gnome.orr.noaa.gov/py_gnome_testdata - - If it successfully downloads the file, it puts it in the user specified - path given in file_ and returns the 'file_' string. - - If file is not found or server is down, it rethrows the HTTPError raised - by urllib2.urlopen - - :param file_: path to the file including filename - :type file_: string - :exception: raises urllib2.HTTPError if server is down or file not found - on server - :returns: returns the string 'file_' once it has been downloaded to - user specified location - """ - - if os.path.exists(file_): - return file_ - - # download file, then return file_ path - (path_, fname) = os.path.split(file_) - if path_ == '': - path_ = '.' # relative to current path - - try: - # Open the file using urlopen, ensuring proper closure of the connection - with urllib_request.urlopen(urljoin(DATA_SERVER, fname)) as resp: - # # progress bar - # widgets = [fname + ': ', - # pb.Percentage(), - # ' ', - # pb.Bar(), - # ' ', - # pb.ETA(), - # ' ', - # pb.FileTransferSpeed(), - # ] - # - # pbar = pb.ProgressBar(widgets=widgets, - # maxval=int(resp.info().getheader('Content-Length')) - # ).start() - - if not os.path.exists(path_): - os.makedirs(path_) - - sz_read = 0 - with open(file_, 'wb') as fh: - # while sz_read < resp.info().getheader('Content-Length') - # goes into infinite recursion so break loop for len(data) == 0 - while True: - data = resp.read(CHUNKSIZE) - - if len(data) == 0: - break - else: - fh.write(data) - sz_read += len(data) - - # if sz_read >= CHUNKSIZE: - # pbar.update(CHUNKSIZE) - - # pbar.finish() - return file_ - - except urllib_request.HTTPError as ex: - ex.msg = ("{0}. '{1}' not found on server or server is down" - .format(ex.msg, fname)) - raise ex diff --git a/gridded/tests/test_io/__init__.py b/gridded/tests/test_io/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gridded/tests/test_io/example_files/example_verdat.verdat b/gridded/tests/test_io/example_files/example_verdat.verdat new file mode 100644 index 0000000..c2043bc --- /dev/null +++ b/gridded/tests/test_io/example_files/example_verdat.verdat @@ -0,0 +1,42 @@ +DOGS METERS + 1, -122.370068, 48.057132, 1.000 + 2, -122.376380, 48.035384, 1.000 + 3, -122.370554, 48.000634, 1.000 + 4, -122.350647, 47.983087, 1.000 + 5, -122.348705, 47.958057, 1.000 + 6, -122.305978, 47.950903, 1.000 + 7, -122.252084, 47.960658, 1.000 + 8, -122.233148, 47.984062, 1.000 + 9, -122.224409, 48.020448, 1.000 + 10, -122.226836, 48.031813, 1.000 + 11, -122.242373, 48.031488, 1.000 + 12, -122.291412, 48.051290, 1.000 + 13, -122.297724, 48.065245, 1.000 + 14, -122.315689, 48.077573, 1.000 + 15, -122.357444, 48.060377, 1.000 + 16, -122.356959, 48.055834, 1.000 + 17, -122.309815, 48.014016, 1.000 + 18, -122.303017, 48.006221, 1.000 + 19, -122.309329, 48.005571, 1.000 + 20, -122.325837, 48.010606, 1.000 + 21, -122.334819, 48.018401, 1.000 + 22, -122.333363, 48.021648, 1.000 + 23, -122.321953, 48.021324, 1.000 + 24, -122.354969, 48.040480, 93.000 + 25, -122.327294, 48.047135, 65.000 + 26, -122.350842, 48.014666, 89.000 + 27, -122.329236, 47.996149, 88.000 + 28, -122.328265, 47.969175, 93.000 + 29, -122.273400, 48.005733, 48.000 + 30, -122.293792, 48.029604, 49.000 + 31, -122.285781, 48.039506, 38.000 + 32, -122.310786, 48.045836, 58.000 + 33, -122.339345, 48.059858, 66.000 + 34, -122.259322, 48.027103, 1.000 + 35, -122.249523, 48.007986, 1.000 + 36, -122.288718, 47.975470, 73.000 + 37, -122.316890, 47.963716, 95.000 + 0, 0.000000, 0.000000, 0.000 +2 +16 +23 diff --git a/gridded/tests/test_io/example_files/example_verdat_no_units.verdat b/gridded/tests/test_io/example_files/example_verdat_no_units.verdat new file mode 100644 index 0000000..f032fa7 --- /dev/null +++ b/gridded/tests/test_io/example_files/example_verdat_no_units.verdat @@ -0,0 +1,40 @@ +DOGS + 1, -122.370068, 48.057132, 1.000 + 2, -122.376380, 48.035384, 1.000 + 3, -122.370554, 48.000634, 1.000 + 4, -122.350647, 47.983087, 1.000 + 5, -122.348705, 47.958057, 1.000 + 6, -122.305978, 47.950903, 1.000 + 7, -122.252084, 47.960658, 1.000 + 8, -122.233148, 47.984062, 1.000 + 9, -122.224409, 48.020448, 1.000 + 10, -122.226836, 48.031813, 1.000 + 11, -122.242373, 48.031488, 1.000 + 12, -122.291412, 48.051290, 1.000 + 13, -122.297724, 48.065245, 1.000 + 14, -122.315689, 48.077573, 1.000 + 15, -122.357444, 48.060377, 1.000 + 16, -122.356959, 48.055834, 1.000 + 17, -122.309815, 48.014016, 1.000 + 18, -122.303017, 48.006221, 1.000 + 19, -122.309329, 48.005571, 1.000 + 20, -122.325837, 48.010606, 1.000 + 21, -122.334819, 48.018401, 1.000 + 22, -122.333363, 48.021648, 1.000 + 23, -122.321953, 48.021324, 1.000 + 24, -122.354969, 48.040480, 93.000 + 25, -122.327294, 48.047135, 65.000 + 26, -122.350842, 48.014666, 89.000 + 27, -122.329236, 47.996149, 88.000 + 28, -122.328265, 47.969175, 93.000 + 29, -122.273400, 48.005733, 48.000 + 30, -122.293792, 48.029604, 49.000 + 31, -122.285781, 48.039506, 38.000 + 32, -122.310786, 48.045836, 58.000 + 33, -122.339345, 48.059858, 66.000 + 34, -122.259322, 48.027103, 1.000 + 35, -122.249523, 48.007986, 1.000 + 36, -122.288718, 47.975470, 73.000 + 37, -122.316890, 47.963716, 95.000 + 0, 0.000000, 0.000000, 0.000 + diff --git a/gridded/tests/test_io/example_files/tiny.verdat b/gridded/tests/test_io/example_files/tiny.verdat new file mode 100644 index 0000000..4246b6f --- /dev/null +++ b/gridded/tests/test_io/example_files/tiny.verdat @@ -0,0 +1,16 @@ +DOGS FEET + 1, -62.242001, 12.775000, 1.000 + 2, -28.990000, 12.775000, 1.000 + 3, -28.990000, 30.645000, 1.000 + 4, -47.669998, 31.774000, 102.000 + 5, -53.500999, 36.661999, 1.000 + 6, -43.978001, 28.400000, 1.000 + 7, -37.164001, 25.445999, 60.000 + 8, -39.804001, 17.983000, 1.000 + 9, -44.109001, 22.204000, 1.000 + 10, -50.821999, 20.202999, 97.000 + 11, -34.911236, 29.293791, 1.000 + 0, 0.000000, 0.000000, 0.000 +2 +5 +9 diff --git a/gridded/tests/test_io/test_verdat.py b/gridded/tests/test_io/test_verdat.py new file mode 100644 index 0000000..9dddd7b --- /dev/null +++ b/gridded/tests/test_io/test_verdat.py @@ -0,0 +1,199 @@ +""" +testing verdat read/write capability +""" + +import os +from pathlib import Path +import random + +import numpy as np + +import pooch + +import gridded +from gridded import io + +from ..utilities import data_file_cache + +DATA_URL = "https://gnome.orr.noaa.gov/py_gnome_testdata/gridded_test_files/" + +HERE = Path(__file__).parent +EXAMPLES = HERE / "example_files" +OUTPUT = HERE / "output" +OUTPUT.mkdir(exist_ok=True) + +test_filename = EXAMPLES / "example_verdat.verdat" +test_filename_no_units = EXAMPLES / "example_verdat_no_units.verdat" +test_filename_tiny = EXAMPLES / "tiny.verdat" + + +def test_read(): + """ + at least it does something + """ + ds = io.load_verdat(test_filename) + + assert isinstance(ds, gridded.Dataset) + + +def test_read_no_units(): + ds = io.load_verdat(test_filename_no_units) + + assert ds.variables['depth'].units == "" + + +def test_read_tiny(): + """ + a small example, so we can realy be sure + """ + ds = io.load_verdat(test_filename_tiny) + + assert isinstance(ds, gridded.Dataset) + + nodes = ds.grid.nodes + assert len(nodes) == 11 + assert np.array_equal(nodes[:3], [(-62.242001, 12.775000), + (-28.990000, 12.775000), + (-28.990000, 30.645000), + ]) + + assert np.array_equal(nodes[-2:], [(-50.821999, 20.202999), + (-34.911236, 29.293791), + ]) + + bounds = ds.grid.boundaries + assert len(bounds) == 9 + print(bounds) + assert tuple(bounds[0]) == (0, 1) + assert tuple(bounds[1]) == (1, 2) + assert tuple(bounds[2]) == (2, 3) + assert tuple(bounds[3]) == (3, 4) + assert tuple(bounds[4]) == (4, 0) + + assert tuple(bounds[5]) == (5, 6) + assert tuple(bounds[6]) == (6, 7) + assert tuple(bounds[7]) == (7, 8) + assert tuple(bounds[8]) == (8, 5) + + depths = ds['depth'].data + + assert np.array_equal(depths, [1.0, + 1.0, + 1.0, + 102.0, + 1.0, + 1.0, + 60.0, + 1.0, + 1.0, + 97.0, + 1.0, + ]) + + +def test_save_verdat(): + infilename = EXAMPLES / "tiny.verdat" + ds = io.load_verdat(infilename) + + outfilename = OUTPUT / "tiny_out.verdat" + + outfilename.unlink(missing_ok=True) + + io.save_verdat(ds, outfilename) + + assert outfilename.is_file() + + # Check at least a little bit if it's a valid verdat + orig_contents = open(infilename).readlines() + contents = open(outfilename).readlines() + for l1, l2 in zip(orig_contents, contents): + norm1 = [s.strip() for s in l1.strip().split(",")] + norm2 = [s.strip() for s in l2.strip().split(",")] + print() + print(norm1) + print(norm2) + + assert norm1 == norm2 + + +def test_order_boundary_segments(): + """ + tests that we can find the order of the boundary segments + """ + # bounds from tiny verdat, randomized + boundaries = np.array([[8, 5], [7, 8], [6, 7], [0, 1], [4, 0], [1, 2], [5, 6], [3, 4], [2, 3]]) + + closed_bounds, open_bounds = io.verdat.order_boundary_segments(boundaries) + + assert not open_bounds + + assert len(closed_bounds) == 2 + + # check the bounds are exactly correct + closed_bounds.sort() + + assert sorted(closed_bounds[0]) == [0, 1, 2, 3, 4] + assert sorted(closed_bounds[1]) == [5, 6, 7, 8] + + +def test_order_boundary_segments_open(): + """ + tests that we can find the order of the boundary segments + + and it will find an open boundary + """ + # bounds from tiny verdat, randomized + boundaries = np.array([[8, 5], [7, 8], [6, 7], [0, 1], [4, 0], [1, 2], [5, 6], [3, 4]]) + + closed_bounds, open_bounds = io.verdat.order_boundary_segments(boundaries) + + assert len(closed_bounds) == 1 + assert len(open_bounds) == 1 + + # check the bounds are exactly correct + closed_bounds.sort() + open_bounds.sort() + print(closed_bounds) + print(open_bounds) + + assert sorted(open_bounds[0]) == [0, 1, 2, 3, 4] + assert sorted(closed_bounds[0]) == [5, 6, 7, 8] + + +def test_order_boundary_segments_none(): + boundaries = np.array([]) + closed_bounds, open_bounds = io.verdat.order_boundary_segments(boundaries) + + assert len(closed_bounds) == 0 + assert len(open_bounds) == 0 + + +def test_general_ugrid_to_verdat_no_depth(): + """ + Loads a regular old UGRID netCDF file, and saves it to verdat + """ + ugrid_file = data_file_cache.fetch("SSCOFS.ugrid.nc") + ds = gridded.Dataset.from_netCDF(ugrid_file) + + outfile = OUTPUT / "SSCOFS.verdat" + outfile.unlink(missing_ok=True) + + io.save_verdat(ds, outfile, depth_var=None) + + assert outfile.is_file() + + contents = open(outfile).readlines() + + assert contents[0] == "DOGS \n" + + + assert contents[-1] == "190\n" + assert contents[-2] == "1\n" + + +if __name__ == "__main__": + test_order_boundary_segments_open() + + + + diff --git a/gridded/tests/test_projected_ugrid.py b/gridded/tests/test_projected_ugrid.py index 8e38cd3..2eef64a 100644 --- a/gridded/tests/test_projected_ugrid.py +++ b/gridded/tests/test_projected_ugrid.py @@ -60,35 +60,40 @@ from gridded import VALID_UGRID_LOCATIONS -from .utilities import get_temp_test_file +from .utilities import data_file_cache +# try: +# data_file = get_temp_test_file("projected_coords_ugrid.nc") +# if data_file is None: +# # skip these tests if the data file couldn't be downloaded +# pytestmark = pytest.mark.skip +# except: # if anything went wrong, skip these. +# pytestmark = pytest.mark.skip -try: - data_file = get_temp_test_file("projected_coords_ugrid.nc") - if data_file is None: - # skip these tests if the data file couldn't be downloaded - pytestmark = pytest.mark.skip -except: # if anything went wrong, skip these. - pytestmark = pytest.mark.skip - +data_file = data_file_cache.fetch("projected_coords_ugrid.nc") def test_load(): """ The file should load without error """ - ds = Dataset(data_file) + ds = Dataset.from_netCDF(data_file) assert isinstance(ds.grid, Grid_U) + print(ds.grid.nodes.max(), ds.grid.nodes.min()) + assert ds.grid.nodes.min() > 148_000 # definitely not lat-lon + def test_find_variables(): """ Does it find the variables? """ - ds = Dataset(data_file) + ds = Dataset.from_netCDF(data_file) var_names = list(ds.variables.keys()) + print(var_names) + all_vars = ['mesh2d_Numlimdt', 'mesh2d_czs', 'mesh2d_diu', diff --git a/gridded/tests/test_read.py b/gridded/tests/test_read.py index 073e051..27dbab3 100644 --- a/gridded/tests/test_read.py +++ b/gridded/tests/test_read.py @@ -4,12 +4,14 @@ Tests for testing a UGrid file read. We really need a **lot** more sample data files... - """ import os +from pathlib import Path +import numpy as np +import netCDF4 from gridded import Dataset from gridded.variable import Variable @@ -18,13 +20,19 @@ from gridded.pysgrid.sgrid import SGrid # from pyugrid import read_netcdf -test_data_dir = get_test_file_dir() +HERE = Path(__file__).parent +test_data_dir = HERE / 'test_data' +output_dir = HERE / 'output' +output_dir.mkdir(exist_ok=True) def test_simple_read(): - """Can it be read at all?""" - with chdir(test_data_dir): - ds = Dataset('UGRIDv0.9_eleven_points.nc') + """ + Can it be read at all? + NOTE: passing the file name into the constructor is now deprecated + """ + + ds = Dataset.from_netCDF(test_data_dir / 'UGRIDv0.9_eleven_points.nc') assert isinstance(ds, Dataset) assert isinstance(ds.grid, UGrid) @@ -35,8 +43,8 @@ def test_read_variables(): UGRIDv0.9_eleven_points.nc file """ - with chdir(test_data_dir): - ds = Dataset('UGRIDv0.9_eleven_points.nc') + + ds = Dataset.from_netCDF(test_data_dir / 'UGRIDv0.9_eleven_points.nc') varnames = list(ds.variables.keys()) varnames.sort() print("variables are:", varnames) @@ -51,195 +59,78 @@ def test_read_variables(): def test_read_variable_attributes(): - with chdir(test_data_dir): - ds = Dataset('UGRIDv0.9_eleven_points.nc') + ds = Dataset(test_data_dir / 'UGRIDv0.9_eleven_points.nc') print(ds.variables['Mesh2_depth'].attributes) assert (ds.variables['Mesh2_depth'].attributes['standard_name'] == 'sea_floor_depth_below_geoid') assert ds.variables['Mesh2_depth'].attributes['units'] == 'm' -def test_read_FVCOM(): - '''Optional test to make sure that files from TAMU and NGOFS are read correctly.''' - with chdir(test_data_dir): - if os.path.exists('COOPS_NGOFS.nc'): - ds = Dataset('COOPS_NGOFS.nc') - print("COOPS_NGOFS variables are:", ds.variables.keys()) - assert isinstance(ds, Dataset) - assert isinstance(ds.grid, UGrid) - assert isinstance(ds.variables, dict) - assert 'u' in ds.variables.keys() - assert 'v' in ds.variables.keys() - - -def test_read_TAMU(): - with chdir(test_data_dir): - if os.path.exists('TAMU.nc'): - ds = Dataset('TAMU.nc') - print("TAMU variables are:", ds.variables.keys()) - assert isinstance(ds, Dataset) - assert isinstance(ds.grid, SGrid) - assert isinstance(ds.variables, dict) - assert 'water_u' in ds.variables.keys() - assert 'water_v' in ds.variables.keys() - - -# def test_get_mesh_names(): -# """ -# Check that it can find the mesh variable names. - -# """ -# with chdir(files): -# nc = netCDF4.Dataset('ElevenPoints_UGRIDv0.9.nc') -# names = read_netcdf.find_mesh_names(nc) -# assert names == [u'Mesh2'] - - -# def test_mesh_not_there(): -# """Test raising Value error with incorrect mesh name.""" -# with pytest.raises(ValueError): -# with chdir(files): -# UGrid.from_ncfile('ElevenPoints_UGRIDv0.9.nc', mesh_name='garbage') - - -# def test_load_grid_from_nc(): -# """Test reading a fairly full example file.""" -# with chdir(files): -# grid = UGrid.from_ncfile('ElevenPoints_UGRIDv0.9.nc') - -# assert grid.mesh_name == 'Mesh2' -# assert grid.nodes.shape == (11, 2) -# assert grid.faces.shape == (13, 3) -# assert grid.face_face_connectivity.shape == (13, 3) -# assert grid.boundaries.shape == (9, 2) -# assert grid.edges is None - - -# def test_read_nodes(): -# """Do we get the right nodes array?""" -# with chdir(files): -# grid = UGrid.from_ncfile('ElevenPoints_UGRIDv0.9.nc') -# assert grid.nodes.shape == (11, 2) - -# # Not ideal to pull specific values out, but how else to test? -# assert np.array_equal(grid.nodes[0, :], (-62.242, 12.774999)) -# assert np.array_equal(grid.nodes[-1, :], (-34.911235, 29.29379)) - - -# def test_read_none_edges(): -# """Do we get the right edge array?""" - -# with chdir(files): -# grid = UGrid.from_ncfile('ElevenPoints_UGRIDv0.9.nc') -# assert grid.edges is None - - -# def test_read_faces(): -# """Do we get the right faces array?""" - -# with chdir(files): -# grid = UGrid.from_ncfile('ElevenPoints_UGRIDv0.9.nc') -# assert grid.faces.shape == (13, 3) - -# # Not ideal to pull specific values out, but how else to test? -# assert np.array_equal(grid.faces[0, :], (2, 3, 10)) -# assert np.array_equal(grid.faces[-1, :], (10, 5, 6)) - - -# def test_read_face_face(): -# """Do we get the right face_face_connectivity array?""" - -# with chdir(files): -# grid = UGrid.from_ncfile('ElevenPoints_UGRIDv0.9.nc') -# assert grid.face_face_connectivity.shape == (13, 3) +# def test_read_FVCOM(): +# '''Optional test to make sure that files from NGOFS are read correctly.''' +# with chdir(test_data_dir): +# if os.path.exists('COOPS_NGOFS.nc'): +# ds = Dataset('COOPS_NGOFS.nc') +# print("COOPS_NGOFS variables are:", ds.variables.keys()) +# assert isinstance(ds, Dataset) +# assert isinstance(ds.grid, UGrid) +# assert isinstance(ds.variables, dict) +# assert 'u' in ds.variables.keys() +# assert 'v' in ds.variables.keys() +# else: +# print("COOPS_NGOFS.nc could not be found") +# assert False -# # Not ideal to pull specific values out, but how else to test? -# assert np.array_equal(grid.face_face_connectivity[0, :], (11, 5, -1)) -# assert np.array_equal(grid.face_face_connectivity[-1, :], (-1, 5, 11)) +# def test_read_TAMU(): +# """ +# Test to see if the TAMU files are read correctly +# """ +# with chdir(test_data_dir): +# if os.path.exists('TAMU.nc'): +# ds = Dataset('TAMU.nc') +# print("TAMU variables are:", ds.variables.keys()) +# assert isinstance(ds, Dataset) +# assert isinstance(ds.grid, SGrid) +# assert isinstance(ds.variables, dict) +# assert 'water_u' in ds.variables.keys() +# assert 'water_v' in ds.variables.keys() +# else: +# print("TAMU.nc could not be found") +# assert False + + +def test_read_variable(): + """ + at least see if one variable can be read :-) -# def test_read_boundaries(): -# """Do we get the right boundaries array?""" - -# with chdir(files): -# grid = UGrid.from_ncfile('ElevenPoints_UGRIDv0.9.nc') -# assert grid.boundaries.shape == (9, 2) - -# # Not ideal to pull specific values out, but how else to test? -# # Note: file is 1-indexed, so these values are adjusted. -# expected_boundaries = [[0, 1], [1, 2], [2, 3], -# [3, 4], [4, 0], [5, 6], -# [6, 7], [7, 8], [8, 5]] -# assert np.array_equal(grid.boundaries, expected_boundaries) - - -# def test_read_face_coordinates(): -# """Do we get the right face_coordinates array?""" - -# with chdir(files): -# grid = UGrid.from_ncfile('ElevenPoints_UGRIDv0.9.nc') -# assert grid.face_coordinates.shape == (13, 2) - -# # Not ideal to pull specific values out, but how else to test? -# assert np.array_equal(grid.face_coordinates[0], -# (-37.1904106666667, 30.57093)) -# assert np.array_equal(grid.face_coordinates[-1], -# (-38.684412, 27.7132626666667)) - - -# def test_read_none_edge_coordinates(): -# """Do we get the right edge_coordinates array?""" -# with chdir(files): -# grid = UGrid.from_ncfile('ElevenPoints_UGRIDv0.9.nc') -# assert grid.edge_coordinates is None - - -# def test_read_none_boundary_coordinates(): -# """Do we get the right boundary_coordinates array?""" -# with chdir(files): -# grid = UGrid.from_ncfile('ElevenPoints_UGRIDv0.9.nc') -# assert grid.boundary_coordinates is None - - -# def test_read_longitude_no_standard_name(): -# with chdir(files): -# grid = UGrid.from_ncfile('no_stand_name_long.nc') -# assert grid.nodes.shape == (11, 2) - -# # Not ideal to pull specific values out, but how else to test? -# assert np.array_equal(grid.nodes[0, :], (-62.242, 12.774999)) -# assert np.array_equal(grid.nodes[-1, :], (-34.911235, 29.29379)) - - -# def test_read_data_keys(): -# with chdir(files): -# grid = UGrid.from_ncfile('ElevenPoints_UGRIDv0.9.nc', load_data=True) -# assert sorted(grid.data.keys()) == [u'boundary_count', -# u'boundary_types', -# u'depth'] - - -# def test_read_data(): -# expected_depth = [1, 1, 1, 102, 1, 1, 60, 1, 1, 97, 1] -# expected_depth_attributes = {'standard_name': 'sea_floor_depth_below_geoid', -# 'units': 'm', -# 'positive': 'down', -# } -# with chdir(files): -# grid = UGrid.from_ncfile('ElevenPoints_UGRIDv0.9.nc', load_data=True) -# assert np.array_equal(grid.data['depth'].data, expected_depth) -# assert grid.data['depth'].attributes == expected_depth_attributes - + this is an old pyugrid test ported over. + """ + expected_depth = [ 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10.] + expected_depth_attributes = {'standard_name': 'sea_floor_depth_below_geoid', + 'units': 'm', + 'positive': 'down', + 'coordinates': 'Mesh2_node_y Mesh2_node_x', + 'grid': 'Bathymetry_Mesh', + 'long_name': 'Bathymetry', + 'type': 'data' + } + ds = Dataset.from_netCDF(test_data_dir / 'UGRIDv0.9_eleven_points_with_depth.nc') + + depth = ds['h'] + assert np.array_equal(depth.data, expected_depth) + assert depth.attributes == expected_depth_attributes + + +def test_read_from_nc_dataset(): + """ + Minimal test, but makes sure you can read from an already + open netCDF4.Dataset. + """ + with netCDF4.Dataset(test_data_dir / 'UGRIDv0.9_eleven_points_with_depth.nc') as nc: + ds = Dataset.from_netCDF(nc) + assert ds.grid.mesh_name == 'Mesh2' + assert ds.grid.nodes.shape == (11, 2) + assert ds.grid.faces.shape == (13, 3) -# def test_read_from_nc_dataset(): -# """ -# Minimal test, but makes sure you can read from an already -# open netCDF4.Dataset. -# """ -# with chdir(files): -# with netCDF4.Dataset('ElevenPoints_UGRIDv0.9.nc') as nc: -# grid = UGrid.from_nc_dataset(nc) -# assert grid.mesh_name == 'Mesh2' -# assert grid.nodes.shape == (11, 2) -# assert grid.faces.shape == (13, 3) diff --git a/gridded/tests/utilities.py b/gridded/tests/utilities.py index a976217..d8dcd9f 100644 --- a/gridded/tests/utilities.py +++ b/gridded/tests/utilities.py @@ -2,26 +2,49 @@ Assorted utilities useful for the tests. """ -import os import contextlib +from pathlib import Path +import glob +import os + +import pooch -try: - import urllib.request as urllib_request # for python 3 -except ImportError: - import urllib2 as urllib_request # for python 2 + +import urllib.request as urllib_request # for python 3 import pytest -import glob -from .get_remote_data import get_datafile + +HERE = Path(__file__).parent +EXAMPLE_DATA = HERE / "example_data" + +# # Files on PYGNOME server -- add them here as needed +data_file_cache = pooch.create( + # Use a local cache folder for the operating system + # path=pooch.os_cache("plumbus"), + path=EXAMPLE_DATA, + # The remote data is on the pygnome server + base_url="https://gnome.orr.noaa.gov/py_gnome_testdata/gridded_test_files/", + # version=version, + # # If this is a development version, get the data from the "main" branch + # version_dev="main", + registry={ + "3D_ROMS_example.nc": "sha256:d802d408bf3925dd77ff582bf906b95062eb65161de7b2290fb8d41537a566b6", + "FVCOM-Erie-OFS-subsetter.nc": "sha256: 96c20ef1f4c463838c86e88baa9eba05aacb2db6fe184dc6d338489c38827567", + "ROMS-WCOFS-OFS-subsetter.nc": "sha256:04af4479331894ab3abbd789fbfc2e4717c39e9c62123942929775a40406b9e9", + "SSCOFS.ugrid.nc": "sha256:0dcea2a2fb6ad87c7cce3ebc475fd2f0430616a5019f54f4adf97391e075e939", + "projected_coords_ugrid.nc": "sha256:019c1469c0583021268dbf1ea3eed97038364a0b7a361bc3f50b6be5f83b1ff2" + }, +) def get_test_file_dir(): """ returns the test file dir path + + This should be replaced with simple code in the tests ... """ - test_file_dir = os.path.join(os.path.dirname(__file__), 'test_data') - return test_file_dir + return Path(__file__).parent / 'test_data' def get_test_cdl_filelist(): @@ -29,32 +52,32 @@ def get_test_cdl_filelist(): return glob.glob(os.path.join(dirpath, '*.cdl')) -def get_temp_test_file(filename): - """ - returns the path to a temporary test file. - - If it exists, it will return it directly. - - If not, it will attempt to download it. - - If it can't download, it will return None - """ - print("getting temp test file") - filepath = os.path.join(os.path.dirname(__file__), - 'temp_data', - filename) - if os.path.isfile(filepath): - print("already there") - return filepath - else: - # attempt to download it - print("trying to download") - try: - get_datafile(filepath) - except urllib_request.HTTPError: - print("got an error trying to download {}:".format(filepath)) - return None - return None +# def get_temp_test_file(filename): +# """ +# returns the path to a temporary test file. + +# If it exists, it will return it directly. + +# If not, it will attempt to download it. + +# If it can't download, it will return None +# """ +# print("getting temp test file") +# filepath = os.path.join(os.path.dirname(__file__), +# 'temp_data', +# filename) +# if os.path.isfile(filepath): +# print("already there") +# return filepath +# else: +# # attempt to download it +# print("trying to download") +# try: +# get_datafile(filepath) +# except urllib_request.HTTPError: +# print("got an error trying to download {}:".format(filepath)) +# return None +# return None @pytest.fixture diff --git a/gridded/variable.py b/gridded/variable.py index e13242c..4c8aebe 100644 --- a/gridded/variable.py +++ b/gridded/variable.py @@ -306,11 +306,12 @@ def __str__(self): def __repr__(self): return ('{0.__class__.__module__}.{0.__class__.__name__}(' - 'name="{0.name}", ' - 'time="{0.time}", ' - 'units="{0.units}", ' - 'data="{0.data}", ' - ')').format(self) + 'name="{0.name}", \n' + 'time="{0.time}", \n' + 'units="{0.units}", \n' + 'location="{0.location}" \n' + 'data=Type:{1}, shape:{0.data.shape}", ' + ')').format(self, type(self.data)) @classmethod def constant(cls, value):