Skip to content

Commit 7528557

Browse files
author
Bob Simons
committed
v1.60
1 parent 0cc69eb commit 7528557

17 files changed

+1297
-332
lines changed

‎WEB-INF/classes/com/cohort/util/Calendar2.java‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -77,23 +77,6 @@ public class Calendar2 {
7777
public final static int ZONE_OFFSET = Calendar.ZONE_OFFSET; //millis
7878
public final static int DST_OFFSET = Calendar.DST_OFFSET; //millis
7979

80-
/*
81-
//for thread safety, always use: synchronized(<itself>) {<use...>}
82-
//do before defaultValue= (below)
83-
private static SimpleDateFormat isoDateTimeFormat =
84-
new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
85-
private static SimpleDateFormat isoDateFormat =
86-
new SimpleDateFormat("yyyy-MM-dd");
87-
private static SimpleDateFormat isoDateHMFormat =
88-
new SimpleDateFormat("yyyy-MM-dd HH:mm");
89-
private static SimpleDateFormat compactDateTimeFormat =
90-
new SimpleDateFormat("yyyyMMddHHmmss");
91-
private static SimpleDateFormat YYYYDDDFormat =
92-
new SimpleDateFormat("yyyyDDD");
93-
private static SimpleDateFormat YYYYMMFormat =
94-
new SimpleDateFormat("yyyyMM");
95-
*/
96-
9780
public final static int MINUTES_PER_DAY = 1440;
9881
public final static int MINUTES_PER_7DAYS = 7 * MINUTES_PER_DAY; //10080
9982
public final static int MINUTES_PER_30DAYS = 30 * MINUTES_PER_DAY; //43200

‎WEB-INF/classes/com/cohort/util/String2.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1159,6 +1159,8 @@ public static boolean isUrl(String url) {
11591159

11601160
/**
11611161
* This indicates if s has just file-name-safe characters (0-9, A-Z, a-z, _, -, .).
1162+
* Note, this does not check for filenames that are too long
1163+
* (Windows has a path+fileName max length of 255 chars).
11621164
*
11631165
* @param s a string, usually a file name
11641166
* @return true if s has just file-name-safe characters (0-9, A-Z, a-z, _, -, .).
@@ -1184,6 +1186,8 @@ public static boolean isFileNameSafe(String s) {
11841186
* See javadocs for java.net.URLEncoder, which describes valid characters
11851187
* (but deals with encoding, whereas this method alters or removes).
11861188
* The result may be shorter than s.
1189+
* Note, this does not check for filenames that are too long
1190+
* (Windows has a path+fileName max length of 255 chars).
11871191
*
11881192
* @param s If s is null, this returns "_null".
11891193
* If s is "", this returns "_".
@@ -1299,6 +1303,8 @@ public static boolean isJsonpNameSafe(String s) {
12991303
* <ul>
13001304
* Note that Java allows Unicode characters, but this does not.
13011305
* See also the safer encodeVariableNameSafe(String s).
1306+
* Note, this does not check for names that are too long
1307+
* (many system have an 80 or 255 char limit).
13021308
*
13031309
* @param s
13041310
* @return a safe variable name (but perhaps two s's lead to the same result)

‎WEB-INF/classes/gov/noaa/pfel/coastwatch/TestAll.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -511,7 +511,9 @@ public static void main(String args[]) throws Throwable {
511511
// Temporarily switching off parts of McAfee : Virus Scan Console (2X speedup!)
512512
// On Access Scanner : All Processes
513513
// Scan Items: check: specified file types only (instead of usual All Files)
514-
// EDDTableFromNcFiles.bobConsolidateGtsppTgz(2014, 11, 2015, 1, false); //first/last year(1990..)/month(1..), testMode
514+
// EDDTableFromNcFiles.bobConsolidateGtsppTgz(1970, 1, 1970, 1, false); //first/last year(1990..)/month(1..), testMode
515+
// EDDTableFromNcFiles.bobConsolidateGtsppTgz(2005, 4, 2005, 4, false); //first/last year(1990..)/month(1..), testMode
516+
// EDDTableFromNcFiles.bobConsolidateGtsppTgz(2012, 2, 2015, 2, false); //first/last year(1990..)/month(1..), testMode
515517
// log file is c:/data/gtspp/log.txt
516518
// 2b) Email the "good" but "impossible" stations to Charles Sun
517519
// [was Melanie Hamilton, now retired]
@@ -553,8 +555,8 @@ public static void main(String args[]) throws Throwable {
553555
// http://coastwatch.pfeg.noaa.gov/erddap/setDatasetFlag.txt?datasetID=erdGtsppBest&flagKey=2369414249
554556
// and make sure the new data and metadata are visible (hence, new dataset has loaded)
555557

556-
// String2.log(EDDTableFromSOS.generateDatasetsXmlFromIOOS(
557-
// "http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS"));
558+
// String2.log(EDDTableFromSOS.generateDatasetsXml(
559+
// "http://data.gcoos.org:8080/52nSOS/sos/kvp", "1.0.0", "IOOS_52N"));
558560
// EDDTableFromSOS.testErddapSos();
559561

560562
//** To update pmelTAO data on/after 9am 2nd day of every month:
@@ -608,7 +610,7 @@ public static void main(String args[]) throws Throwable {
608610
EDDTableFromHyraxFiles.testGenerateDatasetsXml();
609611
EDDTableFromNcFiles.testGenerateDatasetsXml();
610612
EDDTableFromOBIS.testGenerateDatasetsXml();
611-
//EDDTableFromSOS.testGenerateDatasetsXml(); //not up-to-date
613+
EDDTableFromSOS.testGenerateDatasetsXml(true);
612614
EDDTableFromSOS.testGenerateDatasetsXmlFromOneIOOS();
613615
EDDTableFromSOS.testGenerateDatasetsXmlFromIOOS();
614616
EDDTableFromThreddsFiles.testGenerateDatasetsXml();
@@ -1138,7 +1140,7 @@ public static void main(String args[]) throws Throwable {
11381140

11391141
//data
11401142
DataStream.main(null);
1141-
SimpleXMLReader.main(null);
1143+
SimpleXMLReader.test();
11421144
TimePeriods.test();
11431145
FileNameUtility.main(null);
11441146
ParseJSON.test();
@@ -1300,7 +1302,7 @@ public static void main(String args[]) throws Throwable {
13001302
//EDDTableFromNWISDV.test(); //INACTIVE as of 2011-12-16.
13011303
EDDTableFromOBIS.test();
13021304
//EDDTableFromBMDE.test(); //INACTIVE
1303-
EDDTableFromSOS.test();
1305+
EDDTableFromSOS.test(true); //useCachedInfo
13041306
EDDTableFromWFSFiles.test();
13051307
EDDTableCopy.test();
13061308
//EDDTableCopyPost.test(); INACTIVE

‎WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/OQNux10S1day_20050712_x-135_X-105_y22_Y50.nc‎

0 Bytes
Binary file not shown.

‎WEB-INF/classes/gov/noaa/pfel/coastwatch/util/FileVisitorDNLS.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,20 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
136136
return FileVisitResult.CONTINUE;
137137
}
138138

139+
/** Invoked for a file that could not be visited. */
140+
public FileVisitResult visitFileFailed(Path file, IOException exc) {
141+
//2015-03-10 I added this method to override the superclass
142+
//which apparently throws the exception and stops the parent
143+
//SimpleFileVisitor. This class just ignores the error.
144+
//Partial test that this change solves the problem: call
145+
// new FileVisitorSubdir("/")
146+
// on my Windows computer with message analogous to below enabled.
147+
// It shows several files where visitFileFailed.
148+
//Always show message here. It is useful information. (message is just filename)
149+
String2.log("WARNING: FileVisitorDNLS.visitFileFailed: " + exc.getMessage());
150+
return FileVisitResult.CONTINUE;
151+
}
152+
139153
/** A convenience method for using this class.
140154
* The results are in the global table and in the PA variables.
141155
*
@@ -256,7 +270,7 @@ public static void test() throws Throwable {
256270
if (directoryPA.size() < 1000) {
257271
String2.log(directoryPA.size() + " files. Not a good test.");
258272
} else {
259-
Test.ensureBetween(time / (double)directoryPA.size(), 1.5e-3, 8e-3,
273+
Test.ensureBetween(time / (double)directoryPA.size(), 1e-3, 8e-3,
260274
"ms/file (3e-3 expected)");
261275
String dir0 = directoryPA.get(0);
262276
String2.log("backward slash test: dir0=" + dir0);

‎WEB-INF/classes/gov/noaa/pfel/coastwatch/util/FileVisitorSubdir.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,19 @@ public FileVisitResult preVisitDirectory(Path tPath, BasicFileAttributes attrs)
6565
}
6666

6767

68+
/** Invoked for a file that could not be visited. */
69+
public FileVisitResult visitFileFailed(Path file, IOException exc) {
70+
//2015-03-10 I added this method to override the superclass
71+
//which apparently throws the exception and stops the parent
72+
//SimpleFileVisitor. This class just ignores the error.
73+
//A one time test that this change solves the problem: call
74+
// new FileVisitorSubdir("/")
75+
// on my Windows computer with message below enabled.
76+
//Always show message here. It is useful information. (message is just filename)
77+
String2.log("WARNING: FileVisitorSubdir.visitFileFailed: " + exc.getMessage());
78+
return FileVisitResult.CONTINUE;
79+
}
80+
6881
/**
6982
* A convenience method for using this class.
7083
*

‎WEB-INF/classes/gov/noaa/pfel/coastwatch/util/SimpleXMLReader.java‎

Lines changed: 60 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -490,6 +490,19 @@ public void nextTag() throws Exception {
490490

491491
}
492492

493+
/**
494+
* This skips efficiently until the stackSize is toStackSize.
495+
* E.g., call this after an opening tag is read in order to skip efficiently
496+
* to the matching closing tag by calling skipToStackSize(stackSize()).
497+
*/
498+
public void skipToStackSize(int toStackSize) throws Exception {
499+
while (true) {
500+
nextTag();
501+
if (stack.size() == toStackSize)
502+
return;
503+
}
504+
}
505+
493506
/**
494507
* This throws the standard "Unexpected tag" Exception.
495508
* This also calls close().
@@ -539,10 +552,11 @@ public boolean isOpen() {
539552
*
540553
* @param args is ignored
541554
*/
542-
public static void main(String args[]) throws Exception {
555+
public static void test() throws Exception {
543556

544557
SimpleXMLReader xmlReader;
545558
String2.log("SimpleXMLReader will now intentionally throw and catch several exceptions.");
559+
String expected;
546560

547561
//test invalid start of xml
548562
String error = "";
@@ -596,6 +610,28 @@ public static void main(String args[]) throws Exception {
596610
}
597611
Test.ensureTrue(error.indexOf(" End tag </bob> doesn't have a matching start tag.") > 0, "error=" + error);
598612

613+
//test no end tag
614+
error = "";
615+
try {
616+
xmlReader = new SimpleXMLReader(new ByteArrayInputStream(String2.toByteArray(
617+
"<?xml version=\"1.0\" encoding=\"ISO-8859-1\" ?>\n" +
618+
"<testc>\n" +
619+
"")));
620+
xmlReader.nextTag();
621+
xmlReader.nextTag();
622+
} catch (Exception e) {
623+
error = e.toString();
624+
}
625+
expected =
626+
"java.lang.Exception: ERROR in XML file on line #3: Unexpected end of file with non-empty stack: <testc>\n" +
627+
" tag = \n" +
628+
" content = \n" +
629+
" exception = SimpleXMLReader.getNextTag:\n" +
630+
"ERROR:\n" +
631+
"\n" +
632+
"java.lang.Exception: end of file";
633+
Test.ensureEqual(error.substring(0, expected.length()), expected, "error=" + error);
634+
599635
//test un-closed comment
600636
error = "";
601637
try {
@@ -611,7 +647,7 @@ public static void main(String args[]) throws Exception {
611647
String2.log("That was the last Expected Exception.");
612648

613649
//test valid xml
614-
xmlReader = new SimpleXMLReader(new ByteArrayInputStream(String2.toByteArray(
650+
String testXml =
615651
"<?xml version=\"1.0\" encoding=\"ISO-8859-1\" ?>\n" +
616652
"<testr>\n" +
617653
" <level1 att1=value1 att2=\"value 2\" > \n level 1 \r&amp; <!-- comment < > -->text \r\n" +
@@ -621,7 +657,9 @@ public static void main(String args[]) throws Exception {
621657
"\n" +
622658
"\n" +
623659
"\n" +
624-
"</testr>")));
660+
"</testr attr=\"valr\">";
661+
xmlReader = new SimpleXMLReader(new ByteArrayInputStream(
662+
String2.toByteArray(testXml)));
625663
Test.ensureEqual(xmlReader.stackSize(), 0, "a");
626664
xmlReader.nextTag();
627665
Test.ensureEqual(xmlReader.stackSize(), 1, "a");
@@ -674,17 +712,31 @@ public static void main(String args[]) throws Exception {
674712
Test.ensureEqual(xmlReader.topTag(), "/testr", "f");
675713
Test.ensureEqual(xmlReader.content(), "", "f");
676714
Test.ensureEqual(xmlReader.allTags(), "</testr>", "f");
715+
Test.ensureEqual(xmlReader.attributeValue("attr"), "valr", "b");
677716

678717
xmlReader.nextTag();
679718
Test.ensureEqual(xmlReader.stackSize(), 0, "g");
680719
Test.ensureEqual(xmlReader.topTag(), null, "g");
681720
Test.ensureEqual(xmlReader.content(), "", "g");
682721
Test.ensureEqual(xmlReader.allTags(), "", "g");
722+
xmlReader.close();
723+
724+
//skipToStackSize();
725+
//"<testr>\n" +
726+
//" <level1 att1=value1 att2=\"value 2\" > \n level 1 \r&amp; <!-- comment < > -->text \r\n" +
727+
//" </level1>\n" +
728+
String2.log("test skipToClosingTag()");
729+
xmlReader = new SimpleXMLReader(new ByteArrayInputStream(
730+
String2.toByteArray(testXml)));
731+
xmlReader.nextTag();
732+
Test.ensureEqual(xmlReader.topTag(), "testr", "k");
733+
xmlReader.skipToStackSize(xmlReader.stackSize());
734+
Test.ensureEqual(xmlReader.topTag(), "/testr", "f");
735+
Test.ensureEqual(xmlReader.content(), "", "f");
736+
Test.ensureEqual(xmlReader.allTags(), "</testr>", "f");
737+
Test.ensureEqual(xmlReader.attributeValue("attr"), "valr", "b");
738+
xmlReader.close();
683739

684-
String2.log("SimpleXMLReader.main's tests finished successfully.\n");
685-
686-
740+
String2.log("SimpleXMLReader.tests's tests finished successfully.\n");
687741
}
688-
689-
690742
}

‎WEB-INF/classes/gov/noaa/pfel/coastwatch/util/WatchDirectory.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,33 @@ public static void test(boolean doInteractiveTest) throws Throwable {
246246
}
247247
Test.ensureBetween(n, 4, 5, ""); //sometimes the dir event isn't caught
248248

249+
//*** test creating a huge number
250+
//This is allowed on Windows. It doesn't appear to have max number.
251+
// ADVICE TO ADMINS: 2015-03-10
252+
// On Linux computers,
253+
// if you are using <updateEveryNMillis> with EDDGridFromFiles or EDDTableFromFiles
254+
// classes, you may see a problem where a dataset fails to load with the
255+
// error message:
256+
// "IOException: User limit of inotify instances reached or too many open files".
257+
// If so, you can fix this problem by calling (as root):
258+
// echo 10000 > /proc/sys/fs/inotify/max_user_watches
259+
// echo 10000 > /proc/sys/fs/inotify/max_user_instances
260+
// Or, use higher numbers if the problem persists.
261+
// The default for watches is 8192. The default for instances is 128.
262+
//VERY SLOW! NOT USUALLY DONE:
263+
//String2.log("test all dirs on this computer: watchDirectoryAll(\"/\", true)");
264+
//wd = watchDirectoryAll("/", true);
265+
266+
//never finished this test since Windows allows large number of watches.
267+
//n = 10000;
268+
//String2.log("test huge number=" + n);
269+
//WatchDirectory wdar[] = new WatchDirectory[n];
270+
//for (int i = 0; i < n; i++) {
271+
// if (i % 10 == 0) Math2.gc(100); //does gc make the error go away?
272+
// wdar[i] = watchDirectoryAll(watchDir, true);
273+
//}
274+
275+
249276
//*** interactive test
250277
RegexFilenameFilter.regexDelete(watchDir, ".*", true);
251278
Math2.sleep(sleep);

‎WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13698,7 +13698,7 @@ public static void testBasic() throws Throwable {
1369813698
(EDStatic.sosActive? "\"\", " : "") + //currently, it isn't made available via sos
1369913699
(EDStatic.wcsActive? "\"\", " : "") +
1370013700
(EDStatic.wmsActive? "\"\", " : "") +
13701-
(EDStatic.filesActive? "\"\", " : "") +
13701+
(EDStatic.filesActive? "\"http://127.0.0.1:8080/cwexperimental/files/erdGlobecBottle/\", " : "") +
1370213702
(EDStatic.authentication.length() > 0? "\"public\", " : "") +
1370313703
"\"GLOBEC NEP Rosette Bottle Data (2002)\", \"GLOBEC (GLOBal " +
1370413704
"Ocean ECosystems Dynamics) NEP (Northeast Pacific)\\nRosette Bottle Data from " +
@@ -13816,7 +13816,7 @@ public static void testBasic() throws Throwable {
1381613816

1381713817
//files
1381813818
results = SSR.getUrlResponseString(EDStatic.erddapUrl + "/files/");
13819-
Test.ensureTrue(results.indexOf("You can browse and download source data files.") >= 0, "results=\n" + results);
13819+
Test.ensureTrue(results.indexOf("ERDDAP's \"files\" system lets you browse a virtual file system and download source data files.") >= 0, "results=\n" + results);
1382013820
Test.ensureTrue(results.indexOf("WARNING!") >= 0, "results=\n" + results);
1382113821
Test.ensureTrue(results.indexOf("Last modified") >= 0, "results=\n" + results);
1382213822
Test.ensureTrue(results.indexOf("Parent Directory") >= 0, "results=\n" + results);

‎WEB-INF/classes/gov/noaa/pfel/erddap/GenerateDatasetsXml.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -456,16 +456,19 @@ public String doIt(String args[], boolean loop) throws Throwable {
456456

457457
} else if (eddType.equals("EDDTableFromIoosSOS")) {
458458
s1 = get(args, 1, s1, "URL");
459-
s2 = get(args, 2, s2, "SOS server type (IOOS_NDBC, IOOS_NOS)");
459+
s2 = get(args, 2, s2, "SOS version (e.g., 1.0.0)");
460+
s3 = get(args, 3, s3, "SOS server type (52N, IOOS_NDBC, IOOS_NOS)");
460461
String2.log("working...");
461-
printToBoth(EDDTableFromSOS.generateDatasetsXmlFromIOOS(s1, s2));
462+
printToBoth(EDDTableFromSOS.generateDatasetsXmlFromIOOS(
463+
false, s1, s2, s3)); //use cached getCapabilities
462464

463465
} else if (eddType.equals("EDDTableFromSOS")) {
464466
s1 = get(args, 1, s1, "URL");
465-
s2 = get(args, 2, s2, "SOS version (or \"\" for default)");
466-
s3 = get(args, 3, s3, "SOS server type (IOOS_NDBC, IOOS_NOS, OOSTethys, or WHOI)");
467+
s2 = get(args, 2, s2, "SOS version (e.g., 1.0.0)");
468+
s3 = get(args, 3, s3, "SOS server type (52N, IOOS_NDBC, IOOS_NOS, OOSTethys, or WHOI)");
467469
String2.log("working...");
468-
printToBoth(EDDTableFromSOS.generateDatasetsXml(s1, s2, s3));
470+
printToBoth(EDDTableFromSOS.generateDatasetsXml(
471+
false, s1, s2, s3)); //use cached getCapabilities
469472

470473
} else if (eddType.equals("EDDTableFromThreddsFiles")) {
471474
s1 = get(args, 1, s1, "Starting catalog.xml URL");

0 commit comments

Comments
 (0)