Skip to content

Commit 0bc6d18

Browse files
author
Doug Schmidt
authored
Merge pull request #302 from DougSchmidt-AI/feature/PF-1417-PointZillaHistoricalTimezones
Feature/pf 1417 point zilla historical timezones
2 parents 550afbd + ac70900 commit 0bc6d18

18 files changed

+692
-154
lines changed

‎TimeSeries/PublicApis/SdkExamples/PointZilla/App.config

+4
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,10 @@
3333
<assemblyIdentity name="Google.Protobuf" publicKeyToken="a7d26565bac4d604" culture="neutral" />
3434
<bindingRedirect oldVersion="0.0.0.0-3.15.0.0" newVersion="3.15.0.0" />
3535
</dependentAssembly>
36+
<dependentAssembly>
37+
<assemblyIdentity name="NodaTime" publicKeyToken="4226afe0d9b296d1" culture="neutral" />
38+
<bindingRedirect oldVersion="0.0.0.0-1.4.0.0" newVersion="1.4.0.0" />
39+
</dependentAssembly>
3640
</assemblyBinding>
3741
</runtime>
3842
</configuration>

‎TimeSeries/PublicApis/SdkExamples/PointZilla/Context.cs

+5
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,10 @@ public class Context
5050
public List<ExtendedAttributeValue> ExtendedAttributeValues { get; set; } = new List<ExtendedAttributeValue>();
5151
public TimeSeriesType? TimeSeriesType { get; set; }
5252

53+
public DateTimeZone Timezone { get; set; }
54+
public string FindTimezones { get; set; }
55+
public Dictionary<string, string> TimezoneAliases { get; } = new Dictionary<string, string>(StringComparer.InvariantCultureIgnoreCase);
56+
5357
public TimeSeriesIdentifier SourceTimeSeries { get; set; }
5458
public Instant? SourceQueryFrom { get; set; }
5559
public Instant? SourceQueryTo { get; set; }
@@ -87,6 +91,7 @@ public class Context
8791
public Field CsvGradeField { get; set; }
8892
public Field CsvQualifiersField { get; set; }
8993
public Field CsvNotesField { get; set; }
94+
public Field CsvTimezoneField { get; set; }
9095
public string CsvComment { get; set; }
9196
public int CsvSkipRows { get; set; }
9297
public bool CsvHasHeaderRow { get; set; }

‎TimeSeries/PublicApis/SdkExamples/PointZilla/CsvWriter.cs

+4-28
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ public void WritePoints(List<TimeSeriesPoint> points, List<TimeSeriesNote> notes
7878
{
7979
var time = point.Time ?? Instant.MinValue;
8080

81-
var line = $"{InstantPattern.ExtendedIsoPattern.Format(time)}, {point.Value:G12}, {point.GradeCode}, {FormatQualifiers(point.Qualifiers)}";
81+
var line = $"{InstantPattern.ExtendedIso.Format(time)}, {point.Value:G12}, {point.GradeCode}, {FormatQualifiers(point.Qualifiers)}";
8282

8383
if (Context.SaveNotesMode == SaveNotesMode.WithPoints)
8484
{
@@ -116,7 +116,7 @@ public void WritePoints(List<TimeSeriesPoint> points, List<TimeSeriesNote> notes
116116
if (!note.TimeRange.HasValue)
117117
continue;
118118

119-
notesWriter.WriteLine($"{InstantPattern.ExtendedIsoPattern.Format(note.TimeRange.Value.Start)}, {InstantPattern.ExtendedIsoPattern.Format(note.TimeRange.Value.End)}, {CsvEscapedColumn(note.NoteText)}");
119+
notesWriter.WriteLine($"{InstantPattern.ExtendedIso.Format(note.TimeRange.Value.Start)}, {InstantPattern.ExtendedIso.Format(note.TimeRange.Value.End)}, {CsvEscapedColumn(note.NoteText)}");
120120
}
121121
}
122122
}
@@ -133,30 +133,6 @@ private static PublishNote Convert(TimeSeriesNote note)
133133
};
134134
}
135135

136-
public static void SetPointZillaCsvFormat(Context context)
137-
{
138-
// Match PointZilla Export format below
139-
140-
// # CSV data starts at line 15.
141-
// #
142-
// ISO 8601 UTC, Value, Grade, Qualifiers, Notes
143-
// 2015-12-04T00:01:00Z, 3.523200823975, 500, ,
144-
// 2015-12-04T00:02:00Z, 3.525279357147, 500, ,
145-
146-
context.CsvSkipRows = 0;
147-
context.CsvComment = "#";
148-
context.CsvDateTimeField = Field.Parse("ISO 8601 UTC", nameof(context.CsvDateTimeField));
149-
context.CsvDateTimeFormat = null;
150-
context.CsvDateOnlyField = null;
151-
context.CsvTimeOnlyField = null;
152-
context.CsvValueField = Field.Parse("Value", nameof(context.CsvValueField));
153-
context.CsvGradeField = Field.Parse("Grade", nameof(context.CsvGradeField));
154-
context.CsvQualifiersField = Field.Parse("Qualifiers", nameof(context.CsvQualifiersField));
155-
context.CsvNotesField = Field.Parse("Notes", nameof(context.CsvNotesField));
156-
context.CsvIgnoreInvalidRows = true;
157-
context.CsvRealign = false;
158-
}
159-
160136
private TimeSeriesIdentifier CreateTimeSeriesIdentifier()
161137
{
162138
if (Context.SourceTimeSeries != null)
@@ -210,8 +186,8 @@ private static string CreatePeriod(Instant? startTime, Instant? endTime)
210186
private static (string StartText, string EndText) CreatePeriod(Instant start, Instant end)
211187
{
212188
return (
213-
start == Instant.MinValue ? "StartOfRecord" : InstantPattern.ExtendedIsoPattern.Format(start),
214-
end == Instant.MaxValue ? "EndOfRecord" : InstantPattern.ExtendedIsoPattern.Format(end)
189+
start == Instant.MinValue ? "StartOfRecord" : InstantPattern.ExtendedIso.Format(start),
190+
end == Instant.MaxValue ? "EndOfRecord" : InstantPattern.ExtendedIso.Format(end)
215191
);
216192
}
217193

‎TimeSeries/PublicApis/SdkExamples/PointZilla/Field.cs

+7
Original file line numberDiff line numberDiff line change
@@ -38,5 +38,12 @@ private Field(string fieldName, string columnName)
3838
FieldName = fieldName;
3939
ColumnName = columnName;
4040
}
41+
42+
public override string ToString()
43+
{
44+
return HasColumnName
45+
? $"{FieldName}:'{ColumnName}'"
46+
: $"{FieldName}:#{ColumnIndex}";
47+
}
4148
}
4249
}

‎TimeSeries/PublicApis/SdkExamples/PointZilla/Formats.cs

+113
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
using System;
2+
using System.Collections.Generic;
3+
4+
namespace PointZilla
5+
{
6+
public static class Formats
7+
{
8+
public static string Description =>
9+
"Shortcut for known CSV formats. One of 'NG', '3X', or 'PointZilla'. [default: NG]";
10+
11+
public static void SetFormat(Context context, string value)
12+
{
13+
if (!Formatters.TryGetValue(value, out var formatter))
14+
throw new ExpectedException($"'{value}' is an unknown CSV format.");
15+
16+
formatter(context);
17+
}
18+
19+
private static readonly Dictionary<string, Action<Context>> Formatters =
20+
new Dictionary<string, Action<Context>>(StringComparer.InvariantCultureIgnoreCase)
21+
{
22+
{ "NG", SetNgCsvFormat },
23+
{ "3X", Set3XCsvFormat },
24+
{ "PointZilla", SetPointZillaCsvFormat },
25+
};
26+
27+
public static void SetNgCsvFormat(Context context)
28+
{
29+
// Match AQTS 201x Export-from-Springboard CSV format
30+
31+
// # Take [email protected] generated at 2018-09-14 05:03:15 (UTC-07:00) by AQUARIUS 18.3.79.0
32+
// #
33+
// # Time series identifier: Take Volume.CS1004@IM974363
34+
// # Location: 20017_CS1004
35+
// # UTC offset: (UTC+12:00)
36+
// # Value units: m^3
37+
// # Value parameter: Take Volume
38+
// # Interpolation type: Instantaneous Totals
39+
// # Time series type: Basic
40+
// #
41+
// # Export options: Corrected signal from Beginning of Record to End of Record
42+
// #
43+
// # CSV data starts at line 15.
44+
// #
45+
// ISO 8601 UTC, Timestamp (UTC+12:00), Value, Approval Level, Grade, Qualifiers
46+
// 2013-07-01T11:59:59Z,2013-07-01 23:59:59,966.15,Raw - yet to be review,200,
47+
// 2013-07-02T11:59:59Z,2013-07-02 23:59:59,966.15,Raw - yet to be review,200,
48+
// 2013-07-03T11:59:59Z,2013-07-03 23:59:59,966.15,Raw - yet to be review,200,
49+
50+
context.CsvSkipRows = 0;
51+
context.CsvComment = "#";
52+
context.CsvDateTimeField = Field.Parse("ISO 8601 UTC", nameof(context.CsvDateTimeField));
53+
context.CsvDateTimeFormat = null;
54+
context.CsvDateOnlyField = null;
55+
context.CsvTimeOnlyField = null;
56+
context.CsvTimezoneField = null;
57+
context.CsvValueField = Field.Parse("Value", nameof(context.CsvValueField));
58+
context.CsvGradeField = Field.Parse("Grade", nameof(context.CsvGradeField));
59+
context.CsvQualifiersField = Field.Parse("Qualifiers", nameof(context.CsvQualifiersField));
60+
context.CsvIgnoreInvalidRows = true;
61+
context.CsvRealign = false;
62+
}
63+
64+
public static void Set3XCsvFormat(Context context)
65+
{
66+
// Match AQTS 3.x Export format
67+
68+
// ,Take Volume.CS1004@IM974363,Take Volume.CS1004@IM974363,Take Volume.CS1004@IM974363,Take Volume.CS1004@IM974363
69+
// mm/dd/yyyy HH:MM:SS,m^3,,,
70+
// Date-Time,Value,Grade,Approval,Interpolation Code
71+
// 07/01/2013 23:59:59,966.15,200,1,6
72+
// 07/02/2013 23:59:59,966.15,200,1,6
73+
74+
context.CsvComment = null;
75+
context.CsvSkipRows = 2;
76+
context.CsvDateTimeField = Field.Parse("Date-Time", nameof(context.CsvDateTimeField));
77+
context.CsvDateTimeFormat = "MM/dd/yyyy HH:mm:ss";
78+
context.CsvDateOnlyField = null;
79+
context.CsvTimeOnlyField = null;
80+
context.CsvTimezoneField = null;
81+
context.CsvValueField = Field.Parse("Value", nameof(context.CsvValueField));
82+
context.CsvGradeField = Field.Parse("Grade", nameof(context.CsvGradeField));
83+
context.CsvQualifiersField = null;
84+
context.CsvIgnoreInvalidRows = true;
85+
context.CsvRealign = false;
86+
}
87+
88+
public static void SetPointZillaCsvFormat(Context context)
89+
{
90+
// Match PointZilla Export format below
91+
92+
// # CSV data starts at line 15.
93+
// #
94+
// ISO 8601 UTC, Value, Grade, Qualifiers, Notes
95+
// 2015-12-04T00:01:00Z, 3.523200823975, 500, ,
96+
// 2015-12-04T00:02:00Z, 3.525279357147, 500, ,
97+
98+
context.CsvSkipRows = 0;
99+
context.CsvComment = "#";
100+
context.CsvDateTimeField = Field.Parse("ISO 8601 UTC", nameof(context.CsvDateTimeField));
101+
context.CsvDateTimeFormat = null;
102+
context.CsvDateOnlyField = null;
103+
context.CsvTimeOnlyField = null;
104+
context.CsvTimezoneField = null;
105+
context.CsvValueField = Field.Parse("Value", nameof(context.CsvValueField));
106+
context.CsvGradeField = Field.Parse("Grade", nameof(context.CsvGradeField));
107+
context.CsvQualifiersField = Field.Parse("Qualifiers", nameof(context.CsvQualifiersField));
108+
context.CsvNotesField = Field.Parse("Notes", nameof(context.CsvNotesField));
109+
context.CsvIgnoreInvalidRows = true;
110+
context.CsvRealign = false;
111+
}
112+
}
113+
}

‎TimeSeries/PublicApis/SdkExamples/PointZilla/PointReaders/CsvReader.cs

+25-4
Original file line numberDiff line numberDiff line change
@@ -263,12 +263,21 @@ private TimeSeriesPoint ParseExcelRow(DataRow row)
263263
double? value = null;
264264
int? gradeCode = null;
265265
List<string> qualifiers = null;
266+
DateTimeZone zone = null;
266267

267268
if (!string.IsNullOrEmpty(Context.CsvComment) && ((row[0] as string)?.StartsWith(Context.CsvComment) ?? false))
268269
return null;
269270

270271
try
271272
{
273+
ParseExcelStringColumn(row, Context.CsvTimezoneField?.ColumnIndex, text =>
274+
{
275+
if (Context.TimezoneAliases.TryGetValue(text, out var alias))
276+
text = alias;
277+
278+
TimezoneHelper.TryParseDateTimeZone(text, out zone);
279+
});
280+
272281
if (Context.CsvDateOnlyField != null)
273282
{
274283
var dateOnly = DateTime.MinValue;
@@ -281,11 +290,11 @@ private TimeSeriesPoint ParseExcelRow(DataRow row)
281290
ParseExcelColumn<DateTime>(row, Context.CsvTimeOnlyField.ColumnIndex, dateTime => timeOnly = dateTime.TimeOfDay);
282291
}
283292

284-
time = InstantFromDateTime(dateOnly.Add(timeOnly));
293+
time = InstantFromDateTime(dateOnly.Add(timeOnly), () => zone);
285294
}
286295
else
287296
{
288-
ParseExcelColumn<DateTime>(row, Context.CsvDateTimeField.ColumnIndex, dateTime => time = InstantFromDateTime(dateTime));
297+
ParseExcelColumn<DateTime>(row, Context.CsvDateTimeField.ColumnIndex, dateTime => time = InstantFromDateTime(dateTime, () => zone));
289298
}
290299

291300
if (string.IsNullOrEmpty(Context.CsvNanValue))
@@ -451,6 +460,18 @@ private TimeSeriesPoint ParsePoint(string[] fields)
451460
int? gradeCode = null;
452461
List<string> qualifiers = null;
453462
PointType? pointType = null;
463+
DateTimeZone zone = null;
464+
465+
if (Context.CsvTimezoneField != null)
466+
{
467+
ParseField(fields, Context.CsvTimezoneField.ColumnIndex, text =>
468+
{
469+
if (Context.TimezoneAliases.TryGetValue(text, out var alias))
470+
text = alias;
471+
472+
TimezoneHelper.TryParseDateTimeZone(text, out zone);
473+
});
474+
}
454475

455476
if (Context.CsvDateOnlyField != null)
456477
{
@@ -473,7 +494,7 @@ private TimeSeriesPoint ParsePoint(string[] fields)
473494
ParseField(fields, Context.CsvTimeOnlyField.ColumnIndex, text => timeOnly = ParseTimeOnly(text, Context.CsvTimeOnlyFormat));
474495
}
475496

476-
time = InstantFromDateTime(dateOnly.Add(timeOnly));
497+
time = InstantFromDateTime(dateOnly.Add(timeOnly), () => zone);
477498
}
478499
else
479500
{
@@ -485,7 +506,7 @@ private TimeSeriesPoint ParsePoint(string[] fields)
485506
return;
486507
}
487508

488-
time = ParseInstant(text);
509+
time = ParseInstant(text, () => zone);
489510
});
490511
}
491512

‎TimeSeries/PublicApis/SdkExamples/PointZilla/PointReaders/CsvReaderBase.cs

+55-16
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
using System;
2-
using System.Collections.Generic;
32
using System.Linq;
43
using System.Reflection;
5-
using System.Text;
6-
using System.Threading.Tasks;
74
using NodaTime;
85
using NodaTime.Text;
96
using ServiceStack.Logging;
@@ -12,35 +9,77 @@ namespace PointZilla.PointReaders
129
{
1310
public abstract class CsvReaderBase : PointReaderBase
1411
{
12+
// ReSharper disable once PossibleNullReferenceException
1513
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
1614

17-
private InstantPattern TimePattern { get; }
15+
private InstantPattern InstantPattern { get; }
16+
private LocalDateTimePattern LocalDateTimePattern { get; }
1817

1918
protected CsvReaderBase(Context context)
2019
: base(context)
2120
{
22-
TimePattern = string.IsNullOrWhiteSpace(Context.CsvDateTimeFormat)
23-
? InstantPattern.ExtendedIsoPattern
24-
: InstantPattern.CreateWithInvariantCulture(Context.CsvDateTimeFormat);
21+
InstantPattern = HasZoneInfo
22+
? null
23+
: string.IsNullOrWhiteSpace(Context.CsvDateTimeFormat)
24+
? InstantPattern.ExtendedIso
25+
: InstantPattern.CreateWithInvariantCulture(Context.CsvDateTimeFormat);
2526

26-
var isTimeFormatUtc = TimePattern.PatternText.Contains("'Z'");
27+
LocalDateTimePattern = HasZoneInfo
28+
? string.IsNullOrWhiteSpace(Context.CsvDateTimeFormat)
29+
? LocalDateTimePattern.ExtendedIso
30+
: LocalDateTimePattern.CreateWithInvariantCulture(Context.CsvDateTimeFormat)
31+
: null;
32+
33+
var isTimeFormatUtc = (InstantPattern?.PatternText.Contains("'Z'") ?? false) || (LocalDateTimePattern?.PatternText.Contains("'Z'") ?? false);
2734

2835
if (Context.CsvDateOnlyField != null)
2936
{
30-
isTimeFormatUtc = Context.CsvDateOnlyFormat.Contains("Z");
37+
isTimeFormatUtc |= Context.CsvDateOnlyFormat.Contains("Z");
3138
}
3239

33-
DefaultBias = isTimeFormatUtc
34-
? Duration.Zero
35-
: Duration.FromTimeSpan((Context.UtcOffset ?? Offset.FromTicks(DateTimeOffset.Now.Offset.Ticks)).ToTimeSpan());
40+
if (!isTimeFormatUtc)
41+
return;
42+
43+
DefaultBias = Duration.Zero;
44+
45+
if (!HasZoneInfo)
46+
return;
47+
48+
var patterns = new[]
49+
{
50+
InstantPattern?.PatternText,
51+
LocalDateTimePattern?.PatternText,
52+
Context.CsvDateOnlyFormat
53+
}
54+
.Where(s => !string.IsNullOrWhiteSpace(s))
55+
.ToList();
56+
57+
var settingMessage = Context.Timezone != null
58+
? $"/{nameof(context.Timezone)}='{context.Timezone}'"
59+
: $"/{nameof(context.CsvTimezoneField)}={context.CsvTimezoneField}";
60+
61+
Log.Warn($"Ignoring the {settingMessage} value since the time-format patterns \"{string.Join("\" and \"", patterns)}\" contain zone-info.");
62+
Context.Timezone = null;
63+
Context.CsvTimezoneField = null;
3664
}
3765

38-
protected Instant? ParseInstant(string text)
66+
protected Instant? ParseInstant(string text, Func<DateTimeZone> zoneResolver = null)
3967
{
40-
var result = TimePattern.Parse(text);
68+
if (LocalDateTimePattern != null)
69+
{
70+
var result = LocalDateTimePattern.Parse(text);
71+
72+
if (result.Success)
73+
return InstantFromLocalDateTime(result.Value, zoneResolver);
74+
}
4175

42-
if (result.Success)
43-
return result.Value.Minus(DefaultBias);
76+
if (InstantPattern != null)
77+
{
78+
var result = InstantPattern.Parse(text);
79+
80+
if (result.Success)
81+
return result.Value.Minus(DefaultBias);
82+
}
4483

4584
return null;
4685
}

0 commit comments

Comments
 (0)