Skip to content

Commit 8978eb3

Browse files
author
Doug Schmidt
authored
Merge pull request #291 from DougSchmidt-AI/feature/PF-1365-NtdnReportingTweaks
PF-1365 - Minor tweaks
2 parents dc77e81 + b245225 commit 8978eb3

File tree

9 files changed

+167
-22
lines changed

9 files changed

+167
-22
lines changed

‎Samples/DotNetSdk/LabFileImporter/LabFileImporter.csproj

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@
6060
<Reference Include="NodaTime, Version=1.3.0.0, Culture=neutral, PublicKeyToken=4226afe0d9b296d1, processorArchitecture=MSIL">
6161
<HintPath>..\packages\NodaTime.1.3.0\lib\net35-Client\NodaTime.dll</HintPath>
6262
</Reference>
63-
<Reference Include="RestSharp, Version=106.11.7.0, Culture=neutral, PublicKeyToken=598062e77f915f75, processorArchitecture=MSIL">
64-
<HintPath>..\packages\RestSharp.106.11.7\lib\net452\RestSharp.dll</HintPath>
63+
<Reference Include="RestSharp, Version=106.15.0.0, Culture=neutral, PublicKeyToken=598062e77f915f75, processorArchitecture=MSIL">
64+
<HintPath>..\packages\RestSharp.106.15.0\lib\net452\RestSharp.dll</HintPath>
6565
</Reference>
6666
<Reference Include="ServiceStack.Client, Version=5.0.0.0, Culture=neutral, PublicKeyToken=02c12cbda47e6587, processorArchitecture=MSIL">
6767
<HintPath>..\packages\ServiceStack.Client.5.10.4\lib\net45\ServiceStack.Client.dll</HintPath>
@@ -73,7 +73,7 @@
7373
<HintPath>..\packages\ServiceStack.Interfaces.5.10.4\lib\net472\ServiceStack.Interfaces.dll</HintPath>
7474
</Reference>
7575
<Reference Include="ServiceStack.Logging.Log4Net, Version=5.0.0.0, Culture=neutral, PublicKeyToken=02c12cbda47e6587, processorArchitecture=MSIL">
76-
<HintPath>..\packages\ServiceStack.Logging.Log4Net.5.9.0\lib\net45\ServiceStack.Logging.Log4Net.dll</HintPath>
76+
<HintPath>..\packages\ServiceStack.Logging.Log4Net.5.10.4\lib\net45\ServiceStack.Logging.Log4Net.dll</HintPath>
7777
</Reference>
7878
<Reference Include="ServiceStack.Text, Version=5.0.0.0, Culture=neutral, PublicKeyToken=02c12cbda47e6587, processorArchitecture=MSIL">
7979
<HintPath>..\packages\ServiceStack.Text.5.10.4\lib\net45\ServiceStack.Text.dll</HintPath>

‎Samples/DotNetSdk/LabFileImporter/packages.config

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,11 @@
1313
<package id="Microsoft.CSharp" version="4.5.0" targetFramework="net472" />
1414
<package id="Microsoft.Net.Http" version="2.2.29" targetFramework="net472" />
1515
<package id="NodaTime" version="1.3.0" targetFramework="net472" />
16-
<package id="RestSharp" version="106.12.0" targetFramework="net472" />
16+
<package id="RestSharp" version="106.15.0" targetFramework="net472" />
1717
<package id="ServiceStack.Client" version="5.10.4" targetFramework="net472" />
1818
<package id="ServiceStack.HttpClient" version="5.10.4" targetFramework="net472" />
1919
<package id="ServiceStack.Interfaces" version="5.10.4" targetFramework="net472" />
20-
<package id="ServiceStack.Logging.Log4Net" version="5.9.0" targetFramework="net472" />
20+
<package id="ServiceStack.Logging.Log4Net" version="5.10.4" targetFramework="net472" />
2121
<package id="ServiceStack.Text" version="5.10.4" targetFramework="net472" />
2222
<package id="System.Buffers" version="4.5.1" targetFramework="net472" />
2323
<package id="System.Memory" version="4.5.4" targetFramework="net472" />

‎Samples/DotNetSdk/NWFWMD-LabFileImporter/NWFWMD-LabFileImporter.csproj

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,8 @@
5858
<Reference Include="NodaTime, Version=1.3.0.0, Culture=neutral, PublicKeyToken=4226afe0d9b296d1, processorArchitecture=MSIL">
5959
<HintPath>..\packages\NodaTime.1.3.0\lib\net35-Client\NodaTime.dll</HintPath>
6060
</Reference>
61-
<Reference Include="RestSharp, Version=106.11.7.0, Culture=neutral, PublicKeyToken=598062e77f915f75, processorArchitecture=MSIL">
62-
<HintPath>..\packages\RestSharp.106.11.7\lib\net452\RestSharp.dll</HintPath>
61+
<Reference Include="RestSharp, Version=106.15.0.0, Culture=neutral, PublicKeyToken=598062e77f915f75, processorArchitecture=MSIL">
62+
<HintPath>..\packages\RestSharp.106.15.0\lib\net452\RestSharp.dll</HintPath>
6363
</Reference>
6464
<Reference Include="ServiceStack.Client, Version=5.0.0.0, Culture=neutral, PublicKeyToken=02c12cbda47e6587, processorArchitecture=MSIL">
6565
<HintPath>..\packages\ServiceStack.Client.5.10.4\lib\net45\ServiceStack.Client.dll</HintPath>
@@ -71,7 +71,7 @@
7171
<HintPath>..\packages\ServiceStack.Interfaces.5.10.4\lib\net472\ServiceStack.Interfaces.dll</HintPath>
7272
</Reference>
7373
<Reference Include="ServiceStack.Logging.Log4Net, Version=5.0.0.0, Culture=neutral, PublicKeyToken=02c12cbda47e6587, processorArchitecture=MSIL">
74-
<HintPath>..\packages\ServiceStack.Logging.Log4Net.5.9.0\lib\net45\ServiceStack.Logging.Log4Net.dll</HintPath>
74+
<HintPath>..\packages\ServiceStack.Logging.Log4Net.5.10.4\lib\net45\ServiceStack.Logging.Log4Net.dll</HintPath>
7575
</Reference>
7676
<Reference Include="ServiceStack.Text, Version=5.0.0.0, Culture=neutral, PublicKeyToken=02c12cbda47e6587, processorArchitecture=MSIL">
7777
<HintPath>..\packages\ServiceStack.Text.5.10.4\lib\net45\ServiceStack.Text.dll</HintPath>

‎Samples/DotNetSdk/NWFWMD-LabFileImporter/packages.config

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,11 @@
1212
<package id="Microsoft.CSharp" version="4.5.0" targetFramework="net472" />
1313
<package id="Microsoft.Net.Http" version="2.2.29" targetFramework="net472" />
1414
<package id="NodaTime" version="1.3.0" targetFramework="net472" />
15-
<package id="RestSharp" version="106.12.0" targetFramework="net472" />
15+
<package id="RestSharp" version="106.15.0" targetFramework="net472" />
1616
<package id="ServiceStack.Client" version="5.10.4" targetFramework="net472" />
1717
<package id="ServiceStack.HttpClient" version="5.10.4" targetFramework="net472" />
1818
<package id="ServiceStack.Interfaces" version="5.10.4" targetFramework="net472" />
19-
<package id="ServiceStack.Logging.Log4Net" version="5.9.0" targetFramework="net472" />
19+
<package id="ServiceStack.Logging.Log4Net" version="5.10.4" targetFramework="net472" />
2020
<package id="ServiceStack.Text" version="5.10.4" targetFramework="net472" />
2121
<package id="System.Buffers" version="4.5.1" targetFramework="net472" />
2222
<package id="System.Memory" version="4.5.4" targetFramework="net472" />

‎Samples/DotNetSdk/ObservationReportExporter/Exporter.cs

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ public class Exporter
3939
private List<ApplyTagRequest> AppliedTags { get; } = new List<ApplyTagRequest>();
4040

4141
private int ExportedLocations { get; set; }
42+
private int ExportedObservations { get; set; }
4243
private int SkippedLocations { get; set; }
4344
private int Errors { get; set; }
4445
private int DeletedAttachments { get; set; }
@@ -59,7 +60,7 @@ public void Run()
5960
ExportAllLocations();
6061
}
6162

62-
LogAction($"Exported observations to {"location".ToQuantity(ExportedLocations)}, skipping {"unknown location".ToQuantity(SkippedLocations)}, deleting {"existing attachment".ToQuantity(DeletedAttachments)}, with {"detected error".ToQuantity(Errors)} in {stopwatch.Elapsed.Humanize()}.");
63+
LogAction($"Exported {"observation".ToQuantity(ExportedObservations)} using '{ExportTemplate.CustomId}' template into {"location".ToQuantity(ExportedLocations)}, skipping {"unknown location".ToQuantity(SkippedLocations)}, deleting {"existing attachment".ToQuantity(DeletedAttachments)}, with {"detected error".ToQuantity(Errors)} in {stopwatch.Elapsed.Humanize()}.");
6364
}
6465

6566
private void LogAction(string message)
@@ -428,10 +429,6 @@ private void ExportLocation(SamplingLocation location)
428429
DeleteExistingAttachment(locationData, existingAttachment);
429430
}
430431

431-
LogAction($"Exporting observations from '{location.CustomId}' ...");
432-
433-
++ExportedLocations;
434-
435432
var exportRequest = new GetExportObservations
436433
{
437434
EndObservedTime = FromDateTimeOffset(Context.EndTime),
@@ -441,6 +438,21 @@ private void ExportLocation(SamplingLocation location)
441438
AnalyticalGroupIds = AnalyticalGroupIds,
442439
};
443440

441+
var exportedObservationCount = Samples.Get(new GetObservationsV2
442+
{
443+
EndObservedTime = exportRequest.EndObservedTime,
444+
StartObservedTime = exportRequest.StartObservedTime,
445+
SamplingLocationIds = exportRequest.SamplingLocationIds,
446+
ObservedPropertyIds = exportRequest.ObservedPropertyIds,
447+
AnalyticalGroupIds = exportRequest.AnalyticalGroupIds
448+
}).TotalCount;
449+
450+
LogAction($"Exporting {"observation".ToQuantity(exportedObservationCount)} from '{location.CustomId}' ...");
451+
452+
++ExportedLocations;
453+
ExportedObservations += exportedObservationCount;
454+
455+
// Need to hack the URL until WI-4928 is fixed
444456
var url = $"{(Samples.Client as JsonServiceClient)?.BaseUri}{exportRequest.ToGetUrl()}&observationTemplateAttachmentId={ExportTemplate.Attachments.Single().Id}&format=xlsx";
445457

446458
if (Context.DryRun)

‎Samples/DotNetSdk/ObservationReportExporter/ObservationReportExporter.csproj

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -144,8 +144,8 @@
144144
<HintPath>..\packages\System.Memory.4.5.4\lib\net461\System.Memory.dll</HintPath>
145145
</Reference>
146146
<Reference Include="System.Net" />
147-
<Reference Include="System.Net.Http, Version=4.1.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
148-
<HintPath>..\packages\System.Net.Http.4.3.0\lib\net46\System.Net.Http.dll</HintPath>
147+
<Reference Include="System.Net.Http, Version=4.1.1.3, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
148+
<HintPath>..\packages\System.Net.Http.4.3.4\lib\net46\System.Net.Http.dll</HintPath>
149149
<Private>True</Private>
150150
<Private>True</Private>
151151
</Reference>
@@ -215,7 +215,7 @@
215215
</Reference>
216216
<Reference Include="System.ServiceModel" />
217217
<Reference Include="System.Text.RegularExpressions, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
218-
<HintPath>..\packages\System.Text.RegularExpressions.4.3.0\lib\net463\System.Text.RegularExpressions.dll</HintPath>
218+
<HintPath>..\packages\System.Text.RegularExpressions.4.3.1\lib\net463\System.Text.RegularExpressions.dll</HintPath>
219219
<Private>True</Private>
220220
<Private>True</Private>
221221
</Reference>
@@ -248,6 +248,7 @@
248248
<Compile Include="Option.cs" />
249249
<Compile Include="Program.cs" />
250250
<Compile Include="Properties\AssemblyInfo.cs" />
251+
<Compile Include="SingleInstanceGuard.cs" />
251252
</ItemGroup>
252253
<ItemGroup>
253254
<None Include="App.config" />

‎Samples/DotNetSdk/ObservationReportExporter/Program.cs

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
using System.IO;
44
using System.Linq;
55
using System.Reflection;
6+
using System.Security.Cryptography;
7+
using System.Text;
68
using System.Text.RegularExpressions;
79
using System.Xml;
810
using Aquarius.Samples.Client;
@@ -64,17 +66,15 @@ private static void ConfigureLogging()
6466

6567
log4net.Config.XmlConfigurator.Configure(xml.DocumentElement);
6668

67-
// ReSharper disable once PossibleNullReferenceException
68-
_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
69+
_log = LogManager.GetLogger(GetProgramType());
6970

7071
ServiceStack.Logging.LogManager.LogFactory = new Log4NetFactory();
7172
}
7273
}
7374

7475
private static byte[] LoadEmbeddedResource(string path)
7576
{
76-
// ReSharper disable once PossibleNullReferenceException
77-
var resourceName = $"{MethodBase.GetCurrentMethod().DeclaringType.Namespace}.{path}";
77+
var resourceName = $"{GetProgramType().Namespace}.{path}";
7878

7979
using (var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(resourceName))
8080
{
@@ -85,6 +85,11 @@ private static byte[] LoadEmbeddedResource(string path)
8585
}
8686
}
8787

88+
private static Type GetProgramType()
89+
{
90+
return MethodBase.GetCurrentMethod()?.DeclaringType;
91+
}
92+
8893
private static Context ParseArgs(string[] args)
8994
{
9095
var context = new Context();
@@ -349,11 +354,33 @@ public Program(Context context)
349354

350355
private void Run()
351356
{
357+
using (var guard = new SingleInstanceGuard(GetContextName()))
358+
{
359+
if (!guard.IsAnotherInstanceRunning())
360+
{
361+
new Exporter { Context = _context }
362+
.Run();
363+
}
364+
else
365+
{
366+
_log.Warn($"Exiting while another instance of {guard.Name} is running.");
367+
}
368+
}
352369
new Exporter
353370
{
354371
Context = _context
355372
}
356373
.Run();
357374
}
375+
376+
private string GetContextName()
377+
{
378+
var jsonText = _context.ToJson();
379+
380+
using (var sha256 = new SHA256Managed())
381+
{
382+
return BitConverter.ToString(sha256.ComputeHash(Encoding.UTF8.GetBytes(jsonText)));
383+
}
384+
}
358385
}
359386
}

‎Samples/DotNetSdk/ObservationReportExporter/Readme.md

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,28 @@ The intent is to run this tool on a schedule so that the uploaded location attac
1212
- Results can be filtered by optional date range, location, location group, property, or analytical group.
1313
- All filters are cumulative (ie. they are AND-ed together). The more filters you add, the fewer results will be exported.
1414
- An exit code of 0 indicates the export was successful. An exit code greater than 0 indicates an error. This allows for easy error checking when invoked from scripts.
15+
- A log file named `ObservationReportExporter.log` will be created in the same folder as the EXE.
1516

1617
## Requirements
1718

1819
- The .NET 4.7 runtime is required, which is pre-installed on all Windows 10 and Windows Server 2016 systems, and on nearly all up-to-date Windows 7 and Windows Server 2008 systems.
1920
- No installer is needed. It is just a single .EXE which can be run from any folder.
2021

22+
## Basic workflow
23+
24+
Each run of the tool will:
25+
- Validate all the provided options and stop right away if any errors are detected.
26+
- Determine the AQUARIUS Samples locations from which observations will be exported. This is either the list of `-LocationId=` options locations or locations belonging to the `-LocationGroupId=` options.
27+
- For each exported location:
28+
- Find the matching AQUARIUS Time Series location.
29+
- If the Time Series location is not found:
30+
- Log a warning and move on to the next export.
31+
- Otherwise:
32+
- Determine the exported attachment filename (see [controlling the attachment filename](#controlling-the-aqts-attachment-filename) for details).
33+
- Delete any existing location attachments with the same filename.
34+
- Export all the filtered observations from the AQUARIUS Samples location using the named template.
35+
- Upload the exported spreadsheet of location-specific observations to the AQUARIUS Time-Series location.
36+
2137
### Command line option syntax
2238

2339
All command line options are case-insensitive, and support both common shell syntaxes: either `/Name=value` (for CMD.EXE) or `-Name=value` (for bash and PowerShell).
@@ -26,6 +42,22 @@ In addition, the [`@options.txt` syntax](https://github.com/AquaticInformatics/e
2642

2743
Try the `/help` option for a detailed list of options and their default values.
2844

45+
## Running the tool on a periodic schedule
46+
47+
The `ObservationReportExporter.exe` too can be run from Windows Task Scheduler, or other scheduling software.
48+
49+
Typical configuration involves:
50+
- Storing all required command-line options in a single text file, in the same folder as the EXE.
51+
- Specifying the executable as full path to the `ObservationReportExporter.exe` tool.
52+
- Setting the working directory to the folder containing the EXE.
53+
- Setting the arguments to the `@Options.txt` file containing all the required options.
54+
55+
No special Windows account is required to run the tool. All the required credentials are supplied as command-line options, so it is fine to run the tool using a built-in Windows account like LocalSystem (NT_AUTHORITY/SYSTEM).
56+
57+
The tool can be scheduled to run at whatever frequency you would like. The tool will quickly exit if it detects an identical export request already in progress. This allows for simple scheduling for normal loads.
58+
59+
If a complete export cycle normally takes 2 hours, but can sometimes take 4 hours to complete, you can still safely schedule the tool to run every 3 hours, and tool will detect when a previous cycle hasn't completed and won't go crazy.
60+
2961
## Authentication with AQUARIUS Samples
3062

3163
Two options are required to tell the tool how to access your AQUARIUS Samples instance.

‎Samples/DotNetSdk/ObservationReportExporter/SingleInstanceGuard.cs

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
using System;
2+
using System.Reflection;
3+
using System.Threading;
4+
using log4net;
5+
6+
namespace ObservationReportExporter
7+
{
8+
public class SingleInstanceGuard : IDisposable
9+
{
10+
// ReSharper disable once PossibleNullReferenceException
11+
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
12+
13+
private Mutex InstanceMutex { get; set; }
14+
public string Name { get; }
15+
private bool ShouldRelease { get; set; }
16+
17+
public SingleInstanceGuard(string name)
18+
{
19+
Name = $"{ExeHelper.ExeName}.{name}";
20+
InstanceMutex = new Mutex(true, Name);
21+
ShouldRelease = true;
22+
}
23+
24+
public bool IsAnotherInstanceRunning()
25+
{
26+
try
27+
{
28+
var isAnotherInstanceRunning = !InstanceMutex.WaitOne(TimeSpan.Zero, true);
29+
30+
if (isAnotherInstanceRunning)
31+
{
32+
ShouldRelease = false;
33+
}
34+
35+
return isAnotherInstanceRunning;
36+
}
37+
catch (AbandonedMutexException)
38+
{
39+
Log.Debug($"Previous run of the program did not clear the '{Name}' mutex cleanly.");
40+
41+
return false;
42+
}
43+
catch (Exception ex)
44+
{
45+
Log.Warn($"Error occurred while checking if the program is still running:'{ex.Message}'. Will continue.");
46+
}
47+
48+
return false;
49+
}
50+
51+
public void Dispose()
52+
{
53+
Release();
54+
55+
InstanceMutex?.Dispose();
56+
}
57+
58+
private void Release()
59+
{
60+
if (!ShouldRelease)
61+
return;
62+
63+
try
64+
{
65+
InstanceMutex?.ReleaseMutex();
66+
}
67+
catch (Exception e)
68+
{
69+
Log.Warn($"Can't release mutex '{Name}': {e.Message}");
70+
}
71+
}
72+
}
73+
}

0 commit comments

Comments
 (0)