Skip to content

Commit 7d43bb2

Browse files
CorgamCorgam
authored
Yaml files split + DB health checks (#319)
2 parents 041c1f0 + 2777269 commit 7d43bb2

File tree

120 files changed

+244
-113
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

120 files changed

+244
-113
lines changed

‎.github/workflows/deploy_production.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,6 @@ jobs:
8686
scp ./docker-compose.yml ${{secrets.PROD_ENV_SSH_USER}}@${{secrets.PROD_ENV_SSH_HOST}}:/var/lib/bie
8787
scp ./.env.production ${{secrets.PROD_ENV_SSH_USER}}@${{secrets.PROD_ENV_SSH_HOST}}:/var/lib/bie/.env
8888
- name: Connect and Pull
89-
run: ssh ${{secrets.PROD_ENV_SSH_USER}}@${{secrets.PROD_ENV_SSH_HOST}} "cd /var/lib/bie && docker compose down --remove-orphans && docker compose pull && docker compose up -d --remove-orphans && exit"
89+
run: ssh ${{secrets.PROD_ENV_SSH_USER}}@${{secrets.PROD_ENV_SSH_HOST}} "cd /var/lib/bie && docker compose pull && docker compose up -d --remove-orphans --force-recreate && exit"
9090
- name: Cleanup
9191
run: rm -rf ~/.ssh

‎.github/workflows/deploy_test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ jobs:
8989
scp ./.env.test ${{ secrets.TEST_ENV_SSH_USER }}@${{ secrets.TEST_ENV_SSH_HOST }}:/var/lib/bie/.env
9090
9191
- name: Connect and Pull
92-
run: ssh ${{ secrets.TEST_ENV_SSH_USER }}@${{ secrets.TEST_ENV_SSH_HOST }} "cd /var/lib/bie && docker compose down --remove-orphans && docker compose pull && docker compose up -d --remove-orphans && exit"
92+
run: ssh ${{ secrets.TEST_ENV_SSH_USER }}@${{ secrets.TEST_ENV_SSH_HOST }} "cd /var/lib/bie && docker compose pull && docker compose up -d --remove-orphans --force-recreate && exit"
9393

9494
- name: Cleanup
9595
run: rm -rf ~/.ssh

‎backend/api-gateway/Models/DatasetBasicData.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,12 @@ public class DatasetBasicData
66
public string Name { get; set; }
77
public string Description { get; set; }
88
public string Icon { get; set; }
9+
public List<TableData> Tables { get; set; }
10+
}
11+
12+
public class TableData
13+
{
14+
public string Name { get; set; }
15+
public int NumberOfLines { get; set; }
916
}
1017
}

‎backend/sql-database/Dockerfile‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from mcr.microsoft.com/mssql/server:2019-latest
1+
FROM mcr.microsoft.com/mssql/server:2019-latest
22
ENV SA_PASSWORD MyPass@1234
33
ENV ACCEPT_EULA Y
44

@@ -7,15 +7,13 @@ USER root
77
RUN apt-get update && apt-get install -y dos2unix
88

99

10-
copy ./ /
11-
copy ./scripts/CreateDB.sql /setup.sql
10+
COPY ./ /
11+
COPY ./scripts/CreateDB.sql /setup.sql
1212

1313
RUN dos2unix /entrypoint.sh
1414
RUN dos2unix /run-script.sh
1515

1616
RUN chmod +x /run-script.sh
1717

1818
ENTRYPOINT ["/bin/bash","entrypoint.sh"]
19-
CMD ["/opt/mssql/bin/sqlservr"]
20-
21-
19+
CMD ["/opt/mssql/bin/sqlservr"]

‎backend/src/BIE.DataPipeline/Dockerfile‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,27 @@
1+
# Prepare the basline image
12
FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS base
2-
workdir /app
3+
WORKDIR /app
34

4-
expose 445
5+
EXPOSE 445
56

67
USER root
7-
88
RUN apt-get update && apt-get install -y dos2unix
99

10+
# Build the dotnet project
1011
FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build
11-
workdir /app
12+
WORKDIR /app
1213

1314
COPY *.csproj .
1415
RUN dotnet restore
1516

16-
copy . ./
17+
COPY . ./
18+
1719
RUN dotnet publish "BIE.DataPipeline.csproj" --output /app/ --configuration Release
1820

19-
from base as final
20-
workdir /app
21-
copy --from=build /app .
21+
# Run the project
22+
FROM base AS final
23+
WORKDIR /app
24+
COPY --from=build /app .
2225

2326
RUN dos2unix entrypoint.sh
2427

‎backend/src/BIE.DataPipeline/Import/CsvImporter.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ public bool ReadLine(out string nextLine)
146146
//check if the value can be empty
147147
if (dataSourceDescription.table_cols[yamlIndex].is_not_nullable && line[fileIndex] == "")
148148
{
149-
Console.WriteLine("Line does not match not null criteria");
149+
Console.WriteLine("Line does not match not null criteria, skipping...");
150150
//Read next line
151151
builder.Clear();
152152
break;

‎backend/src/BIE.DataPipeline/Import/ShapeImporter.cs‎

Lines changed: 43 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
using ProjNet.CoordinateSystems;
88
using ProjNet.CoordinateSystems.Transformations;
99

10-
1110
namespace BIE.DataPipeline.Import
1211
{
1312
internal class ShapeImporter : IImporter
@@ -19,7 +18,7 @@ internal class ShapeImporter : IImporter
1918

2019
public ShapeImporter(DataSourceDescription? dataSourceDescription)
2120
{
22-
//YAML Arguments:
21+
// YAML Arguments:
2322
this.mDataSourceDescription = dataSourceDescription;
2423

2524
new StringBuilder();
@@ -46,17 +45,47 @@ public ShapeImporter(DataSourceDescription? dataSourceDescription)
4645

4746
private void SetupParser()
4847
{
49-
// handle Zip file:
48+
MemoryStream shpStream = new MemoryStream();
49+
MemoryStream dbfStream = new MemoryStream();
50+
51+
if (mDataSourceDescription.source.type.Equals("filepath"))
52+
{
53+
// handle local Zip file:
54+
Console.WriteLine($"Opening local Zip file {mDataSourceDescription.source.location}");
55+
using (var zipStream = new FileStream(mDataSourceDescription.source.location, FileMode.Open))
56+
{
57+
var zipArchive = new ZipArchive(zipStream, ZipArchiveMode.Read);
58+
ExtractShapeFilesFromZip(zipArchive, shpStream, dbfStream);
59+
}
60+
}
61+
else
62+
{
63+
// handle Zip file from URL:
64+
Console.WriteLine($"Grabbing Webfile {mDataSourceDescription.source.location}");
65+
var client = new HttpClient();
66+
var zipStream = client.GetStreamAsync(mDataSourceDescription.source.location).Result;
67+
68+
Console.WriteLine("Opening the Zip file...");
69+
var zipArchive = new ZipArchive(zipStream, ZipArchiveMode.Read);
70+
ExtractShapeFilesFromZip(zipArchive, shpStream, dbfStream);
71+
}
5072

51-
Console.WriteLine($"Grabbing Webfile {mDataSourceDescription.source.location}");
52-
var client = new HttpClient();
53-
var zipStream = client.GetStreamAsync(mDataSourceDescription.source.location).Result;
73+
shpStream.Position = 0;
74+
dbfStream.Position = 0;
5475

55-
Console.WriteLine("opening Zip file");
56-
var zipArchive = new ZipArchive(zipStream, ZipArchiveMode.Read);
76+
mParser = Shapefile.CreateDataReader(
77+
new ShapefileStreamProviderRegistry(
78+
new ByteStreamProvider(StreamTypes.Shape, shpStream),
79+
new ByteStreamProvider(StreamTypes.Data, dbfStream),
80+
true,
81+
true),
82+
GeometryFactory.Default);
5783

58-
var shpStream = new MemoryStream();
59-
var dbfStream = new MemoryStream();
84+
mHeader = mParser.DbaseHeader;
85+
}
86+
87+
private void ExtractShapeFilesFromZip(ZipArchive zipArchive, MemoryStream shpStream, MemoryStream dbfStream)
88+
{
6089
foreach (var zipArchiveEntry in zipArchive.Entries)
6190
{
6291
if (zipArchiveEntry.Name.EndsWith(".shp"))
@@ -73,34 +102,12 @@ private void SetupParser()
73102
}
74103
}
75104

76-
if (shpStream == null || dbfStream == null)
105+
if (shpStream.Length == 0 || dbfStream.Length == 0)
77106
{
78-
throw new FileNotFoundException("the required .shp and .bdf files could not be found.");
107+
throw new FileNotFoundException("The required .shp and .dbf files could not be found.");
79108
}
80-
81-
shpStream.Position = 0;
82-
dbfStream.Position = 0;
83-
Console.WriteLine("File loaded.");
84-
85-
mParser =
86-
Shapefile.CreateDataReader(
87-
new ShapefileStreamProviderRegistry(
88-
new ByteStreamProvider(
89-
StreamTypes.Shape,
90-
shpStream),
91-
new ByteStreamProvider(
92-
StreamTypes.Data,
93-
dbfStream),
94-
true,
95-
true),
96-
GeometryFactory.Default);
97-
98-
// parser = new ShapefileDataReader(@"D:\datasets\093_Oberpfalz_Hausumringe\hausumringe",
99-
// NetTopologySuite.Geometries.GeometryFactory.Default);
100-
mHeader = mParser.DbaseHeader;
101109
}
102110

103-
104111
/// <summary>
105112
/// reads the next line from the Shapefile and returns it as WKT (Well known text)
106113
/// </summary>
@@ -111,7 +118,7 @@ public bool ReadLine(out string nextLine)
111118
nextLine = "";
112119
if (!mParser.Read())
113120
{
114-
Console.WriteLine("EOF");
121+
Console.WriteLine("Reached EOF, finishing.");
115122
return false;
116123
}
117124

@@ -163,4 +170,4 @@ private Geometry ConvertUtmToLatLong(Geometry polygon)
163170
return polygon;
164171
}
165172
}
166-
}
173+
}

‎backend/src/BIE.DataPipeline/Metadata/MetadataDbHelper.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,14 @@ public bool CreateConnection()
2727
{
2828
try
2929
{
30-
var client = new MongoClient($"mongodb://datapipeline:datapipeline@{mMetaDataDbUrl}/bci-metadata");
30+
var connectionString = $"mongodb://datapipeline:datapipeline@{mMetaDataDbUrl}/bci-metadata";
31+
var client = new MongoClient(connectionString);
3132
mDatabase = client.GetDatabase("bci-metadata");
3233
return true;
3334
}
3435
catch (Exception e)
3536
{
36-
Console.WriteLine("Could not establish Connection to Metadata Database");
37+
Console.WriteLine($"Could not establish the connection to Metadata Database.");
3738
return false;
3839
}
3940
}

‎backend/src/BIE.DataPipeline/Metadata/MetadataObject.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,16 @@ public class MetaData
3131
{
3232
public string Icon { get; set; }
3333
public string Type { get; set; }
34+
35+
/// <summary>
36+
/// Zoom level is higher the closer you look at something. If current zoom level is below this, it shouldn't display any value.
37+
/// </summary>
3438
public int MinZoomLevel { get; set; }
39+
40+
/// <summary>
41+
/// The display property is the property that should be shown in a popup.
42+
/// </summary>
43+
public string DisplayProperty { get; set; }
3544
}
3645

3746
public class TableData

‎backend/src/BIE.DataPipeline/Program.cs‎

Lines changed: 20 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -3,43 +3,40 @@
33
using BIE.DataPipeline.Metadata;
44
using Mono.Options;
55

6-
// setup command line options.
6+
// Setup the command line options
77
var tableInsertBehaviour = InsertBehaviour.none;
8-
98
var filename = HandleCliArguments();
109

11-
Console.WriteLine("Starting datapipeline");
12-
1310
var description = GetDataSourceDescription(filename);
1411
if (description == null)
1512
{
1613
return 1;
1714
}
1815

19-
Console.WriteLine($@"Running with {filename}:
16+
Console.WriteLine($@"Starting the Data Pipeline for {filename}:
2017
type: {description.source.type}
2118
format: {description.source.data_format}
2219
location: {description.source.location}
2320
table name: {description.table_name}
24-
2521
");
2622

23+
// Check if the table insert behaviour is overwritten
2724
if (tableInsertBehaviour != InsertBehaviour.none)
2825
{
2926
description.options.if_table_exists = tableInsertBehaviour;
30-
Console.WriteLine($"Overwriting Description: Using {tableInsertBehaviour} Behaviour for insertion.");
27+
Console.WriteLine($"Overwriting description: Using {tableInsertBehaviour} Behaviour for insertion.");
3128
}
3229

33-
// create connection to database;
30+
// Create the connection to the database
3431
var dbHelper = new DbHelper();
3532

36-
// End if Connection not possible.
33+
// Exit when the connection is not possible
3734
if (!dbHelper.CheckConnection())
3835
{
39-
Console.WriteLine("Could not establish Database Connection, exiting...");
36+
Console.WriteLine("Could not establish database connection, exiting...");
4037
return 1;
4138
}
42-
Console.WriteLine("Established Database Connection.");
39+
Console.WriteLine("Established the database connection.");
4340

4441
// End if Dataset can be skipped
4542
if (DbHelper.CanSkip(description))
@@ -55,8 +52,9 @@
5552
return 1;
5653
}
5754

58-
Console.WriteLine("Starting Importer");
55+
Console.WriteLine("Starting the importer...");
5956

57+
// Import the data based on the specified data type
6058
IImporter importer;
6159
try
6260
{
@@ -80,7 +78,7 @@
8078
}
8179
catch (Exception e)
8280
{
83-
Console.WriteLine("Error While setting up Importer.");
81+
Console.WriteLine("Error while setting up the importer.");
8482
Console.WriteLine(e);
8583
return 1;
8684
}
@@ -94,34 +92,32 @@
9492
{
9593
var line = "";
9694
var notEof = importer.ReadLine(out line);
95+
Console.WriteLine("Inserting the data into the database...");
9796

98-
Console.WriteLine("Inserting into Database");
99-
97+
// Read all lines
10098
var count = 0;
10199
while (notEof)
102100
{
103101
dbHelper.InsertData(line);
104102
notEof = importer.ReadLine(out line);
105103
count++;
106-
Console.Write($"\rLines: {count}");
107104
}
108-
109-
Console.WriteLine();
110-
Console.WriteLine("Finished Inserting");
105+
Console.WriteLine($"Finished inserting {count} lines of data.");
111106
if (description.source.data_format == "SHAPE")
112107
{
113-
Console.WriteLine("Creating Indexes");
108+
Console.WriteLine("Creating indexes...");
114109
dbHelper.CreateIndexes(description);
115-
Console.WriteLine("Indexes Created");
110+
Console.WriteLine("Indexes created.");
116111
}
117112

118-
Console.WriteLine("Updating Metadata");
113+
Console.WriteLine("Updating the metadata...");
119114
metadataDbHelper.UpdateMetadata(description, count);
120-
Console.WriteLine("Metadata updated");
115+
Console.WriteLine("The metadata was updated.");
116+
Console.WriteLine("--------------------------------------------------------------");
121117
}
122118
catch (Exception e)
123119
{
124-
Console.WriteLine("Error inserting into Database:");
120+
Console.WriteLine("Error inserting the data into the database:");
125121
Console.WriteLine(e);
126122
return 1;
127123
}

0 commit comments

Comments
 (0)