Unittests and code cleanup (#293)

Corgam · web-flow · commit 81c719e4095d · 2024-06-18T18:16:53.000+02:00
diff --git a/backend/src/BIE.DataPipeline/BIE.DataPipeline.csproj b/backend/src/BIE.DataPipeline/BIE.DataPipeline.csproj
@@ -43,6 +43,7 @@
 
   <ItemGroup>
     <None Remove="yaml\unitTest.yaml" />
+    <None Remove="yaml\unitTestShort.yaml" />
   </ItemGroup>
 
 </Project>
diff --git a/backend/src/BIE.DataPipeline/Import/CsvImporter.cs b/backend/src/BIE.DataPipeline/Import/CsvImporter.cs
@@ -3,12 +3,14 @@
 using System.Linq;
 using System.Net;
 using System.Reflection.PortableExecutable;
+using System.Runtime.CompilerServices;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using BIE.DataPipeline;
 using Microsoft.VisualBasic.FileIO;
 
+[assembly: InternalsVisibleTo("BIE.Tests")]
 namespace BIE.DataPipeline.Import
 {
     internal class CsvImporter : IImporter
@@ -58,14 +60,22 @@ public CsvImporter(DataSourceDescription? dataSourceDescription)
             }
         }
 
-        //tablename = name
+        /// <summary>
+        /// Returns the SQL table name for the data set given by the yaml file.
+        /// </summary>
+        /// <returns>The SQL table name.</returns>
         public string GetTableName()
         {
             return this.dataSourceDescription.table_name;
         }
 
+        /// <summary>
+        /// Creates a comma seperated sting with all column names for the SQL table.
+        /// </summary>
+        /// <returns>The SQL column name string.</returns>
         public string GetHeaderString()
         {
+            //if the string is not empty the result can be returned instantly
             if (!headerString.Equals(""))
             {
                 return headerString;
@@ -82,6 +92,11 @@ public string GetHeaderString()
             return headerString;
         }
 
+        /// <summary>
+        /// Reads a line from the csv file.
+        /// </summary>
+        /// <param name="nextLine">An output parameter that returns a line or an empty string.</param>
+        /// <returns>A boolean indicating if the end of the file has been reached.</returns>
         public bool ReadLine(out string nextLine)
         {
             string[]? line;
@@ -90,13 +105,6 @@ public bool ReadLine(out string nextLine)
 
             while (builder.Length == 0)
             {
-                // Console.Write($"trying to write");
-
-                // if (parser.EndOfData)
-                // {
-                //     return false;
-                // }
-
                 line = parser.ReadFields();
                 if (line == null)
                 {
@@ -106,7 +114,6 @@ public bool ReadLine(out string nextLine)
 
                 if (line.Length == 0)
                 {
-                    //TODO what to do with empty lines
                     Console.WriteLine("Line is empty");
                     //Read next line
                     nextLine = "";
@@ -116,12 +123,21 @@ public bool ReadLine(out string nextLine)
 
                 foreach (var (i, yamlIndex)in columnIndexes)
                 {
+                    //checks if the line has not enougth content for the expected yaml columns.
+                    if(i >= line.Length)
+                    {
+                        Console.WriteLine("Line does not match the number of expected columns");
+                        //Read next line
+                        builder.Clear();
+                        break;
+                    }
+
                     //check if the value can be empty
                     if (dataSourceDescription.table_cols[yamlIndex].is_not_nullable && line[i] == "")
                     {
                         Console.WriteLine("Line does not match not null criteria");
                         //Read next line
-                        nextLine = "";
+                        builder.Clear();
                         break;
                     }
 
@@ -130,20 +146,16 @@ public bool ReadLine(out string nextLine)
 
                     if (columnTypes[i] == typeof(string))
                     {
-                        // nextLine += $"'{line[i]}',";
                         builder.Append($"'{line[i]}',");
                         continue;
                     }
 
-                    // nextLine += string.Format("{0},", Convert.ChangeType(line[i], columnTypes[i]));
-                    // nextLine += $"{line[i]},";
                     builder.Append($"{line[i]},");
                 }
             }
 
             builder.Length--; // this removes the last comma
 
-            // nextLine = RemoveLastComma(nextLine);
             nextLine = builder.ToString();
 
             return true;
diff --git a/backend/src/BIE.DataPipeline/Import/YamlImporter.cs b/backend/src/BIE.DataPipeline/Import/YamlImporter.cs
@@ -1,9 +1,11 @@
 ﻿using System.ComponentModel;
+using System.Runtime.CompilerServices;
 using System.Text;
 using YamlDotNet.Serialization;
 
 // ReSharper disable InconsistentNaming
 
+[assembly: InternalsVisibleTo("BIE.Tests")]
 namespace BIE.DataPipeline.Import
 {
     public static class YamlImporter
@@ -73,6 +75,22 @@ public char delimiter
         /// </summary>
         public List<DataSourceColumn> table_cols { get; set; }
 
+        public override bool Equals(object? obj)
+        {
+            if (obj == null) return false;
+
+            DataSourceDescription des = obj as DataSourceDescription;
+
+            if (!this.source.Equals(des.source)) return false;
+            if (!this.options.Equals(des.options)) return false;
+            if (!this.table_name.Equals(des.table_name)) return false;
+            for (int i = 0; i < this.table_cols.Count; i++)
+            {
+                if (!this.table_cols[i].Equals(des.table_cols[i])) return false;
+            }
+            return true;
+        }
+
         public class DataSourceLocation
         {
             /// <summary>
@@ -89,6 +107,18 @@ public class DataSourceLocation
             /// the format of the data
             /// </summary>
             public string data_format { get; set; }
+
+            public override bool Equals(object? obj)
+            {
+                if (obj == null) return false;
+
+                DataSourceLocation des = obj as DataSourceLocation;
+
+                if (this.type != des.type) return false;
+                if (this.location != des.location) return false;
+                if (this.data_format != des.data_format) return false;
+                return true;
+            }
         }
 
         public class DataSourceOptions
@@ -116,6 +146,18 @@ public InsertBehaviour if_table_exists
                 get => mIf_table_exists;
                 set => mIf_table_exists = value;
             }
+
+            public override bool Equals(object? obj)
+            {
+                if (obj == null) return false;
+
+                DataSourceOptions des = obj as DataSourceOptions;
+
+                if (this.skip_lines != des.skip_lines) return false;
+                if (this.discard_null_rows != des.discard_null_rows) return false;
+                if (this.if_table_exists != des.if_table_exists) return false;
+                return true;
+            }
         }
 
         public class DataSourceColumn
@@ -145,6 +187,20 @@ public string type
             /// </summary>
             [DefaultValue(false)]
             public bool is_not_nullable { get; set; }
+
+
+            public override bool Equals(object? obj)
+            {
+                if (obj == null) return false;
+
+                DataSourceColumn des = obj as DataSourceColumn;
+
+                if (this.name != des.name) return false;
+                if (this.name_in_table != des.name_in_table) return false;
+                if (this.type != des.type) return false;
+                if (this.is_not_nullable != des.is_not_nullable) return false;
+                return true;
+            }
         }
     }
 
diff --git a/backend/src/BIE.DataPipeline/entrypoint.sh b/backend/src/BIE.DataPipeline/entrypoint.sh
@@ -6,7 +6,7 @@ echo "insert data sets"
 
 directory="./yaml"
 yamlfiles=$(find "$directory" -type f -name "*.yaml")
-excludeFiles=("unitTest.yaml" "example.yaml")
+excludeFiles=("unitTest.yaml" "unitTestShort.yaml")
 
 # Check if yaml_files is not empty
 if [ -n "$yamlfiles" ]; then
diff --git a/backend/src/BIE.DataPipeline/yaml/unitTest.yaml b/backend/src/BIE.DataPipeline/yaml/unitTest.yaml
@@ -1,24 +1,24 @@
 # describe the source
 source:
   # link | filepath
-  type: URL
-  location: https://data.bundesnetzagentur.de/Bundesnetzagentur/SharedDocs/Downloads/DE/Sachgebiete/Energie/Unternehmen_Institutionen/E_Mobilitaet/Ladesaeulenregister.csv
+  type: filepath
+  location: .\..\..\..\testData\unitTest.csv
   data_format: CSV
 options:
   # skip lines at the beginning
-  skip_lines: 10
+  skip_lines: 7
   # discard any rows that have null values
   discard_null_rows: false
   # how to deal with existing table. Options: ignore, replace, skip (default).
   if_table_exists: replace
-table_name: EV_charging_stations
+table_name: unitTestTable
 delimiter: ";"
 
 table_cols:
-  - name: testDefault
+  - name: header1
     name_in_table: testDefault
     is_not_nullable: true
-  - name: testVarChar
+  - name: header2
     name_in_table: testVarChar
     type: VARCHAR
   - name: testBool
diff --git a/backend/src/BIE.DataPipeline/yaml/unitTestShort.yaml b/backend/src/BIE.DataPipeline/yaml/unitTestShort.yaml
@@ -0,0 +1,22 @@
+# describe the source
+source:
+  # link | filepath
+  type: filepath
+  location: .\..\..\..\testData\unitTest.csv
+  data_format: CSV
+options:
+  # skip lines at the beginning
+  skip_lines: 7
+  # discard any rows that have null values
+  discard_null_rows: false
+  # how to deal with existing table. Options: ignore, replace, skip (default).
+  if_table_exists: replace
+table_name: unitTestTable
+delimiter: ";"
+
+table_cols:
+  - name: colName
+    name_in_table: colTableName
+    type: VARCHAR
+  
+  
diff --git a/backend/src/BIE.Tests/CsvImporterTest.cs b/backend/src/BIE.Tests/CsvImporterTest.cs
@@ -0,0 +1,53 @@
+﻿using BIE.DataPipeline.Import;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BIE.Tests
+{
+    internal class CsvImporterTest
+    {
+        private CsvImporter csvImporter;
+
+        [SetUp]
+        public void SetUpCsvImporter()
+        {
+            csvImporter = new CsvImporter(YamlImporter.GetSourceDescription(@".\yaml\unitTest.yaml"));
+        }
+
+        [Test]
+        public void TestGetTableName()
+        {
+            Assert.That(csvImporter.GetTableName(), Is.EqualTo("unitTestTable"));
+        }
+
+        [Test]
+        public void TestGetHeaderString()
+        {
+            string expected = "testDefault,testVarChar,checkValue,testBoolean,testInt,testInteger,testFloat,testDouble,testDecimal";
+            string actual = csvImporter.GetHeaderString();
+            Assert.That(actual, Is.EqualTo(expected));
+        }
+
+        [Test]
+        public void TestReadLine()
+        {
+            string[] expected =
+            {
+                "'name1','1',1.5,true,,,,,",
+                "'name3','3',3.5,true,,,,,",
+                "'name4','4',4.5,false,,,,,",
+                "'name5','5',5.5,false,,,,,",
+                string.Empty,
+            };
+
+            for (int i = 0; i < 5; i++) {
+                string actual = "";
+                csvImporter.ReadLine(out actual);
+                Assert.That(actual, Is.EqualTo(expected[i]));
+            }
+        }
+    }
+}
diff --git a/backend/src/BIE.Tests/ImporterHelperTest.cs b/backend/src/BIE.Tests/ImporterHelperTest.cs
@@ -65,8 +65,8 @@ public void TestRemoveLastBrackets()
         public void TestReadYamlHeader()
         {
             string[] expected = {
-            "testDefault",
-            "testVarChar",
+            "header1",
+            "header2",
             "testBool",
             "testBoolean",
             "testInt",
diff --git a/backend/src/BIE.Tests/YamlImporterTest.cs b/backend/src/BIE.Tests/YamlImporterTest.cs
@@ -0,0 +1,44 @@
+﻿using BIE.DataPipeline.Import;
+using MySqlX.XDevAPI.Common;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using static BIE.DataPipeline.Import.DataSourceDescription;
+
+namespace BIE.Tests
+{
+    internal class YamlImporterTest
+    {
+        [Test]
+        public void TestGetSourceDescription()
+        {
+            DataSourceDescription expected = new DataSourceDescription();
+            DataSourceLocation expectedLocation = new DataSourceLocation();
+            expectedLocation.type = "filepath";
+            expectedLocation.data_format = "CSV";
+            expectedLocation.location = ".\\..\\..\\..\\testData\\unitTest.csv";
+            expected.source = expectedLocation;
+            expected.delimiter = ';';
+            DataSourceOptions expectedOptions = new DataSourceOptions();
+            expectedOptions.discard_null_rows = false;
+            expectedOptions.skip_lines = 7;
+            expectedOptions.if_table_exists = InsertBehaviour.replace;
+            expected.options = expectedOptions;
+            DataSourceColumn expectedColumn = new DataSourceColumn();
+            expectedColumn.is_not_nullable = false;
+            expectedColumn.name = "colName";
+            expectedColumn.name_in_table = "colTableName";
+            expectedColumn.type = "VARCHAR";
+            expected.table_cols = new List<DataSourceColumn>
+            {
+                expectedColumn,
+            };
+            expected.table_name = "unitTestTable";
+
+            DataSourceDescription actual = YamlImporter.GetSourceDescription(@".\yaml\unitTestShort.yaml");
+            Assert.True(actual.Equals(expected));
+        }
+    }
+}
diff --git a/backend/src/BIE.Tests/testData/unitTest.csv b/backend/src/BIE.Tests/testData/unitTest.csv

Original file line number	Diff line number	Diff line change
`@@ -3,12 +3,14 @@`
`3`	`3`	`using System.Linq;`
`4`	`4`	`using System.Net;`
`5`	`5`	`using System.Reflection.PortableExecutable;`
	`6`	`+using System.Runtime.CompilerServices;`
`6`	`7`	`using System.Text;`
`7`	`8`	`using System.Text.RegularExpressions;`
`8`	`9`	`using System.Threading.Tasks;`
`9`	`10`	`using BIE.DataPipeline;`
`10`	`11`	`using Microsoft.VisualBasic.FileIO;`
`11`	`12`
	`13`	`+[assembly: InternalsVisibleTo("BIE.Tests")]`
`12`	`14`	`namespace BIE.DataPipeline.Import`
`13`	`15`	`{`
`14`	`16`	`internal class CsvImporter : IImporter`
`@@ -58,14 +60,22 @@ public CsvImporter(DataSourceDescription? dataSourceDescription)`
`58`	`60`	`}`
`59`	`61`	`}`
`60`	`62`
`61`		`- //tablename = name`
	`63`	`+ /// <summary>`
	`64`	`+ /// Returns the SQL table name for the data set given by the yaml file.`
	`65`	`+ /// </summary>`
	`66`	`+ /// <returns>The SQL table name.</returns>`
`62`	`67`	`public string GetTableName()`
`63`	`68`	`{`
`64`	`69`	`return this.dataSourceDescription.table_name;`
`65`	`70`	`}`
`66`	`71`
	`72`	`+ /// <summary>`
	`73`	`+ /// Creates a comma seperated sting with all column names for the SQL table.`
	`74`	`+ /// </summary>`
	`75`	`+ /// <returns>The SQL column name string.</returns>`
`67`	`76`	`public string GetHeaderString()`
`68`	`77`	`{`
	`78`	`+ //if the string is not empty the result can be returned instantly`
`69`	`79`	`if (!headerString.Equals(""))`
`70`	`80`	`{`
`71`	`81`	`return headerString;`
`@@ -82,6 +92,11 @@ public string GetHeaderString()`
`82`	`92`	`return headerString;`
`83`	`93`	`}`
`84`	`94`
	`95`	`+ /// <summary>`
	`96`	`+ /// Reads a line from the csv file.`
	`97`	`+ /// </summary>`
	`98`	`+ /// <param name="nextLine">An output parameter that returns a line or an empty string.</param>`
	`99`	`+ /// <returns>A boolean indicating if the end of the file has been reached.</returns>`
`85`	`100`	`public bool ReadLine(out string nextLine)`
`86`	`101`	`{`
`87`	`102`	`string[]? line;`
`@@ -90,13 +105,6 @@ public bool ReadLine(out string nextLine)`
`90`	`105`
`91`	`106`	`while (builder.Length == 0)`
`92`	`107`	`{`
`93`		`- // Console.Write($"trying to write");`
`94`		`-`
`95`		`- // if (parser.EndOfData)`
`96`		`- // {`
`97`		`- // return false;`
`98`		`- // }`
`99`		`-`
`100`	`108`	`line = parser.ReadFields();`
`101`	`109`	`if (line == null)`
`102`	`110`	`{`
`@@ -106,7 +114,6 @@ public bool ReadLine(out string nextLine)`
`106`	`114`
`107`	`115`	`if (line.Length == 0)`
`108`	`116`	`{`
`109`		`- //TODO what to do with empty lines`
`110`	`117`	`Console.WriteLine("Line is empty");`
`111`	`118`	`//Read next line`
`112`	`119`	`nextLine = "";`
`@@ -116,12 +123,21 @@ public bool ReadLine(out string nextLine)`
`116`	`123`
`117`	`124`	`foreach (var (i, yamlIndex)in columnIndexes)`
`118`	`125`	`{`
	`126`	`+ //checks if the line has not enougth content for the expected yaml columns.`
	`127`	`+ if(i >= line.Length)`
	`128`	`+ {`
	`129`	`+ Console.WriteLine("Line does not match the number of expected columns");`
	`130`	`+ //Read next line`
	`131`	`+ builder.Clear();`
	`132`	`+ break;`
	`133`	`+ }`
	`134`	`+`
`119`	`135`	`//check if the value can be empty`
`120`	`136`	`if (dataSourceDescription.table_cols[yamlIndex].is_not_nullable && line[i] == "")`
`121`	`137`	`{`
`122`	`138`	`Console.WriteLine("Line does not match not null criteria");`
`123`	`139`	`//Read next line`
`124`		`- nextLine = "";`
	`140`	`+ builder.Clear();`
`125`	`141`	`break;`
`126`	`142`	`}`
`127`	`143`
`@@ -130,20 +146,16 @@ public bool ReadLine(out string nextLine)`
`130`	`146`
`131`	`147`	`if (columnTypes[i] == typeof(string))`
`132`	`148`	`{`
`133`		`- // nextLine += $"'{line[i]}',";`
`134`	`149`	`builder.Append($"'{line[i]}',");`
`135`	`150`	`continue;`
`136`	`151`	`}`
`137`	`152`
`138`		`- // nextLine += string.Format("{0},", Convert.ChangeType(line[i], columnTypes[i]));`
`139`		`- // nextLine += $"{line[i]},";`
`140`	`153`	`builder.Append($"{line[i]},");`
`141`	`154`	`}`
`142`	`155`	`}`
`143`	`156`
`144`	`157`	`builder.Length--; // this removes the last comma`
`145`	`158`
`146`		`- // nextLine = RemoveLastComma(nextLine);`
`147`	`159`	`nextLine = builder.ToString();`
`148`	`160`
`149`	`161`	`return true;`