Metadata Improvements + bug fixes (#320)

Author: Corgam

backend/api-gateway/Controllers/APIGatewayController.cs

@@ -33,7 +33,7 @@ public APIGatewayController(HttpClient httpClient, ILogger<APIGatewayController>
         /// </summary>
         /// <returns>A list of available datasets</returns>
         [HttpGet("getDatasetList")]
-        [ProducesResponseType(typeof(List<DatasetBasicData>), 200)]
+        [ProducesResponseType(typeof(List<MetadataObject.BasicData>), 200)]
         [ProducesResponseType(400)]
         public async Task<IActionResult> GetDataSetList()
         {
@@ -44,7 +44,7 @@ public async Task<IActionResult> GetDataSetList()
 
             var collection = _mongoDBService.GetDatasetsCollection();
             var datasets = await collection.Find(_ => true).ToListAsync();
-            var datasetBasicDataList = datasets.Select(d => d.BasicData).ToList();
+            var datasetBasicDataList = datasets.Select(d => d.basicData).ToList();
 
             return Ok(datasetBasicDataList);
         }
@@ -55,7 +55,7 @@ public async Task<IActionResult> GetDataSetList()
         /// <param name="datasetID">The ID of the dataset</param>
         /// <returns>Metadata for the specified dataset</returns>
         [HttpGet("getDatasetMetadata")]
-        [ProducesResponseType(typeof(DatasetMetadata), 200)]
+        [ProducesResponseType(typeof(MetadataObject), 200)]
         [ProducesResponseType(400)]
         [ProducesResponseType(404)]
         public async Task<IActionResult> GetDatasetMetadata([FromQuery, Required] string datasetID)
@@ -66,14 +66,14 @@ public async Task<IActionResult> GetDatasetMetadata([FromQuery, Required] string
             }
 
             var collection = _mongoDBService.GetDatasetsCollection();
-            var dataset = await collection.Find(d => d.BasicData.DatasetId == datasetID).FirstOrDefaultAsync();
+            var dataset = await collection.Find(d => d.basicData.DatasetId == datasetID).FirstOrDefaultAsync();
 
             if (dataset == null)
             {
                 return NotFound($"Dataset with ID {datasetID} not found.");
             }
 
-            return Ok(dataset.MetaData);
+            return Ok(dataset.additionalData);
         }
 
         /// <summary>

backend/api-gateway/Models/DatasetBasicData.cs

@@ -5,25 +5,25 @@ namespace APIGateway.Models
 
     public class GeoJsonResponse
     {
-        public string type { get; set; }
-        public List<Feature> features { get; set; }
+        public string type { get; set; }  = string.Empty;
+        public List<Feature> features { get; set; } = new List<Feature>();
     }
 
     public class Feature
     {
-        public string type { get; set; }
-        public Geometry geometry { get; set; }
-        public Properties properties { get; set; }
+        public string type { get; set; } = string.Empty;
+        public Geometry geometry { get; set; } = new Geometry();
+        public Properties properties { get; set; } = new Properties();
     }
 
     public class Geometry
     {
-        public string type { get; set; }
-        public List<float> coordinates { get; set; }
+        public string type { get; set; } = string.Empty;
+        public List<float> coordinates { get; set; } = new List<float>();
     }
 
     public class Properties
     {
-        public string name { get; set; }
+        public string name { get; set; } = string.Empty;
     }
 }

backend/api-gateway/Models/DatasetData.cs

@@ -2,29 +2,29 @@
 {
     public class LocationDataRequest
     {
-        public string DatasetId { get; set; }
-        public List<Coordinate> Location { get; set; }
+        public string DatasetId { get; set; } = string.Empty;
+        public List<Coordinate> Location { get; set; } = new List<Coordinate>();
     }
 
     public class Coordinate
     {
-        public double Latitude { get; set; }
-        public double Longitude { get; set; }
+        public double Latitude { get; set; } = 0;
+        public double Longitude { get; set; } = 0;
     }
 
     public class LocationDataResponse
     {
-        public List<DatasetItem> CurrentDatasetData { get; set; }
-        public List<DatasetItem> GeneralData { get; set; }
+        public List<DatasetItem> CurrentDatasetData { get; set; } = new List<DatasetItem>();
+        public List<DatasetItem> GeneralData { get; set; } = new List<DatasetItem>();
     }
 
     public class DatasetItem
     {
-        public string Id { get; set; }
+        public string Id { get; set; } = string.Empty;
 
-        public string Key { get; set; }
-        public string Value { get; set; }
-        public string MapId { get; set; } // Optional  -> for "open in map" functionality
+        public string Key { get; set; } = string.Empty;
+        public string Value { get; set; } = string.Empty;
+        public string MapId { get; set; } = string.Empty; // Optional  -> for "open in map" functionality
     }
 
 

backend/api-gateway/Models/DatasetMetadata.cs

@@ -0,0 +1,55 @@
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace APIGateway.Models;
+
+public class MetadataObject
+{
+    [BsonRepresentation(BsonType.ObjectId)]
+    public string _id { get; set; } = string.Empty;
+    
+    [BsonElement("basicData")]
+    public BasicData basicData { get; set; } = new BasicData();
+
+    [BsonElement("additionalData")]
+    public AdditionalData additionalData { get; set; } = new AdditionalData();
+    
+    // The general and most important data about a dataset.
+    public class BasicData
+    {
+        public string DatasetId { get; set; } = string.Empty;
+        public string Name { get; set; } = string.Empty;
+        public string ShortDescription { get; set; } = string.Empty;
+        public string Icon { get; set; } = string.Empty;
+    }
+
+    // The additional data for each of the datasets, queried at a request.
+    public class AdditionalData
+    {
+        public string Icon { get; set; } = string.Empty;
+        public string Type { get; set; } = string.Empty;
+
+        public string LongDescription { get; set; } = string.Empty;
+
+        // Zoom level is higher the closer you look at something. If current zoom level is below this, it shouldn't display any value.
+        public int MinZoomLevel { get; set; } = 0;
+        
+        // The zoom threshold where areas start to turn into markers
+        public int MarkersThreshold { get; set; } = 0;
+
+        // The display property is the property that should be shown in a popup.
+        public string DisplayProperty { get; set; } = string.Empty;
+
+        // Table data populated by the data pipeline. Contains the name and the size of the all .yaml files correlated to that specific dataset.
+        public List<TableData> Tables { get; set; } = new List<TableData>();
+    }
+    
+    // Table data populated by the data pipeline. Contains the name and the size of the all .yaml files correlated to that specific dataset.
+    public class TableData
+    {
+        // The name of the .yaml file
+        public string Name { get; set; } = string.Empty;
+        // The number of lines of data in that file.
+        public int NumberOfLines { get; set; } = 0;
+    }
+}

backend/api-gateway/Models/GeoJsonResponse.cs

@@ -14,9 +14,9 @@ public MongoDBService(IConfiguration configuration)
             _database = client.GetDatabase("bci-metadata");
         }
 
-        public IMongoCollection<DatasetData> GetDatasetsCollection()
+        public IMongoCollection<MetadataObject> GetDatasetsCollection()
         {
-            return _database.GetCollection<DatasetData>("datasets");
+            return _database.GetCollection<MetadataObject>("datasets");
         }
     }
 }

backend/api-gateway/Models/LocationData.cs

@@ -11,72 +11,85 @@ db.createUser({
 db.createUser({
   user: "datapipeline",
   pwd: "datapipeline",
-  roles: [{role: "readWrite", db: "bci-metadata"}] 
-})
+  roles: [{ role: "readWrite", db: "bci-metadata" }],
+});
 
 // Define the datasets
 const datasets = [
   {
     basicData: {
       DatasetId: "empty_map",
       Name: "Empty Map",
-      Description: "An empty, default map of Germany, with no data loaded.",
-      Tables: [],
+      ShortDescription: `An empty, default map of Germany, with no data loaded.`,
     },
-    metaData: {
-      Type: "marker",
+    additionalData: {
+      Type: "none",
+      LongDescription: `An empty, default map of Germany, with no data loaded. Useful for exploring the map.`,
       MinZoomLevel: -1,
+      MarkersThreshold: -1,
+      DisplayProperty: "",
+      Tables: [],
     },
   },
   {
     basicData: {
       DatasetId: "EV_charging_stations",
       Name: "Charging stations",
-      Description: "Locations of all charging stations in Germany.",
+      ShortDescription: `Locations of all charging stations in Germany.`,
       Icon: '<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" fill="#000000" viewBox="0 0 256 256"><path d="M134.62,123.51a8,8,0,0,1,.81,7.46l-16,40A8,8,0,0,1,104.57,165l11.61-29H96a8,8,0,0,1-7.43-11l16-40A8,8,0,1,1,119.43,91l-11.61,29H128A8,8,0,0,1,134.62,123.51ZM248,86.63V168a24,24,0,0,1-48,0V128a8,8,0,0,0-8-8H176v88h16a8,8,0,0,1,0,16H32a8,8,0,0,1,0-16H48V56A24,24,0,0,1,72,32h80a24,24,0,0,1,24,24v48h16a24,24,0,0,1,24,24v40a8,8,0,0,0,16,0V86.63A8,8,0,0,0,229.66,81L210.34,61.66a8,8,0,0,1,11.32-11.32L241,69.66A23.85,23.85,0,0,1,248,86.63ZM160,208V56a8,8,0,0,0-8-8H72a8,8,0,0,0-8,8V208Z"></path></svg>',
-      Tables: [],
     },
-    metaData: {
+    additionalData: {
       Icon: '<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" fill="#000000" viewBox="0 0 256 256"><path d="M134.62,123.51a8,8,0,0,1,.81,7.46l-16,40A8,8,0,0,1,104.57,165l11.61-29H96a8,8,0,0,1-7.43-11l16-40A8,8,0,1,1,119.43,91l-11.61,29H128A8,8,0,0,1,134.62,123.51ZM248,86.63V168a24,24,0,0,1-48,0V128a8,8,0,0,0-8-8H176v88h16a8,8,0,0,1,0,16H32a8,8,0,0,1,0-16H48V56A24,24,0,0,1,72,32h80a24,24,0,0,1,24,24v48h16a24,24,0,0,1,24,24v40a8,8,0,0,0,16,0V86.63A8,8,0,0,0,229.66,81L210.34,61.66a8,8,0,0,1,11.32-11.32L241,69.66A23.85,23.85,0,0,1,248,86.63ZM160,208V56a8,8,0,0,0-8-8H72a8,8,0,0,0-8,8V208Z"></path></svg>',
       Type: "marker",
+      LongDescription: `A map of EV charging stations displays the locations of electric vehicle charging points located in Germany, helping drivers plan routes and manage charging needs. It is essential for supporting the adoption and convenience of electric vehicles.`,
       MinZoomLevel: 10,
+      MarkersThreshold: -1,
       DisplayProperty: "name",
+      Tables: [],
     },
   },
   {
     basicData: {
       DatasetId: "house_footprints",
       Name: "House Footprints",
-      Description: "Footprints for the houses.",
+      ShortDescription: `Footprints for the houses.`,
       Icon: '<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" fill="#000000" viewBox="0 0 256 256"><path d="M232,56H72V40a8,8,0,0,0-8-8H48A32,32,0,0,0,16,64V176a32,32,0,0,0,32,32H232a8,8,0,0,0,8-8V64A8,8,0,0,0,232,56ZM32,64A16,16,0,0,1,48,48h8v96H48a31.82,31.82,0,0,0-16,4.29ZM224,192H48a16,16,0,0,1,0-32H64a8,8,0,0,0,8-8V72H224ZM104,136a8,8,0,0,0,0,16h16v8a8,8,0,0,0,16,0v-8h24v8a8,8,0,0,0,16,0v-8h16a8,8,0,0,0,0-16H176V120h16a8,8,0,0,0,0-16H176V96a8,8,0,0,0-16,0v8H136V96a8,8,0,0,0-16,0v8H104a8,8,0,0,0,0,16h16v16Zm32-16h24v16H136Z"></path></svg>',
-      Tables: [],
     },
-    metaData: {
+    additionalData: {
       Icon: '<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" fill="#000000" viewBox="0 0 256 256"><path d="M232,56H72V40a8,8,0,0,0-8-8H48A32,32,0,0,0,16,64V176a32,32,0,0,0,32,32H232a8,8,0,0,0,8-8V64A8,8,0,0,0,232,56ZM32,64A16,16,0,0,1,48,48h8v96H48a31.82,31.82,0,0,0-16,4.29ZM224,192H48a16,16,0,0,1,0-32H64a8,8,0,0,0,8-8V72H224ZM104,136a8,8,0,0,0,0,16h16v8a8,8,0,0,0,16,0v-8h24v8a8,8,0,0,0,16,0v-8h16a8,8,0,0,0,0-16H176V120h16a8,8,0,0,0,0-16H176V96a8,8,0,0,0-16,0v8H136V96a8,8,0,0,0-16,0v8H104a8,8,0,0,0,0,16h16v16Zm32-16h24v16H136Z"></path></svg>',
       Type: "marker",
+      LongDescription: `House footprints refer to the outline or ground area covered by a house, typically measured from the exterior walls of the structure. This footprint includes all parts of the house that are in contact with the ground, and is important for planning and zoning purposes, calculating property taxes, and designing land use.`,
       MinZoomLevel: 10,
+      MarkersThreshold: 10,
+      DisplayProperty: "",
+      Tables: [],
     },
   },
   {
     basicData: {
       DatasetId: "actual_use",
       Name: "Actual Use",
-      Description: "The Actual Use (?)",
-      Icon: '<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" fill="#000000" viewBox="0 0 256 256"><path d="M232,56H72V40a8,8,0,0,0-8-8H48A32,32,0,0,0,16,64V176a32,32,0,0,0,32,32H232a8,8,0,0,0,8-8V64A8,8,0,0,0,232,56ZM32,64A16,16,0,0,1,48,48h8v96H48a31.82,31.82,0,0,0-16,4.29ZM224,192H48a16,16,0,0,1,0-32H64a8,8,0,0,0,8-8V72H224ZM104,136a8,8,0,0,0,0,16h16v8a8,8,0,0,0,16,0v-8h24v8a8,8,0,0,0,16,0v-8h16a8,8,0,0,0,0-16H176V120h16a8,8,0,0,0,0-16H176V96a8,8,0,0,0-16,0v8H136V96a8,8,0,0,0-16,0v8H104a8,8,0,0,0,0,16h16v16Zm32-16h24v16H136Z"></path></svg>',
-      Tables: [],
+      ShortDescription: `The division of the land based on its utilization.`,
+      Icon: '<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" fill="#000000" viewBox="0 0 256 256"><path d="M136.83,220.43a8,8,0,0,1-11.09,2.23A183.15,183.15,0,0,0,24,192a8,8,0,0,1,0-16,199.11,199.11,0,0,1,110.6,33.34A8,8,0,0,1,136.83,220.43ZM24,144a8,8,0,0,0,0,16,214.81,214.81,0,0,1,151.17,61.71,8,8,0,1,0,11.2-11.42A230.69,230.69,0,0,0,24,144Zm208,16a216.51,216.51,0,0,0-48.59,5.49q8.24,6.25,16,13.16A201.53,201.53,0,0,1,232,176a8,8,0,0,1,0,16c-6,0-11.93.29-17.85.86q8.32,8.67,15.94,18.14a8,8,0,1,1-12.48,10A247,247,0,0,0,24,128a8,8,0,0,1,0-16,266.33,266.33,0,0,1,48,4.37V80a8,8,0,0,1,3.2-6.4l64-48a8,8,0,0,1,9.6,0l64,48A8,8,0,0,1,216,80v32.49c5.31-.31,10.64-.49,16-.49a8,8,0,0,1,0,16,246.3,246.3,0,0,0-84.26,14.69q9.44,5,18.46,10.78A232.2,232.2,0,0,1,232,144a8,8,0,0,1,0,16ZM120,88h48a8,8,0,0,1,8,8v21.94q11.88-2.56,24-4V84L144,42,88,84v35.81q12.19,3,24,7.18V96A8,8,0,0,1,120,88Zm8.07,45.27A262.48,262.48,0,0,1,160,121.94V104H128v29.24Z"></path></svg>',
     },
-    metaData: {
-      Icon: '<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" fill="#000000" viewBox="0 0 256 256"><path d="M232,56H72V40a8,8,0,0,0-8-8H48A32,32,0,0,0,16,64V176a32,32,0,0,0,32,32H232a8,8,0,0,0,8-8V64A8,8,0,0,0,232,56ZM32,64A16,16,0,0,1,48,48h8v96H48a31.82,31.82,0,0,0-16,4.29ZM224,192H48a16,16,0,0,1,0-32H64a8,8,0,0,0,8-8V72H224ZM104,136a8,8,0,0,0,0,16h16v8a8,8,0,0,0,16,0v-8h24v8a8,8,0,0,0,16,0v-8h16a8,8,0,0,0,0-16H176V120h16a8,8,0,0,0,0-16H176V96a8,8,0,0,0-16,0v8H136V96a8,8,0,0,0-16,0v8H104a8,8,0,0,0,0,16h16v16Zm32-16h24v16H136Z"></path></svg>',
+    additionalData: {
+      Icon: '<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" fill="#000000" viewBox="0 0 256 256"><path d="M136.83,220.43a8,8,0,0,1-11.09,2.23A183.15,183.15,0,0,0,24,192a8,8,0,0,1,0-16,199.11,199.11,0,0,1,110.6,33.34A8,8,0,0,1,136.83,220.43ZM24,144a8,8,0,0,0,0,16,214.81,214.81,0,0,1,151.17,61.71,8,8,0,1,0,11.2-11.42A230.69,230.69,0,0,0,24,144Zm208,16a216.51,216.51,0,0,0-48.59,5.49q8.24,6.25,16,13.16A201.53,201.53,0,0,1,232,176a8,8,0,0,1,0,16c-6,0-11.93.29-17.85.86q8.32,8.67,15.94,18.14a8,8,0,1,1-12.48,10A247,247,0,0,0,24,128a8,8,0,0,1,0-16,266.33,266.33,0,0,1,48,4.37V80a8,8,0,0,1,3.2-6.4l64-48a8,8,0,0,1,9.6,0l64,48A8,8,0,0,1,216,80v32.49c5.31-.31,10.64-.49,16-.49a8,8,0,0,1,0,16,246.3,246.3,0,0,0-84.26,14.69q9.44,5,18.46,10.78A232.2,232.2,0,0,1,232,144a8,8,0,0,1,0,16ZM120,88h48a8,8,0,0,1,8,8v21.94q11.88-2.56,24-4V84L144,42,88,84v35.81q12.19,3,24,7.18V96A8,8,0,0,1,120,88Zm8.07,45.27A262.48,262.48,0,0,1,160,121.94V104H128v29.24Z"></path></svg>',
       Type: "marker",
+      LongDescription: `The Actual Use map describes the use of the earth's surface in four main groups (settlement, traffic, vegetation and water bodies). The division of these main groups into almost 140 different types of use, such as residential areas, road traffic, agriculture or flowing water, enables detailed evaluations and analyses of the use of the earth's surface.`,
       MinZoomLevel: 10,
+      MarkersThreshold: 10,
+      DisplayProperty: "",
+      Tables: [],
     },
   },
 ];
 
 // Iterate over datasets and insert only if DatasetId does not exist
 datasets.forEach((dataset) => {
   if (
-    !db.datasets.findOne({ "basicData.DatasetId": dataset.basicData.DatasetId })
+    !db.datasets.findOne({
+      "basicData.DatasetId": dataset.basicData.DatasetId,
+    })
   ) {
     db.datasets.insertOne(dataset);
   }