Merge pull request #743 from cambridge-cares/467-example-visualization-mapbox-api-credentials-in-separate-file

mdhillman · web-flow · commit 135586a80adc · 2023-07-12T17:03:03.000+01:00
Example visualization Mapbox API credentials from Docker secret files
diff --git a/.github/scripts/dtvf/check-version.sh b/.github/scripts/dtvf/check-version.sh
@@ -30,18 +30,6 @@ if grep -q "$TOKEN" <<< "$VERSION"; then
 fi
 echo "Version does not contain -SNAPSHOT qualifier."
 
-# Check that the README has been updated
-README="./web/digital-twin-vis-framework/README.md"
-TOKEN="$VERSION"
-
-echo "Looking in README for '$VERSION'"
-if ! grep -q "$TOKEN" "$README"; then
-	echo "Main README does not list the correct version of DTVF!"
-	exit -1
-fi
-echo "Main README has been updated."
-
-
 # Check that the change log contains an entry for that version
 CHANGELOG="./web/digital-twin-vis-framework/CHANGELOG.md"
 TOKEN="# $VERSION"
diff --git a/Deploy/stacks/dynamic/stack-clients/docker-compose.yml b/Deploy/stacks/dynamic/stack-clients/docker-compose.yml
@@ -1,7 +1,7 @@
 version: '3.8'
 services:
   stack-client:
-    image: docker.cmclinnovations.com/stack-client${IMAGE_SUFFIX}:1.12.2
+    image: docker.cmclinnovations.com/stack-client${IMAGE_SUFFIX}:1.13.0
     secrets:
       - blazegraph_password
       - postgis_password
diff --git a/Deploy/stacks/dynamic/stack-clients/pom.xml b/Deploy/stacks/dynamic/stack-clients/pom.xml
@@ -7,7 +7,7 @@
 
   <groupId>com.cmclinnovations</groupId>
   <artifactId>stack-clients</artifactId>
-  <version>1.12.2</version>
+  <version>1.13.0</version>
 
   <name>Stack Clients</name>
   <url>https://www.cmclinnovations.com</url>
diff --git a/Deploy/stacks/dynamic/stack-clients/src/main/java/com/cmclinnovations/stack/services/VisualisationService.java b/Deploy/stacks/dynamic/stack-clients/src/main/java/com/cmclinnovations/stack/services/VisualisationService.java
@@ -0,0 +1,18 @@
+package com.cmclinnovations.stack.services;
+
+import com.cmclinnovations.stack.services.config.ServiceConfig;
+
+public class VisualisationService extends ContainerService {
+
+    public static final String TYPE = "visualisation";
+
+    public VisualisationService(String stackName, ServiceConfig config) {
+        super(stackName, config);
+    }
+
+    @Override
+    public void doPreStartUpConfiguration() {
+        ensureOptionalSecret("mapbox_username");
+        ensureOptionalSecret("mapbox_api_key");
+    }
+}
diff --git a/Deploy/stacks/dynamic/stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/visualisation.json b/Deploy/stacks/dynamic/stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/visualisation.json
@@ -1,4 +1,5 @@
 {
+    "type": "visualisation",
     "ServiceSpec": {
         "Name": "visualisation",
         "TaskTemplate": {
@@ -10,6 +11,20 @@
                         "Source": "vis-files",
                         "Target": "/var/www/html"
                     }
+                ],
+                "Secrets": [
+                    {
+                        "SecretName": "mapbox_username",
+                        "File": {
+                            "Name": "/var/www/html/mapbox_username"
+                        }
+                    },
+                    {
+                        "SecretName": "mapbox_api_key",
+                        "File": {
+                            "Name": "/var/www/html/mapbox_api_key"
+                        }
+                    }
                 ]
             }
         }
diff --git a/Deploy/stacks/dynamic/stack-data-uploader/docker-compose.yml b/Deploy/stacks/dynamic/stack-data-uploader/docker-compose.yml
@@ -1,7 +1,7 @@
 version: '3.8'
 services:
   stack-data-uploader:
-    image: docker.cmclinnovations.com/stack-data-uploader${IMAGE_SUFFIX}:1.12.2
+    image: docker.cmclinnovations.com/stack-data-uploader${IMAGE_SUFFIX}:1.13.0
     secrets:
       - blazegraph_password
       - postgis_password
diff --git a/Deploy/stacks/dynamic/stack-data-uploader/pom.xml b/Deploy/stacks/dynamic/stack-data-uploader/pom.xml
@@ -7,7 +7,7 @@
 
   <groupId>com.cmclinnovations</groupId>
   <artifactId>stack-data-uploader</artifactId>
-  <version>1.12.2</version>
+  <version>1.13.0</version>
 
   <name>Stack Data Uploader</name>
   <url>https://www.cmclinnovations.com</url>
@@ -29,7 +29,7 @@
     <dependency>
       <groupId>com.cmclinnovations</groupId>
       <artifactId>stack-clients</artifactId>
-      <version>1.12.2</version>
+      <version>1.13.0</version>
     </dependency>
 
     <dependency>
diff --git a/Deploy/stacks/dynamic/stack-manager/README.md b/Deploy/stacks/dynamic/stack-manager/README.md
@@ -268,11 +268,12 @@ The format of the stack configuration file is as follows:
 
 ## Example - including a visualisation
 
-This example explains how to spin up a DTVF based visualisation container within a stack. The visualisation container requires a volume called `vis-files` to be populated.
+This example explains how to spin up a DTVF based visualisation container within a stack. The visualisation container requires a volume called `vis-files` to be populated and secrets `mapbox_username`, and `mapbox_api_key` to be created.
 The steps to configure the stack are as follows:
 * Enable the visualisation container by adding it to the `services` `includes` list in the stack config file.
 * Specify the sub-directory of the `stack-manager/inputs/data/` folder from which the custom files that configure the visualisation should be copied, in this example the sub-directory is called `webspace`.
 * Copy the custom files that configure the visualisation in to that directory, in this example `stack-manager/inputs/data/webspace`.
+* Create `mapbox_username` and `mapbox_api_key` files in the `stack-manager/inputs/secrets` and populate with the relevant credentials.
 
 The final stack config file should contain the following content:
 ```json
diff --git a/Deploy/stacks/dynamic/stack-manager/docker-compose.yml b/Deploy/stacks/dynamic/stack-manager/docker-compose.yml
@@ -1,7 +1,7 @@
 version: '3.8'
 services:
   stack-manager:
-    image: docker.cmclinnovations.com/stack-manager${IMAGE_SUFFIX}:1.12.2
+    image: docker.cmclinnovations.com/stack-manager${IMAGE_SUFFIX}:1.13.0
     environment:
       EXTERNAL_PORT: "${EXTERNAL_PORT-3838}"
       STACK_BASE_DIR: "${STACK_BASE_DIR-}"
diff --git a/Deploy/stacks/dynamic/stack-manager/pom.xml b/Deploy/stacks/dynamic/stack-manager/pom.xml
@@ -7,7 +7,7 @@
 
   <groupId>com.cmclinnovations</groupId>
   <artifactId>stack-manager</artifactId>
-  <version>1.12.2</version>
+  <version>1.13.0</version>
 
   <name>Stack Manager</name>
   <url>https://www.cmclinnovations.com</url>
@@ -29,7 +29,7 @@
     <dependency>
       <groupId>com.cmclinnovations</groupId>
       <artifactId>stack-clients</artifactId>
-      <version>1.12.2</version>
+      <version>1.13.0</version>
     </dependency>
 
     <dependency>
diff --git a/web/digital-twin-vis-framework/.gitignore b/web/digital-twin-vis-framework/.gitignore
@@ -1,2 +1,5 @@
 *.log
-library/output
+library/output
+
+mapbox_username
+mapbox_api_key
diff --git a/web/digital-twin-vis-framework/CHANGELOG.md b/web/digital-twin-vis-framework/CHANGELOG.md
@@ -1,9 +1,23 @@
 [//]: # (Note that version headers need to start with "# " characters to be picked up by some automated scripts)
+
+# 3.7.0
+### Features:
+* Mapbox username and token can now be read from external files/docker secrets.
+### Bug Fixes:
+* _No bug fixes present._
+
 # 3.6.0
 ### Features:
 * Location of configuration file can now be passed into DTVF.
 ### Bug Fixes:
 * _No bug fixes present._
+
+# 3.5.1
+### Features:
+* _No features present._
+### Bug Fixes:
+* API credentials are now loaded from separate files/docker secrets.
+  
 # 3.5.0
 ### Features:
 * Added ability to specify client-side styling & filtering options for Cesium 3D tilesets.
diff --git a/web/digital-twin-vis-framework/README.md b/web/digital-twin-vis-framework/README.md
@@ -4,7 +4,7 @@ A central framework for Digital Twin Visualisations (the Digital Twin Visualisat
 
 Please note that the DTVF can only plot data that is provided in the accepted formats (more on this below), restricts some features if the data isn't organised (again, more on this below), and only produces a basic visualisation. Developers that want to display non-standard data, or create more complex map interactions are free to use the DTVF as a base, but will likely need to handle these additions themselves.
 
-The current, working version of the DTVF is **3.6.0**.
+The current, working version of the DTVF is listed in the `VERSION` file.
 
 # Technical Overview
 
@@ -18,7 +18,7 @@ At the time of writing the available mapping providers are [Mapbox](https://www.
 
 To use the DTVF, developers just need to provide some JSON configuration files and a `index.html` file that imports the remote DTVF library (along with other dependencies), adds some required HTML elements, then calls the startup routine. Template HTML files can be found within the visualisation examples (listed below).
 
-On startup, the DTVF reads a number of those JSON configuration files (all expected to be local to the visualisation, i.e. next to the `index.html` file), constructs internal, hierarchal representations of data sources and layers, then calls provider specific classes to handle adding those sources and layers to the map for visualisation.
+On start up, the DTVF reads a number of those JSON configuration files (all expected to be local to the visualisation, i.e. next to the `index.html` file), constructs internal, hierarchal representations of data sources and layers, then calls provider specific classes to handle adding those sources and layers to the map for visualisation.
 
 Display of meta and timeseries data is also a feature offered by the DTVF (regardless of the chosen mapping provider), however this functionality is still under development and requires data to be hosted within a [stack instance](https://github.com/cambridge-cares/TheWorldAvatar/tree/main/Deploy/stacks/dynamic/stack-manager), with a specific agent running, and the geospatial data to meet certain standards. If these conditions are not met, any metadata baked directly into the geospatial data is shown instead; it's recommended to use this approach for now.
 
diff --git a/web/digital-twin-vis-framework/VERSION b/web/digital-twin-vis-framework/VERSION
@@ -1 +1 @@
-3.6.0
+3.7.0
diff --git a/web/digital-twin-vis-framework/example-cesium-vis/README.md b/web/digital-twin-vis-framework/example-cesium-vis/README.md
@@ -2,7 +2,7 @@
 
 This example visualisation has been put together to demonstrate the intended use of the centralised Digital Twin Visualisation Framework (DTVF). This framework has been designed to make it easier for users not experienced with Typescript (or the mapping libraries) to quickly & easily put together a new Digital Twin visualisation. It is intended for developers to use this example visualisation to gain an understanding of the DTVF before attempting to create their own visualisation; to do that, this example can be copied and used as a starting point.
 
-It is recommended that you read the [Digital Twin Visualisations](https://github.com/cambridge-cares/TheWorldAvatar/wiki/Digital-Twin-Visualisations) page of the GitHub wiki before continuing with this document. It's also worth noting that this example uses version 3.6.0 of the DTVF, hosted on a remote CMCL server and not the raw TypeScript files within the library directory.
+It is recommended that you read the [Digital Twin Visualisations](https://github.com/cambridge-cares/TheWorldAvatar/wiki/Digital-Twin-Visualisations) page of the GitHub wiki before continuing with this document. It's also worth noting that this example uses version 3.7.0 of the DTVF, hosted on a remote CMCL server and not the raw TypeScript files within the library directory.
 
 <img src="readme-example.JPG" alt="Example of 3D data on a Cesium JS visualisation" width="100%"/>
 
@@ -56,6 +56,14 @@ In addition to these JSON files, areas of the `index.html` file can also be adju
 
 Please note that the `index.html` file also required users to input their Mapbox API key, this is so that the terrain imagery can be pulled from Mapbox's free API rather than using imagery from Cesium ion (which would require a licence).
 
+### Credentials 
+
+In addition to the aforementioned configuration files, two additional files are required to house a Mapbox username and associated API key. Note these are required, even in Cesium visualisations, as the base map imagery is still provided by Mapbox.
+
+To set these two files, either create and populate `mapbox_username`, and `mapbox_api_key` files within the hosted webspace, or use the stack infrastructure to provide these as Docker secrets. You can learn more about the latter by reading [the stack's documentation](https://github.com/cambridge-cares/TheWorldAvatar/tree/main/Deploy/stacks/dynamic/stack-manager#example---including-a-visualisation).
+
+Once present, these files are queried by the DTVF, loading in the required credentials. Note that previous versions of the DTVF required these parameters to be set within each visualisation's `index.html` file, this is no longer required (see the example visualisations to learn about the new format).
+
 ### Data Specification File
 
 The `data.json` file is a core configuration file for the visualisation and defines what data is loaded and shown, so it's worth explaining its formatting a little. Each node represents a group of data. Each group can contain data sources and layers and/or sub-groups. The hierarchy of these groups is completely up to the writer of the file and is used to build the selection tree within the visualisation. The `name` parameter specifies the group's user-facing name, and the `stack` parameter is the base URL for the stack containing that group's metadata (note that if not using the metadata, this parameter can be any old URL).
diff --git a/web/digital-twin-vis-framework/example-cesium-vis/webspace/index.html b/web/digital-twin-vis-framework/example-cesium-vis/webspace/index.html
@@ -1,7 +1,7 @@
 <html>
     <head>
         <!-- ===== CUSTOMISABLE ===== -->
-        <title>DTVF v3.6.0</title>
+        <title>DTVF</title>
         <!-- ===== CUSTOMISABLE ===== -->
 
         <meta charset="utf-8">
@@ -15,7 +15,7 @@
 		<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/3.5.1/chart.js"></script>
         <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.29.4/moment.min.js" ></script>
 		<script src="https://cdnjs.cloudflare.com/ajax/libs/chartjs-adapter-moment/1.0.0/chartjs-adapter-moment.js"></script>
-        <script src="https://kg.cmclinnovations.com/cdn/dtvf/3.6.0/dtvf.min.js"></script>
+        <script src="https://kg.cmclinnovations.com/cdn/dtvf/3.7.0/dtvf.min.js"></script>
 
         <!-- ===== CUSTOMISABLE ===== -->
         <!-- JavaScript files to provide functionality specifically for this visualisation instance can go here. -->
@@ -26,7 +26,7 @@
         <link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.4/css/all.min.css" rel="stylesheet">
         <link href="https://cdn.jsdelivr.net/gh/hummingbird-dev/hummingbird-treeview@v3.0.4/hummingbird-treeview.min.css" rel="stylesheet">
         <link href="https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/themes/smoothness/jquery-ui.css" rel="stylesheet">
-        <link href="https://kg.cmclinnovations.com/cdn/dtvf/3.6.0/dtvf.min.css" rel="stylesheet" />
+        <link href="https://kg.cmclinnovations.com/cdn/dtvf/3.7.0/dtvf.min.css" rel="stylesheet" />
 
         <!-- ===== CUSTOMISABLE ===== -->
         <!-- CSS files to provide styling specifically for this visualisation instance can go here. -->
@@ -156,51 +156,48 @@
         </div>
 
         <!-- Code entry point -->
-        <script>
+        <script type="module">
             $("#sidePanelInner").tabs();
 
-            // ===== CUSTOMISABLE =====
-            // Enter Mapbox account name and API key here, even in Cesium visualisations!
-            MapHandler.MAP_USER = "";
-            MapHandler.MAP_API = "";
-            // ===== CUSTOMISABLE =====
-
             // Create a new manager instance
             var manager = new Manager(MapProvider.CESIUM);
             window.manager = manager;
 
-            // Only start the map after data definitions have been read.
-            var dataPromise = manager.loadDefinitions();
-            dataPromise.then(() => start());
-
-            /**
-             * Starts the visualisation setup process.
-             */
-             function start() {
-                // Initialise the map object
-                manager.initialiseMap();
-
-                // ===== CUSTOMISABLE =====
-                // Set the default content in the "General" tab of the side panel
-                manager.getPanelHandler().setTitle("<h1>Example Visualisation</h1>");
-                manager.getPanelHandler().setContent(`
-                    <p>This example visualisation has been put together to demonstrate the intended use of the centralised Digital Twin Visualisation Framework (DTVF).
-                    This framework has been designed to make it easier for users not experienced with Typescript (or the mapping libraries) to quickly & easily put 
-                    together a new Digital Twin visualisation. It is intended for developers to use this example visualisation to gain an understanding of the DTVF 
-                    before attempting to create their own visualisation; to do that, this example can be copied and used as a starting point.</p>
-                    <p>This centralised framework expects users to have structured their data in a particular format, and provide a number of associated metadata files. 
-                    Before starting, ensure your data meets this format by reading the <a href="https://github.com/cambridge-cares/TheWorldAvatar/wiki/DTVF-v2.0.0" target="_blank">GitHub wiki</a>. 
-                    This page also details how the framework functions, and how to configure your visualisation to use it.</p>
-                    <p>Note that this example has no version number as it is not intended for release; instead it should be updated to reflect the most recent version of 
-                    the framework itself. It's also worth noting that this example uses the currently hosted version of the framework library (i.e. from the CMCL KG website), 
-                    not the files within the repository.</p>
-                `);
-                
-                manager.getPanelHandler().setFooter("The World Avatar, " + new Date().getFullYear());
-                // ===== CUSTOMISABLE =====
-
-                // Save general tab state as default
-                manager.getPanelHandler().storeDefault();
+            // Read credentials
+            manager.readCredentials().then(() => {
+
+                // Only start the map after data definitions have been read.
+                var dataPromise = manager.loadDefinitions();
+                dataPromise.then(() => start());
+
+                /**
+                 * Starts the visualisation setup process.
+                 */
+                function start() {
+                    // Initialise the map object
+                    manager.initialiseMap();
+
+                    // ===== CUSTOMISABLE =====
+                    // Set the default content in the "General" tab of the side panel
+                    manager.getPanelHandler().setTitle("<h1>Example Visualisation</h1>");
+                    manager.getPanelHandler().setContent(`
+                        <p>This example visualisation has been put together to demonstrate the intended use of the centralised Digital Twin Visualisation Framework (DTVF).
+                        This framework has been designed to make it easier for users not experienced with Typescript (or the mapping libraries) to quickly & easily put 
+                        together a new Digital Twin visualisation. It is intended for developers to use this example visualisation to gain an understanding of the DTVF 
+                        before attempting to create their own visualisation; to do that, this example can be copied and used as a starting point.</p>
+                        <p>This centralised framework expects users to have structured their data in a particular format, and provide a number of associated metadata files. 
+                        Before starting, ensure your data meets this format by reading the <a href="https://github.com/cambridge-cares/TheWorldAvatar/wiki/Digital-Twin-Visualisations" target="_blank">GitHub wiki</a>. 
+                        This page also details how the framework functions, and how to configure your visualisation to use it.</p>
+                        <p>Note that this example has no version number as it is not intended for release; instead it should be updated to reflect the most recent version of 
+                        the framework itself. It's also worth noting that this example uses the currently hosted version of the framework library (i.e. from the CMCL KG website), 
+                        not the files within the repository.</p>
+                    `);
+                    
+                    manager.getPanelHandler().setFooter("The World Avatar, " + new Date().getFullYear());
+                    // ===== CUSTOMISABLE =====
+
+                    // Save general tab state as default
+                    manager.getPanelHandler().storeDefault();
 
                  // Load the data definitions
                 let configURL = "./data.json"
@@ -209,7 +206,8 @@
                     // Plot the data
                     manager.plotData();
                 });
-            }
+                }
+            });
         </script>
     </body>
 </html>
diff --git a/web/digital-twin-vis-framework/example-mapbox-vis/README.md b/web/digital-twin-vis-framework/example-mapbox-vis/README.md
diff --git a/web/digital-twin-vis-framework/example-mapbox-vis/webspace/index.html b/web/digital-twin-vis-framework/example-mapbox-vis/webspace/index.html
diff --git a/web/digital-twin-vis-framework/library/package.json b/web/digital-twin-vis-framework/library/package.json
diff --git a/web/digital-twin-vis-framework/library/release.html b/web/digital-twin-vis-framework/library/release.html
diff --git a/web/digital-twin-vis-framework/library/src/ts/manager.ts b/web/digital-twin-vis-framework/library/src/ts/manager.ts
diff --git a/web/digital-twin-vis-framework/library/version b/web/digital-twin-vis-framework/library/version

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`{`
	`2`	`+ "type": "visualisation",`
`2`	`3`	`"ServiceSpec": {`
`3`	`4`	`"Name": "visualisation",`
`4`	`5`	`"TaskTemplate": {`
`@@ -10,6 +11,20 @@`
`10`	`11`	`"Source": "vis-files",`
`11`	`12`	`"Target": "/var/www/html"`
`12`	`13`	`}`
	`14`	`+ ],`
	`15`	`+ "Secrets": [`
	`16`	`+ {`
	`17`	`+ "SecretName": "mapbox_username",`
	`18`	`+ "File": {`
	`19`	`+ "Name": "/var/www/html/mapbox_username"`
	`20`	`+ }`
	`21`	`+ },`
	`22`	`+ {`
	`23`	`+ "SecretName": "mapbox_api_key",`
	`24`	`+ "File": {`
	`25`	`+ "Name": "/var/www/html/mapbox_api_key"`
	`26`	`+ }`
	`27`	`+ }`
`13`	`28`	`]`
`14`	`29`	`}`
`15`	`30`	`}`