Parser to exported dashboard json (#374)

Co-authored-by: Rafał Strzaliński <[email protected]>
Co-authored-by: przemyslaw <[email protected]>
Co-authored-by: Przemek Delewski <[email protected]>

Author: 4 people

bin/unjson-dashboards/.gitignore

@@ -0,0 +1,4 @@
+dashboards/*.ndjson
+mappings/*.json
+screenshots/*.png
+

bin/unjson-dashboards/README.md

@@ -0,0 +1,19 @@
+[INTERNAL ONLY] Checking how Quesma MVP works with provided Kibana dashboards
+======================================================================
+
+:warning: **This is an internal-only document.** Despite our open-source DNA, procedures described here cannot be replayed 
+without access to internal resources.
+
+
+Let's say someone provided you with some Kibana dashboards exported as NDJSON files.
+
+1. Place dashboard json files in `bin/unjson-dashboards/dashboards`
+2. Run `local-dev-chrome.yml` docker compose file from `quesma-examples` repository (private repo)
+3. Open `bin/unjson-dashboards/main.go` in your IDE \
+   We will be running steps by changing `if` statement condition in that file and running go program.  
+
+There are 4 stages there:
+* `extractMappings()` - figures out indices/mappings from the dashboards and writes them to `bin/unjson-dashboards/mappings` directory
+* `ingestAll()` - generates some test data according to the mappings and ingests it using Elasticsearch API to ClickHouse via Quesma  
+* `createDataViews()/importDashboards()` - creates relevant data views in Kibana
+* `visitDashboards()` - opens headless chrome which makes screenshot of these dashboards, saves them in `bin/unjson-dashboards/screenshots` and adds `index.html` so that these can be browsed in a convenient manner.

bin/unjson-dashboards/const.go

@@ -0,0 +1,11 @@
+// Copyright Quesma, licensed under the Elastic License 2.0.
+// SPDX-License-Identifier: Elastic-2.0
+package main
+
+const (
+	kibanaURL             = "http://localhost:5601" // Update this with your Kibana URL
+	kibanaSpace           = "default"               // Change if using a specific space
+	apiKey                = ""                      // Add API key if required
+	ndjsonFolder          = "dashboards"            // Directory containing NDJSON files
+	dashboardListEndpoint = "/api/saved_objects/_find?type=dashboard&per_page=10000"
+)

bin/unjson-dashboards/dashboards/README.md

@@ -0,0 +1 @@
+Place your dashboards in this directory. 

bin/unjson-dashboards/dataviews.go

@@ -0,0 +1,90 @@
+// Copyright Quesma, licensed under the Elastic License 2.0.
+// SPDX-License-Identifier: Elastic-2.0
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+)
+
+func createDataViews() {
+
+	indexes, err := readIndexMappings()
+	if err != nil {
+		log.Fatalf("Error reading index mappings: %v", err)
+	}
+
+	for _, index := range indexes {
+
+		var timestampField string
+		for field, mapping := range index.Properties {
+
+			if mapping.Type == "date" {
+				timestampField = field
+				break
+			}
+		}
+
+		err = CreateDataView(index.Pattern, timestampField)
+		if err != nil {
+			log.Printf("Failed to create Data View: %v", err)
+		}
+
+	}
+
+}
+
+type DataViewRequest struct {
+	Attributes struct {
+		Title         string `json:"title"`                   // Index pattern name
+		TimeFieldName string `json:"timeFieldName,omitempty"` // Optional time field
+	} `json:"attributes"`
+}
+
+// CreateDataView creates a new Data View (Index Pattern) in Kibana
+func CreateDataView(dataViewName string, timeField string) error {
+	// Construct the API URL
+	url := "http://localhost:5601/api/saved_objects/index-pattern"
+
+	// Create the request payload
+	payload := DataViewRequest{}
+	payload.Attributes.Title = dataViewName
+	if timeField != "" {
+		payload.Attributes.TimeFieldName = timeField
+	}
+
+	// Convert to JSON
+	jsonData, err := json.Marshal(payload)
+	if err != nil {
+		return fmt.Errorf("failed to marshal JSON: %v", err)
+	}
+
+	// Create HTTP request
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+	if err != nil {
+		return fmt.Errorf("failed to create request: %v", err)
+	}
+
+	// Set headers
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("kbn-xsrf", "true") // Required to bypass CSRF protection
+
+	// Execute the request
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	// Check response status
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
+		return fmt.Errorf("failed to create Data View, status: %d", resp.StatusCode)
+	}
+
+	log.Printf("✅ Successfully created Data View: %s", dataViewName)
+	return nil
+}