Ingest performance improvements (#1028)

nablaone · web-flow · commit 3d4a9efd4bb9 · 2024-11-27T18:49:35.000Z
This PR can be reviewed commit by commit. It includes 4 improvements:

- **Use go-json insead of standard JSON library**
- **Add regexp cache**
- **Remove remarshalling**
- **Remove remarshalling - continued**


Maybe I should split them into separate PRs.

---------

Signed-off-by: Rafał Strzaliński &lt;rafal@quesma.com&gt;
diff --git a/quesma/ab_testing/collector/diff.go b/quesma/ab_testing/collector/diff.go
@@ -4,8 +4,8 @@ package collector
 
 import (
 	"crypto/sha1"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"quesma/jsondiff"
 	"quesma/quesma/types"
 )
diff --git a/quesma/ab_testing/collector/fanout.go b/quesma/ab_testing/collector/fanout.go
@@ -5,8 +5,8 @@ package collector
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"net/http"
 	"quesma/ingest"
 	"quesma/logger"
diff --git a/quesma/ab_testing/collector/processors.go b/quesma/ab_testing/collector/processors.go
@@ -3,8 +3,8 @@
 package collector
 
 import (
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"quesma/quesma/types"
 	"regexp"
 )
diff --git a/quesma/ast/query_traversal_test.go b/quesma/ast/query_traversal_test.go
@@ -3,8 +3,8 @@
 package ast
 
 import (
-	"encoding/json"
 	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
+	"github.com/goccy/go-json"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"testing"
diff --git a/quesma/buildinfo/build.go b/quesma/buildinfo/build.go
@@ -3,9 +3,9 @@
 package buildinfo
 
 import (
-	"encoding/json"
 	"fmt"
 	"github.com/coreos/go-semver/semver"
+	"github.com/goccy/go-json"
 	"net/http"
 	"time"
 )
diff --git a/quesma/clickhouse/table_discovery.go b/quesma/clickhouse/table_discovery.go
@@ -5,9 +5,9 @@ package clickhouse
 import (
 	"context"
 	"database/sql"
-	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/goccy/go-json"
 	"quesma/common_table"
 	"quesma/end_user_errors"
 	"quesma/logger"
diff --git a/quesma/clickhouse/util.go b/quesma/clickhouse/util.go
@@ -4,8 +4,8 @@ package clickhouse
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"quesma/logger"
 	"quesma/model"
 	"strings"
diff --git a/quesma/elasticsearch/index_resolver.go b/quesma/elasticsearch/index_resolver.go
@@ -4,7 +4,7 @@ package elasticsearch
 
 import (
 	"context"
-	"encoding/json"
+	"github.com/goccy/go-json"
 	"io"
 	"net/http"
 	"quesma/quesma/config"
diff --git a/quesma/eql/e2e/ingest_integration_test.go b/quesma/eql/e2e/ingest_integration_test.go
@@ -6,8 +6,8 @@ package e2e
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"log"
 	"net/http"
 	"time"
diff --git a/quesma/eql/e2e/query_integration_test.go b/quesma/eql/e2e/query_integration_test.go
@@ -6,8 +6,8 @@ package e2e
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"io"
 	"net/http"
 	"quesma/jsonprocessor"
diff --git a/quesma/go.mod b/quesma/go.mod
@@ -31,6 +31,7 @@ require (
 
 require (
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
diff --git a/quesma/go.sum b/quesma/go.sum
@@ -32,6 +32,8 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
 github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
+github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
diff --git a/quesma/health/elastic.go b/quesma/health/elastic.go
@@ -4,8 +4,8 @@ package health
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"io"
 	"net/http"
 	"quesma/elasticsearch"
diff --git a/quesma/ingest/common_table_test.go b/quesma/ingest/common_table_test.go
@@ -4,8 +4,8 @@ package ingest
 
 import (
 	"context"
-	"encoding/json"
 	"github.com/DATA-DOG/go-sqlmock"
+	"github.com/goccy/go-json"
 	"github.com/stretchr/testify/assert"
 	"quesma/clickhouse"
 	"quesma/common_table"
diff --git a/quesma/ingest/processor.go b/quesma/ingest/processor.go
@@ -5,9 +5,9 @@ package ingest
 import (
 	"context"
 	"database/sql"
-	"encoding/json"
 	"fmt"
 	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/goccy/go-json"
 	chLib "quesma/clickhouse"
 	"quesma/comment_metadata"
 	"quesma/common_table"
@@ -490,32 +490,14 @@ func (ip *IngestProcessor) GenerateIngestContent(table *chLib.Table,
 	config *chLib.ChTableConfig,
 	encodings map[schema.FieldEncodingKey]schema.EncodedFieldName) ([]string, types.JSON, []NonSchemaField, error) {
 
-	jsonAsBytesSlice, err := json.Marshal(data)
-
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	// we find all non-schema fields
-	jsonMap, err := types.ParseJSON(string(jsonAsBytesSlice))
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
 	if len(config.Attributes) == 0 {
-		return nil, jsonMap, nil, nil
-	}
-
-	schemaFieldsJson, err := json.Marshal(jsonMap)
-
-	if err != nil {
-		return nil, jsonMap, nil, err
+		return nil, data, nil, nil
 	}
 
-	mDiff := DifferenceMap(jsonMap, table) // TODO change to DifferenceMap(m, t)
+	mDiff := DifferenceMap(data, table) // TODO change to DifferenceMap(m, t)
 
-	if len(mDiff) == 0 && string(schemaFieldsJson) == string(jsonAsBytesSlice) && len(inValidJson) == 0 { // no need to modify, just insert 'js'
-		return nil, jsonMap, nil, nil
+	if len(mDiff) == 0 && len(inValidJson) == 0 { // no need to modify, just insert 'js'
+		return nil, data, nil, nil
 	}
 
 	// check attributes precondition
@@ -546,7 +528,7 @@ func (ip *IngestProcessor) GenerateIngestContent(table *chLib.Table,
 		return nil, nil, nil, err
 	}
 
-	onlySchemaFields := RemoveNonSchemaFields(jsonMap, table)
+	onlySchemaFields := RemoveNonSchemaFields(data, table)
 
 	return alterCmd, onlySchemaFields, nonSchemaFields, nil
 }
diff --git a/quesma/ingest/processor_test.go b/quesma/ingest/processor_test.go
@@ -3,7 +3,7 @@
 package ingest
 
 import (
-	"encoding/json"
+	"github.com/goccy/go-json"
 	"quesma/clickhouse"
 	"quesma/persistence"
 	"quesma/quesma/config"
diff --git a/quesma/ingest/util.go b/quesma/ingest/util.go
@@ -4,8 +4,8 @@ package ingest
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"quesma/clickhouse"
 	"strings"
 )
diff --git a/quesma/jsonprocessor/json_processor_test.go b/quesma/jsonprocessor/json_processor_test.go
@@ -3,7 +3,7 @@
 package jsonprocessor
 
 import (
-	"encoding/json"
+	"github.com/goccy/go-json"
 	"github.com/stretchr/testify/assert"
 	"quesma/quesma/types"
 	"reflect"
diff --git a/quesma/licensing/license_manager.go b/quesma/licensing/license_manager.go
@@ -4,8 +4,8 @@ package licensing
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"io"
 	"net/http"
 	"time"
diff --git a/quesma/model/query_result.go b/quesma/model/query_result.go
@@ -4,8 +4,8 @@ package model
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"quesma/common_table"
 	"quesma/logger"
 	"quesma/schema"
diff --git a/quesma/model/search_response.go b/quesma/model/search_response.go
@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Elastic-2.0
 package model
 
-import "encoding/json"
+import "github.com/goccy/go-json"
 
 type JsonMap = map[string]interface{}
 
diff --git a/quesma/persistence/elastic.go b/quesma/persistence/elastic.go
@@ -4,8 +4,8 @@ package persistence
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"io"
 	"log"
 	"net/http"
diff --git a/quesma/proxy/l4_proxy_test.go b/quesma/proxy/l4_proxy_test.go
@@ -4,8 +4,8 @@ package proxy
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"github.com/stretchr/testify/assert"
 	"log"
 	"net"
diff --git a/quesma/queryparser/dashboard_error.go b/quesma/queryparser/dashboard_error.go
@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Elastic-2.0
 package queryparser
 
-import "encoding/json"
+import "github.com/goccy/go-json"
 
 func BadRequestParseError(err error) []byte {
 	serialized, _ := json.Marshal(DashboardErrorResponse{
diff --git a/quesma/queryparser/query_parser.go b/quesma/queryparser/query_parser.go
@@ -6,8 +6,8 @@ package queryparser
 import (
 	"context"
 	"encoding/hex"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"github.com/k0kubun/pp"
 	"quesma/clickhouse"
 	"quesma/logger"
diff --git a/quesma/queryparser/query_translator_test.go b/quesma/queryparser/query_translator_test.go
@@ -4,7 +4,7 @@ package queryparser
 
 import (
 	"context"
-	"encoding/json"
+	"github.com/goccy/go-json"
 	"github.com/k0kubun/pp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
diff --git a/quesma/quesma/config/env2json_test.go b/quesma/quesma/config/env2json_test.go
@@ -3,7 +3,7 @@
 package config
 
 import (
-	"encoding/json"
+	"github.com/goccy/go-json"
 	"github.com/stretchr/testify/assert"
 	"os"
 	"testing"
diff --git a/quesma/quesma/functionality/bulk/bulk.go b/quesma/quesma/functionality/bulk/bulk.go
@@ -4,8 +4,8 @@ package bulk
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"io"
 	"net/http"
 	"quesma/clickhouse"
diff --git a/quesma/quesma/functionality/bulk/bulk_test.go b/quesma/quesma/functionality/bulk/bulk_test.go
@@ -3,7 +3,7 @@
 package bulk
 
 import (
-	"encoding/json"
+	"github.com/goccy/go-json"
 	"github.com/stretchr/testify/require"
 	"testing"
 )
diff --git a/quesma/quesma/functionality/field_capabilities/field_caps.go b/quesma/quesma/functionality/field_capabilities/field_caps.go
@@ -4,8 +4,8 @@ package field_capabilities
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
+	"github.com/goccy/go-json"
 	"quesma/clickhouse"
 	"quesma/elasticsearch"
 	"quesma/elasticsearch/elasticsearch_field_types"
diff --git a/quesma/quesma/functionality/field_capabilities/field_caps_test.go b/quesma/quesma/functionality/field_capabilities/field_caps_test.go
@@ -3,7 +3,7 @@
 package field_capabilities
 
 import (
-	"encoding/json"
+	"github.com/goccy/go-json"
 	"github.com/stretchr/testify/assert"
 	"quesma/clickhouse"
 	"quesma/model"
diff --git a/quesma/quesma/functionality/terms_enum/terms_enum.go b/quesma/quesma/functionality/terms_enum/terms_enum.go
@@ -4,9 +4,9 @@ package terms_enum
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/goccy/go-json"
 	"quesma/clickhouse"
 	"quesma/end_user_errors"
 	"quesma/logger"
diff --git a/quesma/quesma/functionality/terms_enum/terms_enum_test.go b/quesma/quesma/functionality/terms_enum/terms_enum_test.go
@@ -5,9 +5,9 @@ package terms_enum
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
 	"github.com/DATA-DOG/go-sqlmock"
+	"github.com/goccy/go-json"
 	"github.com/stretchr/testify/assert"
 	"quesma/clickhouse"
 	"quesma/logger"
diff --git a/quesma/quesma/highlight_test.go b/quesma/quesma/highlight_test.go
@@ -4,7 +4,7 @@ package quesma
 
 import (
 	"context"
-	"encoding/json"
+	"github.com/goccy/go-json"
 	"github.com/stretchr/testify/assert"
 	"quesma/clickhouse"
 	"quesma/model"
diff --git a/quesma/quesma/mappings_test.go b/quesma/quesma/mappings_test.go
@@ -3,7 +3,7 @@
 package quesma
 
 import (
-	"encoding/json"
+	"github.com/goccy/go-json"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"quesma/elasticsearch"
diff --git a/quesma/quesma/router.go b/quesma/quesma/router.go
@@ -4,8 +4,8 @@ package quesma
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
+	"github.com/goccy/go-json"
 	"net/http"
 	"quesma/clickhouse"
 	"quesma/elasticsearch"
diff --git a/quesma/quesma/search_ab_testing.go b/quesma/quesma/search_ab_testing.go
diff --git a/quesma/quesma/search_test.go b/quesma/quesma/search_test.go
diff --git a/quesma/quesma/types/json.go b/quesma/quesma/types/json.go
diff --git a/quesma/quesma/types/json_test.go b/quesma/quesma/types/json_test.go
diff --git a/quesma/quesma/types/ndjson.go b/quesma/quesma/types/ndjson.go
diff --git a/quesma/quesma/ui/ab_testing.go b/quesma/quesma/ui/ab_testing.go
diff --git a/quesma/quesma/ui/console_routes.go b/quesma/quesma/ui/console_routes.go
diff --git a/quesma/quesma/ui/es_auth_provider.go b/quesma/quesma/ui/es_auth_provider.go
diff --git a/quesma/quesma/ui/live_tail_drilldown.go b/quesma/quesma/ui/live_tail_drilldown.go
diff --git a/quesma/quesma/ui/telemetry.go b/quesma/quesma/ui/telemetry.go
diff --git a/quesma/stats/ingest_statistics_test.go b/quesma/stats/ingest_statistics_test.go
diff --git a/quesma/telemetry/phone_home.go b/quesma/telemetry/phone_home.go
diff --git a/quesma/telemetry/phone_home_test.go b/quesma/telemetry/phone_home_test.go
diff --git a/quesma/util/utils.go b/quesma/util/utils.go
diff --git a/quesma/util/utils_test.go b/quesma/util/utils_test.go

Original file line number	Diff line number	Diff line change
`@@ -4,8 +4,8 @@ package collector`
`4`	`4`
`5`	`5`	`import (`
`6`	`6`	`"crypto/sha1"`
`7`		`- "encoding/json"`
`8`	`7`	`"fmt"`
	`8`	`+ "github.com/goccy/go-json"`
`9`	`9`	`"quesma/jsondiff"`
`10`	`10`	`"quesma/quesma/types"`
`11`	`11`	`)`
Original file line number	Diff line number	Diff line change
`@@ -3,8 +3,8 @@`
`3`	`3`	`package collector`
`4`	`4`
`5`	`5`	`import (`
`6`		`- "encoding/json"`
`7`	`6`	`"fmt"`
	`7`	`+ "github.com/goccy/go-json"`
`8`	`8`	`"quesma/quesma/types"`
`9`	`9`	`"regexp"`
`10`	`10`	`)`