Introduce unique ids (#1435)

An attempt to introduce unique object IDs in the ClickHouse realm.

A returned document id (`_id` field) would have the following syntax:
```
{hex-encoded timestamp field}qqq{hex-encoded hash of the document}
```
Of course, the `{hex-encoded hash of the document}` lives only in Quesma
memory:
1. It needs to be re-calculated when returning search hits
2. Quesma need to filter out based on it when rendering hits

Therefore when fetching document with specific `_id` we could filter out
documents with matching timestamp and then filter for that with matching
fields, returning the right entry.


This fixes the issue, where JSON view of single document could return a
random object from search hits, not necessarily the one clicked.
<img width="1458" alt="image"
src="https://github.com/user-attachments/assets/0b808dba-0255-4be0-ac5f-acb7a21569ba"
/>

**However** (and it's a pretty big however), the "surrounding document"
view **cannot return the surrounding documents**. While the query
doesn't error it also doesn't return any documents. The experiment to
have that working [is carried out is a separate
PR](#1446), although it's not
clear yet if this way of doing things is going to guarantee 100%
correctness.

Author: mieciu

platform/model/typical_queries/hits.go

@@ -4,6 +4,9 @@ package typical_queries
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"github.com/QuesmaOrg/quesma/platform/clickhouse"
 	"github.com/QuesmaOrg/quesma/platform/common_table"
@@ -12,6 +15,7 @@ import (
 	"github.com/QuesmaOrg/quesma/platform/model"
 	"github.com/QuesmaOrg/quesma/platform/util"
 	"reflect"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -239,9 +243,9 @@ func (query Hits) computeIdForDocument(doc model.SearchHit, defaultID string) st
 	if v, ok := doc.Fields[tsFieldName]; ok {
 		if vv, okk := v[0].(time.Time); okk {
 			// At database level we only compare timestamps with millisecond precision
-			// However in search results we append `q` plus generated digits (we use q because it's not in hex)
-			// so that kibana can iterate over documents in UI
-			pseudoUniqueId = fmt.Sprintf("%xq%s", vv, defaultID)
+			// However in search results we append `qqq` plus generated hash of the source to hex-encoded timestamp
+			sourceHash := fmt.Sprintf("%x", ComputeHash(doc.Source))
+			pseudoUniqueId = fmt.Sprintf("%xqqq%x", vv, sourceHash)
 		} else {
 			logger.WarnWithCtx(query.ctx).Msgf("failed to convert timestamp field [%v] to time.Time", v[0])
 			return defaultID
@@ -250,6 +254,50 @@ func (query Hits) computeIdForDocument(doc model.SearchHit, defaultID string) st
 	return pseudoUniqueId
 }
 
+func ComputeHash(data json.RawMessage) string {
+	var parsed interface{}
+	if err := json.Unmarshal(data, &parsed); err != nil {
+		hash := sha256.Sum256(data)
+		return hex.EncodeToString(hash[:])
+	}
+	normalized := normalizeJSON(parsed)
+	normalizedBytes, err := json.Marshal(normalized)
+	if err != nil {
+		hash := sha256.Sum256(data)
+		return hex.EncodeToString(hash[:])
+	}
+	hash := sha256.Sum256(normalizedBytes)
+	return hex.EncodeToString(hash[:])
+}
+
+// normalizeJSON recursively normalizes JSON structure to ensure consistent ordering for further hashing.
+func normalizeJSON(v interface{}) interface{} {
+	switch val := v.(type) {
+	case map[string]interface{}:
+		keys := make([]string, 0, len(val))
+		for k := range val {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+
+		normalized := make(map[string]interface{})
+		for _, k := range keys {
+			normalized[k] = normalizeJSON(val[k])
+		}
+		return normalized
+
+	case []interface{}:
+		normalized := make([]interface{}, len(val))
+		for i, v := range val {
+			normalized[i] = normalizeJSON(v)
+		}
+		return normalized
+
+	default:
+		return val
+	}
+}
+
 func (query Hits) String() string {
 	return fmt.Sprintf("hits(indexes: %v)", strings.Join(query.indexes, ", "))
 }

platform/parsers/elastic_query_dsl/query_parser.go

@@ -34,8 +34,10 @@ func NewEmptyHighlighter() model.Highlighter {
 }
 
 const (
-	defaultQueryResultSize = 10
-	defaultTrackTotalHits  = 10000
+	defaultQueryResultSize        = 10
+	increasedResultSizeForIdQuery = 10000 // In `_id` query we had to fetch way more documents as these have to be filtered later on, when assembling the result
+	defaultTrackTotalHits         = 10000
+	uuidSeparator                 = "qqq" // Document IDs (_id) fields in quesma ar
 )
 
 func (cw *ClickhouseQueryTranslator) ParseQuery(body types.JSON) (*model.ExecutionPlan, error) {
@@ -322,6 +324,7 @@ func (cw *ClickhouseQueryTranslator) parseIds(queryMap QueryMap) model.SimpleQue
 		return model.NewSimpleQueryInvalid()
 	}
 	ids := make([]string, 0, len(idsRaw))
+	uniqueIds := make([]string, 0, len(idsRaw))
 	for _, id := range idsRaw {
 		if idAsString, ok := id.(string); ok {
 			ids = append(ids, idAsString)
@@ -331,18 +334,19 @@ func (cw *ClickhouseQueryTranslator) parseIds(queryMap QueryMap) model.SimpleQue
 		}
 	}
 
-	// when our generated ID appears in query looks like this: `1d<TRUNCATED>0b8q1`
-	// therefore we need to strip the hex part (before `q`) and convert it to decimal
-	// then we can query at DB level
+	// when our generated ID appears in query looks like this:
+	// `<hex-encoded timestamp>qqq<hex-encoded source hash>`
+	// Therefore we need to convert the hex-encoded timestamp to assemble the SQL query
 	for i, id := range ids {
-		idInHex := strings.Split(id, "q")[0]
+		idInHex := strings.Split(id, uuidSeparator)[0]
 		if idAsStr, err := hex.DecodeString(idInHex); err != nil {
 			logger.Error().Msgf("error parsing document id %s: %v", id, err)
 			return model.NewSimpleQueryInvalid()
 		} else {
 			tsWithoutTZ := strings.TrimSuffix(string(idAsStr), " +0000 UTC")
 			ids[i] = fmt.Sprintf("'%s'", tsWithoutTZ)
 		}
+		uniqueIds = append(uniqueIds, id)
 	}
 
 	var idToSql func(string) (model.Expr, error)
@@ -399,6 +403,7 @@ func (cw *ClickhouseQueryTranslator) parseIds(queryMap QueryMap) model.SimpleQue
 		idsTuple := model.NewTupleExpr(idsAsExprs...)
 		whereStmt = model.NewInfixExpr(model.NewColumnRef(timestampColumnName), " IN ", idsTuple)
 	}
+	cw.UniqueIDsUsedInTheQuery = uniqueIds // a crucial side effect here - queries against _id field requires special treatment
 	return model.NewSimpleQuery(whereStmt, true)
 }
 
@@ -1233,6 +1238,11 @@ func ResolveField(ctx context.Context, fieldName string, schemaInstance schema.S
 }
 
 func (cw *ClickhouseQueryTranslator) parseSize(queryMap QueryMap, defaultSize int) int {
+	if len(cw.UniqueIDsUsedInTheQuery) > 0 {
+		// If this is a unique ID query, we can't limit size at the SQL level,
+		// because we need all matching timestamps that later will be filtered out but looking at hashes computed on hits
+		return increasedResultSizeForIdQuery
+	}
 	sizeRaw, exists := queryMap["size"]
 	if !exists {
 		return defaultSize

platform/parsers/elastic_query_dsl/query_translator.go

@@ -11,6 +11,8 @@ import (
 	"github.com/QuesmaOrg/quesma/platform/parsers/elastic_query_dsl/query_util"
 	"github.com/QuesmaOrg/quesma/platform/schema"
 	"github.com/QuesmaOrg/quesma/platform/util"
+	"slices"
+	"strings"
 )
 
 type JsonMap = map[string]interface{}
@@ -24,7 +26,9 @@ type ClickhouseQueryTranslator struct {
 	Indexes []string
 
 	// TODO this will be removed
-	Table *clickhouse.Table
+	Table                   *clickhouse.Table
+	UniqueIDsUsedInTheQuery []string // A list of UniqueIDs used in the query (via `_id` field), which has to be passed to the JSON response rendering stage.
+	// That's because it has to be used both in the SQL query and during the response rendering, see https://github.com/QuesmaOrg/quesma/pull/1435 for more details.
 }
 
 var completionStatusOK = func() *int { value := 200; return &value }()
@@ -131,7 +135,30 @@ func (cw *ClickhouseQueryTranslator) makeHits(queries []*model.Query, results []
 	hitsPartOfResponse := hitsQuery.Type.TranslateSqlResponseToJson(hitsResultSet)
 
 	hitsResponse := hitsPartOfResponse["hits"].(model.SearchHits)
-	return queriesWithoutHits, resultsWithoutHits, &hitsResponse
+	hits := cw.FilterOutHitsIfThisIsIdQuery(hitsResponse)
+	return queriesWithoutHits, resultsWithoutHits, &hits
+}
+
+// FilterOutHitsIfThisIsIdQuery - If during parsing we have found that this is a query for _id,
+// we filter out hits that are not in the list of UniqueIDsUsedInTheQuery.
+// we only do this filtering based on the doc.Source hash comparison, ignoring the two first UUID parts.
+func (cw *ClickhouseQueryTranslator) FilterOutHitsIfThisIsIdQuery(hits model.SearchHits) model.SearchHits {
+	if len(cw.UniqueIDsUsedInTheQuery) == 0 {
+		return hits // not _id query, proceed as usual
+	}
+	hashesFromQuery := make([]string, 0, len(cw.UniqueIDsUsedInTheQuery))
+	for _, id := range cw.UniqueIDsUsedInTheQuery {
+		hashesFromQuery = append(hashesFromQuery, strings.Split(id, uuidSeparator)[1])
+	}
+	filteredHits := make([]model.SearchHit, 0, len(hits.Hits))
+	for _, hit := range hits.Hits {
+		hash := strings.Split(hit.ID, uuidSeparator)[1]
+		if slices.Contains(hashesFromQuery, hash) {
+			filteredHits = append(filteredHits, hit)
+		}
+	}
+	hits.Hits = filteredHits
+	return hits
 }
 
 func (cw *ClickhouseQueryTranslator) makeTotalCount(queries []*model.Query, results [][]model.QueryResultRow) (queriesWithoutCount []*model.Query, resultsWithoutCount [][]model.QueryResultRow, total *model.Total) {

platform/testdata/requests.go

@@ -2148,7 +2148,7 @@ var TestsSearch = []SearchTestCase{
 					},
 					{
 					  "match_phrase": {
-						"_id": "323032342d30352d32342031333a33323a34372e333037202b3030303020555443q1"
+						"_id": "323032342d30352d32342031333a33323a34372e333037202b3030303020555443qqq111111111111111111111111111111111111111111111111111111111111"
 					  }
 					}
 				  ]
@@ -2386,7 +2386,7 @@ Men\\'s Clothing \\\\ 	%' LIMIT 10`},
 		`{
 			"query": {
 				"ids": {
-					 "values": ["323032342d31322d32312030373a32393a30332e333637202b3030303020555443q1"]
+					 "values": ["323032342d31322d32312030373a32393a30332e333637202b3030303020555443qqq1111111111111111111111111111111111111111"]
 				}
 			},
 			"track_total_hits": false
@@ -2407,8 +2407,8 @@ Men\\'s Clothing \\\\ 	%' LIMIT 10`},
 			"query": {
 				"ids": {
 					 "values": [
-						"323032342d31322d32312030373a32393a30332e333637202b3030303020555443q1",
-						"323032342d31322d32312030373a32393a30322e393932202b3030303020555443q3"
+						"323032342d31322d32312030373a32393a30332e333637202b3030303020555443qqq111111111111111111111111111",
+ 						"323032342d31322d32312030373a32393a30322e393932202b3030303020555443qqq111111111111111111111111111"
 					]
 				}
 			},
@@ -2420,7 +2420,7 @@ Men\\'s Clothing \\\\ 	%' LIMIT 10`},
 			`SELECT "message" ` +
 				`FROM ` + TableName + ` ` +
 				`WHERE "@timestamp" IN tuple(toDateTime64('2024-12-21 07:29:03.367',3), toDateTime64('2024-12-21 07:29:02.992',3)) ` +
-				`LIMIT 10`,
+				`LIMIT 10000`,
 		},
 		[]string{},
 	},
@@ -2429,7 +2429,7 @@ Men\\'s Clothing \\\\ 	%' LIMIT 10`},
 		`{
 			"query": {
 				"ids": {
-					 "values": ["323032342d31322d32312030373a32393a30332e333637303030303030q1"]
+					 "values": ["323032342d31322d32312030373a32393a30332e333637303030303030qqq123qqq11111111111111111111111111111111111111111111111"]
 				}
 			},
 			"track_total_hits": false
@@ -2440,7 +2440,7 @@ Men\\'s Clothing \\\\ 	%' LIMIT 10`},
 			`SELECT "message" ` +
 				`FROM ` + TableName + ` ` +
 				`WHERE "@timestamp" = toDateTime64('2024-12-21 07:29:03.367000000',9) ` +
-				`LIMIT 10`,
+				`LIMIT 10000`,
 		},
 		[]string{},
 	},
@@ -2449,7 +2449,7 @@ Men\\'s Clothing \\\\ 	%' LIMIT 10`},
 		`{
 			"query": {
 				"ids": {
-					 "values": ["323032342d31322d32312030373a32393a30332e313233343536373839q123"]
+					 "values": ["323032342d31322d32312030373a32393a30332e313233343536373839qqq123qqq11111111111111111111111111111111111111111111111"]
 				}
 			},
 			"track_total_hits": false
@@ -2460,7 +2460,7 @@ Men\\'s Clothing \\\\ 	%' LIMIT 10`},
 			`SELECT "message" ` +
 				`FROM ` + TableName + ` ` +
 				`WHERE "@timestamp" = toDateTime64('2024-12-21 07:29:03.123456789',9) ` +
-				`LIMIT 10`,
+				`LIMIT 10000`,
 		},
 		[]string{},
 	},
@@ -2469,7 +2469,7 @@ Men\\'s Clothing \\\\ 	%' LIMIT 10`},
 		`{
 			"query": {
 				"ids": {
-					 "values": ["323032342d31322d32312030373a32393a3033q1"]
+					 "values": ["323032342d31322d32312030373a32393a3033qqq11111111111111111111111111111111111111111111111"]
 				}
 			},
 			"track_total_hits": false
@@ -2480,7 +2480,7 @@ Men\\'s Clothing \\\\ 	%' LIMIT 10`},
 			`SELECT "message" ` +
 				`FROM ` + TableName + ` ` +
 				`WHERE "@timestamp" = toDateTime64('2024-12-21 07:29:03',0) ` +
-				`LIMIT 10`,
+				`LIMIT 10000`,
 		},
 		[]string{},
 	},
@@ -2489,7 +2489,7 @@ Men\\'s Clothing \\\\ 	%' LIMIT 10`},
 		`{
 			"query": {
 				"ids": {
-					 "values": ["323032342d31322d32312030373a32393a30332e33q1"]
+					 "values": ["323032342d31322d32312030373a32393a30332e33qqq11111111111111111111111111111111111111111111111"]
 				}
 			},
 			"track_total_hits": false
@@ -2500,7 +2500,7 @@ Men\\'s Clothing \\\\ 	%' LIMIT 10`},
 			`SELECT "message" ` +
 				`FROM ` + TableName + ` ` +
 				`WHERE "@timestamp" = toDateTime64('2024-12-21 07:29:03.3',1) ` +
-				`LIMIT 10`,
+				`LIMIT 10000`,
 		},
 		[]string{},
 	},