introduce unique ids

Author: mieciu

platform/model/typical_queries/hits.go

@@ -3,7 +3,11 @@
 package typical_queries
 
 import (
+	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"github.com/QuesmaOrg/quesma/platform/clickhouse"
 	"github.com/QuesmaOrg/quesma/platform/common_table"
@@ -241,7 +245,9 @@ func (query Hits) computeIdForDocument(doc model.SearchHit, defaultID string) st
 			// At database level we only compare timestamps with millisecond precision
 			// However in search results we append `q` plus generated digits (we use q because it's not in hex)
 			// so that kibana can iterate over documents in UI
-			pseudoUniqueId = fmt.Sprintf("%xq%s", vv, defaultID)
+			sourceHash := fmt.Sprintf("%x", HashJSON(doc.Source))
+			pseudoUniqueId = fmt.Sprintf("%xqqq%sqqq%x", vv, defaultID, sourceHash)
+			//pseudoUniqueId = fmt.Sprintf("%xq%s", vv, defaultID)
 		} else {
 			logger.WarnWithCtx(query.ctx).Msgf("failed to convert timestamp field [%v] to time.Time", v[0])
 			return defaultID
@@ -250,6 +256,19 @@ func (query Hits) computeIdForDocument(doc model.SearchHit, defaultID string) st
 	return pseudoUniqueId
 }
 
+func HashJSON(data json.RawMessage) string {
+	var buf bytes.Buffer
+	err := json.Compact(&buf, data)
+	if err != nil {
+		hash := sha256.Sum256(data)
+		return hex.EncodeToString(hash[:])
+	}
+	hash := sha256.Sum256(buf.Bytes())
+	eee := hex.EncodeToString(hash[:])
+	println(eee)
+	return hex.EncodeToString(hash[:])
+}
+
 func (query Hits) String() string {
 	return fmt.Sprintf("hits(indexes: %v)", strings.Join(query.indexes, ", "))
 }

platform/parsers/elastic_query_dsl/query_parser.go

@@ -34,7 +34,7 @@ func NewEmptyHighlighter() model.Highlighter {
 }
 
 const (
-	defaultQueryResultSize = 10
+	defaultQueryResultSize = 10000 // TODO looks like we can NOT limit the returned `hits` because we calculate IDs there
 	defaultTrackTotalHits  = 10000
 )
 
@@ -322,6 +322,7 @@ func (cw *ClickhouseQueryTranslator) parseIds(queryMap QueryMap) model.SimpleQue
 		return model.NewSimpleQueryInvalid()
 	}
 	ids := make([]string, 0, len(idsRaw))
+	uniqueIds := make([]string, 0, len(idsRaw)) // to avoid duplicates
 	for _, id := range idsRaw {
 		if idAsString, ok := id.(string); ok {
 			ids = append(ids, idAsString)
@@ -335,14 +336,15 @@ func (cw *ClickhouseQueryTranslator) parseIds(queryMap QueryMap) model.SimpleQue
 	// therefore we need to strip the hex part (before `q`) and convert it to decimal
 	// then we can query at DB level
 	for i, id := range ids {
-		idInHex := strings.Split(id, "q")[0]
+		idInHex := strings.Split(id, "qqq")[0]
 		if idAsStr, err := hex.DecodeString(idInHex); err != nil {
 			logger.Error().Msgf("error parsing document id %s: %v", id, err)
 			return model.NewSimpleQueryInvalid()
 		} else {
 			tsWithoutTZ := strings.TrimSuffix(string(idAsStr), " +0000 UTC")
 			ids[i] = fmt.Sprintf("'%s'", tsWithoutTZ)
 		}
+		uniqueIds = append(uniqueIds, id)
 	}
 
 	var idToSql func(string) (model.Expr, error)
@@ -399,6 +401,7 @@ func (cw *ClickhouseQueryTranslator) parseIds(queryMap QueryMap) model.SimpleQue
 		idsTuple := model.NewTupleExpr(idsAsExprs...)
 		whereStmt = model.NewInfixExpr(model.NewColumnRef(timestampColumnName), " IN ", idsTuple)
 	}
+	cw.UniqueIDs = uniqueIds // TODO a crucial side effect here
 	return model.NewSimpleQuery(whereStmt, true)
 }
 

platform/parsers/elastic_query_dsl/query_translator.go

@@ -11,6 +11,8 @@ import (
 	"github.com/QuesmaOrg/quesma/platform/parsers/elastic_query_dsl/query_util"
 	"github.com/QuesmaOrg/quesma/platform/schema"
 	"github.com/QuesmaOrg/quesma/platform/util"
+	"slices"
+	"strings"
 )
 
 type JsonMap = map[string]interface{}
@@ -24,7 +26,8 @@ type ClickhouseQueryTranslator struct {
 	Indexes []string
 
 	// TODO this will be removed
-	Table *clickhouse.Table
+	Table     *clickhouse.Table
+	UniqueIDs []string // used for hits queries, to filter out hits that are not in the list of IDs
 }
 
 var completionStatusOK = func() *int { value := 200; return &value }()
@@ -130,8 +133,31 @@ func (cw *ClickhouseQueryTranslator) makeHits(queries []*model.Query, results []
 	}
 	hitsPartOfResponse := hitsQuery.Type.TranslateSqlResponseToJson(hitsResultSet)
 
+	// trim hits
+
 	hitsResponse := hitsPartOfResponse["hits"].(model.SearchHits)
-	return queriesWithoutHits, resultsWithoutHits, &hitsResponse
+	hits := cw.RemoveHitsIfDocHashesSet(hitsResponse)
+	return queriesWithoutHits, resultsWithoutHits, &hits
+}
+
+func (cw *ClickhouseQueryTranslator) RemoveHitsIfDocHashesSet(hits model.SearchHits) model.SearchHits {
+	// if we have doc hashes set, we need to remove hits from the response
+	if len(cw.UniqueIDs) == 0 {
+		return hits
+	}
+	docHashes := make([]string, 0, len(cw.UniqueIDs))
+	for _, id := range cw.UniqueIDs {
+		docHashes = append(docHashes, strings.Split(id, "qqq")[2])
+	}
+	filteredHits := make([]model.SearchHit, 0, len(hits.Hits))
+	for _, hit := range hits.Hits {
+		hashOfOfDocInHit := strings.Split(hit.ID, "qqq")[2]
+		if slices.Contains(docHashes, hashOfOfDocInHit) {
+			filteredHits = append(filteredHits, hit)
+		}
+	}
+	hits.Hits = filteredHits
+	return hits
 }
 
 func (cw *ClickhouseQueryTranslator) makeTotalCount(queries []*model.Query, results [][]model.QueryResultRow) (queriesWithoutCount []*model.Query, resultsWithoutCount [][]model.QueryResultRow, total *model.Total) {