From 0e25749b4da06814ae1a9baee16b59f88eb727da Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Tue, 15 Oct 2024 20:29:02 +0200 Subject: [PATCH 01/30] some done --- quesma/persistence/elastic_with_eviction.go | 70 +++++++++++++++++++ quesma/persistence/evictor.go | 18 +++++ quesma/persistence/model.go | 23 ++++++ .../quesma/async_search_storage/in_memory.go | 11 ++- quesma/quesma/async_search_storage/model.go | 5 +- quesma/quesma/search.go | 11 +-- 6 files changed, 125 insertions(+), 13 deletions(-) create mode 100644 quesma/persistence/elastic_with_eviction.go create mode 100644 quesma/persistence/evictor.go diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go new file mode 100644 index 000000000..69199cccc --- /dev/null +++ b/quesma/persistence/elastic_with_eviction.go @@ -0,0 +1,70 @@ +// Copyright Quesma, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 +package persistence + +import "quesma/quesma/config" + +type ElasticDatabaseWithEviction struct { + *ElasticJSONDatabase // maybe remove and copy fields here + EvictorInterface + sizeInBytesLimit int64 +} + +func NewElasticDatabaseWithEviction(cfg config.ElasticsearchConfiguration, + indexName string, sizeInBytesLimit int64) *ElasticDatabaseWithEviction { + + return &ElasticDatabaseWithEviction{ + ElasticJSONDatabase: NewElasticJSONDatabase(cfg, indexName), + EvictorInterface: &Evictor{}, + sizeInBytesLimit: sizeInBytesLimit, + } +} + +// mutexy? or what +func (db *ElasticDatabaseWithEviction) Put(row Sizeable) bool { + bytesNeeded := db.SizeInBytes() + row.SizeInBytes() + if bytesNeeded > db.SizeInBytesLimit() { + docsToEvict, bytesEvicted := db.SelectToEvict(db.getAll(), bytesNeeded-db.SizeInBytesLimit()) + db.evict(docsToEvict) + bytesNeeded -= bytesEvicted + } + + if bytesNeeded <= db.SizeInBytesLimit() { + // put document + return true + } + return false +} + +func (db *ElasticDatabaseWithEviction) Get(id string) (*Sizeable, bool) { + // either use ElasticJSONDatabase.Get or implement own + // doesn't matter + return nil, false +} + +func (db *ElasticDatabaseWithEviction) Delete(id string) { + // mark as deleted, don't actually delete + // (single document deletion is hard in ES, it's done by evictor for entire index) +} + +func (db *ElasticDatabaseWithEviction) DocCount() int { + // send count() query to ES + return 0 +} + +func (db *ElasticDatabaseWithEviction) SizeInBytes() int64 { + return 0 +} + +func (db *ElasticDatabaseWithEviction) SizeInBytesLimit() int64 { + return db.sizeInBytesLimit +} + +func (db *ElasticDatabaseWithEviction) getAll() *basicDocumentInfo { + // send query + return nil +} + +func (db *ElasticDatabaseWithEviction) evict(documents []*basicDocumentInfo) { + +} diff --git a/quesma/persistence/evictor.go b/quesma/persistence/evictor.go new file mode 100644 index 000000000..f33cb5258 --- /dev/null +++ b/quesma/persistence/evictor.go @@ -0,0 +1,18 @@ +// Copyright Quesma, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 +package persistence + +type EvictorInterface interface { + SelectToEvict(documents []*basicDocumentInfo, sizeNeeded int64) (evictThese []*basicDocumentInfo, bytesEvicted int64) +} + +// It's only 1 implementation, which looks well suited for ElasticSearch. +// It can be implemented differently. +type Evictor struct{} + +func (e *Evictor) SelectToEvict(documents []*basicDocumentInfo, sizeNeeded int64) (evictThese []*basicDocumentInfo, bytesEvicted int64) { + if sizeNeeded <= 0 { + return // check if it's empty array or nil + } + +} diff --git a/quesma/persistence/model.go b/quesma/persistence/model.go index b1b16c3b8..85fad970f 100644 --- a/quesma/persistence/model.go +++ b/quesma/persistence/model.go @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Elastic-2.0 package persistence +import "time" + // JSONDatabase is an interface for a database that stores JSON data. // Treat it as `etcd` equivalent rather than `MongoDB`. // The main usage is to store our configuration data, like @@ -15,3 +17,24 @@ type JSONDatabase interface { Get(key string) (string, bool, error) Put(key string, data string) error } + +// T - type of the data to store, e.g. async_search_storage.AsyncRequestResult +type JSONDatabaseWithEviction interface { // for sure JSON? maybe not only json? check + Put(row *Sizeable) error + Get(id string) (*Sizeable, bool) + Delete(id string) + DocCount() int + SizeInBytes() int64 + SizeInBytesLimit() int64 +} + +type basicDocumentInfo struct { + id string + sizeInBytes int64 + timestamp time.Time + markedAsDeleted bool +} + +type Sizeable interface { + SizeInBytes() int64 +} diff --git a/quesma/quesma/async_search_storage/in_memory.go b/quesma/quesma/async_search_storage/in_memory.go index b8e6c3c03..452ce3036 100644 --- a/quesma/quesma/async_search_storage/in_memory.go +++ b/quesma/quesma/async_search_storage/in_memory.go @@ -41,10 +41,19 @@ func (s AsyncSearchStorageInMemory) Delete(id string) { s.idToResult.Delete(id) } -func (s AsyncSearchStorageInMemory) Size() int { +func (s AsyncSearchStorageInMemory) DocCount() int { return s.idToResult.Size() } +func (s AsyncSearchStorageInMemory) SizeInBytes() int { + size := 0 + s.Range(func(key string, value *AsyncRequestResult) bool { + size += len(value.GetResponseBody()) + return true + }) + return size +} + type AsyncQueryContextStorageInMemory struct { idToContext *concurrent.Map[string, *AsyncQueryContext] } diff --git a/quesma/quesma/async_search_storage/model.go b/quesma/quesma/async_search_storage/model.go index 86ceeef88..814b55920 100644 --- a/quesma/quesma/async_search_storage/model.go +++ b/quesma/quesma/async_search_storage/model.go @@ -9,10 +9,11 @@ import ( type AsyncRequestResultStorage interface { Store(id string, result *AsyncRequestResult) - Range(func(key string, value *AsyncRequestResult) bool) // ideally I'd like to get rid of this, but not sure if it's possible Load(id string) (*AsyncRequestResult, bool) Delete(id string) - Size() int + DocCount() int + SizeInBytes() uint64 + SizeInBytesLimit() uint64 } // TODO: maybe merge those 2? diff --git a/quesma/quesma/search.go b/quesma/quesma/search.go index 0942ea2d1..82aa07cc5 100644 --- a/quesma/quesma/search.go +++ b/quesma/quesma/search.go @@ -520,15 +520,6 @@ func (q *QueryRunner) storeAsyncSearch(qmc *ui.QuesmaManagementConsole, id, asyn return } -func (q *QueryRunner) asyncQueriesCumulatedBodySize() int { - size := 0 - q.AsyncRequestStorage.Range(func(key string, value *async_search_storage.AsyncRequestResult) bool { - size += len(value.GetResponseBody()) - return true - }) - return size -} - func (q *QueryRunner) handlePartialAsyncSearch(ctx context.Context, id string) ([]byte, error) { if !strings.Contains(id, tracing.AsyncIdPrefix) { logger.ErrorWithCtx(ctx).Msgf("non quesma async id: %v", id) @@ -573,7 +564,7 @@ func (q *QueryRunner) deleteAsyncSeach(id string) ([]byte, error) { } func (q *QueryRunner) reachedQueriesLimit(ctx context.Context, asyncId string, doneCh chan<- AsyncSearchWithError) bool { - if q.AsyncRequestStorage.Size() < asyncQueriesLimit && q.asyncQueriesCumulatedBodySize() < asyncQueriesLimitBytes { + if q.AsyncRequestStorage.DocCount() < asyncQueriesLimit && q.AsyncRequestStorage.SizeInBytes() < asyncQueriesLimitBytes { return false } err := errors.New("too many async queries") From 5928f10f84d74e7973e72e56267bada63f92a033 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Tue, 5 Nov 2024 16:55:03 +0100 Subject: [PATCH 02/30] Some more done --- quesma/persistence/elastic.go | 5 +- quesma/persistence/elastic_with_eviction.go | 122 ++++++++++++++++---- 2 files changed, 103 insertions(+), 24 deletions(-) diff --git a/quesma/persistence/elastic.go b/quesma/persistence/elastic.go index aa12fae76..73fa1adcb 100644 --- a/quesma/persistence/elastic.go +++ b/quesma/persistence/elastic.go @@ -118,11 +118,10 @@ func (p *ElasticJSONDatabase) List() ([]string, error) { }` resp, err := p.httpClient.Request(context.Background(), "GET", elasticsearchURL, []byte(query)) - + defer resp.Body.Close() if err != nil { return nil, err } - defer resp.Body.Close() jsonAsBytes, err := io.ReadAll(resp.Body) if err != nil { @@ -142,7 +141,7 @@ func (p *ElasticJSONDatabase) List() ([]string, error) { var ids []string // Unmarshal the JSON response var result map[string]interface{} - if err := json.Unmarshal(jsonAsBytes, &result); err != nil { + if err = json.Unmarshal(jsonAsBytes, &result); err != nil { log.Fatalf("Error parsing the response JSON: %s", err) } diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index 69199cccc..e6f23733f 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -2,17 +2,29 @@ // SPDX-License-Identifier: Elastic-2.0 package persistence -import "quesma/quesma/config" +import ( + "bytes" + "context" + "encoding/gob" + "encoding/json" + "fmt" + "io" + "net/http" + "quesma/logger" + "quesma/quesma/config" +) +const MAX_DOC_COUNT = 10000 // prototype TODO: fix/make configurable/idk/etc + +// so far I serialize entire struct and keep only 1 string in ES type ElasticDatabaseWithEviction struct { + ctx context.Context *ElasticJSONDatabase // maybe remove and copy fields here EvictorInterface sizeInBytesLimit int64 } -func NewElasticDatabaseWithEviction(cfg config.ElasticsearchConfiguration, - indexName string, sizeInBytesLimit int64) *ElasticDatabaseWithEviction { - +func NewElasticDatabaseWithEviction(ctx context.Context, cfg config.ElasticsearchConfiguration, indexName string, sizeInBytesLimit int64) *ElasticDatabaseWithEviction { return &ElasticDatabaseWithEviction{ ElasticJSONDatabase: NewElasticJSONDatabase(cfg, indexName), EvictorInterface: &Evictor{}, @@ -21,25 +33,42 @@ func NewElasticDatabaseWithEviction(cfg config.ElasticsearchConfiguration, } // mutexy? or what -func (db *ElasticDatabaseWithEviction) Put(row Sizeable) bool { +func (db *ElasticDatabaseWithEviction) Put(id string, row Sizeable) bool { bytesNeeded := db.SizeInBytes() + row.SizeInBytes() if bytesNeeded > db.SizeInBytesLimit() { - docsToEvict, bytesEvicted := db.SelectToEvict(db.getAll(), bytesNeeded-db.SizeInBytesLimit()) - db.evict(docsToEvict) - bytesNeeded -= bytesEvicted + logger.InfoWithCtx(db.ctx).Msg("Database is full, evicting documents") + //docsToEvict, bytesEvicted := db.SelectToEvict(db.getAll(), bytesNeeded-db.SizeInBytesLimit()) + //db.evict(docsToEvict) + //bytesNeeded -= bytesEvicted } - - if bytesNeeded <= db.SizeInBytesLimit() { + if bytesNeeded > db.SizeInBytesLimit() { // put document - return true + return false } - return false + + serialized, err := db.serialize(row) + if err != nil { + logger.WarnWithCtx(db.ctx).Msg("Error serializing document, id:" + id) + return false + } + + err = db.ElasticJSONDatabase.Put(id, serialized) + if err != nil { + logger.WarnWithCtx(db.ctx).Msgf("Error putting document, id: %s, error: %v", id, err) + return false + } + + return true } -func (db *ElasticDatabaseWithEviction) Get(id string) (*Sizeable, bool) { - // either use ElasticJSONDatabase.Get or implement own - // doesn't matter - return nil, false +// co zwraca? zrobić switch na oba typy jakie teraz mamy? +func (db *ElasticDatabaseWithEviction) Get(id string) (string, bool) { // probably change return type to *Sizeable + value, success, err := db.ElasticJSONDatabase.Get(id) + if err != nil { + logger.WarnWithCtx(db.ctx).Msgf("Error getting document, id: %s, error: %v", id, err) + return "", false + } + return value, success } func (db *ElasticDatabaseWithEviction) Delete(id string) { @@ -47,13 +76,52 @@ func (db *ElasticDatabaseWithEviction) Delete(id string) { // (single document deletion is hard in ES, it's done by evictor for entire index) } -func (db *ElasticDatabaseWithEviction) DocCount() int { - // send count() query to ES - return 0 +func (db *ElasticDatabaseWithEviction) DocCount() (count int, success bool) { + // TODO: add WHERE not_deleted + + // Build the query to get only document IDs + elasticsearchURL := fmt.Sprintf("%s/_search", db.indexName) + query := `{ + "_source": false, + "size": 0, + "track_total_hits": true + }` + + resp, err := db.httpClient.Request(context.Background(), "GET", elasticsearchURL, []byte(query)) + defer resp.Body.Close() + if err != nil { + return + } + + jsonAsBytes, err := io.ReadAll(resp.Body) + if err != nil { + return + } + + switch resp.StatusCode { + case http.StatusOK: + break + default: + logger.WarnWithCtx(db.ctx).Msgf("failed to get from elastic: %s, response status code: %v", string(jsonAsBytes), resp.StatusCode) + return + } + + // Unmarshal the JSON response + var result map[string]interface{} + if err = json.Unmarshal(jsonAsBytes, &result); err != nil { + logger.WarnWithCtx(db.ctx).Msgf("Error parsing the response JSON: %s", err) + return + } + + count = int(result["hits"].(map[string]interface{})["total"].(map[string]interface{})["value"].(float64)) // TODO: add some checks... to prevent panic + return count, true } -func (db *ElasticDatabaseWithEviction) SizeInBytes() int64 { - return 0 +func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, success bool) { + elasticsearchURL := fmt.Sprintf("%s/_search", db.indexName) + + // Build the query to get only document IDs + query := fmt.Sprintf(`{"_source": false, "size": %d}`, MAX_DOC_COUNT) } func (db *ElasticDatabaseWithEviction) SizeInBytesLimit() int64 { @@ -68,3 +136,15 @@ func (db *ElasticDatabaseWithEviction) getAll() *basicDocumentInfo { func (db *ElasticDatabaseWithEviction) evict(documents []*basicDocumentInfo) { } + +func (db *ElasticDatabaseWithEviction) serialize(row Sizeable) (serialized string, err error) { + var b bytes.Buffer + + enc := gob.NewEncoder(&b) // maybe create 1 encoder forever + if err = enc.Encode(row); err != nil { + fmt.Println("Error encoding struct:", err) + return + } + + return b.String(), nil +} From 36613616000870d5432e4663ec454c9ff62a6b06 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Fri, 8 Nov 2024 13:00:29 +0100 Subject: [PATCH 03/30] Before reverse to 1 index --- quesma/persistence/elastic_with_eviction.go | 222 +++++++++++++++++--- quesma/persistence/evictor.go | 30 ++- quesma/persistence/model.go | 32 ++- quesma/persistence/persistence_test.go | 180 +++++++++++++++- 4 files changed, 417 insertions(+), 47 deletions(-) diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index e6f23733f..b511dbd65 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -3,18 +3,19 @@ package persistence import ( - "bytes" "context" - "encoding/gob" "encoding/json" "fmt" "io" "net/http" "quesma/logger" "quesma/quesma/config" + "quesma/quesma/types" + "time" ) -const MAX_DOC_COUNT = 10000 // prototype TODO: fix/make configurable/idk/etc +const MAX_DOC_COUNT = 10000 // prototype TODO: fix/make configurable/idk/etc +const defaultSizeInBytesLimit = int64(1_000_000_000) // 1GB // so far I serialize entire struct and keep only 1 string in ES type ElasticDatabaseWithEviction struct { @@ -26,6 +27,7 @@ type ElasticDatabaseWithEviction struct { func NewElasticDatabaseWithEviction(ctx context.Context, cfg config.ElasticsearchConfiguration, indexName string, sizeInBytesLimit int64) *ElasticDatabaseWithEviction { return &ElasticDatabaseWithEviction{ + ctx: ctx, ElasticJSONDatabase: NewElasticJSONDatabase(cfg, indexName), EvictorInterface: &Evictor{}, sizeInBytesLimit: sizeInBytesLimit, @@ -33,32 +35,61 @@ func NewElasticDatabaseWithEviction(ctx context.Context, cfg config.Elasticsearc } // mutexy? or what -func (db *ElasticDatabaseWithEviction) Put(id string, row Sizeable) bool { - bytesNeeded := db.SizeInBytes() + row.SizeInBytes() +func (db *ElasticDatabaseWithEviction) Put(doc *document) bool { + dbSize, success := db.SizeInBytes() + if !success { + return false + } + fmt.Println("kk dbg Put() dbSize:", dbSize) + bytesNeeded := dbSize + doc.SizeInBytes if bytesNeeded > db.SizeInBytesLimit() { - logger.InfoWithCtx(db.ctx).Msg("Database is full, evicting documents") - //docsToEvict, bytesEvicted := db.SelectToEvict(db.getAll(), bytesNeeded-db.SizeInBytesLimit()) - //db.evict(docsToEvict) - //bytesNeeded -= bytesEvicted + logger.InfoWithCtx(db.ctx).Msgf("Database is full, need %d bytes more. Evicting documents", bytesNeeded-db.SizeInBytesLimit()) + allDocs, ok := db.getAll() + if !ok { + logger.WarnWithCtx(db.ctx).Msg("Error getting all documents") + return false + } + indexesToEvict, bytesEvicted := db.SelectToEvict(allDocs, bytesNeeded-db.SizeInBytesLimit()) + logger.InfoWithCtx(db.ctx).Msgf("Evicting %v indexes, %d bytes", indexesToEvict, bytesEvicted) + db.evict(indexesToEvict) + bytesNeeded -= bytesEvicted } if bytesNeeded > db.SizeInBytesLimit() { // put document return false } - serialized, err := db.serialize(row) + //elasticsearchURL := fmt.Sprintf("%s/_update/%s", db.fullIndexName(), doc.Id) + elasticsearchURL := fmt.Sprintf("%s/_update/%s", db.indexName, doc.Id) + fmt.Println("kk dbg Put() elasticsearchURL:", elasticsearchURL) + + updateContent := types.JSON{} + updateContent["doc"] = doc + updateContent["doc_as_upsert"] = true + + jsonData, err := json.Marshal(updateContent) if err != nil { - logger.WarnWithCtx(db.ctx).Msg("Error serializing document, id:" + id) + logger.WarnWithCtx(db.ctx).Msgf("Error marshalling document: %v", err) return false } - err = db.ElasticJSONDatabase.Put(id, serialized) + resp, err := db.httpClient.Request(context.Background(), "POST", elasticsearchURL, jsonData) if err != nil { - logger.WarnWithCtx(db.ctx).Msgf("Error putting document, id: %s, error: %v", id, err) + logger.WarnWithCtx(db.ctx).Msgf("Error sending request to elastic: %v", err) return false } + defer resp.Body.Close() - return true + switch resp.StatusCode { + case http.StatusCreated, http.StatusOK: + return true + default: + respBody, err := io.ReadAll(resp.Body) + if err != nil { + logger.WarnWithCtx(db.ctx).Msgf("Error reading response body: %v, respBody: %v", err, respBody) + } + return false + } } // co zwraca? zrobić switch na oba typy jakie teraz mamy? @@ -71,36 +102,75 @@ func (db *ElasticDatabaseWithEviction) Get(id string) (string, bool) { // probab return value, success } -func (db *ElasticDatabaseWithEviction) Delete(id string) { +func (db *ElasticDatabaseWithEviction) Delete(id string) bool { // mark as deleted, don't actually delete // (single document deletion is hard in ES, it's done by evictor for entire index) + + // TODO: check if doc exists? + elasticsearchURL := fmt.Sprintf("%s/_update/%s", db.indexName, id) + + updateContent := types.JSON{} + updateContent["doc"] = types.JSON{"markedAsDeleted": true} + updateContent["doc_as_upsert"] = true + + jsonData, err := json.Marshal(updateContent) + if err != nil { + logger.WarnWithCtx(db.ctx).Msgf("Error marshalling document: %v", err) + return false + } + + resp, err := db.httpClient.Request(context.Background(), "POST", elasticsearchURL, jsonData) + if err != nil { + logger.WarnWithCtx(db.ctx).Msgf("Error sending request to elastic: %v", err) + return false + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusCreated, http.StatusOK: + return true + default: + respBody, err := io.ReadAll(resp.Body) + if err != nil { + logger.WarnWithCtx(db.ctx).Msgf("Error reading response body: %v, respBody: %v", err, respBody) + } + return false + } } func (db *ElasticDatabaseWithEviction) DocCount() (count int, success bool) { - // TODO: add WHERE not_deleted - - // Build the query to get only document IDs elasticsearchURL := fmt.Sprintf("%s/_search", db.indexName) query := `{ "_source": false, "size": 0, - "track_total_hits": true + "track_total_hits": true, + "query": { + "term": { + "markedAsDeleted": { + "value": false + } + } + } }` resp, err := db.httpClient.Request(context.Background(), "GET", elasticsearchURL, []byte(query)) - defer resp.Body.Close() if err != nil { return } + defer resp.Body.Close() jsonAsBytes, err := io.ReadAll(resp.Body) if err != nil { return } + fmt.Println("kk dbg DocCount() resp.StatusCode:", resp.StatusCode) + switch resp.StatusCode { case http.StatusOK: break + case http.StatusNoContent, http.StatusNotFound: + return 0, true default: logger.WarnWithCtx(db.ctx).Msgf("failed to get from elastic: %s, response status code: %v", string(jsonAsBytes), resp.StatusCode) return @@ -113,38 +183,122 @@ func (db *ElasticDatabaseWithEviction) DocCount() (count int, success bool) { return } + fmt.Println("kk dbg DocCount() result:", result) + count = int(result["hits"].(map[string]interface{})["total"].(map[string]interface{})["value"].(float64)) // TODO: add some checks... to prevent panic return count, true } func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, success bool) { elasticsearchURL := fmt.Sprintf("%s/_search", db.indexName) + query := `{ + "_source": ["sizeInBytes"], + "size": 10000, + "track_total_hits": true + }` - // Build the query to get only document IDs - query := fmt.Sprintf(`{"_source": false, "size": %d}`, MAX_DOC_COUNT) + resp, err := db.httpClient.Request(context.Background(), "GET", elasticsearchURL, []byte(query)) + if err != nil { + return + } + defer resp.Body.Close() + + jsonAsBytes, err := io.ReadAll(resp.Body) + if err != nil { + return + } + + fmt.Println("kk dbg SizeInBytes() resp.StatusCode:", resp.StatusCode) + + switch resp.StatusCode { + case http.StatusOK: + break + case http.StatusNoContent, http.StatusNotFound: + return 0, true + default: + logger.WarnWithCtx(db.ctx).Msgf("failed to get from elastic: %s, response status code: %v", string(jsonAsBytes), resp.StatusCode) + return + } + + // Unmarshal the JSON response + var result map[string]interface{} + if err = json.Unmarshal(jsonAsBytes, &result); err != nil { + logger.WarnWithCtx(db.ctx).Msgf("Error parsing the response JSON: %s", err) + return + } + + a := make([]int64, 0) + for _, hit := range result["hits"].(map[string]interface{})["hits"].([]interface{}) { + b := sizeInBytes + sizeInBytes += int64(hit.(map[string]interface{})["_source"].(map[string]interface{})["sizeInBytes"].(float64)) // TODO: add checks + a = append(a, sizeInBytes-b) + } + fmt.Println("kk dbg SizeInBytes() sizes in storage:", a) + return sizeInBytes, true } func (db *ElasticDatabaseWithEviction) SizeInBytesLimit() int64 { return db.sizeInBytesLimit } -func (db *ElasticDatabaseWithEviction) getAll() *basicDocumentInfo { - // send query - return nil -} +func (db *ElasticDatabaseWithEviction) getAll() (documents []*document, success bool) { + elasticsearchURL := fmt.Sprintf("%s*/_search", db.indexName) + query := `{ + "_source": { + "excludes": "data" + }, + "size": 10000, + "track_total_hits": true + }` + + resp, err := db.httpClient.Request(context.Background(), "GET", elasticsearchURL, []byte(query)) + if err != nil { + return + } + defer resp.Body.Close() -func (db *ElasticDatabaseWithEviction) evict(documents []*basicDocumentInfo) { + jsonAsBytes, err := io.ReadAll(resp.Body) + if err != nil { + return + } -} + fmt.Println("kk dbg getAll() resp.StatusCode:", resp.StatusCode) -func (db *ElasticDatabaseWithEviction) serialize(row Sizeable) (serialized string, err error) { - var b bytes.Buffer + switch resp.StatusCode { + case http.StatusOK: + break + default: + logger.WarnWithCtx(db.ctx).Msgf("failed to get from elastic: %s, response status code: %v", string(jsonAsBytes), resp.StatusCode) + return + } - enc := gob.NewEncoder(&b) // maybe create 1 encoder forever - if err = enc.Encode(row); err != nil { - fmt.Println("Error encoding struct:", err) + // Unmarshal the JSON response + var result map[string]interface{} + if err = json.Unmarshal(jsonAsBytes, &result); err != nil { + logger.WarnWithCtx(db.ctx).Msgf("Error parsing the response JSON: %s", err) return } - return b.String(), nil + fmt.Println("kk dbg getAll() documents:") + for _, hit := range result["hits"].(map[string]interface{})["hits"].([]interface{}) { + doc := &document{ + Id: hit.(map[string]interface{})["_id"].(string), + Index: hit.(map[string]interface{})["_index"].(string), + SizeInBytes: int64(hit.(map[string]interface{})["_source"].(map[string]interface{})["sizeInBytes"].(float64)), // TODO: add checks + //Timestamp: hit.(map[string]interface{})["_source"].(map[string]interface{})["timestamp"].(time.Time), // TODO: add checks + MarkedAsDeleted: hit.(map[string]interface{})["_source"].(map[string]interface{})["markedAsDeleted"].(bool), // TODO: add checks + } + fmt.Println(doc) + documents = append(documents, doc) + } + return documents, true +} + +func (db *ElasticDatabaseWithEviction) evict(indexes []string) { + // todo +} + +func (db *ElasticDatabaseWithEviction) fullIndexName() string { + now := time.Now().UTC() + return fmt.Sprintf("%s-%d-%d-%d-%d-%d-%d", db.indexName, now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second()) } diff --git a/quesma/persistence/evictor.go b/quesma/persistence/evictor.go index f33cb5258..18f4dd3af 100644 --- a/quesma/persistence/evictor.go +++ b/quesma/persistence/evictor.go @@ -2,17 +2,43 @@ // SPDX-License-Identifier: Elastic-2.0 package persistence +import "fmt" + type EvictorInterface interface { - SelectToEvict(documents []*basicDocumentInfo, sizeNeeded int64) (evictThese []*basicDocumentInfo, bytesEvicted int64) + SelectToEvict(documents []*document, sizeNeeded int64) (indexesToEvict []string, bytesEvicted int64) } // It's only 1 implementation, which looks well suited for ElasticSearch. // It can be implemented differently. type Evictor struct{} -func (e *Evictor) SelectToEvict(documents []*basicDocumentInfo, sizeNeeded int64) (evictThese []*basicDocumentInfo, bytesEvicted int64) { +func (e *Evictor) SelectToEvict(documents []*document, sizeNeeded int64) (indexesToEvict []string, bytesEvicted int64) { if sizeNeeded <= 0 { return // check if it's empty array or nil } + fmt.Println("kk dbg SelectToEvict() sizeNeeded:", sizeNeeded) + + countByIndex := make(map[string]int) + countByIndexMarkedAsDeleted := make(map[string]int) + + for _, doc := range documents { + countByIndex[doc.Index]++ + if doc.MarkedAsDeleted { + countByIndexMarkedAsDeleted[doc.Index]++ + } + } + + for index, markedAsDeletedCnt := range countByIndexMarkedAsDeleted { + if countByIndex[index] == markedAsDeletedCnt { + indexesToEvict = append(indexesToEvict, index) + } + } + + for _, doc := range documents { + if countByIndex[doc.Index] == countByIndexMarkedAsDeleted[doc.Index] { + bytesEvicted += doc.SizeInBytes + } + } + return } diff --git a/quesma/persistence/model.go b/quesma/persistence/model.go index 85fad970f..7b4eecbe7 100644 --- a/quesma/persistence/model.go +++ b/quesma/persistence/model.go @@ -20,21 +20,33 @@ type JSONDatabase interface { // T - type of the data to store, e.g. async_search_storage.AsyncRequestResult type JSONDatabaseWithEviction interface { // for sure JSON? maybe not only json? check - Put(row *Sizeable) error - Get(id string) (*Sizeable, bool) + Put(doc document) bool + Get(id string) (document, bool) Delete(id string) - DocCount() int - SizeInBytes() int64 + DocCount() (int, bool) + SizeInBytes() (int64, bool) SizeInBytesLimit() int64 } +type document struct { + Id string `json:"id"` + Data string `json:"data"` + Index string `json:"index,omitempty"` + SizeInBytes int64 `json:"sizeInBytes"` + Timestamp time.Time `json:"timestamp"` + MarkedAsDeleted bool `json:"markedAsDeleted"` +} + +/* type basicDocumentInfo struct { - id string - sizeInBytes int64 - timestamp time.Time - markedAsDeleted bool + Id string + SizeInBytes int64 + Timestamp time.Time + MarkedAsDeleted bool } -type Sizeable interface { - SizeInBytes() int64 +// mb remove or change impl +func (d *basicDocumentInfo) SizeInBytes() int64 { + return d.SizeInBytesTotal } +*/ diff --git a/quesma/persistence/persistence_test.go b/quesma/persistence/persistence_test.go index 1bde8e446..f841c0b5e 100644 --- a/quesma/persistence/persistence_test.go +++ b/quesma/persistence/persistence_test.go @@ -3,8 +3,11 @@ package persistence import ( + "context" "fmt" + "github.com/stretchr/testify/assert" "net/url" + "quesma/logger" "quesma/quesma/config" "quesma/quesma/types" "testing" @@ -16,7 +19,7 @@ func TestNewElasticPersistence(t *testing.T) { var p JSONDatabase // change to false if you want to test non-trivial persistence - if true { + if false { p = NewStaticJSONDatabase() } else { indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMicro()) @@ -36,6 +39,7 @@ func TestNewElasticPersistence(t *testing.T) { } p = NewElasticJSONDatabase(cfg, indexName) + fmt.Println("??") } m1 := make(types.JSON) @@ -80,3 +84,177 @@ func TestNewElasticPersistence(t *testing.T) { } } + +func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { + const precise = true + + logger.InitSimpleLoggerForTests() + indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) + fmt.Println("indexName:", indexName) + + realUrl, err := url.Parse("http://localhost:9200") + assert.NoError(t, err) + + cfgUrl := config.Url(*realUrl) + cfg := config.ElasticsearchConfiguration{ + Url: &cfgUrl, + User: "", + Password: "", + } + + const bigSizeLimit = int64(1_000_000_000) + db := NewElasticDatabaseWithEviction(context.Background(), cfg, indexName, bigSizeLimit) + + // check initial state + assert.Equal(t, bigSizeLimit, db.SizeInBytesLimit()) + + docCount, ok := db.DocCount() + assert.True(t, ok) + assert.Equal(t, 0, docCount) + + sizeInBytes, ok := db.SizeInBytes() + assert.True(t, ok) + assert.Equal(t, int64(0), sizeInBytes) + + // put first documents + docs := []*document{ + doc("doc1", 100), + doc("doc2", 200), + doc("doc3", 300), + doc("doc4", 400), + doc("doc5", 500), + } + for _, d := range docs { + assert.True(t, db.Put(d)) + } + + if precise { + time.Sleep(4 * time.Second) + docCount, ok = db.DocCount() + assert.True(t, ok) + assert.Equal(t, 5, docCount) + } else { + docCount, ok = db.DocCount() + assert.True(t, ok) + assert.True(t, docCount >= 0) + } + + val, ok := db.Get(docs[0].Id) + fmt.Println(val, ok) + // TODO: deserialize and check content + + db.Delete(docs[1].Id) + db.Delete(docs[3].Id) + + if precise { + time.Sleep(1 * time.Second) + docCount, ok = db.DocCount() + assert.True(t, ok) + assert.Equal(t, 3, docCount) + } else { + docCount, ok = db.DocCount() + assert.True(t, ok) + assert.True(t, docCount >= 0) + } + + assert.Equal(t, bigSizeLimit, db.SizeInBytesLimit()) +} + +const updateTime = 4 * time.Second + +func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { + logger.InitSimpleLoggerForTests() + indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) + + realUrl, err := url.Parse("http://localhost:9200") + assert.NoError(t, err) + + cfgUrl := config.Url(*realUrl) + cfg := config.ElasticsearchConfiguration{ + Url: &cfgUrl, + User: "", + Password: "", + } + + const smallSizeLimit = int64(1200) + db := NewElasticDatabaseWithEviction(context.Background(), cfg, indexName, smallSizeLimit) + fmt.Println("indexName:", indexName, "fullIndexName:", db.fullIndexName()) + + // check initial state + assert.Equal(t, smallSizeLimit, db.SizeInBytesLimit()) + + docCount, ok := db.DocCount() + assert.True(t, ok) + assert.Equal(t, 0, docCount) + + sizeInBytes, ok := db.SizeInBytes() + assert.True(t, ok) + assert.Equal(t, int64(0), sizeInBytes) + + // put first documents + docs := []*document{ + doc("doc1", 200), + doc("doc2", 300), + doc("doc3", 400), + doc("doc4", 500), + doc("doc5", 500), + } + for _, d := range docs[:2] { + fmt.Println("put", d.SizeInBytes, db.Put(d)) + } + time.Sleep(updateTime) + fmt.Println("put", docs[2].SizeInBytes, db.Put(docs[2])) + time.Sleep(updateTime) + + docCount, ok = db.DocCount() + assert.True(t, ok) + assert.Equal(t, 3, docCount) + + db.Delete("doc2") + time.Sleep(updateTime) + + docCount, ok = db.DocCount() + assert.True(t, ok) + assert.Equal(t, 2, docCount) + + put4 := db.Put(docs[4]) + fmt.Println("put", docs[4].SizeInBytes, put4) + assert.False(t, put4) + + time.Sleep(3000 * time.Millisecond) + + docCount, ok = db.DocCount() + assert.True(t, ok) + assert.Equal(t, 3, docCount) + + // + /* + val, ok := db.Get(docs[0].Id) + fmt.Println(val, ok) + // TODO: deserialize and check content + + db.Delete(docs[1].Id) + db.Delete(docs[3].Id) + + time.Sleep(1 * time.Second) + docCount, ok = db.DocCount() + assert.True(t, ok) + assert.Equal(t, 3, docCount) + } else { + docCount, ok = db.DocCount() + assert.True(t, ok) + assert.True(t, docCount >= 0) + } + + + */ + assert.Equal(t, smallSizeLimit, db.SizeInBytesLimit()) +} + +func doc(id string, size int64) *document { + return &document{ + Id: id, + SizeInBytes: size, + Timestamp: time.Now(), + } +} From 56245b8cc8ea49be56fef112f868e0bf95fcd288 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Sun, 10 Nov 2024 16:54:15 +0100 Subject: [PATCH 04/30] some more --- quesma/persistence/elastic_with_eviction.go | 24 +++--- quesma/persistence/model.go | 1 - quesma/quesma/async_search_storage/evictor.go | 43 +++++++++++ .../quesma/async_search_storage/in_elastic.go | 61 ++++++++++++++- .../quesma/async_search_storage/in_memory.go | 74 ++++++------------- quesma/quesma/async_search_storage/model.go | 7 +- 6 files changed, 143 insertions(+), 67 deletions(-) create mode 100644 quesma/quesma/async_search_storage/evictor.go diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index b511dbd65..76ee9fd50 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -14,7 +14,7 @@ import ( "time" ) -const MAX_DOC_COUNT = 10000 // prototype TODO: fix/make configurable/idk/etc +const MAX_DOC_COUNT = 10000 // TODO: fix/make configurable/idk/etc const defaultSizeInBytesLimit = int64(1_000_000_000) // 1GB // so far I serialize entire struct and keep only 1 string in ES @@ -25,9 +25,9 @@ type ElasticDatabaseWithEviction struct { sizeInBytesLimit int64 } -func NewElasticDatabaseWithEviction(ctx context.Context, cfg config.ElasticsearchConfiguration, indexName string, sizeInBytesLimit int64) *ElasticDatabaseWithEviction { +func NewElasticDatabaseWithEviction(cfg config.ElasticsearchConfiguration, indexName string, sizeInBytesLimit int64) *ElasticDatabaseWithEviction { return &ElasticDatabaseWithEviction{ - ctx: ctx, + ctx: context.Background(), ElasticJSONDatabase: NewElasticJSONDatabase(cfg, indexName), EvictorInterface: &Evictor{}, sizeInBytesLimit: sizeInBytesLimit, @@ -35,7 +35,7 @@ func NewElasticDatabaseWithEviction(ctx context.Context, cfg config.Elasticsearc } // mutexy? or what -func (db *ElasticDatabaseWithEviction) Put(doc *document) bool { +func (db *ElasticDatabaseWithEviction) Put(ctx context.Context, doc *document) bool { dbSize, success := db.SizeInBytes() if !success { return false @@ -43,14 +43,14 @@ func (db *ElasticDatabaseWithEviction) Put(doc *document) bool { fmt.Println("kk dbg Put() dbSize:", dbSize) bytesNeeded := dbSize + doc.SizeInBytes if bytesNeeded > db.SizeInBytesLimit() { - logger.InfoWithCtx(db.ctx).Msgf("Database is full, need %d bytes more. Evicting documents", bytesNeeded-db.SizeInBytesLimit()) + logger.InfoWithCtx(ctx).Msgf("Database is full, need %d bytes more. Evicting documents", bytesNeeded-db.SizeInBytesLimit()) allDocs, ok := db.getAll() if !ok { - logger.WarnWithCtx(db.ctx).Msg("Error getting all documents") + logger.WarnWithCtx(ctx).Msg("Error getting all documents") return false } indexesToEvict, bytesEvicted := db.SelectToEvict(allDocs, bytesNeeded-db.SizeInBytesLimit()) - logger.InfoWithCtx(db.ctx).Msgf("Evicting %v indexes, %d bytes", indexesToEvict, bytesEvicted) + logger.InfoWithCtx(ctx).Msgf("Evicting %v indexes, %d bytes", indexesToEvict, bytesEvicted) db.evict(indexesToEvict) bytesNeeded -= bytesEvicted } @@ -69,13 +69,13 @@ func (db *ElasticDatabaseWithEviction) Put(doc *document) bool { jsonData, err := json.Marshal(updateContent) if err != nil { - logger.WarnWithCtx(db.ctx).Msgf("Error marshalling document: %v", err) + logger.WarnWithCtx(ctx).Msgf("Error marshalling document: %v", err) return false } resp, err := db.httpClient.Request(context.Background(), "POST", elasticsearchURL, jsonData) if err != nil { - logger.WarnWithCtx(db.ctx).Msgf("Error sending request to elastic: %v", err) + logger.WarnWithCtx(ctx).Msgf("Error sending request to elastic: %v", err) return false } defer resp.Body.Close() @@ -86,17 +86,17 @@ func (db *ElasticDatabaseWithEviction) Put(doc *document) bool { default: respBody, err := io.ReadAll(resp.Body) if err != nil { - logger.WarnWithCtx(db.ctx).Msgf("Error reading response body: %v, respBody: %v", err, respBody) + logger.WarnWithCtx(ctx).Msgf("Error reading response body: %v, respBody: %v", err, respBody) } return false } } // co zwraca? zrobić switch na oba typy jakie teraz mamy? -func (db *ElasticDatabaseWithEviction) Get(id string) (string, bool) { // probably change return type to *Sizeable +func (db *ElasticDatabaseWithEviction) Get(ctx context.Context, id string) (string, bool) { // probably change return type to *Sizeable value, success, err := db.ElasticJSONDatabase.Get(id) if err != nil { - logger.WarnWithCtx(db.ctx).Msgf("Error getting document, id: %s, error: %v", id, err) + logger.WarnWithCtx(ctx).Msgf("Error getting document, id: %s, error: %v", id, err) return "", false } return value, success diff --git a/quesma/persistence/model.go b/quesma/persistence/model.go index 7b4eecbe7..d93f61ed6 100644 --- a/quesma/persistence/model.go +++ b/quesma/persistence/model.go @@ -18,7 +18,6 @@ type JSONDatabase interface { Put(key string, data string) error } -// T - type of the data to store, e.g. async_search_storage.AsyncRequestResult type JSONDatabaseWithEviction interface { // for sure JSON? maybe not only json? check Put(doc document) bool Get(id string) (document, bool) diff --git a/quesma/quesma/async_search_storage/evictor.go b/quesma/quesma/async_search_storage/evictor.go new file mode 100644 index 000000000..83b266e4b --- /dev/null +++ b/quesma/quesma/async_search_storage/evictor.go @@ -0,0 +1,43 @@ +package async_search_storage + +import ( + "context" + "quesma/logger" + "quesma/quesma/recovery" + "time" +) + +type AsyncQueriesEvictor struct { + ctx context.Context + cancel context.CancelFunc + AsyncRequestStorage AsyncSearchStorageInMemory + AsyncQueriesContexts AsyncQueryContextStorageInMemory +} + +func NewAsyncQueriesEvictor(AsyncRequestStorage AsyncSearchStorageInMemory, AsyncQueriesContexts AsyncQueryContextStorageInMemory) *AsyncQueriesEvictor { + ctx, cancel := context.WithCancel(context.Background()) + return &AsyncQueriesEvictor{ctx: ctx, cancel: cancel, AsyncRequestStorage: AsyncRequestStorage, AsyncQueriesContexts: AsyncQueriesContexts} +} + +func (e *AsyncQueriesEvictor) tryEvictAsyncRequests(timeFun func(time.Time) time.Duration) { + e.AsyncRequestStorage.evict(timeFun) + e.AsyncQueriesContexts.evict(timeFun) +} + +func (e *AsyncQueriesEvictor) AsyncQueriesGC() { + defer recovery.LogPanic() + for { + select { + case <-e.ctx.Done(): + logger.Debug().Msg("evictor stopped") + return + case <-time.After(GCInterval): + e.tryEvictAsyncRequests(elapsedTime) + } + } +} + +func (e *AsyncQueriesEvictor) Close() { + e.cancel() + logger.Info().Msg("AsyncQueriesEvictor Stopped") +} diff --git a/quesma/quesma/async_search_storage/in_elastic.go b/quesma/quesma/async_search_storage/in_elastic.go index 2575a1057..64031e18d 100644 --- a/quesma/quesma/async_search_storage/in_elastic.go +++ b/quesma/quesma/async_search_storage/in_elastic.go @@ -2,4 +2,63 @@ // SPDX-License-Identifier: Elastic-2.0 package async_search_storage -// TODO :( +import ( + "context" + "quesma/persistence" + "quesma/quesma/config" +) + +type AsyncSearchStorageInElastic struct { + db *persistence.ElasticDatabaseWithEviction +} + +func NewAsyncSearchStorageInElastic() AsyncSearchStorageInElastic { + return AsyncSearchStorageInElastic{ + db: persistence.NewElasticDatabaseWithEviction( + config.ElasticsearchConfiguration{}, "async_search", 1_000_000_000), + } +} + +func (s AsyncSearchStorageInElastic) Store(ctx context.Context, id string, result *AsyncRequestResult) { + s.db.Put(ctx, nil) +} + +func (s AsyncSearchStorageInElastic) Load(ctx context.Context, id string) (*AsyncRequestResult, bool) { + _, ok := s.db.Get(ctx, id) + return nil, ok +} + +func (s AsyncSearchStorageInElastic) Delete(id string) { + s.db.Delete(id) +} + +func (s AsyncSearchStorageInElastic) DocCount() int { + cnt, ok := s.db.DocCount() + if !ok { + return -1 + } + return cnt +} + +func (s AsyncSearchStorageInElastic) SizeInBytes() int64 { + size, ok := s.db.SizeInBytes() + if !ok { + return -1 + } + return size +} + +type AsyncQueryContextStorageInElastic struct { + db *persistence.ElasticDatabaseWithEviction +} + +func NewAsyncQueryContextStorageInElastic() AsyncQueryContextStorageInElastic { + return AsyncQueryContextStorageInElastic{ + db: persistence.NewElasticDatabaseWithEviction( + config.ElasticsearchConfiguration{}, "async_search", 1_000_000_000), + } +} + +func (s AsyncQueryContextStorageInElastic) Store(ctx context.Context, id string, context *AsyncQueryContext) { + s.db.Put(ctx, nil) +} diff --git a/quesma/quesma/async_search_storage/in_memory.go b/quesma/quesma/async_search_storage/in_memory.go index ecd3970dd..1821af08e 100644 --- a/quesma/quesma/async_search_storage/in_memory.go +++ b/quesma/quesma/async_search_storage/in_memory.go @@ -12,14 +12,11 @@ import ( "time" ) -const EvictionInterval = 15 * time.Minute -const GCInterval = 1 * time.Minute - type AsyncSearchStorageInMemory struct { idToResult *concurrent.Map[string, *AsyncRequestResult] } -func NewAsyncSearchStorageInMemory() AsyncSearchStorageInMemory { +func NewAsyncSearchStorageInMemory() AsyncSearchStorageInMemory { // change result type to AsyncRequestResultStorage interface return AsyncSearchStorageInMemory{ idToResult: concurrent.NewMap[string, *AsyncRequestResult](), } @@ -54,6 +51,19 @@ func (s AsyncSearchStorageInMemory) SizeInBytes() int { return size } +func (s AsyncSearchStorageInMemory) evict(timeFun func(time.Time) time.Duration) { + var ids []asyncQueryIdWithTime + s.Range(func(key string, value *AsyncRequestResult) bool { + if timeFun(value.added) > EvictionInterval { + ids = append(ids, asyncQueryIdWithTime{id: key, time: value.added}) + } + return true + }) + for _, id := range ids { + s.Delete(id.id) + } +} + type AsyncQueryContextStorageInMemory struct { idToContext *concurrent.Map[string, *AsyncQueryContext] } @@ -68,40 +78,9 @@ func (s AsyncQueryContextStorageInMemory) Store(id string, context *AsyncQueryCo s.idToContext.Store(id, context) } -type AsyncQueriesEvictor struct { - ctx context.Context - cancel context.CancelFunc - AsyncRequestStorage AsyncSearchStorageInMemory - AsyncQueriesContexts AsyncQueryContextStorageInMemory -} - -func NewAsyncQueriesEvictor(AsyncRequestStorage AsyncSearchStorageInMemory, AsyncQueriesContexts AsyncQueryContextStorageInMemory) *AsyncQueriesEvictor { - ctx, cancel := context.WithCancel(context.Background()) - return &AsyncQueriesEvictor{ctx: ctx, cancel: cancel, AsyncRequestStorage: AsyncRequestStorage, AsyncQueriesContexts: AsyncQueriesContexts} -} - -func elapsedTime(t time.Time) time.Duration { - return time.Since(t) -} - -type asyncQueryIdWithTime struct { - id string - time time.Time -} - -func (e *AsyncQueriesEvictor) tryEvictAsyncRequests(timeFun func(time.Time) time.Duration) { - var ids []asyncQueryIdWithTime - e.AsyncRequestStorage.Range(func(key string, value *AsyncRequestResult) bool { - if timeFun(value.added) > EvictionInterval { - ids = append(ids, asyncQueryIdWithTime{id: key, time: value.added}) - } - return true - }) - for _, id := range ids { - e.AsyncRequestStorage.idToResult.Delete(id.id) - } +func (s AsyncQueryContextStorageInMemory) evict(timeFun func(time.Time) time.Duration) { var asyncQueriesContexts []*AsyncQueryContext - e.AsyncQueriesContexts.idToContext.Range(func(key string, value *AsyncQueryContext) bool { + s.idToContext.Range(func(key string, value *AsyncQueryContext) bool { if timeFun(value.added) > EvictionInterval { if value != nil { asyncQueriesContexts = append(asyncQueriesContexts, value) @@ -111,7 +90,7 @@ func (e *AsyncQueriesEvictor) tryEvictAsyncRequests(timeFun func(time.Time) time }) evictedIds := make([]string, 0) for _, asyncQueryContext := range asyncQueriesContexts { - e.AsyncQueriesContexts.idToContext.Delete(asyncQueryContext.id) + s.idToContext.Delete(asyncQueryContext.id) if asyncQueryContext.cancel != nil { evictedIds = append(evictedIds, asyncQueryContext.id) asyncQueryContext.cancel() @@ -122,22 +101,13 @@ func (e *AsyncQueriesEvictor) tryEvictAsyncRequests(timeFun func(time.Time) time } } -func (e *AsyncQueriesEvictor) AsyncQueriesGC() { - defer recovery.LogPanic() - for { - select { - case <-e.ctx.Done(): - logger.Debug().Msg("evictor stopped") - return - case <-time.After(GCInterval): - e.tryEvictAsyncRequests(elapsedTime) - } - } +func elapsedTime(t time.Time) time.Duration { + return time.Since(t) } -func (e *AsyncQueriesEvictor) Close() { - e.cancel() - logger.Info().Msg("AsyncQueriesEvictor Stopped") +type asyncQueryIdWithTime struct { + id string + time time.Time } type AsyncQueryTraceLoggerEvictor struct { diff --git a/quesma/quesma/async_search_storage/model.go b/quesma/quesma/async_search_storage/model.go index 814b55920..5ae1f5aa7 100644 --- a/quesma/quesma/async_search_storage/model.go +++ b/quesma/quesma/async_search_storage/model.go @@ -7,6 +7,9 @@ import ( "time" ) +const EvictionInterval = 15 * time.Minute +const GCInterval = 1 * time.Minute + type AsyncRequestResultStorage interface { Store(id string, result *AsyncRequestResult) Load(id string) (*AsyncRequestResult, bool) @@ -14,11 +17,13 @@ type AsyncRequestResultStorage interface { DocCount() int SizeInBytes() uint64 SizeInBytesLimit() uint64 + + evict(timeFun func(time.Time) time.Duration) } -// TODO: maybe merge those 2? type AsyncQueryContextStorage interface { Store(id string, context *AsyncQueryContext) + evict(timeFun func(time.Time) time.Duration) } type AsyncRequestResult struct { From 29679c39f92d016a1fe588ecd06fdbf14bb0fae2 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Mon, 11 Nov 2024 08:03:15 +0100 Subject: [PATCH 05/30] Almost --- quesma/elasticsearch/client.go | 13 ++ quesma/persistence/elastic_with_eviction.go | 221 +++++++----------- quesma/persistence/model.go | 13 +- quesma/persistence/persistence_test.go | 6 +- .../quesma/async_search_storage/in_elastic.go | 46 ++-- .../quesma/async_search_storage/in_memory.go | 16 +- .../in_memory_fallback_elastic.go | 53 +++++ .../async_search_storage/in_memory_test.go | 33 ++- quesma/quesma/async_search_storage/model.go | 19 ++ 9 files changed, 251 insertions(+), 169 deletions(-) create mode 100644 quesma/quesma/async_search_storage/in_memory_fallback_elastic.go diff --git a/quesma/elasticsearch/client.go b/quesma/elasticsearch/client.go index d178adcd8..bb25d7bb6 100644 --- a/quesma/elasticsearch/client.go +++ b/quesma/elasticsearch/client.go @@ -40,6 +40,19 @@ func (es *SimpleClient) RequestWithHeaders(ctx context.Context, method, endpoint return es.doRequest(ctx, method, endpoint, body, headers) } +func (es *SimpleClient) DoRequestCheckResponseStatus(ctx context.Context, method, endpoint string, body []byte) (resp *http.Response, err error) { + resp, err = es.doRequest(ctx, "GET", endpoint, body, nil) + if err != nil { + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return resp, fmt.Errorf("response code from Elastic is not 200 OK, but %s", resp.Status) + } + return resp, nil +} + func (es *SimpleClient) Authenticate(ctx context.Context, authHeader string) bool { resp, err := es.doRequest(ctx, "GET", "_security/_authenticate", nil, http.Header{"Authorization": {authHeader}}) if err != nil { diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index 76ee9fd50..a7aa1197b 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -5,6 +5,7 @@ package persistence import ( "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -14,8 +15,8 @@ import ( "time" ) -const MAX_DOC_COUNT = 10000 // TODO: fix/make configurable/idk/etc -const defaultSizeInBytesLimit = int64(1_000_000_000) // 1GB +const MAX_DOC_COUNT = 10000 // TODO: fix/make configurable/idk/etc +const defaultHugeSizeInBytesLimit = int64(500_000_000_000) // 500GB // so far I serialize entire struct and keep only 1 string in ES type ElasticDatabaseWithEviction struct { @@ -34,32 +35,28 @@ func NewElasticDatabaseWithEviction(cfg config.ElasticsearchConfiguration, index } } -// mutexy? or what -func (db *ElasticDatabaseWithEviction) Put(ctx context.Context, doc *document) bool { - dbSize, success := db.SizeInBytes() - if !success { - return false +func (db *ElasticDatabaseWithEviction) Put(doc *document) error { + dbSize, err := db.SizeInBytes() + if err != nil { + return err } fmt.Println("kk dbg Put() dbSize:", dbSize) bytesNeeded := dbSize + doc.SizeInBytes if bytesNeeded > db.SizeInBytesLimit() { - logger.InfoWithCtx(ctx).Msgf("Database is full, need %d bytes more. Evicting documents", bytesNeeded-db.SizeInBytesLimit()) - allDocs, ok := db.getAll() - if !ok { - logger.WarnWithCtx(ctx).Msg("Error getting all documents") - return false + logger.Info().Msgf("elastic database: is full, need %d bytes more. Evicting documents", bytesNeeded-db.SizeInBytesLimit()) + allDocs, err := db.getAll() + if err != nil { + return err } indexesToEvict, bytesEvicted := db.SelectToEvict(allDocs, bytesNeeded-db.SizeInBytesLimit()) - logger.InfoWithCtx(ctx).Msgf("Evicting %v indexes, %d bytes", indexesToEvict, bytesEvicted) + logger.Info().Msgf("elastic database: evicting %v indexes, %d bytes", indexesToEvict, bytesEvicted) db.evict(indexesToEvict) bytesNeeded -= bytesEvicted } if bytesNeeded > db.SizeInBytesLimit() { - // put document - return false + return errors.New("elastic database: is full, cannot put document") } - //elasticsearchURL := fmt.Sprintf("%s/_update/%s", db.fullIndexName(), doc.Id) elasticsearchURL := fmt.Sprintf("%s/_update/%s", db.indexName, doc.Id) fmt.Println("kk dbg Put() elasticsearchURL:", elasticsearchURL) @@ -69,40 +66,23 @@ func (db *ElasticDatabaseWithEviction) Put(ctx context.Context, doc *document) b jsonData, err := json.Marshal(updateContent) if err != nil { - logger.WarnWithCtx(ctx).Msgf("Error marshalling document: %v", err) - return false - } - - resp, err := db.httpClient.Request(context.Background(), "POST", elasticsearchURL, jsonData) - if err != nil { - logger.WarnWithCtx(ctx).Msgf("Error sending request to elastic: %v", err) - return false + return err } - defer resp.Body.Close() - switch resp.StatusCode { - case http.StatusCreated, http.StatusOK: - return true - default: - respBody, err := io.ReadAll(resp.Body) - if err != nil { - logger.WarnWithCtx(ctx).Msgf("Error reading response body: %v, respBody: %v", err, respBody) - } - return false + resp, err := db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodPost, elasticsearchURL, jsonData) + if err != nil && resp.StatusCode != http.StatusCreated { + return err } + return nil } // co zwraca? zrobić switch na oba typy jakie teraz mamy? -func (db *ElasticDatabaseWithEviction) Get(ctx context.Context, id string) (string, bool) { // probably change return type to *Sizeable - value, success, err := db.ElasticJSONDatabase.Get(id) - if err != nil { - logger.WarnWithCtx(ctx).Msgf("Error getting document, id: %s, error: %v", id, err) - return "", false - } - return value, success +func (db *ElasticDatabaseWithEviction) Get(id string) (string, error) { // probably change return type to *Sizeable + value, _, err := db.ElasticJSONDatabase.Get(id) + return value, err } -func (db *ElasticDatabaseWithEviction) Delete(id string) bool { +func (db *ElasticDatabaseWithEviction) Delete(id string) error { // mark as deleted, don't actually delete // (single document deletion is hard in ES, it's done by evictor for entire index) @@ -115,30 +95,21 @@ func (db *ElasticDatabaseWithEviction) Delete(id string) bool { jsonData, err := json.Marshal(updateContent) if err != nil { - logger.WarnWithCtx(db.ctx).Msgf("Error marshalling document: %v", err) - return false + return err } - resp, err := db.httpClient.Request(context.Background(), "POST", elasticsearchURL, jsonData) - if err != nil { - logger.WarnWithCtx(db.ctx).Msgf("Error sending request to elastic: %v", err) - return false + resp, err := db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodPost, elasticsearchURL, jsonData) + if err != nil && resp.StatusCode != http.StatusCreated { + return err } - defer resp.Body.Close() + return nil +} - switch resp.StatusCode { - case http.StatusCreated, http.StatusOK: - return true - default: - respBody, err := io.ReadAll(resp.Body) - if err != nil { - logger.WarnWithCtx(db.ctx).Msgf("Error reading response body: %v, respBody: %v", err, respBody) - } - return false - } +func (db *ElasticDatabaseWithEviction) DeleteOld(deleteOlderThan time.Duration) error { + return nil } -func (db *ElasticDatabaseWithEviction) DocCount() (count int, success bool) { +func (db *ElasticDatabaseWithEviction) DocCount() (docCount int, err error) { elasticsearchURL := fmt.Sprintf("%s/_search", db.indexName) query := `{ "_source": false, @@ -153,43 +124,33 @@ func (db *ElasticDatabaseWithEviction) DocCount() (count int, success bool) { } }` - resp, err := db.httpClient.Request(context.Background(), "GET", elasticsearchURL, []byte(query)) + var resp *http.Response + resp, err = db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodGet, elasticsearchURL, []byte(query)) if err != nil { - return + if resp.StatusCode == http.StatusNoContent || resp.StatusCode == http.StatusNotFound { + return 0, nil + } + return -1, err } - defer resp.Body.Close() - jsonAsBytes, err := io.ReadAll(resp.Body) + var jsonAsBytes []byte + jsonAsBytes, err = io.ReadAll(resp.Body) if err != nil { return } - fmt.Println("kk dbg DocCount() resp.StatusCode:", resp.StatusCode) - - switch resp.StatusCode { - case http.StatusOK: - break - case http.StatusNoContent, http.StatusNotFound: - return 0, true - default: - logger.WarnWithCtx(db.ctx).Msgf("failed to get from elastic: %s, response status code: %v", string(jsonAsBytes), resp.StatusCode) - return - } - // Unmarshal the JSON response var result map[string]interface{} if err = json.Unmarshal(jsonAsBytes, &result); err != nil { - logger.WarnWithCtx(db.ctx).Msgf("Error parsing the response JSON: %s", err) return } fmt.Println("kk dbg DocCount() result:", result) - count = int(result["hits"].(map[string]interface{})["total"].(map[string]interface{})["value"].(float64)) // TODO: add some checks... to prevent panic - return count, true + return int(result["hits"].(map[string]interface{})["total"].(map[string]interface{})["value"].(float64)), nil // TODO: add some checks... to prevent panic } -func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, success bool) { +func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, err error) { elasticsearchURL := fmt.Sprintf("%s/_search", db.indexName) query := `{ "_source": ["sizeInBytes"], @@ -197,33 +158,23 @@ func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, success "track_total_hits": true }` - resp, err := db.httpClient.Request(context.Background(), "GET", elasticsearchURL, []byte(query)) + var resp *http.Response + resp, err = db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodGet, elasticsearchURL, []byte(query)) if err != nil { return } - defer resp.Body.Close() - jsonAsBytes, err := io.ReadAll(resp.Body) + var jsonAsBytes []byte + jsonAsBytes, err = io.ReadAll(resp.Body) if err != nil { return } fmt.Println("kk dbg SizeInBytes() resp.StatusCode:", resp.StatusCode) - switch resp.StatusCode { - case http.StatusOK: - break - case http.StatusNoContent, http.StatusNotFound: - return 0, true - default: - logger.WarnWithCtx(db.ctx).Msgf("failed to get from elastic: %s, response status code: %v", string(jsonAsBytes), resp.StatusCode) - return - } - // Unmarshal the JSON response var result map[string]interface{} if err = json.Unmarshal(jsonAsBytes, &result); err != nil { - logger.WarnWithCtx(db.ctx).Msgf("Error parsing the response JSON: %s", err) return } @@ -234,64 +185,68 @@ func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, success a = append(a, sizeInBytes-b) } fmt.Println("kk dbg SizeInBytes() sizes in storage:", a) - return sizeInBytes, true + return sizeInBytes, nil } func (db *ElasticDatabaseWithEviction) SizeInBytesLimit() int64 { return db.sizeInBytesLimit } -func (db *ElasticDatabaseWithEviction) getAll() (documents []*document, success bool) { - elasticsearchURL := fmt.Sprintf("%s*/_search", db.indexName) - query := `{ +func (db *ElasticDatabaseWithEviction) getAll() (documents []*document, err error) { + _ = fmt.Sprintf("%s*/_search", db.indexName) + _ = `{ "_source": { "excludes": "data" }, "size": 10000, "track_total_hits": true }` + /* + db.httpClient. - resp, err := db.httpClient.Request(context.Background(), "GET", elasticsearchURL, []byte(query)) - if err != nil { - return - } - defer resp.Body.Close() + resp, err := db.httpClient.Request(context.Background(), "GET", elasticsearchURL, []byte(query)) + if err != nil { + return + } + defer resp.Body.Close() - jsonAsBytes, err := io.ReadAll(resp.Body) - if err != nil { - return - } + jsonAsBytes, err := io.ReadAll(resp.Body) + if err != nil { + return + } - fmt.Println("kk dbg getAll() resp.StatusCode:", resp.StatusCode) + fmt.Println("kk dbg getAll() resp.StatusCode:", resp.StatusCode) - switch resp.StatusCode { - case http.StatusOK: - break - default: - logger.WarnWithCtx(db.ctx).Msgf("failed to get from elastic: %s, response status code: %v", string(jsonAsBytes), resp.StatusCode) - return - } + switch resp.StatusCode { + case http.StatusOK: + break + default: + logger.WarnWithCtx(db.ctx).Msgf("failed to get from elastic: %s, response status code: %v", string(jsonAsBytes), resp.StatusCode) + return + } - // Unmarshal the JSON response - var result map[string]interface{} - if err = json.Unmarshal(jsonAsBytes, &result); err != nil { - logger.WarnWithCtx(db.ctx).Msgf("Error parsing the response JSON: %s", err) - return - } + // Unmarshal the JSON response + var result map[string]interface{} + if err = json.Unmarshal(jsonAsBytes, &result); err != nil { + logger.WarnWithCtx(db.ctx).Msgf("Error parsing the response JSON: %s", err) + return + } - fmt.Println("kk dbg getAll() documents:") - for _, hit := range result["hits"].(map[string]interface{})["hits"].([]interface{}) { - doc := &document{ - Id: hit.(map[string]interface{})["_id"].(string), - Index: hit.(map[string]interface{})["_index"].(string), - SizeInBytes: int64(hit.(map[string]interface{})["_source"].(map[string]interface{})["sizeInBytes"].(float64)), // TODO: add checks - //Timestamp: hit.(map[string]interface{})["_source"].(map[string]interface{})["timestamp"].(time.Time), // TODO: add checks - MarkedAsDeleted: hit.(map[string]interface{})["_source"].(map[string]interface{})["markedAsDeleted"].(bool), // TODO: add checks + fmt.Println("kk dbg getAll() documents:") + for _, hit := range result["hits"].(map[string]interface{})["hits"].([]interface{}) { + doc := &document{ + Id: hit.(map[string]interface{})["_id"].(string), + Index: hit.(map[string]interface{})["_index"].(string), + SizeInBytes: int64(hit.(map[string]interface{})["_source"].(map[string]interface{})["sizeInBytes"].(float64)), // TODO: add checks + //Timestamp: hit.(map[string]interface{})["_source"].(map[string]interface{})["timestamp"].(time.Time), // TODO: add checks + MarkedAsDeleted: hit.(map[string]interface{})["_source"].(map[string]interface{})["markedAsDeleted"].(bool), // TODO: add checks + } + fmt.Println(doc) + documents = append(documents, doc) } - fmt.Println(doc) - documents = append(documents, doc) - } - return documents, true + + */ + return documents, nil } func (db *ElasticDatabaseWithEviction) evict(indexes []string) { diff --git a/quesma/persistence/model.go b/quesma/persistence/model.go index d93f61ed6..c09b244f5 100644 --- a/quesma/persistence/model.go +++ b/quesma/persistence/model.go @@ -18,12 +18,13 @@ type JSONDatabase interface { Put(key string, data string) error } -type JSONDatabaseWithEviction interface { // for sure JSON? maybe not only json? check - Put(doc document) bool - Get(id string) (document, bool) - Delete(id string) - DocCount() (int, bool) - SizeInBytes() (int64, bool) +type DatabaseWithEviction interface { // for sure JSON? maybe not only json? check + Put(doc document) error + Get(id string) (document, error) + Delete(id string) error + DeleteOld(time.Duration) error + DocCount() (int, error) + SizeInBytes() (int64, error) SizeInBytesLimit() int64 } diff --git a/quesma/persistence/persistence_test.go b/quesma/persistence/persistence_test.go index f841c0b5e..1b48a275d 100644 --- a/quesma/persistence/persistence_test.go +++ b/quesma/persistence/persistence_test.go @@ -87,7 +87,7 @@ func TestNewElasticPersistence(t *testing.T) { func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { const precise = true - + t.Skip() logger.InitSimpleLoggerForTests() indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) fmt.Println("indexName:", indexName) @@ -103,7 +103,7 @@ func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { } const bigSizeLimit = int64(1_000_000_000) - db := NewElasticDatabaseWithEviction(context.Background(), cfg, indexName, bigSizeLimit) + db := NewElasticDatabaseWithEviction(cfg, indexName, bigSizeLimit) // check initial state assert.Equal(t, bigSizeLimit, db.SizeInBytesLimit()) @@ -165,7 +165,7 @@ const updateTime = 4 * time.Second func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { logger.InitSimpleLoggerForTests() indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) - + t.Skip() realUrl, err := url.Parse("http://localhost:9200") assert.NoError(t, err) diff --git a/quesma/quesma/async_search_storage/in_elastic.go b/quesma/quesma/async_search_storage/in_elastic.go index 64031e18d..41be78e0d 100644 --- a/quesma/quesma/async_search_storage/in_elastic.go +++ b/quesma/quesma/async_search_storage/in_elastic.go @@ -3,9 +3,10 @@ package async_search_storage import ( - "context" + "quesma/logger" "quesma/persistence" "quesma/quesma/config" + "time" ) type AsyncSearchStorageInElastic struct { @@ -19,30 +20,47 @@ func NewAsyncSearchStorageInElastic() AsyncSearchStorageInElastic { } } -func (s AsyncSearchStorageInElastic) Store(ctx context.Context, id string, result *AsyncRequestResult) { - s.db.Put(ctx, nil) +func (s AsyncSearchStorageInElastic) Store(id string, result *AsyncRequestResult) { + err := s.db.Put(result.toJSON(id)) + if err != nil { + logger.Warn().Err(err).Msg("failed to store document") + } } -func (s AsyncSearchStorageInElastic) Load(ctx context.Context, id string) (*AsyncRequestResult, bool) { - _, ok := s.db.Get(ctx, id) - return nil, ok +func (s AsyncSearchStorageInElastic) Load(id string) (*AsyncRequestResult, error) { + _, err := s.db.Get(id) + return nil, err } func (s AsyncSearchStorageInElastic) Delete(id string) { - s.db.Delete(id) + err := s.db.Delete(id) + if err != nil { + logger.Warn().Err(err).Msg("failed to delete document") + } } +func (s AsyncSearchStorageInElastic) DeleteOld(t time.Duration) { + err := s.db.DeleteOld(t) + if err != nil { + logger.Warn().Err(err).Msg("failed to delete old documents") + } +} + +// DocCount returns the number of documents in the database, or -1 if the count could not be retrieved. func (s AsyncSearchStorageInElastic) DocCount() int { - cnt, ok := s.db.DocCount() - if !ok { + cnt, err := s.db.DocCount() + if err != nil { + logger.Warn().Err(err).Msg("failed to get document count") return -1 } return cnt } -func (s AsyncSearchStorageInElastic) SizeInBytes() int64 { - size, ok := s.db.SizeInBytes() - if !ok { +// StorageSizeInBytes returns the total size of all documents in the database, or -1 if the size could not be retrieved. +func (s AsyncSearchStorageInElastic) StorageSizeInBytes() int64 { + size, err := s.db.SizeInBytes() + if err != nil { + logger.Warn().Err(err).Msg("failed to get storage size") return -1 } return size @@ -58,7 +76,3 @@ func NewAsyncQueryContextStorageInElastic() AsyncQueryContextStorageInElastic { config.ElasticsearchConfiguration{}, "async_search", 1_000_000_000), } } - -func (s AsyncQueryContextStorageInElastic) Store(ctx context.Context, id string, context *AsyncQueryContext) { - s.db.Put(ctx, nil) -} diff --git a/quesma/quesma/async_search_storage/in_memory.go b/quesma/quesma/async_search_storage/in_memory.go index 1821af08e..d3176a186 100644 --- a/quesma/quesma/async_search_storage/in_memory.go +++ b/quesma/quesma/async_search_storage/in_memory.go @@ -4,6 +4,7 @@ package async_search_storage import ( "context" + "math" "quesma/concurrent" "quesma/logger" "quesma/quesma/recovery" @@ -51,16 +52,20 @@ func (s AsyncSearchStorageInMemory) SizeInBytes() int { return size } +func (s AsyncSearchStorageInMemory) SizeInBytesLimit() uint64 { + return math.MaxUint64 / 16 // some huge number for now, can be changed if we want to limit in-memory storage +} + func (s AsyncSearchStorageInMemory) evict(timeFun func(time.Time) time.Duration) { - var ids []asyncQueryIdWithTime + var ids []string s.Range(func(key string, value *AsyncRequestResult) bool { if timeFun(value.added) > EvictionInterval { - ids = append(ids, asyncQueryIdWithTime{id: key, time: value.added}) + ids = append(ids, key) } return true }) for _, id := range ids { - s.Delete(id.id) + s.Delete(id) } } @@ -105,11 +110,6 @@ func elapsedTime(t time.Time) time.Duration { return time.Since(t) } -type asyncQueryIdWithTime struct { - id string - time time.Time -} - type AsyncQueryTraceLoggerEvictor struct { AsyncQueryTrace *concurrent.Map[string, tracing.TraceCtx] ctx context.Context diff --git a/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go b/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go new file mode 100644 index 000000000..3701c0b64 --- /dev/null +++ b/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go @@ -0,0 +1,53 @@ +package async_search_storage + +import "time" + +type AsyncSearchStorageInMemoryFallbackElastic struct { + inMemory AsyncSearchStorageInMemory + elastic AsyncSearchStorageInElastic +} + +func NewAsyncSearchStorageInMemoryFallbackElastic() AsyncSearchStorageInMemoryFallbackElastic { + return AsyncSearchStorageInMemoryFallbackElastic{ + inMemory: NewAsyncSearchStorageInMemory(), + elastic: NewAsyncSearchStorageInElastic(), + } +} + +func (s AsyncSearchStorageInMemoryFallbackElastic) Store(id string, result *AsyncRequestResult) { + s.inMemory.Store(id, result) + go s.elastic.Store(id, result) +} + +func (s AsyncSearchStorageInMemoryFallbackElastic) Load(id string) (*AsyncRequestResult, error) { + result, ok := s.inMemory.Load(id) + if ok { + return result, nil + } + return s.elastic.Load(id) +} + +func (s AsyncSearchStorageInMemoryFallbackElastic) Delete(id string) { + s.inMemory.Delete(id) + go s.elastic.Delete(id) +} + +// DocCount returns inMemory doc count +func (s AsyncSearchStorageInMemoryFallbackElastic) DocCount() int { + return s.inMemory.DocCount() +} + +// SizeInBytes returns inMemory size in bytes +func (s AsyncSearchStorageInMemoryFallbackElastic) SizeInBytes() int { + return s.inMemory.SizeInBytes() +} + +func (s AsyncSearchStorageInMemoryFallbackElastic) evict(timeFun func(time.Time) time.Duration) { + s.inMemory.evict(timeFun) + go s.elastic.DeleteOld(timeFun(time.Now())) +} + +// SizeInBytesLimit returns inMemory size in bytes limit +func (s AsyncSearchStorageInMemoryFallbackElastic) SizeInBytesLimit() uint64 { + return s.inMemory.SizeInBytesLimit() +} diff --git a/quesma/quesma/async_search_storage/in_memory_test.go b/quesma/quesma/async_search_storage/in_memory_test.go index 7557d6a7a..6117b6fdc 100644 --- a/quesma/quesma/async_search_storage/in_memory_test.go +++ b/quesma/quesma/async_search_storage/in_memory_test.go @@ -3,7 +3,9 @@ package async_search_storage import ( - "github.com/stretchr/testify/assert" + "context" + "fmt" + "github.com/ClickHouse/clickhouse-go/v2" "quesma/concurrent" "testing" "time" @@ -20,7 +22,7 @@ func TestAsyncQueriesEvictorTimePassed(t *testing.T) { return 20 * time.Minute }) - assert.Equal(t, 0, evictor.AsyncRequestStorage.Size()) + //assert.Equal(t, 0, evictor.AsyncRequestStorage.Size()) } func TestAsyncQueriesEvictorStillAlive(t *testing.T) { @@ -35,5 +37,30 @@ func TestAsyncQueriesEvictorStillAlive(t *testing.T) { return time.Second }) - assert.Equal(t, 3, evictor.AsyncRequestStorage.Size()) + //assert.Equal(t, 3, evictor.AsyncRequestStorage.Size()) +} + +const qid = "abc" + +func TestKK(t *testing.T) { + options := clickhouse.Options{Addr: []string{"localhost:9000"}} + a := clickhouse.OpenDB(&options) + ctx := clickhouse.Context(context.Background(), clickhouse.WithQueryID(qid)) + + b, err := a.QueryContext(ctx, "SELECT number FROM (SELECT number FROM numbers(100_000_000_000)) ORDER BY number DESC LIMIT 10") + var q int64 + for b.Next() { + b.Scan(&q) + fmt.Println(q) + } + + fmt.Println(b, "q:", q, err) +} + +func TestCancel(t *testing.T) { + options := clickhouse.Options{Addr: []string{"localhost:9000"}} + a := clickhouse.OpenDB(&options) + + b, err := a.Query("KILL QUERY WHERE query_id= 'dupa'") + fmt.Println(b, err) } diff --git a/quesma/quesma/async_search_storage/model.go b/quesma/quesma/async_search_storage/model.go index 5ae1f5aa7..4ee4f9dfb 100644 --- a/quesma/quesma/async_search_storage/model.go +++ b/quesma/quesma/async_search_storage/model.go @@ -4,6 +4,7 @@ package async_search_storage import ( "context" + "quesma/quesma/types" "time" ) @@ -49,6 +50,15 @@ func (r *AsyncRequestResult) IsCompressed() bool { return r.isCompressed } +func (r *AsyncRequestResult) toJSON(id string) types.JSON { + json := types.JSON{} + json["id"] = id + json["data"] = string(r.responseBody) + json["sizeInBytes"] = uint64(len(r.responseBody)) + uint64(len(id)) + 100 // 100 is a rough upper bound estimate of the size of the rest of the fields + json["added"] = r.added + return json +} + type AsyncQueryContext struct { id string ctx context.Context @@ -59,3 +69,12 @@ type AsyncQueryContext struct { func NewAsyncQueryContext(ctx context.Context, cancel context.CancelFunc, id string) *AsyncQueryContext { return &AsyncQueryContext{ctx: ctx, cancel: cancel, added: time.Now(), id: id} } + +func (c *AsyncQueryContext) toJSON() types.JSON { + json := types.JSON{} + json["id"] = c.id + json["added"] = c.added + clickhouse. + + return json +} \ No newline at end of file From 7656f8b0c4647061a479710965a0ff4e097061ec Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Mon, 11 Nov 2024 09:09:13 +0100 Subject: [PATCH 06/30] 99% done, need to debug tests. --- quesma/elasticsearch/client.go | 4 +- quesma/persistence/elastic_with_eviction.go | 19 ++++-- quesma/persistence/persistence_test.go | 4 +- quesma/quesma/async_search_storage/evictor.go | 6 +- .../quesma/async_search_storage/in_elastic.go | 29 +++++++++- .../quesma/async_search_storage/in_memory.go | 19 +++--- .../in_memory_fallback_elastic.go | 18 +++--- .../async_search_storage/in_memory_test.go | 58 ++++++++++++------- quesma/quesma/async_search_storage/model.go | 11 ++-- 9 files changed, 107 insertions(+), 61 deletions(-) diff --git a/quesma/elasticsearch/client.go b/quesma/elasticsearch/client.go index bb25d7bb6..b17c30a08 100644 --- a/quesma/elasticsearch/client.go +++ b/quesma/elasticsearch/client.go @@ -41,11 +41,10 @@ func (es *SimpleClient) RequestWithHeaders(ctx context.Context, method, endpoint } func (es *SimpleClient) DoRequestCheckResponseStatus(ctx context.Context, method, endpoint string, body []byte) (resp *http.Response, err error) { - resp, err = es.doRequest(ctx, "GET", endpoint, body, nil) + resp, err = es.doRequest(ctx, method, endpoint, body, nil) if err != nil { return } - defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return resp, fmt.Errorf("response code from Elastic is not 200 OK, but %s", resp.Status) @@ -66,6 +65,7 @@ func (es *SimpleClient) Authenticate(ctx context.Context, authHeader string) boo // doRequest can override auth headers specified in the config, use with care! func (es *SimpleClient) doRequest(ctx context.Context, method, endpoint string, body []byte, headers http.Header) (*http.Response, error) { + fmt.Println("full url:", fmt.Sprintf("%s/%s", es.config.Url, endpoint)) req, err := http.NewRequestWithContext(ctx, method, fmt.Sprintf("%s/%s", es.config.Url, endpoint), bytes.NewBuffer(body)) if err != nil { return nil, err diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index a7aa1197b..bfacaf61d 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -35,13 +35,13 @@ func NewElasticDatabaseWithEviction(cfg config.ElasticsearchConfiguration, index } } -func (db *ElasticDatabaseWithEviction) Put(doc *document) error { +func (db *ElasticDatabaseWithEviction) Put(data types.JSON) error { dbSize, err := db.SizeInBytes() if err != nil { return err } fmt.Println("kk dbg Put() dbSize:", dbSize) - bytesNeeded := dbSize + doc.SizeInBytes + bytesNeeded := dbSize + data["sizeInBytes"].(int64) // improve if bytesNeeded > db.SizeInBytesLimit() { logger.Info().Msgf("elastic database: is full, need %d bytes more. Evicting documents", bytesNeeded-db.SizeInBytesLimit()) allDocs, err := db.getAll() @@ -57,11 +57,11 @@ func (db *ElasticDatabaseWithEviction) Put(doc *document) error { return errors.New("elastic database: is full, cannot put document") } - elasticsearchURL := fmt.Sprintf("%s/_update/%s", db.indexName, doc.Id) + elasticsearchURL := fmt.Sprintf("%s/_update/%s", db.indexName, data["id"].(string)) fmt.Println("kk dbg Put() elasticsearchURL:", elasticsearchURL) updateContent := types.JSON{} - updateContent["doc"] = doc + updateContent["doc"] = data updateContent["doc_as_upsert"] = true jsonData, err := json.Marshal(updateContent) @@ -70,7 +70,8 @@ func (db *ElasticDatabaseWithEviction) Put(doc *document) error { } resp, err := db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodPost, elasticsearchURL, jsonData) - if err != nil && resp.StatusCode != http.StatusCreated { + fmt.Println("kk dbg Put() resp:", resp, "err:", err) + if err != nil && (resp == nil || resp.StatusCode != http.StatusCreated) { return err } return nil @@ -126,8 +127,9 @@ func (db *ElasticDatabaseWithEviction) DocCount() (docCount int, err error) { var resp *http.Response resp, err = db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodGet, elasticsearchURL, []byte(query)) + fmt.Println("kk dbg DocCount() resp:", resp, "err:", err, "elastic url:", elasticsearchURL) if err != nil { - if resp.StatusCode == http.StatusNoContent || resp.StatusCode == http.StatusNotFound { + if resp != nil && (resp.StatusCode == http.StatusNoContent || resp.StatusCode == http.StatusNotFound) { return 0, nil } return -1, err @@ -160,9 +162,14 @@ func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, err err var resp *http.Response resp, err = db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodGet, elasticsearchURL, []byte(query)) + fmt.Println("kk dbg SizeInBytes() err:", err, "\nresp:", resp) if err != nil { + if resp != nil && resp.StatusCode == 404 { + return 0, nil + } return } + defer resp.Body.Close() // add everywhere var jsonAsBytes []byte jsonAsBytes, err = io.ReadAll(resp.Body) diff --git a/quesma/persistence/persistence_test.go b/quesma/persistence/persistence_test.go index 1b48a275d..1b3aca1d5 100644 --- a/quesma/persistence/persistence_test.go +++ b/quesma/persistence/persistence_test.go @@ -3,7 +3,6 @@ package persistence import ( - "context" "fmt" "github.com/stretchr/testify/assert" "net/url" @@ -87,7 +86,6 @@ func TestNewElasticPersistence(t *testing.T) { func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { const precise = true - t.Skip() logger.InitSimpleLoggerForTests() indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) fmt.Println("indexName:", indexName) @@ -177,7 +175,7 @@ func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { } const smallSizeLimit = int64(1200) - db := NewElasticDatabaseWithEviction(context.Background(), cfg, indexName, smallSizeLimit) + db := NewElasticDatabaseWithEviction(cfg, indexName, smallSizeLimit) fmt.Println("indexName:", indexName, "fullIndexName:", db.fullIndexName()) // check initial state diff --git a/quesma/quesma/async_search_storage/evictor.go b/quesma/quesma/async_search_storage/evictor.go index 83b266e4b..0ee0248e9 100644 --- a/quesma/quesma/async_search_storage/evictor.go +++ b/quesma/quesma/async_search_storage/evictor.go @@ -10,11 +10,11 @@ import ( type AsyncQueriesEvictor struct { ctx context.Context cancel context.CancelFunc - AsyncRequestStorage AsyncSearchStorageInMemory - AsyncQueriesContexts AsyncQueryContextStorageInMemory + AsyncRequestStorage AsyncRequestResultStorage + AsyncQueriesContexts AsyncQueryContextStorage } -func NewAsyncQueriesEvictor(AsyncRequestStorage AsyncSearchStorageInMemory, AsyncQueriesContexts AsyncQueryContextStorageInMemory) *AsyncQueriesEvictor { +func NewAsyncQueriesEvictor(AsyncRequestStorage AsyncRequestResultStorage, AsyncQueriesContexts AsyncQueryContextStorage) *AsyncQueriesEvictor { ctx, cancel := context.WithCancel(context.Background()) return &AsyncQueriesEvictor{ctx: ctx, cancel: cancel, AsyncRequestStorage: AsyncRequestStorage, AsyncQueriesContexts: AsyncQueriesContexts} } diff --git a/quesma/quesma/async_search_storage/in_elastic.go b/quesma/quesma/async_search_storage/in_elastic.go index 41be78e0d..cc498383a 100644 --- a/quesma/quesma/async_search_storage/in_elastic.go +++ b/quesma/quesma/async_search_storage/in_elastic.go @@ -3,6 +3,8 @@ package async_search_storage import ( + "fmt" + "net/url" "quesma/logger" "quesma/persistence" "quesma/quesma/config" @@ -14,9 +16,19 @@ type AsyncSearchStorageInElastic struct { } func NewAsyncSearchStorageInElastic() AsyncSearchStorageInElastic { + // TODO use passed config + realUrl, err := url.Parse("http://localhost:9200") + if err != nil { + fmt.Println("ERR", err) + } + cfgUrl := config.Url(*realUrl) + cfg := config.ElasticsearchConfiguration{ + Url: &cfgUrl, + User: "", + Password: "", + } return AsyncSearchStorageInElastic{ - db: persistence.NewElasticDatabaseWithEviction( - config.ElasticsearchConfiguration{}, "async_search", 1_000_000_000), + db: persistence.NewElasticDatabaseWithEviction(cfg, "quesma_async_storage", 1_000_000_000), } } @@ -57,7 +69,7 @@ func (s AsyncSearchStorageInElastic) DocCount() int { } // StorageSizeInBytes returns the total size of all documents in the database, or -1 if the size could not be retrieved. -func (s AsyncSearchStorageInElastic) StorageSizeInBytes() int64 { +func (s AsyncSearchStorageInElastic) SpaceInUse() int64 { size, err := s.db.SizeInBytes() if err != nil { logger.Warn().Err(err).Msg("failed to get storage size") @@ -66,6 +78,17 @@ func (s AsyncSearchStorageInElastic) StorageSizeInBytes() int64 { return size } +func (s AsyncSearchStorageInElastic) SpaceMaxAvailable() int64 { + return s.db.SizeInBytesLimit() +} + +func (s AsyncSearchStorageInElastic) evict(timeFun func(time.Time) time.Duration) { + err := s.db.DeleteOld(timeFun(time.Now())) + if err != nil { + logger.Warn().Err(err).Msg("failed to evict documents") + } +} + type AsyncQueryContextStorageInElastic struct { db *persistence.ElasticDatabaseWithEviction } diff --git a/quesma/quesma/async_search_storage/in_memory.go b/quesma/quesma/async_search_storage/in_memory.go index d3176a186..93432fe8c 100644 --- a/quesma/quesma/async_search_storage/in_memory.go +++ b/quesma/quesma/async_search_storage/in_memory.go @@ -4,6 +4,7 @@ package async_search_storage import ( "context" + "fmt" "math" "quesma/concurrent" "quesma/logger" @@ -31,8 +32,11 @@ func (s AsyncSearchStorageInMemory) Range(f func(key string, value *AsyncRequest s.idToResult.Range(f) } -func (s AsyncSearchStorageInMemory) Load(id string) (*AsyncRequestResult, bool) { - return s.idToResult.Load(id) +func (s AsyncSearchStorageInMemory) Load(id string) (*AsyncRequestResult, error) { + if val, ok := s.idToResult.Load(id); ok { + return val, nil + } + return nil, fmt.Errorf("key %s not found", id) } func (s AsyncSearchStorageInMemory) Delete(id string) { @@ -43,17 +47,18 @@ func (s AsyncSearchStorageInMemory) DocCount() int { return s.idToResult.Size() } -func (s AsyncSearchStorageInMemory) SizeInBytes() int { - size := 0 +// in bytes +func (s AsyncSearchStorageInMemory) SpaceInUse() int64 { + size := int64(0) s.Range(func(key string, value *AsyncRequestResult) bool { - size += len(value.GetResponseBody()) + size += int64(len(value.GetResponseBody())) return true }) return size } -func (s AsyncSearchStorageInMemory) SizeInBytesLimit() uint64 { - return math.MaxUint64 / 16 // some huge number for now, can be changed if we want to limit in-memory storage +func (s AsyncSearchStorageInMemory) SpaceMaxAvailable() int64 { + return math.MaxInt64 / 16 // some huge number for now, can be changed if we want to limit in-memory storage } func (s AsyncSearchStorageInMemory) evict(timeFun func(time.Time) time.Duration) { diff --git a/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go b/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go index 3701c0b64..dbfeb17ed 100644 --- a/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go +++ b/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go @@ -20,8 +20,8 @@ func (s AsyncSearchStorageInMemoryFallbackElastic) Store(id string, result *Asyn } func (s AsyncSearchStorageInMemoryFallbackElastic) Load(id string) (*AsyncRequestResult, error) { - result, ok := s.inMemory.Load(id) - if ok { + result, err := s.inMemory.Load(id) + if err == nil { return result, nil } return s.elastic.Load(id) @@ -38,16 +38,16 @@ func (s AsyncSearchStorageInMemoryFallbackElastic) DocCount() int { } // SizeInBytes returns inMemory size in bytes -func (s AsyncSearchStorageInMemoryFallbackElastic) SizeInBytes() int { - return s.inMemory.SizeInBytes() +func (s AsyncSearchStorageInMemoryFallbackElastic) SpaceInUse() int64 { + return s.inMemory.SpaceInUse() +} + +// SizeInBytesLimit returns inMemory size in bytes limit +func (s AsyncSearchStorageInMemoryFallbackElastic) SpaceMaxAvailable() int64 { + return s.inMemory.SpaceMaxAvailable() } func (s AsyncSearchStorageInMemoryFallbackElastic) evict(timeFun func(time.Time) time.Duration) { s.inMemory.evict(timeFun) go s.elastic.DeleteOld(timeFun(time.Now())) } - -// SizeInBytesLimit returns inMemory size in bytes limit -func (s AsyncSearchStorageInMemoryFallbackElastic) SizeInBytesLimit() uint64 { - return s.inMemory.SizeInBytesLimit() -} diff --git a/quesma/quesma/async_search_storage/in_memory_test.go b/quesma/quesma/async_search_storage/in_memory_test.go index 6117b6fdc..525c5472b 100644 --- a/quesma/quesma/async_search_storage/in_memory_test.go +++ b/quesma/quesma/async_search_storage/in_memory_test.go @@ -6,43 +6,56 @@ import ( "context" "fmt" "github.com/ClickHouse/clickhouse-go/v2" - "quesma/concurrent" + "github.com/stretchr/testify/assert" + "quesma/logger" "testing" "time" ) func TestAsyncQueriesEvictorTimePassed(t *testing.T) { - queryContextStorage := NewAsyncQueryContextStorageInMemory() - queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) - evictor := NewAsyncQueriesEvictor(NewAsyncSearchStorageInMemory(), queryContextStorage) - evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{added: time.Now()}) - evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{added: time.Now()}) - evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{added: time.Now()}) - evictor.tryEvictAsyncRequests(func(time.Time) time.Duration { - return 20 * time.Minute - }) + // TODO: add also 3rd storage and nice test for it (remove from memory, but still in elastic) + logger.InitSimpleLoggerForTests() + for _, storage := range []AsyncRequestResultStorage{NewAsyncSearchStorageInMemory(), NewAsyncSearchStorageInElastic()} { + queryContextStorage := NewAsyncQueryContextStorageInMemory() + queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) + evictor := NewAsyncQueriesEvictor(storage, queryContextStorage) + evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{added: time.Now()}) + evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{added: time.Now()}) + evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{added: time.Now()}) + evictor.tryEvictAsyncRequests(func(time.Time) time.Duration { + return 20 * time.Minute + }) - //assert.Equal(t, 0, evictor.AsyncRequestStorage.Size()) + if _, ok := storage.(*AsyncSearchStorageInElastic); ok { + time.Sleep(1 * time.Second) + } + assert.Equal(t, 0, evictor.AsyncRequestStorage.DocCount()) + } } func TestAsyncQueriesEvictorStillAlive(t *testing.T) { - queryContextStorage := NewAsyncQueryContextStorageInMemory() - queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) - evictor := NewAsyncQueriesEvictor(NewAsyncSearchStorageInMemory(), queryContextStorage) - evictor.AsyncRequestStorage.idToResult = concurrent.NewMap[string, *AsyncRequestResult]() - evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{added: time.Now()}) - evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{added: time.Now()}) - evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{added: time.Now()}) - evictor.tryEvictAsyncRequests(func(time.Time) time.Duration { - return time.Second - }) + logger.InitSimpleLoggerForTests() + for _, storage := range []AsyncRequestResultStorage{NewAsyncSearchStorageInMemory(), NewAsyncSearchStorageInElastic()} { + t.Run(fmt.Sprintf("storage: %T", storage), func(t *testing.T) { + queryContextStorage := NewAsyncQueryContextStorageInMemory() + queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) + evictor := NewAsyncQueriesEvictor(storage, queryContextStorage) + evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{added: time.Now()}) + evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{added: time.Now()}) + evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{added: time.Now()}) + evictor.tryEvictAsyncRequests(func(time.Time) time.Duration { + return time.Second + }) - //assert.Equal(t, 3, evictor.AsyncRequestStorage.Size()) + assert.Equal(t, 3, evictor.AsyncRequestStorage.DocCount()) + }) + } } const qid = "abc" func TestKK(t *testing.T) { + t.Skip() options := clickhouse.Options{Addr: []string{"localhost:9000"}} a := clickhouse.OpenDB(&options) ctx := clickhouse.Context(context.Background(), clickhouse.WithQueryID(qid)) @@ -58,6 +71,7 @@ func TestKK(t *testing.T) { } func TestCancel(t *testing.T) { + t.Skip() options := clickhouse.Options{Addr: []string{"localhost:9000"}} a := clickhouse.OpenDB(&options) diff --git a/quesma/quesma/async_search_storage/model.go b/quesma/quesma/async_search_storage/model.go index 4ee4f9dfb..5f110fd12 100644 --- a/quesma/quesma/async_search_storage/model.go +++ b/quesma/quesma/async_search_storage/model.go @@ -13,11 +13,11 @@ const GCInterval = 1 * time.Minute type AsyncRequestResultStorage interface { Store(id string, result *AsyncRequestResult) - Load(id string) (*AsyncRequestResult, bool) + Load(id string) (*AsyncRequestResult, error) Delete(id string) DocCount() int - SizeInBytes() uint64 - SizeInBytesLimit() uint64 + SpaceInUse() int64 + SpaceMaxAvailable() int64 evict(timeFun func(time.Time) time.Duration) } @@ -54,7 +54,7 @@ func (r *AsyncRequestResult) toJSON(id string) types.JSON { json := types.JSON{} json["id"] = id json["data"] = string(r.responseBody) - json["sizeInBytes"] = uint64(len(r.responseBody)) + uint64(len(id)) + 100 // 100 is a rough upper bound estimate of the size of the rest of the fields + json["sizeInBytes"] = int64(len(r.responseBody)) + int64(len(id)) + 100 // 100 is a rough upper bound estimate of the size of the rest of the fields json["added"] = r.added return json } @@ -74,7 +74,6 @@ func (c *AsyncQueryContext) toJSON() types.JSON { json := types.JSON{} json["id"] = c.id json["added"] = c.added - clickhouse. return json -} \ No newline at end of file +} From 49956b02380d2a9678ac9278abbdf52acc96c405 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Mon, 11 Nov 2024 12:34:06 +0100 Subject: [PATCH 07/30] Some more --- quesma/persistence/elastic_with_eviction.go | 70 ++++++------ quesma/persistence/evictor.go | 26 +---- quesma/persistence/model.go | 38 +++---- quesma/persistence/persistence_test.go | 105 +++++++++--------- .../quesma/async_search_storage/in_elastic.go | 6 +- .../in_memory_fallback_elastic.go | 2 +- quesma/quesma/async_search_storage/model.go | 5 +- quesma/quesma/search.go | 4 +- 8 files changed, 118 insertions(+), 138 deletions(-) diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index bfacaf61d..b8d4d99cc 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "math" "net/http" "quesma/logger" "quesma/quesma/config" @@ -35,33 +36,32 @@ func NewElasticDatabaseWithEviction(cfg config.ElasticsearchConfiguration, index } } -func (db *ElasticDatabaseWithEviction) Put(data types.JSON) error { +func (db *ElasticDatabaseWithEviction) Put(document *JSONWithSize) error { dbSize, err := db.SizeInBytes() if err != nil { return err } fmt.Println("kk dbg Put() dbSize:", dbSize) - bytesNeeded := dbSize + data["sizeInBytes"].(int64) // improve + bytesNeeded := dbSize + document.SizeInBytesTotal // improve if bytesNeeded > db.SizeInBytesLimit() { logger.Info().Msgf("elastic database: is full, need %d bytes more. Evicting documents", bytesNeeded-db.SizeInBytesLimit()) allDocs, err := db.getAll() if err != nil { return err } - indexesToEvict, bytesEvicted := db.SelectToEvict(allDocs, bytesNeeded-db.SizeInBytesLimit()) - logger.Info().Msgf("elastic database: evicting %v indexes, %d bytes", indexesToEvict, bytesEvicted) - db.evict(indexesToEvict) + bytesEvicted := db.Evict(allDocs, bytesNeeded-db.SizeInBytesLimit()) + logger.Info().Msgf("elastic database: evicted %d bytes", bytesEvicted) bytesNeeded -= bytesEvicted } if bytesNeeded > db.SizeInBytesLimit() { return errors.New("elastic database: is full, cannot put document") } - elasticsearchURL := fmt.Sprintf("%s/_update/%s", db.indexName, data["id"].(string)) + elasticsearchURL := fmt.Sprintf("%s/_update/%s", db.indexName, document.id) fmt.Println("kk dbg Put() elasticsearchURL:", elasticsearchURL) updateContent := types.JSON{} - updateContent["doc"] = data + updateContent["doc"] = document.JSON updateContent["doc_as_upsert"] = true jsonData, err := json.Marshal(updateContent) @@ -88,26 +88,41 @@ func (db *ElasticDatabaseWithEviction) Delete(id string) error { // (single document deletion is hard in ES, it's done by evictor for entire index) // TODO: check if doc exists? - elasticsearchURL := fmt.Sprintf("%s/_update/%s", db.indexName, id) - - updateContent := types.JSON{} - updateContent["doc"] = types.JSON{"markedAsDeleted": true} - updateContent["doc_as_upsert"] = true - - jsonData, err := json.Marshal(updateContent) - if err != nil { - return err - } - - resp, err := db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodPost, elasticsearchURL, jsonData) - if err != nil && resp.StatusCode != http.StatusCreated { + elasticsearchURL := fmt.Sprintf("%s/_doc/%s", db.indexName, id) + resp, err := db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodDelete, elasticsearchURL, nil) + if err != nil && (resp == nil || resp.StatusCode != http.StatusCreated) { return err } return nil } -func (db *ElasticDatabaseWithEviction) DeleteOld(deleteOlderThan time.Duration) error { - return nil +func (db *ElasticDatabaseWithEviction) DeleteOld(deleteOlderThan time.Duration) (err error) { + if deleteOlderThan < 1*time.Second { + deleteOlderThan = 1 * time.Second + } + + rangeStr := fmt.Sprintf("now-%dm", int(math.Floor(deleteOlderThan.Minutes()))) + if deleteOlderThan < 5*time.Minute { + rangeStr = fmt.Sprintf("now-%ds", int(math.Floor(deleteOlderThan.Seconds()))) + } + + elasticsearchURL := fmt.Sprintf("%s/_delete_by_query", db.indexName) + query := fmt.Sprintf(`{ + "query": { + "range": { + "added": { + "lte": "%s" + } + } + } + }`, rangeStr) + + fmt.Println(query) + + var resp *http.Response + resp, err = db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodPost, elasticsearchURL, []byte(query)) + fmt.Println("kk dbg DocCount() resp:", resp, "err:", err, "elastic url:", elasticsearchURL) + return err } func (db *ElasticDatabaseWithEviction) DocCount() (docCount int, err error) { @@ -115,14 +130,7 @@ func (db *ElasticDatabaseWithEviction) DocCount() (docCount int, err error) { query := `{ "_source": false, "size": 0, - "track_total_hits": true, - "query": { - "term": { - "markedAsDeleted": { - "value": false - } - } - } + "track_total_hits": true }` var resp *http.Response @@ -199,7 +207,7 @@ func (db *ElasticDatabaseWithEviction) SizeInBytesLimit() int64 { return db.sizeInBytesLimit } -func (db *ElasticDatabaseWithEviction) getAll() (documents []*document, err error) { +func (db *ElasticDatabaseWithEviction) getAll() (documents []*JSONWithSize, err error) { _ = fmt.Sprintf("%s*/_search", db.indexName) _ = `{ "_source": { diff --git a/quesma/persistence/evictor.go b/quesma/persistence/evictor.go index 18f4dd3af..88bfc5f90 100644 --- a/quesma/persistence/evictor.go +++ b/quesma/persistence/evictor.go @@ -5,40 +5,18 @@ package persistence import "fmt" type EvictorInterface interface { - SelectToEvict(documents []*document, sizeNeeded int64) (indexesToEvict []string, bytesEvicted int64) + Evict(documents []*JSONWithSize, sizeNeeded int64) (bytesEvicted int64) } // It's only 1 implementation, which looks well suited for ElasticSearch. // It can be implemented differently. type Evictor struct{} -func (e *Evictor) SelectToEvict(documents []*document, sizeNeeded int64) (indexesToEvict []string, bytesEvicted int64) { +func (e *Evictor) Evict(documents []*JSONWithSize, sizeNeeded int64) (bytesEvicted int64) { if sizeNeeded <= 0 { return // check if it's empty array or nil } fmt.Println("kk dbg SelectToEvict() sizeNeeded:", sizeNeeded) - countByIndex := make(map[string]int) - countByIndexMarkedAsDeleted := make(map[string]int) - - for _, doc := range documents { - countByIndex[doc.Index]++ - if doc.MarkedAsDeleted { - countByIndexMarkedAsDeleted[doc.Index]++ - } - } - - for index, markedAsDeletedCnt := range countByIndexMarkedAsDeleted { - if countByIndex[index] == markedAsDeletedCnt { - indexesToEvict = append(indexesToEvict, index) - } - } - - for _, doc := range documents { - if countByIndex[doc.Index] == countByIndexMarkedAsDeleted[doc.Index] { - bytesEvicted += doc.SizeInBytes - } - } - return } diff --git a/quesma/persistence/model.go b/quesma/persistence/model.go index c09b244f5..58767f8c5 100644 --- a/quesma/persistence/model.go +++ b/quesma/persistence/model.go @@ -2,7 +2,10 @@ // SPDX-License-Identifier: Elastic-2.0 package persistence -import "time" +import ( + "quesma/quesma/types" + "time" +) // JSONDatabase is an interface for a database that stores JSON data. // Treat it as `etcd` equivalent rather than `MongoDB`. @@ -19,8 +22,8 @@ type JSONDatabase interface { } type DatabaseWithEviction interface { // for sure JSON? maybe not only json? check - Put(doc document) error - Get(id string) (document, error) + Put(doc JSONWithSize) error + Get(id string) (types.JSON, error) Delete(id string) error DeleteOld(time.Duration) error DocCount() (int, error) @@ -28,25 +31,20 @@ type DatabaseWithEviction interface { // for sure JSON? maybe not only json? che SizeInBytesLimit() int64 } -type document struct { - Id string `json:"id"` - Data string `json:"data"` - Index string `json:"index,omitempty"` - SizeInBytes int64 `json:"sizeInBytes"` - Timestamp time.Time `json:"timestamp"` - MarkedAsDeleted bool `json:"markedAsDeleted"` +type JSONWithSizeInterface interface { + SizeInBytes() int64 } -/* -type basicDocumentInfo struct { - Id string - SizeInBytes int64 - Timestamp time.Time - MarkedAsDeleted bool +type JSONWithSize struct { + types.JSON + id string + SizeInBytesTotal int64 } -// mb remove or change impl -func (d *basicDocumentInfo) SizeInBytes() int64 { - return d.SizeInBytesTotal +func NewJSONWithSize(data types.JSON, id string, sizeInBytesTotal int64) *JSONWithSize { + return &JSONWithSize{ + JSON: data, + id: id, + SizeInBytesTotal: sizeInBytesTotal, + } } -*/ diff --git a/quesma/persistence/persistence_test.go b/quesma/persistence/persistence_test.go index 1b3aca1d5..8449db570 100644 --- a/quesma/persistence/persistence_test.go +++ b/quesma/persistence/persistence_test.go @@ -92,13 +92,8 @@ func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { realUrl, err := url.Parse("http://localhost:9200") assert.NoError(t, err) - cfgUrl := config.Url(*realUrl) - cfg := config.ElasticsearchConfiguration{ - Url: &cfgUrl, - User: "", - Password: "", - } + cfg := config.ElasticsearchConfiguration{Url: &cfgUrl} const bigSizeLimit = int64(1_000_000_000) db := NewElasticDatabaseWithEviction(cfg, indexName, bigSizeLimit) @@ -106,16 +101,16 @@ func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { // check initial state assert.Equal(t, bigSizeLimit, db.SizeInBytesLimit()) - docCount, ok := db.DocCount() - assert.True(t, ok) + docCount, err := db.DocCount() + assert.NoError(t, err) assert.Equal(t, 0, docCount) - sizeInBytes, ok := db.SizeInBytes() - assert.True(t, ok) + sizeInBytes, err := db.SizeInBytes() + assert.NoError(t, err) assert.Equal(t, int64(0), sizeInBytes) // put first documents - docs := []*document{ + docs := []*JSONWithSize{ doc("doc1", 100), doc("doc2", 200), doc("doc3", 300), @@ -123,56 +118,54 @@ func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { doc("doc5", 500), } for _, d := range docs { - assert.True(t, db.Put(d)) + assert.NoError(t, db.Put(d)) } if precise { - time.Sleep(4 * time.Second) - docCount, ok = db.DocCount() - assert.True(t, ok) + time.Sleep(updateTime) + docCount, err = db.DocCount() + assert.NoError(t, err) assert.Equal(t, 5, docCount) } else { - docCount, ok = db.DocCount() - assert.True(t, ok) + docCount, err = db.DocCount() + assert.NoError(t, err) assert.True(t, docCount >= 0) } - val, ok := db.Get(docs[0].Id) + val, ok := db.Get(docs[0].id) fmt.Println(val, ok) // TODO: deserialize and check content - db.Delete(docs[1].Id) - db.Delete(docs[3].Id) + err = db.Delete(docs[1].id) + assert.NoError(t, err) + err = db.Delete(docs[3].id) + assert.NoError(t, err) if precise { - time.Sleep(1 * time.Second) - docCount, ok = db.DocCount() - assert.True(t, ok) + time.Sleep(updateTime) + docCount, err = db.DocCount() + assert.NoError(t, err) assert.Equal(t, 3, docCount) } else { - docCount, ok = db.DocCount() - assert.True(t, ok) + docCount, err = db.DocCount() + assert.NoError(t, err) assert.True(t, docCount >= 0) } assert.Equal(t, bigSizeLimit, db.SizeInBytesLimit()) } -const updateTime = 4 * time.Second +const updateTime = 2 * time.Second func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { logger.InitSimpleLoggerForTests() indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) - t.Skip() + realUrl, err := url.Parse("http://localhost:9200") assert.NoError(t, err) cfgUrl := config.Url(*realUrl) - cfg := config.ElasticsearchConfiguration{ - Url: &cfgUrl, - User: "", - Password: "", - } + cfg := config.ElasticsearchConfiguration{Url: &cfgUrl} const smallSizeLimit = int64(1200) db := NewElasticDatabaseWithEviction(cfg, indexName, smallSizeLimit) @@ -181,16 +174,16 @@ func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { // check initial state assert.Equal(t, smallSizeLimit, db.SizeInBytesLimit()) - docCount, ok := db.DocCount() - assert.True(t, ok) + docCount, err := db.DocCount() + assert.NoError(t, err) assert.Equal(t, 0, docCount) - sizeInBytes, ok := db.SizeInBytes() - assert.True(t, ok) + sizeInBytes, err := db.SizeInBytes() + assert.NoError(t, err) assert.Equal(t, int64(0), sizeInBytes) // put first documents - docs := []*document{ + docs := []*JSONWithSize{ doc("doc1", 200), doc("doc2", 300), doc("doc3", 400), @@ -198,31 +191,33 @@ func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { doc("doc5", 500), } for _, d := range docs[:2] { - fmt.Println("put", d.SizeInBytes, db.Put(d)) + fmt.Println("put", d.SizeInBytesTotal, db.Put(d)) } time.Sleep(updateTime) - fmt.Println("put", docs[2].SizeInBytes, db.Put(docs[2])) + fmt.Println("put", docs[2].SizeInBytesTotal, db.Put(docs[2])) time.Sleep(updateTime) - docCount, ok = db.DocCount() - assert.True(t, ok) + docCount, err = db.DocCount() + assert.NoError(t, err) assert.Equal(t, 3, docCount) - db.Delete("doc2") + err = db.Delete("doc2") + assert.NoError(t, err) + time.Sleep(updateTime) - docCount, ok = db.DocCount() - assert.True(t, ok) + docCount, err = db.DocCount() + assert.NoError(t, err) assert.Equal(t, 2, docCount) - put4 := db.Put(docs[4]) - fmt.Println("put", docs[4].SizeInBytes, put4) - assert.False(t, put4) + err = db.Put(docs[4]) + fmt.Println("put", docs[4].SizeInBytesTotal, err) + assert.NoError(t, err) time.Sleep(3000 * time.Millisecond) - docCount, ok = db.DocCount() - assert.True(t, ok) + docCount, err = db.DocCount() + assert.NoError(t, err) assert.Equal(t, 3, docCount) // @@ -249,10 +244,10 @@ func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { assert.Equal(t, smallSizeLimit, db.SizeInBytesLimit()) } -func doc(id string, size int64) *document { - return &document{ - Id: id, - SizeInBytes: size, - Timestamp: time.Now(), - } +func doc(id string, size int64) *JSONWithSize { + json := types.JSON{} + json["id"] = id + json["sizeInBytes"] = size + json["timestamp"] = time.Now() + return NewJSONWithSize(json, id, size) } diff --git a/quesma/quesma/async_search_storage/in_elastic.go b/quesma/quesma/async_search_storage/in_elastic.go index cc498383a..d1630bfe6 100644 --- a/quesma/quesma/async_search_storage/in_elastic.go +++ b/quesma/quesma/async_search_storage/in_elastic.go @@ -82,10 +82,10 @@ func (s AsyncSearchStorageInElastic) SpaceMaxAvailable() int64 { return s.db.SizeInBytesLimit() } -func (s AsyncSearchStorageInElastic) evict(timeFun func(time.Time) time.Duration) { - err := s.db.DeleteOld(timeFun(time.Now())) +func (s AsyncSearchStorageInElastic) evict() { + err := s.db.DeleteOld(EvictionInterval) if err != nil { - logger.Warn().Err(err).Msg("failed to evict documents") + logger.Warn().Err(err).Msgf("failed to evict documents, err: %v", err) } } diff --git a/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go b/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go index dbfeb17ed..efb619472 100644 --- a/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go +++ b/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go @@ -49,5 +49,5 @@ func (s AsyncSearchStorageInMemoryFallbackElastic) SpaceMaxAvailable() int64 { func (s AsyncSearchStorageInMemoryFallbackElastic) evict(timeFun func(time.Time) time.Duration) { s.inMemory.evict(timeFun) - go s.elastic.DeleteOld(timeFun(time.Now())) + go s.elastic.DeleteOld(EvictionInterval) } diff --git a/quesma/quesma/async_search_storage/model.go b/quesma/quesma/async_search_storage/model.go index 5f110fd12..26a50f7bd 100644 --- a/quesma/quesma/async_search_storage/model.go +++ b/quesma/quesma/async_search_storage/model.go @@ -4,6 +4,7 @@ package async_search_storage import ( "context" + "quesma/persistence" "quesma/quesma/types" "time" ) @@ -50,13 +51,13 @@ func (r *AsyncRequestResult) IsCompressed() bool { return r.isCompressed } -func (r *AsyncRequestResult) toJSON(id string) types.JSON { +func (r *AsyncRequestResult) toJSON(id string) *persistence.JSONWithSize { json := types.JSON{} json["id"] = id json["data"] = string(r.responseBody) json["sizeInBytes"] = int64(len(r.responseBody)) + int64(len(id)) + 100 // 100 is a rough upper bound estimate of the size of the rest of the fields json["added"] = r.added - return json + return persistence.NewJSONWithSize(json, id, json["sizeInBytes"].(int64)) } type AsyncQueryContext struct { diff --git a/quesma/quesma/search.go b/quesma/quesma/search.go index 03a626af4..63e7e3339 100644 --- a/quesma/quesma/search.go +++ b/quesma/quesma/search.go @@ -495,7 +495,7 @@ func (q *QueryRunner) handlePartialAsyncSearch(ctx context.Context, id string) ( logger.ErrorWithCtx(ctx).Msgf("non quesma async id: %v", id) return queryparser.EmptyAsyncSearchResponse(id, false, 503) } - if result, ok := q.AsyncRequestStorage.Load(id); ok { + if result, err := q.AsyncRequestStorage.Load(id); err != nil { if err := result.GetErr(); err != nil { q.AsyncRequestStorage.Delete(id) logger.ErrorWithCtx(ctx).Msgf("error processing async query: %v", err) @@ -534,7 +534,7 @@ func (q *QueryRunner) deleteAsyncSearch(id string) ([]byte, error) { } func (q *QueryRunner) reachedQueriesLimit(ctx context.Context, asyncId string, doneCh chan<- asyncSearchWithError) bool { - if q.AsyncRequestStorage.DocCount() < asyncQueriesLimit && q.AsyncRequestStorage.SizeInBytes() < asyncQueriesLimitBytes { + if q.AsyncRequestStorage.DocCount() < asyncQueriesLimit && q.AsyncRequestStorage.SpaceInUse() < asyncQueriesLimitBytes { return false } err := errors.New("too many async queries") From 144042a92491058f76a4f0f79bbc71c4c1e37214 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Mon, 11 Nov 2024 18:14:29 +0100 Subject: [PATCH 08/30] All tests pass, maybe it works? --- quesma/persistence/elastic_with_eviction.go | 12 ++- quesma/persistence/persistence_test.go | 97 +++++++++---------- quesma/quesma/async_search_storage/evictor.go | 8 +- .../quesma/async_search_storage/in_elastic.go | 24 ++++- .../quesma/async_search_storage/in_memory.go | 8 +- .../in_memory_fallback_elastic.go | 6 +- .../async_search_storage/in_memory_test.go | 86 +++++++++++----- quesma/quesma/async_search_storage/model.go | 12 +-- 8 files changed, 152 insertions(+), 101 deletions(-) diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index b8d4d99cc..73f030dc0 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -78,9 +78,15 @@ func (db *ElasticDatabaseWithEviction) Put(document *JSONWithSize) error { } // co zwraca? zrobić switch na oba typy jakie teraz mamy? -func (db *ElasticDatabaseWithEviction) Get(id string) (string, error) { // probably change return type to *Sizeable - value, _, err := db.ElasticJSONDatabase.Get(id) - return value, err +func (db *ElasticDatabaseWithEviction) Get(id string) ([]byte, error) { // probably change return type to *Sizeable + elasticsearchURL := fmt.Sprintf("%s/_source/%s", db.indexName, id) + resp, err := db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodGet, elasticsearchURL, nil) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + return io.ReadAll(resp.Body) } func (db *ElasticDatabaseWithEviction) Delete(id string) error { diff --git a/quesma/persistence/persistence_test.go b/quesma/persistence/persistence_test.go index 8449db570..84876cfae 100644 --- a/quesma/persistence/persistence_test.go +++ b/quesma/persistence/persistence_test.go @@ -13,6 +13,8 @@ import ( "time" ) +const elasticUpdateTime = 2 * time.Second // time to wait for elastic to update + func TestNewElasticPersistence(t *testing.T) { var p JSONDatabase @@ -81,11 +83,9 @@ func TestNewElasticPersistence(t *testing.T) { if d2["foo"] != "bar" { t.Fatal("expected bar") } - } func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { - const precise = true logger.InitSimpleLoggerForTests() indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) fmt.Println("indexName:", indexName) @@ -121,42 +121,38 @@ func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { assert.NoError(t, db.Put(d)) } - if precise { - time.Sleep(updateTime) - docCount, err = db.DocCount() - assert.NoError(t, err) - assert.Equal(t, 5, docCount) - } else { - docCount, err = db.DocCount() - assert.NoError(t, err) - assert.True(t, docCount >= 0) - } + // check state after put (5 documents + "get" OK) + time.Sleep(elasticUpdateTime) + docCount, err = db.DocCount() + assert.NoError(t, err) + assert.Equal(t, 5, docCount) - val, ok := db.Get(docs[0].id) - fmt.Println(val, ok) - // TODO: deserialize and check content + val, err := db.Get(docs[0].id) + assert.NoError(t, err) + assert.Contains(t, string(val), `"id":"doc1"`) + assert.Contains(t, string(val), `"sizeInBytes":100`) + // delete some documents err = db.Delete(docs[1].id) assert.NoError(t, err) err = db.Delete(docs[3].id) assert.NoError(t, err) - if precise { - time.Sleep(updateTime) - docCount, err = db.DocCount() - assert.NoError(t, err) - assert.Equal(t, 3, docCount) - } else { - docCount, err = db.DocCount() - assert.NoError(t, err) - assert.True(t, docCount >= 0) - } + // doc_count should be 3 and "get" should fail for deleted documents + time.Sleep(elasticUpdateTime) + docCount, err = db.DocCount() + assert.NoError(t, err) + assert.Equal(t, 3, docCount) + val, err = db.Get(docs[1].id) + assert.Error(t, err) + assert.Empty(t, val) + val, err = db.Get(docs[3].id) + assert.Error(t, err) + assert.Empty(t, val) assert.Equal(t, bigSizeLimit, db.SizeInBytesLimit()) } -const updateTime = 2 * time.Second - func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { logger.InitSimpleLoggerForTests() indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) @@ -167,7 +163,7 @@ func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { cfgUrl := config.Url(*realUrl) cfg := config.ElasticsearchConfiguration{Url: &cfgUrl} - const smallSizeLimit = int64(1200) + const smallSizeLimit = int64(1100) db := NewElasticDatabaseWithEviction(cfg, indexName, smallSizeLimit) fmt.Println("indexName:", indexName, "fullIndexName:", db.fullIndexName()) @@ -187,24 +183,28 @@ func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { doc("doc1", 200), doc("doc2", 300), doc("doc3", 400), - doc("doc4", 500), + doc("doc4", 600), doc("doc5", 500), } for _, d := range docs[:2] { fmt.Println("put", d.SizeInBytesTotal, db.Put(d)) } - time.Sleep(updateTime) + time.Sleep(elasticUpdateTime) fmt.Println("put", docs[2].SizeInBytesTotal, db.Put(docs[2])) - time.Sleep(updateTime) + time.Sleep(elasticUpdateTime) docCount, err = db.DocCount() assert.NoError(t, err) assert.Equal(t, 3, docCount) + // storage should be full => error on put + err = db.Put(docs[3]) + assert.Error(t, err) + err = db.Delete("doc2") assert.NoError(t, err) - time.Sleep(updateTime) + time.Sleep(elasticUpdateTime) docCount, err = db.DocCount() assert.NoError(t, err) @@ -214,33 +214,26 @@ func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { fmt.Println("put", docs[4].SizeInBytesTotal, err) assert.NoError(t, err) - time.Sleep(3000 * time.Millisecond) + time.Sleep(elasticUpdateTime) docCount, err = db.DocCount() assert.NoError(t, err) assert.Equal(t, 3, docCount) - // - /* - val, ok := db.Get(docs[0].Id) - fmt.Println(val, ok) - // TODO: deserialize and check content - - db.Delete(docs[1].Id) - db.Delete(docs[3].Id) - - time.Sleep(1 * time.Second) - docCount, ok = db.DocCount() - assert.True(t, ok) - assert.Equal(t, 3, docCount) - } else { - docCount, ok = db.DocCount() - assert.True(t, ok) - assert.True(t, docCount >= 0) - } + val, ok := db.Get(docs[0].id) + fmt.Println(val, ok) + // TODO: deserialize and check content + err = db.Delete(docs[0].id) + assert.NoError(t, err) + err = db.Delete(docs[3].id) + assert.Error(t, err) + + time.Sleep(elasticUpdateTime) + docCount, err = db.DocCount() + assert.NoError(t, err) + assert.Equal(t, 2, docCount) - */ assert.Equal(t, smallSizeLimit, db.SizeInBytesLimit()) } diff --git a/quesma/quesma/async_search_storage/evictor.go b/quesma/quesma/async_search_storage/evictor.go index 0ee0248e9..922d436ca 100644 --- a/quesma/quesma/async_search_storage/evictor.go +++ b/quesma/quesma/async_search_storage/evictor.go @@ -19,9 +19,9 @@ func NewAsyncQueriesEvictor(AsyncRequestStorage AsyncRequestResultStorage, Async return &AsyncQueriesEvictor{ctx: ctx, cancel: cancel, AsyncRequestStorage: AsyncRequestStorage, AsyncQueriesContexts: AsyncQueriesContexts} } -func (e *AsyncQueriesEvictor) tryEvictAsyncRequests(timeFun func(time.Time) time.Duration) { - e.AsyncRequestStorage.evict(timeFun) - e.AsyncQueriesContexts.evict(timeFun) +func (e *AsyncQueriesEvictor) tryEvictAsyncRequests(olderThan time.Duration) { + e.AsyncRequestStorage.evict(olderThan) + e.AsyncQueriesContexts.evict(olderThan) } func (e *AsyncQueriesEvictor) AsyncQueriesGC() { @@ -32,7 +32,7 @@ func (e *AsyncQueriesEvictor) AsyncQueriesGC() { logger.Debug().Msg("evictor stopped") return case <-time.After(GCInterval): - e.tryEvictAsyncRequests(elapsedTime) + e.tryEvictAsyncRequests(EvictionInterval) } } } diff --git a/quesma/quesma/async_search_storage/in_elastic.go b/quesma/quesma/async_search_storage/in_elastic.go index d1630bfe6..e6f5b4ca4 100644 --- a/quesma/quesma/async_search_storage/in_elastic.go +++ b/quesma/quesma/async_search_storage/in_elastic.go @@ -3,11 +3,14 @@ package async_search_storage import ( + "encoding/json" "fmt" + "math/rand" "net/url" "quesma/logger" "quesma/persistence" "quesma/quesma/config" + "strconv" "time" ) @@ -27,8 +30,9 @@ func NewAsyncSearchStorageInElastic() AsyncSearchStorageInElastic { User: "", Password: "", } + i := rand.Int() return AsyncSearchStorageInElastic{ - db: persistence.NewElasticDatabaseWithEviction(cfg, "quesma_async_storage", 1_000_000_000), + db: persistence.NewElasticDatabaseWithEviction(cfg, "quesma_async_storage-"+strconv.Itoa(i), 1_000_000_000), } } @@ -40,8 +44,18 @@ func (s AsyncSearchStorageInElastic) Store(id string, result *AsyncRequestResult } func (s AsyncSearchStorageInElastic) Load(id string) (*AsyncRequestResult, error) { - _, err := s.db.Get(id) - return nil, err + resultAsBytes, err := s.db.Get(id) + if err != nil { + return nil, err + } + + result := AsyncRequestResult{} + err = json.Unmarshal(resultAsBytes, &result) + if err != nil { + return nil, err + } + + return &result, nil } func (s AsyncSearchStorageInElastic) Delete(id string) { @@ -82,8 +96,8 @@ func (s AsyncSearchStorageInElastic) SpaceMaxAvailable() int64 { return s.db.SizeInBytesLimit() } -func (s AsyncSearchStorageInElastic) evict() { - err := s.db.DeleteOld(EvictionInterval) +func (s AsyncSearchStorageInElastic) evict(evictOlderThan time.Duration) { + err := s.db.DeleteOld(evictOlderThan) if err != nil { logger.Warn().Err(err).Msgf("failed to evict documents, err: %v", err) } diff --git a/quesma/quesma/async_search_storage/in_memory.go b/quesma/quesma/async_search_storage/in_memory.go index 93432fe8c..f425eb78c 100644 --- a/quesma/quesma/async_search_storage/in_memory.go +++ b/quesma/quesma/async_search_storage/in_memory.go @@ -61,10 +61,10 @@ func (s AsyncSearchStorageInMemory) SpaceMaxAvailable() int64 { return math.MaxInt64 / 16 // some huge number for now, can be changed if we want to limit in-memory storage } -func (s AsyncSearchStorageInMemory) evict(timeFun func(time.Time) time.Duration) { +func (s AsyncSearchStorageInMemory) evict(evictOlderThan time.Duration) { var ids []string s.Range(func(key string, value *AsyncRequestResult) bool { - if timeFun(value.added) > EvictionInterval { + if time.Since(value.added) > evictOlderThan { ids = append(ids, key) } return true @@ -88,10 +88,10 @@ func (s AsyncQueryContextStorageInMemory) Store(id string, context *AsyncQueryCo s.idToContext.Store(id, context) } -func (s AsyncQueryContextStorageInMemory) evict(timeFun func(time.Time) time.Duration) { +func (s AsyncQueryContextStorageInMemory) evict(evictOlderThan time.Duration) { var asyncQueriesContexts []*AsyncQueryContext s.idToContext.Range(func(key string, value *AsyncQueryContext) bool { - if timeFun(value.added) > EvictionInterval { + if time.Since(value.added) > evictOlderThan { if value != nil { asyncQueriesContexts = append(asyncQueriesContexts, value) } diff --git a/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go b/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go index efb619472..8d8cfb965 100644 --- a/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go +++ b/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go @@ -47,7 +47,7 @@ func (s AsyncSearchStorageInMemoryFallbackElastic) SpaceMaxAvailable() int64 { return s.inMemory.SpaceMaxAvailable() } -func (s AsyncSearchStorageInMemoryFallbackElastic) evict(timeFun func(time.Time) time.Duration) { - s.inMemory.evict(timeFun) - go s.elastic.DeleteOld(EvictionInterval) +func (s AsyncSearchStorageInMemoryFallbackElastic) evict(olderThan time.Duration) { + s.inMemory.evict(olderThan) + go s.elastic.DeleteOld(olderThan) } diff --git a/quesma/quesma/async_search_storage/in_memory_test.go b/quesma/quesma/async_search_storage/in_memory_test.go index 525c5472b..41e35ecb5 100644 --- a/quesma/quesma/async_search_storage/in_memory_test.go +++ b/quesma/quesma/async_search_storage/in_memory_test.go @@ -6,52 +6,90 @@ import ( "context" "fmt" "github.com/ClickHouse/clickhouse-go/v2" + "github.com/k0kubun/pp" "github.com/stretchr/testify/assert" - "quesma/logger" "testing" "time" ) func TestAsyncQueriesEvictorTimePassed(t *testing.T) { // TODO: add also 3rd storage and nice test for it (remove from memory, but still in elastic) - logger.InitSimpleLoggerForTests() - for _, storage := range []AsyncRequestResultStorage{NewAsyncSearchStorageInMemory(), NewAsyncSearchStorageInElastic()} { + storageKinds := []AsyncRequestResultStorage{ + NewAsyncSearchStorageInMemory(), + NewAsyncSearchStorageInElastic(), + NewAsyncSearchStorageInMemoryFallbackElastic(), + } + for _, storage := range storageKinds { queryContextStorage := NewAsyncQueryContextStorageInMemory() queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) evictor := NewAsyncQueriesEvictor(storage, queryContextStorage) evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{added: time.Now()}) evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{added: time.Now()}) evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{added: time.Now()}) - evictor.tryEvictAsyncRequests(func(time.Time) time.Duration { - return 20 * time.Minute - }) - if _, ok := storage.(*AsyncSearchStorageInElastic); ok { - time.Sleep(1 * time.Second) - } + time.Sleep(2 * time.Second) + evictor.tryEvictAsyncRequests(1 * time.Second) + time.Sleep(2 * time.Second) + assert.Equal(t, 0, evictor.AsyncRequestStorage.DocCount()) } } func TestAsyncQueriesEvictorStillAlive(t *testing.T) { - logger.InitSimpleLoggerForTests() - for _, storage := range []AsyncRequestResultStorage{NewAsyncSearchStorageInMemory(), NewAsyncSearchStorageInElastic()} { - t.Run(fmt.Sprintf("storage: %T", storage), func(t *testing.T) { - queryContextStorage := NewAsyncQueryContextStorageInMemory() - queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) - evictor := NewAsyncQueriesEvictor(storage, queryContextStorage) - evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{added: time.Now()}) - evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{added: time.Now()}) - evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{added: time.Now()}) - evictor.tryEvictAsyncRequests(func(time.Time) time.Duration { - return time.Second - }) - - assert.Equal(t, 3, evictor.AsyncRequestStorage.DocCount()) - }) + // TODO: add also 3rd storage and nice test for it (remove from memory, but still in elastic) + storageKinds := []AsyncRequestResultStorage{ + NewAsyncSearchStorageInMemory(), + NewAsyncSearchStorageInElastic(), + NewAsyncSearchStorageInMemoryFallbackElastic(), + } + for _, storage := range storageKinds { + queryContextStorage := NewAsyncQueryContextStorageInMemory() + queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) + evictor := NewAsyncQueriesEvictor(storage, queryContextStorage) + evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{added: time.Now()}) + evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{added: time.Now()}) + evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{added: time.Now()}) + + time.Sleep(2 * time.Second) + evictor.tryEvictAsyncRequests(10 * time.Second) + time.Sleep(2 * time.Second) + + assert.Equal(t, 3, evictor.AsyncRequestStorage.DocCount()) } } +func TestInMemoryFallbackElasticStorage(t *testing.T) { + storage := NewAsyncSearchStorageInMemoryFallbackElastic() + storage.Store("1", &AsyncRequestResult{}) + storage.Store("2", &AsyncRequestResult{}) + storage.Store("3", &AsyncRequestResult{}) + + assert.Equal(t, 0, storage.elastic.DocCount()) // elastic is async, probably shouldn't be updated yet + assert.Equal(t, 3, storage.inMemory.DocCount()) + time.Sleep(2 * time.Second) + assert.Equal(t, 3, storage.elastic.DocCount()) + assert.Equal(t, 3, storage.DocCount()) + + storage.Delete("1") + storage.Delete("2") + assert.Equal(t, 1, storage.DocCount()) + assert.Equal(t, 1, storage.inMemory.DocCount()) + assert.Equal(t, 3, storage.elastic.DocCount()) // elastic is async, probably shouldn't be updated yet + time.Sleep(2 * time.Second) + assert.Equal(t, 1, storage.elastic.DocCount()) + assert.Equal(t, 1, storage.DocCount()) + + // simulate Quesma, and inMemory storage restart + storage.inMemory = NewAsyncSearchStorageInMemory() + assert.Equal(t, 0, storage.DocCount()) + assert.Equal(t, 1, storage.elastic.DocCount()) + + doc, err := storage.Load("1") + pp.Println(err, doc) + assert.Nil(t, doc) + assert.NotNil(t, err) +} + const qid = "abc" func TestKK(t *testing.T) { diff --git a/quesma/quesma/async_search_storage/model.go b/quesma/quesma/async_search_storage/model.go index 26a50f7bd..c1e46b111 100644 --- a/quesma/quesma/async_search_storage/model.go +++ b/quesma/quesma/async_search_storage/model.go @@ -20,19 +20,19 @@ type AsyncRequestResultStorage interface { SpaceInUse() int64 SpaceMaxAvailable() int64 - evict(timeFun func(time.Time) time.Duration) + evict(olderThan time.Duration) } type AsyncQueryContextStorage interface { Store(id string, context *AsyncQueryContext) - evict(timeFun func(time.Time) time.Duration) + evict(olderThan time.Duration) } type AsyncRequestResult struct { - responseBody []byte - added time.Time - isCompressed bool - err error + responseBody []byte `json:"responseBody"` + added time.Time `json:"added"` + isCompressed bool `json:"isCompressed"` + err error `json:"err"` } func NewAsyncRequestResult(responseBody []byte, err error, added time.Time, isCompressed bool) *AsyncRequestResult { From a2c621e0b2cade737cece0eb4115dae41c787fa7 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Mon, 11 Nov 2024 18:15:53 +0100 Subject: [PATCH 09/30] Linter --- quesma/quesma/async_search_storage/evictor.go | 2 ++ .../quesma/async_search_storage/in_memory_fallback_elastic.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/quesma/quesma/async_search_storage/evictor.go b/quesma/quesma/async_search_storage/evictor.go index 922d436ca..498c54e8a 100644 --- a/quesma/quesma/async_search_storage/evictor.go +++ b/quesma/quesma/async_search_storage/evictor.go @@ -1,3 +1,5 @@ +// Copyright Quesma, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 package async_search_storage import ( diff --git a/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go b/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go index 8d8cfb965..28dcd26d0 100644 --- a/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go +++ b/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go @@ -1,3 +1,5 @@ +// Copyright Quesma, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 package async_search_storage import "time" From 576b195fa3c0b6f369f49d63b25a04e9ee56dfef Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Mon, 11 Nov 2024 18:49:49 +0100 Subject: [PATCH 10/30] Minor, now should be all --- .../{in_elastic.go => in_elasticsearch.go} | 42 ++++++---- .../quesma/async_search_storage/in_memory.go | 28 +++---- .../in_memory_fallback_elastic.go | 55 ------------- .../in_memory_fallback_elasticsearch.go | 79 +++++++++++++++++++ .../async_search_storage/in_memory_test.go | 24 +++--- quesma/quesma/async_search_storage/model.go | 7 +- quesma/quesma/dual_write_proxy.go | 4 +- quesma/quesma/search.go | 6 +- 8 files changed, 141 insertions(+), 104 deletions(-) rename quesma/quesma/async_search_storage/{in_elastic.go => in_elasticsearch.go} (59%) delete mode 100644 quesma/quesma/async_search_storage/in_memory_fallback_elastic.go create mode 100644 quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go diff --git a/quesma/quesma/async_search_storage/in_elastic.go b/quesma/quesma/async_search_storage/in_elasticsearch.go similarity index 59% rename from quesma/quesma/async_search_storage/in_elastic.go rename to quesma/quesma/async_search_storage/in_elasticsearch.go index e6f5b4ca4..b15e9e451 100644 --- a/quesma/quesma/async_search_storage/in_elastic.go +++ b/quesma/quesma/async_search_storage/in_elasticsearch.go @@ -14,11 +14,11 @@ import ( "time" ) -type AsyncSearchStorageInElastic struct { +type AsyncRequestResultStorageInElasticsearch struct { db *persistence.ElasticDatabaseWithEviction } -func NewAsyncSearchStorageInElastic() AsyncSearchStorageInElastic { +func NewAsyncRequestResultStorageInElasticsearch() AsyncRequestResultStorage { // TODO use passed config realUrl, err := url.Parse("http://localhost:9200") if err != nil { @@ -31,19 +31,19 @@ func NewAsyncSearchStorageInElastic() AsyncSearchStorageInElastic { Password: "", } i := rand.Int() - return AsyncSearchStorageInElastic{ + return AsyncRequestResultStorageInElasticsearch{ db: persistence.NewElasticDatabaseWithEviction(cfg, "quesma_async_storage-"+strconv.Itoa(i), 1_000_000_000), } } -func (s AsyncSearchStorageInElastic) Store(id string, result *AsyncRequestResult) { +func (s AsyncRequestResultStorageInElasticsearch) Store(id string, result *AsyncRequestResult) { err := s.db.Put(result.toJSON(id)) if err != nil { logger.Warn().Err(err).Msg("failed to store document") } } -func (s AsyncSearchStorageInElastic) Load(id string) (*AsyncRequestResult, error) { +func (s AsyncRequestResultStorageInElasticsearch) Load(id string) (*AsyncRequestResult, error) { resultAsBytes, err := s.db.Get(id) if err != nil { return nil, err @@ -58,14 +58,14 @@ func (s AsyncSearchStorageInElastic) Load(id string) (*AsyncRequestResult, error return &result, nil } -func (s AsyncSearchStorageInElastic) Delete(id string) { +func (s AsyncRequestResultStorageInElasticsearch) Delete(id string) { err := s.db.Delete(id) if err != nil { logger.Warn().Err(err).Msg("failed to delete document") } } -func (s AsyncSearchStorageInElastic) DeleteOld(t time.Duration) { +func (s AsyncRequestResultStorageInElasticsearch) DeleteOld(t time.Duration) { err := s.db.DeleteOld(t) if err != nil { logger.Warn().Err(err).Msg("failed to delete old documents") @@ -73,7 +73,7 @@ func (s AsyncSearchStorageInElastic) DeleteOld(t time.Duration) { } // DocCount returns the number of documents in the database, or -1 if the count could not be retrieved. -func (s AsyncSearchStorageInElastic) DocCount() int { +func (s AsyncRequestResultStorageInElasticsearch) DocCount() int { cnt, err := s.db.DocCount() if err != nil { logger.Warn().Err(err).Msg("failed to get document count") @@ -83,7 +83,7 @@ func (s AsyncSearchStorageInElastic) DocCount() int { } // StorageSizeInBytes returns the total size of all documents in the database, or -1 if the size could not be retrieved. -func (s AsyncSearchStorageInElastic) SpaceInUse() int64 { +func (s AsyncRequestResultStorageInElasticsearch) SpaceInUse() int64 { size, err := s.db.SizeInBytes() if err != nil { logger.Warn().Err(err).Msg("failed to get storage size") @@ -92,24 +92,38 @@ func (s AsyncSearchStorageInElastic) SpaceInUse() int64 { return size } -func (s AsyncSearchStorageInElastic) SpaceMaxAvailable() int64 { +func (s AsyncRequestResultStorageInElasticsearch) SpaceMaxAvailable() int64 { return s.db.SizeInBytesLimit() } -func (s AsyncSearchStorageInElastic) evict(evictOlderThan time.Duration) { +func (s AsyncRequestResultStorageInElasticsearch) evict(evictOlderThan time.Duration) { err := s.db.DeleteOld(evictOlderThan) if err != nil { logger.Warn().Err(err).Msgf("failed to evict documents, err: %v", err) } } -type AsyncQueryContextStorageInElastic struct { +type AsyncQueryContextStorageInElasticsearch struct { db *persistence.ElasticDatabaseWithEviction } -func NewAsyncQueryContextStorageInElastic() AsyncQueryContextStorageInElastic { - return AsyncQueryContextStorageInElastic{ +func NewAsyncQueryContextStorageInElasticsearch() AsyncQueryContextStorage { + return AsyncQueryContextStorageInElasticsearch{ db: persistence.NewElasticDatabaseWithEviction( config.ElasticsearchConfiguration{}, "async_search", 1_000_000_000), } } + +func (s AsyncQueryContextStorageInElasticsearch) Store(context *AsyncQueryContext) { + err := s.db.Put(context.toJSON()) + if err != nil { + logger.Warn().Err(err).Msg("failed to store document") + } +} + +func (s AsyncQueryContextStorageInElasticsearch) evict(evictOlderThan time.Duration) { + err := s.db.DeleteOld(evictOlderThan) + if err != nil { + logger.Warn().Err(err).Msg("failed to delete old documents") + } +} diff --git a/quesma/quesma/async_search_storage/in_memory.go b/quesma/quesma/async_search_storage/in_memory.go index f425eb78c..32a8efc87 100644 --- a/quesma/quesma/async_search_storage/in_memory.go +++ b/quesma/quesma/async_search_storage/in_memory.go @@ -14,41 +14,41 @@ import ( "time" ) -type AsyncSearchStorageInMemory struct { +type AsyncRequestResultStorageInMemory struct { idToResult *concurrent.Map[string, *AsyncRequestResult] } -func NewAsyncSearchStorageInMemory() AsyncSearchStorageInMemory { // change result type to AsyncRequestResultStorage interface - return AsyncSearchStorageInMemory{ +func NewAsyncRequestResultStorageInMemory() AsyncRequestResultStorage { // change result type to AsyncRequestResultStorage interface + return AsyncRequestResultStorageInMemory{ idToResult: concurrent.NewMap[string, *AsyncRequestResult](), } } -func (s AsyncSearchStorageInMemory) Store(id string, result *AsyncRequestResult) { +func (s AsyncRequestResultStorageInMemory) Store(id string, result *AsyncRequestResult) { s.idToResult.Store(id, result) } -func (s AsyncSearchStorageInMemory) Range(f func(key string, value *AsyncRequestResult) bool) { +func (s AsyncRequestResultStorageInMemory) Range(f func(key string, value *AsyncRequestResult) bool) { s.idToResult.Range(f) } -func (s AsyncSearchStorageInMemory) Load(id string) (*AsyncRequestResult, error) { +func (s AsyncRequestResultStorageInMemory) Load(id string) (*AsyncRequestResult, error) { if val, ok := s.idToResult.Load(id); ok { return val, nil } return nil, fmt.Errorf("key %s not found", id) } -func (s AsyncSearchStorageInMemory) Delete(id string) { +func (s AsyncRequestResultStorageInMemory) Delete(id string) { s.idToResult.Delete(id) } -func (s AsyncSearchStorageInMemory) DocCount() int { +func (s AsyncRequestResultStorageInMemory) DocCount() int { return s.idToResult.Size() } // in bytes -func (s AsyncSearchStorageInMemory) SpaceInUse() int64 { +func (s AsyncRequestResultStorageInMemory) SpaceInUse() int64 { size := int64(0) s.Range(func(key string, value *AsyncRequestResult) bool { size += int64(len(value.GetResponseBody())) @@ -57,11 +57,11 @@ func (s AsyncSearchStorageInMemory) SpaceInUse() int64 { return size } -func (s AsyncSearchStorageInMemory) SpaceMaxAvailable() int64 { +func (s AsyncRequestResultStorageInMemory) SpaceMaxAvailable() int64 { return math.MaxInt64 / 16 // some huge number for now, can be changed if we want to limit in-memory storage } -func (s AsyncSearchStorageInMemory) evict(evictOlderThan time.Duration) { +func (s AsyncRequestResultStorageInMemory) evict(evictOlderThan time.Duration) { var ids []string s.Range(func(key string, value *AsyncRequestResult) bool { if time.Since(value.added) > evictOlderThan { @@ -78,14 +78,14 @@ type AsyncQueryContextStorageInMemory struct { idToContext *concurrent.Map[string, *AsyncQueryContext] } -func NewAsyncQueryContextStorageInMemory() AsyncQueryContextStorageInMemory { +func NewAsyncQueryContextStorageInMemory() AsyncQueryContextStorage { return AsyncQueryContextStorageInMemory{ idToContext: concurrent.NewMap[string, *AsyncQueryContext](), } } -func (s AsyncQueryContextStorageInMemory) Store(id string, context *AsyncQueryContext) { - s.idToContext.Store(id, context) +func (s AsyncQueryContextStorageInMemory) Store(context *AsyncQueryContext) { + s.idToContext.Store(context.id, context) } func (s AsyncQueryContextStorageInMemory) evict(evictOlderThan time.Duration) { diff --git a/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go b/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go deleted file mode 100644 index 28dcd26d0..000000000 --- a/quesma/quesma/async_search_storage/in_memory_fallback_elastic.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Quesma, licensed under the Elastic License 2.0. -// SPDX-License-Identifier: Elastic-2.0 -package async_search_storage - -import "time" - -type AsyncSearchStorageInMemoryFallbackElastic struct { - inMemory AsyncSearchStorageInMemory - elastic AsyncSearchStorageInElastic -} - -func NewAsyncSearchStorageInMemoryFallbackElastic() AsyncSearchStorageInMemoryFallbackElastic { - return AsyncSearchStorageInMemoryFallbackElastic{ - inMemory: NewAsyncSearchStorageInMemory(), - elastic: NewAsyncSearchStorageInElastic(), - } -} - -func (s AsyncSearchStorageInMemoryFallbackElastic) Store(id string, result *AsyncRequestResult) { - s.inMemory.Store(id, result) - go s.elastic.Store(id, result) -} - -func (s AsyncSearchStorageInMemoryFallbackElastic) Load(id string) (*AsyncRequestResult, error) { - result, err := s.inMemory.Load(id) - if err == nil { - return result, nil - } - return s.elastic.Load(id) -} - -func (s AsyncSearchStorageInMemoryFallbackElastic) Delete(id string) { - s.inMemory.Delete(id) - go s.elastic.Delete(id) -} - -// DocCount returns inMemory doc count -func (s AsyncSearchStorageInMemoryFallbackElastic) DocCount() int { - return s.inMemory.DocCount() -} - -// SizeInBytes returns inMemory size in bytes -func (s AsyncSearchStorageInMemoryFallbackElastic) SpaceInUse() int64 { - return s.inMemory.SpaceInUse() -} - -// SizeInBytesLimit returns inMemory size in bytes limit -func (s AsyncSearchStorageInMemoryFallbackElastic) SpaceMaxAvailable() int64 { - return s.inMemory.SpaceMaxAvailable() -} - -func (s AsyncSearchStorageInMemoryFallbackElastic) evict(olderThan time.Duration) { - s.inMemory.evict(olderThan) - go s.elastic.DeleteOld(olderThan) -} diff --git a/quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go b/quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go new file mode 100644 index 000000000..58287bfb3 --- /dev/null +++ b/quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go @@ -0,0 +1,79 @@ +// Copyright Quesma, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 +package async_search_storage + +import ( + "time" +) + +type AsyncRequestResultStorageInMemoryFallbackElastic struct { + inMemory AsyncRequestResultStorageInMemory + inElasticsearch AsyncRequestResultStorageInElasticsearch +} + +func NewAsyncSearchStorageInMemoryFallbackElastic() AsyncRequestResultStorageInMemoryFallbackElastic { + return AsyncRequestResultStorageInMemoryFallbackElastic{ + inMemory: NewAsyncRequestResultStorageInMemory().(AsyncRequestResultStorageInMemory), + inElasticsearch: NewAsyncRequestResultStorageInElasticsearch().(AsyncRequestResultStorageInElasticsearch), + } +} + +func (s AsyncRequestResultStorageInMemoryFallbackElastic) Store(id string, result *AsyncRequestResult) { + s.inMemory.Store(id, result) + go s.inElasticsearch.Store(id, result) +} + +func (s AsyncRequestResultStorageInMemoryFallbackElastic) Load(id string) (*AsyncRequestResult, error) { + result, err := s.inMemory.Load(id) + if err == nil { + return result, nil + } + return s.inElasticsearch.Load(id) +} + +func (s AsyncRequestResultStorageInMemoryFallbackElastic) Delete(id string) { + s.inMemory.Delete(id) + go s.inElasticsearch.Delete(id) +} + +// DocCount returns inMemory doc count +func (s AsyncRequestResultStorageInMemoryFallbackElastic) DocCount() int { + return s.inMemory.DocCount() +} + +// SpaceInUse returns inMemory size in bytes +func (s AsyncRequestResultStorageInMemoryFallbackElastic) SpaceInUse() int64 { + return s.inMemory.SpaceInUse() +} + +// SpaceMaxAvailable returns inMemory size in bytes limit +func (s AsyncRequestResultStorageInMemoryFallbackElastic) SpaceMaxAvailable() int64 { + return s.inMemory.SpaceMaxAvailable() +} + +func (s AsyncRequestResultStorageInMemoryFallbackElastic) evict(olderThan time.Duration) { + s.inMemory.evict(olderThan) + go s.inElasticsearch.DeleteOld(olderThan) +} + +type AsyncQueryContextStorageInMemoryFallbackElasticsearch struct { + inMemory AsyncQueryContextStorageInMemory + inElasticsearch AsyncQueryContextStorageInElasticsearch +} + +func NewAsyncQueryContextStorageInMemoryFallbackElasticsearch() AsyncQueryContextStorage { + return AsyncQueryContextStorageInMemoryFallbackElasticsearch{ + inMemory: NewAsyncQueryContextStorageInMemory().(AsyncQueryContextStorageInMemory), + inElasticsearch: NewAsyncQueryContextStorageInElasticsearch().(AsyncQueryContextStorageInElasticsearch), + } +} + +func (s AsyncQueryContextStorageInMemoryFallbackElasticsearch) Store(context *AsyncQueryContext) { + s.inMemory.Store(context) + go s.inElasticsearch.Store(context) +} + +func (s AsyncQueryContextStorageInMemoryFallbackElasticsearch) evict(evictOlderThan time.Duration) { + s.inMemory.evict(evictOlderThan) + go s.inElasticsearch.evict(evictOlderThan) +} diff --git a/quesma/quesma/async_search_storage/in_memory_test.go b/quesma/quesma/async_search_storage/in_memory_test.go index 41e35ecb5..0a0c1f344 100644 --- a/quesma/quesma/async_search_storage/in_memory_test.go +++ b/quesma/quesma/async_search_storage/in_memory_test.go @@ -15,12 +15,12 @@ import ( func TestAsyncQueriesEvictorTimePassed(t *testing.T) { // TODO: add also 3rd storage and nice test for it (remove from memory, but still in elastic) storageKinds := []AsyncRequestResultStorage{ - NewAsyncSearchStorageInMemory(), - NewAsyncSearchStorageInElastic(), + NewAsyncRequestResultStorageInMemory(), + NewAsyncRequestResultStorageInElasticsearch(), NewAsyncSearchStorageInMemoryFallbackElastic(), } for _, storage := range storageKinds { - queryContextStorage := NewAsyncQueryContextStorageInMemory() + queryContextStorage := NewAsyncQueryContextStorageInMemory().(AsyncQueryContextStorageInMemory) queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) evictor := NewAsyncQueriesEvictor(storage, queryContextStorage) evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{added: time.Now()}) @@ -38,12 +38,12 @@ func TestAsyncQueriesEvictorTimePassed(t *testing.T) { func TestAsyncQueriesEvictorStillAlive(t *testing.T) { // TODO: add also 3rd storage and nice test for it (remove from memory, but still in elastic) storageKinds := []AsyncRequestResultStorage{ - NewAsyncSearchStorageInMemory(), - NewAsyncSearchStorageInElastic(), + NewAsyncRequestResultStorageInMemory(), + NewAsyncRequestResultStorageInElasticsearch(), NewAsyncSearchStorageInMemoryFallbackElastic(), } for _, storage := range storageKinds { - queryContextStorage := NewAsyncQueryContextStorageInMemory() + queryContextStorage := NewAsyncQueryContextStorageInMemory().(AsyncQueryContextStorageInMemory) queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) evictor := NewAsyncQueriesEvictor(storage, queryContextStorage) evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{added: time.Now()}) @@ -64,25 +64,25 @@ func TestInMemoryFallbackElasticStorage(t *testing.T) { storage.Store("2", &AsyncRequestResult{}) storage.Store("3", &AsyncRequestResult{}) - assert.Equal(t, 0, storage.elastic.DocCount()) // elastic is async, probably shouldn't be updated yet + assert.Equal(t, 0, storage.inElasticsearch.DocCount()) // inElasticsearch is async, probably shouldn't be updated yet assert.Equal(t, 3, storage.inMemory.DocCount()) time.Sleep(2 * time.Second) - assert.Equal(t, 3, storage.elastic.DocCount()) + assert.Equal(t, 3, storage.inElasticsearch.DocCount()) assert.Equal(t, 3, storage.DocCount()) storage.Delete("1") storage.Delete("2") assert.Equal(t, 1, storage.DocCount()) assert.Equal(t, 1, storage.inMemory.DocCount()) - assert.Equal(t, 3, storage.elastic.DocCount()) // elastic is async, probably shouldn't be updated yet + assert.Equal(t, 3, storage.inElasticsearch.DocCount()) // inElasticsearch is async, probably shouldn't be updated yet time.Sleep(2 * time.Second) - assert.Equal(t, 1, storage.elastic.DocCount()) + assert.Equal(t, 1, storage.inElasticsearch.DocCount()) assert.Equal(t, 1, storage.DocCount()) // simulate Quesma, and inMemory storage restart - storage.inMemory = NewAsyncSearchStorageInMemory() + storage.inMemory = NewAsyncRequestResultStorageInMemory().(AsyncRequestResultStorageInMemory) assert.Equal(t, 0, storage.DocCount()) - assert.Equal(t, 1, storage.elastic.DocCount()) + assert.Equal(t, 1, storage.inElasticsearch.DocCount()) doc, err := storage.Load("1") pp.Println(err, doc) diff --git a/quesma/quesma/async_search_storage/model.go b/quesma/quesma/async_search_storage/model.go index c1e46b111..e5ca810fc 100644 --- a/quesma/quesma/async_search_storage/model.go +++ b/quesma/quesma/async_search_storage/model.go @@ -24,7 +24,7 @@ type AsyncRequestResultStorage interface { } type AsyncQueryContextStorage interface { - Store(id string, context *AsyncQueryContext) + Store(context *AsyncQueryContext) evict(olderThan time.Duration) } @@ -71,10 +71,9 @@ func NewAsyncQueryContext(ctx context.Context, cancel context.CancelFunc, id str return &AsyncQueryContext{ctx: ctx, cancel: cancel, added: time.Now(), id: id} } -func (c *AsyncQueryContext) toJSON() types.JSON { +func (c *AsyncQueryContext) toJSON() *persistence.JSONWithSize { json := types.JSON{} json["id"] = c.id json["added"] = c.added - - return json + return persistence.NewJSONWithSize(json, c.id, 100) // 100 is a rough upper bound estimate of the size of the rest of the fields } diff --git a/quesma/quesma/dual_write_proxy.go b/quesma/quesma/dual_write_proxy.go index a56506be4..321023efc 100644 --- a/quesma/quesma/dual_write_proxy.go +++ b/quesma/quesma/dual_write_proxy.go @@ -118,8 +118,8 @@ func newDualWriteProxy(schemaLoader clickhouse.TableDiscovery, logManager *click logManager: logManager, publicPort: config.PublicTcpPort, asyncQueriesEvictor: async_search_storage.NewAsyncQueriesEvictor( - queryRunner.AsyncRequestStorage.(async_search_storage.AsyncSearchStorageInMemory), - queryRunner.AsyncQueriesContexts.(async_search_storage.AsyncQueryContextStorageInMemory), + queryRunner.AsyncRequestStorage, + queryRunner.AsyncQueriesContexts, ), queryRunner: queryRunner, } diff --git a/quesma/quesma/search.go b/quesma/quesma/search.go index 63e7e3339..a2b9659fc 100644 --- a/quesma/quesma/search.go +++ b/quesma/quesma/search.go @@ -77,8 +77,8 @@ func NewQueryRunner(lm *clickhouse.LogManager, return &QueryRunner{logManager: lm, cfg: cfg, im: im, quesmaManagementConsole: qmc, executionCtx: ctx, cancel: cancel, - AsyncRequestStorage: async_search_storage.NewAsyncSearchStorageInMemory(), - AsyncQueriesContexts: async_search_storage.NewAsyncQueryContextStorageInMemory(), + AsyncRequestStorage: async_search_storage.NewAsyncSearchStorageInMemoryFallbackElastic(), + AsyncQueriesContexts: async_search_storage.NewAsyncQueryContextStorageInMemoryFallbackElasticsearch(), transformationPipeline: TransformationPipeline{ transformers: []model.QueryTransformer{ &SchemaCheckPass{cfg: cfg}, @@ -544,7 +544,7 @@ func (q *QueryRunner) reachedQueriesLimit(ctx context.Context, asyncId string, d } func (q *QueryRunner) addAsyncQueryContext(ctx context.Context, cancel context.CancelFunc, asyncRequestIdStr string) { - q.AsyncQueriesContexts.Store(asyncRequestIdStr, async_search_storage.NewAsyncQueryContext(ctx, cancel, asyncRequestIdStr)) + q.AsyncQueriesContexts.Store(async_search_storage.NewAsyncQueryContext(ctx, cancel, asyncRequestIdStr)) } // This is a HACK From 47229e0e86d5434e7d0dbdf82736761d0ca57b09 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Mon, 11 Nov 2024 18:56:10 +0100 Subject: [PATCH 11/30] Fix linter --- quesma/persistence/elastic.go | 2 +- quesma/persistence/elastic_with_eviction.go | 7 ----- quesma/persistence/persistence_test.go | 4 ++- .../quesma/async_search_storage/in_memory.go | 4 +-- .../async_search_storage/in_memory_test.go | 21 +++++++------- quesma/quesma/async_search_storage/model.go | 28 ++++++------------- quesma/quesma/search.go | 8 +++--- 7 files changed, 29 insertions(+), 45 deletions(-) diff --git a/quesma/persistence/elastic.go b/quesma/persistence/elastic.go index 73fa1adcb..b6a806b5a 100644 --- a/quesma/persistence/elastic.go +++ b/quesma/persistence/elastic.go @@ -118,10 +118,10 @@ func (p *ElasticJSONDatabase) List() ([]string, error) { }` resp, err := p.httpClient.Request(context.Background(), "GET", elasticsearchURL, []byte(query)) - defer resp.Body.Close() if err != nil { return nil, err } + defer resp.Body.Close() jsonAsBytes, err := io.ReadAll(resp.Body) if err != nil { diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index 73f030dc0..d403ecdc8 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -16,9 +16,6 @@ import ( "time" ) -const MAX_DOC_COUNT = 10000 // TODO: fix/make configurable/idk/etc -const defaultHugeSizeInBytesLimit = int64(500_000_000_000) // 500GB - // so far I serialize entire struct and keep only 1 string in ES type ElasticDatabaseWithEviction struct { ctx context.Context @@ -270,10 +267,6 @@ func (db *ElasticDatabaseWithEviction) getAll() (documents []*JSONWithSize, err return documents, nil } -func (db *ElasticDatabaseWithEviction) evict(indexes []string) { - // todo -} - func (db *ElasticDatabaseWithEviction) fullIndexName() string { now := time.Now().UTC() return fmt.Sprintf("%s-%d-%d-%d-%d-%d-%d", db.indexName, now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second()) diff --git a/quesma/persistence/persistence_test.go b/quesma/persistence/persistence_test.go index 84876cfae..527c68a67 100644 --- a/quesma/persistence/persistence_test.go +++ b/quesma/persistence/persistence_test.go @@ -20,7 +20,7 @@ func TestNewElasticPersistence(t *testing.T) { var p JSONDatabase // change to false if you want to test non-trivial persistence - if false { + if true { p = NewStaticJSONDatabase() } else { indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMicro()) @@ -86,6 +86,7 @@ func TestNewElasticPersistence(t *testing.T) { } func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { + t.Skip("passes locally, but requires elasticsearch to be running, so skipping") logger.InitSimpleLoggerForTests() indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) fmt.Println("indexName:", indexName) @@ -154,6 +155,7 @@ func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { } func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { + t.Skip("passes locally, but requires elasticsearch to be running, so skipping") logger.InitSimpleLoggerForTests() indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) diff --git a/quesma/quesma/async_search_storage/in_memory.go b/quesma/quesma/async_search_storage/in_memory.go index 32a8efc87..435f5a305 100644 --- a/quesma/quesma/async_search_storage/in_memory.go +++ b/quesma/quesma/async_search_storage/in_memory.go @@ -51,7 +51,7 @@ func (s AsyncRequestResultStorageInMemory) DocCount() int { func (s AsyncRequestResultStorageInMemory) SpaceInUse() int64 { size := int64(0) s.Range(func(key string, value *AsyncRequestResult) bool { - size += int64(len(value.GetResponseBody())) + size += int64(len(value.ResponseBody)) return true }) return size @@ -64,7 +64,7 @@ func (s AsyncRequestResultStorageInMemory) SpaceMaxAvailable() int64 { func (s AsyncRequestResultStorageInMemory) evict(evictOlderThan time.Duration) { var ids []string s.Range(func(key string, value *AsyncRequestResult) bool { - if time.Since(value.added) > evictOlderThan { + if time.Since(value.Added) > evictOlderThan { ids = append(ids, key) } return true diff --git a/quesma/quesma/async_search_storage/in_memory_test.go b/quesma/quesma/async_search_storage/in_memory_test.go index 0a0c1f344..514e71725 100644 --- a/quesma/quesma/async_search_storage/in_memory_test.go +++ b/quesma/quesma/async_search_storage/in_memory_test.go @@ -16,16 +16,16 @@ func TestAsyncQueriesEvictorTimePassed(t *testing.T) { // TODO: add also 3rd storage and nice test for it (remove from memory, but still in elastic) storageKinds := []AsyncRequestResultStorage{ NewAsyncRequestResultStorageInMemory(), - NewAsyncRequestResultStorageInElasticsearch(), - NewAsyncSearchStorageInMemoryFallbackElastic(), + //NewAsyncRequestResultStorageInElasticsearch(), passes + //NewAsyncSearchStorageInMemoryFallbackElastic(), passes } for _, storage := range storageKinds { queryContextStorage := NewAsyncQueryContextStorageInMemory().(AsyncQueryContextStorageInMemory) queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) evictor := NewAsyncQueriesEvictor(storage, queryContextStorage) - evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{added: time.Now()}) - evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{added: time.Now()}) - evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{added: time.Now()}) + evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{Added: time.Now()}) + evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{Added: time.Now()}) + evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{Added: time.Now()}) time.Sleep(2 * time.Second) evictor.tryEvictAsyncRequests(1 * time.Second) @@ -39,16 +39,16 @@ func TestAsyncQueriesEvictorStillAlive(t *testing.T) { // TODO: add also 3rd storage and nice test for it (remove from memory, but still in elastic) storageKinds := []AsyncRequestResultStorage{ NewAsyncRequestResultStorageInMemory(), - NewAsyncRequestResultStorageInElasticsearch(), - NewAsyncSearchStorageInMemoryFallbackElastic(), + //NewAsyncRequestResultStorageInElasticsearch(), passes + //NewAsyncSearchStorageInMemoryFallbackElastic(), passes } for _, storage := range storageKinds { queryContextStorage := NewAsyncQueryContextStorageInMemory().(AsyncQueryContextStorageInMemory) queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) evictor := NewAsyncQueriesEvictor(storage, queryContextStorage) - evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{added: time.Now()}) - evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{added: time.Now()}) - evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{added: time.Now()}) + evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{Added: time.Now()}) + evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{Added: time.Now()}) + evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{Added: time.Now()}) time.Sleep(2 * time.Second) evictor.tryEvictAsyncRequests(10 * time.Second) @@ -59,6 +59,7 @@ func TestAsyncQueriesEvictorStillAlive(t *testing.T) { } func TestInMemoryFallbackElasticStorage(t *testing.T) { + t.Skip("passes locally, but requires elasticsearch to be running, so skipping") storage := NewAsyncSearchStorageInMemoryFallbackElastic() storage.Store("1", &AsyncRequestResult{}) storage.Store("2", &AsyncRequestResult{}) diff --git a/quesma/quesma/async_search_storage/model.go b/quesma/quesma/async_search_storage/model.go index e5ca810fc..6d52c17ce 100644 --- a/quesma/quesma/async_search_storage/model.go +++ b/quesma/quesma/async_search_storage/model.go @@ -29,34 +29,22 @@ type AsyncQueryContextStorage interface { } type AsyncRequestResult struct { - responseBody []byte `json:"responseBody"` - added time.Time `json:"added"` - isCompressed bool `json:"isCompressed"` - err error `json:"err"` + ResponseBody []byte `json:"responseBody"` + Added time.Time `json:"added"` + IsCompressed bool `json:"isCompressed"` + Err error `json:"err"` } func NewAsyncRequestResult(responseBody []byte, err error, added time.Time, isCompressed bool) *AsyncRequestResult { - return &AsyncRequestResult{responseBody: responseBody, err: err, added: added, isCompressed: isCompressed} -} - -func (r *AsyncRequestResult) GetResponseBody() []byte { - return r.responseBody -} - -func (r *AsyncRequestResult) GetErr() error { - return r.err -} - -func (r *AsyncRequestResult) IsCompressed() bool { - return r.isCompressed + return &AsyncRequestResult{ResponseBody: responseBody, Err: err, Added: added, IsCompressed: isCompressed} } func (r *AsyncRequestResult) toJSON(id string) *persistence.JSONWithSize { json := types.JSON{} json["id"] = id - json["data"] = string(r.responseBody) - json["sizeInBytes"] = int64(len(r.responseBody)) + int64(len(id)) + 100 // 100 is a rough upper bound estimate of the size of the rest of the fields - json["added"] = r.added + json["data"] = string(r.ResponseBody) + json["sizeInBytes"] = int64(len(r.ResponseBody)) + int64(len(id)) + 100 // 100 is a rough upper bound estimate of the size of the rest of the fields + json["added"] = r.Added return persistence.NewJSONWithSize(json, id, json["sizeInBytes"].(int64)) } diff --git a/quesma/quesma/search.go b/quesma/quesma/search.go index dab7ca17d..d65a39098 100644 --- a/quesma/quesma/search.go +++ b/quesma/quesma/search.go @@ -496,15 +496,15 @@ func (q *QueryRunner) handlePartialAsyncSearch(ctx context.Context, id string) ( return queryparser.EmptyAsyncSearchResponse(id, false, 503) } if result, err := q.AsyncRequestStorage.Load(id); err != nil { - if err := result.GetErr(); err != nil { + if result.Err != nil { q.AsyncRequestStorage.Delete(id) logger.ErrorWithCtx(ctx).Msgf("error processing async query: %v", err) return queryparser.EmptyAsyncSearchResponse(id, false, 503) } q.AsyncRequestStorage.Delete(id) // We use zstd to conserve memory, as we have a lot of async queries - if result.IsCompressed() { - buf, err := util.Decompress(result.GetResponseBody()) + if result.IsCompressed { + buf, err := util.Decompress(result.ResponseBody) if err == nil { // Mark trace end is called only when the async query is fully processed // which means that isPartial is false @@ -517,7 +517,7 @@ func (q *QueryRunner) handlePartialAsyncSearch(ctx context.Context, id string) ( // Mark trace end is called only when the async query is fully processed // which means that isPartial is false logger.MarkTraceEndWithCtx(ctx).Msgf("Async query id : %s ended successfully", id) - return result.GetResponseBody(), nil + return result.ResponseBody, nil } else { const isPartial = true logger.InfoWithCtx(ctx).Msgf("async query id : %s partial result", id) From 47cc725d713cafa5bb627517d5c33aa1390588bf Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Mon, 11 Nov 2024 19:25:11 +0100 Subject: [PATCH 12/30] Fix config data for smoke test --- .../async_search_storage/in_elasticsearch.go | 9 ++++----- .../in_memory_fallback_elasticsearch.go | 5 +++-- .../{in_memory_test.go => storage_test.go} | 17 ++++++++++++++++- quesma/quesma/search.go | 2 +- 4 files changed, 24 insertions(+), 9 deletions(-) rename quesma/quesma/async_search_storage/{in_memory_test.go => storage_test.go} (91%) diff --git a/quesma/quesma/async_search_storage/in_elasticsearch.go b/quesma/quesma/async_search_storage/in_elasticsearch.go index b15e9e451..0fba4fddd 100644 --- a/quesma/quesma/async_search_storage/in_elasticsearch.go +++ b/quesma/quesma/async_search_storage/in_elasticsearch.go @@ -4,9 +4,7 @@ package async_search_storage import ( "encoding/json" - "fmt" "math/rand" - "net/url" "quesma/logger" "quesma/persistence" "quesma/quesma/config" @@ -18,9 +16,9 @@ type AsyncRequestResultStorageInElasticsearch struct { db *persistence.ElasticDatabaseWithEviction } -func NewAsyncRequestResultStorageInElasticsearch() AsyncRequestResultStorage { - // TODO use passed config - realUrl, err := url.Parse("http://localhost:9200") +func NewAsyncRequestResultStorageInElasticsearch(cfg config.ElasticsearchConfiguration) AsyncRequestResultStorage { + /* some test config, maybe you'll find it easier to debug with it + realUrl, err := url.Parse("http://localhost:9201") if err != nil { fmt.Println("ERR", err) } @@ -30,6 +28,7 @@ func NewAsyncRequestResultStorageInElasticsearch() AsyncRequestResultStorage { User: "", Password: "", } + */ i := rand.Int() return AsyncRequestResultStorageInElasticsearch{ db: persistence.NewElasticDatabaseWithEviction(cfg, "quesma_async_storage-"+strconv.Itoa(i), 1_000_000_000), diff --git a/quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go b/quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go index 58287bfb3..7498f0dcf 100644 --- a/quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go +++ b/quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go @@ -3,6 +3,7 @@ package async_search_storage import ( + "quesma/quesma/config" "time" ) @@ -11,10 +12,10 @@ type AsyncRequestResultStorageInMemoryFallbackElastic struct { inElasticsearch AsyncRequestResultStorageInElasticsearch } -func NewAsyncSearchStorageInMemoryFallbackElastic() AsyncRequestResultStorageInMemoryFallbackElastic { +func NewAsyncSearchStorageInMemoryFallbackElastic(cfg config.ElasticsearchConfiguration) AsyncRequestResultStorageInMemoryFallbackElastic { return AsyncRequestResultStorageInMemoryFallbackElastic{ inMemory: NewAsyncRequestResultStorageInMemory().(AsyncRequestResultStorageInMemory), - inElasticsearch: NewAsyncRequestResultStorageInElasticsearch().(AsyncRequestResultStorageInElasticsearch), + inElasticsearch: NewAsyncRequestResultStorageInElasticsearch(cfg).(AsyncRequestResultStorageInElasticsearch), } } diff --git a/quesma/quesma/async_search_storage/in_memory_test.go b/quesma/quesma/async_search_storage/storage_test.go similarity index 91% rename from quesma/quesma/async_search_storage/in_memory_test.go rename to quesma/quesma/async_search_storage/storage_test.go index 514e71725..d0904c3a1 100644 --- a/quesma/quesma/async_search_storage/in_memory_test.go +++ b/quesma/quesma/async_search_storage/storage_test.go @@ -8,6 +8,8 @@ import ( "github.com/ClickHouse/clickhouse-go/v2" "github.com/k0kubun/pp" "github.com/stretchr/testify/assert" + "net/url" + "quesma/quesma/config" "testing" "time" ) @@ -60,7 +62,7 @@ func TestAsyncQueriesEvictorStillAlive(t *testing.T) { func TestInMemoryFallbackElasticStorage(t *testing.T) { t.Skip("passes locally, but requires elasticsearch to be running, so skipping") - storage := NewAsyncSearchStorageInMemoryFallbackElastic() + storage := NewAsyncSearchStorageInMemoryFallbackElastic(testConfig()) storage.Store("1", &AsyncRequestResult{}) storage.Store("2", &AsyncRequestResult{}) storage.Store("3", &AsyncRequestResult{}) @@ -93,6 +95,19 @@ func TestInMemoryFallbackElasticStorage(t *testing.T) { const qid = "abc" +func testConfig() config.ElasticsearchConfiguration { + realUrl, err := url.Parse("http://localhost:9201") + if err != nil { + fmt.Println("ERR", err) + } + cfgUrl := config.Url(*realUrl) + return config.ElasticsearchConfiguration{ + Url: &cfgUrl, + User: "", + Password: "", + } +} + func TestKK(t *testing.T) { t.Skip() options := clickhouse.Options{Addr: []string{"localhost:9000"}} diff --git a/quesma/quesma/search.go b/quesma/quesma/search.go index d65a39098..574f47c1a 100644 --- a/quesma/quesma/search.go +++ b/quesma/quesma/search.go @@ -77,7 +77,7 @@ func NewQueryRunner(lm *clickhouse.LogManager, return &QueryRunner{logManager: lm, cfg: cfg, im: im, quesmaManagementConsole: qmc, executionCtx: ctx, cancel: cancel, - AsyncRequestStorage: async_search_storage.NewAsyncSearchStorageInMemoryFallbackElastic(), + AsyncRequestStorage: async_search_storage.NewAsyncSearchStorageInMemoryFallbackElastic(cfg.Elasticsearch), AsyncQueriesContexts: async_search_storage.NewAsyncQueryContextStorageInMemoryFallbackElasticsearch(), transformationPipeline: TransformationPipeline{ transformers: []model.QueryTransformer{ From 385fcb3d03244b7f4188527e025c7a7047350a23 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Tue, 12 Nov 2024 08:05:01 +0100 Subject: [PATCH 13/30] Debug for smoke test --- smoke-test/async_query.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/smoke-test/async_query.go b/smoke-test/async_query.go index 172fedf3e..95a42f4dc 100644 --- a/smoke-test/async_query.go +++ b/smoke-test/async_query.go @@ -286,7 +286,8 @@ func waitForAsyncQuery(timeout time.Duration) { } defer resp.Body.Close() if resp.StatusCode != 200 { - fmt.Printf("async query status is %d %s\n", resp.StatusCode, asyncGetQueryUrlPrefix+asyncQuery.Id) + body, _ = io.ReadAll(resp.Body) + fmt.Printf("async query status is %d %s, resp body: %s\n", resp.StatusCode, asyncGetQueryUrlPrefix+asyncQuery.Id, body) panic("async query status is not 200") } } From c37e8e393a19818078b47452aa645f44c2242af2 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Tue, 12 Nov 2024 09:04:19 +0100 Subject: [PATCH 14/30] Some fixes for smoke test --- quesma/main.go | 2 ++ quesma/persistence/elastic_with_eviction.go | 3 +++ quesma/quesma/async_search_storage/in_elasticsearch.go | 8 +++++--- .../in_memory_fallback_elasticsearch.go | 4 ++-- quesma/quesma/async_search_storage/model.go | 4 +++- quesma/quesma/search.go | 7 +++++-- 6 files changed, 20 insertions(+), 8 deletions(-) diff --git a/quesma/main.go b/quesma/main.go index bc8e00a40..ff6eb612c 100644 --- a/quesma/main.go +++ b/quesma/main.go @@ -53,6 +53,8 @@ func main() { var newConfiguration = config.LoadV2Config() var cfg = newConfiguration.TranslateToLegacyConfig() + fmt.Printf("kk dbg cfg: %+v", cfg) + if err := cfg.Validate(); err != nil { log.Fatalf("error validating configuration: %v", err) } diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index d403ecdc8..26695b4b1 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/k0kubun/pp" "io" "math" "net/http" @@ -34,6 +35,7 @@ func NewElasticDatabaseWithEviction(cfg config.ElasticsearchConfiguration, index } func (db *ElasticDatabaseWithEviction) Put(document *JSONWithSize) error { + pp.Println(db) dbSize, err := db.SizeInBytes() if err != nil { return err @@ -198,6 +200,7 @@ func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, err err a := make([]int64, 0) for _, hit := range result["hits"].(map[string]interface{})["hits"].([]interface{}) { + pp.Println("hit:", hit) b := sizeInBytes sizeInBytes += int64(hit.(map[string]interface{})["_source"].(map[string]interface{})["sizeInBytes"].(float64)) // TODO: add checks a = append(a, sizeInBytes-b) diff --git a/quesma/quesma/async_search_storage/in_elasticsearch.go b/quesma/quesma/async_search_storage/in_elasticsearch.go index 0fba4fddd..9cae5ddff 100644 --- a/quesma/quesma/async_search_storage/in_elasticsearch.go +++ b/quesma/quesma/async_search_storage/in_elasticsearch.go @@ -4,6 +4,7 @@ package async_search_storage import ( "encoding/json" + "fmt" "math/rand" "quesma/logger" "quesma/persistence" @@ -30,6 +31,7 @@ func NewAsyncRequestResultStorageInElasticsearch(cfg config.ElasticsearchConfigu } */ i := rand.Int() + fmt.Println("kk dbg NewAsyncRequestResultStorageInElasticsearch() i:", cfg) return AsyncRequestResultStorageInElasticsearch{ db: persistence.NewElasticDatabaseWithEviction(cfg, "quesma_async_storage-"+strconv.Itoa(i), 1_000_000_000), } @@ -106,10 +108,10 @@ type AsyncQueryContextStorageInElasticsearch struct { db *persistence.ElasticDatabaseWithEviction } -func NewAsyncQueryContextStorageInElasticsearch() AsyncQueryContextStorage { +func NewAsyncQueryContextStorageInElasticsearch(cfg config.ElasticsearchConfiguration) AsyncQueryContextStorage { + fmt.Println("kk dbg NewAsyncQueryContextStorageInElasticsearch() i:", cfg) return AsyncQueryContextStorageInElasticsearch{ - db: persistence.NewElasticDatabaseWithEviction( - config.ElasticsearchConfiguration{}, "async_search", 1_000_000_000), + db: persistence.NewElasticDatabaseWithEviction(cfg, "async_search", 1_000_000_000), } } diff --git a/quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go b/quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go index 7498f0dcf..5cdc15c62 100644 --- a/quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go +++ b/quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go @@ -62,10 +62,10 @@ type AsyncQueryContextStorageInMemoryFallbackElasticsearch struct { inElasticsearch AsyncQueryContextStorageInElasticsearch } -func NewAsyncQueryContextStorageInMemoryFallbackElasticsearch() AsyncQueryContextStorage { +func NewAsyncQueryContextStorageInMemoryFallbackElasticsearch(cfg config.ElasticsearchConfiguration) AsyncQueryContextStorage { return AsyncQueryContextStorageInMemoryFallbackElasticsearch{ inMemory: NewAsyncQueryContextStorageInMemory().(AsyncQueryContextStorageInMemory), - inElasticsearch: NewAsyncQueryContextStorageInElasticsearch().(AsyncQueryContextStorageInElasticsearch), + inElasticsearch: NewAsyncQueryContextStorageInElasticsearch(cfg).(AsyncQueryContextStorageInElasticsearch), } } diff --git a/quesma/quesma/async_search_storage/model.go b/quesma/quesma/async_search_storage/model.go index 6d52c17ce..6ece7e465 100644 --- a/quesma/quesma/async_search_storage/model.go +++ b/quesma/quesma/async_search_storage/model.go @@ -60,8 +60,10 @@ func NewAsyncQueryContext(ctx context.Context, cancel context.CancelFunc, id str } func (c *AsyncQueryContext) toJSON() *persistence.JSONWithSize { + sizeInBytesUpperBoundEstimate := int64(100) json := types.JSON{} json["id"] = c.id json["added"] = c.added - return persistence.NewJSONWithSize(json, c.id, 100) // 100 is a rough upper bound estimate of the size of the rest of the fields + json["sizeInBytes"] = sizeInBytesUpperBoundEstimate + return persistence.NewJSONWithSize(json, c.id, sizeInBytesUpperBoundEstimate) } diff --git a/quesma/quesma/search.go b/quesma/quesma/search.go index 574f47c1a..00bd80383 100644 --- a/quesma/quesma/search.go +++ b/quesma/quesma/search.go @@ -6,6 +6,7 @@ import ( "context" "errors" "fmt" + "github.com/k0kubun/pp" "quesma/ab_testing" "quesma/clickhouse" "quesma/common_table" @@ -75,10 +76,12 @@ func NewQueryRunner(lm *clickhouse.LogManager, ctx, cancel := context.WithCancel(context.Background()) + pp.Println("cfg tuz przed", cfg) + return &QueryRunner{logManager: lm, cfg: cfg, im: im, quesmaManagementConsole: qmc, executionCtx: ctx, cancel: cancel, AsyncRequestStorage: async_search_storage.NewAsyncSearchStorageInMemoryFallbackElastic(cfg.Elasticsearch), - AsyncQueriesContexts: async_search_storage.NewAsyncQueryContextStorageInMemoryFallbackElasticsearch(), + AsyncQueriesContexts: async_search_storage.NewAsyncQueryContextStorageInMemoryFallbackElasticsearch(cfg.Elasticsearch), transformationPipeline: TransformationPipeline{ transformers: []model.QueryTransformer{ &SchemaCheckPass{cfg: cfg}, @@ -495,7 +498,7 @@ func (q *QueryRunner) handlePartialAsyncSearch(ctx context.Context, id string) ( logger.ErrorWithCtx(ctx).Msgf("non quesma async id: %v", id) return queryparser.EmptyAsyncSearchResponse(id, false, 503) } - if result, err := q.AsyncRequestStorage.Load(id); err != nil { + if result, err := q.AsyncRequestStorage.Load(id); result != nil && err != nil { if result.Err != nil { q.AsyncRequestStorage.Delete(id) logger.ErrorWithCtx(ctx).Msgf("error processing async query: %v", err) From 57982de2b475023b009996f1afa472f1759cedf3 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Tue, 12 Nov 2024 09:14:42 +0100 Subject: [PATCH 15/30] unskip all tests --- quesma/persistence/persistence_test.go | 4 ++-- .../async_search_storage/storage_test.go | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/quesma/persistence/persistence_test.go b/quesma/persistence/persistence_test.go index 527c68a67..ef34a91e7 100644 --- a/quesma/persistence/persistence_test.go +++ b/quesma/persistence/persistence_test.go @@ -86,7 +86,7 @@ func TestNewElasticPersistence(t *testing.T) { } func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { - t.Skip("passes locally, but requires elasticsearch to be running, so skipping") + // reskip after merge t.Skip("passes locally, but requires elasticsearch to be running, so skipping") logger.InitSimpleLoggerForTests() indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) fmt.Println("indexName:", indexName) @@ -155,7 +155,7 @@ func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { } func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { - t.Skip("passes locally, but requires elasticsearch to be running, so skipping") + // reskip after merge t.Skip("passes locally, but requires elasticsearch to be running, so skipping") logger.InitSimpleLoggerForTests() indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) diff --git a/quesma/quesma/async_search_storage/storage_test.go b/quesma/quesma/async_search_storage/storage_test.go index d0904c3a1..64a64dd9d 100644 --- a/quesma/quesma/async_search_storage/storage_test.go +++ b/quesma/quesma/async_search_storage/storage_test.go @@ -16,10 +16,13 @@ import ( func TestAsyncQueriesEvictorTimePassed(t *testing.T) { // TODO: add also 3rd storage and nice test for it (remove from memory, but still in elastic) + realUrl, _ := url.Parse("http://localhost:9201") + cfgUrl := config.Url(*realUrl) + cfg := config.ElasticsearchConfiguration{Url: &cfgUrl} storageKinds := []AsyncRequestResultStorage{ NewAsyncRequestResultStorageInMemory(), - //NewAsyncRequestResultStorageInElasticsearch(), passes - //NewAsyncSearchStorageInMemoryFallbackElastic(), passes + NewAsyncRequestResultStorageInElasticsearch(cfg), // passes, reskip after merge + NewAsyncSearchStorageInMemoryFallbackElastic(cfg), // passes, reskip after merge } for _, storage := range storageKinds { queryContextStorage := NewAsyncQueryContextStorageInMemory().(AsyncQueryContextStorageInMemory) @@ -39,10 +42,13 @@ func TestAsyncQueriesEvictorTimePassed(t *testing.T) { func TestAsyncQueriesEvictorStillAlive(t *testing.T) { // TODO: add also 3rd storage and nice test for it (remove from memory, but still in elastic) + realUrl, _ := url.Parse("http://localhost:9201") + cfgUrl := config.Url(*realUrl) + cfg := config.ElasticsearchConfiguration{Url: &cfgUrl} storageKinds := []AsyncRequestResultStorage{ NewAsyncRequestResultStorageInMemory(), - //NewAsyncRequestResultStorageInElasticsearch(), passes - //NewAsyncSearchStorageInMemoryFallbackElastic(), passes + NewAsyncRequestResultStorageInElasticsearch(cfg), // passes, reskip after merge + NewAsyncSearchStorageInMemoryFallbackElastic(cfg), // passes, reskip after merge } for _, storage := range storageKinds { queryContextStorage := NewAsyncQueryContextStorageInMemory().(AsyncQueryContextStorageInMemory) @@ -61,7 +67,7 @@ func TestAsyncQueriesEvictorStillAlive(t *testing.T) { } func TestInMemoryFallbackElasticStorage(t *testing.T) { - t.Skip("passes locally, but requires elasticsearch to be running, so skipping") + // reskip after merge t.Skip("passes locally, but requires elasticsearch to be running, so skipping") storage := NewAsyncSearchStorageInMemoryFallbackElastic(testConfig()) storage.Store("1", &AsyncRequestResult{}) storage.Store("2", &AsyncRequestResult{}) @@ -109,6 +115,7 @@ func testConfig() config.ElasticsearchConfiguration { } func TestKK(t *testing.T) { + // TODO: remove this test t.Skip() options := clickhouse.Options{Addr: []string{"localhost:9000"}} a := clickhouse.OpenDB(&options) @@ -125,6 +132,7 @@ func TestKK(t *testing.T) { } func TestCancel(t *testing.T) { + // TODO: remove this test t.Skip() options := clickhouse.Options{Addr: []string{"localhost:9000"}} a := clickhouse.OpenDB(&options) From 821b8724da74230bfbea07e7ed5c3abc1528b279 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Tue, 12 Nov 2024 10:46:06 +0100 Subject: [PATCH 16/30] Reskip tests --- quesma/persistence/persistence_test.go | 4 +-- .../async_search_storage/storage_test.go | 26 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/quesma/persistence/persistence_test.go b/quesma/persistence/persistence_test.go index ef34a91e7..527c68a67 100644 --- a/quesma/persistence/persistence_test.go +++ b/quesma/persistence/persistence_test.go @@ -86,7 +86,7 @@ func TestNewElasticPersistence(t *testing.T) { } func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { - // reskip after merge t.Skip("passes locally, but requires elasticsearch to be running, so skipping") + t.Skip("passes locally, but requires elasticsearch to be running, so skipping") logger.InitSimpleLoggerForTests() indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) fmt.Println("indexName:", indexName) @@ -155,7 +155,7 @@ func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { } func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { - // reskip after merge t.Skip("passes locally, but requires elasticsearch to be running, so skipping") + t.Skip("passes locally, but requires elasticsearch to be running, so skipping") logger.InitSimpleLoggerForTests() indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) diff --git a/quesma/quesma/async_search_storage/storage_test.go b/quesma/quesma/async_search_storage/storage_test.go index 64a64dd9d..77a33cd5c 100644 --- a/quesma/quesma/async_search_storage/storage_test.go +++ b/quesma/quesma/async_search_storage/storage_test.go @@ -16,13 +16,13 @@ import ( func TestAsyncQueriesEvictorTimePassed(t *testing.T) { // TODO: add also 3rd storage and nice test for it (remove from memory, but still in elastic) - realUrl, _ := url.Parse("http://localhost:9201") - cfgUrl := config.Url(*realUrl) - cfg := config.ElasticsearchConfiguration{Url: &cfgUrl} + //realUrl, _ := url.Parse("http://localhost:9201") + //cfgUrl := config.Url(*realUrl) + //cfg := config.ElasticsearchConfiguration{Url: &cfgUrl} storageKinds := []AsyncRequestResultStorage{ NewAsyncRequestResultStorageInMemory(), - NewAsyncRequestResultStorageInElasticsearch(cfg), // passes, reskip after merge - NewAsyncSearchStorageInMemoryFallbackElastic(cfg), // passes, reskip after merge + //NewAsyncRequestResultStorageInElasticsearch(cfg), // passes, reskip after merge + //NewAsyncSearchStorageInMemoryFallbackElastic(cfg), // passes, reskip after merge } for _, storage := range storageKinds { queryContextStorage := NewAsyncQueryContextStorageInMemory().(AsyncQueryContextStorageInMemory) @@ -42,13 +42,13 @@ func TestAsyncQueriesEvictorTimePassed(t *testing.T) { func TestAsyncQueriesEvictorStillAlive(t *testing.T) { // TODO: add also 3rd storage and nice test for it (remove from memory, but still in elastic) - realUrl, _ := url.Parse("http://localhost:9201") - cfgUrl := config.Url(*realUrl) - cfg := config.ElasticsearchConfiguration{Url: &cfgUrl} + //realUrl, _ := url.Parse("http://localhost:9201") + //cfgUrl := config.Url(*realUrl) + //cfg := config.ElasticsearchConfiguration{Url: &cfgUrl} storageKinds := []AsyncRequestResultStorage{ NewAsyncRequestResultStorageInMemory(), - NewAsyncRequestResultStorageInElasticsearch(cfg), // passes, reskip after merge - NewAsyncSearchStorageInMemoryFallbackElastic(cfg), // passes, reskip after merge + //NewAsyncRequestResultStorageInElasticsearch(cfg), // passes, reskip after merge + //NewAsyncSearchStorageInMemoryFallbackElastic(cfg), // passes, reskip after merge } for _, storage := range storageKinds { queryContextStorage := NewAsyncQueryContextStorageInMemory().(AsyncQueryContextStorageInMemory) @@ -67,7 +67,7 @@ func TestAsyncQueriesEvictorStillAlive(t *testing.T) { } func TestInMemoryFallbackElasticStorage(t *testing.T) { - // reskip after merge t.Skip("passes locally, but requires elasticsearch to be running, so skipping") + t.Skip("passes locally, but requires elasticsearch to be running, so skipping") storage := NewAsyncSearchStorageInMemoryFallbackElastic(testConfig()) storage.Store("1", &AsyncRequestResult{}) storage.Store("2", &AsyncRequestResult{}) @@ -115,7 +115,7 @@ func testConfig() config.ElasticsearchConfiguration { } func TestKK(t *testing.T) { - // TODO: remove this test + // TODO: remove this test after evicting from Clickhouse from UI works t.Skip() options := clickhouse.Options{Addr: []string{"localhost:9000"}} a := clickhouse.OpenDB(&options) @@ -132,7 +132,7 @@ func TestKK(t *testing.T) { } func TestCancel(t *testing.T) { - // TODO: remove this test + // TODO: remove this test after evicting from Clickhouse from UI works t.Skip() options := clickhouse.Options{Addr: []string{"localhost:9000"}} a := clickhouse.OpenDB(&options) From a96bd11208a0147209cf040b3dd1625507230f1b Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Tue, 12 Nov 2024 10:53:33 +0100 Subject: [PATCH 17/30] Style --- quesma/elasticsearch/client.go | 3 +-- quesma/main.go | 2 -- quesma/persistence/elastic.go | 1 + quesma/persistence/elastic_with_eviction.go | 12 ++++++------ 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/quesma/elasticsearch/client.go b/quesma/elasticsearch/client.go index b17c30a08..efc9117fb 100644 --- a/quesma/elasticsearch/client.go +++ b/quesma/elasticsearch/client.go @@ -40,7 +40,7 @@ func (es *SimpleClient) RequestWithHeaders(ctx context.Context, method, endpoint return es.doRequest(ctx, method, endpoint, body, headers) } -func (es *SimpleClient) DoRequestCheckResponseStatus(ctx context.Context, method, endpoint string, body []byte) (resp *http.Response, err error) { +func (es *SimpleClient) DoRequestCheckResponseStatusOK(ctx context.Context, method, endpoint string, body []byte) (resp *http.Response, err error) { resp, err = es.doRequest(ctx, method, endpoint, body, nil) if err != nil { return @@ -65,7 +65,6 @@ func (es *SimpleClient) Authenticate(ctx context.Context, authHeader string) boo // doRequest can override auth headers specified in the config, use with care! func (es *SimpleClient) doRequest(ctx context.Context, method, endpoint string, body []byte, headers http.Header) (*http.Response, error) { - fmt.Println("full url:", fmt.Sprintf("%s/%s", es.config.Url, endpoint)) req, err := http.NewRequestWithContext(ctx, method, fmt.Sprintf("%s/%s", es.config.Url, endpoint), bytes.NewBuffer(body)) if err != nil { return nil, err diff --git a/quesma/main.go b/quesma/main.go index ff6eb612c..bc8e00a40 100644 --- a/quesma/main.go +++ b/quesma/main.go @@ -53,8 +53,6 @@ func main() { var newConfiguration = config.LoadV2Config() var cfg = newConfiguration.TranslateToLegacyConfig() - fmt.Printf("kk dbg cfg: %+v", cfg) - if err := cfg.Validate(); err != nil { log.Fatalf("error validating configuration: %v", err) } diff --git a/quesma/persistence/elastic.go b/quesma/persistence/elastic.go index b6a806b5a..3bc6229a9 100644 --- a/quesma/persistence/elastic.go +++ b/quesma/persistence/elastic.go @@ -118,6 +118,7 @@ func (p *ElasticJSONDatabase) List() ([]string, error) { }` resp, err := p.httpClient.Request(context.Background(), "GET", elasticsearchURL, []byte(query)) + if err != nil { return nil, err } diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index 26695b4b1..d75bf5157 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -68,7 +68,7 @@ func (db *ElasticDatabaseWithEviction) Put(document *JSONWithSize) error { return err } - resp, err := db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodPost, elasticsearchURL, jsonData) + resp, err := db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodPost, elasticsearchURL, jsonData) fmt.Println("kk dbg Put() resp:", resp, "err:", err) if err != nil && (resp == nil || resp.StatusCode != http.StatusCreated) { return err @@ -79,7 +79,7 @@ func (db *ElasticDatabaseWithEviction) Put(document *JSONWithSize) error { // co zwraca? zrobić switch na oba typy jakie teraz mamy? func (db *ElasticDatabaseWithEviction) Get(id string) ([]byte, error) { // probably change return type to *Sizeable elasticsearchURL := fmt.Sprintf("%s/_source/%s", db.indexName, id) - resp, err := db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodGet, elasticsearchURL, nil) + resp, err := db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodGet, elasticsearchURL, nil) if err != nil { return nil, err } @@ -94,7 +94,7 @@ func (db *ElasticDatabaseWithEviction) Delete(id string) error { // TODO: check if doc exists? elasticsearchURL := fmt.Sprintf("%s/_doc/%s", db.indexName, id) - resp, err := db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodDelete, elasticsearchURL, nil) + resp, err := db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodDelete, elasticsearchURL, nil) if err != nil && (resp == nil || resp.StatusCode != http.StatusCreated) { return err } @@ -125,7 +125,7 @@ func (db *ElasticDatabaseWithEviction) DeleteOld(deleteOlderThan time.Duration) fmt.Println(query) var resp *http.Response - resp, err = db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodPost, elasticsearchURL, []byte(query)) + resp, err = db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodPost, elasticsearchURL, []byte(query)) fmt.Println("kk dbg DocCount() resp:", resp, "err:", err, "elastic url:", elasticsearchURL) return err } @@ -139,7 +139,7 @@ func (db *ElasticDatabaseWithEviction) DocCount() (docCount int, err error) { }` var resp *http.Response - resp, err = db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodGet, elasticsearchURL, []byte(query)) + resp, err = db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodGet, elasticsearchURL, []byte(query)) fmt.Println("kk dbg DocCount() resp:", resp, "err:", err, "elastic url:", elasticsearchURL) if err != nil { if resp != nil && (resp.StatusCode == http.StatusNoContent || resp.StatusCode == http.StatusNotFound) { @@ -174,7 +174,7 @@ func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, err err }` var resp *http.Response - resp, err = db.httpClient.DoRequestCheckResponseStatus(context.Background(), http.MethodGet, elasticsearchURL, []byte(query)) + resp, err = db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodGet, elasticsearchURL, []byte(query)) fmt.Println("kk dbg SizeInBytes() err:", err, "\nresp:", resp) if err != nil { if resp != nil && resp.StatusCode == 404 { From 3921080654838e5fbc7f6692a598ef3678fd317b Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Tue, 12 Nov 2024 15:26:59 +0100 Subject: [PATCH 18/30] small add test --- quesma/quesma/async_search_storage/storage_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/quesma/quesma/async_search_storage/storage_test.go b/quesma/quesma/async_search_storage/storage_test.go index 77a33cd5c..5fafe171e 100644 --- a/quesma/quesma/async_search_storage/storage_test.go +++ b/quesma/quesma/async_search_storage/storage_test.go @@ -67,7 +67,7 @@ func TestAsyncQueriesEvictorStillAlive(t *testing.T) { } func TestInMemoryFallbackElasticStorage(t *testing.T) { - t.Skip("passes locally, but requires elasticsearch to be running, so skipping") + //t.Skip("passes locally, but requires elasticsearch to be running, so skipping") storage := NewAsyncSearchStorageInMemoryFallbackElastic(testConfig()) storage.Store("1", &AsyncRequestResult{}) storage.Store("2", &AsyncRequestResult{}) @@ -97,6 +97,11 @@ func TestInMemoryFallbackElasticStorage(t *testing.T) { pp.Println(err, doc) assert.Nil(t, doc) assert.NotNil(t, err) + + doc, err = storage.Load("3") + pp.Println(err, doc) + assert.NotNil(t, doc) + assert.Nil(t, err) } const qid = "abc" From 45239e3f00abb41b7b172dda56b0c881028a4cac Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Fri, 20 Dec 2024 16:36:58 +0100 Subject: [PATCH 19/30] Cleanup --- quesma/persistence/elastic_with_eviction.go | 80 ++++++++++++------- quesma/quesma/async_search_storage/evictor.go | 4 +- .../async_search_storage/in_elasticsearch.go | 7 +- .../quesma/async_search_storage/in_memory.go | 4 +- .../async_search_storage/in_memory_test.go | 39 --------- quesma/quesma/async_search_storage/model.go | 55 +++++++------ .../async_search_storage/storage_test.go | 12 ++- 7 files changed, 95 insertions(+), 106 deletions(-) delete mode 100644 quesma/quesma/async_search_storage/in_memory_test.go diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index d75bf5157..a615b50de 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -11,7 +11,6 @@ import ( "io" "math" "net/http" - "quesma/logger" "quesma/quesma/config" "quesma/quesma/types" "time" @@ -34,30 +33,39 @@ func NewElasticDatabaseWithEviction(cfg config.ElasticsearchConfiguration, index } } +const printDebugElasticDB = false // TODO: remove this + all occurances after final version of the storage + func (db *ElasticDatabaseWithEviction) Put(document *JSONWithSize) error { - pp.Println(db) dbSize, err := db.SizeInBytes() if err != nil { return err } - fmt.Println("kk dbg Put() dbSize:", dbSize) - bytesNeeded := dbSize + document.SizeInBytesTotal // improve + if printDebugElasticDB { + fmt.Println("kk dbg Put() dbSize:", dbSize) + } + bytesNeeded := dbSize + document.SizeInBytesTotal if bytesNeeded > db.SizeInBytesLimit() { - logger.Info().Msgf("elastic database: is full, need %d bytes more. Evicting documents", bytesNeeded-db.SizeInBytesLimit()) - allDocs, err := db.getAll() - if err != nil { - return err - } - bytesEvicted := db.Evict(allDocs, bytesNeeded-db.SizeInBytesLimit()) - logger.Info().Msgf("elastic database: evicted %d bytes", bytesEvicted) - bytesNeeded -= bytesEvicted + /* + TODO: restore after eviction readded + logger.Info().Msgf("elastic database: is full, need %d bytes more. Evicting documents", bytesNeeded-db.SizeInBytesLimit()) + allDocs, err := db.getAll() + if err != nil { + return err + } + bytesEvicted := db.Evict(allDocs, bytesNeeded-db.SizeInBytesLimit()) + logger.Info().Msgf("elastic database: evicted %d bytes", bytesEvicted) + bytesNeeded -= bytesEvicted + + */ } if bytesNeeded > db.SizeInBytesLimit() { return errors.New("elastic database: is full, cannot put document") } elasticsearchURL := fmt.Sprintf("%s/_update/%s", db.indexName, document.id) - fmt.Println("kk dbg Put() elasticsearchURL:", elasticsearchURL) + if printDebugElasticDB { + fmt.Println("kk dbg Put() elasticsearchURL:", elasticsearchURL) + } updateContent := types.JSON{} updateContent["doc"] = document.JSON @@ -69,15 +77,17 @@ func (db *ElasticDatabaseWithEviction) Put(document *JSONWithSize) error { } resp, err := db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodPost, elasticsearchURL, jsonData) - fmt.Println("kk dbg Put() resp:", resp, "err:", err) + if printDebugElasticDB { + fmt.Println("kk dbg Put() resp:", resp, "err:", err) + } if err != nil && (resp == nil || resp.StatusCode != http.StatusCreated) { return err } return nil } -// co zwraca? zrobić switch na oba typy jakie teraz mamy? -func (db *ElasticDatabaseWithEviction) Get(id string) ([]byte, error) { // probably change return type to *Sizeable +// Get TODO: probably change return type to some more useful +func (db *ElasticDatabaseWithEviction) Get(id string) ([]byte, error) { elasticsearchURL := fmt.Sprintf("%s/_source/%s", db.indexName, id) resp, err := db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodGet, elasticsearchURL, nil) if err != nil { @@ -89,10 +99,6 @@ func (db *ElasticDatabaseWithEviction) Get(id string) ([]byte, error) { // proba } func (db *ElasticDatabaseWithEviction) Delete(id string) error { - // mark as deleted, don't actually delete - // (single document deletion is hard in ES, it's done by evictor for entire index) - - // TODO: check if doc exists? elasticsearchURL := fmt.Sprintf("%s/_doc/%s", db.indexName, id) resp, err := db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodDelete, elasticsearchURL, nil) if err != nil && (resp == nil || resp.StatusCode != http.StatusCreated) { @@ -122,11 +128,15 @@ func (db *ElasticDatabaseWithEviction) DeleteOld(deleteOlderThan time.Duration) } }`, rangeStr) - fmt.Println(query) + if printDebugElasticDB { + fmt.Println(query) + } var resp *http.Response resp, err = db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodPost, elasticsearchURL, []byte(query)) - fmt.Println("kk dbg DocCount() resp:", resp, "err:", err, "elastic url:", elasticsearchURL) + if printDebugElasticDB { + fmt.Println("kk dbg DocCount() resp:", resp, "err:", err, "elastic url:", elasticsearchURL) + } return err } @@ -140,7 +150,9 @@ func (db *ElasticDatabaseWithEviction) DocCount() (docCount int, err error) { var resp *http.Response resp, err = db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodGet, elasticsearchURL, []byte(query)) - fmt.Println("kk dbg DocCount() resp:", resp, "err:", err, "elastic url:", elasticsearchURL) + if printDebugElasticDB { + fmt.Println("kk dbg DocCount() resp:", resp, "err:", err, "elastic url:", elasticsearchURL) + } if err != nil { if resp != nil && (resp.StatusCode == http.StatusNoContent || resp.StatusCode == http.StatusNotFound) { return 0, nil @@ -160,13 +172,16 @@ func (db *ElasticDatabaseWithEviction) DocCount() (docCount int, err error) { return } - fmt.Println("kk dbg DocCount() result:", result) + if printDebugElasticDB { + fmt.Println("kk dbg DocCount() result:", result) + } return int(result["hits"].(map[string]interface{})["total"].(map[string]interface{})["value"].(float64)), nil // TODO: add some checks... to prevent panic } func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, err error) { elasticsearchURL := fmt.Sprintf("%s/_search", db.indexName) + // TODO change query to aggregation query := `{ "_source": ["sizeInBytes"], "size": 10000, @@ -175,7 +190,9 @@ func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, err err var resp *http.Response resp, err = db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodGet, elasticsearchURL, []byte(query)) - fmt.Println("kk dbg SizeInBytes() err:", err, "\nresp:", resp) + if printDebugElasticDB { + fmt.Println("kk dbg SizeInBytes() err:", err, "\nresp:", resp) + } if err != nil { if resp != nil && resp.StatusCode == 404 { return 0, nil @@ -190,7 +207,9 @@ func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, err err return } - fmt.Println("kk dbg SizeInBytes() resp.StatusCode:", resp.StatusCode) + if printDebugElasticDB { + fmt.Println("kk dbg SizeInBytes() resp.StatusCode:", resp.StatusCode) + } // Unmarshal the JSON response var result map[string]interface{} @@ -200,12 +219,16 @@ func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, err err a := make([]int64, 0) for _, hit := range result["hits"].(map[string]interface{})["hits"].([]interface{}) { - pp.Println("hit:", hit) + if printDebugElasticDB { + pp.Println("hit:", hit) + } b := sizeInBytes sizeInBytes += int64(hit.(map[string]interface{})["_source"].(map[string]interface{})["sizeInBytes"].(float64)) // TODO: add checks a = append(a, sizeInBytes-b) } - fmt.Println("kk dbg SizeInBytes() sizes in storage:", a) + if printDebugElasticDB { + fmt.Println("kk dbg SizeInBytes() sizes in storage:", a) + } return sizeInBytes, nil } @@ -223,6 +246,7 @@ func (db *ElasticDatabaseWithEviction) getAll() (documents []*JSONWithSize, err "track_total_hits": true }` /* + TODO: restore after eviction readded db.httpClient. resp, err := db.httpClient.Request(context.Background(), "GET", elasticsearchURL, []byte(query)) diff --git a/quesma/quesma/async_search_storage/evictor.go b/quesma/quesma/async_search_storage/evictor.go index 498c54e8a..d53ac07a4 100644 --- a/quesma/quesma/async_search_storage/evictor.go +++ b/quesma/quesma/async_search_storage/evictor.go @@ -33,8 +33,8 @@ func (e *AsyncQueriesEvictor) AsyncQueriesGC() { case <-e.ctx.Done(): logger.Debug().Msg("evictor stopped") return - case <-time.After(GCInterval): - e.tryEvictAsyncRequests(EvictionInterval) + case <-time.After(gcInterval): + e.tryEvictAsyncRequests(evictionInterval) } } } diff --git a/quesma/quesma/async_search_storage/in_elasticsearch.go b/quesma/quesma/async_search_storage/in_elasticsearch.go index 9cae5ddff..b8d489bf8 100644 --- a/quesma/quesma/async_search_storage/in_elasticsearch.go +++ b/quesma/quesma/async_search_storage/in_elasticsearch.go @@ -5,11 +5,9 @@ package async_search_storage import ( "encoding/json" "fmt" - "math/rand" "quesma/logger" "quesma/persistence" "quesma/quesma/config" - "strconv" "time" ) @@ -29,12 +27,15 @@ func NewAsyncRequestResultStorageInElasticsearch(cfg config.ElasticsearchConfigu User: "", Password: "", } - */ i := rand.Int() fmt.Println("kk dbg NewAsyncRequestResultStorageInElasticsearch() i:", cfg) return AsyncRequestResultStorageInElasticsearch{ db: persistence.NewElasticDatabaseWithEviction(cfg, "quesma_async_storage-"+strconv.Itoa(i), 1_000_000_000), } + */ + return AsyncRequestResultStorageInElasticsearch{ + db: persistence.NewElasticDatabaseWithEviction(cfg, defaultElasticDbName, defaultElasticDbStorageLimitInBytes), + } } func (s AsyncRequestResultStorageInElasticsearch) Store(id string, result *AsyncRequestResult) { diff --git a/quesma/quesma/async_search_storage/in_memory.go b/quesma/quesma/async_search_storage/in_memory.go index cecbd4220..0979f8004 100644 --- a/quesma/quesma/async_search_storage/in_memory.go +++ b/quesma/quesma/async_search_storage/in_memory.go @@ -135,7 +135,7 @@ func (e *AsyncQueryTraceLoggerEvictor) Stop() { func (e *AsyncQueryTraceLoggerEvictor) TryFlushHangingAsyncQueryTrace(timeFun func(time.Time) time.Duration) { asyncIds := []string{} e.AsyncQueryTrace.Range(func(key string, value tracing.TraceCtx) bool { - if timeFun(value.Added) > EvictionInterval { + if timeFun(value.Added) > evictionInterval { asyncIds = append(asyncIds, key) logger.Error().Msgf("Async query %s was not finished", key) var formattedLines strings.Builder @@ -154,7 +154,7 @@ func (e *AsyncQueryTraceLoggerEvictor) FlushHangingAsyncQueryTrace(timeFun func( defer recovery.LogPanic() for { select { - case <-time.After(GCInterval): + case <-time.After(gcInterval): e.TryFlushHangingAsyncQueryTrace(timeFun) case <-e.ctx.Done(): logger.Debug().Msg("AsyncQueryTraceLoggerEvictor stopped") diff --git a/quesma/quesma/async_search_storage/in_memory_test.go b/quesma/quesma/async_search_storage/in_memory_test.go deleted file mode 100644 index 59b129d2f..000000000 --- a/quesma/quesma/async_search_storage/in_memory_test.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Quesma, licensed under the Elastic License 2.0. -// SPDX-License-Identifier: Elastic-2.0 -package async_search_storage - -import ( - "github.com/stretchr/testify/assert" - "quesma/util" - "testing" - "time" -) - -func TestAsyncQueriesEvictorTimePassed(t *testing.T) { - queryContextStorage := NewAsyncQueryContextStorageInMemory() - queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) - evictor := NewAsyncQueriesEvictor(NewAsyncSearchStorageInMemory(), queryContextStorage) - evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{added: time.Now()}) - evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{added: time.Now()}) - evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{added: time.Now()}) - evictor.tryEvictAsyncRequests(func(time.Time) time.Duration { - return 20 * time.Minute - }) - - assert.Equal(t, 0, evictor.AsyncRequestStorage.Size()) -} - -func TestAsyncQueriesEvictorStillAlive(t *testing.T) { - queryContextStorage := NewAsyncQueryContextStorageInMemory() - queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) - evictor := NewAsyncQueriesEvictor(NewAsyncSearchStorageInMemory(), queryContextStorage) - evictor.AsyncRequestStorage.idToResult = util.NewSyncMap[string, *AsyncRequestResult]() - evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{added: time.Now()}) - evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{added: time.Now()}) - evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{added: time.Now()}) - evictor.tryEvictAsyncRequests(func(time.Time) time.Duration { - return time.Second - }) - - assert.Equal(t, 3, evictor.AsyncRequestStorage.Size()) -} diff --git a/quesma/quesma/async_search_storage/model.go b/quesma/quesma/async_search_storage/model.go index 6ece7e465..e63ad2658 100644 --- a/quesma/quesma/async_search_storage/model.go +++ b/quesma/quesma/async_search_storage/model.go @@ -9,41 +9,46 @@ import ( "time" ) -const EvictionInterval = 15 * time.Minute -const GCInterval = 1 * time.Minute - -type AsyncRequestResultStorage interface { - Store(id string, result *AsyncRequestResult) - Load(id string) (*AsyncRequestResult, error) - Delete(id string) - DocCount() int - SpaceInUse() int64 - SpaceMaxAvailable() int64 +const ( + evictionInterval = 15 * time.Minute + gcInterval = 1 * time.Minute + defaultElasticDbName = "async_search_storage" + defaultElasticDbStorageLimitInBytes = int64(100 * 1024 * 1024 * 1024) // 100GB +) - evict(olderThan time.Duration) -} +type ( + AsyncRequestResultStorage interface { + Store(id string, result *AsyncRequestResult) + Load(id string) (*AsyncRequestResult, error) + Delete(id string) + DocCount() int + SpaceInUse() int64 + SpaceMaxAvailable() int64 -type AsyncQueryContextStorage interface { - Store(context *AsyncQueryContext) - evict(olderThan time.Duration) -} - -type AsyncRequestResult struct { - ResponseBody []byte `json:"responseBody"` - Added time.Time `json:"added"` - IsCompressed bool `json:"isCompressed"` - Err error `json:"err"` -} + evict(olderThan time.Duration) + } + AsyncQueryContextStorage interface { + Store(context *AsyncQueryContext) + evict(olderThan time.Duration) + } + AsyncRequestResult struct { + ResponseBody []byte `json:"responseBody"` + Added time.Time `json:"added"` + IsCompressed bool `json:"isCompressed"` + Err error `json:"err"` + } +) func NewAsyncRequestResult(responseBody []byte, err error, added time.Time, isCompressed bool) *AsyncRequestResult { return &AsyncRequestResult{ResponseBody: responseBody, Err: err, Added: added, IsCompressed: isCompressed} } func (r *AsyncRequestResult) toJSON(id string) *persistence.JSONWithSize { + const sizeInBytesUpperBoundEstimate = int64(100) // 100 is a rough upper bound estimate of the size of the rest of the fields json := types.JSON{} json["id"] = id json["data"] = string(r.ResponseBody) - json["sizeInBytes"] = int64(len(r.ResponseBody)) + int64(len(id)) + 100 // 100 is a rough upper bound estimate of the size of the rest of the fields + json["sizeInBytes"] = int64(len(r.ResponseBody)) + int64(len(id)) + sizeInBytesUpperBoundEstimate json["added"] = r.Added return persistence.NewJSONWithSize(json, id, json["sizeInBytes"].(int64)) } @@ -60,7 +65,7 @@ func NewAsyncQueryContext(ctx context.Context, cancel context.CancelFunc, id str } func (c *AsyncQueryContext) toJSON() *persistence.JSONWithSize { - sizeInBytesUpperBoundEstimate := int64(100) + const sizeInBytesUpperBoundEstimate = int64(100) json := types.JSON{} json["id"] = c.id json["added"] = c.added diff --git a/quesma/quesma/async_search_storage/storage_test.go b/quesma/quesma/async_search_storage/storage_test.go index 5fafe171e..b17e19e53 100644 --- a/quesma/quesma/async_search_storage/storage_test.go +++ b/quesma/quesma/async_search_storage/storage_test.go @@ -119,9 +119,8 @@ func testConfig() config.ElasticsearchConfiguration { } } -func TestKK(t *testing.T) { - // TODO: remove this test after evicting from Clickhouse from UI works - t.Skip() +func TestEvictingAsyncQuery_1(t *testing.T) { + t.Skip("TODO: automize this test after evicting from Clickhouse from UI works") options := clickhouse.Options{Addr: []string{"localhost:9000"}} a := clickhouse.OpenDB(&options) ctx := clickhouse.Context(context.Background(), clickhouse.WithQueryID(qid)) @@ -136,12 +135,11 @@ func TestKK(t *testing.T) { fmt.Println(b, "q:", q, err) } -func TestCancel(t *testing.T) { - // TODO: remove this test after evicting from Clickhouse from UI works - t.Skip() +func TestEvictingAsyncQuery_2(t *testing.T) { + t.Skip("TODO: automize this test after evicting from Clickhouse from UI works") options := clickhouse.Options{Addr: []string{"localhost:9000"}} a := clickhouse.OpenDB(&options) - b, err := a.Query("KILL QUERY WHERE query_id= 'dupa'") + b, err := a.Query("KILL QUERY WHERE query_id= 'x'") fmt.Println(b, err) } From 51b1e6870cc63748e8dcf9964715c07d5e9c74d0 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Fri, 20 Dec 2024 16:44:06 +0100 Subject: [PATCH 20/30] Cleanup 2 --- quesma/persistence/elastic_with_eviction.go | 10 ++-- quesma/persistence/evictor.go | 15 ++--- quesma/persistence/model.go | 62 ++++++++++----------- 3 files changed, 38 insertions(+), 49 deletions(-) diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index a615b50de..04d2865ee 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -20,7 +20,7 @@ import ( type ElasticDatabaseWithEviction struct { ctx context.Context *ElasticJSONDatabase // maybe remove and copy fields here - EvictorInterface + // EvictorInterface sizeInBytesLimit int64 } @@ -28,8 +28,8 @@ func NewElasticDatabaseWithEviction(cfg config.ElasticsearchConfiguration, index return &ElasticDatabaseWithEviction{ ctx: context.Background(), ElasticJSONDatabase: NewElasticJSONDatabase(cfg, indexName), - EvictorInterface: &Evictor{}, - sizeInBytesLimit: sizeInBytesLimit, + // EvictorInterface: &Evictor{}, + sizeInBytesLimit: sizeInBytesLimit, } } @@ -45,6 +45,7 @@ func (db *ElasticDatabaseWithEviction) Put(document *JSONWithSize) error { } bytesNeeded := dbSize + document.SizeInBytesTotal if bytesNeeded > db.SizeInBytesLimit() { + return errors.New("elastic database: is full, cannot put document") /* TODO: restore after eviction readded logger.Info().Msgf("elastic database: is full, need %d bytes more. Evicting documents", bytesNeeded-db.SizeInBytesLimit()) @@ -58,9 +59,6 @@ func (db *ElasticDatabaseWithEviction) Put(document *JSONWithSize) error { */ } - if bytesNeeded > db.SizeInBytesLimit() { - return errors.New("elastic database: is full, cannot put document") - } elasticsearchURL := fmt.Sprintf("%s/_update/%s", db.indexName, document.id) if printDebugElasticDB { diff --git a/quesma/persistence/evictor.go b/quesma/persistence/evictor.go index 88bfc5f90..fa84e9f12 100644 --- a/quesma/persistence/evictor.go +++ b/quesma/persistence/evictor.go @@ -2,21 +2,16 @@ // SPDX-License-Identifier: Elastic-2.0 package persistence -import "fmt" - type EvictorInterface interface { Evict(documents []*JSONWithSize, sizeNeeded int64) (bytesEvicted int64) } -// It's only 1 implementation, which looks well suited for ElasticSearch. -// It can be implemented differently. +// TODO: Find out how this might work. My old idea doesn't work now, +// as don't remove entire indices, but delete single documents. +// (It turned out consistency was too eventual to rely on it) +// old comment: It's only 1 implementation, which looks well suited for ElasticSearch. It can be implemented differently. type Evictor struct{} func (e *Evictor) Evict(documents []*JSONWithSize, sizeNeeded int64) (bytesEvicted int64) { - if sizeNeeded <= 0 { - return // check if it's empty array or nil - } - fmt.Println("kk dbg SelectToEvict() sizeNeeded:", sizeNeeded) - - return + panic("implement me (or remove)") } diff --git a/quesma/persistence/model.go b/quesma/persistence/model.go index 58767f8c5..9fd2e7be2 100644 --- a/quesma/persistence/model.go +++ b/quesma/persistence/model.go @@ -7,39 +7,35 @@ import ( "time" ) -// JSONDatabase is an interface for a database that stores JSON data. -// Treat it as `etcd` equivalent rather than `MongoDB`. -// The main usage is to store our configuration data, like -// - schema -// - user settings -// - statistics, etc -// -// For each case, we should have a separate database. -type JSONDatabase interface { - List() (keys []string, err error) - Get(key string) (string, bool, error) - Put(key string, data string) error -} - -type DatabaseWithEviction interface { // for sure JSON? maybe not only json? check - Put(doc JSONWithSize) error - Get(id string) (types.JSON, error) - Delete(id string) error - DeleteOld(time.Duration) error - DocCount() (int, error) - SizeInBytes() (int64, error) - SizeInBytesLimit() int64 -} - -type JSONWithSizeInterface interface { - SizeInBytes() int64 -} - -type JSONWithSize struct { - types.JSON - id string - SizeInBytesTotal int64 -} +type ( + // JSONDatabase is an interface for a database that stores JSON data. + // Treat it as `etcd` equivalent rather than `MongoDB`. + // The main usage is to store our configuration data, like + // - schema + // - user settings + // - statistics, etc + // + // For each case, we should have a separate database. + JSONDatabase interface { + List() (keys []string, err error) + Get(key string) (string, bool, error) + Put(key string, data string) error + } + DatabaseWithEviction interface { // for sure JSON? maybe not only json? check + Put(doc *JSONWithSize) error + Get(id string) ([]byte, error) + Delete(id string) error + DeleteOld(time.Duration) error + DocCount() (int, error) + SizeInBytes() (int64, error) + SizeInBytesLimit() int64 + } + JSONWithSize struct { + types.JSON + id string + SizeInBytesTotal int64 + } +) func NewJSONWithSize(data types.JSON, id string, sizeInBytesTotal int64) *JSONWithSize { return &JSONWithSize{ From 89a5180e76e05dac9e77fc93f22eac095fb70eaf Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Fri, 20 Dec 2024 17:20:46 +0100 Subject: [PATCH 21/30] Fix all tests --- quesma/persistence/persistence_test.go | 5 +- .../async_search_storage/in_elasticsearch.go | 9 +- .../async_search_storage/storage_test.go | 112 ++++++++++-------- quesma/quesma/dual_write_proxy_v2.go | 2 +- 4 files changed, 71 insertions(+), 57 deletions(-) diff --git a/quesma/persistence/persistence_test.go b/quesma/persistence/persistence_test.go index 527c68a67..4518c5a17 100644 --- a/quesma/persistence/persistence_test.go +++ b/quesma/persistence/persistence_test.go @@ -86,7 +86,7 @@ func TestNewElasticPersistence(t *testing.T) { } func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { - t.Skip("passes locally, but requires elasticsearch to be running, so skipping") + t.Skip("Test passes locally (20.12.2024), but requires elasticsearch to be running, so skipping for now") logger.InitSimpleLoggerForTests() indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) fmt.Println("indexName:", indexName) @@ -155,7 +155,7 @@ func TestJSONDatabaseWithEviction_noEviction(t *testing.T) { } func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { - t.Skip("passes locally, but requires elasticsearch to be running, so skipping") + t.Skip("Test passes locally (20.12.2024), but requires elasticsearch to be running, so skipping for now") logger.InitSimpleLoggerForTests() indexName := fmt.Sprintf("quesma_test_%d", time.Now().UnixMilli()) @@ -213,7 +213,6 @@ func TestJSONDatabaseWithEviction_withEviction(t *testing.T) { assert.Equal(t, 2, docCount) err = db.Put(docs[4]) - fmt.Println("put", docs[4].SizeInBytesTotal, err) assert.NoError(t, err) time.Sleep(elasticUpdateTime) diff --git a/quesma/quesma/async_search_storage/in_elasticsearch.go b/quesma/quesma/async_search_storage/in_elasticsearch.go index b8d489bf8..62d53cb2b 100644 --- a/quesma/quesma/async_search_storage/in_elasticsearch.go +++ b/quesma/quesma/async_search_storage/in_elasticsearch.go @@ -27,11 +27,10 @@ func NewAsyncRequestResultStorageInElasticsearch(cfg config.ElasticsearchConfigu User: "", Password: "", } - i := rand.Int() - fmt.Println("kk dbg NewAsyncRequestResultStorageInElasticsearch() i:", cfg) - return AsyncRequestResultStorageInElasticsearch{ - db: persistence.NewElasticDatabaseWithEviction(cfg, "quesma_async_storage-"+strconv.Itoa(i), 1_000_000_000), - } + fmt.Println("kk dbg NewAsyncRequestResultStorageInElasticsearch() i:", cfg) + return AsyncRequestResultStorageInElasticsearch{ + db: persistence.NewElasticDatabaseWithEviction(cfg, "quesma_async_storage-"+strconv.Itoa(i), 1_000_000_000), + } */ return AsyncRequestResultStorageInElasticsearch{ db: persistence.NewElasticDatabaseWithEviction(cfg, defaultElasticDbName, defaultElasticDbStorageLimitInBytes), diff --git a/quesma/quesma/async_search_storage/storage_test.go b/quesma/quesma/async_search_storage/storage_test.go index b17e19e53..4af98fbc4 100644 --- a/quesma/quesma/async_search_storage/storage_test.go +++ b/quesma/quesma/async_search_storage/storage_test.go @@ -15,59 +15,73 @@ import ( ) func TestAsyncQueriesEvictorTimePassed(t *testing.T) { - // TODO: add also 3rd storage and nice test for it (remove from memory, but still in elastic) - //realUrl, _ := url.Parse("http://localhost:9201") - //cfgUrl := config.Url(*realUrl) - //cfg := config.ElasticsearchConfiguration{Url: &cfgUrl} storageKinds := []AsyncRequestResultStorage{ NewAsyncRequestResultStorageInMemory(), - //NewAsyncRequestResultStorageInElasticsearch(cfg), // passes, reskip after merge - //NewAsyncSearchStorageInMemoryFallbackElastic(cfg), // passes, reskip after merge + NewAsyncRequestResultStorageInElasticsearch(testConfig()), + NewAsyncSearchStorageInMemoryFallbackElastic(testConfig()), } for _, storage := range storageKinds { - queryContextStorage := NewAsyncQueryContextStorageInMemory().(AsyncQueryContextStorageInMemory) - queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) - evictor := NewAsyncQueriesEvictor(storage, queryContextStorage) - evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{Added: time.Now()}) - evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{Added: time.Now()}) - evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{Added: time.Now()}) - - time.Sleep(2 * time.Second) - evictor.tryEvictAsyncRequests(1 * time.Second) - time.Sleep(2 * time.Second) - - assert.Equal(t, 0, evictor.AsyncRequestStorage.DocCount()) + t.Run(fmt.Sprintf("storage %T", storage), func(t *testing.T) { + _, inMemory := storage.(AsyncRequestResultStorageInMemory) + if !inMemory { + t.Skip("Test passes locally (20.12.2024), but requires elasticsearch to be running, so skipping for now") + } + + queryContextStorage := NewAsyncQueryContextStorageInMemory().(AsyncQueryContextStorageInMemory) + queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) + evictor := NewAsyncQueriesEvictor(storage, queryContextStorage) + evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{Added: time.Now().Add(-2 * time.Second)}) + evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{Added: time.Now().Add(-5 * time.Second)}) + evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{Added: time.Now().Add(2 * time.Second)}) + + if !inMemory { + time.Sleep(2 * time.Second) + } + evictor.tryEvictAsyncRequests(1 * time.Second) + if !inMemory { + time.Sleep(2 * time.Second) + } + + assert.Equal(t, 1, evictor.AsyncRequestStorage.DocCount()) + }) } } func TestAsyncQueriesEvictorStillAlive(t *testing.T) { - // TODO: add also 3rd storage and nice test for it (remove from memory, but still in elastic) - //realUrl, _ := url.Parse("http://localhost:9201") - //cfgUrl := config.Url(*realUrl) - //cfg := config.ElasticsearchConfiguration{Url: &cfgUrl} storageKinds := []AsyncRequestResultStorage{ NewAsyncRequestResultStorageInMemory(), - //NewAsyncRequestResultStorageInElasticsearch(cfg), // passes, reskip after merge - //NewAsyncSearchStorageInMemoryFallbackElastic(cfg), // passes, reskip after merge + NewAsyncRequestResultStorageInElasticsearch(testConfig()), + NewAsyncSearchStorageInMemoryFallbackElastic(testConfig()), } for _, storage := range storageKinds { - queryContextStorage := NewAsyncQueryContextStorageInMemory().(AsyncQueryContextStorageInMemory) - queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) - evictor := NewAsyncQueriesEvictor(storage, queryContextStorage) - evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{Added: time.Now()}) - evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{Added: time.Now()}) - evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{Added: time.Now()}) - - time.Sleep(2 * time.Second) - evictor.tryEvictAsyncRequests(10 * time.Second) - time.Sleep(2 * time.Second) - - assert.Equal(t, 3, evictor.AsyncRequestStorage.DocCount()) + t.Run(fmt.Sprintf("storage %T", storage), func(t *testing.T) { + _, inMemory := storage.(AsyncRequestResultStorageInMemory) + if !inMemory { + t.Skip("Test passes locally (20.12.2024), but requires elasticsearch to be running, so skipping for now") + } + + queryContextStorage := NewAsyncQueryContextStorageInMemory().(AsyncQueryContextStorageInMemory) + queryContextStorage.idToContext.Store("1", &AsyncQueryContext{}) + evictor := NewAsyncQueriesEvictor(storage, queryContextStorage) + evictor.AsyncRequestStorage.Store("1", &AsyncRequestResult{Added: time.Now()}) + evictor.AsyncRequestStorage.Store("2", &AsyncRequestResult{Added: time.Now()}) + evictor.AsyncRequestStorage.Store("3", &AsyncRequestResult{Added: time.Now()}) + + if !inMemory { + time.Sleep(2 * time.Second) + } + evictor.tryEvictAsyncRequests(10 * time.Second) + if !inMemory { + time.Sleep(2 * time.Second) + } + + assert.Equal(t, 3, evictor.AsyncRequestStorage.DocCount()) + }) } } func TestInMemoryFallbackElasticStorage(t *testing.T) { - //t.Skip("passes locally, but requires elasticsearch to be running, so skipping") + t.Skip("Test passes locally (20.12.2024), but requires elasticsearch to be running, so skipping for now") storage := NewAsyncSearchStorageInMemoryFallbackElastic(testConfig()) storage.Store("1", &AsyncRequestResult{}) storage.Store("2", &AsyncRequestResult{}) @@ -122,24 +136,26 @@ func testConfig() config.ElasticsearchConfiguration { func TestEvictingAsyncQuery_1(t *testing.T) { t.Skip("TODO: automize this test after evicting from Clickhouse from UI works") options := clickhouse.Options{Addr: []string{"localhost:9000"}} - a := clickhouse.OpenDB(&options) - ctx := clickhouse.Context(context.Background(), clickhouse.WithQueryID(qid)) + db := clickhouse.OpenDB(&options) + defer db.Close() - b, err := a.QueryContext(ctx, "SELECT number FROM (SELECT number FROM numbers(100_000_000_000)) ORDER BY number DESC LIMIT 10") - var q int64 - for b.Next() { - b.Scan(&q) - fmt.Println(q) + ctx := clickhouse.Context(context.Background(), clickhouse.WithQueryID(qid)) + rows, err := db.QueryContext(ctx, "SELECT number FROM (SELECT number FROM numbers(100_000_000_000)) ORDER BY number DESC LIMIT 10") + var i int64 + for rows.Next() { + rows.Scan(&i) + fmt.Println(i) } - fmt.Println(b, "q:", q, err) + fmt.Println(rows, "i:", i, err) } func TestEvictingAsyncQuery_2(t *testing.T) { t.Skip("TODO: automize this test after evicting from Clickhouse from UI works") options := clickhouse.Options{Addr: []string{"localhost:9000"}} - a := clickhouse.OpenDB(&options) + db := clickhouse.OpenDB(&options) + defer db.Close() - b, err := a.Query("KILL QUERY WHERE query_id= 'x'") - fmt.Println(b, err) + rows, err := db.Query("KILL QUERY WHERE query_id= 'x'") + fmt.Println(rows, err) } diff --git a/quesma/quesma/dual_write_proxy_v2.go b/quesma/quesma/dual_write_proxy_v2.go index 32594025a..6a80b01ee 100644 --- a/quesma/quesma/dual_write_proxy_v2.go +++ b/quesma/quesma/dual_write_proxy_v2.go @@ -122,7 +122,7 @@ func newDualWriteProxyV2(dependencies quesma_api.Dependencies, schemaLoader clic logManager: logManager, publicPort: config.PublicTcpPort, asyncQueriesEvictor: async_search_storage.NewAsyncQueriesEvictor( - queryProcessor.AsyncRequestStorage.(async_search_storage.AsyncSearchStorageInMemory), + queryProcessor.AsyncRequestStorage.(async_search_storage.AsyncRequestResultStorageInMemory), queryProcessor.AsyncQueriesContexts.(async_search_storage.AsyncQueryContextStorageInMemory), ), queryRunner: queryProcessor, From dc40a900ff2509eba7523e7739f27667af0143fe Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Fri, 20 Dec 2024 17:26:31 +0100 Subject: [PATCH 22/30] Fix linter --- quesma/persistence/elastic_with_eviction.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index 04d2865ee..8145707f5 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -234,8 +234,9 @@ func (db *ElasticDatabaseWithEviction) SizeInBytesLimit() int64 { return db.sizeInBytesLimit } +/* TODO: restore after eviction readded, or remove func (db *ElasticDatabaseWithEviction) getAll() (documents []*JSONWithSize, err error) { - _ = fmt.Sprintf("%s*/_search", db.indexName) + _ = fmt.Sprintf("%s/_search", db.indexName) _ = `{ "_source": { "excludes": "data" @@ -243,8 +244,7 @@ func (db *ElasticDatabaseWithEviction) getAll() (documents []*JSONWithSize, err "size": 10000, "track_total_hits": true }` - /* - TODO: restore after eviction readded + db.httpClient. resp, err := db.httpClient.Request(context.Background(), "GET", elasticsearchURL, []byte(query)) @@ -287,10 +287,9 @@ func (db *ElasticDatabaseWithEviction) getAll() (documents []*JSONWithSize, err fmt.Println(doc) documents = append(documents, doc) } - - */ return documents, nil } +*/ func (db *ElasticDatabaseWithEviction) fullIndexName() string { now := time.Now().UTC() From ffdc79ed8c7a5e5754113155db25444112963fd3 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Fri, 20 Dec 2024 17:37:15 +0100 Subject: [PATCH 23/30] Final if manual test passes --- quesma/persistence/elastic_with_eviction.go | 2 +- quesma/quesma/dual_write_proxy_v2.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index 8145707f5..d6b82ff46 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -20,7 +20,7 @@ import ( type ElasticDatabaseWithEviction struct { ctx context.Context *ElasticJSONDatabase // maybe remove and copy fields here - // EvictorInterface + // EvictorInterface TODO: rethink how eviction should work, maybe remove sizeInBytesLimit int64 } diff --git a/quesma/quesma/dual_write_proxy_v2.go b/quesma/quesma/dual_write_proxy_v2.go index 6a80b01ee..6b9c8e434 100644 --- a/quesma/quesma/dual_write_proxy_v2.go +++ b/quesma/quesma/dual_write_proxy_v2.go @@ -122,8 +122,8 @@ func newDualWriteProxyV2(dependencies quesma_api.Dependencies, schemaLoader clic logManager: logManager, publicPort: config.PublicTcpPort, asyncQueriesEvictor: async_search_storage.NewAsyncQueriesEvictor( - queryProcessor.AsyncRequestStorage.(async_search_storage.AsyncRequestResultStorageInMemory), - queryProcessor.AsyncQueriesContexts.(async_search_storage.AsyncQueryContextStorageInMemory), + queryProcessor.AsyncRequestStorage, + queryProcessor.AsyncQueriesContexts, ), queryRunner: queryProcessor, } From cb402dc523a2801424c38a84aa26c3f4b4a72e2b Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Fri, 20 Dec 2024 17:41:18 +0100 Subject: [PATCH 24/30] Last: 1 name --- quesma/persistence/elastic_with_eviction.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index d6b82ff46..6a35a513b 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -215,17 +215,17 @@ func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, err err return } - a := make([]int64, 0) + sizes := make([]int64, 0) for _, hit := range result["hits"].(map[string]interface{})["hits"].([]interface{}) { if printDebugElasticDB { pp.Println("hit:", hit) } b := sizeInBytes sizeInBytes += int64(hit.(map[string]interface{})["_source"].(map[string]interface{})["sizeInBytes"].(float64)) // TODO: add checks - a = append(a, sizeInBytes-b) + sizes = append(sizes, sizeInBytes-b) } if printDebugElasticDB { - fmt.Println("kk dbg SizeInBytes() sizes in storage:", a) + fmt.Println("kk dbg SizeInBytes() sizes in storage:", sizes) } return sizeInBytes, nil } From 1499a49a38c87059036ccba1a8a43b188273aa0b Mon Sep 17 00:00:00 2001 From: Rafal Strzalinski Date: Wed, 12 Feb 2025 11:11:58 +0100 Subject: [PATCH 25/30] Linter --- quesma/quesma/async_search_storage/in_memory.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/quesma/quesma/async_search_storage/in_memory.go b/quesma/quesma/async_search_storage/in_memory.go index 6fb2f2780..740b1bf9e 100644 --- a/quesma/quesma/async_search_storage/in_memory.go +++ b/quesma/quesma/async_search_storage/in_memory.go @@ -107,7 +107,3 @@ func (s AsyncQueryContextStorageInMemory) evict(evictOlderThan time.Duration) { logger.Info().Msgf("Evicted %d async queries : %s", len(evictedIds), strings.Join(evictedIds, ",")) } } - -func elapsedTime(t time.Time) time.Duration { - return time.Since(t) -} From a72f6063d8f0d14612e75b61ffac6b0340e486a2 Mon Sep 17 00:00:00 2001 From: Rafal Strzalinski Date: Wed, 12 Feb 2025 11:43:45 +0100 Subject: [PATCH 26/30] Unify error handling --- quesma/persistence/elastic_with_eviction.go | 41 ++++++++++++++++----- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index 050688228..bd3352a48 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -79,9 +79,15 @@ func (db *ElasticDatabaseWithEviction) Put(document *JSONWithSize) error { if printDebugElasticDB { fmt.Println("kk dbg Put() resp:", resp, "err:", err) } - if err != nil && (resp == nil || resp.StatusCode != http.StatusCreated) { + if err != nil { return err } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + return errors.New("failed to put document, got status: " + resp.Status) + } + return nil } @@ -100,9 +106,13 @@ func (db *ElasticDatabaseWithEviction) Get(id string) ([]byte, error) { func (db *ElasticDatabaseWithEviction) Delete(id string) error { elasticsearchURL := fmt.Sprintf("%s/_doc/%s", db.indexName, id) resp, err := db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodDelete, elasticsearchURL, nil) - if err != nil && (resp == nil || resp.StatusCode != http.StatusCreated) { + if err != nil { return err } + defer resp.Body.Close() + if resp.StatusCode != http.StatusCreated { + return errors.New("failed to delete document") + } return nil } @@ -133,6 +143,15 @@ func (db *ElasticDatabaseWithEviction) DeleteOld(deleteOlderThan time.Duration) var resp *http.Response resp, err = db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodPost, elasticsearchURL, []byte(query)) + if err != nil { + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNotFound { + return errors.New("failed to delete old documents, got status: " + resp.Status) + } + if printDebugElasticDB { fmt.Println("kk dbg DocCount() resp:", resp, "err:", err, "elastic url:", elasticsearchURL) } @@ -153,11 +172,13 @@ func (db *ElasticDatabaseWithEviction) DocCount() (docCount int, err error) { fmt.Println("kk dbg DocCount() resp:", resp, "err:", err, "elastic url:", elasticsearchURL) } if err != nil { - if resp != nil && (resp.StatusCode == http.StatusNoContent || resp.StatusCode == http.StatusNotFound) { - return 0, nil - } return -1, err } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNoContent || resp.StatusCode == http.StatusNotFound { + return 0, nil + } var jsonAsBytes []byte jsonAsBytes, err = io.ReadAll(resp.Body) @@ -193,12 +214,14 @@ func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, err err fmt.Println("kk dbg SizeInBytes() err:", err, "\nresp:", resp) } if err != nil { - if resp != nil && resp.StatusCode == 404 { - return 0, nil - } return } - defer resp.Body.Close() // add everywhere + + defer resp.Body.Close() + + if resp.StatusCode == 404 { + return 0, nil + } var jsonAsBytes []byte jsonAsBytes, err = io.ReadAll(resp.Body) From 6a72ca4102334846fbae6c6fa83dcd9d156adb00 Mon Sep 17 00:00:00 2001 From: Rafal Strzalinski Date: Wed, 12 Feb 2025 13:05:47 +0100 Subject: [PATCH 27/30] Remove gorountines. Simplification --- .../in_memory_fallback_elasticsearch.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go b/quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go index 3c8ddd96e..17c10c302 100644 --- a/quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go +++ b/quesma/quesma/async_search_storage/in_memory_fallback_elasticsearch.go @@ -21,7 +21,7 @@ func NewAsyncSearchStorageInMemoryFallbackElastic(cfg config.ElasticsearchConfig func (s AsyncRequestResultStorageInMemoryFallbackElastic) Store(id string, result *AsyncRequestResult) { s.inMemory.Store(id, result) - go s.inElasticsearch.Store(id, result) + s.inElasticsearch.Store(id, result) } func (s AsyncRequestResultStorageInMemoryFallbackElastic) Load(id string) (*AsyncRequestResult, error) { @@ -34,7 +34,7 @@ func (s AsyncRequestResultStorageInMemoryFallbackElastic) Load(id string) (*Asyn func (s AsyncRequestResultStorageInMemoryFallbackElastic) Delete(id string) { s.inMemory.Delete(id) - go s.inElasticsearch.Delete(id) + s.inElasticsearch.Delete(id) } // DocCount returns inMemory doc count @@ -54,7 +54,7 @@ func (s AsyncRequestResultStorageInMemoryFallbackElastic) SpaceMaxAvailable() in func (s AsyncRequestResultStorageInMemoryFallbackElastic) evict(olderThan time.Duration) { s.inMemory.evict(olderThan) - go s.inElasticsearch.DeleteOld(olderThan) + s.inElasticsearch.DeleteOld(olderThan) } type AsyncQueryContextStorageInMemoryFallbackElasticsearch struct { @@ -71,10 +71,10 @@ func NewAsyncQueryContextStorageInMemoryFallbackElasticsearch(cfg config.Elastic func (s AsyncQueryContextStorageInMemoryFallbackElasticsearch) Store(context *AsyncQueryContext) { s.inMemory.Store(context) - go s.inElasticsearch.Store(context) + s.inElasticsearch.Store(context) } func (s AsyncQueryContextStorageInMemoryFallbackElasticsearch) evict(evictOlderThan time.Duration) { s.inMemory.evict(evictOlderThan) - go s.inElasticsearch.evict(evictOlderThan) + s.inElasticsearch.evict(evictOlderThan) } From 96e9fa6b3bf8454bf9ca4248ae435aefb5d71115 Mon Sep 17 00:00:00 2001 From: Rafal Strzalinski Date: Wed, 12 Feb 2025 13:09:04 +0100 Subject: [PATCH 28/30] fmt --- quesma/persistence/elastic_with_eviction.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index bd3352a48..ee09396f9 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -216,7 +216,7 @@ func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, err err if err != nil { return } - + defer resp.Body.Close() if resp.StatusCode == 404 { From 44b337cdaa3b77c8e94b10ce3d5e444056561afc Mon Sep 17 00:00:00 2001 From: Rafal Strzalinski Date: Wed, 12 Feb 2025 16:18:54 +0100 Subject: [PATCH 29/30] Fix error handling --- quesma/elasticsearch/client.go | 12 ------------ quesma/persistence/elastic_with_eviction.go | 12 ++++++------ 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/quesma/elasticsearch/client.go b/quesma/elasticsearch/client.go index bc5851aa5..34f967b14 100644 --- a/quesma/elasticsearch/client.go +++ b/quesma/elasticsearch/client.go @@ -44,18 +44,6 @@ func (es *SimpleClient) RequestWithHeaders(ctx context.Context, method, endpoint return es.doRequest(ctx, method, endpoint, body, headers) } -func (es *SimpleClient) DoRequestCheckResponseStatusOK(ctx context.Context, method, endpoint string, body []byte) (resp *http.Response, err error) { - resp, err = es.doRequest(ctx, method, endpoint, body, nil) - if err != nil { - return - } - - if resp.StatusCode != http.StatusOK { - return resp, fmt.Errorf("response code from Elastic is not 200 OK, but %s", resp.Status) - } - return resp, nil -} - func (es *SimpleClient) Authenticate(ctx context.Context, authHeader string) bool { var authEndpoint string // This is really suboptimal, and we should find a better way to set this systematically (config perhaps?) diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index ee09396f9..afa074db2 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -75,7 +75,7 @@ func (db *ElasticDatabaseWithEviction) Put(document *JSONWithSize) error { return err } - resp, err := db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodPost, elasticsearchURL, jsonData) + resp, err := db.httpClient.Request(context.Background(), http.MethodPost, elasticsearchURL, jsonData) if printDebugElasticDB { fmt.Println("kk dbg Put() resp:", resp, "err:", err) } @@ -94,7 +94,7 @@ func (db *ElasticDatabaseWithEviction) Put(document *JSONWithSize) error { // Get TODO: probably change return type to some more useful func (db *ElasticDatabaseWithEviction) Get(id string) ([]byte, error) { elasticsearchURL := fmt.Sprintf("%s/_source/%s", db.indexName, id) - resp, err := db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodGet, elasticsearchURL, nil) + resp, err := db.httpClient.Request(context.Background(), http.MethodGet, elasticsearchURL, nil) if err != nil { return nil, err } @@ -105,7 +105,7 @@ func (db *ElasticDatabaseWithEviction) Get(id string) ([]byte, error) { func (db *ElasticDatabaseWithEviction) Delete(id string) error { elasticsearchURL := fmt.Sprintf("%s/_doc/%s", db.indexName, id) - resp, err := db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodDelete, elasticsearchURL, nil) + resp, err := db.httpClient.Request(context.Background(), http.MethodDelete, elasticsearchURL, nil) if err != nil { return err } @@ -142,7 +142,7 @@ func (db *ElasticDatabaseWithEviction) DeleteOld(deleteOlderThan time.Duration) } var resp *http.Response - resp, err = db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodPost, elasticsearchURL, []byte(query)) + resp, err = db.httpClient.Request(context.Background(), http.MethodPost, elasticsearchURL, []byte(query)) if err != nil { return } @@ -167,7 +167,7 @@ func (db *ElasticDatabaseWithEviction) DocCount() (docCount int, err error) { }` var resp *http.Response - resp, err = db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodGet, elasticsearchURL, []byte(query)) + resp, err = db.httpClient.Request(context.Background(), http.MethodGet, elasticsearchURL, []byte(query)) if printDebugElasticDB { fmt.Println("kk dbg DocCount() resp:", resp, "err:", err, "elastic url:", elasticsearchURL) } @@ -209,7 +209,7 @@ func (db *ElasticDatabaseWithEviction) SizeInBytes() (sizeInBytes int64, err err }` var resp *http.Response - resp, err = db.httpClient.DoRequestCheckResponseStatusOK(context.Background(), http.MethodGet, elasticsearchURL, []byte(query)) + resp, err = db.httpClient.Request(context.Background(), http.MethodGet, elasticsearchURL, []byte(query)) if printDebugElasticDB { fmt.Println("kk dbg SizeInBytes() err:", err, "\nresp:", resp) } From 8419c0e1034653289d58f64c954658af03024db3 Mon Sep 17 00:00:00 2001 From: trzysiek Date: Tue, 11 Mar 2025 12:39:30 +0100 Subject: [PATCH 30/30] Fix compilation after merge --- platform/async_search_storage/evictor.go | 4 ++-- platform/async_search_storage/in_elasticsearch.go | 6 +++--- platform/async_search_storage/in_memory.go | 2 -- .../in_memory_fallback_elasticsearch.go | 2 +- platform/async_search_storage/model.go | 4 ++-- platform/async_search_storage/storage_test.go | 2 +- platform/persistence/elastic_with_eviction.go | 4 ++-- platform/persistence/model.go | 2 +- 8 files changed, 12 insertions(+), 14 deletions(-) diff --git a/platform/async_search_storage/evictor.go b/platform/async_search_storage/evictor.go index 87d2d704e..48e0720d6 100644 --- a/platform/async_search_storage/evictor.go +++ b/platform/async_search_storage/evictor.go @@ -4,8 +4,8 @@ package async_search_storage import ( "context" - "github.com/QuesmaOrg/quesma/quesma/logger" - "github.com/QuesmaOrg/quesma/quesma/quesma/recovery" + "github.com/QuesmaOrg/quesma/platform/logger" + "github.com/QuesmaOrg/quesma/platform/recovery" "time" ) diff --git a/platform/async_search_storage/in_elasticsearch.go b/platform/async_search_storage/in_elasticsearch.go index 14dcc0c1a..a3e1aac15 100644 --- a/platform/async_search_storage/in_elasticsearch.go +++ b/platform/async_search_storage/in_elasticsearch.go @@ -5,9 +5,9 @@ package async_search_storage import ( "encoding/json" "fmt" - "github.com/QuesmaOrg/quesma/quesma/logger" - "github.com/QuesmaOrg/quesma/quesma/persistence" - "github.com/QuesmaOrg/quesma/quesma/quesma/config" + "github.com/QuesmaOrg/quesma/platform/config" + "github.com/QuesmaOrg/quesma/platform/logger" + "github.com/QuesmaOrg/quesma/platform/persistence" "time" ) diff --git a/platform/async_search_storage/in_memory.go b/platform/async_search_storage/in_memory.go index 002e99ff7..82e232942 100644 --- a/platform/async_search_storage/in_memory.go +++ b/platform/async_search_storage/in_memory.go @@ -3,10 +3,8 @@ package async_search_storage import ( - "context" "fmt" "github.com/QuesmaOrg/quesma/platform/logger" - "github.com/QuesmaOrg/quesma/platform/recovery" "github.com/QuesmaOrg/quesma/platform/util" "math" "strings" diff --git a/platform/async_search_storage/in_memory_fallback_elasticsearch.go b/platform/async_search_storage/in_memory_fallback_elasticsearch.go index 17c10c302..8f12b39a5 100644 --- a/platform/async_search_storage/in_memory_fallback_elasticsearch.go +++ b/platform/async_search_storage/in_memory_fallback_elasticsearch.go @@ -3,7 +3,7 @@ package async_search_storage import ( - "github.com/QuesmaOrg/quesma/quesma/quesma/config" + "github.com/QuesmaOrg/quesma/platform/config" "time" ) diff --git a/platform/async_search_storage/model.go b/platform/async_search_storage/model.go index 1eaab741e..978101f1b 100644 --- a/platform/async_search_storage/model.go +++ b/platform/async_search_storage/model.go @@ -4,8 +4,8 @@ package async_search_storage import ( "context" - "github.com/QuesmaOrg/quesma/quesma/persistence" - "github.com/QuesmaOrg/quesma/quesma/quesma/types" + "github.com/QuesmaOrg/quesma/platform/persistence" + "github.com/QuesmaOrg/quesma/platform/types" "time" ) diff --git a/platform/async_search_storage/storage_test.go b/platform/async_search_storage/storage_test.go index 0dd1c4977..416a164dd 100644 --- a/platform/async_search_storage/storage_test.go +++ b/platform/async_search_storage/storage_test.go @@ -6,7 +6,7 @@ import ( "context" "fmt" "github.com/ClickHouse/clickhouse-go/v2" - "github.com/QuesmaOrg/quesma/quesma/quesma/config" + "github.com/QuesmaOrg/quesma/platform/config" "github.com/k0kubun/pp" "github.com/stretchr/testify/assert" "net/url" diff --git a/platform/persistence/elastic_with_eviction.go b/platform/persistence/elastic_with_eviction.go index afa074db2..ca107de10 100644 --- a/platform/persistence/elastic_with_eviction.go +++ b/platform/persistence/elastic_with_eviction.go @@ -7,8 +7,8 @@ import ( "encoding/json" "errors" "fmt" - "github.com/QuesmaOrg/quesma/quesma/quesma/config" - "github.com/QuesmaOrg/quesma/quesma/quesma/types" + "github.com/QuesmaOrg/quesma/platform/config" + "github.com/QuesmaOrg/quesma/platform/types" "github.com/k0kubun/pp" "io" "math" diff --git a/platform/persistence/model.go b/platform/persistence/model.go index 11ebb64a6..90a9ed7be 100644 --- a/platform/persistence/model.go +++ b/platform/persistence/model.go @@ -3,7 +3,7 @@ package persistence import ( - "github.com/QuesmaOrg/quesma/quesma/quesma/types" + "github.com/QuesmaOrg/quesma/platform/types" "time" )