vendor deps for full text search

Author: vdimir

backend/_example/memory_store/go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/go-pkgz/lgr v0.10.4
 	github.com/jessevdk/go-flags v1.5.0
 	github.com/stretchr/testify v1.8.1
-	github.com/umputun/remark42/backend v1.11.2
+	github.com/umputun/remark42/backend v1.11.3
 )
 
 require (
@@ -36,6 +36,7 @@ require (
 	golang.org/x/image v0.3.0 // indirect
 	golang.org/x/net v0.5.0 // indirect
 	golang.org/x/sys v0.4.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 

backend/_example/memory_store/go.sum

@@ -444,6 +444,11 @@ func (s *public) searchQueryCtrl(w http.ResponseWriter, r *http.Request) {
 		return encodeJSONWithHTML(commentsWithTotal{Comments: comments, Total: total})
 	})
 
+	if err == service.ErrSearchIsNotEnabled {
+		rest.SendErrorJSON(w, r, http.StatusNotFound, err, "can't perform search request", rest.ErrInternal)
+		return
+	}
+
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't perform search request", rest.ErrInternal)
 		return

backend/app/rest/api/rest_public.go

@@ -256,6 +256,7 @@ func TestRest_Search(t *testing.T) {
 		{"hello world test", "&sort=time&skip=1&limit=1", []int{2}, http.StatusOK},
 		{"", "", []int{}, http.StatusBadRequest},
 		{"test", "&sort=text", []int{}, http.StatusBadRequest},
+		{"test", "&limit=a", []int{}, http.StatusBadRequest},
 		{"test", "&skip=-1", []int{}, http.StatusBadRequest},
 		{"test", "&limit=999999", []int{}, http.StatusBadRequest},
 	}
@@ -299,7 +300,7 @@ func TestRest_SearchDisabledFeature(t *testing.T) {
 	resp, err := http.Get(ts.URL + "/api/v1/search?site=remark42&query=test")
 	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())
-	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+	assert.Equal(t, http.StatusNotFound, resp.StatusCode)
 }
 
 func Test_URLKey(t *testing.T) {

backend/app/rest/api/rest_test.go

@@ -2,7 +2,9 @@ package search
 
 import (
 	"context"
+	"encoding/hex"
 	"fmt"
+	"hash/fnv"
 	"math/rand"
 	"os"
 	"testing"
@@ -113,6 +115,10 @@ func TestSearch_Site(t *testing.T) {
 	require.NoError(t, err)
 	assert.Len(t, res.Keys, 0)
 
+	res, err = searcher.Search(&Request{SiteID: "test-site-not-found", Query: "345", Limit: 3})
+	require.Error(t, err)
+	assert.Nil(t, res)
+
 	// restart service to check that saved index is loaded
 	err = searcher.Close()
 	require.NoError(t, err)
@@ -236,23 +242,39 @@ func TestSearch_IndexStartup(t *testing.T) {
 	searcher, serviceTeardown := createTestService(t, sites)
 	defer serviceTeardown()
 
+	sites = append(sites, "not-indexed")
 	storeEngine, dbTeardown := createDB(t, 42, sites)
 	defer dbTeardown()
 
 	grp := syncs.NewErrSizedGroup(4)
+
 	for _, siteID := range sites {
 		err := IndexSite(context.Background(), siteID, searcher, storeEngine, grp)
 		require.NoError(t, err)
 	}
 	err := grp.Wait()
 	require.NoError(t, err)
 
+	// run index again, should not fail, but do nothing
+	err = IndexSite(context.Background(), sites[0], searcher, storeEngine, grp)
+	require.NoError(t, err)
+
+	err = grp.Wait()
+	require.NoError(t, err)
+
 	for _, siteID := range sites {
+
 		serp, err := searcher.Search(&Request{
 			SiteID: siteID,
 			Query:  "text",
 			Limit:  19,
 		})
+
+		if siteID == "not-indexed" {
+			assert.Error(t, err)
+			continue
+		}
+
 		assert.NoError(t, err)
 		assert.Len(t, serp.Keys, 19)
 	}
@@ -270,6 +292,47 @@ func TestSearch_BadAnalyzerName(t *testing.T) {
 	require.Error(t, err)
 }
 
+func TestSearch_NewDirtyPath(t *testing.T) {
+	idxPath := os.TempDir() + "/search-remark42"
+	_ = os.RemoveAll(idxPath)
+
+	err := os.MkdirAll(idxPath, 0o0755)
+	require.NoError(t, err)
+
+	defer func() { _ = os.RemoveAll(idxPath) }()
+
+	// that's how we calculate site index path in NewService
+	siteIdxPath := idxPath + "/" + hex.EncodeToString(fnv.New32().Sum([]byte("test-size")))
+
+	// try to create index specifying path to file
+	err = os.WriteFile(siteIdxPath, []byte("some_data"), 0o0644)
+	require.NoError(t, err)
+	// file exists, but not a directory
+	_, err = NewService([]string{"test-size"}, ServiceParams{IndexPath: idxPath, Analyzer: "standard"})
+	assert.Error(t, err)
+	assert.ErrorContains(t, err, "index path should be a directory")
+
+	err = os.Remove(siteIdxPath)
+	require.NoError(t, err)
+	err = os.MkdirAll(siteIdxPath, 0o0755)
+	require.NoError(t, err)
+
+	// write some garbage to site index dir
+	err = os.WriteFile(siteIdxPath+"/dirty", []byte("garbage"), 0o0644)
+	require.NoError(t, err)
+
+	// site index dir is dirty
+	_, err = NewService([]string{"test-size"}, ServiceParams{IndexPath: idxPath, Analyzer: "standard"})
+	assert.Error(t, err, "index path should be a directory")
+
+	err = os.Remove(siteIdxPath + "/dirty")
+	require.NoError(t, err)
+
+	// empty index dir
+	_, err = NewService([]string{"test-size"}, ServiceParams{IndexPath: idxPath, Analyzer: "standard"})
+	assert.NoError(t, err)
+}
+
 func createDB(t *testing.T, commentsPerSite int, sites []string) (e engine.Interface, teardown func()) {
 	testDB := os.TempDir() + "/remark-db"
 	_ = os.RemoveAll(testDB)

backend/app/store/search/search_test.go

@@ -83,6 +83,9 @@ var nonAdminUser = store.User{}
 // ErrRestrictedWordsFound returned in case comment text contains restricted words
 var ErrRestrictedWordsFound = fmt.Errorf("comment contains restricted words")
 
+// ErrSearchIsNotEnabled returned on search request if search service was not configured
+var ErrSearchIsNotEnabled = fmt.Errorf("search service is not enabled")
+
 // Create prepares comment and forward to Interface.Create
 func (s *DataStore) Create(comment store.Comment) (commentID string, err error) {
 	if comment, err = s.prepareNewComment(comment); err != nil {
@@ -923,7 +926,7 @@ func (s *DataStore) Last(siteID string, limit int, since time.Time, user store.U
 // Search executes search query and enriches the result with comment content
 func (s *DataStore) Search(siteID, query, sortBy string, limit, skip int) ([]store.Comment, uint64, error) {
 	if s.SearchService == nil {
-		return []store.Comment{}, 0, fmt.Errorf("search service is not enabled")
+		return []store.Comment{}, 0, ErrSearchIsNotEnabled
 	}
 	req := &search.Request{
 		SiteID: siteID,
@@ -941,12 +944,14 @@ func (s *DataStore) Search(siteID, query, sortBy string, limit, skip int) ([]sto
 	comments := make([]store.Comment, 0, len(searchRes.Keys))
 	for _, docKey := range searchRes.Keys {
 		comment, e := s.Get(docKey.Locator, docKey.ID, nonAdminUser)
-		if e != nil {
+		if e != nil || comment.Deleted {
+			searchRes.Total--
 			// ignore error because comment can be deleted from the store (but we still have it in search index)
 			continue
 		}
 		comments = append(comments, comment)
 	}
+
 	return comments, searchRes.Total, nil
 }
 

backend/app/store/service/service.go

@@ -24,6 +24,7 @@ import (
 	"github.com/umputun/remark42/backend/app/store/admin"
 	"github.com/umputun/remark42/backend/app/store/engine"
 	"github.com/umputun/remark42/backend/app/store/image"
+	"github.com/umputun/remark42/backend/app/store/search"
 )
 
 func TestService_CreateFromEmpty(t *testing.T) {
@@ -1616,6 +1617,50 @@ func TestService_alterComment(t *testing.T) {
 	assert.Equal(t, engine.FlagRequest{Flag: engine.Blocked, UserID: "devid"}, engineMock.FlagCalls()[0].Req)
 }
 
+func TestService_Search(t *testing.T) {
+	ks := admin.NewStaticKeyStore("secret 123")
+
+	eng, engTeardown := prepStoreEngine(t)
+	defer engTeardown()
+
+	srch, srchTeardown := prepSearcher(t)
+	defer srchTeardown()
+
+	b := DataStore{Engine: eng, AdminStore: ks}
+
+	{
+		res, total, err := b.Search("radio-t", "text", "", 10, 0)
+		assert.Error(t, err)
+		assert.Equal(t, uint64(0), total)
+		assert.Equal(t, 0, len(res))
+	}
+
+	b.SearchService = srch
+
+	comment := store.Comment{
+		Text:    "some text",
+		User:    store.User{ID: "newUser", Name: "new user"},
+		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+	}
+
+	id, err := b.Create(comment)
+	require.NoError(t, err)
+
+	res, total, err := b.Search("radio-t", "text user:\"new user\"", "", 10, 0)
+	assert.NoError(t, err)
+	assert.Equal(t, uint64(1), total)
+	assert.Equal(t, 1, len(res))
+	assert.Equal(t, "some text", res[0].Text)
+
+	err = b.Delete(comment.Locator, id, store.SoftDelete)
+	require.NoError(t, err)
+
+	res, total, err = b.Search("radio-t", "text user:\"new user\"", "", 10, 0)
+	assert.NoError(t, err)
+	assert.Equal(t, uint64(0), total)
+	assert.Equal(t, 0, len(res))
+}
+
 func Benchmark_ServiceCreate(b *testing.B) {
 	dbFile := fmt.Sprintf("%s/test-remark42-%d.db", os.TempDir(), rand.Intn(9999999999))
 	defer func() { _ = os.Remove(dbFile) }()
@@ -1705,3 +1750,23 @@ func getReq(locator store.Locator, commentID string) engine.GetRequest {
 		CommentID: commentID,
 	}
 }
+
+func prepSearcher(t *testing.T) (s *search.Service, teardown func()) {
+	idxPath, err := os.MkdirTemp("", "test_search_r42")
+	require.NoError(t, err)
+	_ = os.Remove(idxPath)
+
+	s, err = search.NewService([]string{"radio-t"}, search.ServiceParams{
+		IndexPath: idxPath,
+		Analyzer:  "standard",
+	})
+
+	require.NoError(t, err)
+
+	teardown = func() {
+		err := s.Close()
+		require.NoError(t, err)
+		_ = os.RemoveAll(idxPath)
+	}
+	return s, teardown
+}

backend/app/store/service/service_test.go

@@ -38,10 +38,29 @@ require (
 )
 
 require (
-	cloud.google.com/go/compute v1.6.1 // indirect
+	cloud.google.com/go/compute v1.14.0 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	github.com/RoaringBitmap/roaring v0.9.4 // indirect
 	github.com/ajg/form v1.5.1 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
+	github.com/bits-and-blooms/bitset v1.2.0 // indirect
+	github.com/blevesearch/bleve/v2 v2.3.6
+	github.com/blevesearch/bleve_index_api v1.0.5 // indirect
+	github.com/blevesearch/geo v0.1.16 // indirect
+	github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
+	github.com/blevesearch/gtreap v0.1.1 // indirect
+	github.com/blevesearch/mmap-go v1.0.4 // indirect
+	github.com/blevesearch/scorch_segment_api/v2 v2.1.4 // indirect
+	github.com/blevesearch/segment v0.9.0 // indirect
+	github.com/blevesearch/snowballstem v0.9.0 // indirect
+	github.com/blevesearch/upsidedown_store_api v1.0.1 // indirect
+	github.com/blevesearch/vellum v1.0.9 // indirect
+	github.com/blevesearch/zapx/v11 v11.3.7 // indirect
+	github.com/blevesearch/zapx/v12 v12.3.7 // indirect
+	github.com/blevesearch/zapx/v13 v13.3.7 // indirect
+	github.com/blevesearch/zapx/v14 v14.3.7 // indirect
+	github.com/blevesearch/zapx/v15 v15.3.8 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dghubble/oauth1 v0.7.2 // indirect
@@ -51,14 +70,17 @@ require (
 	github.com/go-pkgz/email v0.4.1 // indirect
 	github.com/go-pkgz/expirable-cache v0.1.0 // indirect
 	github.com/go-redis/redis/v8 v8.11.5 // indirect
+	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/golang-lru v0.6.0 // indirect
+	github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede // indirect
 	github.com/klauspost/compress v1.15.2 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/mschoch/smat v0.2.0 // indirect
 	github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -68,11 +90,12 @@ require (
 	github.com/xdg-go/stringprep v1.0.3 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
 	go.mongodb.org/mongo-driver v1.11.1 // indirect
+	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
 	golang.org/x/oauth2 v0.4.0 // indirect
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
+	golang.org/x/sync v0.1.0 // indirect
 	golang.org/x/sys v0.4.0 // indirect
 	golang.org/x/text v0.6.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )