feat(search): integrate MeiliSearch as a new search backend

cmd/karpor/app/options/search_storage.go

@@ -28,10 +28,10 @@ type ElasticSearchConfig struct {
 }
 
 type SearchStorageOptions struct {
-	SearchStorageType      string
-	ElasticSearchAddresses []string
-	ElasticSearchUsername  string
-	ElasticSearchPassword  string
+	SearchStorageType string
+	SearchAddresses   []string
+	SearchUsername    string
+	SearchPassword    string
 }
 
 func NewSearchStorageOptions() *SearchStorageOptions {
@@ -44,9 +44,9 @@ func (o *SearchStorageOptions) Validate() []error {
 
 func (o *SearchStorageOptions) ApplyTo(config *registry.ExtraConfig) error {
 	config.SearchStorageType = o.SearchStorageType
-	config.ElasticSearchAddresses = o.ElasticSearchAddresses
-	config.ElasticSearchUsername = o.ElasticSearchUsername
-	config.ElasticSearchPassword = o.ElasticSearchPassword
+	config.SearchAddresses = o.SearchAddresses
+	config.SearchUsername = o.SearchUsername
+	config.SearchPassword = o.SearchPassword
 	return nil
 }
 
@@ -57,17 +57,17 @@ func (o *SearchStorageOptions) AddFlags(fs *pflag.FlagSet) {
 	}
 
 	fs.StringVar(&o.SearchStorageType, "search-storage-type", "", "The search storage type")
-	fs.StringSliceVar(&o.ElasticSearchAddresses, "elastic-search-addresses", nil, "The elastic search address")
-	fs.StringVar(&o.ElasticSearchUsername, "elastic-search-username", "", "The elastic search username")
-	fs.StringVar(&o.ElasticSearchPassword, "elastic-search-password", "", "The elastic search password")
+	fs.StringSliceVar(&o.SearchAddresses, "search-addresses", nil, "The search address")
+	fs.StringVar(&o.SearchUsername, "search-username", "", "The search username")
+	fs.StringVar(&o.SearchPassword, "search-password", "", "The search password")
 }
 
 // MarshalJSON is custom marshalling function for masking sensitive field values
 func (o SearchStorageOptions) MarshalJSON() ([]byte, error) {
 	type tempOptions SearchStorageOptions
 	o2 := tempOptions(o)
-	if o2.ElasticSearchPassword != "" {
-		o2.ElasticSearchPassword = MaskString
+	if o2.SearchPassword != "" {
+		o2.SearchPassword = MaskString
 	}
 	return json.Marshal(&o2)
 }

cmd/karpor/app/syncer.go

@@ -17,10 +17,10 @@ package app
 import (
 	"context"
 
-	"github.com/KusionStack/karpor/pkg/infra/search/storage/elasticsearch"
+	"github.com/KusionStack/karpor/pkg/kubernetes/registry"
+	"github.com/KusionStack/karpor/pkg/kubernetes/registry/search"
 	"github.com/KusionStack/karpor/pkg/kubernetes/scheme"
 	"github.com/KusionStack/karpor/pkg/syncer"
-	esclient "github.com/elastic/go-elasticsearch/v8"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"k8s.io/klog/v2"
@@ -29,9 +29,12 @@ import (
 )
 
 type syncerOptions struct {
-	MetricsAddr            string
-	ProbeAddr              string
-	ElasticSearchAddresses []string
+	SearchStorageType string
+	MetricsAddr       string
+	ProbeAddr         string
+	SearchAddresses   []string
+	SearchUsername    string
+	SearchPassword    string
 }
 
 func NewSyncerOptions() *syncerOptions {
@@ -41,7 +44,10 @@ func NewSyncerOptions() *syncerOptions {
 func (o *syncerOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&o.MetricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	fs.StringVar(&o.ProbeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
-	fs.StringSliceVar(&o.ElasticSearchAddresses, "elastic-search-addresses", nil, "The elastic search address.")
+	fs.StringSliceVar(&o.SearchAddresses, "search-addresses", nil, "The search engine address.")
+	fs.StringVar(&o.SearchStorageType, "search-storage-type", "", "The search storage type")
+	fs.StringVar(&o.SearchUsername, "search-username", "", "The search username")
+	fs.StringVar(&o.SearchPassword, "search-password", "", "The search password")
 }
 
 func NewSyncerCommand(ctx context.Context) *cobra.Command {
@@ -71,18 +77,19 @@ func run(ctx context.Context, options *syncerOptions) error {
 		return err
 	}
 
-	// TODO: add startup parameters to change the type of storage
-	//nolint:contextcheck
-	es, err := elasticsearch.NewStorage(esclient.Config{
-		Addresses: options.ElasticSearchAddresses,
+	searchStorage, err := search.NewResourceStorage(registry.ExtraConfig{
+		SearchStorageType: options.SearchStorageType,
+		SearchAddresses:   options.SearchAddresses,
+		SearchUsername:    options.SearchUsername,
+		SearchPassword:    options.SearchPassword,
 	})
 	if err != nil {
 		log.Error(err, "unable to init elasticsearch client")
 		return err
 	}
 
 	//nolint:contextcheck
-	if err = syncer.NewSyncReconciler(es).SetupWithManager(mgr); err != nil {
+	if err = syncer.NewSyncReconciler(searchStorage).SetupWithManager(mgr); err != nil {
 		log.Error(err, "unable to create resource syncer")
 		return err
 	}

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/google/gofuzz v1.2.0
 	github.com/google/uuid v1.4.0
 	github.com/hupe1980/go-huggingface v0.0.15
+	github.com/meilisearch/meilisearch-go v0.30.0
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852
 	github.com/pkg/errors v0.9.1
@@ -71,6 +72,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.1.1 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/ajg/form v1.5.1 // indirect
+	github.com/andybalholm/brotli v1.1.1 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
@@ -95,6 +97,7 @@ require (
 	github.com/go-openapi/spec v0.20.11 // indirect
 	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/cel-go v0.12.6 // indirect

go.sum

@@ -54,6 +54,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
+github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves=
 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
@@ -186,6 +188,8 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
+github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
@@ -322,6 +326,8 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM=
 github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/meilisearch/meilisearch-go v0.30.0 h1:J5TKZmfNOQc065+icxN2ShzT8u9F2/v6/gO/4DEw2ek=
+github.com/meilisearch/meilisearch-go v0.30.0/go.mod h1:NYOgjEGt/+oExD+NixreBMqxtIB0kCndXOOgpGhoqEs=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
 github.com/mitchellh/copystructure v1.1.1 h1:Bp6x9R1Wn16SIz3OfeDr0b7RnCG2OB66Y7PQyC/cvq4=
 github.com/mitchellh/copystructure v1.1.1/go.mod h1:EBArHfARyrSWO/+Wyr9zwEkc6XMFB9XyNgFNmRkZZU4=
@@ -435,6 +441,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/swaggo/files/v2 v2.0.0 h1:hmAt8Dkynw7Ssz46F6pn8ok6YmGZqHSVLZ+HQM7i0kw=
@@ -449,6 +456,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xwb1989/sqlparser v0.0.0-20171128062118-da747e0c62c4 h1:w96oitIHwAbUymu2zUSla/82gOKNzpJYkFdwCHE/UOA=
 github.com/xwb1989/sqlparser v0.0.0-20171128062118-da747e0c62c4/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

pkg/core/handler/resourcegroup/resource_group.go

@@ -17,9 +17,10 @@ package resourcegroup
 import (
 	"net/http"
 
+	"github.com/KusionStack/karpor/pkg/infra/search/storage"
+
 	"github.com/KusionStack/karpor/pkg/core/handler"
 	"github.com/KusionStack/karpor/pkg/core/manager/resourcegroup"
-	"github.com/KusionStack/karpor/pkg/infra/search/storage/elasticsearch"
 	"github.com/KusionStack/karpor/pkg/util/ctxutil"
 	"github.com/go-chi/chi/v5"
 	"github.com/pkg/errors"
@@ -58,7 +59,7 @@ func List(resourceGroupMgr *resourcegroup.ResourceGroupManager) http.HandlerFunc
 		// Use the ResourceGroupManager to list resource groups by specified rule.
 		rgs, err := resourceGroupMgr.ListResourceGroupsBy(ctx, name)
 		if err != nil {
-			if errors.Is(err, elasticsearch.ErrResourceGroupNotFound) {
+			if errors.Is(err, storage.ErrResourceGroupNotFound) {
 				handler.NotFoundRender(ctx, w, r, err)
 			} else {
 				handler.FailureRender(ctx, w, r, err)

pkg/core/handler/resourcegrouprule/resource_group_rule.go

@@ -17,9 +17,10 @@ package resourcegrouprule
 import (
 	"net/http"
 
+	"github.com/KusionStack/karpor/pkg/infra/search/storage"
+
 	"github.com/KusionStack/karpor/pkg/core/handler"
 	"github.com/KusionStack/karpor/pkg/core/manager/resourcegroup"
-	"github.com/KusionStack/karpor/pkg/infra/search/storage/elasticsearch"
 	"github.com/KusionStack/karpor/pkg/util/ctxutil"
 	"github.com/go-chi/chi/v5"
 	"github.com/pkg/errors"
@@ -195,7 +196,7 @@ func List(resourceGroupMgr *resourcegroup.ResourceGroupManager) http.HandlerFunc
 		// Use the ResourceGroupManager to list resource group rules.
 		rules, err := resourceGroupMgr.ListResourceGroupRules(ctx)
 		if err != nil {
-			if errors.Is(err, elasticsearch.ErrResourceGroupRuleNotFound) {
+			if errors.Is(err, storage.ErrResourceGroupRuleNotFound) {
 				handler.NotFoundRender(ctx, w, r, err)
 			} else {
 				handler.FailureRender(ctx, w, r, err)

pkg/core/manager/resourcegroup/manager.go

@@ -20,7 +20,6 @@ import (
 
 	"github.com/KusionStack/karpor/pkg/core/entity"
 	"github.com/KusionStack/karpor/pkg/infra/search/storage"
-	"github.com/KusionStack/karpor/pkg/infra/search/storage/elasticsearch"
 )
 
 type ResourceGroupManager struct {
@@ -61,7 +60,7 @@ func (m *ResourceGroupManager) CreateResourceGroupRule(ctx context.Context, rgr
 	if err == nil {
 		return ErrResourceGroupRuleAlreadyExists
 	}
-	if !errors.Is(err, elasticsearch.ErrResourceGroupRuleNotFound) {
+	if !errors.Is(err, storage.ErrResourceGroupRuleNotFound) {
 		return err
 	}
 	// Save the new rule to the storage.

pkg/core/manager/resourcegroup/manager_test.go

@@ -21,7 +21,6 @@ import (
 
 	"github.com/KusionStack/karpor/pkg/core/entity"
 	"github.com/KusionStack/karpor/pkg/infra/search/storage"
-	"github.com/KusionStack/karpor/pkg/infra/search/storage/elasticsearch"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -41,7 +40,7 @@ func (m *mockResourceGroupRuleStorage) GetResourceGroupRule(ctx context.Context,
 	if rule, exists := m.rules[name]; exists {
 		return rule, nil
 	}
-	return nil, elasticsearch.ErrResourceGroupRuleNotFound
+	return nil, storage.ErrResourceGroupRuleNotFound
 }
 
 func (m *mockResourceGroupRuleStorage) ListResourceGroupRules(ctx context.Context) ([]*entity.ResourceGroupRule, error) {
@@ -59,7 +58,7 @@ func (m *mockResourceGroupRuleStorage) SaveResourceGroupRule(ctx context.Context
 
 func (m *mockResourceGroupRuleStorage) DeleteResourceGroupRule(ctx context.Context, name string) error {
 	if _, exists := m.rules[name]; !exists {
-		return elasticsearch.ErrResourceGroupRuleNotFound
+		return storage.ErrResourceGroupRuleNotFound
 	}
 	delete(m.rules, name)
 	return nil
@@ -71,7 +70,7 @@ func (m *mockResourceGroupRuleStorage) CountResourceGroupRules(ctx context.Conte
 
 func (m *mockResourceGroupRuleStorage) ListResourceGroupsBy(ctx context.Context, ruleName string) (*storage.ResourceGroupResult, error) {
 	if _, exists := m.rules[ruleName]; !exists {
-		return nil, elasticsearch.ErrResourceGroupRuleNotFound
+		return nil, storage.ErrResourceGroupRuleNotFound
 	}
 	return &storage.ResourceGroupResult{
 		Groups: []*entity.ResourceGroup{
@@ -393,7 +392,7 @@ func TestResourceGroupManager_DeleteResourceGroupRule(t *testing.T) {
 				// Verify the rule was deleted
 				_, err := manager.GetResourceGroupRule(context.Background(), tt.ruleName)
 				require.Error(t, err)
-				require.ErrorIs(t, err, elasticsearch.ErrResourceGroupRuleNotFound)
+				require.ErrorIs(t, err, storage.ErrResourceGroupRuleNotFound)
 			}
 		})
 	}