diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index b4b4f3a8a2bea..b566f2522bf8b 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1406,9 +1406,9 @@ LEVEL = Info ;; repo indexer by default disabled, since it uses a lot of disk space ;REPO_INDEXER_ENABLED = false ;; -;; repo indexer units, the items to index, could be `sources`, `forks`, `mirrors`, `templates` or any combination of them separated by a comma. +;; repo indexer units, the items to index, could be `sources`, `forks`, `mirrors`, `templates`, `wikis` or any combination of them separated by a comma. ;; If empty then it defaults to `sources` only, as if you'd like to disable fully please see REPO_INDEXER_ENABLED. -;REPO_INDEXER_REPO_TYPES = sources,forks,mirrors,templates +;REPO_INDEXER_REPO_TYPES = sources,forks,mirrors,templates,wikis ;; ;; Code search engine type, could be `bleve` or `elasticsearch`. ;REPO_INDEXER_TYPE = bleve diff --git a/docs/content/administration/config-cheat-sheet.en-us.md b/docs/content/administration/config-cheat-sheet.en-us.md index 2309021f94937..71c812d5c69f4 100644 --- a/docs/content/administration/config-cheat-sheet.en-us.md +++ b/docs/content/administration/config-cheat-sheet.en-us.md @@ -473,7 +473,7 @@ relation to port exhaustion. - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch. Relative paths will be made absolute against _`AppWorkPath`_. - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size). -- `REPO_INDEXER_REPO_TYPES`: **sources,forks,mirrors,templates**: Repo indexer units. The items to index could be `sources`, `forks`, `mirrors`, `templates` or any combination of them separated by a comma. If empty then it defaults to `sources` only, as if you'd like to disable fully please see `REPO_INDEXER_ENABLED`. +- `REPO_INDEXER_REPO_TYPES`: **sources,forks,mirrors,templates,wikis**: Repo indexer units. The items to index could be `sources`, `forks`, `mirrors`, `templates`, `wikis` or any combination of them separated by a comma. If empty then it defaults to `sources` only, as if you'd like to disable fully please see `REPO_INDEXER_ENABLED`. - `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`. - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search. - `REPO_INDEXER_CONN_STR`: ****: Code indexer connection string, available when `REPO_INDEXER_TYPE` is elasticsearch. i.e. http://elastic:password@localhost:9200 diff --git a/models/repo/repo_indexer.go b/models/repo/repo_indexer.go index 6e19d8f937f18..4875354823273 100644 --- a/models/repo/repo_indexer.go +++ b/models/repo/repo_indexer.go @@ -20,6 +20,8 @@ const ( RepoIndexerTypeCode RepoIndexerType = iota // 0 // RepoIndexerTypeStats repository stats indexer RepoIndexerTypeStats // 1 + // RepoIndexerTypeWiki wiki indexer + RepoIndexerTypeWiki // 2 ) // RepoIndexerStatus status of a repo's entry in the repo indexer diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go index d7f735e957db9..54d0ab234d95b 100644 --- a/modules/indexer/code/bleve/bleve.go +++ b/modules/indexer/code/bleve/bleve.go @@ -20,6 +20,7 @@ import ( indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_bleve "code.gitea.io/gitea/modules/indexer/internal/bleve" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/typesniffer" @@ -51,6 +52,7 @@ func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error { // RepoIndexerData data stored in the repo indexer type RepoIndexerData struct { RepoID int64 + IsWiki bool CommitID string Content string Language string @@ -65,7 +67,7 @@ func (d *RepoIndexerData) Type() string { const ( repoIndexerAnalyzer = "repoIndexerAnalyzer" repoIndexerDocType = "repoIndexerDocType" - repoIndexerLatestVersion = 6 + repoIndexerLatestVersion = 7 ) // generateBleveIndexMapping generates a bleve index mapping for the repo indexer @@ -75,6 +77,10 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) { numericFieldMapping.IncludeInAll = false docMapping.AddFieldMappingsAt("RepoID", numericFieldMapping) + boolFieldMapping := bleve.NewBooleanFieldMapping() + boolFieldMapping.IncludeInAll = false + docMapping.AddFieldMappingsAt("IsWiki", boolFieldMapping) + textFieldMapping := bleve.NewTextFieldMapping() textFieldMapping.IncludeInAll = false docMapping.AddFieldMappingsAt("Content", textFieldMapping) @@ -125,7 +131,7 @@ func NewIndexer(indexDir string) *Indexer { } func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, commitSha string, - update internal.FileUpdate, repo *repo_model.Repository, batch *inner_bleve.FlushingBatch, + update internal.FileUpdate, repo *repo_model.Repository, isWiki bool, batch *inner_bleve.FlushingBatch, ) error { // Ignore vendored files in code search if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) { @@ -134,10 +140,15 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro size := update.Size + repoPath := repo.RepoPath() + if isWiki { + repoPath = repo.WikiPath() + } + var err error if !update.Sized { var stdout string - stdout, _, err = git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(update.BlobSha).RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + stdout, _, err = git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(update.BlobSha).RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -147,7 +158,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro } if size > setting.Indexer.MaxIndexerFileSize { - return b.addDelete(update.Filename, repo, batch) + return b.addDelete(update.Filename, repo, isWiki, batch) } if _, err := batchWriter.Write([]byte(update.BlobSha + "\n")); err != nil { @@ -170,9 +181,10 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro if _, err = batchReader.Discard(1); err != nil { return err } - id := internal.FilenameIndexerID(repo.ID, update.Filename) + id := internal.FilenameIndexerID(repo.ID, isWiki, update.Filename) return batch.Index(id, &RepoIndexerData{ RepoID: repo.ID, + IsWiki: isWiki, CommitID: commitSha, Content: string(charset.ToUTF8DropErrors(fileContents, charset.ConvertOpts{})), Language: analyze.GetCodeLanguage(update.Filename, fileContents), @@ -180,34 +192,39 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro }) } -func (b *Indexer) addDelete(filename string, repo *repo_model.Repository, batch *inner_bleve.FlushingBatch) error { - id := internal.FilenameIndexerID(repo.ID, filename) +func (b *Indexer) addDelete(filename string, repo *repo_model.Repository, isWiki bool, batch *inner_bleve.FlushingBatch) error { + id := internal.FilenameIndexerID(repo.ID, isWiki, filename) return batch.Delete(id) } // Index indexes the data -func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { +func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, isWiki bool, sha string, changes *internal.RepoChanges) error { + repoPath := repo.RepoPath() + if isWiki { + repoPath = repo.WikiPath() + } + batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize) if len(changes.Updates) > 0 { // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! - if err := git.EnsureValidGitRepository(ctx, repo.RepoPath()); err != nil { - log.Error("Unable to open git repo: %s for %-v: %v", repo.RepoPath(), repo, err) + if err := git.EnsureValidGitRepository(ctx, repoPath); err != nil { + log.Error("Unable to open git repo: %s for %-v: %v", repoPath, repo, err) return err } - batchWriter, batchReader, cancel := git.CatFileBatch(ctx, repo.RepoPath()) + batchWriter, batchReader, cancel := git.CatFileBatch(ctx, repoPath) defer cancel() for _, update := range changes.Updates { - if err := b.addUpdate(ctx, batchWriter, batchReader, sha, update, repo, batch); err != nil { + if err := b.addUpdate(ctx, batchWriter, batchReader, sha, update, repo, isWiki, batch); err != nil { return err } } cancel() } for _, filename := range changes.RemovedFilenames { - if err := b.addDelete(filename, repo, batch); err != nil { + if err := b.addDelete(filename, repo, isWiki, batch); err != nil { return err } } @@ -215,8 +232,14 @@ func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha st } // Delete deletes indexes by ids -func (b *Indexer) Delete(_ context.Context, repoID int64) error { - query := inner_bleve.NumericEqualityQuery(repoID, "RepoID") +func (b *Indexer) Delete(_ context.Context, repoID int64, isWiki optional.Option[bool]) error { + var query query.Query + query = inner_bleve.NumericEqualityQuery(repoID, "RepoID") + if isWiki.Has() { + wikiQuery := bleve.NewBoolFieldQuery(isWiki.Value()) + wikiQuery.FieldVal = "IsWiki" + query = bleve.NewConjunctionQuery(query, wikiQuery) + } searchRequest := bleve.NewSearchRequestOptions(query, 2147483647, 0, false) result, err := b.inner.Indexer.Search(searchRequest) if err != nil { @@ -264,6 +287,12 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int indexerQuery = keywordQuery } + if opts.IsWiki.Has() { + wikiQuery := bleve.NewBoolFieldQuery(opts.IsWiki.Value()) + wikiQuery.FieldVal = "IsWiki" + indexerQuery = bleve.NewConjunctionQuery(indexerQuery, wikiQuery) + } + // Save for reuse without language filter facetQuery := indexerQuery if len(opts.Language) > 0 { @@ -279,7 +308,7 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int from, pageSize := opts.GetSkipTake() searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false) - searchRequest.Fields = []string{"Content", "RepoID", "Language", "CommitID", "UpdatedAt"} + searchRequest.Fields = []string{"Content", "RepoID", "IsWiki", "Language", "CommitID", "UpdatedAt"} searchRequest.IncludeLocations = true if len(opts.Language) == 0 { diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go index e4622fd66ef95..2d94ddbb0a296 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch.go +++ b/modules/indexer/code/elasticsearch/elasticsearch.go @@ -20,6 +20,7 @@ import ( inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/typesniffer" @@ -29,7 +30,7 @@ import ( ) const ( - esRepoIndexerLatestVersion = 1 + esRepoIndexerLatestVersion = 2 // multi-match-types, currently only 2 types are used // Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types esMultiMatchTypeBestFields = "best_fields" @@ -62,6 +63,10 @@ const ( "type": "long", "index": true }, + "is_wiki": { + "type": "boolean" + "index": true + } "content": { "type": "text", "term_vector": "with_positions_offsets", @@ -84,17 +89,22 @@ const ( }` ) -func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update internal.FileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) { +func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update internal.FileUpdate, repo *repo_model.Repository, isWiki bool) ([]elastic.BulkableRequest, error) { // Ignore vendored files in code search if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) { return nil, nil } size := update.Size + repoPath := repo.RepoPath() + if isWiki { + repoPath = repo.WikiPath() + } + var err error if !update.Sized { var stdout string - stdout, _, err = git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(update.BlobSha).RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + stdout, _, err = git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(update.BlobSha).RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return nil, err } @@ -104,7 +114,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro } if size > setting.Indexer.MaxIndexerFileSize { - return []elastic.BulkableRequest{b.addDelete(update.Filename, repo)}, nil + return []elastic.BulkableRequest{b.addDelete(update.Filename, repo, isWiki)}, nil } if _, err := batchWriter.Write([]byte(update.BlobSha + "\n")); err != nil { @@ -127,7 +137,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro if _, err = batchReader.Discard(1); err != nil { return nil, err } - id := internal.FilenameIndexerID(repo.ID, update.Filename) + id := internal.FilenameIndexerID(repo.ID, isWiki, update.Filename) return []elastic.BulkableRequest{ elastic.NewBulkIndexRequest(). @@ -135,6 +145,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro Id(id). Doc(map[string]any{ "repo_id": repo.ID, + "is_wiki": isWiki, "content": string(charset.ToUTF8DropErrors(fileContents, charset.ConvertOpts{})), "commit_id": sha, "language": analyze.GetCodeLanguage(update.Filename, fileContents), @@ -143,28 +154,33 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro }, nil } -func (b *Indexer) addDelete(filename string, repo *repo_model.Repository) elastic.BulkableRequest { - id := internal.FilenameIndexerID(repo.ID, filename) +func (b *Indexer) addDelete(filename string, repo *repo_model.Repository, isWiki bool) elastic.BulkableRequest { + id := internal.FilenameIndexerID(repo.ID, isWiki, filename) return elastic.NewBulkDeleteRequest(). Index(b.inner.VersionedIndexName()). Id(id) } // Index will save the index data -func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { +func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, isWiki bool, sha string, changes *internal.RepoChanges) error { + repoPath := repo.RepoPath() + if isWiki { + repoPath = repo.WikiPath() + } + reqs := make([]elastic.BulkableRequest, 0) if len(changes.Updates) > 0 { // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! - if err := git.EnsureValidGitRepository(ctx, repo.RepoPath()); err != nil { - log.Error("Unable to open git repo: %s for %-v: %v", repo.RepoPath(), repo, err) + if err := git.EnsureValidGitRepository(ctx, repoPath); err != nil { + log.Error("Unable to open git repo: %s for %-v: %v", repoPath, repo, err) return err } - batchWriter, batchReader, cancel := git.CatFileBatch(ctx, repo.RepoPath()) + batchWriter, batchReader, cancel := git.CatFileBatch(ctx, repoPath) defer cancel() for _, update := range changes.Updates { - updateReqs, err := b.addUpdate(ctx, batchWriter, batchReader, sha, update, repo) + updateReqs, err := b.addUpdate(ctx, batchWriter, batchReader, sha, update, repo, isWiki) if err != nil { return err } @@ -176,7 +192,7 @@ func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha st } for _, filename := range changes.RemovedFilenames { - reqs = append(reqs, b.addDelete(filename, repo)) + reqs = append(reqs, b.addDelete(filename, repo, isWiki)) } if len(reqs) > 0 { @@ -196,9 +212,14 @@ func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha st } // Delete deletes indexes by ids -func (b *Indexer) Delete(ctx context.Context, repoID int64) error { +func (b *Indexer) Delete(ctx context.Context, repoID int64, isWiki optional.Option[bool]) error { + query := elastic.NewBoolQuery() + query = query.Must(elastic.NewTermsQuery("repo_id", repoID)) + if isWiki.Has() { + query = query.Must(elastic.NewTermQuery("is_wiki", isWiki.Value())) + } _, err := b.inner.Client.DeleteByQuery(b.inner.VersionedIndexName()). - Query(elastic.NewTermsQuery("repo_id", repoID)). + Query(query). Do(ctx) return err } @@ -239,7 +260,11 @@ func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) panic(fmt.Sprintf("2===%#v", hit.Highlight)) } - repoID, fileName := internal.ParseIndexerID(hit.Id) + repoID, isWiki, fileName, err := internal.ParseIndexerID(hit.Id) + if err != nil { + return 0, nil, nil, err + } + res := make(map[string]any) if err := json.Unmarshal(hit.Source, &res); err != nil { return 0, nil, nil, err @@ -249,6 +274,7 @@ func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) hits = append(hits, &internal.SearchResult{ RepoID: repoID, + IsWiki: isWiki, Filename: fileName, CommitID: res["commit_id"].(string), Content: res["content"].(string), @@ -299,6 +325,10 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int query = query.Must(repoQuery) } + if opts.IsWiki.Has() { + query = query.Must(elastic.NewTermQuery("is_wiki", opts.IsWiki.Value())) + } + var ( start, pageSize = opts.GetSkipTake() kw = "" + opts.Keyword + "" diff --git a/modules/indexer/code/git.go b/modules/indexer/code/git.go index 2905a540e56c1..0ac290737a91c 100644 --- a/modules/indexer/code/git.go +++ b/modules/indexer/code/git.go @@ -12,20 +12,42 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/indexer/code/internal" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/setting" ) -func getDefaultBranchSha(ctx context.Context, repo *repo_model.Repository) (string, error) { - stdout, _, err := git.NewCommand(ctx, "show-ref", "-s").AddDynamicArguments(git.BranchPrefix + repo.DefaultBranch).RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) +func getDefaultBranchSha(ctx context.Context, repo *repo_model.Repository, isWiki bool) (string, error) { + repoPath := repo.RepoPath() + defaultBranch := repo.DefaultBranch + if isWiki { + repoPath = repo.WikiPath() + defaultBranch = repo.DefaultWikiBranch + } + + stdout, _, err := git.NewCommand(ctx, "show-ref", "-s").AddDynamicArguments(git.BranchPrefix + defaultBranch).RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return "", err } return strings.TrimSpace(stdout), nil } +func getRepoStatus(ctx context.Context, repo *repo_model.Repository, isWiki bool) (*repo_model.RepoIndexerStatus, error) { + indexerType := repo_model.RepoIndexerTypeCode + if isWiki { + indexerType = repo_model.RepoIndexerTypeWiki + } + + return repo_model.GetIndexerStatus(ctx, repo, indexerType) +} + // getRepoChanges returns changes to repo since last indexer update -func getRepoChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*internal.RepoChanges, error) { - status, err := repo_model.GetIndexerStatus(ctx, repo, repo_model.RepoIndexerTypeCode) +func getRepoChanges(ctx context.Context, repo *repo_model.Repository, isWiki bool, revision string) (*internal.RepoChanges, error) { + repoPath := repo.RepoPath() + if isWiki { + repoPath = repo.WikiPath() + } + + status, err := getRepoStatus(ctx, repo, isWiki) if err != nil { return nil, err } @@ -33,14 +55,14 @@ func getRepoChanges(ctx context.Context, repo *repo_model.Repository, revision s needGenesis := len(status.CommitSha) == 0 if !needGenesis { hasAncestorCmd := git.NewCommand(ctx, "merge-base").AddDynamicArguments(status.CommitSha, revision) - stdout, _, _ := hasAncestorCmd.RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + stdout, _, _ := hasAncestorCmd.RunStdString(&git.RunOpts{Dir: repoPath}) needGenesis = len(stdout) == 0 } if needGenesis { - return genesisChanges(ctx, repo, revision) + return genesisChanges(ctx, repo, isWiki, revision) } - return nonGenesisChanges(ctx, repo, revision) + return nonGenesisChanges(ctx, repo, isWiki, status, revision) } func isIndexable(entry *git.TreeEntry) bool { @@ -84,14 +106,23 @@ func parseGitLsTreeOutput(objectFormat git.ObjectFormat, stdout []byte) ([]inter } // genesisChanges get changes to add repo to the indexer for the first time -func genesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*internal.RepoChanges, error) { +func genesisChanges(ctx context.Context, repo *repo_model.Repository, isWiki bool, revision string) (*internal.RepoChanges, error) { var changes internal.RepoChanges - stdout, _, runErr := git.NewCommand(ctx, "ls-tree", "--full-tree", "-l", "-r").AddDynamicArguments(revision).RunStdBytes(&git.RunOpts{Dir: repo.RepoPath()}) + repoPath := repo.RepoPath() + if isWiki { + repoPath = repo.WikiPath() + } + + stdout, _, runErr := git.NewCommand(ctx, "ls-tree", "--full-tree", "-l", "-r").AddDynamicArguments(revision).RunStdBytes(&git.RunOpts{Dir: repoPath}) if runErr != nil { return nil, runErr } - objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName) + repoObjectFormatName := "sha1" + if !isWiki { + repoObjectFormatName = repo.ObjectFormatName + } + objectFormat := git.ObjectFormatFromName(repoObjectFormatName) var err error changes.Updates, err = parseGitLsTreeOutput(objectFormat, stdout) @@ -99,17 +130,22 @@ func genesisChanges(ctx context.Context, repo *repo_model.Repository, revision s } // nonGenesisChanges get changes since the previous indexer update -func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*internal.RepoChanges, error) { - diffCmd := git.NewCommand(ctx, "diff", "--name-status").AddDynamicArguments(repo.CodeIndexerStatus.CommitSha, revision) - stdout, _, runErr := diffCmd.RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) +func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, isWiki bool, indexerStatus *repo_model.RepoIndexerStatus, revision string) (*internal.RepoChanges, error) { + repoPath := repo.RepoPath() + if isWiki { + repoPath = repo.WikiPath() + } + + diffCmd := git.NewCommand(ctx, "diff", "--name-status").AddDynamicArguments(indexerStatus.CommitSha, revision) + stdout, _, runErr := diffCmd.RunStdString(&git.RunOpts{Dir: repoPath}) if runErr != nil { // previous commit sha may have been removed by a force push, so // try rebuilding from scratch log.Warn("git diff: %v", runErr) - if err := (*globalIndexer.Load()).Delete(ctx, repo.ID); err != nil { + if err := (*globalIndexer.Load()).Delete(ctx, repo.ID, optional.Some(isWiki)); err != nil { return nil, err } - return genesisChanges(ctx, repo, revision) + return genesisChanges(ctx, repo, isWiki, revision) } var changes internal.RepoChanges @@ -167,12 +203,16 @@ func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revisio cmd := git.NewCommand(ctx, "ls-tree", "--full-tree", "-l").AddDynamicArguments(revision). AddDashesAndList(updatedFilenames...) - lsTreeStdout, _, err := cmd.RunStdBytes(&git.RunOpts{Dir: repo.RepoPath()}) + lsTreeStdout, _, err := cmd.RunStdBytes(&git.RunOpts{Dir: repoPath}) if err != nil { return nil, err } - objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName) + repoObjectFormatName := "sha1" + if !isWiki { + repoObjectFormatName = repo.ObjectFormatName + } + objectFormat := git.ObjectFormatFromName(repoObjectFormatName) changes.Updates, err = parseGitLsTreeOutput(objectFormat, lsTreeStdout) return &changes, err diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go index ebebf6ba8a28d..1eb4b71542d56 100644 --- a/modules/indexer/code/indexer.go +++ b/modules/indexer/code/indexer.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/indexer/code/elasticsearch" "code.gitea.io/gitea/modules/indexer/code/internal" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/queue" "code.gitea.io/gitea/modules/setting" @@ -38,10 +39,10 @@ func init() { globalIndexer.Store(dummyIndexer) } -func index(ctx context.Context, indexer internal.Indexer, repoID int64) error { +func index(ctx context.Context, indexer internal.Indexer, repoID int64, isWiki bool) error { repo, err := repo_model.GetRepositoryByID(ctx, repoID) if repo_model.IsErrRepoNotExist(err) { - return indexer.Delete(ctx, repoID) + return indexer.Delete(ctx, repoID, optional.None[bool]()) } if err != nil { return err @@ -53,42 +54,74 @@ func index(ctx context.Context, indexer internal.Indexer, repoID int64) error { repoTypes = []string{"sources"} } - // skip forks from being indexed if unit is not present - if !slices.Contains(repoTypes, "forks") && repo.IsFork { - return nil - } + if isWiki { + if !repo.HasWiki() { + // wiki go deleted, so we delete index too + status, err := getRepoStatus(ctx, repo, isWiki) + if err != nil { + return err + } + if status.CommitSha != "" { + if err := indexer.Delete(ctx, repoID, optional.Some(isWiki)); err != nil { + return err + } + } + // ignore empty wikis + return nil + } - // skip mirrors from being indexed if unit is not present - if !slices.Contains(repoTypes, "mirrors") && repo.IsMirror { - return nil - } + // skip wikis from being indexed if unit is not present + if !slices.Contains(repoTypes, "wikis") { + return nil + } + } else { + // ignore empty repos + if repo.IsEmpty { + return nil + } - // skip templates from being indexed if unit is not present - if !slices.Contains(repoTypes, "templates") && repo.IsTemplate { - return nil - } + // skip forks from being indexed if unit is not present + if !slices.Contains(repoTypes, "forks") && repo.IsFork { + return nil + } - // skip regular repos from being indexed if unit is not present - if !slices.Contains(repoTypes, "sources") && !repo.IsFork && !repo.IsMirror && !repo.IsTemplate { - return nil + // skip mirrors from being indexed if unit is not present + if !slices.Contains(repoTypes, "mirrors") && repo.IsMirror { + return nil + } + + // skip templates from being indexed if unit is not present + if !slices.Contains(repoTypes, "templates") && repo.IsTemplate { + return nil + } + + // skip regular repos from being indexed if unit is not present + if !slices.Contains(repoTypes, "sources") && !repo.IsFork && !repo.IsMirror && !repo.IsTemplate { + return nil + } } - sha, err := getDefaultBranchSha(ctx, repo) + sha, err := getDefaultBranchSha(ctx, repo, isWiki) if err != nil { return err } - changes, err := getRepoChanges(ctx, repo, sha) + changes, err := getRepoChanges(ctx, repo, isWiki, sha) if err != nil { return err } else if changes == nil { return nil } - if err := indexer.Index(ctx, repo, sha, changes); err != nil { + if err := indexer.Index(ctx, repo, isWiki, sha, changes); err != nil { return err } - return repo_model.UpdateIndexerStatus(ctx, repo, repo_model.RepoIndexerTypeCode, sha) + indexerType := repo_model.RepoIndexerTypeCode + if isWiki { + indexerType = repo_model.RepoIndexerTypeWiki + } + + return repo_model.UpdateIndexerStatus(ctx, repo, indexerType, sha) } // Init initialize the repo indexer @@ -121,11 +154,11 @@ func Init() { handler := func(items ...*internal.IndexerData) (unhandled []*internal.IndexerData) { indexer := *globalIndexer.Load() for _, indexerData := range items { - log.Trace("IndexerData Process Repo: %d", indexerData.RepoID) - if err := index(ctx, indexer, indexerData.RepoID); err != nil { + log.Trace("IndexerData Process Repo: %d (IsWiki=%v)", indexerData.RepoID, indexerData.IsWiki) + if err := index(ctx, indexer, indexerData.RepoID, indexerData.IsWiki); err != nil { unhandled = append(unhandled, indexerData) if !setting.IsInTesting { - log.Error("Codes indexer handler: index error for repo %v: %v", indexerData.RepoID, err) + log.Error("Codes indexer handler: index error for repo %d (wiki=%v): %v", indexerData.RepoID, indexerData.IsWiki, err) } } } @@ -242,10 +275,12 @@ func Init() { } // UpdateRepoIndexer update a repository's entries in the indexer -func UpdateRepoIndexer(repo *repo_model.Repository) { - indexData := &internal.IndexerData{RepoID: repo.ID} +func UpdateRepoIndexer(repo *repo_model.Repository, isWiki bool) { + indexData := &internal.IndexerData{RepoID: repo.ID, IsWiki: isWiki} if err := indexerQueue.Push(indexData); err != nil { log.Error("Update repo index data %v failed: %v", indexData, err) + } else { + log.Trace("Push repo indexer task repo: %d (isWiki=%v)", repo.ID, isWiki) } } @@ -273,40 +308,47 @@ func populateRepoIndexer(ctx context.Context) { log.Fatal("System error: %v", err) } - var maxRepoID int64 - if maxRepoID, err = db.GetMaxID("repository"); err != nil { - log.Fatal("System error: %v", err) - } - - // start with the maximum existing repo ID and work backwards, so that we - // don't include repos that are created after gitea starts; such repos will - // already be added to the indexer, and we don't need to add them again. - for maxRepoID > 0 { - select { - case <-ctx.Done(): - log.Info("Repository Indexer population shutdown before completion") - return - default: + for _, isWiki := range []bool{false, true} { + indexerType := repo_model.RepoIndexerTypeCode + if isWiki { + indexerType = repo_model.RepoIndexerTypeWiki } - ids, err := repo_model.GetUnindexedRepos(ctx, repo_model.RepoIndexerTypeCode, maxRepoID, 0, 50) - if err != nil { - log.Error("populateRepoIndexer: %v", err) - return - } else if len(ids) == 0 { - break + + var maxRepoID int64 + if maxRepoID, err = db.GetMaxID("repository"); err != nil { + log.Fatal("System error: %v", err) } - for _, id := range ids { + + // start with the maximum existing repo ID and work backwards, so that we + // don't include repos that are created after gitea starts; such repos will + // already be added to the indexer, and we don't need to add them again. + for maxRepoID > 0 { select { case <-ctx.Done(): log.Info("Repository Indexer population shutdown before completion") return default: } - if err := indexerQueue.Push(&internal.IndexerData{RepoID: id}); err != nil { - log.Error("indexerQueue.Push: %v", err) + ids, err := repo_model.GetUnindexedRepos(ctx, indexerType, maxRepoID, 0, 50) + if err != nil { + log.Error("populateRepoIndexer: %v", err) return + } else if len(ids) == 0 { + break + } + for _, id := range ids { + select { + case <-ctx.Done(): + log.Info("Repository Indexer population shutdown before completion") + return + default: + } + if err := indexerQueue.Push(&internal.IndexerData{RepoID: id, IsWiki: isWiki}); err != nil { + log.Error("indexerQueue.Push: %v", err) + return + } + maxRepoID = id - 1 } - maxRepoID = id - 1 } } log.Info("Done (re)populating the repo indexer with existing repositories") diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go index 8975c5ce4083b..3a5e3db3f9dbf 100644 --- a/modules/indexer/code/indexer_test.go +++ b/modules/indexer/code/indexer_test.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/indexer/code/bleve" "code.gitea.io/gitea/modules/indexer/code/elasticsearch" "code.gitea.io/gitea/modules/indexer/code/internal" + "code.gitea.io/gitea/modules/optional" _ "code.gitea.io/gitea/models" _ "code.gitea.io/gitea/models/actions" @@ -29,7 +30,7 @@ func TestMain(m *testing.M) { func testIndexer(name string, t *testing.T, indexer internal.Indexer) { t.Run(name, func(t *testing.T) { var repoID int64 = 1 - err := index(git.DefaultContext, indexer, repoID) + err := index(git.DefaultContext, indexer, repoID, false) assert.NoError(t, err) keywords := []struct { RepoIDs []int64 @@ -93,7 +94,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }) } - assert.NoError(t, indexer.Delete(context.Background(), repoID)) + assert.NoError(t, indexer.Delete(context.Background(), repoID, optional.Some(false))) }) } diff --git a/modules/indexer/code/internal/indexer.go b/modules/indexer/code/internal/indexer.go index c259fcd26eb6f..0aa145433161c 100644 --- a/modules/indexer/code/internal/indexer.go +++ b/modules/indexer/code/internal/indexer.go @@ -10,13 +10,14 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/indexer/internal" + "code.gitea.io/gitea/modules/optional" ) // Indexer defines an interface to index and search code contents type Indexer interface { internal.Indexer - Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *RepoChanges) error - Delete(ctx context.Context, repoID int64) error + Index(ctx context.Context, repo *repo_model.Repository, isWiki bool, sha string, changes *RepoChanges) error + Delete(ctx context.Context, repoID int64, isWiki optional.Option[bool]) error Search(ctx context.Context, opts *SearchOptions) (int64, []*SearchResult, []*SearchResultLanguages, error) } @@ -27,6 +28,8 @@ type SearchOptions struct { IsKeywordFuzzy bool + IsWiki optional.Option[bool] + db.Paginator } @@ -41,11 +44,11 @@ type dummyIndexer struct { internal.Indexer } -func (d *dummyIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *RepoChanges) error { +func (d *dummyIndexer) Index(ctx context.Context, repo *repo_model.Repository, isWiki bool, sha string, changes *RepoChanges) error { return fmt.Errorf("indexer is not ready") } -func (d *dummyIndexer) Delete(ctx context.Context, repoID int64) error { +func (d *dummyIndexer) Delete(ctx context.Context, repoID int64, isWiki optional.Option[bool]) error { return fmt.Errorf("indexer is not ready") } diff --git a/modules/indexer/code/internal/model.go b/modules/indexer/code/internal/model.go index f75263c83cfe0..05de7e930f397 100644 --- a/modules/indexer/code/internal/model.go +++ b/modules/indexer/code/internal/model.go @@ -21,11 +21,13 @@ type RepoChanges struct { // IndexerData represents data stored in the code indexer type IndexerData struct { RepoID int64 + IsWiki bool } // SearchResult result of performing a search in a repo type SearchResult struct { RepoID int64 + IsWiki bool StartIndex int EndIndex int Filename string diff --git a/modules/indexer/code/internal/util.go b/modules/indexer/code/internal/util.go index 689c4f4584b14..414b4a9857f89 100644 --- a/modules/indexer/code/internal/util.go +++ b/modules/indexer/code/internal/util.go @@ -4,29 +4,35 @@ package internal import ( + "fmt" "strings" "code.gitea.io/gitea/modules/indexer/internal" "code.gitea.io/gitea/modules/log" ) -func FilenameIndexerID(repoID int64, filename string) string { - return internal.Base36(repoID) + "_" + filename +func FilenameIndexerID(repoID int64, isWiki bool, filename string) string { + t := "r" + if isWiki { + t = "w" + } + return internal.Base36(repoID) + "_" + t + "_" + filename } -func ParseIndexerID(indexerID string) (int64, string) { - index := strings.IndexByte(indexerID, '_') - if index == -1 { - log.Error("Unexpected ID in repo indexer: %s", indexerID) +func ParseIndexerID(indexerID string) (int64, bool, string, error) { + parts := strings.SplitN(indexerID, "_", 3) + if len(parts) != 3 { + return 0, false, "", fmt.Errorf("unexpected ID in repo indexer: %s", indexerID) } - repoID, _ := internal.ParseBase36(indexerID[:index]) - return repoID, indexerID[index+1:] + repoID, _ := internal.ParseBase36(parts[0]) + isWiki := parts[1] == "w" + return repoID, isWiki, parts[2], nil } func FilenameOfIndexerID(indexerID string) string { - index := strings.IndexByte(indexerID, '_') - if index == -1 { - log.Error("Unexpected ID in repo indexer: %s", indexerID) + _, _, name, err := ParseIndexerID(indexerID) + if err != nil { + log.Error(err.Error()) } - return indexerID[index+1:] + return name } diff --git a/modules/indexer/code/internal/util_test.go b/modules/indexer/code/internal/util_test.go new file mode 100644 index 0000000000000..445397a479e2e --- /dev/null +++ b/modules/indexer/code/internal/util_test.go @@ -0,0 +1,27 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package internal + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFilenameIndexerID(t *testing.T) { + assert.EqualValues(t, "9ix_r_test.txt", FilenameIndexerID(12345, false, "test.txt")) + assert.EqualValues(t, "9ix_w_test.txt", FilenameIndexerID(12345, true, "test.txt")) + assert.EqualValues(t, "n_r_you don't know how to name a file?", FilenameIndexerID(23, false, "you don't know how to name a file?")) +} + +func TestParseIndexerID(t *testing.T) { + repoID, isWiki, filename, err := ParseIndexerID("9ix_r_test.txt") + assert.NoError(t, err) + assert.EqualValues(t, 12345, repoID) + assert.False(t, isWiki) + assert.EqualValues(t, "test.txt", filename) + + _, _, _, err = ParseIndexerID("9ix_r") + assert.Error(t, err) +} diff --git a/modules/indexer/code/search.go b/modules/indexer/code/search.go index 51c7595cf8b25..49e16c5240f06 100644 --- a/modules/indexer/code/search.go +++ b/modules/indexer/code/search.go @@ -126,7 +126,6 @@ func searchResult(result *internal.SearchResult, startIndex, endIndex int) (*Res } // PerformSearch perform a search on a repository -// if isFuzzy is true set the Damerau-Levenshtein distance from 0 to 2 func PerformSearch(ctx context.Context, opts *SearchOptions) (int, []*Result, []*SearchResultLanguages, error) { if opts == nil || len(opts.Keyword) == 0 { return 0, nil, nil, nil diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 15f61502427db..0cb975b25208d 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -76,7 +76,7 @@ func loadIndexerFrom(rootCfg ConfigProvider) { Indexer.IssueIndexerName = sec.Key("ISSUE_INDEXER_NAME").MustString(Indexer.IssueIndexerName) Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false) - Indexer.RepoIndexerRepoTypes = strings.Split(sec.Key("REPO_INDEXER_REPO_TYPES").MustString("sources,forks,mirrors,templates"), ",") + Indexer.RepoIndexerRepoTypes = strings.Split(sec.Key("REPO_INDEXER_REPO_TYPES").MustString("sources,forks,mirrors,templates,wikis"), ",") Indexer.RepoType = sec.Key("REPO_INDEXER_TYPE").MustString("bleve") Indexer.RepoPath = filepath.ToSlash(sec.Key("REPO_INDEXER_PATH").MustString(filepath.ToSlash(filepath.Join(AppDataPath, "indexers/repos.bleve")))) if !filepath.IsAbs(Indexer.RepoPath) { diff --git a/routers/web/explore/code.go b/routers/web/explore/code.go index ecd7c33e016f9..9f7e23f5547ab 100644 --- a/routers/web/explore/code.go +++ b/routers/web/explore/code.go @@ -4,6 +4,7 @@ package explore import ( + "fmt" "net/http" "code.gitea.io/gitea/models/db" @@ -36,11 +37,13 @@ func Code(ctx *context.Context) { keyword := ctx.FormTrim("q") isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true) + wikis := ctx.FormOptionalBool("wikis") ctx.Data["Keyword"] = keyword ctx.Data["Language"] = language ctx.Data["IsFuzzy"] = isFuzzy ctx.Data["PageIsViewCode"] = true + ctx.Data["Wikis"] = wikis if keyword == "" { ctx.HTML(http.StatusOK, tplExploreCode) @@ -81,6 +84,7 @@ func Code(ctx *context.Context) { RepoIDs: repoIDs, Keyword: keyword, IsKeywordFuzzy: isFuzzy, + IsWiki: wikis, Language: language, Paginator: &db.ListOptions{ Page: page, @@ -138,6 +142,9 @@ func Code(ctx *context.Context) { pager := context.NewPagination(total, setting.UI.RepoSearchPagingNum, page, 5) pager.SetDefaultParams(ctx) pager.AddParamString("l", language) + if wikis.Has() { + pager.AddParamString("wikis", fmt.Sprintf("%v", wikis.Value())) + } ctx.Data["Page"] = pager ctx.HTML(http.StatusOK, tplExploreCode) diff --git a/routers/web/repo/search.go b/routers/web/repo/search.go index 0f377a97bb71a..f9311f63e8266 100644 --- a/routers/web/repo/search.go +++ b/routers/web/repo/search.go @@ -4,6 +4,7 @@ package repo import ( + "fmt" "net/http" "code.gitea.io/gitea/models/db" @@ -26,11 +27,13 @@ func Search(ctx *context.Context) { keyword := ctx.FormTrim("q") isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true) + wikis := ctx.FormOptionalBool("wikis") ctx.Data["Keyword"] = keyword ctx.Data["Language"] = language ctx.Data["IsFuzzy"] = isFuzzy ctx.Data["PageIsViewCode"] = true + ctx.Data["Wikis"] = wikis if keyword == "" { ctx.HTML(http.StatusOK, tplSearch) @@ -46,6 +49,7 @@ func Search(ctx *context.Context) { RepoIDs: []int64{ctx.Repo.Repository.ID}, Keyword: keyword, IsKeywordFuzzy: isFuzzy, + IsWiki: wikis, Language: language, Paginator: &db.ListOptions{ Page: page, @@ -69,6 +73,9 @@ func Search(ctx *context.Context) { pager := context.NewPagination(total, setting.UI.RepoSearchPagingNum, page, 5) pager.SetDefaultParams(ctx) pager.AddParamString("l", language) + if wikis.Has() { + pager.AddParamString("wikis", fmt.Sprintf("%v", wikis.Value())) + } ctx.Data["Page"] = pager ctx.HTML(http.StatusOK, tplSearch) diff --git a/routers/web/repo/setting/setting.go b/routers/web/repo/setting/setting.go index e045e3b8dcc09..261ca1f10e1bc 100644 --- a/routers/web/repo/setting/setting.go +++ b/routers/web/repo/setting/setting.go @@ -676,7 +676,13 @@ func SettingsPost(ctx *context.Context) { ctx.Error(http.StatusForbidden) return } - code.UpdateRepoIndexer(ctx.Repo.Repository) + code.UpdateRepoIndexer(ctx.Repo.Repository, false) + case "wiki": + if !setting.Indexer.RepoIndexerEnabled { + ctx.Error(http.StatusForbidden) + return + } + code.UpdateRepoIndexer(ctx.Repo.Repository, true) default: ctx.NotFound("", nil) return diff --git a/routers/web/user/code.go b/routers/web/user/code.go index 785c37b1243c6..581883acc1631 100644 --- a/routers/web/user/code.go +++ b/routers/web/user/code.go @@ -4,6 +4,7 @@ package user import ( + "fmt" "net/http" "code.gitea.io/gitea/models/db" @@ -41,11 +42,13 @@ func CodeSearch(ctx *context.Context) { keyword := ctx.FormTrim("q") isFuzzy := ctx.FormOptionalBool("fuzzy").ValueOrDefault(true) + wikis := ctx.FormOptionalBool("wikis") ctx.Data["Keyword"] = keyword ctx.Data["Language"] = language ctx.Data["IsFuzzy"] = isFuzzy ctx.Data["IsCodePage"] = true + ctx.Data["Wikis"] = wikis if keyword == "" { ctx.HTML(http.StatusOK, tplUserCode) @@ -79,6 +82,7 @@ func CodeSearch(ctx *context.Context) { RepoIDs: repoIDs, Keyword: keyword, IsKeywordFuzzy: isFuzzy, + IsWiki: wikis, Language: language, Paginator: &db.ListOptions{ Page: page, @@ -123,6 +127,9 @@ func CodeSearch(ctx *context.Context) { pager := context.NewPagination(total, setting.UI.RepoSearchPagingNum, page, 5) pager.SetDefaultParams(ctx) pager.AddParamString("l", language) + if wikis.Has() { + pager.AddParamString("wikis", fmt.Sprintf("%v", wikis.Value())) + } ctx.Data["Page"] = pager ctx.HTML(http.StatusOK, tplUserCode) diff --git a/services/indexer/notify.go b/services/indexer/notify.go index f1e21a2d40ed1..4e25a4b6d9960 100644 --- a/services/indexer/notify.go +++ b/services/indexer/notify.go @@ -70,14 +70,14 @@ func (r *indexerNotifier) DeleteComment(ctx context.Context, doer *user_model.Us func (r *indexerNotifier) DeleteRepository(ctx context.Context, doer *user_model.User, repo *repo_model.Repository) { issue_indexer.DeleteRepoIssueIndexer(ctx, repo.ID) if setting.Indexer.RepoIndexerEnabled { - code_indexer.UpdateRepoIndexer(repo) + code_indexer.UpdateRepoIndexer(repo, false) } } func (r *indexerNotifier) MigrateRepository(ctx context.Context, doer, u *user_model.User, repo *repo_model.Repository) { issue_indexer.UpdateRepoIndexer(ctx, repo.ID) if setting.Indexer.RepoIndexerEnabled && !repo.IsEmpty { - code_indexer.UpdateRepoIndexer(repo) + code_indexer.UpdateRepoIndexer(repo, false) } if err := stats_indexer.UpdateRepoIndexer(repo); err != nil { log.Error("stats_indexer.UpdateRepoIndexer(%d) failed: %v", repo.ID, err) @@ -90,7 +90,7 @@ func (r *indexerNotifier) PushCommits(ctx context.Context, pusher *user_model.Us } if setting.Indexer.RepoIndexerEnabled && opts.RefFullName.BranchName() == repo.DefaultBranch { - code_indexer.UpdateRepoIndexer(repo) + code_indexer.UpdateRepoIndexer(repo, false) } if err := stats_indexer.UpdateRepoIndexer(repo); err != nil { log.Error("stats_indexer.UpdateRepoIndexer(%d) failed: %v", repo.ID, err) @@ -103,7 +103,7 @@ func (r *indexerNotifier) SyncPushCommits(ctx context.Context, pusher *user_mode } if setting.Indexer.RepoIndexerEnabled && opts.RefFullName.BranchName() == repo.DefaultBranch { - code_indexer.UpdateRepoIndexer(repo) + code_indexer.UpdateRepoIndexer(repo, false) } if err := stats_indexer.UpdateRepoIndexer(repo); err != nil { log.Error("stats_indexer.UpdateRepoIndexer(%d) failed: %v", repo.ID, err) @@ -112,7 +112,7 @@ func (r *indexerNotifier) SyncPushCommits(ctx context.Context, pusher *user_mode func (r *indexerNotifier) ChangeDefaultBranch(ctx context.Context, repo *repo_model.Repository) { if setting.Indexer.RepoIndexerEnabled && !repo.IsEmpty { - code_indexer.UpdateRepoIndexer(repo) + code_indexer.UpdateRepoIndexer(repo, false) } if err := stats_indexer.UpdateRepoIndexer(repo); err != nil { log.Error("stats_indexer.UpdateRepoIndexer(%d) failed: %v", repo.ID, err) diff --git a/services/wiki/wiki.go b/services/wiki/wiki.go index 1b921a44bdbc5..2aafffea96538 100644 --- a/services/wiki/wiki.go +++ b/services/wiki/wiki.go @@ -18,8 +18,10 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" + code_indexer "code.gitea.io/gitea/modules/indexer/code" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/sync" "code.gitea.io/gitea/modules/util" asymkey_service "code.gitea.io/gitea/services/asymkey" @@ -45,6 +47,7 @@ func InitWiki(ctx context.Context, repo *repo_model.Repository) error { } else if _, _, err = git.NewCommand(ctx, "symbolic-ref", "HEAD").AddDynamicArguments(git.BranchPrefix + repo.DefaultWikiBranch).RunStdString(&git.RunOpts{Dir: repo.WikiPath()}); err != nil { return fmt.Errorf("unable to set default wiki branch to %q: %w", repo.DefaultWikiBranch, err) } + return nil } @@ -86,6 +89,12 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model return err } + defer func() { + if setting.Indexer.RepoIndexerEnabled { + code_indexer.UpdateRepoIndexer(repo, true) + } + }() + if err = validateWebPath(newWikiName); err != nil { return err } @@ -250,6 +259,12 @@ func DeleteWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model return err } + defer func() { + if setting.Indexer.RepoIndexerEnabled { + code_indexer.UpdateRepoIndexer(repo, true) + } + }() + wikiWorkingPool.CheckIn(fmt.Sprint(repo.ID)) defer wikiWorkingPool.CheckOut(fmt.Sprint(repo.ID)) @@ -357,6 +372,11 @@ func DeleteWiki(ctx context.Context, repo *repo_model.Repository) error { } system_model.RemoveAllWithNotice(ctx, "Delete repository wiki", repo.WikiPath()) + + if setting.Indexer.RepoIndexerEnabled { + code_indexer.UpdateRepoIndexer(repo, true) + } + return nil } @@ -364,7 +384,7 @@ func ChangeDefaultWikiBranch(ctx context.Context, repo *repo_model.Repository, n if !git.IsValidRefPattern(newBranch) { return fmt.Errorf("invalid branch name: %s", newBranch) } - return db.WithTx(ctx, func(ctx context.Context) error { + if err := db.WithTx(ctx, func(ctx context.Context) error { repo.DefaultWikiBranch = newBranch if err := repo_model.UpdateRepositoryCols(ctx, repo, "default_wiki_branch"); err != nil { return fmt.Errorf("unable to update database: %w", err) @@ -391,5 +411,12 @@ func ChangeDefaultWikiBranch(ctx context.Context, repo *repo_model.Repository, n return fmt.Errorf("unable to rename default branch: %w", err) } return nil - }) + }); err != nil { + return err + } + + if setting.Indexer.RepoIndexerEnabled { + code_indexer.UpdateRepoIndexer(repo, true) + } + return nil } diff --git a/tests/integration/repo_search_test.go b/tests/integration/repo_search_test.go index cf199e98c2895..d10eaad5df2df 100644 --- a/tests/integration/repo_search_test.go +++ b/tests/integration/repo_search_test.go @@ -58,6 +58,6 @@ func testSearch(t *testing.T, url string, expected []string) { assert.EqualValues(t, expected, filenames) } -func executeIndexer(t *testing.T, repo *repo_model.Repository, op func(*repo_model.Repository)) { - op(repo) +func executeIndexer(t *testing.T, repo *repo_model.Repository, op func(*repo_model.Repository, bool)) { + op(repo, false) }