Fix possible bug when migrating issues/pull requests (#33487)

When migrating issues or pull requests from a big repository, some
issue/pull request maybe deleted when migrating. So that there will be
duplicated issues/pull requests because we are get information with
pagination. This PR introduced a map to record all migrated issue pull
request index when migrating to avoid the failure because of duplicated
records.

Author: lunny

services/migrations/migrate.go

@@ -16,6 +16,7 @@ import (
 	repo_model "code.gitea.io/gitea/models/repo"
 	system_model "code.gitea.io/gitea/models/system"
 	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/container"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/hostmatcher"
 	"code.gitea.io/gitea/modules/log"
@@ -327,6 +328,9 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba
 		messenger("repo.migrate.migrating_issues")
 		issueBatchSize := uploader.MaxBatchInsertSize("issue")
 
+		// because when the migrating is running, some issues maybe removed, so after the next page
+		// some of issue maybe duplicated, so we need to record the inserted issue indexes
+		mapInsertedIssueIndexes := container.Set[int64]{}
 		for i := 1; ; i++ {
 			issues, isEnd, err := downloader.GetIssues(ctx, i, issueBatchSize)
 			if err != nil {
@@ -336,6 +340,14 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba
 				log.Warn("migrating issues is not supported, ignored")
 				break
 			}
+			for i := 0; i < len(issues); i++ {
+				if mapInsertedIssueIndexes.Contains(issues[i].Number) {
+					issues = append(issues[:i], issues[i+1:]...)
+					i--
+					continue
+				}
+				mapInsertedIssueIndexes.Add(issues[i].Number)
+			}
 
 			if err := uploader.CreateIssues(ctx, issues...); err != nil {
 				return err
@@ -381,6 +393,7 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba
 		log.Trace("migrating pull requests and comments")
 		messenger("repo.migrate.migrating_pulls")
 		prBatchSize := uploader.MaxBatchInsertSize("pullrequest")
+		mapInsertedPRIndexes := container.Set[int64]{}
 		for i := 1; ; i++ {
 			prs, isEnd, err := downloader.GetPullRequests(ctx, i, prBatchSize)
 			if err != nil {
@@ -390,6 +403,14 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba
 				log.Warn("migrating pull requests is not supported, ignored")
 				break
 			}
+			for i := 0; i < len(prs); i++ {
+				if mapInsertedPRIndexes.Contains(prs[i].Number) {
+					prs = append(prs[:i], prs[i+1:]...)
+					i--
+					continue
+				}
+				mapInsertedPRIndexes.Add(prs[i].Number)
+			}
 
 			if err := uploader.CreatePullRequests(ctx, prs...); err != nil {
 				return err