fix: ignore non-utf8 words when saving to db

Author: lyingbug

Makefile

@@ -1,4 +1,42 @@
-.PHONY: build run test clean docker-build docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama
+.PHONY: help build run test clean docker-build docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images
+
+# Show help
+help:
+	@echo "WeKnora Makefile 帮助"
+	@echo ""
+	@echo "基础命令:"
+	@echo "  build             构建应用"
+	@echo "  run               运行应用"
+	@echo "  test              运行测试"
+	@echo "  clean             清理构建文件"
+	@echo ""
+	@echo "Docker 命令:"
+	@echo "  docker-build      构建 Docker 镜像"
+	@echo "  docker-run        运行 Docker 容器"
+	@echo "  docker-stop       停止 Docker 容器"
+	@echo "  docker-restart    重启 Docker 容器"
+	@echo ""
+	@echo "服务管理:"
+	@echo "  start-all         启动所有服务"
+	@echo "  stop-all          停止所有服务"
+	@echo "  start-ollama      仅启动 Ollama 服务"
+	@echo ""
+	@echo "镜像构建:"
+	@echo "  build-images      从源码构建所有镜像"
+	@echo "  build-images-app  从源码构建应用镜像"
+	@echo "  build-images-docreader 从源码构建文档读取器镜像"
+	@echo "  build-images-frontend  从源码构建前端镜像"
+	@echo "  clean-images      清理本地镜像"
+	@echo ""
+	@echo "数据库:"
+	@echo "  migrate-up        执行数据库迁移"
+	@echo "  migrate-down      回滚数据库迁移"
+	@echo ""
+	@echo "开发工具:"
+	@echo "  fmt               格式化代码"
+	@echo "  lint              代码检查"
+	@echo "  deps              安装依赖"
+	@echo "  docs              生成 API 文档"
 
 # Go related variables
 BINARY_NAME=WeKnora
@@ -53,6 +91,22 @@ stop-all:
 docker-stop:
 	docker-compose down
 
+# 从源码构建镜像相关命令
+build-images:
+	./scripts/build_images.sh
+
+build-images-app:
+	./scripts/build_images.sh --app
+
+build-images-docreader:
+	./scripts/build_images.sh --docreader
+
+build-images-frontend:
+	./scripts/build_images.sh --frontend
+
+clean-images:
+	./scripts/build_images.sh --clean
+
 # Restart Docker container (stop, rebuild, start)
 docker-restart:
 	docker-compose stop -t 60

frontend/.dockerignore

@@ -0,0 +1,8 @@
+node_modules
+dist
+.git
+.gitignore
+README.md
+.vscode
+*.log
+.DS_Store 

internal/application/repository/chunk.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 
+	"github.com/Tencent/WeKnora/internal/common"
 	"github.com/Tencent/WeKnora/internal/types"
 	"github.com/Tencent/WeKnora/internal/types/interfaces"
 	"gorm.io/gorm"
@@ -21,6 +22,9 @@ func NewChunkRepository(db *gorm.DB) interfaces.ChunkRepository {
 
 // CreateChunks creates multiple chunks in batches
 func (r *chunkRepository) CreateChunks(ctx context.Context, chunks []*types.Chunk) error {
+	for _, chunk := range chunks {
+		chunk.Content = common.CleanInvalidUTF8(chunk.Content)
+	}
 	return r.db.WithContext(ctx).CreateInBatches(chunks, 100).Error
 }
 

internal/application/repository/retriever/postgres/structs.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/Tencent/WeKnora/internal/common"
 	"github.com/Tencent/WeKnora/internal/types"
 	"github.com/pgvector/pgvector-go"
 )
@@ -59,7 +60,7 @@ func toDBVectorEmbedding(indexInfo *types.IndexInfo, additionalParams map[string
 		ChunkID:         indexInfo.ChunkID,
 		KnowledgeID:     indexInfo.KnowledgeID,
 		KnowledgeBaseID: indexInfo.KnowledgeBaseID,
-		Content:         indexInfo.Content,
+		Content:         common.CleanInvalidUTF8(indexInfo.Content),
 	}
 	// Add embedding data if available in additionalParams
 	if additionalParams != nil && slices.Contains(slices.Collect(maps.Keys(additionalParams)), "embedding") {

internal/common/tools.go

@@ -6,6 +6,7 @@ import (
 	"regexp"
 	"slices"
 	"strings"
+	"unicode/utf8"
 )
 
 // ToInterfaceSlice converts a slice of strings to a slice of empty interfaces.
@@ -73,3 +74,27 @@ func ParseLLMJsonResponse(content string, target interface{}) error {
 	// If no code block found, return the original error
 	return err
 }
+
+// CleanInvalidUTF8 移除字符串中的非法 UTF-8 字符和 \x00
+func CleanInvalidUTF8(s string) string {
+	var b strings.Builder
+	b.Grow(len(s))
+
+	for i := 0; i < len(s); {
+		r, size := utf8.DecodeRuneInString(s[i:])
+		if r == utf8.RuneError && size == 1 {
+			// 非法 UTF-8 字节，跳过
+			i++
+			continue
+		}
+		if r == 0 {
+			// NULL 字符 \x00，跳过
+			i += size
+			continue
+		}
+		b.WriteRune(r)
+		i += size
+	}
+
+	return b.String()
+}