diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index af27c9e9..65c09080 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -73,11 +73,16 @@ jobs: VAR_BUILD_DATE=${{ env.VAR_BUILD_DATE }} VAR_GIT_SUMMARY=${{ env.VAR_GIT_SUMMARY }} VAR_GIT_BRANCH=${{ env.VAR_GIT_BRANCH }} - OUTPUT_DIR=/app - BINARY_NAME=share - tags: | - ${{ env.DOCKERHUB_REPO }}:latest - ${{ env.DOCKERHUB_REPO }}:${{ env.APP_VERSION }} + # 只有符合 vX.Y.Z 的稳定版才额外打 latest，预发布版（包含 '-'）只打自身版本 + tags: ${{ fromJson( + concat( + '["', env.DOCKERHUB_REPO, ':', env.APP_VERSION, '"', + contains(env.APP_VERSION, '-') ? + '' : + concat(',"', env.DOCKERHUB_REPO, ':latest"'), + ']' + ) + ) }} cache-from: type=gha cache-to: type=gha,mode=max @@ -91,7 +96,7 @@ jobs: echo "🎉 Docker 镜像构建完成！" echo "" echo "📦 镜像标签:" - echo " - ${{ env.DOCKERHUB_REPO }}:latest" + echo " - ${{ env.DOCKERHUB_REPO }}:${{ contains(env.APP_VERSION, '-') && env.APP_VERSION || 'latest' }}" echo " - ${{ env.DOCKERHUB_REPO }}:${{ env.APP_VERSION }}" echo "" echo "🏗️ 支持的架构:" diff --git a/.gitignore b/.gitignore index bbefb99d..4f47204c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,11 @@ logs output data/share.db +data session.json .idea .vscode +datadir fe/node_modules fe/llm-prompt.txt diff --git a/README.md b/README.md index 703f5fcc..a2073a06 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ CloudPan189 Share 是一款专为天翼云盘设计的智能文件分享管理 **🌐 WebDAV 统一接口** - 提供标准 WebDAV 协议支持，兼容主流客户端 - 统一文件访问入口，简化多链接管理流程 +- 支持文件写入和删除操作（可配置） +- 完整的文件锁定机制，确保并发安全 **💻 全功能网页端** - 现代化文件浏览器界面，支持文件夹导航 @@ -26,6 +28,19 @@ CloudPan189 Share 是一款专为天翼云盘设计的智能文件分享管理 - 流式播放技术，实现视频无缓冲即时观看 - 智能带宽适配，确保播放流畅度 +**📺 STRM文件支持** ⚠️ *测试功能* +- 自动为视频文件生成STRM文件 +- 完美兼容Emby、Jellyfin、Plex等媒体服务器 +- 可自定义支持的视频格式列表 +- 支持一键重建和批量管理 + +**📝 文件管理增强** ⚠️ *测试功能* +- WebDAV写入权限控制，支持文件上传和删除 +- 真实文件存储管理，独立于挂载分享文件 +- 完整的文件操作日志和权限控制 + +> ⚠️ **重要提示**：STRM文件生成和WebDAV写入功能目前处于测试阶段，如遇到问题请及时通过 GitHub Issue 反馈。 + ## 🚀 快速开始 ### Docker 部署（推荐） @@ -34,6 +49,7 @@ docker run -d \ --name cloudpan189-share \ -p 12395:12395 \ -v $(pwd)/data:/app/data \ + -v $(pwd)/datadir:/app/datadir \ --restart unless-stopped \ xxcheng123/cloudpan189-share:latest ``` @@ -139,7 +155,8 @@ npm run build ```yaml port: 12395 # 服务端口 dbFile: "data/share.db" # 数据库文件路径 -logPath: "logs" # 日志文件路径 +logFile: "logs/share.log" # 日志文件路径 +fileDir: "datadir" # 文件存储目录 ``` ### 5. 启动服务 @@ -230,4 +247,4 @@ cloudpan189-share/ --- -⭐ 如果这个项目对您有帮助，请给它一个 Star！ \ No newline at end of file +⭐ 如果这个项目对您有帮助，请给它一个 Star！ diff --git a/configs/config.go b/configs/config.go index 3e97fe13..49fb10f6 100644 --- a/configs/config.go +++ b/configs/config.go @@ -4,4 +4,5 @@ type Config struct { Port int `json:"port,default=12395"` DBFile string `json:"dbFile,default=data/share.db"` LogFile string `json:"logFile,default=logs/share.log"` + FileDir string `json:"fileDir,default=datadir"` } diff --git a/configs/init.go b/configs/init.go index c91852da..aab9544d 100644 --- a/configs/init.go +++ b/configs/init.go @@ -58,6 +58,29 @@ func Init() { } } + // 创建日志目录 + logDir := filepath.Dir(c.LogFile) + if logDir != "" && logDir != "." { + if _, err = os.Stat(logDir); os.IsNotExist(err) { + if err = os.MkdirAll(logDir, 0755); err != nil { + panic(fmt.Sprintf("创建日志目录失败: %v", err)) + } + } else if err != nil { + panic(fmt.Sprintf("检查日志目录失败: %v", err)) + } + } + + // 创建文件存储目录 + if c.FileDir != "" { + if _, err = os.Stat(c.FileDir); os.IsNotExist(err) { + if err = os.MkdirAll(c.FileDir, 0755); err != nil { + panic(fmt.Sprintf("创建文件存储目录失败: %v", err)) + } + } else if err != nil { + panic(fmt.Sprintf("检查文件存储目录失败: %v", err)) + } + } + db, err = gorm.Open(sqlite.Open(c.DBFile), &gorm.Config{ NowFunc: func() time.Time { // 使用中国时区，处理加载失败的情况 @@ -122,6 +145,12 @@ func Init() { shared.MultipleStreamThreadCount = dict.Value.Int() case models.SettingDictKeyMultipleStreamChunkSize: shared.MultipleStreamChunkSize = dict.Value.Int64() + case models.SettingDictKeyStrmFileEnable: + shared.StrmFileEnable = dict.Value.Bool() + case models.SettingDictKeyStrmSupportFileExtList: + shared.StrmSupportFileExtList = dict.Value.StringSlice() + case models.SettingDictKeyFileWritable: + shared.FileWritable = dict.Value.Bool() } } diff --git a/docs/1.quick_start.md b/docs/1.quick_start.md index 40858589..92b27ffa 100644 --- a/docs/1.quick_start.md +++ b/docs/1.quick_start.md @@ -12,6 +12,11 @@ CloudPan189 Share 是一个基于天翼云盘的文件分享管理系统，支 - 支持订阅类型和分享类型链接管理 - 提供 Web 界面进行文件浏览和管理 - 多线程流式加速浏览视频 +- **STRM文件生成**：自动为视频文件生成STRM文件，完美兼容Emby、Jellyfin、Plex等媒体服务器 ⚠️ *测试功能* +- **WebDAV写入支持**：支持通过WebDAV协议上传、删除真实文件 ⚠️ *测试功能* +- **增强的文件管理**：完整的文件锁定机制和权限控制 + +> ⚠️ **重要提示**：STRM文件生成和WebDAV写入功能目前处于测试阶段，使用过程中如遇到问题请及时通过 GitHub Issue 反馈。 ## 环境要求 @@ -30,6 +35,7 @@ CloudPan189 Share 是一个基于天翼云盘的文件分享管理系统，支 # 创建工作目录和数据目录 mkdir -p /opt/cloudpan189-share mkdir -p /opt/cloudpan189-share/data +mkdir -p /opt/cloudpan189-share/datadir # 切换到工作目录 cd /opt/cloudpan189-share @@ -43,6 +49,7 @@ docker run -d \ --name cloudpan189-share \ -p 12395:12395 \ -v $(pwd)/data:/app/data \ + -v $(pwd)/datadir:/app/datadir \ --restart unless-stopped \ xxcheng123/cloudpan189-share:latest ``` @@ -53,6 +60,7 @@ docker run -d \ --name cloudpan189-share \ -p 12395:12395 \ -v $(pwd)/data:/app/data \ + -v $(pwd)/datadir:/app/datadir \ --restart unless-stopped \ docker.1ms.run/xxcheng123/cloudpan189-share:latest ``` @@ -110,13 +118,15 @@ chmod +x share-linux-amd64 ```yaml port: 12395 dbFile: "data/share.db" -logPath: "logs" +logFile: "logs/share.log" +fileDir: "datadir" ``` 配置说明： - `port`: Web 服务监听端口，默认 12395 - `dbFile`: 数据库文件路径，相对于程序运行目录 -- `logPath`: 日志文件存储路径 +- `logFile`: 日志文件路径，相对于程序运行目录 +- `fileDir`: 文件存储目录，相对于程序运行目录 #### 4. 运行程序 @@ -345,6 +355,64 @@ https://cloud.189.cn/t/fieiyq6bErum（访问码：zfr5） - 订阅类型存储：显示订阅号的所有共享内容 - 分享类型存储：显示特定分享链接的文件内容 +### 5. 系统设置 + +点击左侧菜单 "系统设置" 可以配置系统的各项功能： + +#### STRM文件生成设置 + +**什么是STRM文件？** +STRM文件是一种包含媒体文件URL的文本文件，被Emby、Jellyfin、Plex等媒体服务器用来播放远程媒体内容。 + +**配置步骤：** +1. 进入系统设置页面 +2. 找到"STRM文件生成"选项 +3. 开启该功能后，系统会自动为支持的视频文件生成对应的STRM文件 +4. 可以在"STRM支持文件格式"中自定义支持的视频格式 + +**支持的默认格式：** +- 常见视频格式：mp4, mkv, avi, mov, wmv, flv, webm, m4v +- 高清格式：ts, mts, m2ts, m2t, mxf +- 其他格式：3gp, f4v, vob, ogv, divx, xvid, rm, rmvb等 + +**使用场景：** +- 配合Emby/Jellyfin/Plex等媒体服务器使用 +- 通过WebDAV挂载到媒体服务器的媒体库目录 +- 实现远程媒体内容的本地化管理 + +#### WebDAV写入权限设置 + +**功能说明：** +控制是否允许通过WebDAV协议进行文件写入和删除操作。 + +**配置选项：** +- **开启**：允许通过WebDAV上传、修改、删除真实文件 +- **关闭**：WebDAV仅提供只读访问，无法进行写入操作 + +**注意事项：** +- 此功能仅影响真实文件，不包括挂载的分享文件和STRM等虚拟文件 +- 开启写入权限后，建议配合用户认证功能使用 +- 删除父级文件夹时，其中的文件也会被删除 + +#### 其他重要设置 + +**多线程流式下载：** +- 使用多个连接并发下载视频片段 +- 可配置线程数（1-64）和块大小（512KB-8MB） +- 优先级高于本地代理设置 + +**用户认证：** +- 控制WebDAV访问是否需要用户登录 +- 建议在生产环境中开启 + +**本地代理：** +- 服务器代为获取资源再转发给用户 +- 可解决某些网络环境下的访问问题 + +**自动刷新：** +- 定期自动刷新挂载的文件列表 +- 可配置刷新间隔（5-120分钟） + ## 常见问题 ### 1. 扫码授权失败 @@ -449,6 +517,62 @@ kill -9 - 检查数据目录和日志目录是否存在 - 确认程序有读写相关目录的权限 +### 9. STRM文件相关问题 + +**问题现象**：STRM文件无法生成或媒体服务器无法识别 + +**解决方案**： +- 确认在系统设置中已开启"STRM文件生成"功能 +- 检查视频文件格式是否在支持列表中 +- 验证WebDAV挂载路径是否正确 +- 确认媒体服务器有读取STRM文件的权限 +- 尝试使用"重建STRM文件"功能重新生成 + +**问题现象**：媒体服务器播放STRM文件时提示无法访问 + +**解决方案**： +- 检查媒体服务器网络是否能访问CloudPan189 Share服务 +- 确认系统设置中的"基础URL"配置正确 +- 验证天翼云盘令牌是否有效 +- 检查防火墙是否阻止了媒体服务器的访问 + +### 10. WebDAV写入权限问题 + +**问题现象**：无法通过WebDAV上传或删除文件 + +**解决方案**： +- 确认在系统设置中已开启"WebDAV写入权限" +- 检查用户是否有足够的权限（如果开启了用户认证） +- 验证WebDAV客户端是否支持写入操作 +- 确认要操作的是真实文件，而非挂载的分享文件 + +**问题现象**：上传文件后无法在Web界面中看到 + +**解决方案**： +- 刷新Web界面的文件列表 +- 检查文件是否上传到正确的目录 +- 确认文件上传过程中没有出现错误 +- 查看系统日志了解详细错误信息 + +### 11. 多线程流式下载问题 + +**问题现象**：视频播放仍然缓慢或卡顿 + +**解决方案**： +- 确认已开启"多线程流式下载"功能 +- 适当调整线程数，过多的线程可能导致性能下降 +- 调整块大小，较大的块可能提升性能但增加内存使用 +- 检查网络带宽和服务器性能 +- 确认天翼云盘令牌有效且未被限速 + +**问题现象**：开启多线程后系统资源占用过高 + +**解决方案**： +- 降低多线程流线程数设置 +- 减小块大小以降低内存使用 +- 监控服务器CPU和内存使用情况 +- 考虑在服务器性能较低时关闭此功能 + ## 维护管理 ### 1. 数据备份 @@ -480,6 +604,7 @@ docker run -d \ --name cloudpan189-share \ -p 12395:12395 \ -v /opt/cloudpan189-share/data:/app/data \ + -v /opt/cloudpan189-share/datadir:/app/datadir \ --restart unless-stopped \ xxcheng123/cloudpan189-share:latest ``` @@ -505,4 +630,4 @@ docker logs cloudpan189-share > /dev/null 2>&1 - 提交 GitHub Issue - 参与社区讨论 ---- \ No newline at end of file +--- diff --git a/etc/config.yaml b/etc/config.yaml index 1918d18d..7711e84c 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -1,3 +1,4 @@ port: 12395 dbFile: "data/share.db" -logPath: "logs" \ No newline at end of file +logFile: "logs/share.log" +fileDir: "datadir" \ No newline at end of file diff --git a/fe/src/api/file.ts b/fe/src/api/file.ts index 6f647243..4e44a90f 100644 --- a/fe/src/api/file.ts +++ b/fe/src/api/file.ts @@ -26,17 +26,58 @@ export interface FileItem { } } +// 获取文件请求参数接口 +export interface GetFileRequest { + includeAutoGenerateStrmFile?: boolean // 是否包括STRM文件 +} + +// 删除文件响应接口 +export interface DeleteFileResponse { + code: number + message: string +} + // 文件API export const fileApi = { // 获取文件/文件夹信息 - getFile: (path: string = ''): Promise => { + getFile: (path: string = '', options?: GetFileRequest): Promise => { // 处理路径，确保正确的API调用 + let url = '' if (path) { // 对路径进行编码，但保留路径分隔符 const encodedPath = path.split('/').map(segment => encodeURIComponent(segment)).join('/') - return api.get(`/open_file/${encodedPath}`) + url = `/open_file/${encodedPath}` } else { - return api.get('/open_file') + url = '/open_file' } + + // 如果有查询参数，添加到URL中 + if (options) { + const params = new URLSearchParams() + + if (options.includeAutoGenerateStrmFile !== undefined) { + params.append('includeAutoGenerateStrmFile', options.includeAutoGenerateStrmFile.toString()) + } + + const queryString = params.toString() + if (queryString) { + url += `?${queryString}` + } + } + + return api.get(url) + }, + + // 删除文件/文件夹 + deleteFile: (path: string): Promise => { + if (!path) { + throw new Error('删除路径不能为空') + } + + // 对路径进行编码，但保留路径分隔符 + const encodedPath = path.split('/').map(segment => encodeURIComponent(segment)).join('/') + const url = `/open_file/${encodedPath}` + + return api.delete(url) } } \ No newline at end of file diff --git a/fe/src/api/setting.ts b/fe/src/api/setting.ts index db98b272..cb81838d 100644 --- a/fe/src/api/setting.ts +++ b/fe/src/api/setting.ts @@ -18,6 +18,9 @@ export interface Setting { autoRefreshMinutes: number // 自动刷新间隔（分钟） multipleStreamThreadCount: number // 多线程流线程数 multipleStreamChunkSize: number // 多线程流块大小 + strmFileEnable: boolean // STRM文件启用状态 + strmSupportFileExtList: string[] // STRM支持的文件扩展名列表 + fileWritable: boolean // 文件可写状态 } export interface InitSystemRequest { @@ -60,17 +63,32 @@ export interface ModifyAutoRefreshMinutesRequest { autoRefreshMinutes: number } -// 新增：修改多线程流线程数请求 +// 修改多线程流线程数请求 export interface ModifyMultipleStreamThreadCountRequest { multipleStreamThreadCount: number // 1-64之间 } -// 新增：修改多线程流块大小请求 +// 修改多线程流块大小请求 export interface ModifyMultipleStreamChunkSizeRequest { multipleStreamChunkSize: number // 512KB-32MB之间 } -// 新增：修改操作的通用响应 +// 切换STRM文件启用状态请求 +export interface ToggleStrmFileEnableRequest { + strmFileEnable: boolean +} + +// 修改STRM支持文件扩展名列表请求 +export interface ModifyStrmSupportFileExtListRequest { + strmSupportFileExtList: string[] // 可选，不传或空数组表示清空列表 +} + +// 新增：切换文件可写状态请求 +export interface ToggleFileWritableRequest { + fileWritable: boolean +} + +// 修改操作的通用响应 export interface ModifyResponse { rowsAffected: number } @@ -127,16 +145,31 @@ export const settingApi = { return api.post('/setting/modify_auto_refresh_minutes', data) }, - // 新增：修改多线程流线程数 + // 修改多线程流线程数 modifyMultipleStreamThreadCount: (data: ModifyMultipleStreamThreadCountRequest): Promise => { return api.post('/setting/modify_multiple_stream_thread_count', data) }, - // 新增：修改多线程流块大小 + // 修改多线程流块大小 modifyMultipleStreamChunkSize: (data: ModifyMultipleStreamChunkSizeRequest): Promise => { return api.post('/setting/modify_multiple_stream_chunk_size', data) }, + // 切换STRM文件启用状态 + toggleStrmFileEnable: (data: ToggleStrmFileEnableRequest): Promise => { + return api.post('/setting/toggle_strm_file_enable', data) + }, + + // 修改STRM支持文件扩展名列表 + modifyStrmSupportFileExtList: (data: ModifyStrmSupportFileExtListRequest): Promise => { + return api.post('/setting/modify_strm_support_file_ext_list', data) + }, + + // 新增：切换文件可写状态 + toggleFileWritable: (data: ToggleFileWritableRequest): Promise => { + return api.post('/setting/toggle_file_writable', data) + }, + // 初始化系统 initSystem: (data: InitSystemRequest): Promise => { return api.post('/setting/init_system', data) diff --git a/fe/src/api/storage.ts b/fe/src/api/storage.ts index 0475031e..a625f955 100644 --- a/fe/src/api/storage.ts +++ b/fe/src/api/storage.ts @@ -115,6 +115,10 @@ export interface SearchResponse { data: SearchItem[] } +export interface ClearRealFileResponse { + message: string +} + // 存储API export const storageApi = { // 添加存储 @@ -145,5 +149,10 @@ export const storageApi = { // 搜索文件 search: (params: SearchRequest): Promise => { return api.get('/storage/file/search', { params }) + }, + + // 清空本地真实存储 + clearRealFile: (): Promise => { + return api.post('/storage/clear_real_file') } -} \ No newline at end of file +} diff --git a/fe/src/components/Icons.vue b/fe/src/components/Icons.vue index 4a21b56c..9de17e5e 100644 --- a/fe/src/components/Icons.vue +++ b/fe/src/components/Icons.vue @@ -2,156 +2,186 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/go.mod b/go.mod index 210c9e1f..8cb77b1f 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/xxcheng123/multistreamer v1.0.1 github.com/zeromicro/go-zero v1.8.5 go.uber.org/zap v1.27.0 + golang.org/x/net v0.35.0 golang.org/x/sync v0.16.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gorm.io/datatypes v1.2.6 @@ -63,7 +64,6 @@ require ( go.uber.org/multierr v1.10.0 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.33.0 // indirect - golang.org/x/net v0.35.0 // indirect golang.org/x/sys v0.30.0 // indirect golang.org/x/text v0.22.0 // indirect google.golang.org/protobuf v1.36.5 // indirect diff --git a/internal/consts/ctx_key.go b/internal/consts/ctx_key.go new file mode 100644 index 00000000..6492af9b --- /dev/null +++ b/internal/consts/ctx_key.go @@ -0,0 +1,13 @@ +package consts + +const ( + CtxKeyFileId = "x_fid" + CtxKeyParentId = "x_pid" + CtxKeyGroupId = "x_gid" + CtxKeyFullPaths = "x_full_paths" + + // CtxKeyGroupFileSet 用户组可以访问的顶级文件 + CtxKeyGroupFileSet = "x_group_file_set" + + CtxKeyFilename = "x_filename" +) diff --git a/internal/consts/file_addition.go b/internal/consts/file_addition.go new file mode 100644 index 00000000..cea48ad3 --- /dev/null +++ b/internal/consts/file_addition.go @@ -0,0 +1,24 @@ +package consts + +const ( + // FileAdditionKeyFilePath 文件真实物理存储的路径 + FileAdditionKeyFilePath = "file_path" + // FileAdditionKeyCloudToken 文件使用哪个令牌ID下载 + FileAdditionKeyCloudToken = "cloud_token" + // FileAdditionKeyFileId 云盘文件ID + FileAdditionKeyFileId = "file_id" + // FileAdditionKeyShareId 分享ID + FileAdditionKeyShareId = "share_id" + // FileAdditionKeySubscribeUser 订阅用户 + FileAdditionKeySubscribeUser = "subscribe_user" + // FileAdditionKeyShareCode 分享码 + FileAdditionKeyShareCode = "share_code" + // FileAdditionKeyAccessCode 访问码 + FileAdditionKeyAccessCode = "access_code" + // FileAdditionKeyShareMode 分享模式 + FileAdditionKeyShareMode = "share_mode" + // FileAdditionKeyShareType 分享类型 + FileAdditionKeyShareType = "share_type" + // FileAdditionKeyIsFolder 是否为文件夹 + FileAdditionKeyIsFolder = "is_folder" +) diff --git a/internal/fs/clear.go b/internal/fs/clear.go new file mode 100644 index 00000000..db62b253 --- /dev/null +++ b/internal/fs/clear.go @@ -0,0 +1,64 @@ +package fs + +import ( + "context" + "github.com/xxcheng123/cloudpan189-share/configs" + "github.com/xxcheng123/cloudpan189-share/internal/models" + "go.uber.org/zap" + "os" + "path/filepath" + "strconv" +) + +func (f *fs) ClearOsType(ctx context.Context, osTypes ...string) (int64, error) { + if len(osTypes) == 0 { + return 0, WhereParamInvalid + } + + // 检查是否需要删除真实文件 + needDeleteReal := false + for _, osType := range osTypes { + if osType == models.OsTypeRealFile { + needDeleteReal = true + + break + } + } + + // 删除真实文件夹 + if needDeleteReal { + f.clearAllRealFileFolders() + } + + // 删除数据库记录 + result := f.getDB(ctx).Where("os_type IN ?", osTypes).Delete(&models.VirtualFile{}) + if result.Error != nil { + f.logger.Error("删除文件失败", zap.Error(result.Error)) + return 0, result.Error + } + + return result.RowsAffected, nil +} + +func (f *fs) clearAllRealFileFolders() { + baseDir := configs.GetConfig().FileDir + + entries, err := os.ReadDir(baseDir) + if err != nil { + f.logger.Error("读取根目录失败", zap.Error(err)) + return + } + + for _, entry := range entries { + if entry.IsDir() { + if _, err = strconv.ParseInt(entry.Name(), 10, 64); err == nil { + folderPath := filepath.Join(baseDir, entry.Name()) + if err := os.RemoveAll(folderPath); err != nil { + f.logger.Error("删除文件夹失败", zap.String("path", folderPath), zap.Error(err)) + } else { + f.logger.Debug("删除文件夹", zap.String("path", folderPath)) + } + } + } + } +} diff --git a/internal/fs/create.go b/internal/fs/create.go new file mode 100644 index 00000000..232ee72b --- /dev/null +++ b/internal/fs/create.go @@ -0,0 +1,120 @@ +package fs + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/google/uuid" + "github.com/samber/lo" + "github.com/xxcheng123/cloudpan189-share/internal/models" + "github.com/xxcheng123/cloudpan189-share/internal/shared" + "go.uber.org/zap" + "gorm.io/datatypes" + "gorm.io/gorm/clause" +) + +func (f *fs) Create(ctx context.Context, pid int64, file File) (id int64, err error) { + if file == nil { + return 0, FileNil + } + + if pid == 0 { + return 0, RootDirProhibitsCreateFile + } + + file.ParentId = pid + + if err = f.getDB(ctx).Create(file).Error; err != nil { + f.logger.Error("创建文件失败", zap.Error(err)) + + return 0, err + } + + if strmFile, ok := GetStrm(file); ok { + _, _ = f.Create(ctx, pid, strmFile) + } + + return file.ID, nil +} + +func (f *fs) BatchCreate(ctx context.Context, pid int64, files ...File) (count int64, err error) { + if len(files) == 0 { + return 0, FileNil + } + + if pid == 0 { + return 0, RootDirProhibitsCreateFile + } + + // 先设置 ParentId + for _, file := range files { + file.ParentId = pid + } + + if result := f.getDB(ctx).Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "parent_id"}, {Name: "name"}}, + DoNothing: true, + }).CreateInBatches(files, len(files)); result.Error != nil { + f.logger.Error("批量创建文件失败", zap.Error(result.Error)) + + return 0, result.Error + } else { + strmFiles := make([]File, 0) + + for _, file := range files { + if strmFile, ok := GetStrm(file); ok { + strmFiles = append(strmFiles, strmFile) + } + } + + _, _ = f.BatchCreate(ctx, pid, strmFiles...) + + return result.RowsAffected, nil + } +} + +func GetStrm(f File) (File, bool) { + if f.IsFolder == 1 { + return nil, false + } + + if !shared.StrmFileEnable || f.OsType != models.OsTypeFile { + return nil, false + } + + extName := strings.TrimPrefix(filepath.Ext(f.Name), ".") + if len(shared.StrmSupportFileExtList) > 0 && lo.IndexOf(shared.StrmSupportFileExtList, extName) == -1 { + return nil, false + } + + now := time.Now() + + // 计算 size - 添加空值检查 + baseURL := "" + if shared.Setting != nil { + baseURL = shared.Setting.BaseURL + } + size := len(fmt.Sprintf("%s/api/file_download?id=%d&random=%s&sign=12345678123456781234567812345678×tamp=-1", baseURL, f.ID, uuid.NewString())) + + strmFile := &models.VirtualFile{ + ParentId: f.ParentId, + LinkId: f.ID, + Name: strings.TrimSuffix(f.Name, filepath.Ext(f.Name)) + ".strm", + IsTop: f.IsTop, + Size: int64(size), + IsFolder: f.IsFolder, + Hash: "-", + CreateDate: now.Format(time.DateTime), + ModifyDate: now.Format(time.DateTime), + OsType: models.OsTypeStrmFile, + Addition: make(datatypes.JSONMap), + Rev: now.Format("20060102150405"), + CreatedAt: now, + UpdatedAt: now, + } + + return strmFile, true +} diff --git a/internal/fs/delete.go b/internal/fs/delete.go new file mode 100644 index 00000000..50e2eca1 --- /dev/null +++ b/internal/fs/delete.go @@ -0,0 +1,172 @@ +package fs + +import ( + "context" + "fmt" + "path/filepath" + "strconv" + "os" + + "github.com/pkg/errors" + "github.com/xxcheng123/cloudpan189-share/configs" + "github.com/xxcheng123/cloudpan189-share/internal/consts" + "github.com/xxcheng123/cloudpan189-share/internal/models" + "go.uber.org/zap" + "gorm.io/gorm" +) + +func (f *fs) Delete(ctx context.Context, id int64) error { + return f.getDB(ctx).Transaction(func(tx *gorm.DB) error { + return f.deleteRecursively(ctx, tx, id) + }) +} + +func (f *fs) deleteRecursively(ctx context.Context, tx *gorm.DB, id int64) error { + // 先获取当前节点信息 + var current models.VirtualFile + if err := tx.Where("id = ?", id).First(¤t).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + f.logger.Warn("文件不存在，跳过删除", zap.Int64("id", id)) + + return nil + } + + return fmt.Errorf("获取文件信息失败 id=%d: %w", id, err) + } + + // 如果是文件夹，先递归删除所有子节点 + if current.IsFolder == 1 { + children, err := f.getChildren(ctx, tx, id) + if err != nil { + return fmt.Errorf("获取子节点失败: %w", err) + } + + for _, child := range children { + if err := f.deleteRecursively(ctx, tx, child.ID); err != nil { + f.logger.Error("删除子文件失败", + zap.Int64("parent_id", id), + zap.Int64("child_id", child.ID), + zap.Error(err)) + return err + } + } + } + + // 删除真实文件/文件夹 + if err := f.deleteRealFileOrFolder(¤t); err != nil { + f.logger.Error("删除真实文件失败", + zap.Int64("id", id), + zap.String("name", current.Name), + zap.Bool("is_folder", current.IsFolder == 1), + zap.Error(err)) + // 如果物理文件已经不存在，记录警告但继续删除数据库记录 + if !os.IsNotExist(err) { + return fmt.Errorf("删除真实文件失败: %w", err) + } + + f.logger.Warn("物理文件不存在，继续删除数据库记录", zap.Int64("id", id)) + } + + // 删除数据库记录 + if err := tx.Where("id = ?", id).Or("link_id = ?", id).Delete(&models.VirtualFile{}).Error; err != nil { + f.logger.Error("删除数据库记录失败", zap.Int64("id", id), zap.Error(err)) + return fmt.Errorf("删除数据库记录失败 id=%d: %w", id, err) + } + + f.logger.Debug("成功删除文件", + zap.Int64("id", id), + zap.String("name", current.Name), + zap.Bool("is_folder", current.IsFolder == 1)) + + return nil +} + +func (f *fs) deleteRealFileOrFolder(file *models.VirtualFile) error { + if file.IsFolder == 1 { + // 删除文件夹 + return f.deleteRealFolder(file.ID) + } else { + // 删除文件 - 从 addition.file_path 获取路径 + filePath := f.getFilePathFromAddition(file) + if filePath == "" { + f.logger.Warn("文件路径为空，跳过删除物理文件", + zap.Int64("id", file.ID), + zap.String("name", file.Name)) + + return nil + } + + return f.deleteRealFile(filePath) + } +} + +// 从 addition 字段中获取文件路径 +func (f *fs) getFilePathFromAddition(file *models.VirtualFile) string { + if file.Addition == nil { + return "" + } + + if filePath, ok := file.Addition[consts.FileAdditionKeyFilePath].(string); ok { + return filePath + } + + f.logger.Warn("addition中没有file_path字段", + zap.Int64("id", file.ID), + zap.String("name", file.Name), + zap.Any("addition", file.Addition)) + + return "" +} + +func (f *fs) getChildren(ctx context.Context, tx *gorm.DB, parentID int64) ([]models.VirtualFile, error) { + var children []models.VirtualFile + err := tx.Where("parent_id = ?", parentID).Find(&children).Error + if err != nil { + return nil, fmt.Errorf("查询子节点失败 parent_id=%d: %w", parentID, err) + } + return children, nil +} + +func (f *fs) deleteRealFile(filePath string) error { + // filePath 已经是完整路径，或者是相对于 FileDir 的路径 + var fullPath string + if filepath.IsAbs(filePath) { + fullPath = filePath + } else { + fullPath = filepath.Join(configs.GetConfig().FileDir, filePath) + } + + // 检查文件是否存在 + if _, err := os.Stat(fullPath); os.IsNotExist(err) { + f.logger.Warn("文件不存在，跳过删除", zap.String("path", fullPath)) + + return nil + } + + if err := os.Remove(fullPath); err != nil { + return fmt.Errorf("删除文件失败 %s: %w", fullPath, err) + } + + f.logger.Debug("成功删除文件", zap.String("path", fullPath)) + + return nil +} + +func (f *fs) deleteRealFolder(folderId int64) error { + folderPath := filepath.Join(configs.GetConfig().FileDir, strconv.FormatInt(folderId, 10)) + + // 检查文件夹是否存在 + if _, err := os.Stat(folderPath); os.IsNotExist(err) { + f.logger.Warn("文件夹不存在，跳过删除", zap.String("path", folderPath)) + + return nil + } + + if err := os.RemoveAll(folderPath); err != nil { + return fmt.Errorf("删除文件夹失败 %s: %w", folderPath, err) + } + + f.logger.Debug("成功删除文件夹", zap.String("path", folderPath)) + + return nil +} diff --git a/internal/fs/err.go b/internal/fs/err.go new file mode 100644 index 00000000..d8b2c9ae --- /dev/null +++ b/internal/fs/err.go @@ -0,0 +1,9 @@ +package fs + +import "github.com/pkg/errors" + +var ( + FileNil = errors.New("file can not be nil") + RootDirProhibitsCreateFile = errors.New("root dir can not create file") + WhereParamInvalid = errors.New("where param invalid") +) diff --git a/internal/fs/fs.go b/internal/fs/fs.go new file mode 100644 index 00000000..fed8f7c3 --- /dev/null +++ b/internal/fs/fs.go @@ -0,0 +1,62 @@ +package fs + +import ( + "context" + "github.com/xxcheng123/cloudpan189-share/internal/models" + "go.uber.org/zap" + "gorm.io/gorm" +) + +type File = *models.VirtualFile + +type FS interface { + Create(ctx context.Context, pid int64, file File) (id int64, err error) + BatchCreate(ctx context.Context, pid int64, files ...File) (count int64, err error) + Delete(ctx context.Context, id int64) error + Children(ctx context.Context, id int64) ([]File, error) + Update(ctx context.Context, id int64, mp map[string]interface{}) error + // ClearOsType 清空指定osType的所有文件 + ClearOsType(ctx context.Context, osTypes ...string) (int64, error) +} + +type fs struct { + db *gorm.DB + logger *zap.Logger +} + +func NewFS(db *gorm.DB, logger *zap.Logger) FS { + return &fs{ + db: db, + logger: logger, + } +} + +func (f *fs) getDB(ctx context.Context) *gorm.DB { + return f.db.WithContext(ctx).Model(&models.VirtualFile{}) +} + +func (f *fs) Children(ctx context.Context, id int64) ([]File, error) { + var children []*models.VirtualFile + + if err := f.getDB(ctx).Where("parent_id = ?", id).Find(&children).Error; err != nil { + f.logger.Error("获取文件子节点失败", zap.Error(err)) + + return nil, err + } + + return children, nil +} + +func (f *fs) Update(ctx context.Context, id int64, mp map[string]interface{}) error { + if mp == nil { + return nil + } + + if err := f.getDB(ctx).Where("id = ?", id).Updates(mp).Error; err != nil { + f.logger.Error("更新文件失败", zap.Error(err)) + + return err + } + + return nil +} diff --git a/internal/jobs/diff.go b/internal/jobs/diff.go new file mode 100644 index 00000000..a912cd81 --- /dev/null +++ b/internal/jobs/diff.go @@ -0,0 +1,767 @@ +package jobs + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + + "github.com/bytedance/gopkg/util/gopool" + "github.com/samber/lo" + "github.com/xxcheng123/cloudpan189-interface/client" + "github.com/xxcheng123/cloudpan189-share/internal/consts" + "github.com/xxcheng123/cloudpan189-share/internal/fs" + "github.com/xxcheng123/cloudpan189-share/internal/models" + "github.com/xxcheng123/cloudpan189-share/internal/pkgs/utils" + "github.com/xxcheng123/cloudpan189-share/internal/shared" + "go.uber.org/zap" + "gorm.io/gorm" +) + +var ( + allowAutoDelOsTypes = []string{ + models.OsTypeFolder, + models.OsTypeFile, + models.OsTypeSubscribe, + models.OsTypeSubscribeShare, + models.OsTypeShare, + } +) + +type diffWorker struct { + logger *zap.Logger + db *gorm.DB + startId int64 // 扫描开始时的文件ID，用于断点续传 + thread int // 线程数 + client client.Client + fs fs.FS +} + +func newDiffWorker(logger *zap.Logger, db *gorm.DB, startId int64) *diffWorker { + thread := shared.Setting.JobThreadCount + if thread <= 0 { + thread = 1 + } + + return &diffWorker{ + logger: logger, + db: db, + startId: startId, + thread: thread, + client: client.New(), + fs: fs.NewFS(db, logger.With(zap.String("owner", "diffWorker"))), + } +} + +func (w *diffWorker) execute(ctx context.Context, file File, deep bool) (err error) { + if file == nil { + return nil + } + + var ( + newFiles = make([]File, 0) + ) + + switch file.OsType { + case models.OsTypeSubscribe: + newFiles, err = w.getSubscribeUserFiles(ctx, file) + case models.OsTypeSubscribeShare: + newFiles, err = w.getSubscribeShareFiles(ctx, file) + case models.OsTypeShare: + newFiles, err = w.getShareFiles(ctx, file) + default: + return nil + } + + _ = shared.UpdateJobProgress(w.startId, 0, int64(len(newFiles))) + defer func() { + _ = shared.UpdateJobProgress(w.startId, int64(len(newFiles)), 0) + }() + + if err != nil { + return fmt.Errorf("failed to scan files for %s(%d): %w", file.Name, file.ID, err) + } + + oldFiles := make([]File, 0) + if err = w.db.WithContext(ctx).Where("parent_id = ?", file.ID).Find(&oldFiles).Error; err != nil { + return fmt.Errorf("failed to query db files: %w", err) + } + + // 创建映射表，用于快速查找 + newFileMap := make(map[string]File) + for _, item := range newFiles { + key := item.Name + newFileMap[key] = item + } + + oldFileMap := make(map[string]File) + for _, item := range oldFiles { + key := item.Name + oldFileMap[key] = item + } + + // 找出需要新增的文件 + var filesToCreate []File + // 找出需要更新的文件 + var filesToUpdateMap = map[int64]map[string]any{} + // 找出需要删除的文件 + var filesToDelete []File + // 文件没有更新，但是需要深度扫描 + var filesToDeep []File + + // 遍历扫描到的文件，找出新增和更新的文件 + for name, newFile := range newFileMap { + if oldFile, exists := oldFileMap[name]; exists { + // 文件存在，检查是否需要更新（通过Rev比较） + if oldFile.Rev != newFile.Rev { + w.logger.Debug("文件存在差异 - rev changed", + zap.String("parent", file.Name), + zap.String("file_name", name), + zap.String("old_rev", oldFile.Rev), + zap.String("new_rev", newFile.Rev)) + + mp := map[string]any{ + "name": newFile.Name, + "rev": newFile.Rev, + "size": newFile.Size, + "hash": strings.ToLower(newFile.Hash), + "modify_date": newFile.ModifyDate, + } + + filesToUpdateMap[oldFile.ID] = mp + } else if oldFile.IsFolder == 1 && deep { + filesToDeep = append(filesToDeep, oldFile) + } + } else { + w.logger.Debug("发现新文件", + zap.String("parent", file.Name), + zap.String("file_name", name), + zap.String("rev", newFile.Rev)) + // 文件不存在，需要新增 + newFile.ParentId = file.ID + filesToCreate = append(filesToCreate, newFile) + } + } + + // 遍历数据库中的文件，找出需要删除的文件 + for name, dbFile := range oldFileMap { + if _, exists := newFileMap[name]; !exists && + dbFile.IsTop != 1 && + // 允许自动删除的文件类型 + lo.IndexOf(allowAutoDelOsTypes, dbFile.OsType) > -1 { + w.logger.Debug("文件不存在 - 删除", + zap.String("parent", file.Name), + zap.String("file_name", name), + zap.Int64("file_id", dbFile.ID), + zap.String("rev", dbFile.Rev)) + // 扫描结果中不存在该文件，需要删除 + + filesToDelete = append(filesToDelete, dbFile) + } + } + + var errs = make([]error, 0) + + // 新增文件 + if len(filesToCreate) > 0 { + count, err := w.fs.BatchCreate(ctx, file.ID, filesToCreate...) + if err != nil { + w.logger.Error("批量创建子文件失败", + zap.Error(err), + zap.Int64("count", count), + zap.Int64("file_id", file.ID), + zap.String("file_name", file.Name)) + + errs = append(errs, fmt.Errorf("批量创建子文件失败: %w", err)) + } else { + w.logger.Info("批量创建子文件成功", + zap.Int64("count", count), + zap.Int64("file_id", file.ID), + zap.String("file_name", file.Name)) + } + } + + // 更新文件 + for id, item := range filesToUpdateMap { + if err = w.fs.Update(ctx, id, item); err != nil { + w.logger.Error("更新文件失败", + zap.Error(err), + zap.String("file_name", item["name"].(string)), + zap.Int64("file_id", id)) + + errs = append(errs, fmt.Errorf("更新文件失败: %w", err)) + } else { + w.logger.Debug("更新文件成功", + zap.String("file_name", item["name"].(string)), + ) + } + } + + for _, item := range filesToDelete { + if err = w.fs.Delete(ctx, item.ID); err != nil { + w.logger.Error("删除文件失败", + zap.Error(err), + zap.String("file_name", item.Name), + zap.Int64("file_id", item.ID)) + + errs = append(errs, fmt.Errorf("删除文件失败: %w", err)) + } else { + w.logger.Debug("删除文件成功", + zap.String("file_name", item.Name), + ) + } + } + + folders := filesToDeep + for _, item := range filesToCreate { + if item.IsFolder == 1 { + folders = append(folders, item) + } + } + + if len(folders) > 0 { + if err = w.processSubfoldersParallel(ctx, folders, deep); err != nil { + errs = append(errs, err) + } + } + + return errors.Join(errs...) +} + +// processSubfoldersParallel 并行处理子文件夹 +func (w *diffWorker) processSubfoldersParallel(ctx context.Context, folders []File, deep bool) error { + if len(folders) == 0 { + return nil + } + + // 限制并发数量，避免创建过多goroutine + maxWorkers := w.thread + if maxWorkers <= 0 { + maxWorkers = 3 // 默认3个并发 + } + + // 如果文件夹数量少于线程数，使用文件夹数量 + if len(folders) < maxWorkers { + maxWorkers = len(folders) + } + + // 创建工作池 + folderChan := make(chan File, len(folders)) + errorChan := make(chan error, len(folders)) + + var wg sync.WaitGroup + + // 启动worker goroutines + for i := 0; i < maxWorkers; i++ { + wg.Add(1) + + workerID := i + gopool.Go(func() { + defer wg.Done() + w.logger.Debug("subfolder worker started", zap.Int("worker_id", workerID)) + + for folder := range folderChan { + select { + case <-ctx.Done(): + errorChan <- ctx.Err() + + return + default: + w.logger.Debug("processing subfolder", + zap.Int("worker_id", workerID), + zap.String("folder_name", folder.Name), + zap.Int64("folder_id", folder.ID)) + + if err := w.execute(ctx, folder, deep); err != nil { + w.logger.Error("failed to process subfolder", + zap.Int("worker_id", workerID), + zap.String("folder_name", folder.Name), + zap.Error(err)) + errorChan <- fmt.Errorf("failed to handle subfolder %s: %w", folder.Name, err) + } else { + w.logger.Debug("subfolder processed successfully", + zap.Int("worker_id", workerID), + zap.String("folder_name", folder.Name)) + } + } + } + }) + } + + // 发送任务到channel + go func() { + defer close(folderChan) + for _, folder := range folders { + select { + case <-ctx.Done(): + return + case folderChan <- folder: + } + } + }() + + // 等待所有worker完成 + go func() { + wg.Wait() + close(errorChan) + }() + + // 收集错误 + var errs []error + for err := range errorChan { + if err != nil { + errs = append(errs, err) + } + } + + if len(errs) > 0 { + return errors.Join(errs...) + } + + return nil +} + +func (w *diffWorker) getSubscribeUserFiles(ctx context.Context, f File) ([]File, error) { + _userId, ok := f.Addition[consts.FileAdditionKeySubscribeUser] + if !ok { + return nil, errors.New("no subscribe_user") + } + + userId := utils.String(_userId) + + var ( + pageNum int64 = 1 + pageSize int64 = 200 + files = make([]File, 0) + ) + + resp, err := w.client.GetUpResourceShare(ctx, userId, pageNum, pageSize) + if err != nil { + return nil, fmt.Errorf("failed to get first page: %w", err) + } + + if resp.Data != nil { + for _, v := range resp.Data.FileList { + files = append(files, &models.VirtualFile{ + ParentId: f.ID, + Name: v.Name, + IsTop: 0, + Size: v.Size, + IsFolder: int8(v.Folder), + Hash: strings.ToLower(v.Md5), + CreateDate: v.CreateDate, + ModifyDate: v.LastOpTime, + OsType: models.OsTypeSubscribeShare, + Addition: map[string]any{ + consts.FileAdditionKeySubscribeUser: userId, + consts.FileAdditionKeyShareId: v.ShareId, + consts.FileAdditionKeyFileId: v.Id, + }, + Rev: v.Rev, + }) + } + } + + if resp.Data != nil && int64(len(files)) < resp.Data.Count { + var ( + mu sync.Mutex + wg sync.WaitGroup + errs []error + allFiles [][]File + ) + + totalPages := (resp.Data.Count + pageSize - 1) / pageSize + allFiles = make([][]File, totalPages-1) + + for i := int64(2); i <= totalPages; i++ { + wg.Add(1) + go func(pageNum int64, index int) { + defer wg.Done() + + subResp, subErr := w.client.GetUpResourceShare(ctx, userId, pageNum, pageSize) + if subErr != nil { + mu.Lock() + errs = append(errs, fmt.Errorf("failed to get page %d: %w", pageNum, subErr)) + mu.Unlock() + return + } + + if subResp.Data != nil { + var pageFiles []File + for _, v := range subResp.Data.FileList { + pageFiles = append(pageFiles, &models.VirtualFile{ + ParentId: f.ID, + Name: v.Name, + IsTop: 0, + Size: v.Size, + IsFolder: int8(v.Folder), + Hash: strings.ToLower(v.Md5), + CreateDate: v.CreateDate, + ModifyDate: v.LastOpTime, + OsType: models.OsTypeSubscribeShare, + Addition: map[string]any{ + consts.FileAdditionKeySubscribeUser: userId, + consts.FileAdditionKeyShareId: v.ShareId, + consts.FileAdditionKeyFileId: v.Id, + }, + Rev: v.Rev, + }) + } + allFiles[index] = pageFiles + } + }(i, int(i-2)) + } + + wg.Wait() + + if len(errs) > 0 { + return nil, errors.Join(errs...) + } + + for _, pageFiles := range allFiles { + files = append(files, pageFiles...) + } + } + + return files, nil +} + +func (w *diffWorker) getSubscribeShareFiles(ctx context.Context, f File) ([]File, error) { + _userId, ok := f.Addition[consts.FileAdditionKeySubscribeUser] + if !ok { + return nil, errors.New("no subscribe_user") + } + + _shareId, ok := f.Addition[consts.FileAdditionKeyShareId] + if !ok { + return nil, errors.New("no share_id") + } + + _fileId, ok := f.Addition[consts.FileAdditionKeyFileId] + if !ok { + return nil, errors.New("no file_id") + } + + var ( + userId = utils.String(_userId) + shareId, _ = utils.Int64(_shareId) + fileId = utils.String(_fileId) + pageNum = 1 + pageSize = 200 + files = make([]File, 0) + ) + + resp, err := w.client.ListShareDir(ctx, shareId, client.String(fileId), func(req *client.ListShareFileRequest) { + req.PageNum = pageNum + req.PageSize = pageSize + }) + + if err != nil { + return nil, fmt.Errorf("failed to get first page: %w", err) + } + + for _, v := range resp.FileListAO.FolderList { + files = append(files, &models.VirtualFile{ + ParentId: f.ID, + Name: v.Name, + IsTop: 0, + Size: 0, + IsFolder: 1, + Hash: "", + CreateDate: v.CreateDate, + ModifyDate: v.LastOpTime, + OsType: models.OsTypeSubscribeShare, + Addition: map[string]any{ + consts.FileAdditionKeySubscribeUser: userId, + consts.FileAdditionKeyShareId: shareId, + consts.FileAdditionKeyFileId: v.Id, + }, + Rev: v.Rev, + }) + } + + for _, v := range resp.FileListAO.FileList { + files = append(files, &models.VirtualFile{ + ParentId: f.ID, + Name: v.Name, + IsTop: 0, + Size: v.Size, + IsFolder: 0, + Hash: strings.ToLower(v.Md5), + CreateDate: v.CreateDate, + ModifyDate: v.LastOpTime, + OsType: models.OsTypeFile, + Addition: map[string]any{ + consts.FileAdditionKeySubscribeUser: userId, + consts.FileAdditionKeyShareId: shareId, + consts.FileAdditionKeyFileId: v.Id, + }, + Rev: v.Rev, + }) + } + + if int64(len(files)) < resp.FileListAO.Count { + var ( + mu sync.Mutex + wg sync.WaitGroup + errs []error + allFiles [][]File + ) + + totalPages := (resp.FileListAO.Count + int64(pageSize) - 1) / int64(pageSize) + allFiles = make([][]File, totalPages-1) + + for i := int64(2); i <= totalPages; i++ { + wg.Add(1) + go func(pageNum int64, index int) { + defer wg.Done() + + subResp, subErr := w.client.ListShareDir(ctx, shareId, client.String(fileId), func(req *client.ListShareFileRequest) { + req.PageNum = int(pageNum) + req.PageSize = pageSize + }) + if subErr != nil { + mu.Lock() + errs = append(errs, fmt.Errorf("failed to get page %d: %w", pageNum, subErr)) + mu.Unlock() + return + } + + var pageFiles []File + for _, v := range subResp.FileListAO.FolderList { + pageFiles = append(pageFiles, &models.VirtualFile{ + ParentId: f.ID, + Name: v.Name, + IsTop: 0, + Size: 0, + IsFolder: 1, + Hash: "", + CreateDate: v.CreateDate, + ModifyDate: v.LastOpTime, + OsType: models.OsTypeSubscribeShare, + Addition: map[string]any{ + consts.FileAdditionKeySubscribeUser: userId, + consts.FileAdditionKeyShareId: shareId, + consts.FileAdditionKeyFileId: v.Id, + }, + Rev: v.Rev, + }) + } + + for _, v := range subResp.FileListAO.FileList { + pageFiles = append(pageFiles, &models.VirtualFile{ + ParentId: f.ID, + Name: v.Name, + IsTop: 0, + Size: v.Size, + IsFolder: 0, + Hash: strings.ToLower(v.Md5), + CreateDate: v.CreateDate, + ModifyDate: v.LastOpTime, + OsType: models.OsTypeFile, + Addition: map[string]any{ + consts.FileAdditionKeySubscribeUser: userId, + consts.FileAdditionKeyShareId: shareId, + consts.FileAdditionKeyFileId: v.Id, + }, + Rev: v.Rev, + }) + } + allFiles[index] = pageFiles + }(i, int(i-2)) + } + + wg.Wait() + + if len(errs) > 0 { + return nil, errors.Join(errs...) + } + + for _, pageFiles := range allFiles { + files = append(files, pageFiles...) + } + } + + return files, nil +} + +func (w *diffWorker) getShareFiles(ctx context.Context, f File) ([]File, error) { + var vv, ok = f.Addition[consts.FileAdditionKeyShareId] + if !ok { + return nil, errors.New("no share_id") + } + + shareId, _ := utils.Int64(vv) + + vv, ok = f.Addition[consts.FileAdditionKeyFileId] + if !ok { + return nil, errors.New("no file_id") + } + + fileId := utils.String(vv) + + vv, ok = f.Addition[consts.FileAdditionKeyShareMode] + if !ok { + return nil, errors.New("no share_mode") + } + + shareMode, _ := utils.Int(vv) + + vv, ok = f.Addition[consts.FileAdditionKeyAccessCode] + if !ok { + return nil, errors.New("no access_code") + } + + accessCode := utils.String(vv) + + vv, ok = f.Addition[consts.FileAdditionKeyIsFolder] + if !ok { + return nil, errors.New("no is_folder") + } + + var ( + pageNum = 1 + pageSize = 200 + files = make([]File, 0) + addMpFn = func(mp map[string]any) map[string]any { + mp[consts.FileAdditionKeyShareId] = shareId + mp[consts.FileAdditionKeyShareMode] = shareMode + mp[consts.FileAdditionKeyAccessCode] = accessCode + + return mp + } + ) + + resp, err := w.client.ListShareDir(ctx, shareId, client.String(fileId), func(req *client.ListShareFileRequest) { + req.PageNum = pageNum + req.PageSize = pageSize + req.IsFolder, _ = utils.Bool(vv) + req.AccessCode = accessCode + req.ShareMode = shareMode + }) + + if err != nil { + return nil, fmt.Errorf("failed to get first page: %w", err) + } + + for _, v := range resp.FileListAO.FolderList { + files = append(files, &models.VirtualFile{ + ParentId: f.ID, + Name: v.Name, + IsTop: 0, + Size: 0, + IsFolder: 1, + Hash: "", + CreateDate: v.CreateDate, + ModifyDate: v.LastOpTime, + OsType: models.OsTypeShare, + Addition: addMpFn(map[string]any{ + consts.FileAdditionKeyFileId: v.Id, + consts.FileAdditionKeyIsFolder: true, + }), + Rev: v.Rev, + }) + } + + for _, v := range resp.FileListAO.FileList { + files = append(files, &models.VirtualFile{ + ParentId: f.ID, + Name: v.Name, + IsTop: 0, + Size: v.Size, + IsFolder: 0, + Hash: strings.ToLower(v.Md5), + CreateDate: v.CreateDate, + ModifyDate: v.LastOpTime, + OsType: models.OsTypeFile, + Addition: addMpFn(map[string]any{ + consts.FileAdditionKeyFileId: v.Id, + consts.FileAdditionKeyIsFolder: false, + }), + Rev: v.Rev, + }) + } + + if int64(len(files)) < resp.FileListAO.Count { + var ( + mu sync.Mutex + wg sync.WaitGroup + errs []error + allFiles [][]File + ) + + totalPages := (resp.FileListAO.Count + int64(pageSize) - 1) / int64(pageSize) + allFiles = make([][]File, totalPages-1) + + for i := int64(2); i <= totalPages; i++ { + wg.Add(1) + go func(pageNum int64, index int) { + defer wg.Done() + + subResp, subErr := w.client.ListShareDir(ctx, shareId, client.String(fileId), func(req *client.ListShareFileRequest) { + req.PageNum = int(pageNum) + req.PageSize = pageSize + }) + if subErr != nil { + mu.Lock() + errs = append(errs, fmt.Errorf("failed to get page %d: %w", pageNum, subErr)) + mu.Unlock() + return + } + + var pageFiles []File + for _, v := range subResp.FileListAO.FolderList { + pageFiles = append(pageFiles, &models.VirtualFile{ + ParentId: f.ID, + Name: v.Name, + IsTop: 0, + Size: 0, + IsFolder: 1, + Hash: "", + CreateDate: v.CreateDate, + ModifyDate: v.LastOpTime, + OsType: models.OsTypeShare, + Addition: addMpFn(map[string]any{ + consts.FileAdditionKeyFileId: v.Id, + consts.FileAdditionKeyIsFolder: true, + }), + Rev: v.Rev, + }) + } + + for _, v := range subResp.FileListAO.FileList { + pageFiles = append(pageFiles, &models.VirtualFile{ + ParentId: f.ID, + Name: v.Name, + IsTop: 0, + Size: v.Size, + IsFolder: 0, + Hash: strings.ToLower(v.Md5), + CreateDate: v.CreateDate, + ModifyDate: v.LastOpTime, + OsType: models.OsTypeFile, + Addition: addMpFn(map[string]any{ + consts.FileAdditionKeyFileId: v.Id, + consts.FileAdditionKeyIsFolder: false, + }), + Rev: v.Rev, + }) + } + allFiles[index] = pageFiles + }(i, int(i-2)) + } + + wg.Wait() + + if len(errs) > 0 { + return nil, errors.Join(errs...) + } + + for _, pageFiles := range allFiles { + files = append(files, pageFiles...) + } + } + + return files, nil +} diff --git a/internal/jobs/job_auto_login.go b/internal/jobs/job_auto_login.go index 911fd7fa..71a76677 100644 --- a/internal/jobs/job_auto_login.go +++ b/internal/jobs/job_auto_login.go @@ -3,6 +3,9 @@ package jobs import ( "context" "fmt" + "sync" + "time" + "github.com/bytedance/gopkg/util/gopool" "github.com/samber/lo" "github.com/tickstep/cloudpan189-api/cloudpan" @@ -10,8 +13,6 @@ import ( "github.com/xxcheng123/cloudpan189-share/internal/pkgs/utils" "go.uber.org/zap" "gorm.io/gorm" - "sync" - "time" ) type AutoLoginJob struct { @@ -25,7 +26,7 @@ type AutoLoginJob struct { func NewAutoLoginJob(db *gorm.DB, logger *zap.Logger) Job { return &AutoLoginJob{ db: db, - logger: logger.With(zap.String("job", "scan_file")), + logger: logger.With(zap.String("job", "auto_login")), } } @@ -50,7 +51,7 @@ func (s *AutoLoginJob) Start(ctx context.Context) error { // 查询过期时间还剩7天的 token var tokens = make([]*models.CloudToken, 0) - if err := s.db.WithContext(ctx).Where("login_type = ?", models.LoginTypePassword).Where("expires_in < ?", time.Now().Unix()-7*24*3600).Find(&tokens).Error; err != nil { + if err := s.db.WithContext(ctx).Where("login_type = ?", models.LoginTypePassword).Where("expires_in < ?", time.Now().Unix()+7*24*3600).Find(&tokens).Error; err != nil { s.logger.Error("query cloud token error", zap.Error(err)) continue @@ -59,9 +60,9 @@ func (s *AutoLoginJob) Start(ctx context.Context) error { retryTimesMap := make(map[int64]int) tokens = lo.Filter(tokens, func(token *models.CloudToken, index int) bool { - val, err := utils.Int(token.Addition) + val, err := utils.Int(token.Addition[models.CloudTokenAdditionAutoLoginTimes]) if err != nil { - return false + val = 0 // 如果获取失败，默认为0次重试 } retryTimesMap[token.ID] = val diff --git a/internal/jobs/job_scan_file.go b/internal/jobs/job_scan_file.go index 6e15f3c7..0134f8d0 100644 --- a/internal/jobs/job_scan_file.go +++ b/internal/jobs/job_scan_file.go @@ -5,19 +5,21 @@ import ( "errors" "fmt" "runtime/debug" - "strings" "sync" "time" + "github.com/xxcheng123/cloudpan189-share/internal/fs" + "github.com/bytedance/gopkg/util/gopool" "github.com/xxcheng123/cloudpan189-interface/client" "github.com/xxcheng123/cloudpan189-share/internal/models" - "github.com/xxcheng123/cloudpan189-share/internal/pkgs/utils" "github.com/xxcheng123/cloudpan189-share/internal/shared" "go.uber.org/zap" "gorm.io/gorm" ) +type File = *models.VirtualFile + type ScanFileJob struct { db *gorm.DB running bool @@ -45,10 +47,6 @@ func (s *ScanFileJob) Start(ctx context.Context) error { defer s.mu.Unlock() - if s.running { - return ErrJobRunning - } - s.running = true gopool.Go(func() { @@ -86,28 +84,46 @@ func (s *ScanFileJob) doJob(ctx context.Context) bool { case msg := <-shared.ScanJobRead(): _ = shared.RunningStat(msg.Msg.ID) defer shared.FinishStat(msg.Msg.ID) - s.logger.Info("scan file job received message", zap.Any("msg", msg)) - // 先检查文件还在不在 - var count int64 - s.db.WithContext(ctx).Model(new(models.VirtualFile)).Where("id = ?", msg.Msg.ID).Count(&count) - if count == 0 { - s.logger.Error("file not found", zap.Int64("file_id", msg.Msg.ID)) - - return true + s.logger.Info("开始执行扫描文件任务", zap.Any("msg", msg)) + if msg.Msg.ID != 0 { + // 先检查文件还在不在 + var count int64 + s.db.WithContext(ctx).Model(new(models.VirtualFile)).Where("id = ?", msg.Msg.ID).Count(&count) + if count == 0 { + s.logger.Error("file not found", zap.Int64("file_id", msg.Msg.ID)) + + return true + } } switch msg.Type { case shared.ScanJobTypeRefresh: - if err := newScanWorker(s).start(ctx, msg.Msg, false); err != nil { - s.logger.Error("handle file error", zap.Error(err)) + if err := newDiffWorker(s.logger, s.db, msg.Msg.ID).execute(ctx, msg.Msg, false); err != nil { + s.logger.Error("扫描文件任务执行失败", zap.Error(err)) } case shared.ScanJobTypeDeepRefresh: - if err := newScanWorker(s).start(ctx, msg.Msg, true); err != nil { - s.logger.Error("handle file error", zap.Error(err)) + if err := newDiffWorker(s.logger, s.db, msg.Msg.ID).execute(ctx, msg.Msg, true); err != nil { + s.logger.Error("深度扫描文件任务执行失败", zap.Error(err)) } case shared.ScanJobTypeDel: - if err := s.recursiveDelete(ctx, []*models.VirtualFile{msg.Msg}); err != nil { - s.logger.Error("delete file error", zap.Error(err)) + if err := fs.NewFS(s.db, s.logger).Delete(ctx, msg.Msg.ID); err != nil { + s.logger.Error("删除文件任务执行失败", zap.Error(err)) + } + case shared.ScanJobRebuildStrm: + if err := s.buildAllStrm(ctx); err != nil { + s.logger.Error("rebuild stream error", zap.Error(err)) + } + case shared.ScanJobClearStrm: + if count, err := fs.NewFS(s.db, s.logger).ClearOsType(ctx, models.OsTypeStrmFile); err != nil { + s.logger.Error("删除strm文件任务执行失败", zap.Error(err)) + } else { + s.logger.Info("删除strm文件成功", zap.Int64("count", count)) + } + case shared.ScanJobClearRealFile: + if count, err := fs.NewFS(s.db, s.logger).ClearOsType(ctx, models.OsTypeRealFile); err != nil { + s.logger.Error("删除真存储文件任务执行失败", zap.Error(err)) + } else { + s.logger.Info("删除真存储文件成功", zap.Int64("count", count)) } } case <-time.After(refreshMinutes * time.Minute): @@ -124,9 +140,11 @@ func (s *ScanFileJob) doJob(ctx context.Context) bool { } func (s *ScanFileJob) Stop() { + s.mu.Lock() + defer s.mu.Unlock() + if s.running { s.cancel() - s.running = false } } @@ -142,7 +160,7 @@ func (s *ScanFileJob) scanTop(ctx context.Context) error { for _, f := range topFiles { _ = shared.RunningStat(f.ID) - if err := newScanWorker(s).start(ctx, f, false); err != nil { + if err := newDiffWorker(s.logger, s.db, f.ID).execute(ctx, f, false); err != nil { errs = append(errs, err) } shared.FinishStat(f.ID) @@ -155,162 +173,43 @@ func (s *ScanFileJob) scanTop(ctx context.Context) error { return nil } -func newScanWorker(job *ScanFileJob) *scanWorker { - return &scanWorker{ - job: job, - logger: job.logger, - db: job.db, - thread: shared.Setting.JobThreadCount, - } -} - -type scanWorker struct { - job *ScanFileJob - logger *zap.Logger - db *gorm.DB - id int64 - // 多线程 - thread int -} - -func (w *scanWorker) start(ctx context.Context, f *models.VirtualFile, deep bool) (err error) { - w.id = f.ID - - return w.execute(ctx, f, deep) -} - -func (w *scanWorker) execute(ctx context.Context, f *models.VirtualFile, deep bool) (err error) { - var ( - scannedFiles = make([]*models.VirtualFile, 0) - ) - - switch f.OsType { - case models.OsTypeSubscribe: - scannedFiles, err = w.job.getSubscribeUserFiles(ctx, f) - case models.OsTypeSubscribeShare: - scannedFiles, err = w.job.getSubscribeShareFiles(ctx, f) - case models.OsTypeShare: - scannedFiles, err = w.job.getShareFiles(ctx, f) - default: - return errors.New("unsupported os type") - } - - _ = shared.UpdateJobProgress(w.id, 0, int64(len(scannedFiles))) - defer func() { - _ = shared.UpdateJobProgress(w.id, int64(len(scannedFiles)), 0) - }() - - if err != nil { - return fmt.Errorf("failed to scan files for %s(%d): %w", f.Name, f.ID, err) +func (s *ScanFileJob) buildStrm(ctx context.Context, f *models.VirtualFile) error { + if f.IsFolder != 1 { + return nil } - // 数据缓存文件 - var dbFiles = make([]*models.VirtualFile, 0) - if err = w.db.WithContext(ctx).Where("parent_id = ?", f.ID).Find(&dbFiles).Error; err != nil { - return fmt.Errorf("failed to query db files: %w", err) - } + subFiles := make([]*models.VirtualFile, 0) - // 创建映射表，用于快速查找 - scannedFileMap := make(map[string]*models.VirtualFile) - for _, file := range scannedFiles { - key := file.Name - scannedFileMap[key] = file + if err := s.db.WithContext(ctx).Where("parent_id = ?", f.ID).Find(&subFiles).Error; err != nil { + return err } - dbFileMap := make(map[string]*models.VirtualFile) - for _, file := range dbFiles { - key := file.Name - dbFileMap[key] = file - } + strmFilesToCreate := make([]*models.VirtualFile, 0) - // 找出需要新增的文件 - var filesToCreate []*models.VirtualFile - // 找出需要更新的文件 - var filesToUpdate []*models.VirtualFile - // 找出需要删除的文件 - var filesToDelete []*models.VirtualFile - - // 遍历扫描到的文件，找出新增和更新的文件 - for name, scannedFile := range scannedFileMap { - if dbFile, exists := dbFileMap[name]; exists { - // 文件存在，检查是否需要更新（通过Rev比较） - if dbFile.Rev != scannedFile.Rev { - w.logger.Debug("file needs update - rev changed", - zap.String("parent", f.Name), - zap.String("file_name", name), - zap.String("old_rev", dbFile.Rev), - zap.String("new_rev", scannedFile.Rev)) - // Rev不同，需要更新 - dbFile.Name = scannedFile.Name - dbFile.Rev = scannedFile.Rev - dbFile.Size = scannedFile.Size - dbFile.Hash = strings.ToLower(scannedFile.Hash) - dbFile.CreateDate = scannedFile.CreateDate - dbFile.ModifyDate = scannedFile.ModifyDate - dbFile.UpdatedAt = time.Now() - filesToUpdate = append(filesToUpdate, dbFile) - } else if deep { - filesToUpdate = append(filesToUpdate, dbFile) - } - } else { - w.logger.Debug("new file found - not in database", - zap.String("parent", f.Name), - zap.String("file_name", name), - zap.String("rev", scannedFile.Rev)) - // 文件不存在，需要新增 - scannedFile.ParentId = f.ID - filesToCreate = append(filesToCreate, scannedFile) - } - } + var errs []error - // 遍历数据库中的文件，找出需要删除的文件 - for name, dbFile := range dbFileMap { - if _, exists := scannedFileMap[name]; !exists && dbFile.IsTop != 1 { - w.logger.Debug("file to be deleted - not in remote", - zap.String("parent", f.Name), - zap.String("file_name", name), - zap.Int64("file_id", dbFile.ID), - zap.String("rev", dbFile.Rev)) - // 扫描结果中不存在该文件，需要删除 - filesToDelete = append(filesToDelete, dbFile) + for _, subFile := range subFiles { + if strmFile, ok := fs.GetStrm(subFile); ok { + strmFilesToCreate = append(strmFilesToCreate, strmFile) } - } - var errs = make([]error, 0) + if subFile.IsFolder == 1 { + if err := s.buildStrm(ctx, subFile); err != nil { + s.logger.Error("failed to build strm", zap.Error(err), zap.String("file_name", subFile.Name)) - // 批量处理新增文件 - if len(filesToCreate) > 0 { - if err = w.db.WithContext(ctx).CreateInBatches(filesToCreate, 100).Error; err != nil { - errs = append(errs, fmt.Errorf("failed to create files: %w", err)) - } - } - - // 批量处理更新文件 - for _, file := range filesToUpdate { - if err = w.db.WithContext(ctx).Save(file).Error; err != nil { - errs = append(errs, fmt.Errorf("failed to update file %s: %w", file.Name, err)) - } - } - - // 递归删除文件（包括子文件和子文件夹） - if len(filesToDelete) > 0 { - if err = w.job.recursiveDelete(ctx, filesToDelete); err != nil { - errs = append(errs, fmt.Errorf("failed to delete files: %w", err)) + errs = append(errs, fmt.Errorf("failed to build strm for %s: %w", subFile.Name, err)) + } } } - // 递归处理子文件夹 - allFiles := append(filesToCreate, filesToUpdate...) - folders := make([]*models.VirtualFile, 0) - for _, file := range allFiles { - if file.IsFolder == 1 { // 如果是文件夹 - folders = append(folders, file) - } - } + if len(strmFilesToCreate) > 0 { + count, err := fs.NewFS(s.db, s.logger).BatchCreate(ctx, f.ID, strmFilesToCreate...) + if err != nil { + s.logger.Error("failed to create strm files", zap.Error(err)) - if len(folders) > 0 { - if err = w.processSubfoldersParallel(ctx, folders, deep); err != nil { - errs = append(errs, err) + errs = append(errs, fmt.Errorf("failed to create strm files: %w", err)) + } else { + s.logger.Info("创建strm文件成功", zap.Int64("count", count)) } } @@ -321,90 +220,27 @@ func (w *scanWorker) execute(ctx context.Context, f *models.VirtualFile, deep bo return nil } -// processSubfoldersParallel 并行处理子文件夹 -func (w *scanWorker) processSubfoldersParallel(ctx context.Context, folders []*models.VirtualFile, deep bool) error { - if len(folders) == 0 { - return nil - } +func (s *ScanFileJob) buildAllStrm(ctx context.Context) error { + s.logger.Info("start build all strm") - // 限制并发数量，避免创建过多goroutine - maxWorkers := w.thread - if maxWorkers <= 0 { - maxWorkers = 3 // 默认3个并发 - } - - // 如果文件夹数量少于线程数，使用文件夹数量 - if len(folders) < maxWorkers { - maxWorkers = len(folders) + if err := s.clearAllStrm(ctx); err != nil { + return err } - // 创建工作池 - folderChan := make(chan *models.VirtualFile, len(folders)) - errorChan := make(chan error, len(folders)) - - var wg sync.WaitGroup - - // 启动worker goroutines - for i := 0; i < maxWorkers; i++ { - wg.Add(1) - - workerID := i - gopool.Go(func() { - defer wg.Done() - w.logger.Debug("subfolder worker started", zap.Int("worker_id", workerID)) - - for folder := range folderChan { - select { - case <-ctx.Done(): - errorChan <- ctx.Err() - - return - default: - w.logger.Debug("processing subfolder", - zap.Int("worker_id", workerID), - zap.String("folder_name", folder.Name), - zap.Int64("folder_id", folder.ID)) - - if err := w.execute(ctx, folder, deep); err != nil { - w.logger.Error("failed to process subfolder", - zap.Int("worker_id", workerID), - zap.String("folder_name", folder.Name), - zap.Error(err)) - errorChan <- fmt.Errorf("failed to handle subfolder %s: %w", folder.Name, err) - } else { - w.logger.Debug("subfolder processed successfully", - zap.Int("worker_id", workerID), - zap.String("folder_name", folder.Name)) - } - } - } - }) + // 读取所有顶层文件 + var topFiles = make([]*models.VirtualFile, 0) + if err := s.db.WithContext(ctx).Where("is_top = 1").Find(&topFiles).Error; err != nil { + return err } - // 发送任务到channel - go func() { - defer close(folderChan) - for _, folder := range folders { - select { - case <-ctx.Done(): - return - case folderChan <- folder: - } - } - }() - - // 等待所有worker完成 - go func() { - wg.Wait() - close(errorChan) - }() + var errs = make([]error, 0) - // 收集错误 - var errs []error - for err := range errorChan { - if err != nil { + for _, f := range topFiles { + _ = shared.RunningStat(f.ID) + if err := s.buildStrm(ctx, f); err != nil { errs = append(errs, err) } + shared.FinishStat(f.ID) } if len(errs) > 0 { @@ -414,492 +250,12 @@ func (w *scanWorker) processSubfoldersParallel(ctx context.Context, folders []*m return nil } -// recursiveDelete 递归删除文件及其所有子文件和子文件夹 -func (s *ScanFileJob) recursiveDelete(ctx context.Context, files []*models.VirtualFile) error { - if len(files) == 0 { - return nil - } - - var fileIDs []int64 - var folderIDs []int64 - var errs = make([]error, 0) - - // 分离文件和文件夹ID - for _, file := range files { - fileIDs = append(fileIDs, file.ID) - if file.IsFolder == 1 { - folderIDs = append(folderIDs, file.ID) - } - } - - // 如果有文件夹，需要先递归删除其子文件 - if len(folderIDs) > 0 { - var childFiles []*models.VirtualFile - if err := s.db.WithContext(ctx).Where("parent_id IN ?", folderIDs).Find(&childFiles).Error; err != nil { - errs = append(errs, fmt.Errorf("failed to find child files: %w", err)) - } else { - // 递归删除子文件 - if len(childFiles) > 0 { - if err := s.recursiveDelete(ctx, childFiles); err != nil { - errs = append(errs, err) - } - } - } - } +func (s *ScanFileJob) clearAllStrm(ctx context.Context) error { + if err := s.db.WithContext(ctx).Where("os_type = ?", models.OsTypeStrmFile).Delete(&models.VirtualFile{}).Error; err != nil { + s.logger.Error("failed to delete strm files", zap.Error(err)) - // 删除当前层级的所有文件 - if err := s.db.WithContext(ctx).Where("id IN ?", fileIDs).Delete(&models.VirtualFile{}).Error; err != nil { - errs = append(errs, fmt.Errorf("failed to delete files: %w", err)) - } - - if len(errs) > 0 { - return errors.Join(errs...) + return err } return nil } - -func (s *ScanFileJob) getSubscribeUserFiles(ctx context.Context, f *models.VirtualFile) ([]*models.VirtualFile, error) { - _userId, ok := f.Addition["subscribe_user"] - if !ok { - return nil, errors.New("no subscribe_user") - } - - userId := utils.String(_userId) - - var ( - pageNum int64 = 1 - pageSize int64 = 200 - files = make([]*models.VirtualFile, 0) - ) - - resp, err := s.client.GetUpResourceShare(ctx, userId, pageNum, pageSize) - if err != nil { - return nil, fmt.Errorf("failed to get first page: %w", err) - } - - if resp.Data != nil { - for _, v := range resp.Data.FileList { - files = append(files, &models.VirtualFile{ - ParentId: f.ID, - Name: v.Name, - IsTop: 0, - Size: v.Size, - IsFolder: int8(v.Folder), - Hash: strings.ToLower(v.Md5), - CreateDate: v.CreateDate, - ModifyDate: v.LastOpTime, - OsType: models.OsTypeSubscribeShare, - Addition: map[string]any{ - "subscribe_user": userId, - "share_id": v.ShareId, - "file_id": v.Id, - }, - Rev: v.Rev, - }) - } - } - - if resp.Data != nil && int64(len(files)) < resp.Data.Count { - var ( - mu sync.Mutex - wg sync.WaitGroup - errs []error - allFiles [][]*models.VirtualFile - ) - - totalPages := (resp.Data.Count + pageSize - 1) / pageSize - allFiles = make([][]*models.VirtualFile, totalPages-1) - - for i := int64(2); i <= totalPages; i++ { - wg.Add(1) - go func(pageNum int64, index int) { - defer wg.Done() - - subResp, subErr := s.client.GetUpResourceShare(ctx, userId, pageNum, pageSize) - if subErr != nil { - mu.Lock() - errs = append(errs, fmt.Errorf("failed to get page %d: %w", pageNum, subErr)) - mu.Unlock() - return - } - - if subResp.Data != nil { - var pageFiles []*models.VirtualFile - for _, v := range subResp.Data.FileList { - pageFiles = append(pageFiles, &models.VirtualFile{ - ParentId: f.ID, - Name: v.Name, - IsTop: 0, - Size: v.Size, - IsFolder: int8(v.Folder), - Hash: strings.ToLower(v.Md5), - CreateDate: v.CreateDate, - ModifyDate: v.LastOpTime, - OsType: models.OsTypeSubscribeShare, - Addition: map[string]any{ - "subscribe_user": userId, - "share_id": v.ShareId, - "file_id": v.Id, - }, - Rev: v.Rev, - }) - } - allFiles[index] = pageFiles - } - }(i, int(i-2)) - } - - wg.Wait() - - if len(errs) > 0 { - return nil, errors.Join(errs...) - } - - for _, pageFiles := range allFiles { - files = append(files, pageFiles...) - } - } - - return files, nil -} - -func (s *ScanFileJob) getSubscribeShareFiles(ctx context.Context, f *models.VirtualFile) ([]*models.VirtualFile, error) { - _userId, ok := f.Addition["subscribe_user"] - if !ok { - return nil, errors.New("no subscribe_user") - } - - _shareId, ok := f.Addition["share_id"] - if !ok { - return nil, errors.New("no share_id") - } - - _fileId, ok := f.Addition["file_id"] - if !ok { - return nil, errors.New("no file_id") - } - - var ( - userId = utils.String(_userId) - shareId, _ = utils.Int64(_shareId) - fileId = utils.String(_fileId) - pageNum = 1 - pageSize = 200 - files = make([]*models.VirtualFile, 0) - ) - - resp, err := s.client.ListShareDir(ctx, shareId, client.String(fileId), func(req *client.ListShareFileRequest) { - req.PageNum = pageNum - req.PageSize = pageSize - }) - - if err != nil { - return nil, fmt.Errorf("failed to get first page: %w", err) - } - - for _, v := range resp.FileListAO.FolderList { - files = append(files, &models.VirtualFile{ - ParentId: f.ID, - Name: v.Name, - IsTop: 0, - Size: 0, - IsFolder: 1, - Hash: "", - CreateDate: v.CreateDate, - ModifyDate: v.LastOpTime, - OsType: models.OsTypeSubscribeShare, - Addition: map[string]any{ - "subscribe_user": userId, - "share_id": shareId, - "file_id": v.Id, - }, - Rev: v.Rev, - }) - } - - for _, v := range resp.FileListAO.FileList { - files = append(files, &models.VirtualFile{ - ParentId: f.ID, - Name: v.Name, - IsTop: 0, - Size: v.Size, - IsFolder: 0, - Hash: strings.ToLower(v.Md5), - CreateDate: v.CreateDate, - ModifyDate: v.LastOpTime, - OsType: models.OsTypeFile, - Addition: map[string]any{ - "subscribe_user": userId, - "share_id": shareId, - "file_id": v.Id, - }, - Rev: v.Rev, - }) - } - - if int64(len(files)) < resp.FileListAO.Count { - var ( - mu sync.Mutex - wg sync.WaitGroup - errs []error - allFiles [][]*models.VirtualFile - ) - - totalPages := (resp.FileListAO.Count + int64(pageSize) - 1) / int64(pageSize) - allFiles = make([][]*models.VirtualFile, totalPages-1) - - for i := int64(2); i <= totalPages; i++ { - wg.Add(1) - go func(pageNum int64, index int) { - defer wg.Done() - - subResp, subErr := s.client.ListShareDir(ctx, shareId, client.String(fileId), func(req *client.ListShareFileRequest) { - req.PageNum = int(pageNum) - req.PageSize = pageSize - }) - if subErr != nil { - mu.Lock() - errs = append(errs, fmt.Errorf("failed to get page %d: %w", pageNum, subErr)) - mu.Unlock() - return - } - - var pageFiles []*models.VirtualFile - for _, v := range subResp.FileListAO.FolderList { - pageFiles = append(pageFiles, &models.VirtualFile{ - ParentId: f.ID, - Name: v.Name, - IsTop: 0, - Size: 0, - IsFolder: 1, - Hash: "", - CreateDate: v.CreateDate, - ModifyDate: v.LastOpTime, - OsType: models.OsTypeSubscribeShare, - Addition: map[string]any{ - "subscribe_user": userId, - "share_id": shareId, - "file_id": v.Id, - }, - Rev: v.Rev, - }) - } - - for _, v := range subResp.FileListAO.FileList { - pageFiles = append(pageFiles, &models.VirtualFile{ - ParentId: f.ID, - Name: v.Name, - IsTop: 0, - Size: v.Size, - IsFolder: 0, - Hash: strings.ToLower(v.Md5), - CreateDate: v.CreateDate, - ModifyDate: v.LastOpTime, - OsType: models.OsTypeFile, - Addition: map[string]any{ - "subscribe_user": userId, - "share_id": shareId, - "file_id": v.Id, - }, - Rev: v.Rev, - }) - } - allFiles[index] = pageFiles - }(i, int(i-2)) - } - - wg.Wait() - - if len(errs) > 0 { - return nil, errors.Join(errs...) - } - - for _, pageFiles := range allFiles { - files = append(files, pageFiles...) - } - } - - return files, nil -} - -func (s *ScanFileJob) getShareFiles(ctx context.Context, f *models.VirtualFile) ([]*models.VirtualFile, error) { - // need fileId、shareId、isFolder、shareMode、accessCode、 - var vv, ok = f.Addition["share_id"] - if !ok { - return nil, errors.New("no share_id") - } - - shareId, _ := utils.Int64(vv) - - vv, ok = f.Addition["file_id"] - if !ok { - return nil, errors.New("no file_id") - } - - fileId := utils.String(vv) - - vv, ok = f.Addition["share_mode"] - if !ok { - return nil, errors.New("no share_mode") - } - - shareMode, _ := utils.Int(vv) - - vv, ok = f.Addition["access_code"] - if !ok { - return nil, errors.New("no access_code") - } - - accessCode := utils.String(vv) - - vv, ok = f.Addition["is_folder"] - if !ok { - return nil, errors.New("no is_folder") - } - - var ( - pageNum = 1 - pageSize = 200 - files = make([]*models.VirtualFile, 0) - addMpFn = func(mp map[string]any) map[string]any { - mp["share_id"] = shareId - mp["share_mode"] = shareMode - mp["access_code"] = accessCode - - return mp - } - ) - - resp, err := s.client.ListShareDir(ctx, shareId, client.String(fileId), func(req *client.ListShareFileRequest) { - req.PageNum = pageNum - req.PageSize = pageSize - req.IsFolder, _ = utils.Bool(vv) - req.AccessCode = accessCode - req.ShareMode = shareMode - }) - - if err != nil { - return nil, fmt.Errorf("failed to get first page: %w", err) - } - - for _, v := range resp.FileListAO.FolderList { - files = append(files, &models.VirtualFile{ - ParentId: f.ID, - Name: v.Name, - IsTop: 0, - Size: 0, - IsFolder: 1, - Hash: "", - CreateDate: v.CreateDate, - ModifyDate: v.LastOpTime, - OsType: models.OsTypeShare, - Addition: addMpFn(map[string]any{ - "file_id": v.Id, - "is_folder": true, - }), - Rev: v.Rev, - }) - } - - for _, v := range resp.FileListAO.FileList { - files = append(files, &models.VirtualFile{ - ParentId: f.ID, - Name: v.Name, - IsTop: 0, - Size: v.Size, - IsFolder: 0, - Hash: strings.ToLower(v.Md5), - CreateDate: v.CreateDate, - ModifyDate: v.LastOpTime, - OsType: models.OsTypeFile, - Addition: addMpFn(map[string]any{ - "file_id": v.Id, - "is_folder": false, - }), - Rev: v.Rev, - }) - } - - if int64(len(files)) < resp.FileListAO.Count { - var ( - mu sync.Mutex - wg sync.WaitGroup - errs []error - allFiles [][]*models.VirtualFile - ) - - totalPages := (resp.FileListAO.Count + int64(pageSize) - 1) / int64(pageSize) - allFiles = make([][]*models.VirtualFile, totalPages-1) - - for i := int64(2); i <= totalPages; i++ { - wg.Add(1) - go func(pageNum int64, index int) { - defer wg.Done() - - subResp, subErr := s.client.ListShareDir(ctx, shareId, client.String(fileId), func(req *client.ListShareFileRequest) { - req.PageNum = int(pageNum) - req.PageSize = pageSize - }) - if subErr != nil { - mu.Lock() - errs = append(errs, fmt.Errorf("failed to get page %d: %w", pageNum, subErr)) - mu.Unlock() - return - } - - var pageFiles []*models.VirtualFile - for _, v := range subResp.FileListAO.FolderList { - files = append(files, &models.VirtualFile{ - ParentId: f.ID, - Name: v.Name, - IsTop: 0, - Size: 0, - IsFolder: 1, - Hash: "", - CreateDate: v.CreateDate, - ModifyDate: v.LastOpTime, - OsType: models.OsTypeShare, - Addition: addMpFn(map[string]any{ - "file_id": v.Id, - "is_folder": true, - }), - Rev: v.Rev, - }) - } - - for _, v := range subResp.FileListAO.FileList { - files = append(files, &models.VirtualFile{ - ParentId: f.ID, - Name: v.Name, - IsTop: 0, - Size: v.Size, - IsFolder: 0, - Hash: strings.ToLower(v.Md5), - CreateDate: v.CreateDate, - ModifyDate: v.LastOpTime, - OsType: models.OsTypeFile, - Addition: addMpFn(map[string]any{ - "file_id": v.Id, - "is_folder": false, - }), - Rev: v.Rev, - }) - } - allFiles[index] = pageFiles - }(i, int(i-2)) - } - - wg.Wait() - - if len(errs) > 0 { - return nil, errors.Join(errs...) - } - - for _, pageFiles := range allFiles { - files = append(files, pageFiles...) - } - } - - return files, nil -} diff --git a/internal/models/mount.go b/internal/models/mount.go index 32e823ed..0b680538 100644 --- a/internal/models/mount.go +++ b/internal/models/mount.go @@ -6,17 +6,22 @@ import ( "gorm.io/datatypes" ) +type OsType = string + const ( OsTypeFolder = "folder" OsTypeFile = "file" OsTypeSubscribe = "subscribe" OsTypeSubscribeShare = "subscribe_share" OsTypeShare = "share" + OsTypeRealFile = "real_file" + OsTypeStrmFile = "strm_file" ) type VirtualFile struct { ID int64 `gorm:"primaryKey" json:"id"` ParentId int64 `gorm:"column:parent_id;type:bigint(20);not null;default:0;uniqueIndex:parent_name_unique" json:"parentId"` + LinkId int64 `gorm:"column:link_id;type:bigint(20);default:0;index:link_id_index" json:"linkId"` // 关联id，用于strm文件，当文件被删除后，实现关联的 strm 文件快速删除 Name string `gorm:"column:name;type:varchar(1024);not null;uniqueIndex:parent_name_unique" json:"name"` IsTop int8 `gorm:"column:is_top;type:tinyint(1);default:0" json:"isTop"` // 是否最顶层文件夹 Size int64 `gorm:"column:size;type:bigint(20);default:0" json:"size"` @@ -24,7 +29,7 @@ type VirtualFile struct { Hash string `gorm:"column:hash;type:varchar(64);default:''" json:"hash"` CreateDate string `gorm:"column:create_date;type:varchar(20);default:CURRENT_TIMESTAMP" json:"createDate"` ModifyDate string `gorm:"column:modify_date;type:varchar(20);default:CURRENT_TIMESTAMP" json:"modifyDate"` - OsType string `gorm:"column:os_type;type:varchar(20);default:'folder'" json:"osType"` // 读取文件的方式 + OsType OsType `gorm:"column:os_type;type:varchar(20);default:'folder'" json:"osType"` // 读取文件的方式 Addition datatypes.JSONMap `gorm:"column:addition;type:json" json:"addition"` Rev string `gorm:"column:rev;type:varchar(64);default:''" json:"rev"` // 版本 用于下次扫描时知道当前文件是删除还是修改还是新增 //IsDelete int8 `gorm:"column:is_delete;type:tinyint(1);default:0" json:"-"` // 删除标记 延迟删除 diff --git a/internal/models/setting.go b/internal/models/setting.go index c6442ca0..a7178e3e 100644 --- a/internal/models/setting.go +++ b/internal/models/setting.go @@ -1,9 +1,11 @@ package models import ( - "gorm.io/gorm" + "encoding/json" "strconv" "time" + + "gorm.io/gorm" ) type Setting struct { @@ -56,6 +58,14 @@ func (s SettingDictValue) Float64() float64 { return f } +func (s SettingDictValue) StringSlice() []string { + var slice []string + + _ = json.Unmarshal([]byte(string(s)), &slice) + + return slice +} + type SettingDict struct { ID int64 `gorm:"primaryKey" json:"id"` Key string `gorm:"column:key;type:varchar(255);not null" json:"key"` @@ -72,11 +82,26 @@ func (s *SettingDict) TableName() string { const ( SettingDictKeyMultipleStreamThreadCount = "multiple_stream_thread_count" SettingDictKeyMultipleStreamChunkSize = "multiple_stream_chunk_size" + SettingDictKeyStrmFileEnable = "strm_file_enable" + SettingDictKeyStrmSupportFileExtList = "strm_support_file_ext_list" + SettingDictKeyFileWritable = "file_writable" ) const ( DefaultMultipleStreamThreadCount = 6 DefaultMultipleStreamChunkSize = 1024 * 1024 * 4 + DefaultStrmFileEnable = false + DefaultFileWritable = false +) + +var ( + DefaultStrmSupportFileExtList = []string{ + "mp4", "mkv", "avi", "mov", "wmv", "flv", "webm", "m4v", + "mpg", "mpeg", "m2v", "m4p", "m4b", "ts", "mts", "m2ts", "m2t", + "mxf", "dv", "dvr-ms", "asf", "3gp", "3g2", "f4v", "f4p", "f4a", "f4b", + "vob", "ogv", "ogg", "divx", "xvid", "rm", "rmvb", "dat", "nsv", + "qt", "amv", "mpv", "m1v", "svi", "viv", "fli", "flc", + } ) func (s *SettingDict) query(db *gorm.DB, key string) (string, error) { @@ -144,3 +169,62 @@ func (s *SettingDict) GetMultipleStreamChunkSize(db *gorm.DB) int64 { func (s *SettingDict) SetMultipleStreamChunkSize(db *gorm.DB, value int64) *gorm.DB { return s.store(db, SettingDictKeyMultipleStreamChunkSize, strconv.FormatInt(value, 10), "int64") } + +func (s *SettingDict) GetStrmFileEnable(db *gorm.DB) bool { + value, err := s.query(db, SettingDictKeyStrmFileEnable) + if err != nil { + return DefaultStrmFileEnable + } + + var v bool + + if v, err = strconv.ParseBool(value); err != nil { + return DefaultStrmFileEnable + } + + return v +} + +func (s *SettingDict) SetStrmFileEnable(db *gorm.DB, value bool) *gorm.DB { + return s.store(db, SettingDictKeyStrmFileEnable, strconv.FormatBool(value), "bool") +} + +func (s *SettingDict) GetStrmSupportFileExtList(db *gorm.DB) []string { + value, err := s.query(db, SettingDictKeyStrmSupportFileExtList) + if err != nil { + return DefaultStrmSupportFileExtList + } + + var v []string + + if err = json.Unmarshal([]byte(value), &v); err != nil { + return DefaultStrmSupportFileExtList + } + + return v +} + +func (s *SettingDict) SetStrmSupportFileExtList(db *gorm.DB, value []string) *gorm.DB { + b, _ := json.Marshal(value) + + return s.store(db, SettingDictKeyStrmSupportFileExtList, string(b), "json") +} + +func (s *SettingDict) GetFileWritable(db *gorm.DB) bool { + value, err := s.query(db, SettingDictKeyFileWritable) + if err != nil { + return DefaultFileWritable + } + + var v bool + + if v, err = strconv.ParseBool(value); err != nil { + return DefaultFileWritable + } + + return v +} + +func (s *SettingDict) SetFileWritable(db *gorm.DB, value bool) *gorm.DB { + return s.store(db, SettingDictKeyFileWritable, strconv.FormatBool(value), "bool") +} diff --git a/internal/pkgs/utils/get_value.go b/internal/pkgs/utils/get_value.go index 014d0d88..816bbafc 100644 --- a/internal/pkgs/utils/get_value.go +++ b/internal/pkgs/utils/get_value.go @@ -95,3 +95,18 @@ func String(value interface{}) string { return fmt.Sprintf("%v", v) } } + +func StringSlice(value interface{}) []string { + switch v := value.(type) { + case []string: + return v + case []interface{}: + var s []string + for _, v := range v { + s = append(s, String(v)) + } + return s + default: + return []string{String(v)} + } +} diff --git a/internal/pkgs/utils/rand.go b/internal/pkgs/utils/rand.go index 0ebc9109..2fe98a79 100644 --- a/internal/pkgs/utils/rand.go +++ b/internal/pkgs/utils/rand.go @@ -1,19 +1,40 @@ package utils import ( - "math/rand" - "time" + "crypto/rand" + "math/big" ) // GenerateRandomPassword 生成随机密码的辅助函数 func GenerateRandomPassword(length int) string { const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*" - rand.Seed(time.Now().UnixNano()) password := make([]byte, length) for i := range password { - password[i] = charset[rand.Intn(len(charset))] + num, err := rand.Int(rand.Reader, big.NewInt(int64(len(charset)))) + if err != nil { + // 如果加密随机数生成失败，回退到时间种子 + return generateFallbackPassword(length) + } + password[i] = charset[num.Int64()] + } + + return string(password) +} + +// generateFallbackPassword 回退的密码生成方法 +func generateFallbackPassword(length int) string { + const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*" + + // 使用当前时间的纳秒作为种子 + seed := make([]byte, 8) + _, _ = rand.Read(seed) + + password := make([]byte, length) + for i := range password { + num, _ := rand.Int(rand.Reader, big.NewInt(int64(len(charset)))) + password[i] = charset[num.Int64()] } return string(password) diff --git a/internal/router/router.go b/internal/router/router.go index c4bca678..52a4c220 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -2,11 +2,12 @@ package router import ( "fmt" - "github.com/xxcheng123/cloudpan189-share/internal/services/usergroup" "io/fs" "net/http" "strings" + "github.com/xxcheng123/cloudpan189-share/internal/services/usergroup" + "github.com/gin-gonic/gin" embed "github.com/xxcheng123/cloudpan189-share" "github.com/xxcheng123/cloudpan189-share/configs" @@ -87,6 +88,9 @@ func StartHTTPServer() error { settingRouter.POST("/modify_auto_refresh_minutes", settingService.ModifyAutoRefreshMinutes()) settingRouter.POST("/modify_multiple_stream_thread_count", settingService.ModifyMultipleStreamThreadCount()) settingRouter.POST("/modify_multiple_stream_chunk_size", settingService.ModifyMultipleStreamChunkSize()) + settingRouter.POST("/toggle_strm_file_enable", settingService.ToggleStrmFileEnable()) + settingRouter.POST("/modify_strm_support_file_ext_list", settingService.ModifyStrmSupportFileExtList()) + settingRouter.POST("/toggle_file_writable", settingService.ToggleFileWritable()) openapiRouter.POST("/setting/init_system", settingService.InitSystem()) } @@ -97,18 +101,69 @@ func StartHTTPServer() error { storageRouter.POST("/delete", storageService.Delete()) storageRouter.POST("/modify_token", storageService.ModifyToken()) storageRouter.GET("/list", storageService.List()) + storageRouter.POST("/clear_real_file", storageService.ClearRealFile()) openapiRouter.POST("/storage/deep_refresh_file", userService.AuthMiddleware(models.PermissionBase), storageService.DeepRefreshFile()) openapiRouter.GET("/storage/file/search", userService.AuthMiddleware(models.PermissionBase), storageService.Search()) } { - openapiRouter.GET("/open_file/*path", userService.AuthMiddleware(models.PermissionBase), universalFsService.Open("/", "json")) - davMethods := []string{"GET", "HEAD", "POST", "PUT", "DELETE", "OPTIONS", "PROPFIND", "MKCOL", "MOVE", "LOCK", "UNLOCK"} + openapiRouter.GET("/open_file/*path", userService.AuthMiddleware(models.PermissionBase), universalFsService.BaseMiddleware(), universalFsService.Open("/", "json")) + openapiRouter.DELETE("/open_file/*path", userService.AuthMiddleware(models.PermissionBase), universalFsService.BaseMiddleware(), universalFsService.Delete()) + + davMethods := []string{"GET", "HEAD", "POST", "OPTIONS", "PROPFIND", "MKCOL", "MOVE", "LOCK", "UNLOCK"} + + handler := []gin.HandlerFunc{ + userService.BasicAuthMiddleware(models.PermissionBase), + universalFsService.DavMiddleware(), + universalFsService.BaseMiddleware(), + } + + registry := []struct { + path string + prefix string + format string + handlers []gin.HandlerFunc + }{ + { + "/dav/*path", + "/dav", + "dav", + handler, + }, + { + "/dav", + "/dav", + "dav", + handler, + }, + { + "/strm_dav/*path", + "/strm_dav", + "strm_dav", + handler, + }, + { + "/strm_dav", + "/strm_dav", + "strm_dav", + handler, + }, + } + for _, method := range davMethods { - engine.Handle(method, "/dav/*path", userService.BasicAuthMiddleware(models.PermissionDavRead), universalFsService.DavMiddleware(), universalFsService.Open("/dav", "dav")) - engine.Handle(method, "/dav", userService.BasicAuthMiddleware(models.PermissionDavRead), universalFsService.DavMiddleware(), universalFsService.Open("/dav", "dav")) + for _, r := range registry { + engine.Handle(method, r.path, append(r.handlers, universalFsService.Open(r.prefix, r.format))...) + } + } + + for _, r := range registry { + engine.Handle(http.MethodPut, r.path, append(r.handlers, universalFsService.Put())...) + } + for _, r := range registry { + engine.Handle(http.MethodDelete, r.path, append(r.handlers, universalFsService.Delete())...) } + openapiRouter.GET("/file_download", universalFsService.FileDownload()) } diff --git a/internal/services/setting/service.go b/internal/services/setting/service.go index aaab6282..0730521b 100644 --- a/internal/services/setting/service.go +++ b/internal/services/setting/service.go @@ -25,6 +25,9 @@ type Service interface { ModifyAutoRefreshMinutes() gin.HandlerFunc ModifyMultipleStreamThreadCount() gin.HandlerFunc ModifyMultipleStreamChunkSize() gin.HandlerFunc + ToggleStrmFileEnable() gin.HandlerFunc + ModifyStrmSupportFileExtList() gin.HandlerFunc + ToggleFileWritable() gin.HandlerFunc } type service struct { diff --git a/internal/services/setting/service_get.go b/internal/services/setting/service_get.go index 17407a2b..31ee9bcb 100644 --- a/internal/services/setting/service_get.go +++ b/internal/services/setting/service_get.go @@ -11,9 +11,12 @@ import ( type getResponse struct { *models.Setting - RunTimes int64 `json:"runTimes"` // 已经运行的时间 - MultipleStreamThreadCount int `json:"multipleStreamThreadCount"` - MultipleStreamChunkSize int64 `json:"multipleStreamChunkSize"` + RunTimes int64 `json:"runTimes"` // 已经运行的时间 + MultipleStreamThreadCount int `json:"multipleStreamThreadCount"` + MultipleStreamChunkSize int64 `json:"multipleStreamChunkSize"` + StrmFileEnable bool `json:"strmFileEnable"` + StrmSupportFileExtList []string `json:"strmSupportFileExtList"` + FileWritable bool `json:"fileWritable"` } func (s *service) Get() gin.HandlerFunc { @@ -23,6 +26,9 @@ func (s *service) Get() gin.HandlerFunc { RunTimes: time.Now().Unix() - s.starTime.Unix(), MultipleStreamThreadCount: shared.MultipleStreamThreadCount, MultipleStreamChunkSize: shared.MultipleStreamChunkSize, + StrmFileEnable: shared.StrmFileEnable, + StrmSupportFileExtList: shared.StrmSupportFileExtList, + FileWritable: shared.FileWritable, }) } } diff --git a/internal/services/setting/service_modify_strm_support_file_ext_list.go b/internal/services/setting/service_modify_strm_support_file_ext_list.go new file mode 100644 index 00000000..b7d76a70 --- /dev/null +++ b/internal/services/setting/service_modify_strm_support_file_ext_list.go @@ -0,0 +1,67 @@ +package setting + +import ( + "fmt" + "github.com/gin-gonic/gin" + "github.com/xxcheng123/cloudpan189-share/internal/models" + "github.com/xxcheng123/cloudpan189-share/internal/shared" + "net/http" +) + +type modifyStrmSupportFileExtListRequest struct { + StrmSupportFileExtList []string `json:"strmSupportFileExtList"` +} + +type modifyStrmSupportFileExtListResponse struct { + RowsAffected int64 `json:"rowsAffected"` +} + +func (s *service) ModifyStrmSupportFileExtList() gin.HandlerFunc { + return func(ctx *gin.Context) { + var req = new(modifyStrmSupportFileExtListRequest) + + if err := ctx.ShouldBindJSON(&req); err != nil { + ctx.JSON(http.StatusBadRequest, gin.H{ + "code": http.StatusBadRequest, + "msg": "参数错误", + }) + + return + } + + // 直接使用传入的切片，如果为空就是空切片 + var fileExtList []string = req.StrmSupportFileExtList + if fileExtList == nil { + fileExtList = make([]string, 0) + } + + // 如果传入了具体的扩展名列表，验证每个扩展名不为空 + if len(req.StrmSupportFileExtList) > 0 { + for _, ext := range req.StrmSupportFileExtList { + if ext == "" { + ctx.JSON(http.StatusBadRequest, gin.H{ + "code": http.StatusBadRequest, + "msg": "文件扩展名不能为空", + }) + return + } + } + } + + result := new(models.SettingDict).SetStrmSupportFileExtList(s.db.WithContext(ctx), fileExtList) + if result.Error != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{ + "code": http.StatusInternalServerError, + "msg": fmt.Sprintf("修改失败：%s", result.Error.Error()), + }) + + return + } + + shared.StrmSupportFileExtList = fileExtList + + ctx.JSON(http.StatusOK, modifyStrmSupportFileExtListResponse{ + RowsAffected: result.RowsAffected, + }) + } +} diff --git a/internal/services/setting/service_toggle_file_writable.go b/internal/services/setting/service_toggle_file_writable.go new file mode 100644 index 00000000..9f4c9662 --- /dev/null +++ b/internal/services/setting/service_toggle_file_writable.go @@ -0,0 +1,48 @@ +package setting + +import ( + "fmt" + "github.com/gin-gonic/gin" + "github.com/xxcheng123/cloudpan189-share/internal/models" + "github.com/xxcheng123/cloudpan189-share/internal/shared" + "net/http" +) + +type toggleFileWritableRequest struct { + FileWritable bool `json:"fileWritable"` +} + +type toggleFileWritableResponse struct { + RowsAffected int64 `json:"rowsAffected"` +} + +func (s *service) ToggleFileWritable() gin.HandlerFunc { + return func(ctx *gin.Context) { + var req = new(toggleFileWritableRequest) + + if err := ctx.ShouldBindJSON(&req); err != nil { + ctx.JSON(http.StatusBadRequest, gin.H{ + "code": http.StatusBadRequest, + "msg": "参数错误", + }) + + return + } + + result := new(models.SettingDict).SetFileWritable(s.db.WithContext(ctx), req.FileWritable) + if result.Error != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{ + "code": http.StatusInternalServerError, + "msg": fmt.Sprintf("修改失败：%s", result.Error.Error()), + }) + + return + } + + shared.FileWritable = req.FileWritable + + ctx.JSON(http.StatusOK, toggleFileWritableResponse{ + RowsAffected: result.RowsAffected, + }) + } +} diff --git a/internal/services/setting/service_toggle_strm_file_enable.go b/internal/services/setting/service_toggle_strm_file_enable.go new file mode 100644 index 00000000..eda90d12 --- /dev/null +++ b/internal/services/setting/service_toggle_strm_file_enable.go @@ -0,0 +1,68 @@ +package setting + +import ( + "fmt" + "github.com/gin-gonic/gin" + "github.com/xxcheng123/cloudpan189-share/internal/models" + "github.com/xxcheng123/cloudpan189-share/internal/shared" + "net/http" +) + +type toggleStrmFileEnableRequest struct { + StrmFileEnable bool `json:"strmFileEnable"` +} + +type toggleStrmFileEnableResponse struct { + RowsAffected int64 `json:"rowsAffected"` +} + +func (s *service) ToggleStrmFileEnable() gin.HandlerFunc { + return func(ctx *gin.Context) { + var req = new(toggleStrmFileEnableRequest) + + if err := ctx.ShouldBindJSON(&req); err != nil { + ctx.JSON(http.StatusBadRequest, gin.H{ + "code": http.StatusBadRequest, + "msg": "参数错误", + }) + + return + } + + result := new(models.SettingDict).SetStrmFileEnable(s.db.WithContext(ctx), req.StrmFileEnable) + if result.Error != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{ + "code": http.StatusInternalServerError, + "msg": fmt.Sprintf("修改失败：%s", result.Error.Error()), + }) + + return + } + + if req.StrmFileEnable { + if err := shared.ScanJobPublish(shared.ScanJobRebuildStrm, new(models.VirtualFile)); err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{ + "code": http.StatusInternalServerError, + "msg": fmt.Sprintf("下发扫描任务失败：%s", err.Error()), + }) + + return + } + } else { + if err := shared.ScanJobPublish(shared.ScanJobClearStrm, new(models.VirtualFile)); err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{ + "code": http.StatusInternalServerError, + "msg": fmt.Sprintf("下发清除任务失败：%s", err.Error()), + }) + + return + } + } + + shared.StrmFileEnable = req.StrmFileEnable + + ctx.JSON(http.StatusOK, toggleStrmFileEnableResponse{ + RowsAffected: result.RowsAffected, + }) + } +} diff --git a/internal/services/storage/service.go b/internal/services/storage/service.go index da8646c0..530541d9 100644 --- a/internal/services/storage/service.go +++ b/internal/services/storage/service.go @@ -16,6 +16,7 @@ type Service interface { ModifyToken() gin.HandlerFunc DeepRefreshFile() gin.HandlerFunc Search() gin.HandlerFunc + ClearRealFile() gin.HandlerFunc } type service struct { diff --git a/internal/services/storage/service_add.go b/internal/services/storage/service_add.go index f12ab5f2..d77d02d7 100644 --- a/internal/services/storage/service_add.go +++ b/internal/services/storage/service_add.go @@ -7,6 +7,7 @@ import ( "github.com/gin-gonic/gin" "github.com/pkg/errors" "github.com/xxcheng123/cloudpan189-interface/client" + "github.com/xxcheng123/cloudpan189-share/internal/consts" "github.com/xxcheng123/cloudpan189-share/internal/models" "github.com/xxcheng123/cloudpan189-share/internal/pkgs/utils" "github.com/xxcheng123/cloudpan189-share/internal/shared" @@ -114,7 +115,7 @@ func (s *service) Add() gin.HandlerFunc { } if req.CloudToken != 0 { - m.Addition["cloud_token"] = req.CloudToken + m.Addition[consts.FileAdditionKeyCloudToken] = req.CloudToken } if req.Protocol == "subscribe" { @@ -128,7 +129,7 @@ func (s *service) Add() gin.HandlerFunc { return } - m.Addition["subscribe_user"] = req.SubscribeUser + m.Addition[consts.FileAdditionKeySubscribeUser] = req.SubscribeUser } else if req.Protocol == "share" { var opts []client.GetShareInfoOption @@ -169,13 +170,13 @@ func (s *service) Add() gin.HandlerFunc { return } - m.Addition["share_id"] = resp.ShareId - m.Addition["share_code"] = req.ShareCode - m.Addition["access_code"] = req.ShareAccessCode - m.Addition["share_mode"] = resp.ShareMode - m.Addition["share_type"] = resp.ShareType - m.Addition["file_id"] = resp.FileId - m.Addition["is_folder"] = resp.IsFolder + m.Addition[consts.FileAdditionKeyShareId] = resp.ShareId + m.Addition[consts.FileAdditionKeyShareCode] = req.ShareCode + m.Addition[consts.FileAdditionKeyAccessCode] = req.ShareAccessCode + m.Addition[consts.FileAdditionKeyShareMode] = resp.ShareMode + m.Addition[consts.FileAdditionKeyShareType] = resp.ShareType + m.Addition[consts.FileAdditionKeyFileId] = resp.FileId + m.Addition[consts.FileAdditionKeyIsFolder] = resp.IsFolder } pid, err := s.findOrCreateAncestors(ctx, req.LocalPath) diff --git a/internal/services/storage/service_clear_real_file.go b/internal/services/storage/service_clear_real_file.go new file mode 100644 index 00000000..f3f2020c --- /dev/null +++ b/internal/services/storage/service_clear_real_file.go @@ -0,0 +1,34 @@ +package storage + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/xxcheng123/cloudpan189-share/internal/models" + "github.com/xxcheng123/cloudpan189-share/internal/shared" +) + +type clearRealFileResponse struct { + Message string `json:"message"` +} + +func (s *service) ClearRealFile() gin.HandlerFunc { + return func(ctx *gin.Context) { + // 使用根虚拟文件来触发清空任务 + rootFile := new(models.VirtualFile) + + if err := shared.ScanJobPublish(shared.ScanJobClearRealFile, rootFile); err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{ + "code": http.StatusInternalServerError, + "msg": fmt.Sprintf("下发清空本地真实存储任务失败：%s", err.Error()), + }) + + return + } + + ctx.JSON(http.StatusOK, clearRealFileResponse{ + Message: "清空本地真实存储任务已发送", + }) + } +} diff --git a/internal/services/storage/service_modify_token.go b/internal/services/storage/service_modify_token.go index 4a5b39a1..85f12a6a 100644 --- a/internal/services/storage/service_modify_token.go +++ b/internal/services/storage/service_modify_token.go @@ -3,6 +3,7 @@ package storage import ( "net/http" + "github.com/xxcheng123/cloudpan189-share/internal/consts" "github.com/xxcheng123/cloudpan189-share/internal/models" "github.com/gin-gonic/gin" @@ -48,7 +49,7 @@ func (s *service) ModifyToken() gin.HandlerFunc { return } - file.Addition["cloud_token"] = req.CloudToken + file.Addition[consts.FileAdditionKeyCloudToken] = req.CloudToken result := s.db.Model(&models.VirtualFile{}).Where("id = ?", req.ID).Update("addition", file.Addition) diff --git a/internal/services/universalfs/service_dav.go b/internal/services/universalfs/dav.go similarity index 79% rename from internal/services/universalfs/service_dav.go rename to internal/services/universalfs/dav.go index 856eadd8..e6851f02 100644 --- a/internal/services/universalfs/service_dav.go +++ b/internal/services/universalfs/dav.go @@ -1,7 +1,9 @@ package universalfs import ( + "errors" "fmt" + "golang.org/x/net/webdav" "net/http" "net/url" "path" @@ -9,66 +11,19 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/xxcheng123/cloudpan189-share/internal/types" + "go.uber.org/zap" ) -func (s *service) DavMiddleware() gin.HandlerFunc { - return func(ctx *gin.Context) { - if ctx.GetHeader("Depth") == "" { - ctx.Request.Header.Add("Depth", "1") - } else if ctx.GetHeader("Depth") == "infinity" { - ctx.JSON(http.StatusBadRequest, gin.H{ - "code": http.StatusBadRequest, - "message": "`infinity` is not allowed", - }) - - ctx.Abort() - - return - } - - if ctx.GetHeader("X-Litmus") == "props: 3 (propfind_invalid2)" { - ctx.JSON(http.StatusBadRequest, gin.H{ - "code": http.StatusBadRequest, - "message": "Invalid property name", - }) - - ctx.Abort() - - return - } - - switch ctx.Request.Method { - case "PROPFIND": - ctx.Next() - case "GET", "HEAD", "POST": - ctx.Next() - case "OPTIONS": - allow := "OPTIONS, HEAD, GET, POST, PROPFIND" - ctx.Header("Allow", allow) - // http://www.webdav.org/specs/rfc4918.html#dav.compliance.classes - ctx.Header("DAV", "1, 2") - // http://msdn.microsoft.com/en-au/library/cc250217.aspx - ctx.Header("MS-Author-Via", "DAV") - - ctx.Abort() - default: - ctx.JSON(http.StatusMethodNotAllowed, gin.H{ - "code": http.StatusMethodNotAllowed, - "message": "Method not allowed", - }) - - ctx.Abort() - } - } -} - func (s *service) responseDav(ctx *gin.Context, fileInfo *FileInfo) { switch ctx.Request.Method { case "GET", "HEAD", "POST": if fileInfo.IsFolder == 1 { - ctx.JSON(http.StatusMethodNotAllowed, gin.H{ - "code": http.StatusMethodNotAllowed, - "message": "Method not allowed", + s.logger.Warn("尝试对文件夹执行不支持的操作", zap.String("method", ctx.Request.Method), zap.String("path", ctx.Request.URL.Path)) + + ctx.JSON(http.StatusMethodNotAllowed, types.ErrResponse{ + Code: http.StatusMethodNotAllowed, + Message: "文件夹不支持此操作", }) return @@ -327,3 +282,58 @@ func escapeXML(s string) string { s = strings.ReplaceAll(s, "'", "'") return s } + +const ( + lockTimeout = 30 * time.Minute +) + +func (s *service) lock(now time.Time, root string) (token string, status int, err error) { + token, err = s.LockSystem.Create(now, webdav.LockDetails{ + Root: root, + Duration: lockTimeout, + ZeroDepth: true, + }) + if err != nil { + if errors.Is(err, webdav.ErrLocked) { + s.logger.Warn("资源已被锁定", zap.String("root", root), zap.Error(err)) + return "", webdav.StatusLocked, err + } + s.logger.Error("创建锁失败", zap.String("root", root), zap.Error(err)) + return "", http.StatusInternalServerError, err + } + return token, 0, nil +} + +func (s *service) confirmLocks(src, dst string) (release func(), status int, err error) { + now, srcToken, dstToken := time.Now(), "", "" + if src != "" { + srcToken, status, err = s.lock(now, src) + if err != nil { + s.logger.Error("锁定源路径失败", zap.String("src", src), zap.Error(err)) + return nil, status, err + } + } + if dst != "" { + dstToken, status, err = s.lock(now, dst) + if err != nil { + if srcToken != "" { + _ = s.LockSystem.Unlock(now, srcToken) + } + s.logger.Error("锁定目标路径失败", zap.String("dst", dst), zap.Error(err)) + return nil, status, err + } + } + + return func() { + if dstToken != "" { + if err := s.LockSystem.Unlock(now, dstToken); err != nil { + s.logger.Warn("解锁目标路径失败", zap.String("dst", dst), zap.String("token", dstToken), zap.Error(err)) + } + } + if srcToken != "" { + if err := s.LockSystem.Unlock(now, srcToken); err != nil { + s.logger.Warn("解锁源路径失败", zap.String("src", src), zap.String("token", srcToken), zap.Error(err)) + } + } + }, 0, nil +} diff --git a/internal/services/universalfs/middleware.go b/internal/services/universalfs/middleware.go new file mode 100644 index 00000000..eaded7f5 --- /dev/null +++ b/internal/services/universalfs/middleware.go @@ -0,0 +1,209 @@ +package universalfs + +import ( + mapset "github.com/deckarep/golang-set/v2" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" + "github.com/xxcheng123/cloudpan189-share/internal/consts" + "github.com/xxcheng123/cloudpan189-share/internal/models" + "github.com/xxcheng123/cloudpan189-share/internal/pkgs/utils" + "github.com/xxcheng123/cloudpan189-share/internal/shared" + "github.com/xxcheng123/cloudpan189-share/internal/types" + "go.uber.org/zap" + "gorm.io/gorm" + "net/http" +) + +func (s *service) BaseMiddleware() gin.HandlerFunc { + return func(ctx *gin.Context) { + rawPath := ctx.Param("path") + + paths, err := utils.SplitPath(rawPath) + if err != nil { + ctx.JSON(http.StatusBadRequest, types.ErrResponse{ + Code: http.StatusBadRequest, + Message: err.Error(), + }) + + ctx.Abort() + + return + } + + var ( + pid int64 + fid int64 + gid = ctx.GetInt64(consts.CtxKeyGroupId) + fullPaths = make([]string, 0) + ) + + var ( + groupFileSet = mapset.NewSet[int64]() + ) + + if gid != 0 { + groupFiles := make([]*models.Group2File, 0) + + if err = s.db.WithContext(ctx).Model(new(models.Group2File)).Where("group_id", gid).Find(&groupFiles).Error; err != nil { + s.logger.Error("获取用户组文件关系失败", zap.Error(err)) + + ctx.JSON(http.StatusInternalServerError, types.ErrResponse{ + Code: http.StatusInternalServerError, + Message: "获取用户组文件关系失败", + }) + + ctx.Abort() + + return + } + + for _, groupFile := range groupFiles { + groupFileSet.Add(groupFile.FileId) + } + } + + if len(paths) == 0 { + fid = 0 + pid = -1 + } else { + for idx, p := range paths { + var tmpFile = new(models.VirtualFile) + if err = s.db.WithContext(ctx).Where("parent_id", fid).Where("name", p).First(tmpFile).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + if ctx.Request.Method == http.MethodPut && len(paths)-1 == idx { + // 这个情况表示要写入文件，但是文件不存在 + // 取到父级目录的ID + pid = fid + fid = -1 + ctx.Set(consts.CtxKeyFilename, p) + + break + } + + s.logger.Warn("文件未找到", zap.String("path", rawPath), zap.String("filename", p)) + + ctx.JSON(http.StatusNotFound, types.ErrResponse{ + Code: http.StatusNotFound, + Message: "文件未找到", + }) + + ctx.Abort() + + return + } + + s.logger.Error("查询文件失败", zap.Error(err), zap.String("path", rawPath), zap.String("filename", p)) + + ctx.JSON(http.StatusBadRequest, types.ErrResponse{ + Code: http.StatusBadRequest, + Message: err.Error(), + }) + + ctx.Abort() + + return + } + + if tmpFile.IsTop == 1 && gid != 0 && !groupFileSet.Contains(tmpFile.ID) { + // 没有权限 + s.logger.Warn("用户无权限访问文件", zap.Int64("gid", gid), zap.Int64("fileId", tmpFile.ID), zap.String("filename", p)) + + ctx.JSON(http.StatusForbidden, types.ErrResponse{ + Code: http.StatusForbidden, + Message: "无权限访问", + }) + + ctx.Abort() + + return + } + + fullPaths = append(fullPaths, p) + pid = tmpFile.ParentId + fid = tmpFile.ID + } + } + + ctx.Set(consts.CtxKeyFileId, fid) + ctx.Set(consts.CtxKeyParentId, pid) + ctx.Set(consts.CtxKeyGroupId, gid) + ctx.Set(consts.CtxKeyFullPaths, fullPaths) + ctx.Set(consts.CtxKeyGroupFileSet, groupFileSet) + } +} + +func (s *service) DavMiddleware() gin.HandlerFunc { + return func(ctx *gin.Context) { + if ctx.GetHeader("Depth") == "" { + ctx.Request.Header.Add("Depth", "1") + } else if ctx.GetHeader("Depth") == "infinity" { + s.logger.Warn("不支持infinity深度查询", zap.String("userAgent", ctx.GetHeader("User-Agent"))) + + ctx.JSON(http.StatusBadRequest, types.ErrResponse{ + Code: http.StatusBadRequest, + Message: "不支持infinity深度查询", + }) + + ctx.Abort() + + return + } + + if ctx.GetHeader("X-Litmus") == "props: 3 (propfind_invalid2)" { + s.logger.Warn("无效的属性名称", zap.String("userAgent", ctx.GetHeader("User-Agent"))) + + ctx.JSON(http.StatusBadRequest, types.ErrResponse{ + Code: http.StatusBadRequest, + Message: "无效的属性名称", + }) + + ctx.Abort() + + return + } + + switch ctx.Request.Method { + case "PROPFIND": + ctx.Next() + case "GET", "HEAD", "POST": + ctx.Next() + case "PUT", "DELETE": + if !shared.FileWritable { + s.logger.Warn("文件写入功能已禁用", zap.String("method", ctx.Request.Method), zap.String("path", ctx.Request.URL.Path)) + + ctx.JSON(http.StatusMethodNotAllowed, types.ErrResponse{ + Code: http.StatusMethodNotAllowed, + Message: "文件写入功能已禁用", + }) + + ctx.Abort() + + return + } + + ctx.Next() + case "OPTIONS": + allow := "OPTIONS, HEAD, GET, POST, PROPFIND" + if shared.FileWritable { + allow += ", PUT, DELETE" + } + + ctx.Header("Allow", allow) + // http://www.webdav.org/specs/rfc4918.html#dav.compliance.classes + ctx.Header("DAV", "1, 2") + // http://msdn.microsoft.com/en-au/library/cc250217.aspx + ctx.Header("MS-Author-Via", "DAV") + + ctx.Abort() + default: + s.logger.Warn("不支持的HTTP方法", zap.String("method", ctx.Request.Method), zap.String("path", ctx.Request.URL.Path)) + + ctx.JSON(http.StatusMethodNotAllowed, types.ErrResponse{ + Code: http.StatusMethodNotAllowed, + Message: "不支持的HTTP方法", + }) + + ctx.Abort() + } + } +} diff --git a/internal/services/universalfs/service.go b/internal/services/universalfs/service.go index df7b9227..9195fd53 100644 --- a/internal/services/universalfs/service.go +++ b/internal/services/universalfs/service.go @@ -1,12 +1,8 @@ package universalfs import ( - "crypto/md5" - "encoding/hex" - "net/url" - "sort" - "strconv" - "strings" + "github.com/xxcheng123/cloudpan189-share/internal/fs" + "golang.org/x/net/webdav" "time" "github.com/gin-gonic/gin" @@ -21,22 +17,29 @@ type Service interface { Open(prefix string, format string) gin.HandlerFunc FileDownload() gin.HandlerFunc DavMiddleware() gin.HandlerFunc + BaseMiddleware() gin.HandlerFunc + Put() gin.HandlerFunc + Delete() gin.HandlerFunc } type service struct { - db *gorm.DB - logger *zap.Logger - startTime time.Time - cache *cache.Cache - g singleflight.Group + db *gorm.DB + logger *zap.Logger + startTime time.Time + cache *cache.Cache + g singleflight.Group + LockSystem webdav.LockSystem + fs fs.FS } func NewService(db *gorm.DB, logger *zap.Logger) Service { return &service{ - db: db, - logger: logger, - startTime: time.Now(), - cache: cache.New(time.Minute, time.Minute*10), + db: db, + logger: logger, + startTime: time.Now(), + cache: cache.New(time.Minute, time.Minute*10), + LockSystem: webdav.NewMemLS(), + fs: fs.NewFS(db, logger), } } @@ -51,47 +54,3 @@ type FileInfo struct { type ReadSession struct { CloudTokenID int64 `json:"cloudTokenId"` } - -func enc(values url.Values, key string) url.Values { - if !values.Has("timestamp") { - timestamp := time.Now().Add(time.Hour * 6).Unix() - values.Set("timestamp", strconv.FormatInt(timestamp, 10)) - } - - // 排序并生成签名字符串 - keys := make([]string, 0, len(values)) - for k := range values { - keys = append(keys, k) - } - - sort.Strings(keys) - - // 构建待签名字符串 - var signStr strings.Builder - for _, k := range keys { - signStr.WriteString(k + "=" + values.Get(k) + "&") - } - signStr.WriteString("key=" + key) - - // 计算MD5 - h := md5.New() - h.Write([]byte(signStr.String())) - sign := hex.EncodeToString(h.Sum(nil)) - - values.Set("sign", sign) - return values -} - -// 验证签名是否有效 -func verify(values url.Values, key string) bool { - // 获取并移除签名 - sign := values.Get("sign") - values.Del("sign") - - // 重新计算签名 - newValues := enc(values, key) - newSign := newValues.Get("sign") - - // 比较签名是否一致 - return sign == newSign -} diff --git a/internal/services/universalfs/service_delete.go b/internal/services/universalfs/service_delete.go new file mode 100644 index 00000000..52738fe5 --- /dev/null +++ b/internal/services/universalfs/service_delete.go @@ -0,0 +1,81 @@ +package universalfs + +import ( + "github.com/gin-gonic/gin" + "github.com/xxcheng123/cloudpan189-share/internal/consts" + "github.com/xxcheng123/cloudpan189-share/internal/models" + "github.com/xxcheng123/cloudpan189-share/internal/types" + "go.uber.org/zap" + "net/http" +) + +func (s *service) Delete() gin.HandlerFunc { + return func(ctx *gin.Context) { + rawPath := ctx.Param("path") + + release, status, err := s.confirmLocks(rawPath, "") + if err != nil { + s.logger.Error("确认文件锁失败", zap.String("path", rawPath), zap.Error(err)) + + ctx.JSON(status, types.ErrResponse{ + Code: status, + Message: err.Error(), + }) + + return + } + + defer release() + + var fid = ctx.GetInt64(consts.CtxKeyFileId) + if fid <= 0 { + s.logger.Warn("尝试删除不存在的文件", zap.String("path", rawPath), zap.Int64("fileId", fid)) + + ctx.JSON(http.StatusNotFound, types.ErrResponse{ + Code: http.StatusNotFound, + Message: "文件不存在", + }) + + return + } + + file := new(models.VirtualFile) + + if err = s.db.WithContext(ctx).Where("id = ?", fid).First(file).Error; err != nil { + s.logger.Error("查询文件信息失败", zap.String("path", rawPath), zap.Int64("fileId", fid), zap.Error(err)) + + ctx.JSON(http.StatusNotFound, types.ErrResponse{ + Code: http.StatusNotFound, + Message: "文件不存在", + }) + + return + } + + if file.IsTop == 1 { + s.logger.Warn("尝试删除挂载点文件", zap.String("path", rawPath), zap.Int64("fileId", fid), zap.String("fileName", file.Name)) + + ctx.JSON(http.StatusBadRequest, types.ErrResponse{ + Code: http.StatusBadRequest, + Message: "挂载点文件请在后台存储管理删除", + }) + + return + } + + if err = s.fs.Delete(ctx, fid); err != nil { + s.logger.Error("删除文件失败", zap.String("path", rawPath), zap.Int64("fileId", fid), zap.String("fileName", file.Name), zap.Error(err)) + + ctx.JSON(http.StatusInternalServerError, types.ErrResponse{ + Code: http.StatusInternalServerError, + Message: "删除失败", + }) + + return + } + + s.logger.Info("文件删除成功", zap.String("path", rawPath), zap.Int64("fileId", fid), zap.String("fileName", file.Name)) + + ctx.Status(http.StatusNoContent) + } +} diff --git a/internal/services/universalfs/service_file_download.go b/internal/services/universalfs/service_file_download.go index 08c0aa24..30ba27c5 100644 --- a/internal/services/universalfs/service_file_download.go +++ b/internal/services/universalfs/service_file_download.go @@ -6,19 +6,25 @@ import ( "io" "net/http" "net/url" + "os" + "path" "strconv" "strings" "time" "github.com/gin-gonic/gin" "github.com/pkg/errors" + "go.uber.org/zap" + "gorm.io/gorm" + "github.com/xxcheng123/cloudpan189-interface/client" + "github.com/xxcheng123/cloudpan189-share/configs" + "github.com/xxcheng123/cloudpan189-share/internal/consts" "github.com/xxcheng123/cloudpan189-share/internal/models" "github.com/xxcheng123/cloudpan189-share/internal/pkgs/utils" "github.com/xxcheng123/cloudpan189-share/internal/shared" + "github.com/xxcheng123/cloudpan189-share/internal/types" "github.com/xxcheng123/multistreamer" - "go.uber.org/zap" - "gorm.io/gorm" ) type fileDownloadRequest struct { @@ -28,30 +34,57 @@ type fileDownloadRequest struct { Sign string `form:"sign" binding:"required"` } +type DoResult struct { + Content string + HttpCode int + Err error +} + +// 全局HTTP客户端，复用连接 +var globalHTTPClient = &http.Client{ + Timeout: 0, + Transport: &http.Transport{ + DisableKeepAlives: false, + MaxIdleConns: 200, + MaxIdleConnsPerHost: 20, + MaxConnsPerHost: 50, + IdleConnTimeout: 120 * time.Second, + TLSHandshakeTimeout: 5 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + DisableCompression: true, + WriteBufferSize: 128 * 1024, + ReadBufferSize: 128 * 1024, + ForceAttemptHTTP2: false, + }, +} + func (s *service) FileDownload() gin.HandlerFunc { return func(ctx *gin.Context) { var req = new(fileDownloadRequest) if err := ctx.ShouldBindQuery(req); err != nil { - ctx.JSON(http.StatusBadRequest, gin.H{ - "code": http.StatusBadRequest, - "message": err.Error(), + s.logger.Warn("文件下载请求参数错误", zap.Error(err)) + ctx.JSON(http.StatusBadRequest, types.ErrResponse{ + Code: http.StatusBadRequest, + Message: err.Error(), }) return } - if req.TimeStamp < time.Now().Unix() { - ctx.JSON(http.StatusUnauthorized, gin.H{ - "code": http.StatusUnauthorized, - "message": "timeStamp is expired", + if req.TimeStamp != -1 && req.TimeStamp < time.Now().Unix() { + s.logger.Warn("文件下载请求时间戳已过期", + zap.Int64("timestamp", req.TimeStamp), + zap.Int64("fileId", req.ID)) + ctx.JSON(http.StatusUnauthorized, types.ErrResponse{ + Code: http.StatusUnauthorized, + Message: "请求时间戳已过期", }) return } key := shared.Setting.SaltKey - values := url.Values{ "id": []string{strconv.FormatInt(req.ID, 10)}, "timestamp": []string{strconv.FormatInt(req.TimeStamp, 10)}, @@ -60,120 +93,158 @@ func (s *service) FileDownload() gin.HandlerFunc { } if !verify(values, key) { - ctx.JSON(http.StatusUnauthorized, gin.H{ - "code": http.StatusUnauthorized, - "message": "sign is invalid", + s.logger.Warn("文件下载请求签名验证失败", + zap.Int64("fileId", req.ID), + zap.String("sign", req.Sign)) + ctx.JSON(http.StatusUnauthorized, types.ErrResponse{ + Code: http.StatusUnauthorized, + Message: "签名验证失败", }) return } if v, ok := s.cache.Get(fmt.Sprintf("file::url::%d", req.ID)); ok { - ctx.Header("X-Download-Url-Cache", "true") - - s.doResponse(ctx, v.(string)) - - return + if url, ok := v.(string); ok { + ctx.Header("X-Download-Url-Cache", "true") + s.doResponse(ctx, url) + return + } else { + s.logger.Warn("缓存中的URL格式错误", zap.Int64("fileId", req.ID)) + // 继续执行，重新获取下载链接 + } } - _result, err, _ := s.g.Do(fmt.Sprintf("file::url::%d", req.ID), func() (interface{}, error) { - u, httpCode, err := s.getFileDownloadURL(ctx, req.ID) - return &DoResult{ - URL: u, - HttpCode: httpCode, - Err: err, - }, nil - }) - - if err != nil { - ctx.JSON(http.StatusInternalServerError, gin.H{ - "code": http.StatusInternalServerError, - "message": err.Error(), + file := &models.VirtualFile{} + if err := s.db.WithContext(ctx).Where("id = ?", req.ID).First(file).Error; err != nil { + s.logger.Error("查询文件信息失败", zap.Int64("fileId", req.ID), zap.Error(err)) + ctx.JSON(http.StatusNotFound, types.ErrResponse{ + Code: http.StatusNotFound, + Message: "文件未找到", }) return } - result := _result.(*DoResult) - - if result.Err != nil { - ctx.JSON(result.HttpCode, gin.H{ - "code": result.HttpCode, - "message": result.Err.Error(), - }) + if file.OsType == models.OsTypeRealFile { + s.handleRealFileDownload(ctx, file, req.ID) return } - s.doResponse(ctx, result.URL) + s.handleCloudFileDownload(ctx, req.ID) } } -// 全局HTTP客户端，复用连接 -var globalHTTPClient = &http.Client{ - Timeout: 0, - Transport: &http.Transport{ - DisableKeepAlives: false, - MaxIdleConns: 200, // 增加连接池 - MaxIdleConnsPerHost: 20, // 每个host更多连接 - MaxConnsPerHost: 50, // 总连接数限制 - IdleConnTimeout: 120 * time.Second, - TLSHandshakeTimeout: 5 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - DisableCompression: true, // 禁用压缩 - WriteBufferSize: 128 * 1024, // 128KB写缓冲 - ReadBufferSize: 128 * 1024, // 128KB读缓冲 - ForceAttemptHTTP2: false, // 禁用HTTP2 - }, +func (s *service) handleRealFileDownload(ctx *gin.Context, file *models.VirtualFile, fileID int64) { + v, ok := file.Addition[consts.FileAdditionKeyFilePath] + if !ok { + s.logger.Error("真实文件路径信息缺失", + zap.Int64("fileId", fileID), + zap.String("fileName", file.Name)) + ctx.JSON(http.StatusInternalServerError, types.ErrResponse{ + Code: http.StatusInternalServerError, + Message: "文件路径不存在", + }) + + return + } + + filePath, ok := v.(string) + if !ok { + s.logger.Error("真实文件路径格式错误", + zap.Int64("fileId", fileID), + zap.String("fileName", file.Name)) + ctx.JSON(http.StatusInternalServerError, types.ErrResponse{ + Code: http.StatusInternalServerError, + Message: "文件路径格式错误", + }) + + return + } + fullPath := path.Join(configs.GetConfig().FileDir, filePath) + + if _, err := os.Stat(fullPath); os.IsNotExist(err) { + s.logger.Error("真实文件不存在", + zap.Int64("fileId", fileID), + zap.String("fileName", file.Name), + zap.String("fullPath", fullPath)) + ctx.JSON(http.StatusNotFound, types.ErrResponse{ + Code: http.StatusNotFound, + Message: "文件不存在", + }) + + return + } + + filename := file.Name + ctx.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"; filename*=UTF-8''%s", + filename, url.QueryEscape(filename))) + + s.logger.Info("开始下载真实文件", + zap.Int64("fileId", fileID), + zap.String("fileName", file.Name), + zap.String("fullPath", fullPath)) + + ctx.File(fullPath) } -func (s *service) doResponse(ctx *gin.Context, url string) { - if shared.Setting.MultipleStream { - ctx.Header("X-Transfer-Type", "multi_stream") - ctx.Header("X-Transfer-Chunk-Size", strconv.FormatInt(shared.MultipleStreamChunkSize, 10)) - ctx.Header("X-Transfer-Chunk-Size-Format", utils.FormatBytes(shared.MultipleStreamChunkSize)) - ctx.Header("X-Transfer-Thread-Count", strconv.Itoa(shared.MultipleStreamThreadCount)) - - httpReq := ctx.Request.Header.Clone() - httpReq.Set("Accept-Encoding", "identity") - httpReq.Del("Content-Type") - - streamer, err := multistreamer.NewStreamer(ctx, - url, - httpReq, - multistreamer.WithLogger(s.logger), - multistreamer.WithThreads(shared.MultipleStreamThreadCount), - multistreamer.WithChunkSize(shared.MultipleStreamChunkSize), - ) - if err != nil { - s.logger.Error("初始化失败", zap.Error(err), zap.String("url", url)) - - ctx.JSON(http.StatusInternalServerError, gin.H{ - "code": http.StatusInternalServerError, - "message": fmt.Sprintf("初始化失败: %v", err), - }) +func (s *service) handleCloudFileDownload(ctx *gin.Context, fileID int64) { + _result, err, _ := s.g.Do(fmt.Sprintf("file::url::%d", fileID), func() (interface{}, error) { + u, httpCode, err := s.getFileDownloadURL(ctx, fileID) - return - } + return &DoResult{ + Content: u, + HttpCode: httpCode, + Err: err, + }, nil + }) - for k, v := range streamer.GetResponseHeader() { - ctx.Header(k, v[0]) - } + if err != nil { + s.logger.Error("获取文件下载链接失败", zap.Int64("fileId", fileID), zap.Error(err)) + ctx.JSON(http.StatusInternalServerError, types.ErrResponse{ + Code: http.StatusInternalServerError, + Message: err.Error(), + }) - ctx.Status(streamer.HTTPCode()) + return + } - if err = streamer.Transfer(ctx, ctx.Writer); err != nil { - if s.isConnectionError(err) { - s.logger.Info("客户端连接断开", - zap.String("url", url), - ) - } else { - s.logger.Error("文件下载转发失败", - zap.Error(err), - zap.String("url", url), - ) - } - } + result, ok := _result.(*DoResult) + if !ok { + s.logger.Error("类型断言失败", zap.Int64("fileId", fileID)) + ctx.JSON(http.StatusInternalServerError, types.ErrResponse{ + Code: http.StatusInternalServerError, + Message: "内部错误", + }) + return + } + + if result.Err != nil { + s.logger.Error("处理文件下载请求失败", + zap.Int64("fileId", fileID), + zap.Int("httpCode", result.HttpCode), + zap.Error(result.Err)) + ctx.JSON(result.HttpCode, types.ErrResponse{ + Code: result.HttpCode, + Message: result.Err.Error(), + }) + + return + } + + if result.HttpCode == http.StatusOK { + ctx.String(http.StatusOK, result.Content) + + return + } + + s.doResponse(ctx, result.Content) +} + +func (s *service) doResponse(ctx *gin.Context, url string) { + if shared.Setting.MultipleStream { + s.handleMultiStreamResponse(ctx, url) } else if shared.Setting.LocalProxy { s.handleLocalProxy(ctx, url) } else { @@ -182,107 +253,131 @@ func (s *service) doResponse(ctx *gin.Context, url string) { } } -// 单独处理 LocalProxy 逻辑 +func (s *service) handleMultiStreamResponse(ctx *gin.Context, url string) { + ctx.Header("X-Transfer-Type", "multi_stream") + ctx.Header("X-Transfer-Chunk-Size", strconv.FormatInt(shared.MultipleStreamChunkSize, 10)) + ctx.Header("X-Transfer-Chunk-Size-Format", utils.FormatBytes(shared.MultipleStreamChunkSize)) + ctx.Header("X-Transfer-Thread-Count", strconv.Itoa(shared.MultipleStreamThreadCount)) + + httpReq := ctx.Request.Header.Clone() + httpReq.Set("Accept-Encoding", "identity") + httpReq.Del("Content-Type") + + streamer, err := multistreamer.NewStreamer(ctx, + url, + httpReq, + multistreamer.WithLogger(s.logger), + multistreamer.WithThreads(shared.MultipleStreamThreadCount), + multistreamer.WithChunkSize(shared.MultipleStreamChunkSize), + ) + if err != nil { + s.logger.Error("多线程流初始化失败", zap.Error(err), zap.String("url", url)) + ctx.JSON(http.StatusInternalServerError, types.ErrResponse{ + Code: http.StatusInternalServerError, + Message: fmt.Sprintf("多线程流初始化失败: %v", err), + }) + + return + } + + for k, v := range streamer.GetResponseHeader() { + ctx.Header(k, v[0]) + } + + ctx.Status(streamer.HTTPCode()) + + if err = streamer.Transfer(ctx, ctx.Writer); err != nil { + if s.isConnectionError(err) { + s.logger.Info("客户端连接断开", zap.String("url", url)) + } else { + s.logger.Error("多线程流文件传输失败", zap.Error(err), zap.String("url", url)) + } + } +} + func (s *service) handleLocalProxy(ctx *gin.Context, url string) { start := time.Now() ctx.Header("X-Transfer-Type", "local_proxy") - // 使用请求的上下文，支持取消 req, err := http.NewRequestWithContext(ctx.Request.Context(), http.MethodGet, url, nil) if err != nil { - s.logger.Error("创建代理请求失败", zap.Error(err)) - ctx.JSON(http.StatusInternalServerError, gin.H{ - "code": http.StatusInternalServerError, - "message": err.Error(), + s.logger.Error("创建本地代理请求失败", zap.Error(err), zap.String("url", url)) + ctx.JSON(http.StatusInternalServerError, types.ErrResponse{ + Code: http.StatusInternalServerError, + Message: err.Error(), }) + return } - // 智能复制请求头 s.copyOptimizedHeaders(ctx.Request.Header, req.Header) - // 检查是否是Range请求 rangeHeader := ctx.Request.Header.Get("Range") isRangeRequest := rangeHeader != "" if isRangeRequest { - // Range请求使用更激进的优化 req.Header.Set("Connection", "keep-alive") } - // 发送请求 resp, err := globalHTTPClient.Do(req) if err != nil { - // 检查是否是上下文取消（客户端断开） if ctx.Request.Context().Err() != nil { s.logger.Info("客户端断开连接", zap.String("url", url)) + return } - s.logger.Error("代理请求失败", zap.Error(err), zap.String("url", url)) - ctx.JSON(http.StatusInternalServerError, gin.H{ - "code": http.StatusInternalServerError, - "message": fmt.Sprintf("代理请求失败: %v", err), + s.logger.Error("本地代理请求失败", zap.Error(err), zap.String("url", url)) + ctx.JSON(http.StatusInternalServerError, types.ErrResponse{ + Code: http.StatusInternalServerError, + Message: fmt.Sprintf("本地代理请求失败: %v", err), }) + return } defer resp.Body.Close() - // 优化响应头复制 s.copyOptimizedResponseHeaders(resp.Header, ctx) - ctx.Status(resp.StatusCode) - // 立即刷新响应头 if flusher, ok := ctx.Writer.(http.Flusher); ok { flusher.Flush() } - // 根据请求类型选择不同的传输策略 var copyErr error - if isRangeRequest || resp.ContentLength < 1024*1024 { // 小于1MB - // 小文件或Range请求：快速传输 + if isRangeRequest || resp.ContentLength < 1024*1024 { copyErr = s.fastCopy(ctx, ctx.Writer, resp.Body) } else { - // 大文件：流式传输 copyErr = s.streamCopy(ctx, ctx.Writer, resp.Body) } - // 性能监控 duration := time.Since(start) if copyErr != nil { if s.isConnectionError(copyErr) { s.logger.Info("客户端连接断开", zap.String("url", url), - zap.Duration("duration", duration), - ) + zap.Duration("duration", duration)) } else { - s.logger.Error("文件下载转发失败", + s.logger.Error("本地代理文件传输失败", zap.Error(copyErr), zap.String("url", url), - zap.Duration("duration", duration), - ) + zap.Duration("duration", duration)) } } else { - s.logger.Info("代理请求完成", + s.logger.Info("本地代理请求完成", zap.String("url", url), zap.Duration("duration", duration), - zap.String("user_agent", ctx.Request.UserAgent()), - ) + zap.String("user_agent", ctx.Request.UserAgent())) - // 如果超过5秒，记录警告 if duration > 5*time.Second { - s.logger.Warn("代理请求较慢", + s.logger.Warn("本地代理请求响应较慢", zap.String("url", url), - zap.Duration("duration", duration), - ) + zap.Duration("duration", duration)) } } } -// 优化的请求头复制 func (s *service) copyOptimizedHeaders(src, dst http.Header) { - // 只复制必要的头 importantHeaders := []string{ "Range", "If-Range", "If-Modified-Since", "If-None-Match", "User-Agent", "Accept", "Accept-Encoding", "Authorization", @@ -296,9 +391,7 @@ func (s *service) copyOptimizedHeaders(src, dst http.Header) { } } -// 优化的响应头复制 func (s *service) copyOptimizedResponseHeaders(src http.Header, ctx *gin.Context) { - // 重要的响应头 importantHeaders := []string{ "Content-Type", "Content-Length", "Content-Range", "Accept-Ranges", "Last-Modified", "ETag", "Cache-Control", @@ -311,15 +404,12 @@ func (s *service) copyOptimizedResponseHeaders(src http.Header, ctx *gin.Context } } - // 添加性能优化头 ctx.Header("Connection", "keep-alive") ctx.Header("Keep-Alive", "timeout=120, max=100") } -// 快速复制（小文件） func (s *service) fastCopy(ctx *gin.Context, dst io.Writer, src io.Reader) error { - // 使用更大的缓冲区一次性读取 - buf := make([]byte, 256*1024) // 256KB + buf := make([]byte, 256*1024) _, err := io.CopyBuffer(dst, src, buf) if flusher, ok := dst.(http.Flusher); ok { @@ -329,9 +419,7 @@ func (s *service) fastCopy(ctx *gin.Context, dst io.Writer, src io.Reader) error return err } -// 优化的流式复制 func (s *service) streamCopy(ctx *gin.Context, dst io.Writer, src io.Reader) error { - // 使用更大的缓冲区 (128KB) buf := make([]byte, 128*1024) flusher, canFlush := dst.(http.Flusher) @@ -339,14 +427,12 @@ func (s *service) streamCopy(ctx *gin.Context, dst io.Writer, src io.Reader) err flushInterval := 0 for { - // 检查连接状态 select { case <-ctx.Request.Context().Done(): return ctx.Request.Context().Err() default: } - // 设置读取超时 if conn, ok := src.(interface{ SetReadDeadline(time.Time) error }); ok { conn.SetReadDeadline(time.Now().Add(30 * time.Second)) } @@ -364,7 +450,6 @@ func (s *service) streamCopy(ctx *gin.Context, dst io.Writer, src io.Reader) err written += int64(nw) flushInterval++ - // 每16KB或每10次写入就刷新一次 if canFlush && (flushInterval%10 == 0 || written%16384 == 0) { flusher.Flush() } @@ -378,7 +463,6 @@ func (s *service) streamCopy(ctx *gin.Context, dst io.Writer, src io.Reader) err } } - // 最终刷新 if canFlush { flusher.Flush() } @@ -386,7 +470,6 @@ func (s *service) streamCopy(ctx *gin.Context, dst io.Writer, src io.Reader) err return nil } -// 检查是否是连接相关的错误 func (s *service) isConnectionError(err error) bool { if err == nil { return false @@ -415,24 +498,18 @@ func (s *service) isConnectionError(err error) bool { return false } -// ConnectionMonitorMiddleware 连接监控中间件 func (s *service) ConnectionMonitorMiddleware() gin.HandlerFunc { return func(ctx *gin.Context) { - // 创建一个可以取消的上下文 ctxWithCancel, cancel := context.WithCancel(ctx.Request.Context()) defer cancel() - // 替换请求上下文 ctx.Request = ctx.Request.WithContext(ctxWithCancel) - // 使用 recover 捕获 panic defer func() { if r := recover(); r != nil { if s.isConnectionError(fmt.Errorf("%v", r)) { - // 连接断开导致的 panic，记录日志但不报错 s.logger.Info("连接断开导致的异常", zap.Any("error", r)) } else { - // 其他 panic 正常处理 s.logger.Error("处理请求时发生异常", zap.Any("error", r)) if !ctx.Writer.Written() { ctx.AbortWithStatus(http.StatusInternalServerError) @@ -445,76 +522,131 @@ func (s *service) ConnectionMonitorMiddleware() gin.HandlerFunc { } } -type DoResult struct { - URL string - HttpCode int - Err error -} - func (s *service) getFileDownloadURL(ctx context.Context, id int64) (string, int, error) { var file = new(models.VirtualFile) if err := s.db.WithContext(ctx).Where("id", id).First(file).Error; err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { - return "", http.StatusNotFound, errors.New("file not found") + s.logger.Warn("获取下载链接时文件未找到", zap.Int64("fileId", id)) + + return "", http.StatusNotFound, errors.New("文件未找到") } + s.logger.Error("查询文件信息失败", zap.Int64("fileId", id), zap.Error(err)) + return "", http.StatusInternalServerError, err } - var cloudTokenId int64 - - var scanFile = file + if file.OsType == models.OsTypeStrmFile { + s.logger.Info("生成STRM文件下载链接", + zap.Int64("fileId", id), + zap.Int64("linkId", file.LinkId)) - for { - if v, ok := scanFile.Addition["cloud_token"]; ok { - cloudTokenId, _ = utils.Int64(v) - - break - } - - if scanFile.ParentId == 0 { - return "", http.StatusBadRequest, errors.New("当前资源没有绑定用于获取播放链接的令牌") - } - - var parent = new(models.VirtualFile) - if err := s.db.WithContext(ctx).Where("id", scanFile.ParentId).First(parent).Error; err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return "", http.StatusNotFound, errors.New("file not found") - } - - return "", http.StatusBadRequest, errors.New("当前资源没有绑定用于获取播放链接的令牌") - } + return s.generateDownloadURLWithNeverExpire(file.LinkId), http.StatusOK, nil + } - scanFile = parent + cloudTokenId, err := s.findCloudTokenId(ctx, file) + if err != nil { + return "", http.StatusBadRequest, err } - var ct = new(models.CloudToken) + ct := new(models.CloudToken) if err := s.db.WithContext(ctx).Where("id", cloudTokenId).First(ct).Error; err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { + s.logger.Error("云盘令牌未找到", + zap.Int64("fileId", id), + zap.Int64("cloudTokenId", cloudTokenId)) + return "", http.StatusBadRequest, errors.New("绑定的令牌查找失败，可能被删除或隐藏") } + s.logger.Error("查询云盘令牌失败", + zap.Int64("fileId", id), + zap.Int64("cloudTokenId", cloudTokenId), + zap.Error(err)) + return "", http.StatusInternalServerError, err } - fileId := utils.String(file.Addition["file_id"]) + fileId := utils.String(file.Addition[consts.FileAdditionKeyFileId]) + s.logger.Info("开始获取云盘文件下载链接", + zap.Int64("fileId", id), + zap.String("cloudFileId", fileId)) - result, err := client.New().WithToken(client.NewAuthToken(ct.AccessToken, ct.ExpiresIn)).GetFileDownload(ctx, client.String(fileId), func(req *client.GetFileDownloadRequest) { - if v, ok := file.Addition["share_id"]; ok { - req.ShareId, _ = utils.Int64(v) - } - }) + result, err := client.New().WithToken(client.NewAuthToken(ct.AccessToken, ct.ExpiresIn)). + GetFileDownload(ctx, client.String(fileId), func(req *client.GetFileDownloadRequest) { + if v, ok := file.Addition[consts.FileAdditionKeyShareId]; ok { + req.ShareId, _ = utils.Int64(v) + } + }) if err != nil { + s.logger.Error("获取云盘文件下载链接失败", + zap.Int64("fileId", id), + zap.String("cloudFileId", fileId), + zap.Error(err)) + return "", http.StatusInternalServerError, err } resp, err := http.Get(result.FileDownloadUrl) if err != nil { + s.logger.Error("请求云盘下载链接失败", + zap.Int64("fileId", id), + zap.String("downloadUrl", result.FileDownloadUrl), + zap.Error(err)) + return "", http.StatusInternalServerError, err } + defer resp.Body.Close() + + finalUrl := resp.Request.URL.String() + s.cache.Set(fmt.Sprintf("file::url::%d", file.ID), finalUrl, time.Minute) + + s.logger.Info("成功获取文件下载链接", + zap.Int64("fileId", id), + zap.String("finalUrl", finalUrl)) + + return finalUrl, http.StatusFound, nil +} + +func (s *service) findCloudTokenId(ctx context.Context, file *models.VirtualFile) (int64, error) { + var cloudTokenId int64 + scanFile := file - s.cache.Set(fmt.Sprintf("file::url::%d", file.ID), resp.Request.URL.String(), time.Minute) + for { + if v, ok := scanFile.Addition[consts.FileAdditionKeyCloudToken]; ok { + cloudTokenId, _ = utils.Int64(v) + break + } + + if scanFile.ParentId == 0 { + s.logger.Error("文件未绑定云盘令牌", + zap.Int64("fileId", file.ID), + zap.String("fileName", file.Name)) + + return 0, errors.New("当前资源没有绑定用于获取播放链接的令牌") + } + + var parent = new(models.VirtualFile) + if err := s.db.WithContext(ctx).Where("id", scanFile.ParentId).First(parent).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + s.logger.Error("查找父级文件失败", + zap.Int64("fileId", file.ID), + zap.Int64("parentId", scanFile.ParentId)) + + return 0, errors.New("文件未找到") + } + + s.logger.Error("查询父级文件信息失败", + zap.Int64("fileId", file.ID), + zap.Int64("parentId", scanFile.ParentId), + zap.Error(err)) + + return 0, errors.New("当前资源没有绑定用于获取播放链接的令牌") + } + + scanFile = parent + } - return resp.Request.URL.String(), http.StatusFound, nil + return cloudTokenId, nil } diff --git a/internal/services/universalfs/service_open.go b/internal/services/universalfs/service_open.go index da59322f..fb5496d0 100644 --- a/internal/services/universalfs/service_open.go +++ b/internal/services/universalfs/service_open.go @@ -1,11 +1,7 @@ package universalfs import ( - "encoding/json" - "errors" "fmt" - "github.com/samber/lo" - "go.uber.org/zap" "net/http" "net/url" "path" @@ -16,108 +12,70 @@ import ( mapset "github.com/deckarep/golang-set/v2" "github.com/gin-gonic/gin" "github.com/google/uuid" + "github.com/samber/lo" + "go.uber.org/zap" + + "github.com/xxcheng123/cloudpan189-share/internal/consts" "github.com/xxcheng123/cloudpan189-share/internal/models" "github.com/xxcheng123/cloudpan189-share/internal/pkgs/utils" "github.com/xxcheng123/cloudpan189-share/internal/shared" - "gorm.io/gorm" + "github.com/xxcheng123/cloudpan189-share/internal/types" ) +type openRequest struct { + IncludeAutoGenerateStrmFile bool `form:"includeAutoGenerateStrmFile"` +} + func (s *service) Open(prefix string, format string) gin.HandlerFunc { return func(ctx *gin.Context) { - rawPath := ctx.Param("path") + var req = new(openRequest) + if err := ctx.ShouldBindQuery(req); err != nil { + s.logger.Warn("文件浏览请求参数错误", zap.Error(err)) - paths, err := utils.SplitPath(rawPath) - if err != nil { - ctx.JSON(http.StatusBadRequest, gin.H{ - "code": http.StatusBadRequest, - "message": err.Error(), + ctx.JSON(http.StatusBadRequest, types.ErrResponse{ + Code: http.StatusBadRequest, + Message: err.Error(), }) return } - var pid int64 - var fullPaths = make([]string, 0) - var session = new(ReadSession) - var file = new(models.VirtualFile) - var ( - groupId = ctx.GetInt64("group_id") - groupFileSet = mapset.NewSet[int64]() + fid = ctx.GetInt64(consts.CtxKeyFileId) + pid = ctx.GetInt64(consts.CtxKeyParentId) + gid = ctx.GetInt64(consts.CtxKeyGroupId) ) - if groupId != 0 { - groupFiles := make([]*models.Group2File, 0) - - if err = s.db.WithContext(ctx).Model(new(models.Group2File)).Where("group_id", groupId).Find(&groupFiles).Error; err != nil { - s.logger.Error("get group files failure", zap.Error(err)) - - ctx.JSON(http.StatusInternalServerError, gin.H{ - "code": http.StatusInternalServerError, - "message": "获取用户组文件关系失败", - }) - - return - } + file, err := s.getFileInfo(ctx, fid, pid) + if err != nil { + s.logger.Error("获取文件信息失败", + zap.Int64("fileId", fid), + zap.Int64("parentId", pid), + zap.Error(err)) + ctx.JSON(http.StatusBadRequest, types.ErrResponse{ + Code: http.StatusBadRequest, + Message: err.Error(), + }) - for _, groupFile := range groupFiles { - groupFileSet.Add(groupFile.FileId) - } + return } - if len(paths) == 0 { - file = &models.VirtualFile{ - ID: 0, - ParentId: -1, - Name: "root", - OsType: models.OsTypeFolder, - CreateDate: s.startTime.Format(time.DateTime), - ModifyDate: s.startTime.Format(time.DateTime), - CreatedAt: s.startTime, - UpdatedAt: s.startTime, - Rev: s.startTime.Format("20060102150405"), - IsFolder: 1, - Addition: map[string]interface{}{}, - } - } else { - for _, p := range paths { - var tmpFile = new(models.VirtualFile) - if err = s.db.WithContext(ctx).Where("parent_id", pid).Where("name", p).First(tmpFile).Error; err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - ctx.JSON(http.StatusNotFound, gin.H{ - "code": http.StatusNotFound, - "message": "file not found", - }) - - return - } - - ctx.JSON(http.StatusBadRequest, gin.H{ - "code": http.StatusBadRequest, - "message": err.Error(), - }) - - return - } else { - if v, ok := tmpFile.Addition["cloud_token"]; ok { - session.CloudTokenID, _ = v.(json.Number).Int64() - } - } - - if tmpFile.IsTop == 1 && groupId != 0 && !groupFileSet.Contains(tmpFile.ID) { - // 没有权限 - ctx.JSON(http.StatusForbidden, gin.H{ - "code": http.StatusForbidden, - "message": "no permission", - }) - - return - } - - fullPaths = append(fullPaths, p) - pid = tmpFile.ID - file = tmpFile - } + var ( + vPath, _ = ctx.Get(consts.CtxKeyFullPaths) + fullPaths = utils.StringSlice(vPath) + vGroupFileSet, _ = ctx.Get(consts.CtxKeyGroupFileSet) + ) + + groupFileSet, ok := vGroupFileSet.(mapset.Set[int64]) + if !ok { + s.logger.Error("获取用户组文件集合失败", + zap.Int64("fileId", fid), + zap.Int64("parentId", pid)) + ctx.JSON(http.StatusInternalServerError, types.ErrResponse{ + Code: http.StatusInternalServerError, + Message: "获取用户组文件集合失败", + }) + return } f := &FileInfo{ @@ -127,64 +85,152 @@ func (s *service) Open(prefix string, format string) gin.HandlerFunc { } if file.IsFolder == 1 { - var list = make([]*models.VirtualFile, 0) - if err = s.db.WithContext(ctx).Where("parent_id", file.ID).Find(&list).Error; err != nil { - ctx.JSON(http.StatusBadRequest, gin.H{ - "code": http.StatusBadRequest, - "message": err.Error(), + if err := s.loadFolderChildren(ctx, f, gid, groupFileSet, format, req); err != nil { + s.logger.Error("加载文件夹子项失败", + zap.Int64("fileId", file.ID), + zap.String("fileName", file.Name), + zap.Error(err)) + ctx.JSON(http.StatusBadRequest, types.ErrResponse{ + Code: http.StatusBadRequest, + Message: err.Error(), }) return } + } else { + f.DownloadURL = s.generateDownloadURL(file.ID) + } - for _, v := range list { - f.Children = append(f.Children, &FileInfo{ - VirtualFile: v, - Path: path.Join(f.Path, v.Name), - Href: utils.PathEscape(f.Path, v.Name), - }) - } + s.responseByFormat(ctx, f, format) + } +} - if groupId != 0 { - f.Children = lo.Filter(f.Children, func(item *FileInfo, _ int) bool { - if item.IsTop == 1 && !groupFileSet.Contains(item.ID) { - return false - } +func (s *service) getFileInfo(ctx *gin.Context, fid, pid int64) (*models.VirtualFile, error) { + if pid == -1 && fid == 0 { + // 根目录特殊处理 + return &models.VirtualFile{ + ID: 0, + ParentId: -1, + Name: "root", + OsType: models.OsTypeFolder, + CreateDate: s.startTime.Format(time.DateTime), + ModifyDate: s.startTime.Format(time.DateTime), + CreatedAt: s.startTime, + UpdatedAt: s.startTime, + Rev: s.startTime.Format("20060102150405"), + IsFolder: 1, + Addition: map[string]interface{}{}, + }, nil + } - return true - }) - } + file := new(models.VirtualFile) + if err := s.db.WithContext(ctx).Where("id", fid).First(&file).Error; err != nil { + return nil, err + } - sort.Slice(f.Children, func(i, j int) bool { - if f.Children[i].IsFolder != f.Children[j].IsFolder { - return f.Children[i].IsFolder > f.Children[j].IsFolder - } + return file, nil +} - return f.Children[i].Rev > f.Children[j].Rev - }) - } else { - values := enc(url.Values{ - "id": []string{fmt.Sprintf("%d", file.ID)}, - "random": []string{uuid.NewString()}, - }, shared.Setting.SaltKey) - - baseURL := shared.Setting.BaseURL - if baseURL == "" { - scheme := "http" - if ctx.Request.TLS != nil { - scheme = "https" - } - baseURL = fmt.Sprintf("%s://%s", scheme, ctx.Request.Host) +func (s *service) loadFolderChildren(ctx *gin.Context, f *FileInfo, gid int64, groupFileSet mapset.Set[int64], format string, req *openRequest) error { + var list = make([]*models.VirtualFile, 0) + if err := s.db.WithContext(ctx).Where("parent_id", f.ID).Find(&list).Error; err != nil { + return err + } + + // 构建子项列表 + for _, v := range list { + f.Children = append(f.Children, &FileInfo{ + VirtualFile: v, + Path: path.Join(f.Path, v.Name), + Href: utils.PathEscape(f.Path, v.Name), + }) + } + + // 应用权限过滤 + if gid != 0 { + f.Children = lo.Filter(f.Children, func(item *FileInfo, _ int) bool { + if item.IsTop == 1 && !groupFileSet.Contains(item.ID) { + return false } - f.DownloadURL = fmt.Sprintf("%s/api/file_download?%s", baseURL, values.Encode()) + return true + }) + } + + // 根据格式过滤STRM文件 + s.filterByFormat(f, format, req) + + // 排序：文件夹优先，然后按修改时间倒序 + sort.Slice(f.Children, func(i, j int) bool { + if f.Children[i].IsFolder != f.Children[j].IsFolder { + return f.Children[i].IsFolder > f.Children[j].IsFolder } - switch format { - case "dav": - s.responseDav(ctx, f) - default: - ctx.JSON(http.StatusOK, f) + return f.Children[i].Rev > f.Children[j].Rev + }) + + return nil +} + +func (s *service) filterByFormat(f *FileInfo, format string, req *openRequest) { + switch format { + case "dav": + // DAV格式：过滤掉STRM文件 + f.Children = lo.Filter(f.Children, func(item *FileInfo, _ int) bool { + return item.OsType != models.OsTypeStrmFile + }) + case "strm_dav": + // STRM DAV格式：过滤掉被STRM文件链接的原文件 + var linkIds = make([]int64, 0) + for _, item := range f.Children { + if item.OsType == models.OsTypeStrmFile { + linkIds = append(linkIds, item.LinkId) + } } + + f.Children = lo.Filter(f.Children, func(item *FileInfo, _ int) bool { + return lo.IndexOf(linkIds, item.ID) == -1 + }) + case "json": + // JSON格式：根据参数决定是否包含自动生成的STRM文件 + f.Children = lo.Filter(f.Children, func(item *FileInfo, _ int) bool { + if !req.IncludeAutoGenerateStrmFile && item.OsType == models.OsTypeStrmFile { + return false + } + + return true + }) + } +} + +func (s *service) responseByFormat(ctx *gin.Context, f *FileInfo, format string) { + switch format { + case "dav", "strm_dav": + s.responseDav(ctx, f) + default: + ctx.JSON(http.StatusOK, f) } } + +func (s *service) generateDownloadURL(fid int64) string { + values := enc(url.Values{ + "id": []string{fmt.Sprintf("%d", fid)}, + "random": []string{uuid.NewString()}, + }, shared.Setting.SaltKey) + + baseURL := shared.Setting.BaseURL + + return fmt.Sprintf("%s/api/file_download?%s", baseURL, values.Encode()) +} + +func (s *service) generateDownloadURLWithNeverExpire(fid int64) string { + values := enc(url.Values{ + "id": []string{fmt.Sprintf("%d", fid)}, + "random": []string{uuid.NewString()}, + "timestamp": []string{"-1"}, + }, shared.Setting.SaltKey) + + baseURL := shared.Setting.BaseURL + + return fmt.Sprintf("%s/api/file_download?%s", baseURL, values.Encode()) +} diff --git a/internal/services/universalfs/service_put.go b/internal/services/universalfs/service_put.go new file mode 100644 index 00000000..ec338793 --- /dev/null +++ b/internal/services/universalfs/service_put.go @@ -0,0 +1,254 @@ +package universalfs + +import ( + "fmt" + "io" + "net/http" + "os" + "path" + "strconv" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "go.uber.org/zap" + "gorm.io/datatypes" + + "github.com/xxcheng123/cloudpan189-share/configs" + "github.com/xxcheng123/cloudpan189-share/internal/consts" + "github.com/xxcheng123/cloudpan189-share/internal/models" + "github.com/xxcheng123/cloudpan189-share/internal/types" +) + +func (s *service) Put() gin.HandlerFunc { + return func(ctx *gin.Context) { + returnCode, err := s.put(ctx) + if err != nil { + s.logger.Error("文件上传失败", + zap.String("path", ctx.Param("path")), + zap.Int("code", returnCode), + zap.Error(err)) + ctx.JSON(returnCode, types.ErrResponse{ + Code: returnCode, + Message: err.Error(), + }) + return + } + + ctx.Writer.WriteHeader(returnCode) + } +} + +func (s *service) put(ctx *gin.Context) (int, error) { + rawPath := ctx.Param("path") + now := time.Now() + + release, status, err := s.confirmLocks(rawPath, "") + if err != nil { + return status, err + } + defer release() + + var ( + pid = ctx.GetInt64(consts.CtxKeyParentId) + fid = ctx.GetInt64(consts.CtxKeyFileId) + filename = ctx.GetString(consts.CtxKeyFilename) + ) + + if err = s.ensureParentDirectory(pid); err != nil { + return s.handleDirectoryError(err) + } + + if fid == -1 { + return s.createNewFile(ctx, pid, filename, now) + } + + return s.updateExistingFile(ctx, fid, now) +} + +func (s *service) ensureParentDirectory(pid int64) error { + dirName := path.Join(configs.GetConfig().FileDir, strconv.FormatInt(pid, 10)) + if err := os.Mkdir(dirName, 0777); err != nil && !os.IsExist(err) { + s.logger.Error("创建父级目录失败", + zap.Int64("parentId", pid), + zap.String("dirName", dirName), + zap.Error(err)) + + return err + } + + return nil +} + +func (s *service) handleDirectoryError(err error) (int, error) { + if os.IsNotExist(err) { + return http.StatusConflict, err + } + + return http.StatusMethodNotAllowed, err +} + +func (s *service) createNewFile(ctx *gin.Context, pid int64, filename string, now time.Time) (int, error) { + writeFilename := uuid.NewString() + ".bin" + dirName := path.Join(configs.GetConfig().FileDir, strconv.FormatInt(pid, 10)) + realPath := path.Join(dirName, writeFilename) + + file, err := os.OpenFile(realPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + s.logger.Error("创建新文件失败", + zap.String("realPath", realPath), + zap.String("filename", filename), + zap.Error(err)) + + return s.handleFileError(err) + } + + fileInfo, err := s.writeFileContent(file, ctx.Request.Body) + if err != nil { + s.logger.Error("写入文件内容失败", + zap.String("filename", filename), + zap.Error(err)) + return http.StatusMethodNotAllowed, err + } + + if err := s.saveNewFileRecord(ctx, pid, filename, fileInfo.Size(), writeFilename, now); err != nil { + s.logger.Error("保存文件记录失败", + zap.String("filename", filename), + zap.Error(err)) + return http.StatusInternalServerError, err + } + + s.logger.Info("新文件创建成功", + zap.String("filename", filename), + zap.Int64("size", fileInfo.Size()), + zap.String("realPath", realPath)) + + return http.StatusCreated, nil +} + +func (s *service) updateExistingFile(ctx *gin.Context, fid int64, now time.Time) (int, error) { + file := &models.VirtualFile{} + if err := s.db.WithContext(ctx).Where("id=?", fid).First(file).Error; err != nil { + s.logger.Error("查询文件记录失败", zap.Int64("fileId", fid), zap.Error(err)) + + return http.StatusNotFound, err + } + + if file.OsType != models.OsTypeRealFile { + s.logger.Warn("尝试更新非真实文件", + zap.Int64("fileId", fid), + zap.String("fileName", file.Name), + zap.String("osType", file.OsType)) + + return http.StatusMethodNotAllowed, fmt.Errorf("文件类型不允许更新") + } + + realPath, err := s.getRealFilePath(file) + if err != nil { + s.logger.Error("获取文件真实路径失败", + zap.Int64("fileId", fid), + zap.String("fileName", file.Name), + zap.Error(err)) + + return http.StatusMethodNotAllowed, err + } + + fullPath := path.Join(configs.GetConfig().FileDir, realPath) + f, err := os.OpenFile(fullPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + s.logger.Error("打开文件失败", + zap.String("fullPath", fullPath), + zap.Error(err)) + + return s.handleFileError(err) + } + + fileInfo, err := s.writeFileContent(f, ctx.Request.Body) + if err != nil { + s.logger.Error("更新文件内容失败", + zap.Int64("fileId", fid), + zap.String("fileName", file.Name), + zap.Error(err)) + + return http.StatusMethodNotAllowed, err + } + + if err := s.updateFileRecord(ctx, fid, fileInfo.Size(), now); err != nil { + s.logger.Error("更新文件记录失败", + zap.Int64("fileId", fid), + zap.Error(err)) + + return http.StatusInternalServerError, err + } + + s.logger.Info("文件更新成功", + zap.Int64("fileId", fid), + zap.String("fileName", file.Name), + zap.Int64("size", fileInfo.Size())) + + return http.StatusCreated, nil +} + +func (s *service) handleFileError(err error) (int, error) { + if os.IsNotExist(err) { + return http.StatusConflict, err + } + return http.StatusNotFound, err +} + +func (s *service) writeFileContent(file *os.File, body io.Reader) (os.FileInfo, error) { + defer func() { + _ = file.Close() + }() + + _, copyErr := io.Copy(file, body) + if copyErr != nil { + return nil, copyErr + } + + fileInfo, statErr := file.Stat() + if statErr != nil { + return nil, statErr + } + + return fileInfo, nil +} + +func (s *service) getRealFilePath(file *models.VirtualFile) (string, error) { + vRealPath, ok := file.Addition[consts.FileAdditionKeyFilePath] + if !ok { + return "", fmt.Errorf("文件路径不存在") + } + + realPath, ok := vRealPath.(string) + if !ok { + return "", fmt.Errorf("文件路径格式错误") + } + + return realPath, nil +} + +func (s *service) saveNewFileRecord(ctx *gin.Context, pid int64, filename string, size int64, writeFilename string, now time.Time) error { + return s.db.WithContext(ctx).Create(&models.VirtualFile{ + ParentId: pid, + Name: filename, + IsTop: 0, + Size: size, + Hash: "-", + OsType: models.OsTypeRealFile, + Addition: datatypes.JSONMap{ + consts.FileAdditionKeyFilePath: fmt.Sprintf("%d/%s", pid, writeFilename), + }, + Rev: now.Format("20060102150405"), + CreateDate: now.Format(time.DateTime), + ModifyDate: now.Format(time.DateTime), + }).Error +} + +func (s *service) updateFileRecord(ctx *gin.Context, fid int64, size int64, now time.Time) error { + return s.db.WithContext(ctx).Model(new(models.VirtualFile)).Where("id=?", fid).Updates(map[string]any{ + "size": size, + "modify_date": now.Format(time.DateTime), + "rev": now.Format("20060102150405"), + }).Error +} diff --git a/internal/services/universalfs/util.go b/internal/services/universalfs/util.go new file mode 100644 index 00000000..658d5a6b --- /dev/null +++ b/internal/services/universalfs/util.go @@ -0,0 +1,55 @@ +package universalfs + +import ( + "crypto/md5" + "encoding/hex" + "net/url" + "sort" + "strconv" + "strings" + "time" +) + +func enc(values url.Values, key string) url.Values { + if !values.Has("timestamp") { + timestamp := time.Now().Add(time.Hour * 6).Unix() + values.Set("timestamp", strconv.FormatInt(timestamp, 10)) + } + + // 排序并生成签名字符串 + keys := make([]string, 0, len(values)) + for k := range values { + keys = append(keys, k) + } + + sort.Strings(keys) + + // 构建待签名字符串 + var signStr strings.Builder + for _, k := range keys { + signStr.WriteString(k + "=" + values.Get(k) + "&") + } + signStr.WriteString("key=" + key) + + // 计算MD5 + h := md5.New() + h.Write([]byte(signStr.String())) + sign := hex.EncodeToString(h.Sum(nil)) + + values.Set("sign", sign) + return values +} + +// 验证签名是否有效 +func verify(values url.Values, key string) bool { + // 获取并移除签名 + sign := values.Get("sign") + values.Del("sign") + + // 重新计算签名 + newValues := enc(values, key) + newSign := newValues.Get("sign") + + // 比较签名是否一致 + return sign == newSign +} diff --git a/internal/services/user/service_auth.go b/internal/services/user/service_auth.go index 93ae05e4..37b05395 100644 --- a/internal/services/user/service_auth.go +++ b/internal/services/user/service_auth.go @@ -2,6 +2,7 @@ package user import ( "errors" + "github.com/xxcheng123/cloudpan189-share/internal/consts" "net/http" "strings" @@ -114,6 +115,7 @@ func (s *service) AuthMiddleware(permission uint8) gin.HandlerFunc { ctx.Set("username", username) ctx.Set("permissions", user.Permissions) ctx.Set("group_id", user.GroupID) + ctx.Set(consts.CtxKeyGroupId, user.GroupID) ctx.Next() } diff --git a/internal/shared/scan_job.go b/internal/shared/scan_job.go index a67796a8..0da6d280 100644 --- a/internal/shared/scan_job.go +++ b/internal/shared/scan_job.go @@ -22,7 +22,7 @@ var scanJobInstance = &scanJob{ var ( ErrScanJobFull = errors.New("执行任务队列已满，请稍后再试") ErrJobConflict = errors.New("当前任务或与已存在任务冲突") - ErrJobNotFound = fmt.Errorf("") + ErrJobNotFound = errors.New("任务未找到") ErrInvalidStatus = fmt.Errorf("invalid job status transition") ) @@ -32,6 +32,9 @@ const ( ScanJobTypeDel ScanJobType = "del" ScanJobTypeRefresh ScanJobType = "refresh" // 添加时或普通刷新时调用 ScanJobTypeDeepRefresh ScanJobType = "deep_refresh" // 递归刷新时调用 rev 相同也继续扫描 + ScanJobRebuildStrm ScanJobType = "rebuild_strm" + ScanJobClearStrm ScanJobType = "clear_strm" + ScanJobClearRealFile ScanJobType = "clear_real_file" ) type ScanMsg struct { diff --git a/internal/shared/setting.go b/internal/shared/setting.go index 2a89d776..97654bbb 100644 --- a/internal/shared/setting.go +++ b/internal/shared/setting.go @@ -5,6 +5,9 @@ import "github.com/xxcheng123/cloudpan189-share/internal/models" var Setting = &models.Setting{} var ( - MultipleStreamThreadCount int = models.DefaultMultipleStreamThreadCount - MultipleStreamChunkSize int64 = models.DefaultMultipleStreamChunkSize + MultipleStreamThreadCount int = models.DefaultMultipleStreamThreadCount + MultipleStreamChunkSize int64 = models.DefaultMultipleStreamChunkSize + StrmFileEnable bool = models.DefaultStrmFileEnable + StrmSupportFileExtList []string = models.DefaultStrmSupportFileExtList + FileWritable bool = models.DefaultFileWritable ) diff --git a/internal/types/resp.go b/internal/types/resp.go new file mode 100644 index 00000000..7aff48e8 --- /dev/null +++ b/internal/types/resp.go @@ -0,0 +1,6 @@ +package types + +type ErrResponse struct { + Code int `json:"code"` + Message string `json:"message"` +}