From 43d66c7a36ca4c2ccb270072b8d04ec1cd02bcd9 Mon Sep 17 00:00:00 2001 From: Jonathan Jogenfors Date: Thu, 6 Feb 2025 14:58:18 +0100 Subject: [PATCH] ignore perm errors --- e2e/docker-compose.yml | 3 + e2e/src/api/specs/library.e2e-spec.ts | 30 ++- server/package-lock.json | 190 +++++++++++++++--- server/package.json | 2 +- server/src/repositories/storage.repository.ts | 15 ++ 5 files changed, 210 insertions(+), 30 deletions(-) diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index c925f6b3d07bd..953be64dc3202 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -32,6 +32,9 @@ services: - database ports: - 2285:2285 + cap_drop: + # We need this to perform testing on permission errors + - DAC_OVERRIDE redis: image: redis:6.2-alpine@sha256:905c4ee67b8e0aa955331960d2aa745781e6bd89afc44a8584bfd13bc890f0ae diff --git a/e2e/src/api/specs/library.e2e-spec.ts b/e2e/src/api/specs/library.e2e-spec.ts index e2e69529fbb9d..a081a132ae00c 100644 --- a/e2e/src/api/specs/library.e2e-spec.ts +++ b/e2e/src/api/specs/library.e2e-spec.ts @@ -1,5 +1,5 @@ import { LibraryResponseDto, LoginResponseDto, getAllLibraries, scanLibrary } from '@immich/sdk'; -import { cpSync, existsSync, rmSync, unlinkSync } from 'node:fs'; +import { chmodSync, cpSync, existsSync, promises, rmSync, unlinkSync } from 'node:fs'; import { Socket } from 'socket.io-client'; import { userDto, uuidDto } from 'src/fixtures'; import { errorDto } from 'src/responses'; @@ -492,6 +492,34 @@ describe('/libraries', () => { utils.removeImageFile(`${testAssetDir}/temp/folder${char}2/asset2.png`); }); + it('should handle permission errors on import paths without error', async () => { + const library = await utils.createLibrary(admin.accessToken, { + ownerId: admin.userId, + importPaths: [`${testAssetDirInternal}/temp/`], + }); + + const stat = await promises.stat(`${testAssetDir}/temp/directoryA`); + const mode = stat.mode; + + chmodSync(`${testAssetDir}/temp/directoryB`, 0o000); + + const { status } = await request(app) + .post(`/libraries/${library.id}/scan`) + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + expect(status).toBe(204); + + await utils.waitForQueueFinish(admin.accessToken, 'library'); + + chmodSync(`${testAssetDir}/temp/directoryB`, mode); + + const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id }); + + expect(assets.count).toBe(1); + expect(assets.items.find((asset) => asset.originalPath.includes('directoryA'))).toBeDefined(); + expect(assets.items.find((asset) => asset.originalPath.includes('directoryB'))).not.toBeDefined(); + }); + it('should reimport a modified file', async () => { const library = await utils.createLibrary(admin.accessToken, { ownerId: admin.userId, diff --git a/server/package-lock.json b/server/package-lock.json index c1fe902dfa0b1..274016778f41b 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -34,7 +34,7 @@ "class-validator": "^0.14.0", "cookie-parser": "^1.4.6", "exiftool-vendored": "^28.3.1", - "fast-glob": "^3.3.2", + "fast-glob": "github:etnoy/fast-glob#built", "fluent-ffmpeg": "^2.1.2", "geo-tz": "^8.0.0", "handlebars": "^4.7.8", @@ -2466,38 +2466,38 @@ } }, "node_modules/@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-4.0.1.tgz", + "integrity": "sha512-vAkI715yhnmiPupY+dq+xenu5Tdf2TBQ66jLvBIcCddtz+5Q8LbMKaf9CIJJreez8fQ8fgaY+RaywQx8RJIWpw==", "license": "MIT", "dependencies": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" + "@nodelib/fs.stat": "4.0.0", + "run-parallel": "^1.2.0" }, "engines": { - "node": ">= 8" + "node": ">=18.18.0" } }, "node_modules/@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-4.0.0.tgz", + "integrity": "sha512-ctr6bByzksKRCV0bavi8WoQevU6plSp2IkllIsEqaiKe2mwNNnaluhnRhcsgGZHrrHk57B3lf95MkLMO3STYcg==", "license": "MIT", "engines": { - "node": ">= 8" + "node": ">=18.18.0" } }, "node_modules/@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-3.0.1.tgz", + "integrity": "sha512-nIh/M6Kh3ZtOmlY00DaUYB4xeeV6F3/ts1l29iwl3/cfyY/OuCfUx+v08zgx8TKPTifXRcjjqVQ4KB2zOYSbyw==", "license": "MIT", "dependencies": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" + "@nodelib/fs.scandir": "4.0.1", + "fastq": "^1.15.0" }, "engines": { - "node": ">= 8" + "node": ">=18.18.0" } }, "node_modules/@nuxtjs/opencollective": { @@ -5559,6 +5559,44 @@ "typescript": ">=4.8.4 <5.8.0" } }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", @@ -5569,6 +5607,23 @@ "balanced-match": "^1.0.0" } }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", @@ -8701,19 +8756,30 @@ "license": "MIT" }, "node_modules/fast-glob": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", - "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "version": "4.0.0", + "resolved": "git+ssh://git@github.com/etnoy/fast-glob.git#7d0d2f912eeadcbc3e4af55249cbbcabe8d0808b", "license": "MIT", "dependencies": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", + "@nodelib/fs.stat": "^4.0.0", + "@nodelib/fs.walk": "^3.0.0", + "glob-parent": "^6.0.2", + "merge2": "^1.4.1", "micromatch": "^4.0.8" }, "engines": { - "node": ">=8.6.0" + "node": ">=18.18.0" + } + }, + "node_modules/fast-glob/node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" } }, "node_modules/fast-json-stable-stringify": { @@ -8744,9 +8810,9 @@ "license": "BSD-3-Clause" }, "node_modules/fastq": { - "version": "1.17.1", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", - "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.0.tgz", + "integrity": "sha512-7SFSRCNjBQIZH/xZR3iy5iQYR8aGBE0h3VG6/cwlbrpdciNYBMotQav8c1XI3HjHH+NikUpP53nPdlZSdWmFzA==", "license": "ISC", "dependencies": { "reusify": "^1.0.4" @@ -14287,6 +14353,74 @@ "tailwindcss": ">=3.4.17" } }, + "node_modules/tailwindcss/node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "license": "MIT", + "peer": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/tailwindcss/node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/tailwindcss/node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/tailwindcss/node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/tailwindcss/node_modules/fast-glob/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "license": "ISC", + "peer": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/tailwindcss/node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", diff --git a/server/package.json b/server/package.json index 7bbc9a2f239df..dba1be1b0476c 100644 --- a/server/package.json +++ b/server/package.json @@ -60,7 +60,7 @@ "class-validator": "^0.14.0", "cookie-parser": "^1.4.6", "exiftool-vendored": "^28.3.1", - "fast-glob": "^3.3.2", + "fast-glob": "github:etnoy/fast-glob#built", "fluent-ffmpeg": "^2.1.2", "geo-tz": "^8.0.0", "handlebars": "^4.7.8", diff --git a/server/src/repositories/storage.repository.ts b/server/src/repositories/storage.repository.ts index 6766f442b88cc..db015fdc47a15 100644 --- a/server/src/repositories/storage.repository.ts +++ b/server/src/repositories/storage.repository.ts @@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common'; import archiver from 'archiver'; import chokidar, { WatchOptions } from 'chokidar'; import { escapePath, glob, globStream } from 'fast-glob'; +import { ErrnoException } from 'fast-glob/out/types'; import { constants, createReadStream, createWriteStream, existsSync, mkdirSync } from 'node:fs'; import fs from 'node:fs/promises'; import path from 'node:path'; @@ -170,6 +171,19 @@ export class StorageRepository implements IStorageRepository { }); } + private errorHandler = (error: ErrnoException) => { + if (error.code === 'ENOENT') { + this.logger.warn(`Path ${error.path} does not exist, ignoring.`); + return true; + } else if (error.code === 'EACCES') { + this.logger.warn(`Permission denied for path ${error.path}, ignoring.`); + return true; + } + + this.logger.error(`Error while walking path ${error.path}: ${error.message}`); + return false; + }; + async *walk(walkOptions: WalkOptionsDto): AsyncGenerator { const { pathsToCrawl, exclusionPatterns, includeHidden } = walkOptions; if (pathsToCrawl.length === 0) { @@ -185,6 +199,7 @@ export class StorageRepository implements IStorageRepository { onlyFiles: true, dot: includeHidden, ignore: exclusionPatterns, + errorHandler: this.errorHandler, }); let batch: string[] = [];