feat: extended metadata downloader

README.md

@@ -185,9 +185,20 @@ Copy `sake/.env.example` to `sake/.env` and fill in the values you need.
 
 - `VITE_ALLOWED_HOSTS` - comma-separated host overrides for Vite/dev setups
 - `ACTIVATED_PROVIDERS` - comma-separated search providers
+- `ACTIVATED_METADATA_PROVIDERS` - comma-separated metadata providers, for example `googlebooks,openlibrary,hardcover`
+- `GOOGLE_BOOKS_API_KEY` - optional Google Books key for higher rate limits
+- `HARDCOVER_API_TOKEN` - optional server-wide token required for the Hardcover metadata provider
+- `METADATA_PROVIDER_TIMEOUT_MS` - optional metadata provider timeout in milliseconds
 - `BODY_SIZE_LIMIT` - upload/body size limit
 
 If `ACTIVATED_PROVIDERS` is unset, blank, or contains no valid values, search stays disabled and the search UI remains hidden.
+If `ACTIVATED_METADATA_PROVIDERS` is unset, blank, or contains no valid values, on-demand metadata lookup stays disabled and the metadata update UI remains hidden.
+
+Metadata provider notes:
+
+- `googlebooks` works without a key; `GOOGLE_BOOKS_API_KEY` only improves rate limits.
+- `openlibrary` works without a key.
+- `hardcover` is skipped unless `HARDCOVER_API_TOKEN` is set.
 
 Accepted provider names:
 
@@ -210,6 +221,10 @@ S3_SECRET_ACCESS_KEY=your-secret-access-key
 S3_FORCE_PATH_STYLE=false
 
 ACTIVATED_PROVIDERS=anna,openlib,gutenberg
+ACTIVATED_METADATA_PROVIDERS=googlebooks,openlibrary
+GOOGLE_BOOKS_API_KEY=
+HARDCOVER_API_TOKEN=
+METADATA_PROVIDER_TIMEOUT_MS=
 VITE_ALLOWED_HOSTS=
 BODY_SIZE_LIMIT=Infinity
 ```
@@ -228,6 +243,10 @@ S3_SECRET_ACCESS_KEY=sakeadminsecret
 S3_FORCE_PATH_STYLE=true
 
 ACTIVATED_PROVIDERS=anna,openlib,gutenberg
+ACTIVATED_METADATA_PROVIDERS=googlebooks,openlibrary
+GOOGLE_BOOKS_API_KEY=
+HARDCOVER_API_TOKEN=
+METADATA_PROVIDER_TIMEOUT_MS=
 VITE_ALLOWED_HOSTS=
 BODY_SIZE_LIMIT=Infinity
 ```

sake/.env.docker.selfhosted

@@ -12,4 +12,8 @@ S3_FORCE_PATH_STYLE=true
 VITE_ALLOWED_HOSTS=
 
 ACTIVATED_PROVIDERS=zlib,anna,openlib,gutenberg
+ACTIVATED_METADATA_PROVIDERS=
+GOOGLE_BOOKS_API_KEY=
+HARDCOVER_API_TOKEN=
+METADATA_PROVIDER_TIMEOUT_MS=
 BODY_SIZE_LIMIT=Infinity

sake/.env.example

@@ -13,4 +13,8 @@ S3_FORCE_PATH_STYLE=
 VITE_ALLOWED_HOSTS=
 
 ACTIVATED_PROVIDERS=
-BODY_SIZE_LIMIT=Infinity
+ACTIVATED_METADATA_PROVIDERS=
+GOOGLE_BOOKS_API_KEY=
+HARDCOVER_API_TOKEN=
+METADATA_PROVIDER_TIMEOUT_MS=
+BODY_SIZE_LIMIT=Infinity

sake/src/lib/server/application/composition.ts

@@ -1,3 +1,4 @@
+import { env } from '$env/dynamic/private';
 import { ZLibraryClient } from '$lib/server/infrastructure/clients/ZLibraryClient';
 import { S3Storage } from '$lib/server/infrastructure/storage/S3Storage';
 import { BookRepository } from '$lib/server/infrastructure/repositories/BookRepository';
@@ -86,8 +87,6 @@ import { getActivatedSearchProviders } from '$lib/server/config/activatedProvide
 import { SEARCH_PROVIDER_IDS } from '$lib/types/Search/Provider';
 import { MetadataAggregatorService } from '$lib/server/application/services/MetadataAggregatorService';
 import { ExternalBookMetadataService } from '$lib/server/application/services/ExternalBookMetadataService';
-import { GoogleBooksMetadataProvider } from '$lib/server/infrastructure/metadata-providers/googleBooksMetadataProvider';
-import { OpenLibraryMetadataProvider } from '$lib/server/infrastructure/metadata-providers/openLibraryMetadataProvider';
 import { createMetadataProviders } from '$lib/server/infrastructure/metadata-providers/metadataProviderFactory';
 import { getActivatedMetadataProviders } from '$lib/server/config/activatedMetadataProviders';
 import { SearchMetadataCandidatesUseCase } from '$lib/server/application/use-cases/SearchMetadataCandidatesUseCase';
@@ -120,17 +119,16 @@ export const deviceProgressDownloadRepository = new DeviceProgressDownloadReposi
 export const bookProgressHistoryRepository = new BookProgressHistoryRepository();
 export const managedBookCoverService = new ManagedBookCoverService(storage);
 
-export const baselineMetadataAggregator = new MetadataAggregatorService([
-	new GoogleBooksMetadataProvider(),
-	new OpenLibraryMetadataProvider()
-]);
+export const activatedMetadataProviders = createMetadataProviders(getActivatedMetadataProviders(), {
+	googleBooksApiKey: env.GOOGLE_BOOKS_API_KEY,
+	hardcoverApiToken: env.HARDCOVER_API_TOKEN,
+	isbnDbApiKey: env.ISBNDB_API_KEY
+});
+export const activatedMetadataAggregator = new MetadataAggregatorService(activatedMetadataProviders);
 export const externalBookMetadataService = new ExternalBookMetadataService(
-	baselineMetadataAggregator
+	activatedMetadataAggregator
 );
 
-export const activatedMetadataProviders = createMetadataProviders(getActivatedMetadataProviders());
-export const activatedMetadataAggregator = new MetadataAggregatorService(activatedMetadataProviders);
-
 export const downloadBookUseCase = new DownloadBookUseCase(
 	zlibraryClient,
 	bookRepository,
@@ -144,7 +142,9 @@ export const queueDownloadUseCase = new QueueDownloadUseCase(downloadQueue);
 export const queueSearchBookUseCase = new QueueSearchBookUseCase(downloadQueue);
 export const getQueueStatusUseCase = new GetQueueStatusUseCase(downloadQueue);
 export const zlibrarySearchUseCase = new ZLibrarySearchUseCase(zlibraryClient);
-export const lookupSearchBookMetadataUseCase = new LookupSearchBookMetadataUseCase();
+export const lookupSearchBookMetadataUseCase = new LookupSearchBookMetadataUseCase(
+	externalBookMetadataService
+);
 const activeSearchProviders = getActivatedSearchProviders();
 const searchProviderDependencies = { zlibrary: zlibraryClient };
 const activeSearchProviderInstances = createSearchProviders(
@@ -210,7 +210,9 @@ export const uploadLibraryBookCoverUseCase = new UploadLibraryBookCoverUseCase(
 export const putLibraryFileUseCase = new PutLibraryFileUseCase(
 	storage,
 	bookRepository,
-	managedBookCoverService
+	managedBookCoverService,
+	undefined,
+	externalBookMetadataService
 );
 export const exportDeviceLibraryBookUseCase = new ExportDeviceLibraryBookUseCase(
 	bookRepository,

sake/src/lib/server/application/services/ExternalBookMetadataService.ts

@@ -1,6 +1,6 @@
 import { MetadataAggregatorService } from '$lib/server/application/services/MetadataAggregatorService';
-import { GoogleBooksMetadataProvider } from '$lib/server/infrastructure/metadata-providers/googleBooksMetadataProvider';
-import { OpenLibraryMetadataProvider } from '$lib/server/infrastructure/metadata-providers/openLibraryMetadataProvider';
+import { sanitizeMetadataDescription } from '$lib/server/application/services/MetadataDescriptionSanitizer';
+import type { MetadataCandidate } from '$lib/server/application/ports/MetadataProviderPort';
 
 export interface ExternalBookMetadata {
 	googleBooksId: string | null;
@@ -49,16 +49,15 @@ function extractAmazonAsin(identifier: string | null): string | null {
 	return null;
 }
 
+function candidateDescription(candidate: MetadataCandidate): string | null {
+	return sanitizeMetadataDescription(candidate.description, candidate.descriptionFormat);
+}
+
 export class ExternalBookMetadataService {
 	private readonly aggregator: MetadataAggregatorService;
 
 	constructor(aggregator?: MetadataAggregatorService) {
-		this.aggregator =
-			aggregator ??
-			new MetadataAggregatorService([
-				new GoogleBooksMetadataProvider(),
-				new OpenLibraryMetadataProvider()
-			]);
+		this.aggregator = aggregator ?? new MetadataAggregatorService([]);
 	}
 
 	async lookup(input: ExternalBookMetadataLookupInput): Promise<ExternalBookMetadata> {
@@ -83,7 +82,7 @@ export class ExternalBookMetadataService {
 			openLibraryKey: olCandidate?.identifiers.openLibraryKey ?? null,
 			amazonAsin: extractAmazonAsin(input.identifier),
 			cover: bestCoverUrl,
-			description: pickFirst(...candidates.map((c) => c.description)),
+			description: pickFirst(...candidates.map(candidateDescription)),
 			publisher: pickFirst(...candidates.map((c) => c.publisher)),
 			series: pickFirst(...candidates.map((c) => c.series)),
 			volume: null,

sake/src/lib/server/application/services/ManagedBookCoverService.ts

@@ -21,6 +21,7 @@ const IMAGE_CONTENT_TYPE_TO_EXTENSION = new Map<string, string>([
 	['image/avif', 'avif']
 ]);
 
+export const MIN_MANAGED_BOOK_COVER_BYTES = 1024;
 export const MAX_MANAGED_BOOK_COVER_BYTES = 10 * 1024 * 1024;
 
 export interface ManagedBookCoverResult {
@@ -146,6 +147,48 @@ function extensionFromImageContentType(contentType: string | null): string | nul
 	return IMAGE_CONTENT_TYPE_TO_EXTENSION.get(contentType) ?? null;
 }
 
+function hasImageMagicBytes(buffer: Buffer, contentType: string): boolean {
+	switch (contentType) {
+		case 'image/jpeg':
+		case 'image/jpg':
+			return buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff;
+		case 'image/png':
+			return (
+				buffer.length >= 8 &&
+				buffer[0] === 0x89 &&
+				buffer[1] === 0x50 &&
+				buffer[2] === 0x4e &&
+				buffer[3] === 0x47 &&
+				buffer[4] === 0x0d &&
+				buffer[5] === 0x0a &&
+				buffer[6] === 0x1a &&
+				buffer[7] === 0x0a
+			);
+		case 'image/gif': {
+			if (buffer.length < 6) {
+				return false;
+			}
+			const signature = buffer.subarray(0, 6).toString('ascii');
+			return signature === 'GIF87a' || signature === 'GIF89a';
+		}
+		case 'image/webp':
+			return (
+				buffer.length >= 12 &&
+				buffer.subarray(0, 4).toString('ascii') === 'RIFF' &&
+				buffer.subarray(8, 12).toString('ascii') === 'WEBP'
+			);
+		case 'image/avif':
+			return (
+				buffer.length >= 12 &&
+				buffer.subarray(4, 8).toString('ascii') === 'ftyp' &&
+				(buffer.subarray(8, 12).toString('ascii') === 'avif' ||
+					buffer.subarray(8, 12).toString('ascii') === 'avis')
+			);
+		default:
+			return false;
+	}
+}
+
 function parseProtocolRelativeOrAbsoluteUrl(value: string): URL | null {
 	try {
 		if (value.startsWith('//')) {
@@ -296,8 +339,20 @@ export class ManagedBookCoverService {
 			return { managedUrl: null, sourceUrl: null };
 		}
 
+		if (extensionFromImageContentType(contentType) === null) {
+			this.serviceLogger.warn(
+				{
+					event: 'library.cover.buffer.unsupported_image_type',
+					bookStorageKey: input.bookStorageKey,
+					contentType
+				},
+				'Managed cover buffer used an unsupported image type'
+			);
+			return { managedUrl: null, sourceUrl: null };
+		}
+
 		if (
-			input.coverBuffer.byteLength === 0 ||
+			input.coverBuffer.byteLength < MIN_MANAGED_BOOK_COVER_BYTES ||
 			input.coverBuffer.byteLength > MAX_MANAGED_BOOK_COVER_BYTES
 		) {
 			this.serviceLogger.warn(
@@ -306,7 +361,20 @@ export class ManagedBookCoverService {
 					bookStorageKey: input.bookStorageKey,
 					byteLength: input.coverBuffer.byteLength
 				},
-				'Managed cover buffer was empty or too large'
+				'Managed cover buffer was too small or too large'
+			);
+			return { managedUrl: null, sourceUrl: null };
+		}
+
+		if (!hasImageMagicBytes(input.coverBuffer, contentType)) {
+			this.serviceLogger.warn(
+				{
+					event: 'library.cover.buffer.invalid_signature',
+					bookStorageKey: input.bookStorageKey,
+					contentType,
+					byteLength: input.coverBuffer.byteLength
+				},
+				'Managed cover buffer did not match the declared image type'
 			);
 			return { managedUrl: null, sourceUrl: null };
 		}
@@ -411,7 +479,10 @@ export class ManagedBookCoverService {
 				response,
 				maxBytes: MAX_MANAGED_BOOK_COVER_BYTES
 			});
-			if (coverRead.exceededLimit || coverRead.byteLength === 0) {
+			if (
+				coverRead.exceededLimit ||
+				coverRead.byteLength < MIN_MANAGED_BOOK_COVER_BYTES
+			) {
 				this.serviceLogger.warn(
 					{
 						event: 'library.cover.fetch.invalid_size',
@@ -420,7 +491,22 @@ export class ManagedBookCoverService {
 						sourceUrl: resolvedSourceUrl,
 						byteLength: coverRead.byteLength
 					},
-					'Managed cover fetch returned an empty or oversized payload'
+					'Managed cover fetch returned a too-small or oversized payload'
+				);
+				return { managedUrl: null, sourceUrl: resolvedSourceUrl };
+			}
+
+			if (!hasImageMagicBytes(coverRead.buffer, contentType)) {
+				this.serviceLogger.warn(
+					{
+						event: 'library.cover.fetch.invalid_signature',
+						bookStorageKey: input.bookStorageKey,
+						provider: input.provider,
+						sourceUrl: resolvedSourceUrl,
+						contentType,
+						byteLength: coverRead.byteLength
+					},
+					'Managed cover fetch returned bytes that did not match the declared image type'
 				);
 				return { managedUrl: null, sourceUrl: resolvedSourceUrl };
 			}