From 80bfee494dc26240b04d5bc65b95bba5a9e06237 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Tue, 23 Dec 2025 17:40:08 +0100 Subject: [PATCH 1/2] feat: add book status field Add status field to books (complete, reading, abandoned, on_hold) with: - Database migration to add status column - API endpoint to update book status - UI components for status display and editing - Status badges in book cards and table views - KOReader plugin updates for status sync --- apps/server/src/ai/open-ai-service.ts | 36 +++-- apps/server/src/books/books-repository.ts | 6 +- apps/server/src/books/books-router.ts | 29 ++++ apps/server/src/db/factories/book-factory.ts | 1 + .../20251223100000_add_status_to_book.ts | 13 ++ apps/server/src/koplugin/koplugin-router.ts | 4 +- apps/server/src/upload/upload-service.ts | 6 +- apps/web/src/api/books.ts | 6 +- apps/web/src/components/status-icons/index.ts | 1 + .../components/status-icons/status-icons.tsx | 127 ++++++++++++++++++ apps/web/src/pages/book-page/book-card.tsx | 115 +++++++++++++++- apps/web/src/pages/books-page/books-cards.tsx | 34 ++++- apps/web/src/pages/books-page/books-table.tsx | 32 ++++- packages/common/types/book.ts | 4 + plugins/koinsight.koplugin/const.lua | 2 +- plugins/koinsight.koplugin/db_reader.lua | 101 ++++++++++++++ plugins/koinsight.koplugin/upload.lua | 13 +- 17 files changed, 499 insertions(+), 31 deletions(-) create mode 100644 apps/server/src/db/migrations/20251223100000_add_status_to_book.ts create mode 100644 apps/web/src/components/status-icons/index.ts create mode 100644 apps/web/src/components/status-icons/status-icons.tsx diff --git a/apps/server/src/ai/open-ai-service.ts b/apps/server/src/ai/open-ai-service.ts index 8558f95f..40f8826b 100644 --- a/apps/server/src/ai/open-ai-service.ts +++ b/apps/server/src/ai/open-ai-service.ts @@ -1,6 +1,4 @@ import OpenAI from 'openai'; -import { zodResponseFormat } from 'openai/helpers/zod'; -import { z } from 'zod'; let openAIClient: OpenAI | undefined; if (process.env.OPENAI_API_KEY && process.env.OPENAI_PROJECT_ID && process.env.OPENAI_ORG_ID) { @@ -11,27 +9,43 @@ if (process.env.OPENAI_API_KEY && process.env.OPENAI_PROJECT_ID && process.env.O }); } -const BookInsights = z.object({ - genres: z.string().array(), - summary: z.string(), -}); +type BookInsights = { + genres: string[]; + summary: string; +}; -export async function getBookInsights(bookTitle: string, bookAuthor: string) { - const completion = await openAIClient?.beta.chat.completions.parse({ +export async function getBookInsights( + bookTitle: string, + bookAuthor: string +): Promise { + if (!openAIClient) { + return null; + } + + const completion = await openAIClient.chat.completions.create({ model: 'gpt-4o', messages: [ { role: 'system', content: - 'You are an expert librarian. You know everything about every book. Respond with details about the book given the title and author', + 'You are an expert librarian. You know everything about every book. Respond with details about the book given the title and author. Return a JSON object with "genres" (array of strings) and "summary" (string).', }, { role: 'user', content: `What can you tell me about the book ${bookTitle} by ${bookAuthor}`, }, ], - response_format: zodResponseFormat(BookInsights, 'book_insights'), + response_format: { type: 'json_object' }, }); - return completion?.choices[0].message.parsed; + const content = completion.choices[0].message.content; + if (!content) { + return null; + } + + try { + return JSON.parse(content) as BookInsights; + } catch { + return null; + } } diff --git a/apps/server/src/books/books-repository.ts b/apps/server/src/books/books-repository.ts index f21cfe21..29c285cb 100644 --- a/apps/server/src/books/books-repository.ts +++ b/apps/server/src/books/books-repository.ts @@ -1,5 +1,5 @@ import { BookGenre, BookWithData } from '@koinsight/common/types'; -import { Book } from '@koinsight/common/types/book'; +import { Book, BookStatus } from '@koinsight/common/types/book'; import { BookDevice } from '@koinsight/common/types/book-device'; import { Genre } from '@koinsight/common/types/genre'; import { sum } from 'ramda'; @@ -122,4 +122,8 @@ export class BooksRepository { static async setReferencePages(id: number, referencePages: number | null) { return db('book').where({ id }).update({ reference_pages: referencePages }); } + + static async setStatus(id: number, status: BookStatus) { + return db('book').where({ id }).update({ status }); + } } diff --git a/apps/server/src/books/books-router.ts b/apps/server/src/books/books-router.ts index 730b4461..481333ee 100644 --- a/apps/server/src/books/books-router.ts +++ b/apps/server/src/books/books-router.ts @@ -1,9 +1,12 @@ +import { BookStatus } from '@koinsight/common/types/book'; import { NextFunction, Request, Response, Router } from 'express'; import { BooksRepository } from './books-repository'; import { BooksService } from './books-service'; import { coversRouter } from './covers/covers-router'; import { getBookById } from './get-book-by-id-middleware'; +const VALID_STATUSES: BookStatus[] = ['complete', 'reading', 'abandoned', 'on_hold', null]; + const router = Router(); router.use('/:bookId/cover', coversRouter); @@ -101,4 +104,30 @@ router.put('/:bookId/reference_pages', getBookById, async (req: Request, res: Re } }); +/** + * Updates a book's status (complete, reading, abandoned, on_hold, or null) + */ +router.put('/:bookId/status', getBookById, async (req: Request, res: Response) => { + const book = req.book!; + const { status } = req.body; + + if (status === undefined) { + res.status(400).json({ error: 'Missing required fields' }); + return; + } + + if (!VALID_STATUSES.includes(status)) { + res.status(400).json({ error: 'Invalid status value' }); + return; + } + + try { + await BooksRepository.setStatus(book.id, status); + res.status(200).json({ message: 'Status updated' }); + } catch (error) { + console.error(error); + res.status(500).json({ error: 'Failed to update status' }); + } +}); + export { router as booksRouter }; diff --git a/apps/server/src/db/factories/book-factory.ts b/apps/server/src/db/factories/book-factory.ts index a47b11c2..0fd23e93 100644 --- a/apps/server/src/db/factories/book-factory.ts +++ b/apps/server/src/db/factories/book-factory.ts @@ -13,6 +13,7 @@ export function fakeBook(overrides: Partial = {}): FakeBook { series: faker.book.series(), language: faker.location.language().alpha2, soft_deleted: false, + status: null, ...overrides, }; diff --git a/apps/server/src/db/migrations/20251223100000_add_status_to_book.ts b/apps/server/src/db/migrations/20251223100000_add_status_to_book.ts new file mode 100644 index 00000000..17d622ed --- /dev/null +++ b/apps/server/src/db/migrations/20251223100000_add_status_to_book.ts @@ -0,0 +1,13 @@ +import type { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + return knex.schema.alterTable('book', (table) => { + table.string('status').defaultTo(null); + }); +} + +export async function down(knex: Knex): Promise { + return knex.schema.alterTable('book', (table) => { + table.dropColumn('status'); + }); +} diff --git a/apps/server/src/koplugin/koplugin-router.ts b/apps/server/src/koplugin/koplugin-router.ts index 0f99775f..2eb560a8 100644 --- a/apps/server/src/koplugin/koplugin-router.ts +++ b/apps/server/src/koplugin/koplugin-router.ts @@ -10,12 +10,12 @@ import { UploadService } from '../upload/upload-service'; // Router for KoInsight koreader plugin const router = Router(); -const REQUIRED_PLUGIN_VERSION = '0.1.0'; +const REQUIRED_PLUGIN_VERSION = '0.2.0'; const rejectOldPluginVersion = (req: Request, res: Response, next: NextFunction) => { const { version } = req.body; - if (!version || version !== '0.1.0') { + if (!version || version !== '0.2.0') { res.status(400).json({ error: `Unsupported plugin version. Version must be ${REQUIRED_PLUGIN_VERSION}. Please update your KOReader koinsight.koplugin`, }); diff --git a/apps/server/src/upload/upload-service.ts b/apps/server/src/upload/upload-service.ts index a0dca5ec..c18c05a4 100644 --- a/apps/server/src/upload/upload-service.ts +++ b/apps/server/src/upload/upload-service.ts @@ -46,10 +46,14 @@ export class UploadService { authors: book.authors, series: book.series, language: book.language, + status: book.status ?? null, })); await Promise.all( - newBooks.map(({ id, ...book }) => trx('book').insert(book).onConflict('md5').ignore()) + newBooks.map(({ id, ...book }) => { + const query = trx('book').insert(book).onConflict('md5'); + return book.status ? query.merge(['status']) : query.ignore(); + }) ); const hasUnknownDevices = diff --git a/apps/web/src/api/books.ts b/apps/web/src/api/books.ts index 7f792e90..bcfbc7d8 100644 --- a/apps/web/src/api/books.ts +++ b/apps/web/src/api/books.ts @@ -1,4 +1,4 @@ -import { Book, BookWithData } from '@koinsight/common/types'; +import { Book, BookStatus, BookWithData } from '@koinsight/common/types'; import useSWR from 'swr'; import { API_URL, fetchFromAPI } from './api'; @@ -37,3 +37,7 @@ export function uploadBookCover(bookId: Book['id'], formData: FormData) { headers: { Accept: 'multipart/form-data' }, }); } + +export async function updateBookStatus(id: Book['id'], status: BookStatus) { + return fetchFromAPI<{ message: string }>(`books/${id}/status`, 'PUT', { status }); +} diff --git a/apps/web/src/components/status-icons/index.ts b/apps/web/src/components/status-icons/index.ts new file mode 100644 index 00000000..ea848969 --- /dev/null +++ b/apps/web/src/components/status-icons/index.ts @@ -0,0 +1 @@ +export { AbandonedIcon, CompletedIcon, OnHoldIcon, ReadingIcon } from './status-icons'; diff --git a/apps/web/src/components/status-icons/status-icons.tsx b/apps/web/src/components/status-icons/status-icons.tsx new file mode 100644 index 00000000..f7c93010 --- /dev/null +++ b/apps/web/src/components/status-icons/status-icons.tsx @@ -0,0 +1,127 @@ +import { Tooltip } from '@mantine/core'; +import { JSX } from 'react'; + +type StatusIconProps = { + size?: number; + withTooltip?: boolean; +}; + +export function CompletedIcon({ size = 16, withTooltip = true }: StatusIconProps): JSX.Element { + const icon = ( + + + + ); + + if (withTooltip) { + return ( + + {icon} + + ); + } + return icon; +} + +export function ReadingIcon({ size = 16, withTooltip = true }: StatusIconProps): JSX.Element { + const icon = ( + + + + ); + + if (withTooltip) { + return ( + + {icon} + + ); + } + return icon; +} + +export function OnHoldIcon({ size = 16, withTooltip = true }: StatusIconProps): JSX.Element { + const icon = ( + + + + ); + + if (withTooltip) { + return ( + + {icon} + + ); + } + return icon; +} + +export function AbandonedIcon({ size = 16, withTooltip = true }: StatusIconProps): JSX.Element { + const icon = ( + + + + ); + + if (withTooltip) { + return ( + + {icon} + + ); + } + return icon; +} diff --git a/apps/web/src/pages/book-page/book-card.tsx b/apps/web/src/pages/book-page/book-card.tsx index 73ae35f2..0ab896e2 100644 --- a/apps/web/src/pages/book-page/book-card.tsx +++ b/apps/web/src/pages/book-page/book-card.tsx @@ -1,9 +1,25 @@ -import { BookWithData } from '@koinsight/common/types'; -import { Flex, Group, Image, Title, Tooltip } from '@mantine/core'; +import { BookStatus, BookWithData } from '@koinsight/common/types'; +import { Button, Flex, Group, Image, Menu, Title, Tooltip } from '@mantine/core'; import { useMediaQuery } from '@mantine/hooks'; -import { IconBooks, IconCalendar, IconHighlight, IconNote, IconUser } from '@tabler/icons-react'; +import { + IconBooks, + IconCalendar, + IconChevronDown, + IconHighlight, + IconNote, + IconUser, + IconX, +} from '@tabler/icons-react'; import { JSX } from 'react'; +import { useSWRConfig } from 'swr'; import { API_URL } from '../../api/api'; +import { updateBookStatus } from '../../api/books'; +import { + AbandonedIcon, + CompletedIcon, + OnHoldIcon, + ReadingIcon, +} from '../../components/status-icons'; import { formatRelativeDate } from '../../utils/dates'; import style from './book-card.module.css'; @@ -12,8 +28,44 @@ type BookCardProps = { book: BookWithData; }; +const getStatusIcon = (status: BookStatus, size = 16) => { + switch (status) { + case 'complete': + return ; + case 'reading': + return ; + case 'on_hold': + return ; + case 'abandoned': + return ; + default: + return null; + } +}; + +const getStatusLabel = (status: BookStatus) => { + switch (status) { + case 'complete': + return 'Completed'; + case 'reading': + return 'Reading'; + case 'on_hold': + return 'On Hold'; + case 'abandoned': + return 'Abandoned'; + default: + return 'Set Status'; + } +}; + export function BookCard({ book }: BookCardProps): JSX.Element { const media = useMediaQuery(`(max-width: 62em)`); + const { mutate } = useSWRConfig(); + + const handleStatusChange = async (status: BookStatus) => { + await updateBookStatus(book.id, status); + mutate(`books/${book.id}`); + }; return ( @@ -32,7 +84,62 @@ export function BookCard({ book }: BookCardProps): JSX.Element { {book.authors ?? 'N/A'} - {book.title} + + {book.title} + {book.status && getStatusIcon(book.status, 24)} + + + + + + + + Reading status + } + onClick={() => handleStatusChange('complete')} + color={book.status === 'complete' ? 'green' : undefined} + > + Completed + + } + onClick={() => handleStatusChange('reading')} + color={book.status === 'reading' ? 'blue' : undefined} + > + Reading + + } + onClick={() => handleStatusChange('on_hold')} + color={book.status === 'on_hold' ? 'yellow' : undefined} + > + On Hold + + } + onClick={() => handleStatusChange('abandoned')} + color={book.status === 'abandoned' ? 'red' : undefined} + > + Abandoned + + + } + onClick={() => handleStatusChange(null)} + > + Clear status + + + diff --git a/apps/web/src/pages/books-page/books-cards.tsx b/apps/web/src/pages/books-page/books-cards.tsx index 02b2b273..fc3727f0 100644 --- a/apps/web/src/pages/books-page/books-cards.tsx +++ b/apps/web/src/pages/books-page/books-cards.tsx @@ -1,15 +1,36 @@ -import { BookWithData } from '@koinsight/common/types'; -import { Box, Group, Image, Progress, Text, Tooltip } from '@mantine/core'; +import { BookStatus, BookWithData } from '@koinsight/common/types'; +import { Box, Flex, Group, Image, Progress, Text, Tooltip } from '@mantine/core'; import { useMediaQuery } from '@mantine/hooks'; import { IconBooks, IconEyeClosed, IconProgress, IconUser } from '@tabler/icons-react'; import C from 'clsx'; import { JSX } from 'react'; import { useNavigate } from 'react-router'; import { API_URL } from '../../api/api'; +import { + AbandonedIcon, + CompletedIcon, + OnHoldIcon, + ReadingIcon, +} from '../../components/status-icons'; import { getBookPath } from '../../routes'; import style from './books-cards.module.css'; +const StatusIcon = ({ status }: { status: BookStatus }) => { + switch (status) { + case 'complete': + return ; + case 'reading': + return ; + case 'on_hold': + return ; + case 'abandoned': + return ; + default: + return null; + } +}; + type BooksCardsProps = { books: BookWithData[]; }; @@ -52,9 +73,12 @@ export function BooksCards({ books }: BooksCardsProps): JSX.Element { color="koinsight" /> - - {book.title} - + + + {book.title} + + {book.status && } + diff --git a/apps/web/src/pages/books-page/books-table.tsx b/apps/web/src/pages/books-page/books-table.tsx index 3c19d3d3..ba26ff2e 100644 --- a/apps/web/src/pages/books-page/books-table.tsx +++ b/apps/web/src/pages/books-page/books-table.tsx @@ -1,14 +1,35 @@ -import { BookWithData } from '@koinsight/common/types'; +import { BookStatus, BookWithData } from '@koinsight/common/types'; import { Anchor, Flex, Image, Progress, Stack, Table, Tooltip } from '@mantine/core'; import { useMediaQuery } from '@mantine/hooks'; import { IconEyeClosed } from '@tabler/icons-react'; import { JSX } from 'react'; import { NavLink } from 'react-router'; import { API_URL } from '../../api/api'; +import { + AbandonedIcon, + CompletedIcon, + OnHoldIcon, + ReadingIcon, +} from '../../components/status-icons'; import { getBookPath } from '../../routes'; import { formatRelativeDate, getDuration, shortDuration } from '../../utils/dates'; import style from './books-table.module.css'; +const StatusIcon = ({ status }: { status: BookStatus }) => { + switch (status) { + case 'complete': + return ; + case 'reading': + return ; + case 'on_hold': + return ; + case 'abandoned': + return ; + default: + return null; + } +}; + type BooksTableProps = { books: BookWithData[]; }; @@ -56,9 +77,12 @@ export function BooksTable({ books }: BooksTableProps): JSX.Element { /> - - {book.title} - + + + {book.title} + + {book.status && } + {book.authors ?? 'Unknown author'} {book.series !== 'N/A' ? ` ยทย ${book.series}` : ''} diff --git a/packages/common/types/book.ts b/packages/common/types/book.ts index 4a4f2e25..9a490394 100644 --- a/packages/common/types/book.ts +++ b/packages/common/types/book.ts @@ -1,3 +1,5 @@ +export type BookStatus = 'complete' | 'reading' | 'abandoned' | 'on_hold' | null; + export type KoReaderBook = { id: number; md5: string; @@ -11,6 +13,7 @@ export type KoReaderBook = { language: string; total_read_time: number; total_read_pages: number; + status?: BookStatus; }; export type DbBook = { @@ -25,4 +28,5 @@ export type DbBook = { export type Book = DbBook & { soft_deleted: boolean; reference_pages: number | null; + status: BookStatus; }; diff --git a/plugins/koinsight.koplugin/const.lua b/plugins/koinsight.koplugin/const.lua index 72fe2799..86c57fb1 100644 --- a/plugins/koinsight.koplugin/const.lua +++ b/plugins/koinsight.koplugin/const.lua @@ -1,5 +1,5 @@ local const = {} -const.VERSION = "0.1.0" +const.VERSION = "0.2.0" return const diff --git a/plugins/koinsight.koplugin/db_reader.lua b/plugins/koinsight.koplugin/db_reader.lua index e0778a79..5434d26f 100644 --- a/plugins/koinsight.koplugin/db_reader.lua +++ b/plugins/koinsight.koplugin/db_reader.lua @@ -1,8 +1,12 @@ local SQ3 = require("lua-ljsqlite3/init") local DataStorage = require("datastorage") +local LuaSettings = require("luasettings") local logger = require("logger") +local lfs = require("libs/libkoreader-lfs") local db_location = DataStorage:getSettingsDir() .. "/statistics.sqlite3" +local docsettings_dir = DataStorage:getDocSettingsDir() +local home_dir = DataStorage:getDataDir() -- Where books are typically stored local KoInsightDbReader = {} @@ -76,4 +80,101 @@ function KoInsightDbReader.progressData() return results end +-- Recursively scan a directory for .sdr folders containing metadata.lua +local function scanForSidecarFiles(dir, results) + results = results or {} + + local ok, iter, dir_obj = pcall(lfs.dir, dir) + if not ok then + return results + end + + for entry in iter, dir_obj do + if entry ~= "." and entry ~= ".." then + local full_path = dir .. "/" .. entry + local mode = lfs.attributes(full_path, "mode") + + if mode == "directory" then + if entry:match("%.sdr$") then + -- This is a sidecar directory, look for metadata*.lua files + local sdr_ok, sdr_iter, sdr_obj = pcall(lfs.dir, full_path) + if sdr_ok then + for sdr_entry in sdr_iter, sdr_obj do + if sdr_entry:match("^metadata.*%.lua$") then + table.insert(results, full_path .. "/" .. sdr_entry) + break -- Only need one metadata file per sdr + end + end + end + else + -- Recurse into subdirectory + scanForSidecarFiles(full_path, results) + end + end + end + end + + return results +end + +-- Read book statuses from sidecar files and return a mapping of md5 -> status +function KoInsightDbReader.getBookStatuses() + local statuses = {} + local sidecar_files = {} + + -- Scan both docsettings directory and home/books directory for sidecar files + local dirs_to_scan = { docsettings_dir, home_dir } + + for _, dir in ipairs(dirs_to_scan) do + logger.info("[KoInsight] Scanning for sidecar files in:", dir) + scanForSidecarFiles(dir, sidecar_files) + end + + logger.info("[KoInsight] Found", #sidecar_files, "sidecar files total") + + for _, metadata_path in ipairs(sidecar_files) do + logger.dbg("[KoInsight] Reading metadata from:", metadata_path) + local ok, settings = pcall(function() + return LuaSettings:open(metadata_path) + end) + + if ok and settings then + -- Get the MD5 checksum (stored at root level as partial_md5_checksum) + local md5 = settings:readSetting("partial_md5_checksum") + local summary = settings:readSetting("summary") + + logger.dbg("[KoInsight] partial_md5_checksum:", md5 or "nil") + logger.dbg("[KoInsight] summary:", summary and "found" or "nil") + + if md5 then + if summary and summary.status then + local status = summary.status + + -- Normalize status values + if status == "complete" or status == "completed" then + status = "complete" + elseif status == "on hold" then + status = "on_hold" + end + + statuses[md5] = status + logger.info("[KoInsight] Found status for", md5, ":", status) + else + logger.dbg("[KoInsight] No status set for book:", md5) + end + end + else + logger.warn("[KoInsight] Failed to read metadata:", metadata_path) + end + end + + -- Count table entries (# doesn't work for tables with string keys) + local count = 0 + for _ in pairs(statuses) do + count = count + 1 + end + logger.info("[KoInsight] Loaded statuses for", count, "books") + return statuses +end + return KoInsightDbReader diff --git a/plugins/koinsight.koplugin/upload.lua b/plugins/koinsight.koplugin/upload.lua index 572e57b6..6a853c66 100644 --- a/plugins/koinsight.koplugin/upload.lua +++ b/plugins/koinsight.koplugin/upload.lua @@ -50,9 +50,20 @@ end function send_statistics_data(server_url, silent) local url = server_url .. API_UPLOAD_LOCATION + -- Get book data and statuses + local books = KoInsightDbReader.bookData() + local statuses = KoInsightDbReader.getBookStatuses() + + -- Merge statuses into book data + for _, book in ipairs(books) do + if book.md5 and statuses[book.md5] then + book.status = statuses[book.md5] + end + end + local body = { stats = KoInsightDbReader.progressData(), - books = KoInsightDbReader.bookData(), + books = books, version = const.VERSION, } From 6783064be165e252bc627417e403594a4afe8121 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Sat, 27 Dec 2025 22:09:42 +0100 Subject: [PATCH 2/2] fix: improve plugin robustness and fix typo - Fix "successfull" typo in server upload response - Increase HTTP timeout to 60 seconds for slow uploads - Add retry logic with exponential backoff (3 retries: 1s, 2s, 4s) - Fix database connection leaks with pcall cleanup pattern - Fix undefined `message` variable in auto-sync error handler - Fix unsafe error message construction in call_api - Make get_md5_by_id a local function --- apps/server/src/koplugin/koplugin-router.ts | 2 +- plugins/koinsight.koplugin/call_api.lua | 110 ++++++++++------ plugins/koinsight.koplugin/db_reader.lua | 137 +++++++++++++------- plugins/koinsight.koplugin/main.lua | 2 +- 4 files changed, 161 insertions(+), 90 deletions(-) diff --git a/apps/server/src/koplugin/koplugin-router.ts b/apps/server/src/koplugin/koplugin-router.ts index 2eb560a8..4d4cdc7d 100644 --- a/apps/server/src/koplugin/koplugin-router.ts +++ b/apps/server/src/koplugin/koplugin-router.ts @@ -56,7 +56,7 @@ router.post('/import', rejectOldPluginVersion, async (req, res) => { console.debug('Importing books:', koreaderBooks); console.debug('Importing page stats:', newPageStats); await UploadService.uploadStatisticData(koreaderBooks, newPageStats); - res.status(200).json({ message: 'Upload successfull' }); + res.status(200).json({ message: 'Upload successful' }); } catch (err) { console.error(err); res.status(500).json({ error: 'Error importing data' }); diff --git a/plugins/koinsight.koplugin/call_api.lua b/plugins/koinsight.koplugin/call_api.lua index eae089ad..0ab781d3 100644 --- a/plugins/koinsight.koplugin/call_api.lua +++ b/plugins/koinsight.koplugin/call_api.lua @@ -8,7 +8,10 @@ local JSON = require("json") local InfoMessage = require("ui/widget/infomessage") local _ = require("gettext") -function response_not_valid(content) +local MAX_RETRIES = 3 +local RETRY_DELAYS = {1, 2, 4} -- seconds, exponential backoff + +local function response_not_valid(content) logger.err("[KoInsight] callApi: response was not valid JSON", content) UIManager:show(InfoMessage:new({ text = _("Server response is not valid."), @@ -18,58 +21,83 @@ end return function(method, url, headers, body, filepath, quiet) quiet = quiet or false - local sink = {} - local request = { - method = method, - } + local last_error + for attempt = 1, MAX_RETRIES do + local sink = {} + local request = { + method = method, + url = url, + headers = headers or {}, + sink = ltn12.sink.table(sink), + } - request.url = url - request.headers = headers or {} + if body ~= nil then + request.source = ltn12.source.string(body) + end - request.sink = ltn12.sink.table(sink) - socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT) + if attempt == 1 then + logger.dbg("[KoInsight] callApi:", request.method, request.url) + else + logger.info("[KoInsight] callApi: retry attempt", attempt, "of", MAX_RETRIES) + end - if body ~= nil then - request.source = ltn12.source.string(body) - end + socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, 60) + local code, resp_headers, status = socket.skip(1, http.request(request)) + socketutil:reset_timeout() - logger.dbg("[KoInsight] callApi:", request.method, request.url) + -- If we got a response (success or HTTP error), process it + if resp_headers ~= nil then + -- If the request returned successfully + if code == 200 then + local content = table.concat(sink) - local code, resp_headers, status = socket.skip(1, http.request(request)) - socketutil:reset_timeout() + if content == nil or content == "" or string.sub(content, 1, 1) ~= "{" then + response_not_valid(content) + return false, "empty_response" + end - -- Raise error if network is unavailable - if resp_headers == nil then - logger.err("[KoInsight] callApi: network error", status or code) - return false, "network_error" - end + local ok, result = pcall(JSON.decode, content) - -- If the request returned successfully - if code == 200 then - local content = table.concat(sink) + if ok and result then + return true, result + else + response_not_valid(content) + return false, "invalid_response" + end + else + -- HTTP error - don't retry, server received the request + if not quiet then + local content = table.concat(sink) + local error_detail = "" + local decode_ok, decoded = pcall(JSON.decode, content) + if decode_ok and type(decoded) == "table" and decoded.error then + error_detail = ": " .. tostring(decoded.error) + end + logger.err("[KoInsight] callApi: HTTP error", status or code, resp_headers) + UIManager:show(InfoMessage:new({ + text = _("Server error") .. error_detail, + })) + end - if content == nil or content == "" or string.sub(content, 1, 1) ~= "{" then - response_not_valid(content) - return false, "empty_response" + return false, "http_error", code + end end - local ok, result = pcall(JSON.decode, content) + -- Network error - retry with delay + last_error = status or code + logger.warn("[KoInsight] Network error, attempt", attempt, "of", MAX_RETRIES, ":", last_error) - if ok and result then - return true, result - else - response_not_valid(content) - return false, "invalid_response" - end - else - if not quiet then - logger.err("[KoInsight] callApi: HTTP error", status or code, resp_headers, result) - UIManager:show(InfoMessage:new({ - text = _("Server error" .. (result and ": " .. result["error"] or "")), - })) + if attempt < MAX_RETRIES then + socket.sleep(RETRY_DELAYS[attempt]) end + end - logger.err("[KoInsight] callApi: HTTP error", status or code, resp_headers) - return false, "http_error", code + -- All retries exhausted + logger.err("[KoInsight] callApi: network error after", MAX_RETRIES, "attempts:", last_error) + if not quiet then + UIManager:show(InfoMessage:new({ + text = _("Network error. Please check your connection."), + })) end + return false, "network_error" end diff --git a/plugins/koinsight.koplugin/db_reader.lua b/plugins/koinsight.koplugin/db_reader.lua index 5434d26f..f0210408 100644 --- a/plugins/koinsight.koplugin/db_reader.lua +++ b/plugins/koinsight.koplugin/db_reader.lua @@ -6,38 +6,56 @@ local lfs = require("libs/libkoreader-lfs") local db_location = DataStorage:getSettingsDir() .. "/statistics.sqlite3" local docsettings_dir = DataStorage:getDocSettingsDir() -local home_dir = DataStorage:getDataDir() -- Where books are typically stored +local home_dir = DataStorage:getDataDir() -- Where KOReader is installed + +-- Get user's configured home directory from KOReader settings +local user_home_dir = G_reader_settings:readSetting("home_dir") + +-- Common library paths on different devices +local library_paths = { + "/mnt/us/documents", -- Kindle + "/mnt/onboard", -- Kobo + "/storage/emulated/0", -- Android +} local KoInsightDbReader = {} function KoInsightDbReader.bookData() local conn = SQ3.open(db_location) - local result, rows = conn:exec("SELECT * FROM book") - local books = {} - - for i = 1, rows do - local book = { - id = tonumber(result[1][i]), - title = result[2][i], - authors = result[3][i], - notes = tonumber(result[4][i]), - last_open = tonumber(result[5][i]), - highlights = tonumber(result[6][i]), - pages = tonumber(result[7][i]), - series = result[8][i], - language = result[9][i], - md5 = result[10][i], - total_read_time = tonumber(result[11][i]), - total_read_pages = tonumber(result[12][i]), - } - table.insert(books, book) - end + local ok, books_or_err = pcall(function() + local result, rows = conn:exec("SELECT * FROM book") + local books = {} + + for i = 1, rows do + local book = { + id = tonumber(result[1][i]), + title = result[2][i], + authors = result[3][i], + notes = tonumber(result[4][i]), + last_open = tonumber(result[5][i]), + highlights = tonumber(result[6][i]), + pages = tonumber(result[7][i]), + series = result[8][i], + language = result[9][i], + md5 = result[10][i], + total_read_time = tonumber(result[11][i]), + total_read_pages = tonumber(result[12][i]), + } + table.insert(books, book) + end + return books + end) conn:close() - return books + + if not ok then + logger.err("[KoInsight] Error reading book data:", books_or_err) + return {} + end + return books_or_err end -function get_md5_by_id(books, target_id) +local function get_md5_by_id(books, target_id) for _, book in ipairs(books) do if book.id == target_id then return book.md5 @@ -47,37 +65,44 @@ function get_md5_by_id(books, target_id) end function KoInsightDbReader.progressData() + local book_data = KoInsightDbReader.bookData() + local device_id = G_reader_settings:readSetting("device_id") + local conn = SQ3.open(db_location) - local result, rows = conn:exec("SELECT * FROM page_stat_data") - local results = {} + local ok, results_or_err = pcall(function() + local result, rows = conn:exec("SELECT * FROM page_stat_data") + local results = {} - local book_data = KoInsightDbReader.bookData() + for i = 1, rows do + local book_id = tonumber(result[1][i]) + local book_md5 = get_md5_by_id(book_data, book_id) - local device_id = G_reader_settings:readSetting("device_id") + if book_md5 == nil then + logger.warn("[KoInsight] Book MD5 not found in book data:" .. book_id) + goto continue + end - for i = 1, rows do - local book_id = tonumber(result[1][i]) - local book_md5 = get_md5_by_id(book_data, book_id) + table.insert(results, { + page = tonumber(result[2][i]), + start_time = tonumber(result[3][i]), + duration = tonumber(result[4][i]), + total_pages = tonumber(result[5][i]), + book_md5 = book_md5, + device_id = device_id, + }) - if book_md5 == nil then - logger.warn("[KoInsight] Book MD5 not found in book data:" .. book_id) - goto continue + ::continue:: end - table.insert(results, { - page = tonumber(result[2][i]), - start_time = tonumber(result[3][i]), - duration = tonumber(result[4][i]), - total_pages = tonumber(result[5][i]), - book_md5 = book_md5, - device_id = device_id, - }) + return results + end) + conn:close() - ::continue:: + if not ok then + logger.err("[KoInsight] Error reading progress data:", results_or_err) + return {} end - - conn:close() - return results + return results_or_err end -- Recursively scan a directory for .sdr folders containing metadata.lua @@ -122,8 +147,26 @@ function KoInsightDbReader.getBookStatuses() local statuses = {} local sidecar_files = {} - -- Scan both docsettings directory and home/books directory for sidecar files - local dirs_to_scan = { docsettings_dir, home_dir } + -- Build list of directories to scan (avoiding duplicates) + local dirs_to_scan = {} + local seen = {} + + local function add_dir(dir) + if dir and dir ~= "" and not seen[dir] then + seen[dir] = true + table.insert(dirs_to_scan, dir) + end + end + + -- Add core directories + add_dir(docsettings_dir) + add_dir(home_dir) + add_dir(user_home_dir) + + -- Add common library paths + for _, path in ipairs(library_paths) do + add_dir(path) + end for _, dir in ipairs(dirs_to_scan) do logger.info("[KoInsight] Scanning for sidecar files in:", dir) diff --git a/plugins/koinsight.koplugin/main.lua b/plugins/koinsight.koplugin/main.lua index e5b38ccd..dc949c98 100644 --- a/plugins/koinsight.koplugin/main.lua +++ b/plugins/koinsight.koplugin/main.lua @@ -137,7 +137,7 @@ function koinsight:performSyncOnSuspend() end) if not success then - message = "Error during auto sync: " .. tostring(error_msg) + local message = "Error during auto sync: " .. tostring(error_msg) logger.err("[KoInsight] " .. message) UIManager:show(InfoMessage:new({ text = _(message),