chunkr/test.toml at main · sguzman/chunkr · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
# Test configuration for `cargo test --test pipeline`.
# The test overrides paths/collection/index at runtime, but uses these settings.

[logging]
level = "info"

[paths]
calibre_root     = "/drive/calibre/en_nonfiction"
chunk_root       = "/tmp/chunkr-test/chunked"
examples_cfr_dir = "examples/cfr"
extract_root     = "/tmp/chunkr-test/extract"
state_dir        = "/tmp/chunkr-test/state"

[extract]
extensions      = [ "epub", "pdf" ]
metadata_layout = "{format}/{title_slug}.json"
output_layout   = "{format}/{title_slug}.txt"
skip_existing   = true
write_metadata  = false

[extract.epub]
backend           = "pandoc"
chapter_split     = true
join_parts        = true
keep_parts        = false
max_chapter_bytes = 2000000
max_file_bytes    = 20000000
pandoc_bin        = "pandoc"
toc_depth         = 3

[extract.pdf]
backend                  = "docling"
docling_bin              = "/home/admin/Code/AI/docling/.venv/bin/python"
docling_device           = "cuda"
docling_pdf_backend      = "dlparse_v4"
docling_pipeline         = "standard"
docling_script           = "/home/admin/Code/AI/docling/docling/cli/main.py"
docling_table_mode       = "accurate"
docling_tables           = true
docling_threads          = 16
document_timeout_seconds = 600
low_quality_force_ocr    = false
low_quality_table_mode   = "fast"
low_quality_tables       = false
low_quality_use_ocr      = false
max_file_bytes           = 20000000
max_pages_per_pass       = 50
ocr_engine               = "tesseract"
ocr_fallback             = true
ocr_lang                 = "eng"
page_batch_size          = 1
pdffonts_bin             = "pdffonts"
pdfinfo_bin              = "pdfinfo"
pdftotext_bin            = "pdftotext"
scan_force_ocr           = true
scan_table_mode          = "fast"
scan_tables              = false
skip_oversize            = false
split_text_extraction    = true
text_alpha_ratio_min     = 0.65
text_first               = true
text_good_min_chars      = 120
text_low_min_chars       = 40
text_sample_pages        = 3

[chunk]
chunk_overlap_chars = 200
collapse_whitespace = true
emit_jsonl          = true
max_chunk_chars     = 7200
max_paragraph_chars = 8000
min_paragraph_chars = 80
normalize_unicode   = true
strip_headers       = true
target_chunk_chars  = 6000

[chunk.metadata]
include_authors     = true
include_calibre_id  = true
include_language    = true
include_published   = true
include_source_path = true
include_title       = true

[insert]
batch_size         = 128
max_parallel_files = 16
retry_backoff_ms   = 500
retry_max          = 3

[insert.qdrant]
api_key           = ""
collection        = "chunkr_test"
create_collection = false
distance          = "Cosine"
url               = "http://127.0.0.1:6333"
vector_size       = 384
wait              = false

[insert.quickwit]
commit_at_end          = true
commit_mode            = "auto"
commit_timeout_seconds = 30
index_id               = "chunkr_test"
url                    = "http://127.0.0.1:7280"

[insert.embeddings]
base_url                = "http://127.0.0.1:11434"
cache_max_entries       = 20000
global_max_concurrency  = 16
max_concurrency         = 4
max_input_chars         = 400
model                   = "qllama/bge-small-en-v1.5:latest"
provider                = "ollama"
request_batch_size      = 8
request_timeout_seconds = 120

[calibre]
library_path = "/drive/calibre/en_nonfiction"
library_url  = "http://127.0.0.1:8081/#en_nonfiction"
state_path   = ""

[calibre.content_server]
password = "admin"
username = "admin"

[calibre.scoring]
authors_weight     = 1
comments_weight    = 1
cover_weight       = 1
identifiers_weight = 2
isbn_weight        = 2
pubdate_weight     = 1
publisher_weight   = 1
tags_weight        = 1
title_weight       = 1

[dups]
ext = [
  "azw3",
  "djvu",
  "epub",
  "mobi",
  "pdf",
]
follow_symlinks = false
hash_algorithm = "xxhash64"
include_sidecars = false
min_size = 1024
output = "json"
threads = 8

[dup_stats]
mode = "human"

[dedup]
dry_run  = true
min_size = 1024