Skip to content

Commit 333d281

Browse files
tballisontballison
authored
small updates to tika-eval (#2881)
1 parent 8a7728a commit 333d281

5 files changed

Lines changed: 472 additions & 6 deletions

File tree

‎.skills/tika-eval-compare.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,8 +88,9 @@ java -jar <tika-eval>/tika-eval-app-*.jar Compare \
8888
| `-a` | Directory of "before" extracts (required) |
8989
| `-b` | Directory of "after" extracts (required) |
9090
| `-d` | H2 database path (temp file if omitted) |
91-
| `-r` | Auto-run Report + tar.gz after Compare |
91+
| `-r` | Auto-run Report + tgz the reports dir (`<reportsDir>.tgz`) after Compare |
9292
| `-rd` | Reports output directory (default: `reports`) |
93+
| `-z` | Gzip the H2 db (`<db>.mv.db.gz`) after Compare for transfer; requires `-d` (no-op + warning for a temp db). Combine with `-r` to package both. |
9394
| `-n` | Number of worker threads |
9495

9596
## Step 3 — Review Results

‎docs/modules/ROOT/pages/advanced/integration-testing/tika-eval-regression.adoc‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,8 @@ java -jar tika-eval/tika-eval-app/target/tika-eval-app-{tika-version}.jar \
191191

192192
The `Compare` subcommand keyword is optional — the CLI infers it from
193193
the `-a` / `-b` flags. The `-r` flag both runs the Report stage and
194-
zips the resulting reports directory for easy archiving.
194+
creates a `.tgz` archive of the resulting reports directory
195+
(`<reportsDir>.tgz`) for easy archiving.
195196

196197
Options:
197198

@@ -203,9 +204,13 @@ Options:
203204
tika-eval will create `\{label}.mv.db` and a `\{label}-reports/` dir
204205
alongside. Persist the db if you want to re-run Report later.
205206
* `-r` / `--report` — automatically run the Report step after Compare,
206-
and zip the reports directory.
207+
and tgz the reports directory to `<reportsDir>.tgz`.
207208
* `-rd` / `--reportsDir` — explicit reports directory (overrides the
208209
default derived from `-d`).
210+
* `-z` / `--gzip` — gzip the H2 db file to `<db>.mv.db.gz` after Compare
211+
so it can be transferred. Requires `-d` (no-op with a warning for a
212+
temp db or a non-file jdbc connection (e.g. `jdbc:h2:mem:`, tcp)). Combine with `-r` to package
213+
both the reports and the db.
209214
* `-n` / `--numWorkers` — comparison worker count.
210215
* `-c` / `--config` — optional tika-eval JSON config.
211216

@@ -296,7 +301,7 @@ java -jar tika-eval-app-{tika-version}.jar \
296301
-b ~/data/extracts/cc-html-sample-B
297302
----
298303
+
299-
Produces `cc-html-29k-A-vs-B-reports/` plus a `.tar.gz` of the same
304+
Produces `cc-html-29k-A-vs-B-reports/` plus a `.tgz` of the same
300305
alongside `cc-html-29k-A-vs-B.mv.db`.
301306

302307
For a 29 K-file HTML sample on a typical workstation (8 forked workers,

‎tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ public class ExtractComparerRunner {
9292
.addOption(Option.builder("m").longOpt("maxExtractLength").hasArg().desc("maximum extract length").get())
9393
.addOption(Option.builder("r").longOpt("report").desc("automatically run Report and tgz after Compare").get())
9494
.addOption(Option.builder("rd").longOpt("reportsDir").hasArg().desc("directory for reports (default: 'reports')").get())
95+
.addOption(Option.builder("z").longOpt("gzip").desc("gzip the H2 db file (<db>.mv.db.gz) after Compare for transfer; requires -d").get())
9596
;
9697
}
9798

@@ -128,15 +129,23 @@ public static void main(String[] args) throws Exception {
128129
if (commandLine.hasOption('r')) {
129130
String reportsDir = commandLine.getOptionValue("rd", "reports");
130131
LOG.info("Running Report...");
131-
ResultsReporter.main(new String[]{"-d", dbPath, "-rd", reportsDir});
132+
if (dbPath.startsWith("jdbc:")) {
133+
ResultsReporter.main(new String[]{"-jdbc", dbPath, "-rd", reportsDir});
134+
} else {
135+
ResultsReporter.main(new String[]{"-d", dbPath, "-rd", reportsDir});
136+
}
132137
Path reportsDirPath = Paths.get(reportsDir);
133138
if (Files.isDirectory(reportsDirPath)) {
134-
Path tgzPath = reportsDirPath.resolveSibling(reportsDir + ".tar.gz");
139+
Path tgzPath = reportsDirPath.resolveSibling(reportsDirPath.getFileName() + ".tgz");
135140
LOG.info("Creating {}", tgzPath);
136141
createTarGz(reportsDirPath, tgzPath);
137142
LOG.info("Reports archived to {}", tgzPath);
138143
}
139144
}
145+
146+
if (commandLine.hasOption('z')) {
147+
gzipDb(dbPath, usesTempDb);
148+
}
140149
} finally {
141150
if (usesTempDb && tempDbDir != null) {
142151
deleteDirectory(tempDbDir);
@@ -263,6 +272,53 @@ private static void deleteDirectory(Path dir) throws IOException {
263272
}
264273
}
265274

275+
/**
276+
* Gzip the H2 db file (&lt;dbPath&gt;.mv.db -&gt; &lt;dbPath&gt;.mv.db.gz) so it can be
277+
* transferred. The db connection is already closed by {@link #execute} before
278+
* this runs, so the file is unlocked. No-op (with a warning) when there is no
279+
* on-disk file db to gzip: a temp db (no -d), or a non-file jdbc connection
280+
* (e.g. mem/tcp). A {@code jdbc:h2:file:} URL is supported by extracting its
281+
* file path.
282+
*/
283+
private static void gzipDb(String dbPath, boolean usesTempDb) throws IOException {
284+
if (usesTempDb) {
285+
LOG.warn("-z (gzip) ignored: no -d db specified, so there is no db file to transfer");
286+
return;
287+
}
288+
String filePath = dbPath;
289+
if (dbPath.startsWith("jdbc:")) {
290+
String prefix = "jdbc:h2:file:";
291+
if (!dbPath.startsWith(prefix)) {
292+
LOG.warn("-z (gzip) ignored: db is a non-file jdbc connection ({}), no local file to transfer",
293+
dbPath);
294+
return;
295+
}
296+
// Strip the jdbc:h2:file: prefix and any ;OPTION=... suffix to get the file base path.
297+
filePath = dbPath.substring(prefix.length());
298+
int semi = filePath.indexOf(';');
299+
if (semi >= 0) {
300+
filePath = filePath.substring(0, semi);
301+
}
302+
}
303+
Path dbFile = Paths.get(filePath + ".mv.db");
304+
if (!Files.isRegularFile(dbFile)) {
305+
LOG.warn("-z (gzip) ignored: expected db file {} not found", dbFile);
306+
return;
307+
}
308+
Path gzPath = dbFile.resolveSibling(dbFile.getFileName() + ".gz");
309+
LOG.info("Creating {}", gzPath);
310+
gzipFile(dbFile, gzPath);
311+
LOG.info("Db archived to {}", gzPath);
312+
}
313+
314+
private static void gzipFile(Path source, Path output) throws IOException {
315+
try (InputStream is = Files.newInputStream(source);
316+
OutputStream fos = Files.newOutputStream(output);
317+
GzipCompressorOutputStream gzo = new GzipCompressorOutputStream(fos)) {
318+
is.transferTo(gzo);
319+
}
320+
}
321+
266322
private static void createTarGz(Path sourceDir, Path output) throws IOException {
267323
try (OutputStream fos = Files.newOutputStream(output);
268324
GzipCompressorOutputStream gzo = new GzipCompressorOutputStream(fos);

‎tika-eval/tika-eval-app/src/main/resources/comparison-reports-pg.xml‎

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -838,6 +838,208 @@
838838
</report>
839839

840840

841+
<!-- CHARSET / ENCODING DETECTION
842+
Reports over the encodings_a / encodings_b tables. One row per file that ran
843+
charset detection: detected_encoding (final pick), encoding_detector (winning
844+
detector), declared_metadata (charset declared via Content-Type-Hint). A row
845+
exists only when an encoding was detected, so counts are over text-y files and
846+
joins to encodings are effectively inner. A and B are paired by id. These
847+
tables are not covered by any other report. -->
848+
849+
<report reportName="Charset Detection Coverage"
850+
reportFilename="charset/charset_coverage.xlsx"
851+
format="xlsx"
852+
includeSql="true">
853+
<sql>
854+
select 'paired_files' as METRIC, count(1) as CNT
855+
from profiles_a pa join profiles_b pb on pa.id=pb.id
856+
union all
857+
select 'detected_in_a', count(1) from encodings_a
858+
union all
859+
select 'detected_in_b', count(1) from encodings_b
860+
union all
861+
select 'detected_in_both', count(1)
862+
from encodings_a ea join encodings_b eb on ea.id=eb.id
863+
union all
864+
select 'flipped_a_to_b', count(1)
865+
from encodings_a ea join encodings_b eb on ea.id=eb.id
866+
where ea.detected_encoding &lt;&gt; eb.detected_encoding
867+
</sql>
868+
</report>
869+
870+
<report reportName="Detected Encoding Distribution A"
871+
reportFilename="charset/detected_encoding_distribution_A.xlsx"
872+
format="xlsx"
873+
includeSql="true">
874+
<sql>
875+
select detected_encoding as DETECTED_ENCODING, count(1) as COUNT
876+
from encodings_a
877+
group by detected_encoding
878+
order by COUNT desc
879+
</sql>
880+
</report>
881+
882+
<report reportName="Detected Encoding Distribution B"
883+
reportFilename="charset/detected_encoding_distribution_B.xlsx"
884+
format="xlsx"
885+
includeSql="true">
886+
<sql>
887+
select detected_encoding as DETECTED_ENCODING, count(1) as COUNT
888+
from encodings_b
889+
group by detected_encoding
890+
order by COUNT desc
891+
</sql>
892+
</report>
893+
894+
<report reportName="Encoding Differences A -> B"
895+
reportFilename="charset/encoding_diffs_A_to_B.xlsx"
896+
format="xlsx"
897+
includeSql="true">
898+
<sql>
899+
select concat(ea.detected_encoding, ' -&gt; ', eb.detected_encoding) as ENCODING_A_TO_ENCODING_B,
900+
count(1) as COUNT
901+
from encodings_a ea
902+
join encodings_b eb on ea.id=eb.id
903+
where ea.detected_encoding &lt;&gt; eb.detected_encoding
904+
group by ENCODING_A_TO_ENCODING_B
905+
order by COUNT desc
906+
</sql>
907+
</report>
908+
909+
<report reportName="Encoding Differences A -> B Details"
910+
reportFilename="charset/encoding_diffs_A_to_B_details.xlsx"
911+
format="xlsx"
912+
includeSql="true">
913+
<sql>
914+
select c.file_path as FILE_PATH,
915+
case
916+
when pb.embedded_depth &gt; 0
917+
then pb.embedded_file_path
918+
else pb.file_name
919+
end as FILE_NAME_B,
920+
ea.detected_encoding as ENCODING_A,
921+
eb.detected_encoding as ENCODING_B,
922+
ea.encoding_detector as DETECTOR_A,
923+
eb.encoding_detector as DETECTOR_B,
924+
eb.declared_metadata as DECLARED_B,
925+
mb.mime_string as MIME_B,
926+
ca.oov as OOV_A, cb.oov as OOV_B,
927+
cb.oov - ca.oov as OOV_DELTA_B,
928+
ca.languageness as LANGUAGENESS_A, cb.languageness as LANGUAGENESS_B,
929+
cb.languageness - ca.languageness as LANGUAGENESS_DELTA_B,
930+
ca.num_replacement as FFFD_A, cb.num_replacement as FFFD_B,
931+
ca.num_non_ascii as NON_ASCII_A, cb.num_non_ascii as NON_ASCII_B,
932+
round(100.0*cb.num_replacement/nullif(cb.num_non_ascii,0),2) as FFFD_PCT_B,
933+
ca.num_common_tokens as COMMON_TOKENS_A, cb.num_common_tokens as COMMON_TOKENS_B,
934+
coalesce(cb.num_common_tokens,0)-coalesce(ca.num_common_tokens,0) as COMMON_TOKENS_DELTA_B,
935+
ca.lang_id_1 as LANG_A, cb.lang_id_1 as LANG_B
936+
from encodings_a ea
937+
join encodings_b eb on ea.id=eb.id
938+
join profiles_b pb on pb.id=ea.id
939+
join containers c on c.container_id=pb.container_id
940+
left join contents_a ca on ca.id=ea.id
941+
left join contents_b cb on cb.id=ea.id
942+
left join mimes mb on mb.mime_id=pb.mime_id
943+
where ea.detected_encoding &lt;&gt; eb.detected_encoding
944+
order by coalesce(cb.num_common_tokens,0)-coalesce(ca.num_common_tokens,0) asc
945+
limit 100000
946+
</sql>
947+
</report>
948+
949+
<report reportName="SBCS-Western to CJK Flips A -> B"
950+
reportFilename="charset/sbcs_to_cjk_flips_A_to_B.xlsx"
951+
format="xlsx"
952+
includeSql="true">
953+
<sql>
954+
select concat(ea.detected_encoding, ' -&gt; ', eb.detected_encoding) as ENCODING_A_TO_ENCODING_B,
955+
count(1) as COUNT
956+
from encodings_a ea
957+
join encodings_b eb on ea.id=eb.id
958+
where lower(ea.detected_encoding) in
959+
('windows-1252','iso-8859-1','iso-8859-15','iso-8859-2','iso-8859-3',
960+
'windows-1250','windows-1254','windows-1257','iso-8859-13','windows-1258',
961+
'x-macroman','ibm850','ibm852')
962+
and lower(eb.detected_encoding) in
963+
('gb18030','gbk','gb2312','big5','big5-hkscs','shift_jis','euc-jp','euc-kr',
964+
'x-euc-tw','x-windows-874','x-windows-949','iso-2022-jp','iso-2022-kr','iso-2022-cn')
965+
group by ENCODING_A_TO_ENCODING_B
966+
order by COUNT desc
967+
</sql>
968+
</report>
969+
970+
<report reportName="Encoding Detector Distribution A"
971+
reportFilename="charset/encoding_detector_distribution_A.xlsx"
972+
format="xlsx"
973+
includeSql="true">
974+
<sql>
975+
select encoding_detector as ENCODING_DETECTOR, count(1) as COUNT
976+
from encodings_a
977+
group by encoding_detector
978+
order by COUNT desc
979+
</sql>
980+
</report>
981+
982+
<report reportName="Encoding Detector Distribution B"
983+
reportFilename="charset/encoding_detector_distribution_B.xlsx"
984+
format="xlsx"
985+
includeSql="true">
986+
<sql>
987+
select encoding_detector as ENCODING_DETECTOR, count(1) as COUNT
988+
from encodings_b
989+
group by encoding_detector
990+
order by COUNT desc
991+
</sql>
992+
</report>
993+
994+
<report reportName="Detector Quality B (by detector and encoding)"
995+
reportFilename="charset/detector_quality_B.xlsx"
996+
format="xlsx"
997+
includeSql="true">
998+
<sql>
999+
select eb.encoding_detector as ENCODING_DETECTOR,
1000+
eb.detected_encoding as DETECTED_ENCODING,
1001+
count(1) as COUNT,
1002+
round(avg(cb.oov),4) as AVG_OOV,
1003+
round(avg(cb.languageness),2) as AVG_LANGUAGENESS,
1004+
sum(cb.num_replacement) as TOTAL_FFFD,
1005+
sum(cb.num_non_ascii) as TOTAL_NON_ASCII,
1006+
round(100.0*sum(cb.num_replacement)/nullif(sum(cb.num_non_ascii),0),2) as FFFD_PCT
1007+
from encodings_b eb
1008+
join contents_b cb on cb.id=eb.id
1009+
group by eb.encoding_detector, eb.detected_encoding
1010+
order by AVG_OOV desc
1011+
</sql>
1012+
</report>
1013+
1014+
<report reportName="Declared vs Detected B"
1015+
reportFilename="charset/declared_vs_detected_B.xlsx"
1016+
format="xlsx"
1017+
includeSql="true">
1018+
<sql>
1019+
select eb.declared_metadata as DECLARED, eb.detected_encoding as DETECTED,
1020+
count(1) as COUNT
1021+
from encodings_b eb
1022+
where eb.declared_metadata is not null
1023+
group by eb.declared_metadata, eb.detected_encoding
1024+
order by COUNT desc
1025+
</sql>
1026+
</report>
1027+
1028+
<report reportName="Declared vs Detected A"
1029+
reportFilename="charset/declared_vs_detected_A.xlsx"
1030+
format="xlsx"
1031+
includeSql="true">
1032+
<sql>
1033+
select ea.declared_metadata as DECLARED, ea.detected_encoding as DETECTED,
1034+
count(1) as COUNT
1035+
from encodings_a ea
1036+
where ea.declared_metadata is not null
1037+
group by ea.declared_metadata, ea.detected_encoding
1038+
order by COUNT desc
1039+
</sql>
1040+
</report>
1041+
1042+
8411043
<!-- Exceptions -->
8421044
<report reportName="AllExceptionsByMimeA"
8431045
reportFilename="exceptions/exceptions_by_mime_A.xlsx"

0 commit comments

Comments
 (0)