Skip to content

Commit f09b37d

Browse files
bluczblucz
authored
Merge pull request #74 from opra-project/cleanup/import-pipeline
Clean up import pipeline: remove dead code, fix CI, add tests
2 parents 3e455b2 + d51d1d3 commit f09b37d

17 files changed

+247
-600
lines changed

‎.github/workflows/import.yml‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,20 +41,47 @@ jobs:
4141
with:
4242
deno-version: v1.x
4343

44+
- name: Download Oratory CSV
45+
id: oratory_csv
46+
if: github.event.inputs.source != 'autoeq'
47+
env:
48+
ORATORY_CSV_URL: ${{ secrets.ORATORY_CSV_URL }}
49+
run: |
50+
if [ -z "$ORATORY_CSV_URL" ]; then
51+
echo "::warning::ORATORY_CSV_URL secret not set, skipping oratory import"
52+
echo "oratory_csv_available=false" >> $GITHUB_OUTPUT
53+
exit 0
54+
fi
55+
# Convert Dropbox share link to direct download (dl=0 -> dl=1)
56+
DIRECT_URL="${ORATORY_CSV_URL/dl=0/dl=1}"
57+
curl -fsSL -o /tmp/oratory.csv "$DIRECT_URL"
58+
echo "oratory_csv_available=true" >> $GITHUB_OUTPUT
59+
echo "Downloaded oratory CSV ($(wc -l < /tmp/oratory.csv) lines)"
60+
4461
- name: Determine import flags
4562
id: flags
63+
env:
64+
SOURCE: ${{ github.event.inputs.source }}
65+
DRY_RUN: ${{ github.event.inputs.dry_run }}
66+
ORATORY_CSV_AVAILABLE: ${{ steps.oratory_csv.outputs.oratory_csv_available }}
4667
run: |
4768
FLAGS=""
48-
# Handle both manual trigger (workflow_dispatch) and scheduled runs
49-
SOURCE="${{ github.event.inputs.source }}"
50-
DRY_RUN="${{ github.event.inputs.dry_run }}"
5169
5270
if [ "$SOURCE" = "autoeq" ]; then
5371
FLAGS="--autoeq-only"
5472
elif [ "$SOURCE" = "oratory" ]; then
5573
FLAGS="--oratory-only"
5674
fi
5775
76+
# Add oratory CSV path if available
77+
if [ "$ORATORY_CSV_AVAILABLE" = "true" ]; then
78+
FLAGS="$FLAGS --oratory-csv /tmp/oratory.csv"
79+
elif [ "$SOURCE" != "autoeq" ]; then
80+
# Oratory CSV not available and not autoeq-only: fall back to autoeq-only
81+
echo "::warning::Oratory CSV not available, falling back to autoeq-only"
82+
FLAGS="--autoeq-only"
83+
fi
84+
5885
if [ "$DRY_RUN" = "true" ]; then
5986
FLAGS="$FLAGS --dry-run"
6087
fi
@@ -64,6 +91,8 @@ jobs:
6491
6592
- name: Run import
6693
id: import
94+
env:
95+
OPRA_CACHE_DIR: ${{ runner.temp }}/opra-cache
6796
run: |
6897
set +e # Don't exit on error
6998
deno run --allow-all tools/import.ts ${{ steps.flags.outputs.flags }} --output-json > import-summary.json 2>&1
@@ -85,7 +114,9 @@ jobs:
85114
86115
- name: Check for changes
87116
id: changes
88-
if: steps.flags.outputs.is_dry_run != 'true'
117+
env:
118+
IS_DRY_RUN: ${{ steps.flags.outputs.is_dry_run }}
119+
if: env.IS_DRY_RUN != 'true'
89120
run: |
90121
if [ -n "$(git status --porcelain)" ]; then
91122
echo "has_changes=true" >> $GITHUB_OUTPUT

‎CLAUDE.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,6 @@ database/vendors/<vendor_slug>/
6060
## Key Tools
6161

6262
- `tools/dist.ts` - Main build script (generates dist/ from database/)
63-
- `tools/import_autoeq.ts` - Imports data from AutoEQ project
64-
- `tools/merge.ts` - Merges database entries
65-
- `tools/preprocess_oratory.ts` - Preprocesses oratory1990 data
63+
- `tools/import.ts` - Unified import entry point for external data sources
64+
- `tools/autoeq/import.ts` - Imports data from the AutoEQ project
65+
- `tools/oratory/import.ts` - CSV-driven oratory1990 import with PDF parsing

‎deno.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
"tasks": {
44
"test": "deno test --allow-read --allow-write --allow-env --allow-net tests/",
55
"test:utils": "deno test --allow-read tests/utils_test.ts",
6-
"test:oratory": "deno test --allow-read --allow-write tests/oratory_*.ts",
7-
"test:autoeq": "deno test --allow-read tests/import_autoeq_test.ts",
6+
"test:oratory": "deno test --allow-read --allow-write --allow-env tests/oratory_*.ts tests/import_oratory_*.ts",
7+
"test:autoeq": "deno test --allow-read --allow-write --allow-env tests/import_autoeq_*.ts",
88
"test:coverage": "deno test --allow-read --allow-write --allow-env --allow-net --coverage=coverage tests/",
99
"build": "deno run --allow-read --allow-write --allow-run tools/dist.ts",
1010
"build:validate": "deno run --allow-read --allow-write --allow-run tools/dist.ts --validate-only",

‎tests/asset_paths_test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
* Run with: deno test --allow-read tests/asset_paths_test.ts
55
*/
66

7-
import { assertEquals } from "https://deno.land/std@0.203.0/assert/mod.ts";
7+
import { assertEquals } from "https://deno.land/std@0.224.0/assert/mod.ts";
88

99
import {
1010
getAssetPath,

‎tests/import_autoeq_test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import {
88
assertEquals,
99
assertAlmostEquals,
1010
assertThrows,
11-
} from "https://deno.land/std@0.203.0/assert/mod.ts";
11+
} from "https://deno.land/std@0.224.0/assert/mod.ts";
1212

1313
// Import from shared modules - no more code duplication!
1414
import { parseParametricEQ, mapTypeToSubtype } from "../tools/autoeq/parse_eq.ts";

‎tests/import_autoeq_unchanged_test.ts‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/**
2+
* Tests for AutoEQ unchanged-EQ detection
3+
*
4+
* Run with: deno test --allow-read --allow-write --allow-env tests/import_autoeq_unchanged_test.ts
5+
*/
6+
7+
import {
8+
assertEquals,
9+
} from "https://deno.land/std@0.224.0/assert/mod.ts";
10+
import { join } from "https://deno.land/std@0.224.0/path/mod.ts";
11+
import { ensureDir } from "https://deno.land/std@0.224.0/fs/mod.ts";
12+
13+
import { importAutoEQ } from "../tools/autoeq/import.ts";
14+
15+
// Helper to create a minimal AutoEQ source structure
16+
async function createTestSource(
17+
baseDir: string,
18+
measurer: string,
19+
type: string,
20+
productName: string,
21+
eqContent: string,
22+
) {
23+
const dir = join(baseDir, measurer, type, productName);
24+
await ensureDir(dir);
25+
await Deno.writeTextFile(
26+
join(dir, `${productName} ParametricEQ.txt`),
27+
eqContent,
28+
);
29+
}
30+
31+
const SAMPLE_EQ = `Preamp: -6.2 dB
32+
Filter 1: ON PK Fc 31 Hz Gain 4.5 dB Q 1.41
33+
Filter 2: ON PK Fc 62 Hz Gain -1.2 dB Q 1.00
34+
`;
35+
36+
const MODIFIED_EQ = `Preamp: -7.0 dB
37+
Filter 1: ON PK Fc 31 Hz Gain 5.0 dB Q 1.41
38+
Filter 2: ON PK Fc 62 Hz Gain -1.5 dB Q 1.00
39+
`;
40+
41+
Deno.test("importAutoEQ - detects unchanged EQ on second import", async () => {
42+
const tmpDir = await Deno.makeTempDir({ prefix: "opra_test_" });
43+
const srcDir = join(tmpDir, "source");
44+
const targetDir = join(tmpDir, "database");
45+
46+
try {
47+
await createTestSource(
48+
srcDir,
49+
"TestMeasurer",
50+
"over-ear",
51+
"TestVendor TestProduct",
52+
SAMPLE_EQ,
53+
);
54+
55+
// First import: should create new EQ
56+
const result1 = await importAutoEQ(srcDir, targetDir);
57+
assertEquals(result1.stats.newEqs, 1);
58+
assertEquals(result1.stats.updatedEqs, 0);
59+
assertEquals(result1.stats.unchangedEqs, 0);
60+
61+
// Second import with same content: should detect unchanged
62+
const result2 = await importAutoEQ(srcDir, targetDir);
63+
assertEquals(result2.stats.newEqs, 0);
64+
assertEquals(result2.stats.updatedEqs, 0);
65+
assertEquals(result2.stats.unchangedEqs, 1);
66+
} finally {
67+
await Deno.remove(tmpDir, { recursive: true });
68+
}
69+
});
70+
71+
Deno.test("importAutoEQ - detects updated EQ when content changes", async () => {
72+
const tmpDir = await Deno.makeTempDir({ prefix: "opra_test_" });
73+
const srcDir = join(tmpDir, "source");
74+
const targetDir = join(tmpDir, "database");
75+
76+
try {
77+
// First import
78+
await createTestSource(
79+
srcDir,
80+
"TestMeasurer",
81+
"over-ear",
82+
"TestVendor TestProduct",
83+
SAMPLE_EQ,
84+
);
85+
const result1 = await importAutoEQ(srcDir, targetDir);
86+
assertEquals(result1.stats.newEqs, 1);
87+
88+
// Modify the source EQ
89+
await createTestSource(
90+
srcDir,
91+
"TestMeasurer",
92+
"over-ear",
93+
"TestVendor TestProduct",
94+
MODIFIED_EQ,
95+
);
96+
97+
// Second import with changed content: should detect update
98+
const result2 = await importAutoEQ(srcDir, targetDir);
99+
assertEquals(result2.stats.newEqs, 0);
100+
assertEquals(result2.stats.updatedEqs, 1);
101+
assertEquals(result2.stats.unchangedEqs, 0);
102+
} finally {
103+
await Deno.remove(tmpDir, { recursive: true });
104+
}
105+
});

‎tests/import_oratory_dryrun_test.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/**
2+
* Tests for Oratory import dry-run behavior
3+
*
4+
* Run with: deno test --allow-read --allow-write --allow-env tests/import_oratory_dryrun_test.ts
5+
*/
6+
7+
import {
8+
assertEquals,
9+
} from "https://deno.land/std@0.224.0/assert/mod.ts";
10+
import { join } from "https://deno.land/std@0.224.0/path/mod.ts";
11+
12+
import { importOratory } from "../tools/oratory/import.ts";
13+
14+
Deno.test("importOratory dry-run - counts only valid rows", async () => {
15+
const tmpDir = await Deno.makeTempDir({ prefix: "opra_test_" });
16+
const csvPath = join(tmpDir, "test.csv");
17+
18+
// CSV with 3 rows: 2 valid, 1 missing link
19+
const csvContent = `Brand;Model;Comment;Target;Link
20+
Sony;WH-1000XM4;0;1;https://dropbox.com/test1.pdf?dl=0
21+
Sennheiser;HD650;0;1;https://dropbox.com/test2.pdf?dl=0
22+
BadRow;NoLink;0;1;
23+
`;
24+
25+
try {
26+
await Deno.writeTextFile(csvPath, csvContent);
27+
28+
const result = await importOratory(tmpDir, {
29+
csvPath,
30+
cacheDir: join(tmpDir, "pdf_cache"),
31+
dryRun: true,
32+
});
33+
34+
// Should count 2 valid rows, not 3 total rows
35+
assertEquals(result.stats.newEqs, 2);
36+
assertEquals(result.stats.errors, 0);
37+
} finally {
38+
await Deno.remove(tmpDir, { recursive: true });
39+
}
40+
});

‎tests/known_vendors_test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
import {
88
assertEquals,
9-
} from "https://deno.land/std@0.203.0/assert/mod.ts";
9+
} from "https://deno.land/std@0.224.0/assert/mod.ts";
1010

1111
import { KNOWN_VENDORS, VENDOR_ALIASES } from "../tools/known_vendors.ts";
1212

‎tests/oratory_parse_pdf_test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import {
88
assertEquals,
99
assertAlmostEquals,
10-
} from "https://deno.land/std@0.203.0/assert/mod.ts";
10+
} from "https://deno.land/std@0.224.0/assert/mod.ts";
1111

1212
import {
1313
parseOratoryPdfText,

‎tests/utils_test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
import {
88
assertEquals,
9-
} from "https://deno.land/std@0.203.0/assert/mod.ts";
9+
} from "https://deno.land/std@0.224.0/assert/mod.ts";
1010

1111
import {
1212
generateSlug,

0 commit comments

Comments
 (0)