🧪 test: Milestone: all test-storage pass ✨

Author: vict0rsch

test/data/urls.json

@@ -75,7 +75,7 @@
         "https://hal.science/hal-03171076",
         "https://hal.science/hal-03171076/file/paper_22.pdf",
         {
-            "venue": "DGMM"
+            "venue": "International Joint Conference on Discrete Geometry and Mathematical Morphology"
         }
     ],
     "ijcai": [
@@ -115,7 +115,8 @@
         "https://www.mdpi.com/2076-328X/13/12/961",
         "https://www.mdpi.com/2076-328X/13/12/961/pdf",
         {
-            "noPdf": true
+            "noPdf": true,
+            "venue": "Behavioral Sciences"
         }
     ],
     "nature": [
@@ -141,7 +142,8 @@
         "https://academic.oup.com/brain/article/147/3/743/7617466",
         "https://watermark02.silverchair.com/awae043.pdf?token=AQECAHi208BE49Ooan9kkhW_Ercy7Dm3ZL_9Cf3qfKAc485ysgAAA1QwggNQBgkqhkiG9w0BBwagggNBMIIDPQIBADCCAzYGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM7kysuY_cs4rR2-jhAgEQgIIDB4P9PivhWOCC8mFmhsZNyZflrzom2JHSmlFrmTEMfpOjewEbs6ujIgmf466LnSL_8BoyFCpXuwfCTp_EVwkQmBfdgVdL8ckkyt6k46nOXKAGKPHOzcv0EDIJaj6R3V9RUMikRSzI4A09fYlKZKE5LmYUxx41i85wYw00VErOHEmDV_yhQXD2VL168iDCDHCJii7QgbCh-m6s6uN-6PB78FXInRQihR4RgscNUHtb3zVU50Hiic0yXYp0wpkcC6j30tAFJlG9fTi4ZYJApvXw6dFTWL_z1kgn0YEg05TNqyCBA3KykBhcZfwB_e0nFiezf1K4ymsgseRyBd5y-e2SRdFw1cpYToGo2gvj53mKqRyWf2R-arEcNKHsx88WHfJsccUXU-ypM2l_jMGfJaKId3FXNXpOSd5zsUy4JoZ93tQgTlllxa9nOsAN9JwcC1y7wHocJ6YuornKusv7ERioglKJdwnbgXb8TUcYxlaiGiPMjkFubq97D9v3hJxsdvjCubkHrvaBG8GUvGd5rASrCUZjPnPzqL52UYKaDOHWQidk26mVV3rbJ1t9GnqZ1VXyiRO0PYChR8hQ46hpsgSmODx7LNBe1Um515BjcTKY6x6Vo-QXEUvj6EQ2yk_8KPwriOAXbKv2CJZNejsVVpx7v07K2XhFwNWthSOILMcV_abQxZxOToNPf59mPK_rTCW_RJkCZdJuklSyR6jY0EXIjPEKJBfIgsRn584ek1jNbOmAAy5qi9DDwPp-8XP-ctS5iN1U6G0bf5XW1umDCKwKrwuHAHgVc-ABiorChqpBPen3NhBPgCQBMyKr0u3_dBjiIrr599qn5a8U0ZsPOyCw1_RDThlP_qMsEdOvGZDwvmQtOQqD_rYgGh2RFDPjOR-GKix6pjFcwLYImoWc0iCjX1JhQJw4ReKI5F6n2Uavhp2Tsz8YrgSAZPDN8skS8Wqqp2cM886EkywyF6h6izg74nD5mYnzK6RUNM2Nu4_MV0jJ1xqsw2B-ou41yMDIQWd7mUqngBA7Jhg",
         {
-            "noPdf": true
+            "noPdf": true,
+            "venue": "Brain"
         }
     ],
     "pmc": [

test/test-storage.js

@@ -148,6 +148,8 @@ describe("Test paper detection and storage", function () {
             expect(missingTests).toEqual([]);
         });
     });
+
+    for (const [orderIdx, order] of orders.entries()) {
         describe("Parsing order: " + order, function () {
             before(async function () {
                 // create browser
@@ -159,16 +161,17 @@ describe("Test paper detection and storage", function () {
                 // visit all relevant urls
                 // all abstracts then all pdfs
 
-                const indices = order === "abs;pdf" ? [0, 1] : [1, 0];
+                const sourceOrder = order === "abs;pdf" ? [0, 1] : [1, 0];
 
-                for (const t of indices) {
-                    for (const [idx, targets] of Object.values(urls).entries()) {
+                for (const sourceOrderIdx of sourceOrder) {
+                    for (const [targetIdx, targets] of Object.values(urls).entries()) {
                         // for each target url (abstract, pdf), visit the url
                         // and wait a little for it to load
-
+                        const isPDF = sourceOrderIdx === 1;
+                        if (isPDF && targets[2]?.noPdf) continue;
                         // filter out the additional test configs
                         const targetUrls = targets.filter((u) => typeof u === "string");
-                        if (t >= targetUrls.length) {
+                        if (sourceOrderIdx >= targetUrls.length) {
                             continue;
                         }
                         if (targets.length > 2) {
@@ -179,9 +182,14 @@ describe("Test paper detection and storage", function () {
                                 continue;
                             }
                         }
-                        const target = targetUrls[t];
+                        // TODO: handle no pdf but still check abstracts
+                        const target = targetUrls[sourceOrderIdx];
                         // log prefix
-                        const n = idx + (o > 0 ? 1 - t : t) * nUrls + 1;
+                        const n =
+                            targetIdx +
+                            (orderIdx > 0 ? 1 - sourceOrderIdx : sourceOrderIdx) *
+                                nUrls +
+                            1;
                         const prefix = `${" ".repeat(6)}(${n}/${nUrls * 2})`;
                         console.log(`${prefix} Going to: ${target}`);
 
@@ -231,8 +239,11 @@ describe("Test paper detection and storage", function () {
                     const filteredSources = sources.filter(
                         (s) => !ignoreSingleOrder(s, urls, order)
                     );
-                    const memoryCounts = allAttributes(memoryPapers, "count");
-                    expect(memoryCounts.every((c) => c >= 2)).toBeTruthy();
+                    for (const paper of Object.values(memoryPapers)) {
+                        let targetMinCount = 2;
+                        if (urls[paper.source][2]?.noPdf) targetMinCount--;
+                        expect(paper.count).toBeGreaterThanOrEqual(targetMinCount);
+                    }
                 });
 
                 it("No undefined keys", async function () {
@@ -265,10 +276,12 @@ describe("Test paper detection and storage", function () {
                             expect(papers?.length).toBe(1);
                         });
 
-                        it("#count is 2", function () {
+                        it("#count is appropriate for the source", function () {
                             const paper = paperForSource(source, memoryPapers);
                             expect(paper).toBeDefined();
-                            expect(paper?.count).toBeGreaterThanOrEqual(2);
+                            let sourceCount = 2;
+                            if (urls[source][2]?.noPdf) sourceCount--;
+                            expect(paper?.count).toBeGreaterThanOrEqual(sourceCount);
                         });
 
                         // more tests parameterized in the 3rd item in the list for this source

test/testConfig.yaml

@@ -12,7 +12,7 @@ keepOpen:
 # |> test-storage, test-duplicates
 pageTimeout:
   type: int
-  defaultValue: 1000
+  defaultValue: 100
 
 # Max number of sources to iterate through for debug. -1 is all sources.
 # |> test-storage, test-duplicates