fix: use lastActionId for matomo_log_action (#9)

Author: revolunet

src/lib.mjs

@@ -95,12 +95,14 @@ export const copyTableBatch = async ({
  * Fetch valid action IDs for a site
  * @param {SourceDB} sourceDB
  * @param {number} idsite
+ * @param {number} [lastActionId]
  * @param {(msg: string) => void} [log]
  * @returns {Promise<Set<number>>}
  */
 export const fetchValidActionIds = async (
   sourceDB,
   idsite,
+  lastActionId = 0,
   log = console.log,
 ) => {
   const validActions = new Set();
@@ -115,15 +117,23 @@ export const fetchValidActionIds = async (
       .distinct()
       .where("idsite", "=", idsite)
       .where("server_time", ">=", new Date("2025-01-01"))
+      .where(({ eb }) =>
+        eb.or([
+          eb("idaction_name", ">", lastActionId),
+          eb("idaction_url", ">", lastActionId),
+        ]),
+      )
       .limit(ACTION_BATCH)
       .offset(actionOffset)
       .execute();
 
     if (batch.length === 0) break;
 
     for (const r of batch) {
-      if (r.idaction_name !== null) validActions.add(r.idaction_name);
-      if (r.idaction_url !== null) validActions.add(r.idaction_url);
+      if (r.idaction_name !== null && r.idaction_name > lastActionId)
+        validActions.add(r.idaction_name);
+      if (r.idaction_url !== null && r.idaction_url > lastActionId)
+        validActions.add(r.idaction_url);
     }
     actionOffset += ACTION_BATCH;
     log(` - fetched ${actionOffset} rows, ${validActions.size} unique actions`);
@@ -192,7 +202,7 @@ export const copyActions = async ({
       .values(sourceRows)
       .executeTakeFirst();
     log(
-      ` - inserted ${insertion.numInsertedOrUpdatedRows} (processed ${chunkIndex}/${validActionsArray.length})`,
+      ` - inserted ${insertion.numInsertedOrUpdatedRows} (processed ${Math.min(chunkIndex, validActionsArray.length)}/${validActionsArray.length})`,
     );
   }
 
@@ -278,7 +288,7 @@ export const importSite = async ({
 
   // Copy actions
   log(`copy matomo_log_action from ${lastActionId}...`);
-  const validActions = await fetchValidActionIds(sourceDB, idsite, log);
+  const validActions = await fetchValidActionIds(sourceDB, idsite, lastActionId, log);
   await copyActions({
     sourceDB,
     targetDB,

src/lib.test.mjs

@@ -240,14 +240,14 @@ describe("copyTableBatch", () => {
 describe("fetchValidActionIds", () => {
   it("should return empty set when no actions found", async () => {
     const sourceDB = createMockDB({ selectResults: [[]] });
-    const result = await fetchValidActionIds(sourceDB, 1, () => {});
+    const result = await fetchValidActionIds(sourceDB, 1, 0, () => {});
 
     assert.strictEqual(result.size, 0);
   });
 
   it("should query matomo_log_link_visit_action with correct columns", async () => {
     const sourceDB = createMockDB({ selectResults: [[]] });
-    await fetchValidActionIds(sourceDB, 1, () => {});
+    await fetchValidActionIds(sourceDB, 1, 0, () => {});
 
     assert.deepStrictEqual(sourceDB._calls.selectFrom, [
       "matomo_log_link_visit_action",
@@ -262,7 +262,7 @@ describe("fetchValidActionIds", () => {
 
   it("should apply idsite and date filters", async () => {
     const sourceDB = createMockDB({ selectResults: [[]] });
-    await fetchValidActionIds(sourceDB, 42, () => {});
+    await fetchValidActionIds(sourceDB, 42, 0, () => {});
 
     // Check idsite filter
     assert.deepStrictEqual(sourceDB._calls.where[0], ["idsite", "=", 42]);
@@ -273,7 +273,7 @@ describe("fetchValidActionIds", () => {
 
   it("should use pagination with limit and offset", async () => {
     const sourceDB = createMockDB({ selectResults: [[]] });
-    await fetchValidActionIds(sourceDB, 1, () => {});
+    await fetchValidActionIds(sourceDB, 1, 0, () => {});
 
     assert.deepStrictEqual(sourceDB._calls.limit, [1000]);
     assert.deepStrictEqual(sourceDB._calls.offset, [0]);
@@ -291,14 +291,34 @@ describe("fetchValidActionIds", () => {
       ],
     });
 
-    const result = await fetchValidActionIds(sourceDB, 1, () => {});
+    const result = await fetchValidActionIds(sourceDB, 1, 0, () => {});
 
     assert.strictEqual(result.size, 4);
     assert.ok(result.has(1));
     assert.ok(result.has(2));
     assert.ok(result.has(3));
     assert.ok(result.has(4));
   });
+
+  it("should only collect action IDs greater than lastActionId", async () => {
+    const sourceDB = createMockDB({
+      selectResults: [
+        [
+          { idaction_name: 5, idaction_url: 10 },
+          { idaction_name: 3, idaction_url: 15 },
+        ],
+        [],
+      ],
+    });
+
+    const result = await fetchValidActionIds(sourceDB, 1, 8, () => {});
+
+    assert.strictEqual(result.size, 2);
+    assert.ok(result.has(10));
+    assert.ok(result.has(15));
+    assert.ok(!result.has(5));
+    assert.ok(!result.has(3));
+  });
 });
 
 describe("copyActions", () => {