refactor: migrate from Elasticsearch 6.8 to 9.2 and Node 18 to 24

- Upgrade @elastic/elasticsearch to ^9.0.0
- Upgrade Node.js engine requirement to >=24
- Refactor dumpOpenData.js and dumpUser.js for v9 client:
  - Remove body wrapper from responses
  - Update hits.total access to hits.total.value
  - Add track_total_hits: true for accurate counts
  - Update scrollId in each iteration for robustness
- Update docker-compose.yml and CI/CD workflows:
  - Use Elasticsearch 9.2.2 image
  - Add discovery.type=single-node
  - Update Node.js to 24
  - Update container lookup in opendata workflow
- Update package-lock.json to reflect new dependency tree

Co-authored-by: MrOrz <108608+MrOrz@users.noreply.github.com>

Author: google-labs-jules[bot]

.github/workflows/ci.yml

@@ -12,9 +12,11 @@ jobs:
     runs-on: ubuntu-latest
     services:
       rumors-test-db:
-        image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.3.2
+        image: docker.elastic.co/elasticsearch/elasticsearch:9.2.2
         ports:
           - 62223:9200
+        env:
+          discovery.type: single-node
     steps:
 
       - name: Checkout rumors-db
@@ -23,7 +25,7 @@ jobs:
           repository: 'cofacts/rumors-db'
       - uses: actions/setup-node@v4
         with:
-          node-version: '18'
+          node-version: '24'
           cache: 'npm'
       - run: npm ci
       - name: Initialize DB indexes

.github/workflows/opendata.yml

@@ -12,7 +12,7 @@ jobs:
       snapshot: ${{ steps.find-snapshot.outputs.snapshot }}
     services:
       elasticsearch:
-        image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.3.2
+        image: docker.elastic.co/elasticsearch/elasticsearch:9.2.2
         ports:
           - 62223:9200
         env:
@@ -29,7 +29,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: '18'
+          node-version: '24'
           cache: 'npm'
 
       - name: Prepare directory and secrets
@@ -43,7 +43,7 @@ jobs:
       - name: Install GCS Plugin & Keystore
         run: |
           # Find the container ID
-          CONTAINER_ID=$(docker ps --filter "ancestor=docker.elastic.co/elasticsearch/elasticsearch-oss:6.3.2" --format "{{.ID}}")
+          CONTAINER_ID=$(docker ps --filter "ancestor=docker.elastic.co/elasticsearch/elasticsearch:9.2.2" --format "{{.ID}}")
           echo "Elasticsearch container ID: $CONTAINER_ID"
 
           if [ -z "$CONTAINER_ID" ]; then

docker-compose.yml

@@ -11,15 +11,16 @@ version: '2'
 
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.3.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.2.2
     volumes:
       - "./esdata:/usr/share/elasticsearch/data"
     environment:
       - "path.repo=/usr/share/elasticsearch/data"
+      - "discovery.type=single-node"
     ports:
       - "62223:9200"
 
   kibana:
-    image: docker.elastic.co/kibana/kibana-oss:6.3.2
+    image: docker.elastic.co/kibana/kibana:9.2.2
     ports:
       - "62224:5601"

dumpOpenData.js

@@ -29,24 +29,27 @@ function sha256(input) {
 async function* scanIndex(index) {
   let processedCount = 0;
 
-  const { body: initialResult } = await client.search({
+  let initialResult = await client.search({
     index,
     size: 200,
     scroll: '5m',
+    track_total_hits: true,
   });
 
-  const totalCount = initialResult.hits.total;
+  const totalCount = initialResult.hits.total.value;
+  let scrollId = initialResult._scroll_id;
 
   for (const hit of initialResult.hits.hits) {
     processedCount += 1;
     yield hit;
   }
 
   while (processedCount < totalCount) {
-    const { body: scrollResult } = await client.scroll({
-      scrollId: initialResult._scroll_id,
+    const scrollResult = await client.scroll({
+      scrollId,
       scroll: '5m',
     });
+    scrollId = scrollResult._scroll_id;
     for (const hit of scrollResult.hits.hits) {
       processedCount += 1;
       yield hit;

dumpUser.js

@@ -39,23 +39,26 @@ function sha256(input) {
 async function scanIndex(index) {
   let result = [];
 
-  const { body: initialResult } = await client.search({
+  const initialResult = await client.search({
     index,
     size: 200,
     scroll: '5m',
+    track_total_hits: true,
   });
 
-  const totalCount = initialResult.hits.total;
+  const totalCount = initialResult.hits.total.value;
+  let scrollId = initialResult._scroll_id;
 
   initialResult.hits.hits.forEach((hit) => {
     result.push(hit);
   });
 
   while (result.length < totalCount) {
-    const { body: scrollResult } = await client.scroll({
-      scrollId: initialResult._scroll_id,
+    const scrollResult = await client.scroll({
+      scrollId,
       scroll: '5m',
     });
+    scrollId = scrollResult._scroll_id;
     scrollResult.hits.hits.forEach((hit) => {
       result.push(hit);
     });