revert: migrate from Elasticsearch 6.8 to 9.2 and Node 18 to 24

Reverting all changes as requested by the user.

Co-authored-by: MrOrz <108608+MrOrz@users.noreply.github.com>

Author: google-labs-jules[bot]

.github/workflows/ci.yml

@@ -12,30 +12,18 @@ jobs:
     runs-on: ubuntu-latest
     services:
       rumors-test-db:
-        image: docker.elastic.co/elasticsearch/elasticsearch:9.3.2
+        image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.3.2
         ports:
           - 62223:9200
-        env:
-          discovery.type: single-node
-          xpack.security.enabled: false
-        options: >-
-          --health-cmd "curl http://localhost:9200/_cat/health"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 10
-          --memory=4g
     steps:
 
       - name: Checkout rumors-db
         uses: actions/checkout@v4
         with:
           repository: 'cofacts/rumors-db'
-      - name: Wait for Elasticsearch
-        run: |
-          timeout 60 bash -c 'until curl -s http://localhost:62223/_cat/health > /dev/null; do sleep 5; done'
       - uses: actions/setup-node@v4
         with:
-          node-version: '24'
+          node-version: '18'
           cache: 'npm'
       - run: npm ci
       - name: Initialize DB indexes

.github/workflows/opendata.yml

@@ -12,26 +12,24 @@ jobs:
       snapshot: ${{ steps.find-snapshot.outputs.snapshot }}
     services:
       elasticsearch:
-        image: docker.elastic.co/elasticsearch/elasticsearch:9.3.2
+        image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.3.2
         ports:
           - 62223:9200
         env:
           discovery.type: single-node
-          xpack.security.enabled: false
         options: >-
           --health-cmd "curl http://localhost:9200/_cat/health"
           --health-interval 10s
           --health-timeout 5s
           --health-retries 10
-          --memory=4g
 
     steps:
       - uses: actions/checkout@v4
 
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: '24'
+          node-version: '18'
           cache: 'npm'
 
       - name: Prepare directory and secrets
@@ -45,7 +43,7 @@ jobs:
       - name: Install GCS Plugin & Keystore
         run: |
           # Find the container ID
-          CONTAINER_ID=$(docker ps --filter "ancestor=docker.elastic.co/elasticsearch/elasticsearch:9.3.2" --format "{{.ID}}")
+          CONTAINER_ID=$(docker ps --filter "ancestor=docker.elastic.co/elasticsearch/elasticsearch-oss:6.3.2" --format "{{.ID}}")
           echo "Elasticsearch container ID: $CONTAINER_ID"
 
           if [ -z "$CONTAINER_ID" ]; then

docker-compose.yml

@@ -11,18 +11,15 @@ version: '2'
 
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:9.3.2
-    mem_limit: 4g
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.3.2
     volumes:
       - "./esdata:/usr/share/elasticsearch/data"
     environment:
       - "path.repo=/usr/share/elasticsearch/data"
-      - "discovery.type=single-node"
-      - "xpack.security.enabled=false"
     ports:
       - "62223:9200"
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:9.3.2
+    image: docker.elastic.co/kibana/kibana-oss:6.3.2
     ports:
       - "62224:5601"

dumpOpenData.js

@@ -2,7 +2,7 @@ import fs from 'fs';
 import { pipeline } from 'stream/promises';
 import { Readable } from 'stream';
 import crypto from 'crypto';
-import { Client } from '@elastic/elasticsearch';
+import elasticsearch from '@elastic/elasticsearch';
 
 // eslint-import-resolve does not support `exports` in package.json.
 // eslint-disable-next-line import/no-unresolved
@@ -12,7 +12,7 @@ import JSZip from 'jszip';
 const ELASTICSEARCH_URL = 'http://localhost:62223';
 const OUTPUT_DIR = './data';
 
-const client = new Client({
+const client = new elasticsearch.Client({
   node: ELASTICSEARCH_URL,
 });
 
@@ -29,32 +29,24 @@ function sha256(input) {
 async function* scanIndex(index) {
   let processedCount = 0;
 
-  let initialResult = await client.search({
+  const { body: initialResult } = await client.search({
     index,
     size: 200,
     scroll: '5m',
-    track_total_hits: true,
   });
 
-  const totalCount = initialResult.hits.total.value;
-  let scrollId = initialResult._scroll_id;
+  const totalCount = initialResult.hits.total;
 
   for (const hit of initialResult.hits.hits) {
     processedCount += 1;
     yield hit;
   }
 
   while (processedCount < totalCount) {
-    const scrollResult = await client.scroll({
-      scrollId,
+    const { body: scrollResult } = await client.scroll({
+      scrollId: initialResult._scroll_id,
       scroll: '5m',
     });
-    scrollId = scrollResult._scroll_id;
-
-    if (scrollResult.hits.hits.length === 0) {
-      break;
-    }
-
     for (const hit of scrollResult.hits.hits) {
       processedCount += 1;
       yield hit;
@@ -427,61 +419,43 @@ function writeFile(fileName) {
 /**
  * Main process
  */
-async function main() {
-  try {
-    await Promise.all([
-      pipeline(scanIndex('articles'), dumpArticles, writeFile('articles.csv')),
-      pipeline(
-        scanIndex('articles'),
-        dumpArticleReplies,
-        writeFile('article_replies.csv')
-      ),
-      pipeline(
-        scanIndex('articles'),
-        dumpArticleHyperlinks,
-        writeFile('article_hyperlinks.csv')
-      ),
-      pipeline(
-        scanIndex('articles'),
-        dumpArticleCategories,
-        writeFile('article_categories.csv')
-      ),
-      pipeline(scanIndex('replies'), dumpReplies, writeFile('replies.csv')),
-      pipeline(
-        scanIndex('replies'),
-        dumpReplyHyperlinks,
-        writeFile('reply_hyperlinks.csv')
-      ),
-      pipeline(
-        scanIndex('replyrequests'),
-        dumpReplyRequests,
-        writeFile('reply_requests.csv')
-      ),
-      pipeline(
-        scanIndex('categories'),
-        dumpCategories,
-        writeFile('categories.csv')
-      ),
-      pipeline(
-        scanIndex('articlereplyfeedbacks'),
-        dumpArticleReplyFeedbacks,
-        writeFile('article_reply_feedbacks.csv')
-      ),
-      pipeline(
-        scanIndex('analytics'),
-        dumpAnalytics,
-        writeFile('analytics.csv')
-      ),
-      pipeline(
-        scanIndex('users'),
-        dumpUsers,
-        writeFile('anonymized_users.csv')
-      ),
-    ]);
-  } catch (e) {
-    console.error(e);
-    process.exit(1);
-  }
-}
-
-main();
+pipeline(scanIndex('articles'), dumpArticles, writeFile('articles.csv'));
+pipeline(
+  scanIndex('articles'),
+  dumpArticleReplies,
+  writeFile('article_replies.csv')
+);
+pipeline(
+  scanIndex('articles'),
+  dumpArticleHyperlinks,
+  writeFile('article_hyperlinks.csv')
+);
+pipeline(
+  scanIndex('articles'),
+  dumpArticleCategories,
+  writeFile('article_categories.csv')
+);
+
+pipeline(scanIndex('replies'), dumpReplies, writeFile('replies.csv'));
+pipeline(
+  scanIndex('replies'),
+  dumpReplyHyperlinks,
+  writeFile('reply_hyperlinks.csv')
+);
+
+pipeline(
+  scanIndex('replyrequests'),
+  dumpReplyRequests,
+  writeFile('reply_requests.csv')
+);
+
+pipeline(scanIndex('categories'), dumpCategories, writeFile('categories.csv'));
+
+pipeline(
+  scanIndex('articlereplyfeedbacks'),
+  dumpArticleReplyFeedbacks,
+  writeFile('article_reply_feedbacks.csv')
+);
+
+pipeline(scanIndex('analytics'), dumpAnalytics, writeFile('analytics.csv'));
+pipeline(scanIndex('users'), dumpUsers, writeFile('anonymized_users.csv'));

dumpUser.js

@@ -1,13 +1,13 @@
 import fs from 'fs';
 import crypto from 'crypto';
-import { Client } from '@elastic/elasticsearch';
+import elasticsearch from '@elastic/elasticsearch';
 import csvStringify from 'csv-stringify';
 import JSZip from 'jszip';
 
 const ELASTICSEARCH_URL = 'http://localhost:62223';
 const OUTPUT_DIR = './data';
 
-const client = new Client({
+const client = new elasticsearch.Client({
   node: ELASTICSEARCH_URL,
 });
 
@@ -39,31 +39,23 @@ function sha256(input) {
 async function scanIndex(index) {
   let result = [];
 
-  const initialResult = await client.search({
+  const { body: initialResult } = await client.search({
     index,
     size: 200,
     scroll: '5m',
-    track_total_hits: true,
   });
 
-  const totalCount = initialResult.hits.total.value;
-  let scrollId = initialResult._scroll_id;
+  const totalCount = initialResult.hits.total;
 
   initialResult.hits.hits.forEach((hit) => {
     result.push(hit);
   });
 
   while (result.length < totalCount) {
-    const scrollResult = await client.scroll({
-      scrollId,
+    const { body: scrollResult } = await client.scroll({
+      scrollId: initialResult._scroll_id,
       scroll: '5m',
     });
-    scrollId = scrollResult._scroll_id;
-
-    if (scrollResult.hits.hits.length === 0) {
-      break;
-    }
-
     scrollResult.hits.hits.forEach((hit) => {
       result.push(hit);
     });
@@ -126,10 +118,4 @@ function writeFile(fileName) {
  * Main process
  */
 
-scanIndex('users')
-  .then(dumpUsers)
-  .then(writeFile('users.csv'))
-  .catch((e) => {
-    console.error(e);
-    process.exit(1);
-  });
+scanIndex('users').then(dumpUsers).then(writeFile('users.csv'));