Merge pull request #6300 from Countly/ar2rsawseen/next

MongoDB based offline geocoder

Author: ar2rsawseen

.github/workflows/main.yml

@@ -161,7 +161,9 @@ jobs:
 
       - name: Copy code
         shell: bash
-        run: cp -rf ./* /opt/countly
+        run: |
+          cp -rf ./* /opt/countly
+          cp ./bin/docker/postinstall.sh /etc/my_init.d
 
       - name: Remove plugin tests
         shell: bash

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 25.xx
+Dependencies:
+- Remove SQLite
+
 ## Version 25.03.6
 Enterprise Features:
 - [journey_engine] Publish toggle is converted into button and rearranged

Dockerfile-api

@@ -32,7 +32,7 @@ RUN apt-get update && \
 	cd /usr/src && \
 	wget https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tar.xz && \
 	tar -xf Python-3.8.12.tar.xz && \
-	apt-get install -y build-essential sudo zlib1g-dev libssl3 libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev && \
+	apt-get install -y build-essential sudo zlib1g-dev libssl3 libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev curl libbz2-dev && \
 	cd Python-3.8.12 && \
 	./configure --enable-optimizations --enable-shared && \
 	make && \
@@ -44,7 +44,6 @@ RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/downlo
 	dpkg -i /tmp/tini.deb && \
 	\
 	# modify standard distribution
-	apt-get update && apt-get install -y sqlite3 && \
 	./bin/docker/modify.sh && \
 	\
 	# preinstall
@@ -58,7 +57,7 @@ RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/downlo
 	\
 	# cleanup & chown
 	npm remove -y --no-save mocha nyc should supertest && \
-	apt-get remove -y git gcc g++ make automake autoconf libtool pkg-config unzip sqlite3 wget && \
+	apt-get remove -y git gcc g++ make automake autoconf libtool pkg-config unzip wget && \
 	apt-get install -y libgbm-dev libgbm1 gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils && \
 	apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
 	rm -rf test /tmp/* /tmp/.??* /var/tmp/* /var/tmp/.??* /var/log/* /root/.npm && \

Dockerfile-centos-api

@@ -46,7 +46,7 @@ RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/downlo
 	yum install -y epel-release && \
 	yum install -y pango.x86_64 libXcomposite.x86_64 libXcursor.x86_64 libXdamage.x86_64 libXext.x86_64 libXi.x86_64 libXtst.x86_64 cups-libs.x86_64 libXScrnSaver.x86_64 libXrandr.x86_64 GConf2.x86_64 alsa-lib.x86_64 atk.x86_64 gtk3.x86_64 xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc && \
 	yum install -y https://pkgs.sysadmins.ws/el8/base/x86_64/raven-release-1.0-2.el8.noarch.rpm && \
-	yum install -y wget openssl-devel make git libsqlite* sqlite unzip bzip2 && \
+	yum install -y wget openssl-devel make git unzip bzip2 && \
 	# modify standard distribution
 	./bin/docker/modify.sh && \
 	\

Dockerfile-centos-frontend

@@ -44,7 +44,7 @@ RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/downlo
         yum install -y epel-release && \
         yum install -y pango.x86_64 libXcomposite.x86_64 libXcursor.x86_64 libXdamage.x86_64 libXext.x86_64 libXi.x86_64 libXtst.x86_64 cups-libs.x86_64 libXScrnSaver.x86_64 libXrandr.x86_64 GConf2.x86_64 alsa-lib.x86_64 atk.x86_64 gtk3.x86_64 xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc && \
 	      yum install -y https://pkgs.sysadmins.ws/el8/base/x86_64/raven-release-1.0-2.el8.noarch.rpm && \
-        yum install -y wget openssl-devel make git sqlite libsqlite* unzip bzip2 && \
+        yum install -y wget openssl-devel make git unzip bzip2 && \
         # modify standard distribution
         ./bin/docker/modify.sh && \
         \

Dockerfile-core

@@ -29,7 +29,7 @@ RUN useradd -r -M -U -d /opt/countly -s /bin/false countly && \
     apt-get update && \
     apt-get install -y \
         # standard
-        build-essential libkrb5-dev git sqlite3 wget sudo \
+        build-essential libkrb5-dev git wget sudo \
         # nginx
         nginx \
         # puppeteer
@@ -76,7 +76,7 @@ RUN useradd -r -M -U -d /opt/countly -s /bin/false countly && \
     chown -R countly:countly /opt/countly && \
     # cleanup
     npm remove -y --no-save mocha nyc should supertest && \
-    apt-get remove -y build-essential libkrb5-dev sqlite3 wget \
+    apt-get remove -y build-essential libkrb5-dev wget \
         gcc g++ make binutils autoconf automake autotools-dev libtool pkg-config zlib1g-dev libcunit1-dev libssl-dev libxml2-dev libev-dev libevent-dev libjansson-dev libjemalloc-dev cython python3-dev python-setuptools && \
     apt-get install -y gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils && \
     apt-get clean && \

Dockerfile-frontend

@@ -28,7 +28,7 @@ RUN apt-get update && \
 	cd /usr/src && \
 	wget https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tar.xz && \
 	tar -xf Python-3.8.12.tar.xz && \
-	apt-get install -y build-essential sudo zlib1g-dev libssl3 libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev && \
+	apt-get install -y build-essential sudo zlib1g-dev libssl3 libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev curl libbz2-dev && \
 	cd Python-3.8.12 && \
 	./configure --enable-optimizations --enable-shared && \
 	make && \
@@ -40,7 +40,6 @@ RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/downlo
 	dpkg -i /tmp/tini.deb && \
 	\
 	# modify standard distribution
-	apt-get update && apt-get install -y sqlite3 && \
 	./bin/docker/modify.sh && \
 	\
 	# preinstall
@@ -56,7 +55,7 @@ RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/downlo
 	\
 	# cleanup & chown
 	npm remove -y --no-save mocha nyc should supertest puppeteer && \
-	apt-get remove -y git gcc g++ make automake autoconf libtool pkg-config unzip sqlite3 wget && \
+	apt-get remove -y git gcc g++ make automake autoconf libtool pkg-config unzip wget && \
 	apt-get autoremove -y && \
 	apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
 	rm -rf test /tmp/* /tmp/.??* /var/tmp/* /var/tmp/.??* /var/log/* /root/.npm && \

api/parts/data/usage.js

@@ -7,7 +7,7 @@
 var usage = {},
     common = require('./../../utils/common.js'),
     geoip = require('geoip-lite'),
-    geocoder = require('offline-geocoder')(),
+    geocoder = require('./../../../bin/offline-geocoder/src/index.js')(),
     log = require('../../utils/log.js')('api:usage'),
     async = require('async'),
     plugins = require('../../../plugins/pluginManager.js'),

bin/countly.install_rhel.sh

@@ -14,7 +14,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 bash "$DIR/scripts/logo.sh";
 
 # prerequisite per release
-sudo dnf install -y wget openssl-devel make git sqlite unzip bzip2
+sudo dnf install -y wget openssl-devel make git unzip bzip2
 
 sudo dnf install -y python3-pip
 sudo pip3 install pip --upgrade
@@ -87,7 +87,7 @@ sudo systemctl start sendmail > /dev/null || echo "sendmail service does not exi
 
 #install npm modules
 npm config set prefix "$DIR/../.local/"
-( cd "$DIR/.."; npm install argon2; npm install sqlite3 --build-from-source; npm install; )
+( cd "$DIR/.."; npm install argon2; npm install; )
 
 #install numactl
 sudo dnf install -y numactl
@@ -156,6 +156,8 @@ node "$DIR/scripts/install_plugins"
 
 #load city data into database
 nodejs "$DIR/scripts/loadCitiesInDb.js"
+nodejs "$DIR/offline-geocoder/scripts/download_geonames_data.js"
+nodejs "$DIR/offline-geocoder/scripts/import_geonames_mongodb.js"
 
 #get web sdk
 sudo countly update sdk-web

bin/countly.install_travis.sh

@@ -19,7 +19,7 @@ apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927
 #update package index
 apt-get update
 
-apt-get install -y build-essential git sqlite3 unzip shellcheck
+apt-get install -y build-essential git unzip shellcheck
 
 if apt-cache pkgnames | grep -q python-software-properties; then
     apt-get install -y python-software-properties
@@ -116,6 +116,8 @@ bash "$DIR/scripts/countly.install.plugins.sh"
 
 #load city data into database
 nodejs "$DIR/scripts/loadCitiesInDb.js"
+nodejs "$DIR/offline-geocoder/scripts/download_geonames_data.js"
+nodejs "$DIR/offline-geocoder/scripts/import_geonames_mongodb.js"
 
 #compile scripts for production
 countly task dist-all