Merge pull request #6375 from Countly/sync-master-into-the-next

Sync master into the next

Author: pnrgenc

.github/workflows/main.yml

@@ -244,96 +244,13 @@ jobs:
           /sbin/my_init &
           node bin/scripts/test.connection.js
           npx grunt mochaTest       
-  ui-test-dashboard:
-    runs-on: ubuntu-22.04-4core
 
-    services:
-      mongodb:
-        image: mongo:8.0
-        options: >-
-          --health-cmd mongosh
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 27017:27017
-
-    container:
-      image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }}
-      env:
-        COUNTLY_CONFIG__MONGODB_HOST: mongodb
-        COUNTLY_CONFIG_API_PREVENT_JOBS: true
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install Chrome
-        shell: bash
-        run: |
-          apt update
-          apt install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb wget
-          wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb -O /tmp/chrome.deb
-          apt install -y /tmp/chrome.deb
-
-      - name: Copy code
-        shell: bash
-        run: |
-          rm -rf /opt/countly/frontend
-          rm -rf /opt/countly/plugins/old-ui-compatibility
-          cp -rf ./* /opt/countly
-          cp /opt/countly/frontend/express/config.sample.js /opt/countly/frontend/express/config.js
-          cp /opt/countly/frontend/express/public/javascripts/countly/countly.config.sample.js /opt/countly/frontend/express/public/javascripts/countly/countly.config.js
-
-      - name: Prepare files to use correct MongoDB host
-        shell: bash
-        run: "sed -i 's/mongosh --quiet/mongosh --host mongodb --quiet/' /opt/countly/bin/backup/import_events.sh && sed -i 's/mongoimport --db/mongoimport --host mongodb --db/' /opt/countly/bin/backup/import_events.sh"
-
-      - name: NPM install
-        shell: bash
-        working-directory: /opt/countly
-        run: npm install
-
-      - name: Prepare environment
-        shell: bash
-        working-directory: /opt/countly
-        run: |
-          sed -i 's/port: 3001,/port: 3001, workers: 1,/' /opt/countly/api/config.js
-          cp "./plugins/plugins.default.json" "/opt/countly/plugins/plugins.json"
-          npm install
-          sudo countly task dist-all
-          bash bin/backup/import_events.sh
-          bash bin/scripts/countly.prepare.ce.tests.sh
-          countly add_user '${{ secrets.CYPRESS_USER_EMAIL }}' '${{ secrets.CYPRESS_USER_PASSWORD }}'
-          mongosh --host mongodb --eval 'db.getSiblingDB("countly").members.updateOne({username: "${{ secrets.CYPRESS_USER_EMAIL }}"}, {$set: {username: "${{ secrets.CYPRESS_USER_USERNAME }}", subscribe_newsletter: true}});'
-          mongosh --host mongodb --eval 'db.getSiblingDB("countly").plugins.updateOne({_id: "plugins"}, {$set: {"frontend.countly_tracking": true}});'
-          cd ui-tests
-          echo '{"username": "${{ secrets.CYPRESS_USER_USERNAME }}","email": "${{ secrets.CYPRESS_USER_EMAIL }}","password": "${{ secrets.CYPRESS_USER_PASSWORD }}"}' > cypress/fixtures/user.json
-          sed -i 's/00000000-0000-0000-0000-000000000000/${{ secrets.CYPRESS_KEY }}/g' package.json
-          cp cypress.config.sample.js cypress.config.js
-          sed -i 's/000000/${{ secrets.CYPRESS_PROJECT_ID }}/g' cypress.config.js
-
-      - name: Run UI tests
-        shell: bash
-        working-directory: /opt/countly
-        run: |
-          /sbin/my_init &
-          cd ui-tests
-          npm install
-          xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" \
-          npm run cy:run:dashboard
-
-      - name: Upload UI tests artifacts
-        if: ${{ failure() }}
-        shell: bash
-        working-directory: /opt/countly/ui-tests/cypress
-        run: |
-          ARTIFACT_ARCHIVE_NAME="$(date '+%Y%m%d-%H.%M')_${GITHUB_REPOSITORY#*/}_CI#${{ github.run_number }}.tar.gz"
-          mkdir -p screenshots videos
-          tar zcvf "$ARTIFACT_ARCHIVE_NAME" screenshots videos
-          curl -o /tmp/uploader.log -u "${{ secrets.BOX_UPLOAD_AUTH }}" ${{ secrets.BOX_UPLOAD_PATH }} -T "$ARTIFACT_ARCHIVE_NAME"
+  ui-tests:
+    runs-on: ubuntu-latest
 
-  ui-test-onboarding:
-    runs-on: ubuntu-22.04-4core
+    strategy:
+      matrix:
+        test_type: [dashboard, onboarding]
 
     services:
       mongodb:
@@ -371,7 +288,7 @@ jobs:
           cp -rf ./* /opt/countly
           cp /opt/countly/frontend/express/config.sample.js /opt/countly/frontend/express/config.js
           cp /opt/countly/frontend/express/public/javascripts/countly/countly.config.sample.js /opt/countly/frontend/express/public/javascripts/countly/countly.config.js
-          
+
       - name: Prepare files to use correct MongoDB host
         shell: bash
         run: "sed -i 's/mongosh --quiet/mongosh --host mongodb --quiet/' /opt/countly/bin/backup/import_events.sh && sed -i 's/mongoimport --db/mongoimport --host mongodb --db/' /opt/countly/bin/backup/import_events.sh"
@@ -389,7 +306,13 @@ jobs:
           cp "./plugins/plugins.default.json" "/opt/countly/plugins/plugins.json"
           npm install
           sudo countly task dist-all
+          bash bin/backup/import_events.sh
           bash bin/scripts/countly.prepare.ce.tests.sh
+          if [ "${{ matrix.test_type }}" = "dashboard" ]; then
+            countly add_user '${{ secrets.CYPRESS_USER_EMAIL }}' '${{ secrets.CYPRESS_USER_PASSWORD }}'
+            mongosh --host mongodb --eval 'db.getSiblingDB("countly").members.updateOne({username: "${{ secrets.CYPRESS_USER_EMAIL }}"}, {$set: {username: "${{ secrets.CYPRESS_USER_USERNAME }}", subscribe_newsletter: true}});'
+            mongosh --host mongodb --eval 'db.getSiblingDB("countly").plugins.updateOne({_id: "plugins"}, {$set: {"frontend.countly_tracking": true}});'
+          fi
           cd ui-tests
           echo '{"username": "${{ secrets.CYPRESS_USER_USERNAME }}","email": "${{ secrets.CYPRESS_USER_EMAIL }}","password": "${{ secrets.CYPRESS_USER_PASSWORD }}"}' > cypress/fixtures/user.json
           sed -i 's/00000000-0000-0000-0000-000000000000/${{ secrets.CYPRESS_KEY }}/g' package.json
@@ -403,15 +326,18 @@ jobs:
           /sbin/my_init &
           cd ui-tests
           npm install
-          xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" \
-          npm run cy:run:onboarding
+          if [ "${{ matrix.test_type }}" = "dashboard" ]; then
+            xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" npm run cy:run:dashboard
+          else
+            xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" npm run cy:run:onboarding
+          fi
 
       - name: Upload UI tests artifacts
         if: ${{ failure() }}
         shell: bash
         working-directory: /opt/countly/ui-tests/cypress
         run: |
-          ARTIFACT_ARCHIVE_NAME="$(date '+%Y%m%d-%H.%M')_${GITHUB_REPOSITORY#*/}_CI#${{ github.run_number }}.tar.gz"
+          ARTIFACT_ARCHIVE_NAME="$(date '+%Y%m%d-%H.%M')_${GITHUB_REPOSITORY#*/}_CI#${{ github.run_number }}_${{ matrix.test_type }}.tar.gz"
           mkdir -p screenshots videos
           tar zcvf "$ARTIFACT_ARCHIVE_NAME" screenshots videos
           curl -o /tmp/uploader.log -u "${{ secrets.BOX_UPLOAD_AUTH }}" ${{ secrets.BOX_UPLOAD_PATH }} -T "$ARTIFACT_ARCHIVE_NAME"

Dockerfile-api

@@ -30,15 +30,15 @@ RUN apt-get update && \
 RUN apt-get update && \
 	apt-get upgrade -y && \
 	cd /usr/src && \
-	wget https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tar.xz && \
-	tar -xf Python-3.8.12.tar.xz && \
-	apt-get install -y build-essential sudo zlib1g-dev libssl3 libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev curl libbz2-dev && \
-	cd Python-3.8.12 && \
+	wget https://www.python.org/ftp/python/3.8.20/Python-3.8.20.tar.xz && \
+	tar -xf Python-3.8.20.tar.xz && \
+	apt-get install -y build-essential sudo zlib1g-dev libssl3 libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev && \
+	cd Python-3.8.20 && \
 	./configure --enable-optimizations --enable-shared && \
 	make && \
 	make altinstall && \
-	ldconfig /usr/src/Python-3.8.12 && \
-	ln -s /usr/local/bin/python3.8 /usr/bin/python3
+	ldconfig /usr/src/Python-3.8.20 && \
+	ln -sf /usr/local/bin/python3.8 /usr/bin/python3
 
 RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" && \
 	dpkg -i /tmp/tini.deb && \
@@ -54,13 +54,16 @@ RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/downlo
 	HOME=/tmp npm install argon2 --build-from-source --unsafe-perm=true --allow-root && \
 	./bin/docker/preinstall.sh && \
 	bash /opt/countly/bin/scripts/detect.init.sh && \
+        python3.8 -m pip install --upgrade pip setuptools && \
 	\
 	# cleanup & chown
 	npm remove -y --no-save mocha nyc should supertest && \
-	apt-get remove -y git gcc g++ make automake autoconf libtool pkg-config unzip wget && \
+        apt-get purge -y python3 python3-pip python3-distutils python3-setuptools && \
+	apt-get remove -y git gcc g++ make automake autoconf libtool pkg-config unzip sqlite3 wget && \
 	apt-get install -y libgbm-dev libgbm1 gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils && \
 	apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-	rm -rf test /tmp/* /tmp/.??* /var/tmp/* /var/tmp/.??* /var/log/* /root/.npm && \
+        apt-get autoremove -y && \
+	rm -rf test /tmp/* /tmp/.??* /var/tmp/* /var/tmp/.??* /var/log/* /root/.npm /usr/local/lib/python3.8/ensurepip/_bundled/*.whl /usr/src/Python* /var/lib/dpkg/info/python3* && \
 	\
 	# temporary to remove npm bug message
 	mkdir /.npm && chown -R 1001:0 /.npm && \
@@ -75,4 +78,4 @@ USER 1001:0
 
 ENTRYPOINT ["/usr/bin/tini", "-v", "--"]
 
-CMD ["/opt/countly/bin/docker/cmd.sh"]
+CMD ["/opt/countly/bin/docker/cmd.sh"]

Dockerfile-frontend

@@ -26,15 +26,15 @@ RUN apt-get update && \
 RUN apt-get update && \
 	apt-get upgrade -y && \
 	cd /usr/src && \
-	wget https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tar.xz && \
-	tar -xf Python-3.8.12.tar.xz && \
-	apt-get install -y build-essential sudo zlib1g-dev libssl3 libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev curl libbz2-dev && \
-	cd Python-3.8.12 && \
+	wget https://www.python.org/ftp/python/3.8.20/Python-3.8.20.tar.xz && \
+	tar -xf Python-3.8.20.tar.xz && \
+	apt-get install -y build-essential sudo zlib1g-dev libssl3 libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev && \
+	cd Python-3.8.20 && \
 	./configure --enable-optimizations --enable-shared && \
 	make && \
 	make altinstall && \
-	ldconfig /usr/src/Python-3.8.12 && \
-	ln -s /usr/local/bin/python3.8 /usr/bin/python3
+	ldconfig /usr/src/Python-3.8.20 && \
+	ln -sf /usr/local/bin/python3.8 /usr/bin/python3
 
 RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" && \
 	dpkg -i /tmp/tini.deb && \
@@ -51,14 +51,17 @@ RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/downlo
 	HOME=/tmp npm install argon2 --build-from-source --unsafe-perm=true --allow-root && \
 	./bin/docker/preinstall.sh && \
 	bash /opt/countly/bin/scripts/detect.init.sh && \
-	countly update sdk-web && \
+	python3.8 -m pip install --upgrade pip setuptools && \
+        countly update sdk-web && \
 	\
 	# cleanup & chown
 	npm remove -y --no-save mocha nyc should supertest puppeteer && \
-	apt-get remove -y git gcc g++ make automake autoconf libtool pkg-config unzip wget && \
+	apt-get purge -y python3 python3-pip python3-distutils python3-setuptools && \
+        apt-get remove -y git gcc g++ make automake autoconf libtool pkg-config unzip sqlite3 wget && \
 	apt-get autoremove -y && \
 	apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-	rm -rf test /tmp/* /tmp/.??* /var/tmp/* /var/tmp/.??* /var/log/* /root/.npm && \
+        apt-get autoremove -y && \
+        rm -rf test /tmp/* /tmp/.??* /var/tmp/* /var/tmp/.??* /var/log/* /root/.npm /usr/local/lib/python3.8/ensurepip/_bundled/*.whl /usr/src/Python* /var/lib/dpkg/info/python3* && \
 	\
 	# temporary to remove npm bug message
 	mkdir /.npm && chown -R 1001:0 /.npm && \