Migrate to QGIS 3

- Migrate QLR handling from QgsMapLayer.fromLayerDefinition
  to QgsLayerDefinition.loadLayerDefinitionLayers
- Upgrade docker image to use qgis/qgis:release-3_14

Author: lucernae

.travis.yml

@@ -3,7 +3,7 @@ sudo: required
 language: python
 
 python:
-  - "2.7"
+  - "3.6"
 
 env:
   global:
@@ -23,9 +23,6 @@ env:
       HEADLESS_COMMAND=dev
       SUBSEQUENT_RUN_TESTING=True
 
-virtualenv:
-  system_site_packages: true
-
 services:
   - docker
 

deployment/ansible/development/group_vars/all.travis.yml

@@ -16,7 +16,7 @@ interpreters:
 inasafe:
   repo: https://github.com/inasafe/inasafe.git
   remote: upstream
-  version: inasafe_4
+  version: maintenance-fix
   depth: 1
 
 inasafe_headless_worker:

deployment/docker-headless/Dockerfile

@@ -1,5 +1,5 @@
 #--------- Generic stuff all our Dockerfiles should start with so we get caching ------------
-FROM kartoza/qgis-desktop:2.18
+FROM qgis/qgis:release-3_14
 
 RUN apt-get -y update; apt-get -y --force-yes install pwgen git inotify-tools
 
@@ -9,7 +9,10 @@ RUN apt-get -y update; apt-get -y --force-yes install pwgen git inotify-tools
 RUN apt-get update -y; apt-get install -y --force-yes openssh-server sudo
 RUN mkdir /var/run/sshd
 RUN echo 'root:docker' | chpasswd
-RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+# Comment out PermitRootLogin setting, whatever it is
+RUN sed -i 's/^PermitRootLogin */#PermitRootLogin /' /etc/ssh/sshd_config
+# Write out PermitRootLogin setting at the end
+RUN echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
 
 # SSH login fix. Otherwise user is kicked off after login
 RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
@@ -21,8 +24,7 @@ RUN echo "export VISIBLE=now" >> /etc/profile
 
 #-------------Application Specific Stuff ----------------------------------------------------
 # Install git, xvfb
-RUN apt-get -y update; apt-get -y --force-yes install git xvfb python-setuptools python-dev libssl-dev libffi-dev python-scipy
-RUN easy_install pip==9.0.1
+RUN apt-get -y update; apt-get -y --force-yes install git xvfb python3-setuptools python3-dev libssl-dev libffi-dev python3-scipy
 # Copy ubuntu fonts
 RUN apt-get -y update; apt-get -y --force-yes install wget unzip
 ADD ubuntu-font-family-0.83.zip /ubuntu-font-family-0.83.zip
@@ -31,8 +33,9 @@ RUN mv ubuntu-font-family-0.83 /usr/share/fonts/truetype/ubuntu-font-family
 RUN fc-cache -f -v
 
 # This image instance uses dist-packages directory
-ADD sitecustomize.py /usr/local/lib/python2.7/dist-packages/sitecustomize.py
+ADD sitecustomize.py /usr/lib/python3/dist-packages/sitecustomize.py
 ADD REQUIREMENTS.txt /REQUIREMENTS.txt
+RUN ln -s /usr/bin/pip3 /usr/bin/pip
 RUN pip install -r REQUIREMENTS.txt
 
 ADD docker-entrypoint.sh /docker-entrypoint.sh

deployment/docker-headless/docker-entrypoint.sh

@@ -2,7 +2,7 @@
 
 # Wait run xvfb
 while [ -z  "$(pidof /usr/bin/Xvfb)" ]; do
-  start-stop-daemon --start -b -x /usr/bin/Xvfb ${DISPLAY}
+  start-stop-daemon --start -b -x /usr/bin/Xvfb ${DISPLAY} -- -screen 0 1024x768x24 -ac +extension GLX +render -noreset -nolisten tcp
   sleep 5
 done
 

deployment/production/docker/headless/Dockerfile

@@ -1,12 +1,11 @@
 #--------- Generic stuff all our Dockerfiles should start with so we get caching ------------
-FROM kartoza/qgis-desktop:2.18
+FROM qgis/qgis:release-3_14
 
 RUN apt-get -y update; apt-get -y --force-yes install pwgen git inotify-tools
 
 #-------------Application Specific Stuff ----------------------------------------------------
 # Install git, xvfb
-RUN apt-get -y update; apt-get -y --force-yes install git xvfb python-setuptools python-dev libssl-dev libffi-dev python-scipy
-RUN easy_install pip==9.0.1
+RUN apt-get -y update; apt-get -y --force-yes install git xvfb python3-setuptools python3-dev libssl-dev libffi-dev python3-scipy
 # Copy ubuntu fonts
 RUN apt-get -y update; apt-get -y --force-yes install wget unzip
 ADD ubuntu-font-family-0.83.zip /ubuntu-font-family-0.83.zip
@@ -15,6 +14,7 @@ RUN mv ubuntu-font-family-0.83 /usr/share/fonts/truetype/ubuntu-font-family
 RUN fc-cache -f -v
 
 ADD REQUIREMENTS.txt /REQUIREMENTS.txt
+RUN ln -s /usr/bin/pip3 /usr/bin/pip
 RUN pip install -r REQUIREMENTS.txt
 
 ADD docker-entrypoint.sh /docker-entrypoint.sh
@@ -23,7 +23,7 @@ RUN chmod +x /docker-entrypoint.sh
 # Install InaSAFE Core
 RUN mkdir -p /usr/share/qgis/python/plugins
 WORKDIR /usr/share/qgis/python/plugins
-ARG INASAFE_CORE_TAG=version-4_4_0
+ARG INASAFE_CORE_TAG=develop
 RUN git clone --branch ${INASAFE_CORE_TAG} --depth 1 --recursive https://github.com/inasafe/inasafe.git inasafe
 
 # Install InaSAFE Headless

deployment/production/docker/headless/docker-entrypoint.sh

@@ -2,11 +2,12 @@
 
 # Wait run xvfb
 while [ -z  "$(pidof /usr/bin/Xvfb)" ]; do
-  start-stop-daemon --start -b -x /usr/bin/Xvfb ${DISPLAY}
+  start-stop-daemon --start -b -x /usr/bin/Xvfb ${DISPLAY} -- -screen 0 1024x768x24 -ac +extension GLX +render -noreset -nolisten tcp
   sleep 5
 done
 
 cp -n /home/app/headless/celeryconfig_sample.py /home/app/headless/celeryconfig.py
+echo "Config file copied"
 
 if [ $# -eq 2 ] && [ $1 = "prod" ] && [ $2 = "inasafe-headless-worker" ]; then
     /usr/local/bin/celery -A headless.celery_app worker -l info -Q inasafe-headless -n inasafe-headless.%h

src/headless/REQUIREMENTS.txt

@@ -1,8 +1,11 @@
 requests>=2.6.2
 tzlocal==1.2
-celery==4.1.0
-nosexcover
+hammock==0.2.4
 nose2==0.6.5
 coverage==4.4.2
+threadpool==1.3.2
+pyinotify==0.9.6
+celery==4.1.1
 raven==5.29.0
 mock==2.0.0
+nose==1.3.7

src/headless/celery_app.py

@@ -1,5 +1,6 @@
 # coding=utf-8
 import importlib
+from importlib import reload
 import json
 import os
 

src/headless/tasks/inasafe_analysis.py

@@ -5,10 +5,10 @@
 
 from copy import deepcopy
 from datetime import datetime
-from PyQt4.QtCore import QUrl
+from qgis.PyQt.QtCore import QUrl
 
 from qgis.core import (
-    QgsCoordinateReferenceSystem, QgsMapLayerRegistry, QgsProject)
+    QgsCoordinateReferenceSystem, QgsProject)
 
 from safe.definitions.constants import (
     PREPARE_SUCCESS, ANALYSIS_SUCCESS, MULTI_EXPOSURE_ANALYSIS_FLAG)
@@ -118,7 +118,7 @@ def inasafe_analysis(
     """
     # Clean up layer registry before using
     # In case previous task exited prematurely before cleanup
-    layer_registry = QgsMapLayerRegistry.instance()
+    layer_registry = QgsProject.instance()
     layer_registry.removeAllMapLayers()
 
     impact_function = ImpactFunction()
@@ -213,7 +213,7 @@ def inasafe_multi_exposure_analysis(
     """
     # Clean up layer registry before using
     # In case previous task exited prematurely before cleanup
-    layer_registry = QgsMapLayerRegistry.instance()
+    layer_registry = QgsProject.instance()
     layer_registry.removeAllMapLayers()
 
     multi_exposure_if = MultiExposureImpactFunction()
@@ -324,7 +324,7 @@ def generate_report(
     """
     # Clean up layer registry before using
     # In case previous task exited prematurely before cleanup
-    layer_registry = QgsMapLayerRegistry.instance()
+    layer_registry = QgsProject.instance()
     layer_registry.removeAllMapLayers()
 
     output_metadata = read_iso19115_metadata(impact_layer_uri)
@@ -344,14 +344,14 @@ def generate_report(
         root = QgsProject.instance().layerTreeRoot()
 
         group_analysis = root.insertGroup(0, impact_function.name)
-        group_analysis.setVisible(True)
+        group_analysis.setItemVisibilityChecked(True)
         group_analysis.setCustomProperty(
             MULTI_EXPOSURE_ANALYSIS_FLAG, True)
 
         for layer in impact_function.outputs:
-            QgsMapLayerRegistry.instance().addMapLayer(layer, False)
+            QgsProject.instance().addMapLayer(layer, False)
             layer_node = group_analysis.addLayer(layer)
-            layer_node.setVisible(False)
+            layer_node.setItemVisibilityChecked(False)
 
             # set layer title if any
             try:
@@ -362,7 +362,7 @@ def generate_report(
 
         for analysis in impact_function.impact_functions:
             detailed_group = group_analysis.insertGroup(0, analysis.name)
-            detailed_group.setVisible(True)
+            detailed_group.setItemVisibilityChecked(True)
             add_impact_layers_to_canvas(analysis, group=detailed_group)
     else:
         impact_function = (

src/headless/tasks/inasafe_wrapper.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 """Task for InaSAFE Headless."""
-
+from importlib import reload
 from headless.celery_app import app, start_inasafe
 from headless.tasks import inasafe_analysis
 from headless.utils import get_headless_logger