included systemd sample configuration files

apps/spectral_indices/tasks.py

@@ -69,7 +69,7 @@ def pixel_drill(task_id=None):
     datasets = [single_pixel[band].values.transpose() for band in plot_measurements] + [clear_mask]
     data_labels = [stringcase.uppercase("{}".format(band)) for band in plot_measurements] + ["Clear"]
     titles = [stringcase.uppercase("{}".format(band)) for band in plot_measurements] + ["Clear Mask"]
-    style = ['r-o', 'g-o', 'b-o', 'c-o', 'm-o', 'y-o', 'k-o', '.']
+    style = ['ro', 'go', 'bo', 'co', 'mo', 'yo', 'ko', '.']
 
     task.plot_path = os.path.join(task.get_result_path(), "plot_path.png")
     create_2d_plot(task.plot_path, dates=dates, datasets=datasets, data_labels=data_labels, titles=titles, style=style)

config/systemd/celery

@@ -0,0 +1,21 @@
+CELERYD_NODES="task_processing data_cube_manager"
+
+CELERY_BIN="/home/localuser/Datacube/datacube_env/bin/celery"
+
+CELERY_APP="data_cube_ui"
+DJANGO_SETTINGS_MODULE="data_cube_ui.settings"
+
+# CELERYD_CHDIR: see WorkingDirectory in unit file
+
+# extra command-line arguments for the worker
+CELERYD_OPTS="-c:task_processing 10 -c:data_cube_manager 2 --max-tasks-per-child:data_cube_manager=1 -Q:data_cube_manager data_cube_manager -Ofair"
+
+CELERYD_LOG_LEVEL="INFO"
+
+CELERYD_LOG_FILE="/var/log/celery/%n%I.log"
+CELERYD_PID_FILE="/var/run/celery/%n.pid"
+
+#CELERY_USER: see User in unit file
+#CELERY_GROUP: see Group in unit file
+
+#CELERY_CREATE_DIRS: ???

config/systemd/celery-beat

@@ -0,0 +1,19 @@
+CELERY_BIN="/home/localuser/Datacube/datacube_env/bin/celery"
+
+CELERY_APP="data_cube_ui"
+DJANGO_SETTINGS_MODULE="data_cube_ui.settings"
+
+# CELERYD_CHDIR: see WorkingDirectory in unit file
+
+# extra command-line arguments for the beat service
+CELERYBEAT_OPTS=""
+
+CELERYBEAT_LOG_LEVEL="INFO"
+
+CELERYBEAT_LOG_FILE="/var/log/celery/celery-beat.log"
+CELERYBEAT_PID_FILE="/var/run/celery/celery-beat.pid"
+
+#CELERY_USER: see User in unit file
+#CELERY_GROUP: see Group in unit file
+
+#CELERY_CREATE_DIRS: ???

config/systemd/celery-beat.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=Celery Beat Service
+After=network.target
+
+[Service]
+User=localuser
+Group=localuser
+EnvironmentFile=/etc/conf.d/celery-beat
+WorkingDirectory=/home/localuser/Datacube/data_cube_ui
+ExecStart=/bin/sh -c '${CELERY_BIN} beat \
+    -A ${CELERY_APP} \
+    --pidfile=${CELERYBEAT_PID_FILE} \
+    --logfile=${CELERYBEAT_LOG_FILE} \
+    --loglevel=${CELERYBEAT_LOG_LEVEL} \
+    --workdir=${CELERYBEAT_WORK_DIR} \
+    ${CELERYBEAT_OPTS}'
+ExecStop=/bin/kill -s TERM $MAINPID
+
+[Install]
+WantedBy=multi-user.target

config/systemd/celery.conf

@@ -0,0 +1,2 @@
+d /var/log/celery 0755 localuser localuser -
+d /var/run/celery 0755 localuser localuser -

config/systemd/celery.service

@@ -0,0 +1,27 @@
+[Unit]
+Description=Celery Service
+After=network.target
+
+[Service]
+Type=forking
+User=localuser
+Group=localuser
+EnvironmentFile=/etc/conf.d/celery
+WorkingDirectory=/home/localuser/Datacube/data_cube_ui
+ExecStart=/bin/sh -c '${CELERY_BIN} multi start ${CELERYD_NODES} \
+    -A ${CELERY_APP} \
+    --pidfile=${CELERYD_PID_FILE} \
+    --logfile=${CELERYD_LOG_FILE} \
+    --loglevel=${CELERYD_LOG_LEVEL} \
+    ${CELERYD_OPTS}'
+ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait ${CELERYD_NODES} \
+    --pidfile=${CELERYD_PID_FILE}'
+ExecReload=/bin/sh -c '${CELERY_BIN} multi restart ${CELERYD_NODES} \
+    -A ${CELERY_APP} \
+    --pidfile=${CELERYD_PID_FILE} \
+    --logfile=${CELERYD_LOG_FILE} \
+    --loglevel=${CELERYD_LOG_LEVEL} \
+    ${CELERYD_OPTS}'
+
+[Install]
+WantedBy=multi-user.target

docs/datacube_install.md

@@ -118,7 +118,8 @@ pip install Cython
 pip install netcdf4
 ```
 
-Please note that the installed gdal version should be as close to your system gdal version as possible, printed with:
+Please note that the installed gdal version should be as close to your system gdal version as possible.
+We try to install a non-existent version (99999999999) to have pip print all available version.
 
 ```
 gdalinfo --version

docs/ingestion.md

@@ -39,6 +39,11 @@ To index and ingest data into the Data Cube, the following prerequisites must be
   * All code is checked out and you have a virtual environment in the correct directories: `~/Datacube/{data_cube_ui, data_cube_notebooks, datacube_env, agdc-v2}`
 * This guide will use a Landsat 7 SR product as an example. Please download a Landsat SR product from [our AWS site](http://ec2-52-201-154-0.compute-1.amazonaws.com/datacube/data/LE071950542015121201T1-SC20170427222707.tar.gz). We are providing a Landsat 7 Collection 1 scene over Ghana for our examples. This will ensure that the entire workflow can be completed by all users. Place the .tar.gz file in your `/datacube/original_data` directory.
 
+Note that the ingestion file hyperlinked above by "our AWS site" can be downloaded with the command:<br> 
+```
+wget http://ec2-52-201-154-0.compute-1.amazonaws.com/datacube/data/LE071950542015121201T1-SC20170427222707.tar.gz
+```
+
 If you have not yet completed our Data Cube Installation Guide, please do so before continuing.
 
 <a name="product_definition"></a> Product Definitions

docs/notebook_install.md

@@ -52,37 +52,6 @@ pip install lcmap-pyccd
 pip install folium
 ```
 
-We'll now need to manually download and install Basemap from source. Create a temporary directory in your home directory with the following commands and download the required .tar.gz
-
-```
-mkdir ~/temp
-cd ~/temp
-wget 'http://downloads.sourceforge.net/project/matplotlib/matplotlib-toolkits/basemap-1.0.7/basemap-1.0.7.tar.gz'
-```
-
-Now that we have the .tar.gz in the correct location, run the following commands to uncompress Basemap, install GEOS (included with Basemap) and install Basemap. This will produce a considerable amount of console output, but unless there are errors there is nothing to worry about.
-
-```
-#ensure that you're in the virtual environment. If not, activate with 'source ~/Datacube/datacube_env/bin/activate'
-
-#this uncompresses basemap and moves into the correct dir.
-tar -xvf basemap-*
-cd basemap-*
-
-#now we need to install geos - included in basemap.
-cd geos-*
-export GEOS_DIR=~/
-./configure --prefix=$GEOS_DIR
-make
-make install  
-
-#now install basemap
-cd ..
-python setup.py install
-```
-
-Now that Basemap has been successfully installed, we can move on to configuring our notebook server.
-
 <a name="configuration"></a> Configuration
 ========  
 
@@ -112,7 +81,9 @@ c.NotebookApp.open_browser = False
 c.NotebookApp.port = 8888
 ```
 
-Save the file and then run the notebook server with:
+Save the file and then run the notebook server with the following command.
+If this fails with a permissions error (`OSError: [...] Permission denied [...]`),
+run the command `export XDG_RUNTIME_DIR=""`.
 
 ```
 cd ~/Datacube/data_cube_notebooks

docs/ui_install.md

@@ -90,22 +90,13 @@ sudo apt-get install -y tmux
 Next, you'll need various Python packages responsible for the entire application:
 
 ```
-pip install django
+pip install django==1.11.13
 pip install redis
 pip install imageio
 pip install django-bootstrap3
 pip install matplotlib
 pip install stringcase
-```
-
-The UI relies on a slightly newer version of Celery that has not yet been pushed to Pypi - This is due to a bug that was fixed a few days after their latest release.
-
-```
-source ~/Datacube/datacube_env/bin/activate
-cd ~/Datacube
-git clone https://github.com/celery/celery.git
-cd celery
-python setup.py install
+pip install celery
 ```
 
 You will also need to create a base directory structure for results:
@@ -211,8 +202,8 @@ In this configuration file, note that all of the paths are absolute. If you used
 We'll now copy the configuration files where they need to be. The '.datacube.conf' file is copied to the home directory for consistency.
 
 ```
-cp config/.datacube.conf ~/.datacube.conf
-sudo cp config/dc_ui.conf /etc/apache2/sites-available/dc_ui.conf
+cp ~/Datacube/data_cube_ui/config/.datacube.conf ~/.datacube.conf
+sudo cp ~/Datacube/data_cube_ui/config/dc_ui.conf /etc/apache2/sites-available/dc_ui.conf
 ```
 
 The next step is to edit the credentials found in the Django settings. Open the 'settings.py' found at `~/Datacube/data_cube_ui/data_cube_ui/settings.py`:
@@ -343,9 +334,10 @@ celery -A data_cube_ui beat
 
 To test the workers we will need to add an area and dataset that you have ingested to the UI's database. This will happen in a separate section.
 
-This process can be automated and daemonized with the following snippet:
+This process can be automated and daemonized with the following snippet if your
+system uses Upstart/SysVInit:
 
-```
+```shell
 sudo cp config/celeryd_conf /etc/default/data_cube_ui && sudo cp config/celeryd /etc/init.d/data_cube_ui
 sudo chmod 777 /etc/init.d/data_cube_ui
 sudo chmod 644 /etc/default/data_cube_ui
@@ -360,7 +352,29 @@ sudo /etc/init.d/celerybeat start
 
 This is done in the initial setup script.
 
-You can alias the /etc/init.d/* script as whatever you like - you can start, stop, kill, restart, etc. the workers using this script.
+You can alias the `/etc/init.d/*` script as whatever you like - you can start, stop, kill, restart, etc. the workers using this script.
+
+
+If your Linux uses Systemd (e.g. Debian >= 8, Ubuntu >= 15.04), you can use the
+files provided in `config/systemd` instead (not included in the initial setup script!):
+```shell
+sudo install -vDm644 config/systemd/celery.service /etc/systemd/system/celery.service
+sudo install -vDm644 config/systemd/celery /etc/conf.d/celery
+
+sudo install -vDm644 config/systemd/celery-beat.service /etc/systemd/system/celery-beat.service
+sudo install -vDm644 config/systemd/celery-beat /etc/conf.d/celery-beat
+
+sudo install -vDm644 config/systemd/celery.conf /etc/tmpfiles.d/celery.conf
+sudo systemd-tmpfiles --create
+
+sudo systemctl daemon-reload
+sudo systemctl enable --now celery celery-beat
+```
+
+Starting, Stopping, Restarting etc. is now done with systemctl, e.g.:
+```
+sudo systemctl restart celery
+```
 
 
 <a name="system_overview"></a> Task System Overview
@@ -371,8 +385,8 @@ The worker system can seem complex at first, but the basic workflow is shown bel
 * The Django view receives form data from the web page. This form data is processed into a Query model for the application
 * The main Celery worker receives a task with a Query model and pulls all of the required parameters from this model
 * Using predefined chunking options, the main Celery task splits the parameters (latitude, longitude, time) into smaller chunks
-* These smaller chunks of (latitude, longitude, time) are sent off to the Celery slave processes - there should be more slave processes than master processes
-* The Celery slave processes load in the data in the parameters they received and perform some analysis. The results are saved to disk and the paths are returned
+* These smaller chunks of (latitude, longitude, time) are sent off to the Celery worker processes - there should be more worker processes than master processes
+* The Celery worker processes load in the data in the parameters they received and perform some analysis. The results are saved to disk and the paths are returned
 * The master process waits until all chunks have been processed then loads all of the result chunks. The chunks are combined into a single product and saved to disk
 * The master process uses the data product to create images and data products and saves them to disk, deleting all the remnant chunk products
 * The master process creates a Result and Metadata model based on what was just created and returns the details to the browser
@@ -393,22 +407,25 @@ source ~/Datacube/datacube_env/bin/activate
 cd ~/Datacube/data_cube_ui
 python manage.py shell
 
-from utils import data_access_api
+from utils.data_cube_utilities import data_access_api
 
 dc = data_access_api.DataAccessApi()
 
 dc.get_datacube_metadata('ls7_ledaps_general','LANDSAT_7')
 ```
 
 Record the latitude and longitude extents.
+They should be:
+lat=(7.745543874267876, 9.617183768731897)
+lon=(-3.5136704023069685, -1.4288602909212722)
 
 Go back to the admin page, select dc_algorithm->Areas, delete all of the initial areas, then click the 'Add Area' button.
 
 For the Area Id, enter 'general', or whatever area you've named that is prepended by the 'ls7_ledaps_'. Give the area a name as well.
 
 Enter the latitude and longitude bounds in all of the latitude/longitude min/max fields for both the top and the detail fields.
 
-For all of the imagery fields, enter '/static/assets/images/black.png'/static/assets/images/black.png' - this will give a black area, but will still highlight our area.
+For all of the imagery fields, enter '/static/assets/images/black.png' - this will give a black area, but will still highlight our area.
 
 Select LANDSAT_7 in the satellites field and save your new area.
 
@@ -443,6 +460,10 @@ If you are having trouble diagnosing issues with the UI, feel free to contact us
 ========  
 ----  
 
+If you daemonized the UI, the first thing to try after any above advice when experiencing issues with the UI is 
+to restart the UI: `sudo service data_cube_ui restart`
+
+
 Q: 	
  >I’m getting a “Permission denied error.”  How do I fix this?  
 
@@ -472,4 +493,18 @@ Q:
 
 A:  
 >	Run the Django migrations to ensure that you have the latest database schemas. If you have updated recently, ensure that you have a database table for each app - if any are missing, run 'python manage.py makemigrations {app_name}' followed by 'python manage.py migrate'.
+
+Q:
+ > How do I refresh the Data Cube Visualization tool?<br/>
+ > My regions are not showing up in the Data Cube Visualization tool.
+
+A:
+ > Activate the Data Cube virtual environment:<br/>
+ > `source ~/Datacube/datacube_env/bin/activate`<br/>
+ > enter the Django console:<br/>
+ > `cd ~/Datacube/data_cube_ui`<br/>
+ > `python manage.py shell`<br/>
+ > then run this function, which should update the cache:<br/>
+ > `import apps.data_cube_manager.tasks as dcmt`<br/>
+ > `dcmt.update_data_cube_details()`
 ---