start worker

.github/workflows/ubuntu.yml

@@ -0,0 +1,48 @@
+name: Tests Ubuntu
+
+# Run this workflow every time a new commit pushed to your repository
+on: [push, pull_request]
+
+jobs:
+  # Set the job key. The key is displayed as the job name
+  # when a job name is not provided
+  test:
+    # Name the Job
+    name: Tests Ubuntu
+    # Set the type of machine to run on
+    runs-on: ubuntu-latest
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        ports:
+        - 3306:3306
+    steps:
+      # Checks out a copy of your repository on the ubuntu-latest machine
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '2.x'
+
+      - name: Install Python modules
+        run: pip install twisted mysqlclient
+
+      - name: Test SQLite
+        run: python tests/main_test.py
+
+      - name: Verify MySQL connection from host
+        run: mysql --host 127.0.0.1 --port 3306 -uroot -proot -e "CREATE DATABASE base"
+
+      - name: Test MySQL
+        run: |
+          echo "[server]" > coalition.ini
+          echo "db_type=mysql" >> coalition.ini
+          echo "db_mysql_host=127.0.0.1" >> coalition.ini
+          echo "db_mysql_user=root" >> coalition.ini
+          echo "db_mysql_password=root" >> coalition.ini
+          echo "db_mysql_base=base" >> coalition.ini
+          python tests/main_test.py

.github/workflows/win.yml

@@ -0,0 +1,29 @@
+name: Tests Windows
+
+# Run this workflow every time a new commit pushed to your repository
+on: [push, pull_request]
+
+jobs:
+  # Set the job key. The key is displayed as the job name
+  # when a job name is not provided
+  test:
+    # Name the Job
+    name: Tests Windows
+    # Set the type of machine to run on
+    runs-on: windows-latest
+
+    steps:
+      # Checks out a copy of your repository on the ubuntu-latest machine
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '2.x'
+
+      - name: Install Python modules
+        run: pip install twisted pypiwin32
+
+      - name: Test
+        run: python tests/main_test.py

.gitignore

@@ -0,0 +1,9 @@
+*.pyc
+*.log
+log/*
+docs/*
+*.db
+*.cache/
+_build
+_static
+_templates

.travis.yml

@@ -0,0 +1,19 @@
+language: python
+cache: pip
+python:
+- "2.7"
+services:
+  - mysql
+before_install:
+- sudo apt-get update -qq
+- sudo apt-get install -qq python-sqlite
+- mysql -e 'CREATE DATABASE base;'
+install: "pip install -r requirements.txt"
+script:
+- echo [server] > coalition.ini
+- echo db_type=sqlite >> coalition.ini
+- tests/main_test.py
+- echo [server] > coalition.ini
+- echo db_type=mysql >> coalition.ini
+- echo db_mysql_user=travis >> coalition.ini
+- tests/main_test.py

Dockerfile

@@ -0,0 +1,11 @@
+FROM debian:buster-slim
+WORKDIR /usr/src/app
+
+RUN apt-get update && apt-get install -y python python-pip python-mysqldb libldap2-dev libsasl2-dev && rm -rf /var/lib/apt/lists/* 
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+CMD [ "python", "./server.py", "--verbose" ]
+EXPOSE 19211

README.rst

@@ -0,0 +1,48 @@
+.. |badge-doc| image:: https://readthedocs.org/projects/coalition/badge/?version=latest
+   :target: http://coalition.readthedocs.io/en/latest/?badge=latest
+   :alt: Documentation Status
+
+.. |badge-size| image:: https://reposs.herokuapp.com/?path=https://github.com/MercenariesEngineering/coalition
+
+.. |badge-version| image:: https://badge.fury.io/gh/MercenariesEngineering%2Fcoalition.svg
+   :target: https://badge.fury.io/gh/MercenariesEngineering%2Fcoalition
+
+.. |badge-coverage| image:: https://coveralls.io/repos/github/MercenariesEngineering/coalition/badge.svg?branch=development
+   :target: https://coveralls.io/github/MercenariesEngineering/coalition?branch=development
+
+.. |badge-tests| image:: https://travis-ci.org/MercenariesEngineering/coalition.svg?branch=master
+
+|badge-doc| |badge-size| |badge-version| |badge-coverage| |badge-tests|
+
+`Full online documentation is availlble on ReadTheDocs <http://coalition.readthedocs.io/en/latest/>`_.
+
+Coalition
+=========
+
+**Coalition** is a lightweight open source **job manager** client-server application whose role is to control **job execution in a set of computers**. A computer is acting as a **server** centralizing the list of jobs to be done. A set of physical (or virtual, eg. in the cloud) computers acting as **workers** shall be deployed, raising the global grid system ressources.
+
+The server waits for incoming workers connections. Workers ask the server for a job to do. When the server is asked by a worker for a job, he decides which job to attribute according to simple **affinity rules**. The worker is now aware of which job it has to do. The worker executes the job. When the job is done, the worker informs the server of the job's execution status and ask for a new job.
+
+*Coalition* should not be used on the public Internet but on **private LANs**, **cloud VLANs** or **VPN** for security reasons.
+
+*Coalition* has been successfully used in production notably for **renderfarms**.
+
+*Coalition* provides:
+
+- **Broadcast discovery** for workers to find the server whithout configuration;
+- **RESTfull python API** based on `Twisted matrix <https://twistedmatrix.com>`_ for program to program communication;
+- **Cloud ready** configuration to manage starting/termination of workers in the cloud;
+- **Web interface** for humans to control jobs, workers, affinities and view status and logs;
+- **Database** interface for sqlite and mysql;
+- **Logging** system;
+- **Email notification** system;
+- **Access Control List** when connected to a **LDAP** server;
+- **Unittests** of critical code parts;
+- **Source code** and **documentation** on `the development platform <https://github.com/MercenariesEngineering/coalition>`_.
+
+The current stable version are 3.8 and 3.10.
+
+The development version is |current-version|.
+
+.. |current-version| include:: version
+

_cloud_aws.ini

@@ -0,0 +1,62 @@
+# Configuration file for aws cloud
+
+[authentication]
+# Aws ssh key pair name
+keyname=
+# Accesskey
+accesskey=
+# Secretkey
+secretaccesskey=
+
+[storage]
+# Storage name
+name=
+# Mountpoint in the worker
+mountpoint=/mnt/bucket
+# Location of the guerilla installer in the storage
+guerillarenderfilename=srv/guerilla_render_2.0.0a13_linux64.tar.gz
+# Location of the coalition installer in the storage
+coalitionfilename=srv/coalition.tar.gz
+
+[coalition]
+# Coalition server IP
+ip =
+# Coalition server port
+port = 19211
+# Maximum number of simultaneous workers
+workerinstancemax=3
+# Delay in seconds between creation of instances.
+# This prevents massive instances creation for big list of short time jobs.
+# Default is 30 seconds.
+workerinstancestartdelay=30
+# Minimum lifetime in seconds before allowing the termination of useless
+# worker instances. Since an instance requires several minutes to start,
+# this option offers the possibility of keeping instances ready even during
+# a short time without jobs.
+# Default is 900 seconds = 15 minutes.
+workerinstanceminimumlifetime=900
+
+[worker]
+# Prefix for the new instance name
+nameprefix=cloud-
+spot=true
+# Instance type
+# https://aws.amazon.com/ec2/instance-types/
+# http://www.ec2instances.info/
+instancetype=m3.medium
+# Aws image, for instance debian-stretch-amd64-hvm-2016-09-23-08-48-ebs
+imageid=ami-2f40bd40
+# Aws subnet
+subnetid=
+# Aws instance profile
+iaminstanceprofile=
+# Aws security group
+securitygroupid=
+availabilityzone=
+[spot]
+# http://docs.aws.amazon.com/cli/latest/reference/ec2/request-spot-instances.html
+# https://aws.amazon.com/ec2/spot/pricing/
+spotprice=10
+instancecount=1
+type=one-time
+

_cloud_gcloud.ini

@@ -0,0 +1,58 @@
+# Configuration file for google cloud
+
+[authentication]
+# Project name
+;project=guerilla-cloud
+# Location of json key file for service user got from developper interface
+;keyfile=guerilla-cloud-34bf64e0149b.json
+# Service account
+;[email protected]
+;scopes=default
+
+
+[storage]
+# Storage name
+;name=guerilla-cloud-bucket
+# Mountpoint in the worker
+;mountpoint=/mnt/bucket
+# Location of the coalition installer in the storage
+;coalitionpackage=srv/coalition.tar.gz
+
+[coalition]
+# Coalition server IP
+;ip = 10.132.0.2
+# Coalition server port
+;port = 19211
+# Maximum number of simultaneous workers
+;workerinstancemax=3
+# Delay in seconds between creation of instances.
+# This prevents massive instances creation for big list of short time jobs.
+# Default is 30 seconds.
+;workerinstancestartdelay=30
+# Minimum lifetime in seconds before allowing the termination of useless
+# worker instances. Since an instance requires several minutes to start,
+# this option offers the possibility of keeping instances ready even during
+# a short time without jobs.
+# Default is 900 seconds = 15 minutes.
+;workerinstanceminimumlifetime=900
+
+[main_program]
+;package=srv/guerilla_render_2.0.0a13_linux64.tar.gz
+;environment=GUERILLA=/usr/local/bin/guerillarender/data/usr/local/guerilla GUERILLA_CLOUD_ROOT=/mnt/bucket
+
+[worker]
+# Install dir fr coalition and main program
+;installdir=/usr/local/bin
+# Prefix for the new instance name
+;nameprefix=cloud-
+;zone=europe-west1-d
+;machinetype=f1-micro
+;subnet=default
+;preemptible=true
+# maintenancepolicy must be TERMINATE if preemptible is true
+;maintenancepolicy=TERMINATE
+;image=debian-8-jessie-v20170308
+;imageproject=debian-cloud
+;bootdisksize=10
+;bootdisktype=pd-standard
+

_cloud_qarnot.ini

@@ -0,0 +1,51 @@
+# Configuration file for qarnot cloud
+
+[authentication]
+; client_token = 
+
+[storage]
+# Storage name
+;name=guerilla-cloud-bucket
+# Mountpoint in the worker
+;mountpoint=/mnt/bucket
+# Location of the coalition installer in the storage
+;coalitionpackage=srv/coalition.tar.gz
+
+[coalition]
+# Coalition server IP
+;ip = 10.132.0.2
+# Coalition server port
+;port = 19211
+# Maximum number of simultaneous workers
+;workerinstancemax=3
+# Delay in seconds between creation of instances.
+# This prevents massive instances creation for big list of short time jobs.
+# Default is 30 seconds.
+;workerinstancestartdelay=30
+# Minimum lifetime in seconds before allowing the termination of useless
+# worker instances. Since an instance requires several minutes to start,
+# this option offers the possibility of keeping instances ready even during
+# a short time without jobs.
+# Default is 900 seconds = 15 minutes.
+;workerinstanceminimumlifetime=900
+
+[main_program]
+;package=srv/guerilla_render_2.0.0a13_linux64.tar.gz
+;environment=GUERILLA=/usr/local/bin/guerillarender/data/usr/local/guerilla GUERILLA_CLOUD_ROOT=/mnt/bucket
+
+[worker]
+# Install dir fr coalition and main program
+;installdir=/usr/local/bin
+# Prefix for the new instance name
+;nameprefix=cloud-
+;zone=europe-west1-d
+;machinetype=f1-micro
+;subnet=default
+;preemptible=true
+# maintenancepolicy must be TERMINATE if preemptible is true
+;maintenancepolicy=TERMINATE
+;image=debian-8-jessie-v20170308
+;imageproject=debian-cloud
+;bootdisksize=10
+;bootdisktype=pd-standard
+