From ba4da35caa93f875b4ffeb14f5de3539e5d6083a Mon Sep 17 00:00:00 2001 From: Aleksandr Volochnev Date: Mon, 6 Jan 2020 19:19:44 +0100 Subject: [PATCH 1/5] Added CI tests for travis --- .travis.yml | 40 ++++++++++++++++++++++++++++++---------- Dockerfile | 29 ++++------------------------- docker-compose.ci.yml | 22 ++++++++++++++++++++++ docker-compose.yaml | 20 ++++++++++---------- 4 files changed, 66 insertions(+), 45 deletions(-) create mode 100644 docker-compose.ci.yml diff --git a/.travis.yml b/.travis.yml index ff38c17..96774c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,36 @@ dist: xenial # required for Python >= 3.7 language: generic +sudo: required +services: + - docker # Build the docker image script: - - docker build -t ${TRAVIS_COMMIT} . +- travis_fold start docker_build +- docker build -t ${TRAVIS_COMMIT} -t killrvideo-python-local . +- travis_fold end docker_build +- travis_fold start docker_dependencies_up +- docker-compose -f docker-compose.ci.yml up -d +- sleep 300 +- docker-compose -f docker-compose.ci.yml logs dse-config +- travis_fold end docker_dependencies_up +- travis_fold start test +- docker run --network killrvideo-python_default killrvideo/killrvideo-integration-tests +- travis_fold end test # If successful, see if we need to publish also after_success: - - test -z $TRAVIS_TAG && travis_terminate 0 - - docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:${TRAVIS_TAG} - - docker login -u $DOCKER_USER -p $DOCKER_PASS - - docker push killrvideo/killrvideo-python:${TRAVIS_TAG} - - "[ \"$(git tag --sort=-v:refname | grep -P \"^\\d+.\\d+.\\d+$\" | head -n1)\" == \"$TRAVIS_TAG\" ] && { docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:latest; docker push killrvideo/killrvideo-python:latest; }" -# Sudo required for doing docker build -sudo: required -services: - - docker +- "[ \"$TRAVIS_EVENT_TYPE\" = \"cron\" ] && { echo \"Ignore nightly builds\"; travis_terminate 0; }" +- test -z $TRAVIS_TAG && { echo "Ignore non-tagged builds"; travis_terminate 0; } +- docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:${TRAVIS_TAG} +- echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin +- docker push killrvideo/killrvideo-python:${TRAVIS_TAG} +- "[ \"$(git tag --sort=-v:refname | grep -P \"^\\d+.\\d+.\\d+$\" | head -n1)\" == \"$TRAVIS_TAG\" ] && { docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:latest; docker push killrvideo/killrvideo-python:latest; }" + +after_failure: +- travis_fold start logs +- docker-compose -f docker-compose.ci.yml logs dse-config backend +- travis_fold end logs env: global: @@ -23,3 +38,8 @@ env: - secure: "N7FzHIc8S/HyMOmEe9AGlMDDpvPG4+fQMX4J2Ws60PJno/g6h5Gifk0D7DS8zAgOWw7y1U+r8WXefRNGXj1WCFmTrchJhbNoURvqZ/yhrBWoHjEW3f4hxPw0KowAqHf6ZBrhGhoDTPJ+rz4YAnbVjMu10S+1/55EjkrlSyDQ9EwHgTjo63k8hz1dRjMpGzxp4i9XRUc2qQZchFCT+H8c5RQqlQjG4SOhKfp+igVhCWO86U0PoJEMR7y6nYCpSVLdnX2W9oUyQI9VxCh35bnxRVgGCX71eqoyqxS3AQ0LLqY8Yy4PiPzkRCSGFX9qtbgrLd+t5osZsdFyX3YfzmBr3dVCmQwkoHcOM2L3HaKtfPCsni92ukEoVBpU39ajZl8qpCt5QcRn7CpqLQmNeh9xfxlvoyfeR/DTlqKpFy/l6kXgFjrIdssXA5qSiAV873nW4vZerBGkwmWnsvojYA4zHMaT3fvFaFCfT9G2haLGyPF2ZYVpgl40Ficu1w7MrED8Zenou9lOvwDaG+g6vzGTBXwpie7yxPjSrVZi4OK7io574esG3nUwHcqDzEOar1FeBgDbo1I60Dw4C3uwJuM0fbBZo1uZ122DEnwjkzuojJl4r3V+OzFJEF9SgtZTJjZ84yYDXbJ+BjANf72d75c3hwP9Y3ppFunmeKCTcb4aMVs=" # DOCKER_PASS - secure: "F8sNKMia3lYxc7X+vn7448CxX3z0Df06RJPBChM7oAREP2fxvOEzgipgMbHPx4POcdSr60aDSJfDfXpTPRylFkkNK8d733HQdvsLA1lyVoHr6pKRjU8QNz4AMMIxHNhunMTQ5XfJEn9Q+a5qmcsho9HOCA4rfhT/thKIC9QlX/qNme4v27n0vjvqh7cUHU2gV5ykZHf324WBRrzHgYyEe0ieNJ6Tvmxf+Kzm3JB6X20uqst48tS/pPDAkdQPnlu4kWprFwRfKboSjLQjgnnHY+zZLvgvP7kyfO817M5n7iH5WAD1jK4o1stb0tSM/1u4SwssxIyLOKJeclCfY9XkG3DjtNuU+43MUKsI8Q/39fzqW39nxYUCcM4kbW5rWZUCz9PfCs3MOd4xhgX7+mf9xT66HclsJcdbqyrty41rSB0/x8T2LtzoW/Q6UD2GNHimkRu2cTDI2aV1HxbZjqGlFxK5x77CPQAuFxtbnaUH6pSoi3fiYyJiivLLTGzeisA2qpjeSMS2+q3KZIpcqaDIMV4slvfng7gurm2TvtlEtwG1ITJ01dokcXNy+vcw/3go606Aemr/bK2Aot00X0/HDQaX1/v0rJVWXW/c3Gcso6Jzmi1C9vp8gTOr5closfqZFyDJFC1Wuv+vdAzW4MMuwAuc4DnoiwQ6Gm8LamV9oy8=" + +notifications: + slack: + rooms: + secure: Eh/HI7+7KAUlzFpl9ViMUpExt4YXXWa5rEcwW3h/zIAo/UVghstuoCnk/CWbY/4SyrBeFO7iGcAc396GrBHbkJA25Iz+z3M7VYI/v22di1P8DeWguLNxbe1/wwCAbneYlxLbpPmkOKuFVho5NWDF1QgGszWONRM6DyiFiPlMF84nUcZEHP9exRELGqY64Cg3gfaKMXBn+QSZ5dzNH5cgVrWsGmPSeVBSTWZnZUYHWFKTbgnkY0bs1aHQefRAvl2CCZY/eFPJtcM9oIyIOiuX0NVz7h971tFJdJ9zpTesmaSVOjhCT00mUEJnPPI+zFJxLyIiXHjwROZXzCZL3J21DuWAb3+3ndVe6f47ao/4VNR/Q1TNJGIw5ewkA7RHHW9fegF27OzF+ddfd88ExFvBh1PvJXlAO3c050/2dIqDw7ovhIZWHFchUjSaoV6o5mkgv8/rEu76o3cpdAa78kgZ2tqzDQSquIY55eyHY1vDRusLYii5QZkXzblN9Hm/VvWuM//X99XtoFfnloeLcARWu6ze0fnsBeexK6iQMuUqseLZTjF3tnFqVe2cArA7Swwkh77fRWT3jm35FX1SHaT/kecsMiPY/E7sBfFGHjHVqSfEwMYXhJ6O3aur0UssB+UpuzWu+yBCkf7TienwzimXMrvOGnE++dSOxx6mXRaJBhI= diff --git a/Dockerfile b/Dockerfile index 0147e46..2aca02d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,35 +1,14 @@ # Dockerfile FROM python:3.7 -ARG KILLRVIDEO_DOCKER_IP -ENV KILLRVIDEO_DOCKER_IP ${KILLRVIDEO_DOCKER_IP} - -ARG KILLRVIDEO_HOST_IP -ENV KILLRVIDEO_HOST_IP ${KILLRVIDEO_HOST_IP} - -ARG KILLRVIDEO_DSE_USERNAME -ENV KILLRVIDEO_DSE_USERNAME ${KILLRVIDEO_DSE_USERNAME} - -ARG KILLRVIDEO_DSE_PASSWORD -ENV KILLRVIDEO_DSE_PASSWORD ${KILLRVIDEO_DSE_PASSWORD} +COPY killrvideo/ requirements.txt /app/ +WORKDIR /app # Install app dependencies -RUN pip install dse-driver -RUN pip install dse-graph -RUN pip install protobuf -RUN pip install grpcio -RUN pip install python-etcd -RUN pip install time-uuid -RUN pip install validate-email -RUN pip install sortedcontainers -RUN pip install nltk -RUN python -m nltk.downloader stopwords -RUN pip install kafka-python +RUN pip install -r requirements.txt \ + && python -m nltk.downloader stopwords # Create app directory -COPY killrvideo/ /app -WORKDIR /app - ENV PYTHONPATH "${PYTHONPATH}:/${WORKDIR}" EXPOSE 50101 diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml new file mode 100644 index 0000000..6c0f0c9 --- /dev/null +++ b/docker-compose.ci.yml @@ -0,0 +1,22 @@ +version: '3' +services: + backend: + image: killrvideo-python-local + depends_on: + - dse + + dse: + image: datastax/dse-server:6.7.4 + command: [ -s -g ] + environment: + DS_LICENSE: accept + cap_add: + - IPC_LOCK + ulimits: + memlock: -1 + + dse-config: + image: killrvideo/killrvideo-dse-config:3.0.0 + depends_on: + - dse + \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index d3f1a1f..513afaa 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -13,18 +13,9 @@ version: '3' services: - web: # Web Interface - image: killrvideo/killrvideo-web:3.0.1 - ports: - - "3000:3000" # Exposes port to be available externally - depends_on: - - backend # Needs Backend to be running - environment: - KILLRVIDEO_LOGGING_LEVEL: debug # More logging - # Service implementation - use docker-compose-backend-external.yaml if running services in IDE backend: - image: killrvideo/killrvideo-python + build: . ports: - "50101:50101" # Exposes port to be available. depends_on: @@ -33,6 +24,15 @@ services: environment: KILLRVIDEO_LOGGING_LEVEL: debug # More logging + web: # Web Interface + image: killrvideo/killrvideo-web:3.0.1 + ports: + - "3000:3000" # Exposes port to be available externally + depends_on: + - backend # Needs Backend to be running + environment: + KILLRVIDEO_LOGGING_LEVEL: debug # More logging + # Datastax Studio provides convenient web interface to work directly with DataStax Enterprise studio: image: datastax/dse-studio:6.7.0 From 16216c4beca68d644703ef386e58fc481f486063 Mon Sep 17 00:00:00 2001 From: Aleksandr Volochnev Date: Tue, 7 Jan 2020 11:43:39 +0100 Subject: [PATCH 2/5] Added kafka + zookeeper to dc.ci.yaml --- docker-compose.ci.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 6c0f0c9..0431e0a 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -19,4 +19,21 @@ services: image: killrvideo/killrvideo-dse-config:3.0.0 depends_on: - dse - \ No newline at end of file + + zookeeper: + image: wurstmeister/zookeeper + ports: + - 2181:2181 + + # Start a Kafka Broker for Killrvideo + kafka: + image: wurstmeister/kafka + depends_on: + - zookeeper + environment: + KAFKA_LISTENERS: PLAINTEXT://:9092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CREATE_TOPICS: "topic-kv-videoCreation:1:1,topic-kv-userCreation:1:1,topic-kv-videoRating:1:1,topic-kv-commentCreation:1:1" + ports: + - 9092:9092 From ee34fb79100b0eca88a83ad2bc8489c3947895cf Mon Sep 17 00:00:00 2001 From: Aleks Volochnev Date: Mon, 30 Mar 2020 13:16:01 +0200 Subject: [PATCH 3/5] Updated travis to release minor and major version docker images --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 96774c5..83fdbd5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,9 @@ after_success: - docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:${TRAVIS_TAG} - echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin - docker push killrvideo/killrvideo-python:${TRAVIS_TAG} -- "[ \"$(git tag --sort=-v:refname | grep -P \"^\\d+.\\d+.\\d+$\" | head -n1)\" == \"$TRAVIS_TAG\" ] && { docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:latest; docker push killrvideo/killrvideo-python:latest; }" +- "[ \"$(git tag --sort=-v:refname | grep -P \"^\\d+.\\d+.\\d+$\" | head -n1)\" == \"$TRAVIS_TAG\" ] && { docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:latest; docker push killrvideo/killrvideo-python:latest; }" +- "[ \"$(git tag --sort=-v:refname | grep -P \"^\\d+.\\d+.\\d+$\" | head -n1)\" == \"$TRAVIS_TAG\" ] && { docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:$(echo $TRAVIS_TAG | cut -d'.' -f 1); docker push killrvideo/killrvideo-python:$(echo $TRAVIS_TAG | cut -d'.' -f 1); }" +- "[ \"$(git tag --sort=-v:refname | grep -P \"^\\d+.\\d+.\\d+$\" | head -n1)\" == \"$TRAVIS_TAG\" ] && { docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-python:$(echo $TRAVIS_TAG | cut -d'.' -f 1).$(echo $TRAVIS_TAG | cut -d'.' -f 2); docker push killrvideo/killrvideo-python:$(echo $TRAVIS_TAG | cut -d'.' -f 1).$(echo $TRAVIS_TAG | cut -d'.' -f 2); }" after_failure: - travis_fold start logs From be9fa7a4cdfe3faee0816e4d48fa50cb4fd55dea Mon Sep 17 00:00:00 2001 From: Aleks Volochnev Date: Wed, 1 Apr 2020 11:25:46 +0200 Subject: [PATCH 4/5] Update docker-compose.ci.yml Soften reqs for dse-config --- docker-compose.ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 0431e0a..b75150d 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -16,7 +16,7 @@ services: memlock: -1 dse-config: - image: killrvideo/killrvideo-dse-config:3.0.0 + image: killrvideo/killrvideo-dse-config:3 depends_on: - dse From 527cab7ba41d39780f87e845c5131aa3bd7cfadd Mon Sep 17 00:00:00 2001 From: Aleks Volochnev Date: Wed, 1 Apr 2020 11:27:27 +0200 Subject: [PATCH 5/5] Update docker-compose.yaml Soften reqs for dse-config, web and generator --- docker-compose.yaml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 513afaa..27dcba7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,16 +1,5 @@ version: '3' -# -# docker-compose.yaml -# Simple compose file for running the Python services in Docker along with other containers comprising the -# KillrVideo. -# For more complex configuration options including metrics, external volumes, OpsCenter, -# etc., see https://github.com/KillrVideo/killrvideo-docker-common -# -# Note: the default configuration here uses the LATEST version of the killrvideo-python -# Docker image, which can be built using scripts/docker_build.sh. -# - services: # Service implementation - use docker-compose-backend-external.yaml if running services in IDE @@ -25,7 +14,7 @@ services: KILLRVIDEO_LOGGING_LEVEL: debug # More logging web: # Web Interface - image: killrvideo/killrvideo-web:3.0.1 + image: killrvideo/killrvideo-web:3 ports: - "3000:3000" # Exposes port to be available externally depends_on: @@ -60,14 +49,14 @@ services: # One-Time Bootstrap Container, configures DSE to have required keyspaces etc. dse-config: - image: killrvideo/killrvideo-dse-config:3.0.0 + image: killrvideo/killrvideo-dse-config:3 depends_on: - dse # Needs DSE to be running # Sample Data Generator, imitates behaviour of users on the killrVideo website. # Adds comments and rates videos, upload new videos and so on. generator: - image: killrvideo/killrvideo-generator:3.0.1 + image: killrvideo/killrvideo-generator:3 depends_on: - backend # Needs Backend to be running environment: