wait for service containers to come up on their designated tcp socket (#304)

* wait for service containers to come up on their designated tcp socket

* use container_ip in integration tests

Author: charless-splunk

tests/helpers/assertions.py

@@ -1,5 +1,6 @@
 import os
 import re
+import socket
 
 
 def has_datapoint_with_metric_name(fake_services, metric_name):
@@ -81,3 +82,11 @@ def any_metric_has_any_dim_key(fake_services, metrics, dim_keys):
                     print("Found metric \"%s\" with dimension key \"%s\"." % (dp.metric, dim.key))
                     return True
     return False
+
+
+def tcp_socket_open(host, port):
+    """
+    Returns True if there is an open TCP socket at the given host/port
+    """
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    return sock.connect_ex((host, port)) == 0

tests/monitors/apache_test.py

@@ -3,7 +3,7 @@
 import string
 
 from tests.helpers import fake_backend
-from tests.helpers.util import wait_for, run_agent, run_service
+from tests.helpers.util import wait_for, run_agent, run_service, container_ip
 from tests.helpers.assertions import *
 
 apache_config = string.Template("""
@@ -15,7 +15,9 @@
 
 def test_apache():
     with run_service("apache") as apache_container:
-        config = apache_config.substitute(host=apache_container.attrs["NetworkSettings"]["IPAddress"])
+        host = container_ip(apache_container)
+        config = apache_config.substitute(host=host)
+        assert wait_for(p(tcp_socket_open, host, 80), 60), "service didn't start"
 
         with run_agent(config) as [backend, _, _]:
             assert wait_for(p(has_datapoint_with_dim, backend, "plugin", "apache")), "Didn't get apache datapoints"

tests/monitors/cassandra_test.py

@@ -3,7 +3,7 @@
 import string
 
 from tests.helpers import fake_backend
-from tests.helpers.util import wait_for, run_agent, run_service
+from tests.helpers.util import wait_for, run_agent, run_service, container_ip
 from tests.helpers.assertions import *
 
 cassandra_config = string.Template("""
@@ -17,7 +17,7 @@
 
 def test_cassandra():
     with run_service("cassandra") as cassandra_cont:
-        config = cassandra_config.substitute(host=cassandra_cont.attrs["NetworkSettings"]["IPAddress"])
+        config = cassandra_config.substitute(host=container_ip(cassandra_cont))
 
         # Wait for the JMX port to be open in the container
         assert wait_for(p(container_cmd_exit_0, cassandra_cont,

tests/monitors/couchbase_test.py

@@ -2,8 +2,8 @@
 import string
 
 from tests.helpers import fake_backend
-from tests.helpers.util import wait_for, run_agent, run_service
-from tests.helpers.assertions import has_datapoint_with_dim
+from tests.helpers.util import wait_for, run_agent, run_service, container_ip
+from tests.helpers.assertions import has_datapoint_with_dim, tcp_socket_open
 
 couchbase_config = string.Template("""
 monitors:
@@ -18,8 +18,10 @@
 
 def test_couchbase():
     with run_service("couchbase", hostname="node1.cluster") as couchbase_container:
-        config = couchbase_config.substitute(host=couchbase_container.attrs["NetworkSettings"]["IPAddress"])
+        host = container_ip(couchbase_container)
+        config = couchbase_config.substitute(host=host)
+        assert wait_for(p(tcp_socket_open, host, 8091), 60), "service didn't start"
 
         with run_agent(config) as [backend, _, _]:
-            assert wait_for(p(has_datapoint_with_dim, backend, "plugin", "couchbase"), 40), \
+            assert wait_for(p(has_datapoint_with_dim, backend, "plugin", "couchbase")), \
                    "Didn't get couchbase datapoints"

tests/monitors/etcd_test.py

@@ -3,7 +3,7 @@
 import string
 
 from tests.helpers import fake_backend
-from tests.helpers.util import wait_for, run_agent, run_container
+from tests.helpers.util import wait_for, run_agent, run_container, container_ip
 from tests.helpers.assertions import *
 
 
@@ -19,7 +19,9 @@
 
 def test_etcd_monitor():
     with run_container("quay.io/coreos/etcd:v2.3.8", command=ETCD_COMMAND) as etcd_cont:
-        config = etcd_config.substitute(host=etcd_cont.attrs["NetworkSettings"]["IPAddress"])
+        host = container_ip(etcd_cont)
+        config = etcd_config.substitute(host=host)
+        assert wait_for(p(tcp_socket_open, host, 2379), 60), "service didn't start"
 
         with run_agent(config) as [backend, _, _]:
             assert wait_for(p(has_datapoint_with_dim, backend, "plugin", "etcd")), "Didn't get etcd datapoints"

tests/monitors/mongodb_test.py

@@ -3,7 +3,7 @@
 import string
 
 from tests.helpers import fake_backend
-from tests.helpers.util import wait_for, run_agent, run_container
+from tests.helpers.util import wait_for, run_agent, run_container, container_ip
 from tests.helpers.assertions import *
 
 mongo_config = string.Template("""
@@ -16,7 +16,9 @@
 
 def test_mongo():
     with run_container("mongo:3.6") as mongo_cont:
-        config = mongo_config.substitute(host=mongo_cont.attrs["NetworkSettings"]["IPAddress"])
+        host = container_ip(mongo_cont)
+        config = mongo_config.substitute(host=host)
+        assert wait_for(p(tcp_socket_open, host, 27017), 60), "service didn't start"
 
         with run_agent(config) as [backend, _, _]:
             assert wait_for(p(has_datapoint_with_dim, backend, "plugin", "mongo")), "Didn't get mongo datapoints"

tests/monitors/nginx_test.py

@@ -3,7 +3,7 @@
 import string
 
 from tests.helpers import fake_backend
-from tests.helpers.util import wait_for, run_agent, run_service
+from tests.helpers.util import wait_for, run_agent, run_service, container_ip
 from tests.helpers.assertions import *
 
 nginx_config = string.Template("""
@@ -15,7 +15,9 @@
 
 def test_nginx():
     with run_service("nginx") as nginx_container:
-        config = nginx_config.substitute(host=nginx_container.attrs["NetworkSettings"]["IPAddress"])
+        host = container_ip(nginx_container)
+        config = nginx_config.substitute(host=host)
+        assert wait_for(p(tcp_socket_open, host, 80), 60), "service didn't start"
 
         with run_agent(config) as [backend, _, _]:
             assert wait_for(p(has_datapoint_with_dim, backend, "plugin", "nginx")), "Didn't get nginx datapoints"

tests/monitors/postgresql_test.py

@@ -3,7 +3,7 @@
 import string
 
 from tests.helpers import fake_backend
-from tests.helpers.util import wait_for, run_agent, run_container
+from tests.helpers.util import wait_for, run_agent, run_container, container_ip
 from tests.helpers.assertions import *
 
 config_temp = string.Template("""
@@ -41,7 +41,9 @@
 
 def test_postgresql():
     with run_container("postgres:10", environment=env) as cont:
-        config = config_temp.substitute(host=cont.attrs["NetworkSettings"]["IPAddress"])
+        host = container_ip(cont)
+        config = config_temp.substitute(host=host)
+        assert wait_for(p(tcp_socket_open, host, 5432), 60), "service didn't start"
 
         with run_agent(config) as [backend, _, _]:
             assert wait_for(p(has_datapoint_with_dim, backend, "plugin", "postgresql")), "Didn't get postgresql datapoints"

tests/monitors/rabbitmq_test.py

@@ -3,14 +3,9 @@
 import string
 
 from tests.helpers import fake_backend
-from tests.helpers.util import wait_for, run_agent, run_container
+from tests.helpers.util import wait_for, run_agent, run_container, container_ip
 from tests.helpers.assertions import *
 
-def wait_for_rabbit_to_start(cont):
-    # 3D38 is 15672 in hex, the port we need to be listening
-    assert wait_for(p(container_cmd_exit_0, cont, "sh -c 'cat /proc/net/tcp | grep 3D38'"), 40), "rabbitmq didn't start"
-
-
 rabbitmq_config = string.Template("""
 monitors:
   - type: collectd/rabbitmq
@@ -24,9 +19,9 @@ def wait_for_rabbit_to_start(cont):
 
 def test_rabbitmq():
     with run_container("rabbitmq:3.6-management") as rabbitmq_cont:
-        host = rabbitmq_cont.attrs["NetworkSettings"]["IPAddress"]
+        host = container_ip(rabbitmq_cont)
         config = rabbitmq_config.substitute(host=host)
-        wait_for_rabbit_to_start(rabbitmq_cont)
+        assert wait_for(p(tcp_socket_open, host, 15672), 60), "service didn't start"
 
         with run_agent(config) as [backend, _, _]:
             assert wait_for(p(has_datapoint_with_dim, backend, "plugin", "rabbitmq")), "Didn't get rabbitmq datapoints"
@@ -35,9 +30,9 @@ def test_rabbitmq():
 
 def test_rabbitmq_broker_name():
     with run_container("rabbitmq:3.6-management") as rabbitmq_cont:
-        host = rabbitmq_cont.attrs["NetworkSettings"]["IPAddress"]
+        host = container_ip(rabbitmq_cont)
         config = rabbitmq_config.substitute(host=host)
-        wait_for_rabbit_to_start(rabbitmq_cont)
+        assert wait_for(p(tcp_socket_open, host, 15672), 60), "service didn't start"
 
         with run_agent("""
 monitors: