chore(vdev): refactor to avoid temp files and leverage docker APIs

scripts/e2e/opentelemetry-logs/compose.yaml

@@ -81,3 +81,8 @@ services:
 volumes:
   vector_target:
     external: true
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/amqp/compose.yaml

@@ -13,3 +13,8 @@ services:
       - RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=false
     volumes:
       - ../../..:/code
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/aws/compose.yaml

@@ -12,3 +12,8 @@ services:
     volumes:
     - $DOCKER_SOCKET:/var/run/docker.sock
     - $HOME/.aws/:/home/.aws/
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/azure/compose.yaml

@@ -6,3 +6,8 @@ services:
     command: azurite --blobHost 0.0.0.0 --loose
     volumes:
     - /var/run:/var/run
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/clickhouse/compose.yaml

@@ -3,3 +3,8 @@ version: '3'
 services:
   clickhouse:
     image: docker.io/clickhouse/clickhouse-server:${CONFIG_VERSION}
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/databend/compose.yaml

@@ -19,3 +19,8 @@ services:
       - minio
     healthcheck:
       test: "curl -f localhost:8080/v1/health || exit 1"
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/datadog-agent/compose.yaml

@@ -27,3 +27,8 @@ services:
     - DD_CMD_PORT=5002
     - DD_USE_DOGSTATSD=false
     - DD_HOSTNAME=datadog-trace-agent
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/datadog-traces/compose.yaml

@@ -29,3 +29,8 @@ services:
     - DD_APM_MAX_MEMORY=0
     - DD_APM_MAX_CPU_PERCENT=0
     - DD_HOSTNAME=datadog-trace-agent-to-vector
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/dnstap/compose.yaml

@@ -13,3 +13,8 @@ services:
 
 volumes:
   dnstap-sockets: {}
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/elasticsearch/compose.yaml

@@ -25,3 +25,8 @@ services:
     - ES_JAVA_OPTS=-Xms400m -Xmx400m
     volumes:
     - ../../../tests/data/ca:/usr/share/elasticsearch/config/certs:ro
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/eventstoredb/compose.yaml

@@ -6,3 +6,8 @@ services:
     command: --insecure --stats-period-sec=1
     volumes:
     - ../../../tests/data:/etc/vector:ro
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/gcp/compose.yaml

@@ -15,3 +15,8 @@ services:
     command:
     - -p
     - /public.pem
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/greptimedb/compose.yaml

@@ -6,3 +6,8 @@ services:
     command: "standalone start --http-addr=0.0.0.0:4000 --rpc-addr=0.0.0.0:4001"
     healthcheck:
       test: "curl -f localhost:4000/health || exit 1"
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/http-client/compose.yaml

@@ -29,3 +29,8 @@ services:
     - ../../../tests/data/http-client/serve:/data
     - ../../../tests/data/ca/intermediate_server/certs/dufs-https-chain.cert.pem:/certs/ca.cert.pem
     - ../../../tests/data/ca/intermediate_server/private/dufs-https.key.pem:/certs/ca.key.pem
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/humio/compose.yaml

@@ -3,3 +3,8 @@ version: '3'
 services:
   humio:
     image: docker.io/humio/humio-single-node-demo:${CONFIG_VERSION}
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/influxdb/compose.yaml

@@ -19,3 +19,8 @@ services:
     command: influxd --reporting-disabled
     environment:
     - INFLUXDB_REPORTING_DISABLED=true
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/kafka/compose.yaml

@@ -34,3 +34,8 @@ services:
     - ../../../tests/data/ca/intermediate_server/private/kafka.pass:/etc/kafka/secrets/kafka.pass:ro
     - ../../../tests/data/ca/intermediate_server/private/kafka.p12:/etc/kafka/secrets/kafka.p12:ro
     - ../../../tests/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/logstash/compose.yaml

@@ -12,3 +12,8 @@ services:
     - /dev/null:/usr/share/logstash/pipeline/logstash.yml
     - ../../../tests/data/host.docker.internal.crt:/tmp/logstash.crt
     - ../../../tests/data/logstash/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/loki/compose.yaml

@@ -4,3 +4,8 @@ services:
   loki:
     image: docker.io/grafana/loki:${CONFIG_VERSION}
     command: -config.file=/etc/loki/local-config.yaml -auth.enabled=true
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/mongodb/compose.yaml

@@ -30,3 +30,8 @@ services:
     - MONGODB_INITIAL_PRIMARY_PORT_NUMBER=27017
     - MONGODB_INITIAL_PRIMARY_ROOT_PASSWORD=toor
     - MONGODB_REPLICA_SET_KEY=vector
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/nats/compose.yaml

@@ -50,3 +50,8 @@ services:
     - /usr/share/nats/config/nats-jetstream.conf
     volumes:
     - ../../../tests/data/nats:/usr/share/nats/config
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/opentelemetry/compose.yaml

@@ -5,3 +5,8 @@ services:
     image: docker.io/otel/opentelemetry-collector-contrib:${CONFIG_VERSION}
     volumes:
     - ../../../tests/data/opentelemetry/config.yaml:/etc/otelcol-contrib/config.yaml
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/postgres/compose.yaml

@@ -14,3 +14,8 @@ services:
 
 volumes:
   socket: {}
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/prometheus/compose.yaml

@@ -21,3 +21,8 @@ services:
     command: --config.file=/etc/vector/prometheus.yaml
     volumes:
     - ../../../tests/data:/etc/vector:ro
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/pulsar/compose.yaml

@@ -16,3 +16,8 @@ services:
       - ../../../tests/data/ca/intermediate_server/private/pulsar.key.pem:/etc/pulsar/certs/pulsar.key.pem:ro
       - ../../../tests/data//ca/intermediate_server/certs/pulsar.cert.pem:/etc/pulsar/certs/pulsar.cert.pem:ro
       - ../../../tests/data/ca/intermediate_server/certs/ca-chain.cert.pem:/etc/pulsar/certs/ca-chain.cert.pem:ro
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/redis/compose.yaml

@@ -24,3 +24,8 @@ services:
             echo "sentinel failover-timeout vector 5000" >> /etc/sentinel.conf &&
             echo "sentinel parallel-syncs vector 1" >> /etc/sentinel.conf &&
             redis-sentinel /etc/sentinel.conf'
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/shutdown/compose.yaml

@@ -34,3 +34,8 @@ services:
     - ../../../tests/data/ca/intermediate_server/private/kafka.pass:/etc/kafka/secrets/kafka.pass:ro
     - ../../../tests/data/ca/intermediate_server/private/kafka.p12:/etc/kafka/secrets/kafka.p12:ro
     - ../../../tests/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/splunk/compose.yaml

@@ -13,3 +13,8 @@ services:
     - 8000:8000
     - 8088:8088
     - 8089:8089
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/webhdfs/compose.yaml

@@ -30,3 +30,8 @@ services:
       interval: 5s
       timeout: 5s
       retries: 3
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

vdev/src/commands/compose_tests/active_projects.rs

@@ -0,0 +1,60 @@
+use anyhow::{Context, Result};
+use std::{collections::HashSet, process::Command};
+
+use crate::testing::{config::ComposeTestConfig, docker::CONTAINER_TOOL};
+
+/// Query Docker Compose for active projects
+pub(super) fn load_active_projects() -> Result<HashSet<String>> {
+    let output = Command::new(CONTAINER_TOOL.clone())
+        .args(["compose", "ls", "--format", "json"])
+        .output()
+        .with_context(|| "Failed to list compose projects")?;
+
+    if !output.status.success() {
+        return Ok(HashSet::new());
+    }
+
+    let projects: Vec<serde_json::Value> = serde_json::from_slice(&output.stdout)
+        .with_context(|| "Failed to parse docker compose ls output")?;
+
+    Ok(projects
+        .iter()
+        .filter_map(|project| {
+            project
+                .get("Name")
+                .and_then(|n| n.as_str())
+                .map(String::from)
+        })
+        .collect())
+}
+
+/// Find the active environment for a given integration by matching Docker Compose project names
+pub(super) fn find_active_environment(
+    active_projects: &HashSet<String>,
+    prefix: &str,
+    config: &ComposeTestConfig,
+) -> Option<String> {
+    for project_name in active_projects {
+        if let Some(sanitized_env_name) = project_name.strip_prefix(prefix) {
+            // The project name has dots replaced with hyphens, so we need to check
+            // all environments to find a match after applying the same sanitization
+            for env_name in config.environments().keys() {
+                if env_name.replace('.', "-") == sanitized_env_name {
+                    return Some(env_name.to_string());
+                }
+            }
+        }
+    }
+    None
+}
+
+/// Find the active environment for a given integration by querying Docker Compose
+pub(super) fn find_active_environment_for_integration(
+    directory: &str,
+    integration: &str,
+    config: &ComposeTestConfig,
+) -> Result<Option<String>> {
+    let active_projects = load_active_projects()?;
+    let prefix = format!("vector-{directory}-{integration}-");
+    Ok(find_active_environment(&active_projects, &prefix, config))
+}

vdev/src/commands/compose_tests/mod.rs

@@ -1,3 +1,5 @@
+mod active_projects;
+
 pub(crate) mod ci_paths;
 pub(crate) mod show;
 pub(crate) mod start;