Test Automation Performance Improvements + Streamlining (#489)

* works for amqp

* updated makefile to use docker compose

* make clean now tears everything down

* ipp and http working

* ntp and pop

* postgres

* tests run and python script works

* Use python instead of shell test script and fail a test immediately if it's failing

* cleanup unused setup/cleanup scripts

* build runner and service_base in parallel

* make integration tests manually triggerable

* remove comment

* integration test manual run

* fix caps

* push an intermediate build image so ipp builds faster

* allow user to select a subset of modules to test

* Allow user to select only a few test modules

* update testing README

* remove extraneous scripts

* Revert "push an intermediate build image so ipp builds faster"

This reverts commit fc20f3b.

* comment makefile and remove Python2 instructions in README

* handle todo and typo fix

* test.py cleanup

* docker-compose cleanup

* add black to make lint

* lint python

Author: phillip-stephens

.github/workflows/integration-test.yml

@@ -4,6 +4,7 @@ on:
     branches:
       - master
   pull_request:
+  workflow_dispatch:
 
 jobs:
   go-test:

Dockerfile

@@ -1,6 +1,6 @@
 ## Build image ##
 ARG GO_VERSION=1.23
-FROM golang:${GO_VERSION}-alpine3.21 as build
+FROM golang:${GO_VERSION}-alpine3.21 AS build
 
 # System dependencies
 RUN apk add --no-cache make
@@ -16,7 +16,7 @@ COPY . .
 RUN make all
 
 ## Runtime image ##
-FROM alpine:3.21 as run
+FROM alpine:3.21 AS run
 
 COPY --from=build /usr/src/zgrab2/cmd/zgrab2/zgrab2 /usr/bin/zgrab2
 

Makefile

@@ -9,7 +9,7 @@ TEST_MODULES ?=
 
 all: zgrab2
 
-.PHONY: all clean integration-test integration-test-clean docker-runner gofmt test
+.PHONY: all clean integration-test integration-test-clean gofmt test
 
 # Test currently only runs on the modules folder because some of the 
 # third-party libraries in lib (e.g. http) are failing.
@@ -19,23 +19,26 @@ test:
 
 lint:
 	gofmt -s -w $(shell find . -type f -name '*.go'| grep -v "/.template/")
+	black .
 
 zgrab2: $(GO_FILES)
 	cd cmd/zgrab2 && go build && cd ../..
 	rm -f zgrab2
 	ln -s cmd/zgrab2/zgrab2$(EXECUTABLE_EXTENSION) zgrab2
 
-docker-runner: clean
-	make -C docker-runner
-
-integration-test: docker-runner
+integration-test:
 	rm -rf zgrab-output
-	TEST_MODULES=$(TEST_MODULES) ./integration_tests/test.sh
+	docker compose -p zgrab -f integration_tests/docker-compose.yml build --no-cache service_base # ensure the apt cache is up to date
+	docker compose -p zgrab -f integration_tests/docker-compose.yml build $(TEST_MODULES)
+	docker compose -p zgrab -f integration_tests/docker-compose.yml up -d $(TEST_MODULES)
+	sleep 10 # Wait for services to start
+	TEST_MODULES="$(TEST_MODULES)" python3 integration_tests/test.py
+	# Shut off the services
+	docker compose -p zgrab -f integration_tests/docker-compose.yml down
 
 integration-test-clean:
 	rm -rf zgrab-output
-	./integration_tests/cleanup.sh
-	make -C docker-runner clean
+	docker compose -p zgrab -f integration_tests/docker-compose.yml down
 
 clean:
 	cd cmd/zgrab2 && go clean

README.md

@@ -168,21 +168,25 @@ To add a schema for the new module, add a module under schemas, and update [`zgr
 See [zgrab2_schemas/README.md](zgrab2_schemas/README.md) for details.
 
 ### Integration tests
-To add integration tests for the new module, run `integration_tests/new.sh [your_new_protocol_name]`.
-This will add stub shell scripts in `integration_tests/your_new_protocol_name`; update these as needed.
+To add integration tests for the new module, you'll need to add a test service to scan against to `integration_tests/docker-compose.yml` and a `test.sh` in a folder named after your module.
+Follow the examples in `integration_tests/.template` to create the necessary files.
 See [integration_tests/mysql/*](integration_tests/mysql) for an example.
 The only hard requirement is that the `test.sh` script drops its output in `$ZGRAB_OUTPUT/[your-module]/*.json`, so that it can be validated against the schema.
 
 #### How to Run Integration Tests
 
-To run integration tests, you must have [Docker](https://www.docker.com/) and **Python 2** on host installed. Then, you can follow the following steps to run integration tests:
+To run integration tests, you must have [Docker](https://www.docker.com/) and **Python 3** on host installed. Then, you can follow the following steps to run integration tests:
 
 ```shell
-go get github.com/jmespath/jp && go build github.com/jmespath/jp
-# or, sudo wget https://github.com/jmespath/jp/releases/download/0.2.1/jp-linux-amd64 -O /usr/local/bin/jp && sudo chmod +x /usr/local/bin/jp
-pip2 install --user zschema
-pip2 install --user -r requirements.txt
-make integration-test
+# Install Python dependencies
+sudo apt update
+sudo apt install -y python3 jp python3-pip
+python3 -m venv venv
+source venv/bin/activate
+# Install Python dependencies
+pip install zschema
+pip install -r requirements.txt
+make integration-test-clean; make integration-test
 ```
 
 Running the integration tests will generate quite a bit of debug output. To ensure that tests completed successfully, you can check for a successful exit code after the tests complete:
@@ -192,6 +196,13 @@ echo $?
 0
 ```
 
+To just run a single/few module's integration tests, you can use the `TEST_MODULES` env. var.:
+
+```shell
+make integration-test-clean; TEST_MODULES="http" make integration-test
+make integration-test-clean; TEST_MODULES="http ssh" make integration-test
+```
+
 Refer to our [Github Actions workflow](.github/workflows/integration-test.yml) for an example of how to prepare environment for integration tests.
 
 ## License

docker-runner/Makefile

@@ -7,4 +7,4 @@
 : "${CONTAINER_NAME:?}"
 
 set -e
-echo 'target' | docker run --rm -i --link $CONTAINER_NAME:target zgrab2_runner $@
+echo 'target' | docker run --rm -i --network container:$CONTAINER_NAME zgrab2_runner $@

docker-runner/base-apt-update.sh

@@ -1,22 +1,15 @@
-Template integration test scripts, used by `new.sh`.
+Template integration test scripts
+
+Specify your module's service in the `docker-compose.yml` file.
 
  - All scripts start with the default `CONTAINER_NAME` variable matching
    the standard format (`zgrab_<module name>`).
 
- - `setup.sh` tries to launch the container; if it fails, then it tries
-   to build the docker image from the `Dockerfile` in
-   `integration_tests/<module_name>/container`; then it tries to launch
-   the container again. **This must be modified by all new modules**.
-
  - `test.sh` runs the container using the `docker-runner.sh` script and
    dumps the output in `zgrab-output/<module name>/<module_name>.json`.
    Afterwards it dumps the container logs to stdout. This can be used
    as-is, it only provides a single test case with no arguments.
 
- - `cleanup.sh` runs `docker stop $CONTAINER_NAME`. This should tear
-   down everything started in `setup.sh`. In the case where there is
-   only a single container, it can be used as-is.
-
  - `schema.py` registers the `scan_response_type` and provides a 
    skeleton that can be filled out. **This must be modified for all new
    modules**.