Add kinesis integration testing

Makefile

@@ -0,0 +1,9 @@
+.PHONY: all compile test
+
+compile:
+	rebar3 compile
+
+test:
+	cd test && \
+	docker compose rm --volumes --force && \
+	docker compose up --build --abort-on-container-exit

test/bin/dump_to_dir

@@ -0,0 +1,23 @@
+#!/bin/bash
+if [ $# -ne 2 ]; then
+    echo "usage: $(basename $0) <stream name> <directory>" >&2
+    exit 1
+fi
+STREAM_NAME=$1
+DIR=$2
+aws kinesis list-shards --stream-name ${STREAM_NAME} \
+      --query 'Shards[*].[ShardId, SequenceNumberRange.StartingSequenceNumber]' --output text |
+    while read shard sequence; do
+        iterator=$(aws kinesis get-shard-iterator --stream-name ${STREAM_NAME} \
+                       --shard-id $shard --shard-iterator-type AT_SEQUENCE_NUMBER \
+                       --starting-sequence-number $sequence \
+                       --query "ShardIterator" --output text)
+        aws kinesis get-records --stream-name ${STREAM_NAME} \
+            --shard-iterator ${iterator} --query "Records[].[PartitionKey,Data]" --output text |
+            while read key data; do
+                echo -n "${key} $(echo -n ${data} | wc -c)"
+                out="${DIR}/$key"
+                echo "${data}" | base64 -d > ${out}
+                echo " --> $(cat ${out} | wc -c) bytes"
+            done
+    done

test/bin/load_from_dir

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -x
+if [ $# -ne 2 ]; then
+    echo "usage: $(basename $0) <stream name> <directory>" >&2
+    exit 1
+fi
+STREAM_NAME=$1
+DIR=$2
+JSON=$(mktemp)
+cat <<EOF >$JSON
+{
+  "StreamName": "${STREAM_NAME}",
+  "Records": [
+EOF
+SEP=""
+for file in $DIR/*; do
+    if [ -f $file ]; then
+        path=$(realpath $file)
+        echo -ne "${SEP}" >>$JSON
+        cat <<EOF >>$JSON
+    {
+      "PartitionKey": "${file}",
+EOF
+        echo -n "      \"Data\": \"$(base64 -w 0 ${path})\"    }" >> $JSON
+        SEP=",\\n"
+    fi
+done
+cat <<EOF >>$JSON
+
+  ]
+}
+EOF
+
+aws kinesis put-records --stream-name ${STREAM_NAME} --cli-input-json file://${JSON}

test/compose.yml

@@ -0,0 +1,68 @@
+services:
+  ca:
+    build: ./compose/ca
+    volumes:
+      - type: volume
+        source: certs
+        target: /certs
+    healthcheck:
+      test: ["CMD", "/ca/check_cert.sh"]
+      interval: 1s
+    stop_signal: SIGKILL
+  tests:
+    build: ./compose/test_runner
+    depends_on:
+      ca:
+        condition: service_healthy
+      kinesis:
+        condition: service_started
+    networks:
+      - local_kinesis
+    volumes:
+      - type: volume
+        source: certs
+        target: /certs
+      - type: bind
+        source: ${TEST_DIR:-../}
+        target: /home/work/code
+    environment:
+      ERL_AFLAGS: "-enable-feature all"
+      AWS_CBOR_DISABLE: 1
+    command: ${TEST_COMMAND:-rebar3 do clean,compile,eunit,ct}
+  dynamo:
+    build: ./compose/dynamo
+    depends_on:
+      ca:
+        condition: service_healthy
+    domainname: dynamodb.us-east-1.amazonaws.com
+    networks:
+      local_kinesis:
+        aliases:
+          - dynamodb.us-east-1.amazonaws.com
+    volumes:
+      - type: volume
+        source: certs
+        target: /usr/local/lib/node_modules/dynalite/ssl
+    stop_signal: SIGKILL
+  kinesis:
+    build: ./compose/kinesis
+    depends_on:
+      ca:
+        condition: service_healthy
+    domainname: kinesis.us-east-1.amazaonaws.com
+    networks:
+      local_kinesis:
+        aliases:
+          - kinesis.us-east-1.amazonaws.com
+          - 000000000000.data-kinesis.us-east-1.amazonaws.com
+    volumes:
+      - type: volume
+        source: certs
+        target: /usr/local/lib/node_modules/kinesalite/ssl
+    stop_signal: SIGKILL
+
+networks:
+  local_kinesis:
+
+volumes:
+  certs:

test/compose/ca/Dockerfile

@@ -0,0 +1,17 @@
+FROM buildpack-deps:bullseye
+
+RUN DEBIAN_FRONTEND=noninteractive \
+    apt-get update && \
+    apt-get install -y openssl
+
+RUN mkdir /ca
+RUN mkdir /certs
+WORKDIR /ca
+
+COPY cacert.sh /ca
+COPY check_cert.sh /ca
+COPY ca.conf /ca
+
+HEALTHCHECK --interval=5s CMD /ca/check_cert.sh
+
+CMD ./cacert.sh

test/compose/ca/ca.conf

@@ -0,0 +1,25 @@
+[ ca ]
+default_ca = ca_default
+[ ca_default ]
+dir = ./ca
+certs = $dir
+new_certs_dir = $dir/ca.db.certs
+database = $dir/ca.db.index
+serial = $dir/ca.db.serial
+RANDFILE = $dir/ca.db.rand
+certificate = $dir/ca.crt
+private_key = $dir/ca.key
+default_days = 365
+default_crl_days = 30
+default_md = sha256
+preserve = no
+policy = generic_policy
+copy_extensions = copy
+[ generic_policy ]
+countryName = optional
+stateOrProvinceName = optional
+localityName = optional
+organizationName = optional
+organizationalUnitName = optional
+commonName = optional
+emailAddress = optional

test/compose/ca/cacert.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+set -x
+
+KEY_SIZE=2048
+
+rm -rf ca
+rm /certs/*
+mkdir -p ca/ca.db.certs
+touch ca/ca.db.index
+echo "1234" > ca/ca.db.serial
+openssl genrsa -des3 -passout pass:foobar -out ca/ca.key $KEY_SIZE || exit 1
+
+openssl req -new -x509 -passin pass:foobar \
+        -subj "/C=US/ST=CA/O=ACME/CN=acme.fake/[email protected]" \
+        -days 10000 -key ca/ca.key -out ca/ca.crt || exit 2
+
+openssl genrsa -out server-key.pem $KEY_SIZE || exit 3
+
+REGION_DOMAIN="us-east-1.amazonaws.com"
+mk_san() {
+    cat <<EOF | while read sub; do echo -n "${SEP}DNS:${sub}.${REGION_DOMAIN}"; SEP=","; done
+kinesis
+dynamodb
+000000000000.data-kinesis
+EOF
+}
+SAN=$(mk_san)
+echo $SAN
+openssl req -new \
+        -subj "/C=US/ST=CA/O=Nile Web Services/CN=*.$REGION_DOMAIN" \
+        -addext "subjectAltName=$SAN" \
+        -key server-key.pem -out server-csr.pem || exit 4
+
+openssl ca -config ca.conf -passin pass:foobar -batch -out server-crt.pem -infiles server-csr.pem || exit 5
+
+cp server*.pem /certs
+cp ca/ca.crt /certs/ca-crt.pem
+cp ca/ca.key /certs/ca-key.pem
+
+chown nobody.nogroup /certs/*
+chmod a+r /certs/server-crt.pem /certs/ca-key.pem
+
+sleep infinity

test/compose/ca/check_cert.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+SRV_KEY_MOD=$(openssl rsa -noout -modulus -in /certs/server-key.pem)
+SRV_CRT_MOD=$(openssl x509 -noout -modulus -in /certs/server-crt.pem)
+CA_KEY_MOD=$(openssl rsa -passin pass:foobar -noout -modulus -in /certs/ca-key.pem)
+CA_CRT_MOD=$(openssl x509 -noout -modulus -in /certs/ca-crt.pem)
+
+if [ "$SRV_KEY_MOD" != "$SRV_CRT_MOD" ]; then
+    echo "Server cert/key mismatch" 2>&1
+    exit 1
+fi
+if [ "$CA_KEY_MOD" != "$CA_CRT_MOD" ]; then
+    echo "CA cert/key mismatch" 2>&1
+    exit 1
+fi
+
+openssl verify -CAfile /certs/ca-crt.pem /certs/server-crt.pem || exit 3
+

test/compose/dynamo/Dockerfile

@@ -0,0 +1,19 @@
+FROM docker.io/node:20.3-bullseye
+
+ENV AWS_DEFAULT_REGION "us-east-1"
+ENV NODE_ENV production
+ENV NPM_CONFIG_CACHE /data
+
+WORKDIR /data
+
+RUN ls
+
+RUN mkdir -p /.npm /data && \
+    npm install -g [email protected] && \
+    chown -R nobody:nogroup /data /.npm /data
+
+USER nobody
+
+VOLUME ["/data"]
+
+CMD /usr/local/bin/dynalite --ssl --port 443

test/compose/kinesis/Dockerfile

@@ -0,0 +1,20 @@
+FROM docker.io/node:18-bullseye
+
+ENV AWS_DEFAULT_REGION "us-east-1"
+ENV NODE_ENV production
+#ENV NODE_PENDING_DEPRECATION 1
+ENV NPM_CONFIG_CACHE /data
+
+WORKDIR /data
+
+RUN ls
+
+RUN mkdir -p /.npm /data && \
+    npm install -g [email protected] && \
+    chown -R nobody:nogroup /data /.npm /data
+
+USER nobody
+
+VOLUME ["/data"]
+
+CMD /usr/local/bin/kinesalite --port 443 --ssl --path /data --shardLimit 100 --createStreamMs 50 --deleteStreamMs 50  --updateStreamMs 50

test/compose/test_runner/Dockerfile

@@ -0,0 +1,25 @@
+FROM erlang:25
+
+RUN DEBIAN_FRONTEND=noninteractive \
+    apt-get update && \
+    apt-get install -y build-essential maven gosu unzip curl python3-pip python3-venv bind9-dnsutils less socat
+
+RUN mkdir /home/work
+WORKDIR /home/work
+
+RUN mkdir /build && cd /build && \
+    pip3 install 'flit_core>=3.7.1,<3.8.1' && \
+    curl https://awscli.amazonaws.com/awscli.tar.gz -o awscli.tar.gz && \
+    tar xzf awscli.tar.gz && \
+    cd $(ls | grep awscli-) && \
+    ./configure --with-download-deps && \
+    make && make install && aws --version
+
+RUN cd /bin && \
+    wget https://s3.amazonaws.com/rebar3/rebar3 && \
+    chmod +x rebar3
+
+RUN touch /
+
+COPY entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

test/compose/test_runner/entrypoint.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+set -x
+
+######################################
+#                                    #
+#   Trust generated CA Certificate   #
+#                                    #
+######################################
+
+openssl rsa -noout -modulus -in /certs/server-key.pem
+openssl x509 -noout -modulus -in /certs/server-crt.pem
+openssl rsa -passin pass:foobar -noout -modulus -in /certs/ca-key.pem
+openssl x509 -noout -modulus -in /certs/ca-crt.pem
+
+cp /certs/ca-crt.pem /usr/local/share/ca-certificates/acme.crt
+update-ca-certificates
+
+for cafile in $(find /usr/local/lib/aws-cli -name 'cacert.pem'); do
+    cp --no-clobber $cafile "${cafile}.orig"
+    cp "${cafile}.orig" $cafile
+    openssl x509 -in /certs/ca-crt.pem -text >> $cafile
+done
+
+openssl s_client -connect dynamodb.us-east-1.amazonaws.com:443
+export AWS_ACCESS_KEY_ID=phony
+export AWS_SECRET_ACCESS_KEY=fake
+export AWS_DEFAULT_REGION=us-east-1
+
+dig kinesis.us-east-1.amazonaws.com
+
+git config --global --add safe.directory /home/work
+
+aws kinesis list-streams || exit $?
+aws dynamodb list-tables || exit $?
+
+###########################################
+#                                         #
+#  Create and use a user that matches     #
+#  uid/gid with the owner of the work dir #
+#                                         #
+###########################################
+
+if [ "$(id -u)" -eq "0" ]; then
+    uid=$(ls -nd ./code | cut -d' ' -f3)
+    gid=$(ls -nd ./code | cut -d' ' -f4)
+    if [ "$uid" -eq "0" -o "$gid" -eq "0" ]; then
+        groupadd work_user
+        useradd -g work_user work_user -d /home/work
+        cp -r /home/work/code /home/work/copy
+        chown -R work_user:work_user /home/work/copy
+        cd /home/work/copy
+        set +x
+        cat <<EOF
+
+*********************************************************************************
+**                                                                             **
+**                            N  O  T  I  C  E                                 **
+**                                                                             **
+** --------------------------------------------------------------------------- **
+**                                                                             **
+** You are running docker as root, and your working dir is owned by root.      **
+** This is probaby due to using docker-desktop (maybe on MacOs or Windows).    **
+**                                                                             **
+** Output files will not be present in your working dir. To retrieve, run:     **
+**                                                                             **
+** docker compose -f test/compose.yml cp tests:/home/work/copy/_build ./_out   **
+**                                                                             **
+*********************************************************************************
+
+EOF
+        set -x
+    else
+        groupadd -g $gid work_user
+        useradd -u $uid -g $gid work_user -d /home/work
+        cd /home/work/code
+    fi
+    chown work_user:work_user /home/work
+    # insert gosu command into $@
+    ls -ld /home/work
+    set -- gosu work_user "$@"
+fi
+
+SHELL="/bin/bash" eval "$@"