Configure AppVeyor to deliver native Windows containers of Pebble and Challtestsrv (#213)

Fixes #208. As a prerequisite, #212 must be merged before.

This PR configure an AppVeyor pipeline to build and publish native Windows
containers of Pebble and Challtestsrv on Docker Hub. This pipeline execute
the same logic that his Linux counterpart on Travis CI: go build, go
test using chisel2 and `acme` module from Certbot, then docker build +
docker push.

The CI part (build + test) is executed for every commit on every branch
(including pull requests). The CD part (docker deploy) is executed
only for tags pushed on Pebble repository. When triggered, the CD will
publish two native Windows dockers on the `letsencrypt/pebble` and
`letsencrypt/pebble-challtestsrv` Docker Hub projects: * the pinned
release tag `X.X.X-nanoserver-sac2016`, that corresponds to the tag
released on the GIT repository (Linux counterpart is `X.X.X`) * the
rolling release tag `nanoserver-sac2016` that is the latest pinned release
(Linux counterpart is `latest`)

Aside AppVeyor configuration, I also modified chisel2 to be runnable on
both Python 2.x and Python 3.x, because Certbot needs to run on Python
3.x on Windows. I also made some lights corrections on the Travis CI
pipeline to make it live nicely with AppVeyor.

Once this PR have been merged, one thing will need to be done from
the Pebble maintainers that I cannot do myself. Indeed, two secured
environment variables are required by this pipeline: `DOCKER_USER` and
`DOCKER_PASS` that are the Docker Hub credentials used to push the image
triggered by a GIT tag push, to trigger a release.

These credentials need to be set with the login/password
of a user authorized to push to `letsencrypt/pebble` and
`letsencrypt/pebble-challtestsrv` Docker Hub projects. To do so, go to
the AppVeyor Pebble project > Settings > Environment, and add the two
environment variables (name + value) in the "Environment variables"
section. **Be sure to check the lock "Toggle variable encryption" for
each environment variable.**

This can be done later after this PR have been merged, because as I said,
the credentials are required only when a tag, and so a new release of
Pebble, is pushed.

Author: adferrand

.travis/publish.sh .ci/publish_linux.sh.travis/publish.sh renamed to .ci/publish_linux.sh

@@ -1,13 +1,6 @@
 #!/usr/bin/env bash
 set -e
 
-if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then
-    echo "Publishing..."
-else
-    echo "Skipping publishing"
-    exit 0
-fi
-
 docker login -u "$DOCKER_USER" -p "$DOCKER_PASS"
 
 BASE_NAMES=(pebble pebble-challtestsrv)
@@ -17,15 +10,15 @@ for BASE_NAME in "${BASE_NAMES[@]}"; do
     echo "Updating docker ${IMAGE_NAME} image..."
 
     # create docker image
-    docker build -t "${IMAGE_NAME}:temp" -f "docker/${BASE_NAME}/Dockerfile" .
+    docker build -t "${IMAGE_NAME}:temp" -f "docker/${BASE_NAME}/linux.Dockerfile" .
 
     # push images
     if [ -n "${TRAVIS_TAG}" ]; then
         echo "Try to publish image: ${IMAGE_NAME}:${TRAVIS_TAG}"
         docker tag "${IMAGE_NAME}:temp" "${IMAGE_NAME}:${TRAVIS_TAG}"
         docker push "${IMAGE_NAME}:${TRAVIS_TAG}"
 
-        echo "Try to publish image: ${IMAGE_NAME}:latest"
+        echo "Try to publish rolling image: ${IMAGE_NAME}:latest"
         docker tag "${IMAGE_NAME}:${TRAVIS_TAG}" "${IMAGE_NAME}:latest"
         docker push "${IMAGE_NAME}:latest"
     fi

.ci/publish_windows.ps1

@@ -0,0 +1,28 @@
+$ErrorActionPreference = 'Stop'
+if ($env:APPVEYOR_REPO_TAG -ne "true") {
+    "Skipping publishing because this is not a tagged commit"
+} else {
+    "Publishing the tagged commit..."
+
+    $ErrorActionPreference = 'SilentlyContinue'
+    docker login -u="$env:DOCKER_USER" -p="$env:DOCKER_PASS"
+    $ErrorActionPreference = 'Stop'
+
+    $basenames = @("pebble", "pebble-challtestsrv")
+    foreach ($basename in $basenames) {
+        $image_name = "letsencrypt/$basename"
+        $tag = "$env:APPVEYOR_REPO_TAG_NAME-nanoserver-sac2016"
+
+        "Updating docker $basename image ..."
+
+        docker build -t="$image_name`:temp" -f="docker/$basename/windows.Dockerfile" .
+
+        "Try to publish image: $image_name`:$tag"
+        docker tag "$image_name`:temp" "$image_name`:$tag"
+        docker push "$image_name`:$tag"
+
+        "Try to publish rolling image: $image_name`:nanoserver-sac2016"
+        docker tag "$image_name`:temp" "$image_name`:nanoserver-sac2016"
+        docker push "$image_name`:nanoserver-sac2016"
+    }
+}

.travis.yml

@@ -1,6 +1,7 @@
 language: go
 
 sudo: true
+dist: xenial
 addons:
   hosts:
     - example.letsencrypt.org
@@ -37,8 +38,6 @@ install:
 
 before_script:
   - pebble &
-  # Wait for pebble to come up
-  - until </dev/tcp/localhost/14000 ; do sleep 0.1 ; done
 
 script:
   - go mod download
@@ -54,7 +53,7 @@ script:
 
 deploy:
   - provider: script
-    script: bash .travis/publish.sh
+    script: bash .ci/publish_linux.sh
     skip_cleanup: true
     on:
       repo: letsencrypt/pebble

appveyor.yml

@@ -1,4 +1,38 @@
-build: off
+image: Visual Studio 2017
+
+hosts:
+  example.letsencrypt.org: 127.0.0.1
+  elpmaxe.letsencrypt.org: 127.0.0.1
+
+environment:
+  GO111MODULE: on
+  PATH: C:\Python37;C:\msys64\mingw64\bin;%USERPROFILE%\go\bin;%PATH%
+
+install:
+  - git clone --single-branch --depth=1 -b master https://github.com/certbot/certbot
+  - cd certbot
+  - python tools\venv3.py
+  - venv3\Scripts\activate.bat
+  - cd ..
+
+before_build:
+  # Install `golangci-lint` using go get (installer not available for Windows)
+  - go get github.com/golangci/golangci-lint/cmd/[email protected]
+
+build_script:
+  - go install -v -mod=vendor ./...
+
+after_build:
+  - ps: $PebbleProcess = Start-Process pebble -PassThru
 
 test_script:
-  - ps: Write-Host "Hello, world!"
+  - go mod download
+  # Vet Go source code using the linter config (see .golang-ci.yml)
+  - golangci-lint run
+  # Run project unit tests (with the race detector enabled)
+  - go test -mod=vendor -v -race ./...
+  # Perform a test issuance with chisel2.py
+  - cmd /c "set REQUESTS_CA_BUNDLE=./test/certs/pebble.minica.pem && python .\test\chisel2.py example.letsencrypt.org elpmaxe.letsencrypt.org"
+
+deploy_script:
+  - ps: .ci\publish_windows.ps1

docker/pebble-challtestsrv/Dockerfile …ker/pebble-challtestsrv/linux.Dockerfiledocker/pebble-challtestsrv/Dockerfile renamed to docker/pebble-challtestsrv/linux.Dockerfile docker/pebble-challtestsrv/Dockerfile renamed to docker/pebble-challtestsrv/linux.Dockerfile

@@ -0,0 +1,17 @@
+FROM golang:1.11-nanoserver-sac2016 as builder
+
+ENV CGO_ENABLED=0 GOFLAGS=-mod=vendor
+
+WORKDIR /pebble-src
+COPY . .
+
+RUN go install -v ./cmd/pebble-challtestsrv/...
+
+## main
+FROM mcr.microsoft.com/windows/nanoserver:sac2016
+
+COPY --from=builder /gopath/bin/pebble-challtestsrv.exe /gopath/bin/pebble-challtestsrv.exe
+
+RUN powershell.exe -Command $path = $env:path + ';c:\gopath\bin'; Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment\' -Name Path -Value $path
+
+CMD [ "/pebble-challtestsrv" ]

docker/pebble-challtestsrv/windows.Dockerfile

@@ -0,0 +1,18 @@
+FROM golang:1.11-nanoserver-sac2016 as builder
+
+ENV CGO_ENABLED=0 GOFLAGS=-mod=vendor
+
+WORKDIR /pebble-src
+COPY . .
+
+RUN go install -v ./cmd/pebble/...
+
+## main
+FROM mcr.microsoft.com/windows/nanoserver:sac2016
+
+COPY --from=builder /gopath/bin/pebble.exe /gopath/bin/pebble.exe
+COPY --from=builder /pebble-src/test/ /test/
+
+RUN powershell.exe -Command $path = $env:path + ';c:\gopath\bin'; Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment\' -Name Path -Value $path
+
+CMD [ "/pebble" ]

docker/pebble/Dockerfile docker/pebble/linux.Dockerfiledocker/pebble/Dockerfile renamed to docker/pebble/linux.Dockerfile

@@ -8,14 +8,16 @@
 $ pip install -r requirements.txt
 $ python chisel.py foo.com bar.com
 """
-import json
+from __future__ import print_function
 import logging
 import os
+import ssl
 import sys
 import signal
 import threading
 import time
-import urllib2
+
+import requests
 
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives.asymmetric import rsa
@@ -44,6 +46,16 @@
 SET_TXT = "http://localhost:8055/set-txt"
 CLEAR_TXT = "http://localhost:8055/clear-txt"
 
+def wait_for_acme_server():
+    """Wait for directory URL set in the DIRECTORY env variable to respond"""
+    while True:
+        try:
+            if requests.get(DIRECTORY).status_code == 200:
+                return
+        except requests.exceptions.ConnectionError:
+            pass
+        time.sleep(0.1)
+
 def make_client(email=None):
     """Build an acme.Client and register a new account with a random key."""
     key = josepy.JWKRSA(key=rsa.generate_private_key(65537, 2048, default_backend()))
@@ -120,18 +132,16 @@ def do_dns_challenges(client, authzs):
         name, value = (c.validation_domain_name(a.body.identifier.value),
             c.validation(client.net.key))
         cleanup_hosts.append(name)
-        urllib2.urlopen(SET_TXT,
-            data=json.dumps({
-                "host": name + ".",
-                "value": value,
-            })).read()
+        requests.post(SET_TXT, json={
+            "host": name + ".",
+            "value": value
+        }).raise_for_status()
         client.answer_challenge(c, c.response(client.net.key))
     def cleanup():
         for host in cleanup_hosts:
-            urllib2.urlopen(CLEAR_TXT,
-                data=json.dumps({
-                    "host": host + ".",
-                })).read()
+            requests.post(CLEAR_TXT, json={
+                "host": host + "."
+            }).raise_for_status()
     return cleanup
 
 def do_http_challenges(client, authzs):
@@ -153,10 +163,11 @@ def cleanup():
         # Loop until the HTTP01Server is ready.
         while True:
             try:
-                urllib2.urlopen("http://localhost:%d" % port)
-                break
-            except urllib2.URLError:
-                time.sleep(0.1)
+                if requests.get("http://localhost:{0}".format(port)).status_code == 200:
+                    break
+            except requests.exceptions.ConnectionError:
+                pass
+            time.sleep(0.1)
 
         for chall_body in challs:
             client.answer_challenge(chall_body, chall_body.response(client.net.key))
@@ -191,10 +202,11 @@ def expect_problem(problem_type, func):
     signal.signal(signal.SIGINT, signal.SIG_DFL)
     domains = sys.argv[1:]
     if len(domains) == 0:
-        print __doc__
+        print(__doc__)
         sys.exit(0)
     try:
+        wait_for_acme_server()
         auth_and_issue(domains)
-    except messages.Error, e:
-        print e
+    except messages.Error as e:
+        print(e)
         sys.exit(1)