Go validation

.github/workflows/pull-request.yaml

@@ -15,5 +15,5 @@ jobs:
       - name: Run clang-format style check for C/C++/Protobuf programs.
         uses: jidicula/clang-format-action@f62da5e3d3a2d88ff364771d9d938773a618ab5e
         with:
-          check-path: 'src'
+          check-path: 'src/nginx_module'
           clang-format-version: '16'

.gitignore

@@ -2,3 +2,7 @@ nginx-*
 vcpkg
 .vscode
 **/.DS_Store
+src/validator/firetail-validator.so
+src/validator/firetail-validator.h
+*.swo
+*.swp

README.md

@@ -29,6 +29,62 @@ The Firetail NGINX Module will be receiving 401 responses as you have not yet co
 
 
 
+### Request Validation
+
+To demonstrate request validation a `POST /proxy/profile/{username}/comment` operation is defined in the provided [appspec.yml](./dev/appspec.yml), with one profile defined in the provided [nginx.conf](./dev/nginx.conf): `POST /proxy/profile/alice/comment`. A `proxy_pass` directive is used to forward requests to `/profile/alice/comment` as the request body validation will not occur on locations where the `return` directive is used.
+
+Making a curl request to `POST /profile/alice/comment` should yield the following result, which validates successfully against the provided appspec:
+
+```bash
+curl localhost:8080/proxy/profile/alice/comment -X POST -H "Content-Type: application/json" -d '{"comment":"Hello world!"}
+```
+
+```json
+{"message":"Success!"}
+```
+
+If you alter the request body in any way that deviates from the appspec you will receive an error response from the Firetail nginx module instead:
+
+```bash
+curl localhost:8080/proxy/profile/alice/comment -X POST -H "Content-Type: application/json" -d '{"comment":12345}'
+```
+
+```json
+{"code":400,"title":"something's wrong with your request body","detail":"the request's body did not match your appspec: request body has an error: doesn't match the schema: Error at \"/comment\": field must be set to string or not be present\nSchema:\n  {\n    \"type\": \"string\"\n  }\n\nValue:\n  \"number, integer\"\n"}
+```
+
+
+
+### Response Validation
+
+To demonstrate response validation a `GET /profile/{username}` operation is defined in the provided [appspec.yml](./dev/appspec.yml), and two profiles are defined in the provided [nginx.conf](./dev/nginx.conf): `GET /profile/alice` and `GET /profile/bob`.
+
+Making a curl request to `GET /profile/alice` should yield the following result, which validates successfully against the provided appspec, as it returns only Alice's username and friend count:
+
+```bash
+curl localhost:8080/profile/alice
+```
+
+```json
+{"username":"alice", "friends": 123456789}
+```
+
+Making a curl request to `GET /profile/bob` will yield a different result, as the response body defined in our nginx.conf erroneously includes Bob's address. This does not validate against our appspec, so the response body is overwritten by the Firetail middleware, which can protect Bob from having his personally identifiable information disclosed by our faulty application. For the purposes of this demo, the Firetail library's debugging responses are enabled so we get a verbose explanation of the problem:
+
+```bash
+curl localhost:8080/profile/bob
+```
+
+```json
+{
+    "code": 500,
+    "title": "internal server error",
+    "detail": "the response's body did not match your appspec: response body doesn't match the schema: property \"address\" is unsupported\nSchema:\n  {\n    \"additionalProperties\": false,\n    \"properties\": {\n      \"friends\": {\n        \"minimum\": 0,\n        \"type\": \"integer\"\n      },\n      \"username\": {\n        \"type\": \"string\"\n      }\n    },\n    \"type\": \"object\"\n  }\n\nValue:\n  {\n    \"address\": \"Oh dear, this shouldn't be public!\",\n    \"friends\": 123456789,\n    \"username\": \"bob\"\n  }\n"
+}
+```
+
+
+
 ### VSCode
 
 For local development with VSCode you'll probably want to download the nginx tarball matching the version you're developing for, and configure it:
@@ -76,7 +132,7 @@ You can then use the `configure` command to generate a `makefile` to build the d
 
 ```bash
 cd nginx-1.24.0
-./configure --with-compat --add-dynamic-module=../src
+./configure --with-compat --add-dynamic-module=../src/nginx_module
 make modules
 ```
 
@@ -86,6 +142,10 @@ You will then need to install the Firetail NGINX Module's dependencies, [curl](h
 make modules
 ```
 
+The Firetail NGINX module is also dependent upon a validator module, written in Go.
+
+// TODO: docs for building the Golang validator 
+
 
 
 ## Configuration

dev/Dockerfile

@@ -1,4 +1,18 @@
-FROM debian:bullseye-slim AS build
+FROM golang:1.21.0-bullseye AS build-golang
+
+# Make a /src and /dist directory, with our workdir set to /src
+WORKDIR /src
+RUN mkdir /dist
+
+# Copy in our go source for the validator
+COPY src/validator /src
+
+# Build the go source as a c-shared lib, outputting to /dist as /dist/validator.so
+# NOTE: this will also create a /dist/validator.h which is not needed, so it's discarded it afterwards
+RUN CGO_ENABLED=1 go build -buildmode c-shared -o /dist/firetail-validator.so .
+RUN rm /dist/firetail-validator.h
+
+FROM debian:bullseye-slim AS build-c
 
 ENV NGINX_VERSION 1.24.0
 
@@ -12,18 +26,20 @@ RUN curl http://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz -o /tmp/nginx-$
 RUN apt install -y build-essential libpcre++-dev zlib1g-dev libcurl4-openssl-dev libjson-c-dev
 
 # Build our dynamic module
-COPY src /tmp/ngx-firetail-module
+COPY src/nginx_module /tmp/ngx-firetail-module
 RUN cd /tmp/nginx-${NGINX_VERSION} && \
     ./configure --with-compat --add-dynamic-module=/tmp/ngx-firetail-module && \
     make modules
 
 # Copy our dynamic module & its dependencies into the image for the nginx version we want to use
 FROM nginx:1.24.0 AS firetail-nginx
 RUN apt-get update && apt-get install -y libjson-c-dev
-COPY --from=build /tmp/nginx-${NGINX_VERSION}/objs/* /etc/nginx/modules/
+COPY --from=build-golang /dist/* /etc/nginx/modules/
+COPY --from=build-c /tmp/nginx-${NGINX_VERSION}/objs/* /etc/nginx/modules/
 
 # An image for local dev with a custom nginx.conf and index.html
 FROM firetail-nginx as firetail-nginx-dev
+COPY dev/appspec.yml /etc/nginx/appspec.yml
 COPY dev/nginx.conf /etc/nginx/nginx.conf
 COPY dev/index.html /usr/share/nginx/html/
 CMD ["nginx-debug", "-g", "daemon off;"]

dev/appspec.yml

@@ -0,0 +1,181 @@
+openapi: 3.0.1
+info:
+  title: Firetail Nginx Module Example
+  version: "0.1"
+paths:
+  /:
+    get:
+      summary: Returns an index.html
+      responses:
+        "200":
+          description: An index.html file
+          content:
+            text/plain: {}
+  /health:
+    get:
+      summary: Returns the status of the server
+      responses:
+        "200":
+          $ref: "#/components/responses/Healthy"
+  /notfound:
+    get:
+      summary: Returns a mock 404 response
+      responses:
+        "404":
+          $ref: "#/components/responses/NotFound"
+  /unhealthy:
+    get:
+      summary: Returns a mock 400 response
+      responses:
+        "500":
+          $ref: "#/components/responses/Unhealthy"
+  /proxy/health:
+    get:
+      summary: Returns the status of the server
+      responses:
+        "200":
+          $ref: "#/components/responses/Healthy"
+  /proxy/notfound:
+    get:
+      summary: Returns a mock 404 response
+      responses:
+        "404":
+          $ref: "#/components/responses/NotFound"
+  /proxy/unhealthy:
+    get:
+      summary: Returns a mock 400 response
+      responses:
+        "500":
+          $ref: "#/components/responses/Unhealthy"
+  /profile/{username}:
+    get:
+      summary: Returns a user's profile
+      parameters:
+        - in: path
+          name: username
+          required: true
+          schema:
+            description: The username of the user whose profile should be returned
+            type: string
+            minLength: 3
+            maxLength: 20
+      responses:
+        "200":
+          description: A user's profile
+          content:
+            application/json:
+              schema:
+                type: object
+                additionalProperties: false
+                properties:
+                  username:
+                    type: string
+                  friends:
+                    type: integer
+                    minimum: 0
+  /profile/{username}/comment:
+    post:
+      summary: Post a comment on a user's profile
+      requestBody:
+        description: The comment to be added to the user's profile
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required: ["comment"]
+              additionalProperties: false
+              properties:
+                comment:
+                  type: string
+      parameters:
+        - in: path
+          name: username
+          required: true
+          schema:
+            description: The username of the user whose profile should be returned
+            type: string
+            minLength: 3
+            maxLength: 20
+      responses:
+        "201":
+          description: Comment created
+          content:
+            application/json:
+              schema:
+                type: object
+                additionalProperties: False
+                properties:
+                  message:
+                    type: string
+  /proxy/profile/{username}/comment:
+    post:
+      required: true
+      summary: Post a comment on a user's profile
+      requestBody:
+        description: The comment to be added to the user's profile
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required: ["comment"]
+              additionalProperties: false
+              properties:
+                comment:
+                  type: string
+      parameters:
+        - in: path
+          name: username
+          required: true
+          schema:
+            description: The username of the user whose profile should be returned
+            type: string
+            minLength: 3
+            maxLength: 20
+      responses:
+        "201":
+          description: Comment created
+          content:
+            application/json:
+              schema:
+                type: object
+                additionalProperties: False
+                properties:
+                  message:
+                    type: string
+components:
+  responses:
+    Healthy:
+      description: A mocked response from a /health endpoint on a healthy service
+      content:
+        application/json:
+          schema:
+            type: object
+            additionalProperties: false
+            properties:
+              message:
+                type: string
+                enum: ["I'm healthy! 💖"]
+    Unhealthy:
+      description: A mocked response from a /health endpoint on an unhealthy service
+      content:
+        application/json:
+          schema:
+            type: object
+            additionalProperties: false
+            properties:
+              message:
+                type: string
+                enum: ["I'm unhealthy! 🤒"]
+    NotFound:
+      description: A mocked 404 response
+      content:
+        application/json:
+          schema:
+            type: object
+            additionalProperties: false
+            properties:
+              message:
+                type: string
+                enum: ["Not Found 🤷"]

dev/nginx.conf

@@ -1,5 +1,6 @@
 # Step 1: Load the Firetail NGINX Module
 load_module modules/ngx_firetail_module.so;
+# error_log stderr debug;
 
 events {}
 
@@ -9,6 +10,7 @@ http {
   # Step 2: Provide your Firetail API token to the Firetail NGINX Module
   # You should use lua-nginx-module to pull the API token in from an environment variable here
   firetail_api_token "YOUR-API-TOKEN";
+  firetail_url "https://api.logging.eu-west-1.prod.firetail.app/logs/bulk";
 
   server {
     listen       80;
@@ -24,23 +26,43 @@ http {
     }
 
     location /notfound {
-      return 404 '{"message":"Not Found"}';
+      return 404 '{"message":"Not Found 🤷"}';
     }
 
     location /unhealthy {
       return 500 '{"message":"I\'m unhealthy! 🤒"}';
     }
 
-    location /health-proxy {
+    location /profile/alice {
+      return 200 '{"username":"alice", "friends": 123456789}';
+    }
+
+    location /profile/bob {
+      return 200 '{"username":"bob", "friends": 123456789, "address":"Oh dear, this shouldn\'t be public!"}';
+    }
+
+    location /profile/alice/comment {
+      return 201 '{"message":"Success!"}';
+    }
+
+    location /proxy/health {
       proxy_pass http://localhost:80/health;
     }
 
-    location /notfound-proxy {
+    location /proxy/notfound {
       proxy_pass http://localhost:80/notexists;
     }
 
-    location /unhealthy-proxy {
+    location /proxy/unhealthy {
       proxy_pass http://localhost:80/unhealth;
     }
+
+    location /proxy/profile/alice/comment {
+      proxy_pass http://localhost:80/profile/alice/comment;
+    }
+
+    location /proxy/profile/bob {
+      proxy_pass http://localhost:80/profile/bob;
+    }
   }
-}
+}