From 237a2d24d6589745f8732ec19074b53d65ad8a7b Mon Sep 17 00:00:00 2001 From: theteacat Date: Thu, 22 Aug 2024 14:44:53 +0100 Subject: [PATCH 1/3] Overhaul docs, move Dockerfile & add dist to .gitignore --- .gitignore | 1 + dev/Dockerfile => Dockerfile | 11 +-- README.md | 132 +++++++++++++++++++++-------------- 3 files changed, 86 insertions(+), 58 deletions(-) rename dev/Dockerfile => Dockerfile (86%) diff --git a/.gitignore b/.gitignore index 3d16ad9..05cb4d2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ src/validator/firetail-validator.so src/validator/firetail-validator.h *.swo *.swp +dist diff --git a/dev/Dockerfile b/Dockerfile similarity index 86% rename from dev/Dockerfile rename to Dockerfile index 75f9ccf..6bbaef1 100644 --- a/dev/Dockerfile +++ b/Dockerfile @@ -1,3 +1,5 @@ +ARG NGINX_VERSION=1.24.0 + FROM golang:1.21.0-bullseye AS build-golang # Make a /src and /dist directory, with our workdir set to /src @@ -13,8 +15,7 @@ RUN CGO_ENABLED=1 go build -buildmode c-shared -o /dist/firetail-validator.so . RUN rm /dist/firetail-validator.h FROM debian:bullseye-slim AS build-c - -ENV NGINX_VERSION 1.24.0 +ARG NGINX_VERSION # Curl the tarball of the nginx version matching that of the container we want to modify RUN apt update && apt install -y curl @@ -32,10 +33,10 @@ RUN cd /tmp/nginx-${NGINX_VERSION} && \ make modules # Copy our dynamic module & its dependencies into the image for the nginx version we want to use -FROM nginx:1.24.0 AS firetail-nginx +FROM nginx:${NGINX_VERSION} AS firetail-nginx RUN apt-get update && apt-get install -y libjson-c-dev -COPY --from=build-golang /dist/* /etc/nginx/modules/ -COPY --from=build-c /tmp/nginx-${NGINX_VERSION}/objs/* /etc/nginx/modules/ +COPY --from=build-golang /dist/firetail-validator.so /etc/nginx/modules/ +COPY --from=build-c /tmp/nginx-${NGINX_VERSION}/objs/ngx_firetail_module.so /etc/nginx/modules/ # An image for local dev with a custom nginx.conf and index.html FROM firetail-nginx as firetail-nginx-dev diff --git a/README.md b/README.md index ffe9c62..974e5d6 100644 --- a/README.md +++ b/README.md @@ -4,18 +4,55 @@ The Firetail NGINX Module is a [dynamic module](https://docs.nginx.com/nginx/adm +## Build Process + +The easiest way to attain the binaries for the FireTail NGINX module is to use [the provided Dockerfile](./Dockerfile). By default, it uses NGINX 1.24.0 - you need to ensure this matches the version of NGINX you are using. You can adjust the version used via the build argument NGINX_VERSION. For example, to build for NGINX 1.25.0: + +```bash +git clone git@github.com:FireTail-io/firetail-nginx-module.git +cd firetail-nginx-module +docker build -t firetail-nginx . --target firetail-nginx --build-arg="NGINX_VERSION=1.25.0" +docker run -v ./dist:/dist firetail-nginx cp /usr/lib/nginx/modules/{ngx_firetail_module.so,firetail-validator.so} /dist +``` + +You should then find the binaries `ngx_firetail_module.so` and `firetail-validator.so` in a `dist` directory, which you will need to place in your NGINX instances' modules directory - Conventionally this is `/etc/nginx/modules` - then load it using [the `load_module` directive](https://nginx.org/en/docs/ngx_core_module.html#load_module). + +If you want to DIY, you can follow [the guidance below](#DIY-Build-Process). + + + +## Configuration + +To setup the Firetail NGINX Module, you will need to modify your `nginx.conf` to load it using the `load_module` directive: + +``` +load_module modules/ngx_firetail_module.so; +``` + +You can then use the `firetail_api_token` directive to provide your Firetail logging API token inside a http block like so: + +``` + firetail_api_token "YOUR-API-TOKEN"; +``` + +See [dev/nginx.conf](./dev/nginx.conf) for an example of this in action. + +You should use a module such as the [ngx_http_lua_module](https://github.com/openresty/lua-nginx-module) to avoid placing plaintext credentials in your `nginx.conf`, and instead make use of [system environment variables](https://github.com/openresty/lua-nginx-module#system-environment-variable-support). + + + ## Local Development -A [Dockerfile](./dev/Dockerfile) is provided which will build the module, install it in [an NGINX docker image](https://hub.docker.com/_/nginx), and setup a custom [nginx.conf](./dev/nginx.conf) and [index.html](./dev/index.html). It should be as simple as: +A [Dockerfile](./Dockerfile) is provided which will build the module, install it in [an NGINX docker image](https://hub.docker.com/_/nginx), and setup a custom [nginx.conf](./dev/nginx.conf) and [index.html](./dev/index.html). It should be as simple as: ```bash git clone git@github.com:FireTail-io/firetail-nginx-module.git cd firetail-nginx-module -docker build -t firetail-nginx . --target firetail-nginx-dev -f dev/Dockerfile +docker build -t firetail-nginx . --target firetail-nginx-dev docker run -p 8080:80 firetail-nginx ``` -When you first make a request to [localhost:8080](http://localhost:8080) you will likely see a log similar to the following: +When you first make a request to [localhost:8080](http://localhost:8080) you will likely eventually see a log similar to the following: ``` 2023/07/14 11:44:37 [debug] 29#29: *1 Status code from POST request to Firetail /logs/bulk endpoint: 401 @@ -29,6 +66,31 @@ The Firetail NGINX Module will be receiving 401 responses as you have not yet co +### VSCode + +For local development with VSCode you'll probably want to download the nginx tarball matching the version you're developing for, and configure it: + +```bash +curl -O http://nginx.org/download/nginx-1.24.0.tar.gz +tar xvzf nginx-1.24.0.tar.gz +cd nginx-1.24.0 +./configure +``` + +Then install VSCode's [Microsoft C/C++ Extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode.cpptools). By default it should happily be finding your nginx headers. If it's not, go to its settings using the command `C/C++: Edit configurations (UI)` and add the following `includePath`: + +``` +${workspaceFolder}/** +``` + +You'll then need to install some of nginx's dependencies (`pcre2`), and the dependenceis of the firetail NGINX module itself (`curl` and `json-c`). You can use [vcpkg](https://vcpkg.io/) for this: + +```bash +vcpkg install pcre2 curl json-c +``` + + + ### Request Validation To demonstrate request validation a `POST /proxy/profile/{username}/comment` operation is defined in the provided [appspec.yml](./dev/appspec.yml), with one profile defined in the provided [nginx.conf](./dev/nginx.conf): `POST /proxy/profile/alice/comment`. A `proxy_pass` directive is used to forward requests to `/profile/alice/comment` as the request body validation will not occur on locations where the `return` directive is used. @@ -85,34 +147,11 @@ curl localhost:8080/profile/bob -### VSCode - -For local development with VSCode you'll probably want to download the nginx tarball matching the version you're developing for, and configure it: - -```bash -curl -O http://nginx.org/download/nginx-1.24.0.tar.gz -tar xvzf nginx-1.24.0.tar.gz -cd nginx-1.24.0 -./configure -``` - -Then install VSCode's [Microsoft C/C++ Extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode.cpptools). By default it should happily be finding your nginx headers. If it's not, go to its settings using the command `C/C++: Edit configurations (UI)` and add the following `includePath`: - -``` -${workspaceFolder}/** -``` - -You'll then need to install some of nginx's dependencies (`pcre2`), and the dependenceis of the firetail NGINX module itself (`curl` and `json-c`). You can use [vcpkg](https://vcpkg.io/) for this: -```bash -vcpkg install pcre2 curl json-c -``` +## DIY Build Process - -## Compilation - -You can follow the [documentation on the NGINX blog](https://www.nginx.com/blog/compiling-dynamic-modules-nginx-plus/) for how to compile third-party dynamic modules for NGINX and NGINX Plus. +If you want to DIY, you can follow the [documentation on the NGINX blog](https://www.nginx.com/blog/compiling-dynamic-modules-nginx-plus/) for how to compile third-party dynamic modules for NGINX and NGINX Plus. First clone this repository: @@ -126,43 +165,30 @@ Then obtain the open source NGINX release matching the version of NGINX you use: ```bash curl -O http://nginx.org/download/nginx-1.24.0.tar.gz tar xvzf nginx-1.24.0.tar.gz +cd nginx-1.24.0 ``` -You can then use the `configure` command to generate a `makefile` to build the dynamic module: +Then install the Firetail NGINX Module's dependencies, [curl](https://github.com/curl/curl) and [json-c](https://github.com/json-c/json-c). + +You can then use the `configure` command to generate a `makefile` to build the dynamic module. You may need to use the `--with-ld-opt` and `--with-cc-opt` options. ```bash -cd nginx-1.24.0 -./configure --with-compat --add-dynamic-module=../src/nginx_module -make modules +./configure --with-compat --add-dynamic-module=../src/nginx_module --with-ld-opt="-L/opt/homebrew/lib" --with-cc-opt="-I/opt/homebrew/include" ``` -You will then need to install the Firetail NGINX Module's dependencies, [curl](https://github.com/curl/curl) and [json-c](https://github.com/json-c/json-c), before running: +The `configure` command will create a makefile you can then use to build the module: ```bash make modules ``` -The Firetail NGINX module is also dependent upon a validator module, written in Go. - -// TODO: docs for building the Golang validator - - - -## Configuration - -To setup the Firetail NGINX Module, you will need to modify your `nginx.conf` to load it using the `load_module` directive: +This should yield a file called `ngx_firetail_module.so` in the `objs` directory - this is the plugin, however it is dependant on a second binary which also needs to be compiled: -``` -load_module modules/ngx_firetail_module.so; -``` - -You can then use the `firetail_api_token` directive to provide your Firetail logging API token inside a http block like so: - -``` - firetail_api_token "YOUR-API-TOKEN"; +```bash +cd ../src/validator +CGO_ENABLED=1 go build -buildmode c-shared -o firetail-validator.so . ``` -See [dev/nginx.conf](./dev/nginx.conf) for an example of this in action. - -You should use a module such as the [ngx_http_lua_module](https://github.com/openresty/lua-nginx-module) to avoid placing plaintext credentials in your `nginx.conf`, and instead make use of [system environment variables](https://github.com/openresty/lua-nginx-module#system-environment-variable-support). +This should yield a file called `firetail-validator.so`. +You will need to take copies of the `ngx_firetail_module.so` and `firetail-validator.so` binaries and place them in `/etc/nginx/modules/`. From 30330b394eda994aae8ece35af5c8ef30ef03f88 Mon Sep 17 00:00:00 2001 From: theteacat Date: Thu, 22 Aug 2024 14:45:39 +0100 Subject: [PATCH 2/3] Format & remove unecessary log --- src/validator/main.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/validator/main.go b/src/validator/main.go index 5ba34a8..b0f49a8 100644 --- a/src/validator/main.go +++ b/src/validator/main.go @@ -96,10 +96,10 @@ func ValidateRequestBody(specBytes unsafe.Pointer, specLength C.int, //export ValidateResponseBody func ValidateResponseBody(urlCharPtr unsafe.Pointer, - urlLength C.int, - tokenCharPtr unsafe.Pointer, tokenLength C.int, - reqBodyCharPtr unsafe.Pointer, reqBodyLength C.int, - specBytes unsafe.Pointer, specLength C.int, + urlLength C.int, + tokenCharPtr unsafe.Pointer, tokenLength C.int, + reqBodyCharPtr unsafe.Pointer, reqBodyLength C.int, + specBytes unsafe.Pointer, specLength C.int, resBodyCharPtr unsafe.Pointer, resBodyLength C.int, pathCharPtr unsafe.Pointer, pathLength C.int, statusCode C.int, @@ -109,7 +109,6 @@ func ValidateResponseBody(urlCharPtr unsafe.Pointer, tokenSlice := C.GoBytes(tokenCharPtr, tokenLength) urlSlice := C.GoBytes(urlCharPtr, urlLength) - log.Println("URL: ", string(urlSlice)) trimTokenSlice := strings.TrimSpace(string(tokenSlice)) trimUrlSlice := strings.TrimSpace(string(urlSlice)) From 29f2e7edd29523bcf89684b5f610ccbda856ed96 Mon Sep 17 00:00:00 2001 From: theteacat Date: Thu, 22 Aug 2024 14:55:28 +0100 Subject: [PATCH 3/3] Detail expected appspec location --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 974e5d6..2e3a976 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ See [dev/nginx.conf](./dev/nginx.conf) for an example of this in action. You should use a module such as the [ngx_http_lua_module](https://github.com/openresty/lua-nginx-module) to avoid placing plaintext credentials in your `nginx.conf`, and instead make use of [system environment variables](https://github.com/openresty/lua-nginx-module#system-environment-variable-support). +Once you've configured your `nginx.conf` you will also need to provide an OpenAPI specification. The FireTail NGINX Module expects to find your OpenAPI specification at `/etc/nginx/appspec.yml`. + ## Local Development