Merge pull request #21 from Sebitosh/infrastructure-apache2-modsec2

Utility change: infrastructure for apache2 with modsec2 on ubuntu

Author: airween

README.md

@@ -697,6 +697,46 @@ SecAction "id:100020,phase:2 pass, setenv:'after=789'"
 ```
 
 ## Run the tool
+### Run the entire toolchain
+
+#### Required:
+* [albedo](https://github.com/coreruleset/albedo)
+* [go-ftw](https://github.com/coreruleset/go-ftw)
+* apache2 with the modsecurity and proxy modules for using the `apache2_ubuntu` + ModSecurity V2 infrastructure
+
+To run the tests on a provided configuration, run the tool:
+
+~~~bash
+$ ./mrts/mrts.py
+usage: mrts.py [-h] -i /path/to/infra/ -r /path/to/mrts/*.yaml -e /path/to/mrts/rules/ -t /path/to/mrts/tests/ [-c]
+               [-f /path/to/mrts/ftw.mrts.config.yaml] [-v]
+mrts.py: error: the following arguments are required: -i/--infrastructure, -r/--rulesdef, -e/--expdir, -t/--testdir
+~~~
+
+As you can see there are few command line arguments.
+* `-i` - WAF infrastructure files
+* `-r` - rules definition files
+* `-e` - export directory where rules will be written
+* `-t` - export test directory where tests will be written
+* `-c` - clean previously generated rule and test files
+* `-f` - `go-ftw` custom configuration file, if you don't want to use the default file provided in the infrastructure directory
+* `-v` - verbose output
+
+For running without a custom `go-ftw` configuration, run the `mrts.py` script from the root directory of the project (or else provide a ftw configuration file with a correct relative path).
+
+~~~bash
+$ ./mrts/mrts.py -i config_infra/apache2_ubuntu/ -r config_tests/ -e generated/rules/ -t generated/tests/regression/tests/
+Generate rules and tests
+Launch backend
+Launch infrastructure
+Executing test set...
+🎉🎉🎉 Success: test set passed
+Backend shutdown
+Infrastructure shutdown
+MRTS completed
+~~~
+
+### Rule and test generation
 
 To generate the rules and their tests, run the tool:
 

config_infra/.gitignore

@@ -0,0 +1,6 @@
+logfile: 'config_infra/apache2_ubuntu/infra/log/error.log'
+logmarkerheadername: 'X-MRTS-TEST'
+logtype:
+  name: 'apache'
+  timeregex:  '\[([A-Z][a-z]{2} [A-z][a-z]{2} \d{1,2} \d{1,2}\:\d{1,2}\:\d{1,2}\.\d+? \d{4})\]'
+  timeformat: 'ddd MMM DD HH:mm:ss.S YYYY'

config_infra/apache2_ubuntu/ftw.mrts.config.yaml

@@ -0,0 +1,228 @@
+# This is the main Apache server configuration file.  It contains the
+# configuration directives that give the server its instructions.
+# See http://httpd.apache.org/docs/2.4/ for detailed information about
+# the directives and /usr/share/doc/apache2/README.Debian about Debian specific
+# hints.
+#
+#
+# Summary of how the Apache 2 configuration works in Debian:
+# The Apache 2 web server configuration in Debian is quite different to
+# upstream's suggested way to configure the web server. This is because Debian's
+# default Apache2 installation attempts to make adding and removing modules,
+# virtual hosts, and extra configuration directives as flexible as possible, in
+# order to make automating the changes and administering the server as easy as
+# possible.
+
+# It is split into several files forming the configuration hierarchy outlined
+# below, all located in the /etc/apache2/ directory:
+#
+#	/etc/apache2/
+#	|-- apache2.conf
+#	|	`--  ports.conf
+#	|-- mods-enabled
+#	|	|-- *.load
+#	|	`-- *.conf
+#	|-- conf-enabled
+#	|	`-- *.conf
+# 	`-- sites-enabled
+#	 	`-- *.conf
+#
+#
+# * apache2.conf is the main configuration file (this file). It puts the pieces
+#   together by including all remaining configuration files when starting up the
+#   web server.
+#
+# * ports.conf is always included from the main configuration file. It is
+#   supposed to determine listening ports for incoming connections which can be
+#   customized anytime.
+#
+# * Configuration files in the mods-enabled/, conf-enabled/ and sites-enabled/
+#   directories contain particular configuration snippets which manage modules,
+#   global configuration fragments, or virtual host configurations,
+#   respectively.
+#
+#   They are activated by symlinking available configuration files from their
+#   respective *-available/ counterparts. These should be managed by using our
+#   helpers a2enmod/a2dismod, a2ensite/a2dissite and a2enconf/a2disconf. See
+#   their respective man pages for detailed information.
+#
+# * The binary is called apache2. Due to the use of environment variables, in
+#   the default configuration, apache2 needs to be started/stopped with
+#   /etc/init.d/apache2 or apache2ctl. Calling /usr/bin/apache2 directly will not
+#   work with the default configuration.
+
+
+# Global configuration
+#
+
+#
+# ServerRoot: The top of the directory tree under which the server's
+# configuration, error, and log files are kept.
+#
+# NOTE!  If you intend to place this on an NFS (or otherwise network)
+# mounted filesystem then please read the Mutex documentation (available
+# at <URL:http://httpd.apache.org/docs/2.4/mod/core.html#mutex>);
+# you will save yourself a lot of trouble.
+#
+# Do NOT add a slash at the end of the directory path.
+#
+#ServerRoot "/etc/apache2"
+
+#
+# The accept serialization lock file MUST BE STORED ON A LOCAL DISK.
+#
+#Mutex file:${APACHE_LOCK_DIR} default
+
+#
+# The directory where shm and other runtime files will be stored.
+#
+
+DefaultRuntimeDir ${APACHE_RUN_DIR}
+
+#
+# PidFile: The file in which the server should record its process
+# identification number when it starts.
+# This needs to be set in /etc/apache2/envvars
+#
+PidFile run/apache2.pid
+
+#
+# Timeout: The number of seconds before receives and sends time out.
+#
+Timeout 300
+
+#
+# KeepAlive: Whether or not to allow persistent connections (more than
+# one request per connection). Set to "Off" to deactivate.
+#
+KeepAlive On
+
+#
+# MaxKeepAliveRequests: The maximum number of requests to allow
+# during a persistent connection. Set to 0 to allow an unlimited amount.
+# We recommend you leave this number high, for maximum performance.
+#
+MaxKeepAliveRequests 100
+
+#
+# KeepAliveTimeout: Number of seconds to wait for the next request from the
+# same client on the same connection.
+#
+KeepAliveTimeout 5
+
+
+# These need to be set in /etc/apache2/envvars
+User ${APACHE_RUN_USER}
+Group ${APACHE_RUN_GROUP}
+
+#
+# HostnameLookups: Log the names of clients or just their IP addresses
+# e.g., www.apache.org (on) or 204.62.129.132 (off).
+# The default is off because it'd be overall better for the net if people
+# had to knowingly turn this feature on, since enabling it means that
+# each client request will result in AT LEAST one lookup request to the
+# nameserver.
+#
+HostnameLookups Off
+
+# ErrorLog: The location of the error log file.
+# If you do not specify an ErrorLog directive within a <VirtualHost>
+# container, error messages relating to that virtual host will be
+# logged here.  If you *do* define an error logfile for a <VirtualHost>
+# container, that host's errors will be logged there and not here.
+#
+
+ErrorLog log/error.log
+
+#
+# LogLevel: Control the severity of messages logged to the error_log.
+# Available values: trace8, ..., trace1, debug, info, notice, warn,
+# error, crit, alert, emerg.
+# It is also possible to configure the log level for particular modules, e.g.
+# "LogLevel info ssl:warn"
+#
+LogLevel warn
+
+# Include module configuration:
+IncludeOptional mods-enabled/*.load
+IncludeOptional mods-enabled/*.conf
+
+# Include list of ports to listen on
+Include ports.conf
+
+
+# Sets the default security model of the Apache2 HTTPD server. It does
+# not allow access to the root filesystem outside of /usr/share and /var/www.
+# The former is used by web applications packaged in Debian,
+# the latter may be used for local directories served by the web server. If
+# your system is serving content from a sub-directory in /srv you must allow
+# access here, or in any related virtual host.
+<Directory />
+	Options FollowSymLinks
+	AllowOverride None
+	Require all denied
+</Directory>
+
+<Directory /usr/share>
+	AllowOverride None
+	Require all granted
+</Directory>
+
+<Directory /var/www/>
+	Options Indexes FollowSymLinks
+	AllowOverride None
+	Require all granted
+</Directory>
+
+#<Directory /srv/>
+#	Options Indexes FollowSymLinks
+#	AllowOverride None
+#	Require all granted
+#</Directory>
+
+
+
+
+# AccessFileName: The name of the file to look for in each directory
+# for additional configuration directives.  See also the AllowOverride
+# directive.
+#
+AccessFileName .htaccess
+
+#
+# The following lines prevent .htaccess and .htpasswd files from being
+# viewed by Web clients.
+#
+<FilesMatch "^\.ht">
+	Require all denied
+</FilesMatch>
+
+
+#
+# The following directives define some format nicknames for use with
+# a CustomLog directive.
+#
+# These deviate from the Common Log Format definitions in that they use %O
+# (the actual bytes sent including headers) instead of %b (the size of the
+# requested file), because the latter makes it impossible to detect partial
+# requests.
+#
+# Note that the use of %{X-Forwarded-For}i instead of %h is not recommended.
+# Use mod_remoteip instead.
+#
+LogFormat "%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined
+LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" combined
+LogFormat "%h %l %u %t \"%r\" %>s %O" common
+LogFormat "%{Referer}i -> %U" referer
+LogFormat "%{User-agent}i" agent
+
+# Include of directories ignores editors' and dpkg's backup files,
+# see README.Debian for details.
+
+# Include generic snippets of statements
+IncludeOptional conf-enabled/*.conf
+
+# Include the virtual host configurations:
+IncludeOptional sites-enabled/*.conf
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

config_infra/apache2_ubuntu/infra/apache2.conf

@@ -0,0 +1,8 @@
+# Read the documentation before enabling AddDefaultCharset.
+# In general, it is only a good idea if you know that all your files
+# have this encoding. It will override any encoding given in the files
+# in meta http-equiv or xml encoding tags.
+
+#AddDefaultCharset UTF-8
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

config_infra/apache2_ubuntu/infra/conf-available/charset.conf

@@ -0,0 +1,81 @@
+# Customizable error responses come in three flavors:
+# 1) plain text
+# 2) local redirects
+# 3) external redirects
+#
+# Some examples:
+#ErrorDocument 500 "The server made a boo boo."
+#ErrorDocument 404 /missing.html
+#ErrorDocument 404 "/cgi-bin/missing_handler.pl"
+#ErrorDocument 402 http://www.example.com/subscription_info.html
+#
+
+#
+# Putting this all together, we can internationalize error responses.
+#
+# We use Alias to redirect any /error/HTTP_<error>.html.var response to
+# our collection of by-error message multi-language collections.  We use
+# includes to substitute the appropriate text.
+#
+# You can modify the messages' appearance without changing any of the
+# default HTTP_<error>.html.var files by adding the line:
+#
+#Alias /error/include/ "/your/include/path/"
+#
+# which allows you to create your own set of files by starting with the
+# /usr/share/apache2/error/include/ files and copying them to /your/include/path/,
+# even on a per-VirtualHost basis.  If you include the Alias in the global server
+# context, is has to come _before_ the 'Alias /error/ ...' line.
+#
+# The default include files will display your Apache version number and your
+# ServerAdmin email address regardless of the setting of ServerSignature.
+#
+# WARNING: The configuration below will NOT work out of the box if you have a
+#		  SetHandler directive in a <Location /> context somewhere. Adding
+#		  the following three lines AFTER the <Location /> context should
+#		  make it work in most cases:
+#		  <Location /error/>
+#			 SetHandler none
+#		  </Location>
+#
+# The internationalized error documents require mod_alias, mod_include
+# and mod_negotiation.  To activate them, uncomment the following 37 lines.
+
+#<IfModule mod_negotiation.c>
+#	<IfModule mod_include.c>
+#		<IfModule mod_alias.c>
+#
+#			Alias /error/ "/usr/share/apache2/error/"
+#
+#			<Directory "/usr/share/apache2/error">
+#				Options IncludesNoExec
+#				AddOutputFilter Includes html
+#				AddHandler type-map var
+#				Order allow,deny
+#				Allow from all
+#				LanguagePriority en cs de es fr it nl sv pt-br ro
+#				ForceLanguagePriority Prefer Fallback
+#			</Directory>
+#
+#			ErrorDocument 400 /error/HTTP_BAD_REQUEST.html.var
+#			ErrorDocument 401 /error/HTTP_UNAUTHORIZED.html.var
+#			ErrorDocument 403 /error/HTTP_FORBIDDEN.html.var
+#			ErrorDocument 404 /error/HTTP_NOT_FOUND.html.var
+#			ErrorDocument 405 /error/HTTP_METHOD_NOT_ALLOWED.html.var
+#			ErrorDocument 408 /error/HTTP_REQUEST_TIME_OUT.html.var
+#			ErrorDocument 410 /error/HTTP_GONE.html.var
+#			ErrorDocument 411 /error/HTTP_LENGTH_REQUIRED.html.var
+#			ErrorDocument 412 /error/HTTP_PRECONDITION_FAILED.html.var
+#			ErrorDocument 413 /error/HTTP_REQUEST_ENTITY_TOO_LARGE.html.var
+#			ErrorDocument 414 /error/HTTP_REQUEST_URI_TOO_LARGE.html.var
+#			ErrorDocument 415 /error/HTTP_UNSUPPORTED_MEDIA_TYPE.html.var
+#			ErrorDocument 500 /error/HTTP_INTERNAL_SERVER_ERROR.html.var
+#			ErrorDocument 501 /error/HTTP_NOT_IMPLEMENTED.html.var
+#			ErrorDocument 502 /error/HTTP_BAD_GATEWAY.html.var
+#			ErrorDocument 503 /error/HTTP_SERVICE_UNAVAILABLE.html.var
+#			ErrorDocument 506 /error/HTTP_VARIANT_ALSO_VARIES.html.var
+#		</IfModule>
+#	</IfModule>
+#</IfModule>
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

config_infra/apache2_ubuntu/infra/conf-available/localized-error-pages.conf

@@ -0,0 +1,4 @@
+# Define an access log for VirtualHosts that don't define their own logfile
+CustomLog log/other_vhosts_access.log vhost_combined
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet