Merge pull request #1690 from CactuseSecurity/develop

Develop new stable release 5.6.9

Author: tpurschke

.vscode/launch.json

@@ -50,12 +50,13 @@
                 "PYTHONPATH": "${PYTHONPATH}:${workspaceRoot}"
             },
             "args": [
-                "-m55",
-                "-d0",
+                "-m8",
+                "-d4",
                 "-f",
+                //"-ihttps://fwodemodata.cactus.de/demo04_cpr8x.json",
                 //"-c",
                 //  41 - lab fortimanager
-                //"-i/home/tim/tmp/configs/mgm_id_53_config_native.json.8.anon",
+                //"-i/tmp/mgm_id_8_config_native.json",
                 //"-i/home/tim/tmp/configs/mgm_id_56_config_native.json.anon", // all rules, all objs local mgm_id 4
                 // "-i/home/tim/tmp/configs/mgm_id_56_config_native.json.anon", // 30k rules, local mgm_id 45
                 //"-i/home/tim/tmp/configs/mgm_id_52_config_native.json.anon", // many gateways, local mgm_id 49

ansible.cfg

@@ -15,4 +15,6 @@ fact_caching = jsonfile
 fact_caching_timeout = 7200
 
 [ssh_connection]
-pipelining = True
+# from ansible 2.10 onwards, pipelining does not work anymore
+# instead the ansible playbook must be run via sudo
+pipelining = False

documentation/auth/README.md

@@ -183,9 +183,6 @@ insert into hdb_catalog.hdb_permission (table_schema, table_name, role_name, per
         "ssh_port",
         "ssh_user",
         "last_import_md5_complete_config",
-        "last_import_md5_rules",
-        "last_import_md5_objects",
-        "last_import_md5_users",
         "do_not_import",
         "clearing_import_ran",
         "force_initial_import",

documentation/database/database-docu.md

@@ -38,3 +38,14 @@ FROM pg_stat_activity
 WHERE pg_stat_activity.datname = '[Database to copy]'
 AND pid <> pg_backend_pid();
 ```
+
+### quick and dirty debugging
+
+```
+...$ sudo su - postgres
+...$ psql -U postgres
+postgres-# \c fworchdb
+fworchdb-# \dt
+fworchdb-# \x
+fworchdb-# SELECT * FROM my_table;
+```

documentation/installer/install-advanced.md

@@ -9,14 +9,14 @@ always change into the firewwall-orchestrator directory before starting the inst
 The following switch can be used to set the type of installation to perform
 
 ```console
-ansible-playbook -e "installation_mode=upgrade" site.yml -K
+sudo ansible-playbook -e "installation_mode=upgrade" site.yml -K
 ```
 
 If you want to drop the database and re-install from scratch, do the following:
 
 ```console
-ansible-playbook -e "installation_mode=uninstall" site.yml -K
-ansible-playbook -e "installation_mode=new" site.yml -K
+sudo ansible-playbook -e "installation_mode=uninstall" site.yml -K
+sudo ansible-playbook -e "installation_mode=new" site.yml -K
 ```
 
 installation_mode options:
@@ -30,7 +30,7 @@ installation_mode options:
 e.g. with IP 1.2.3.4, listening on port 3128<br>
 
 ```console
-ansible-playbook -e "http_proxy=http://1.2.3.4:3128 https_proxy=http://1.2.3.4:3128" site.yml -K
+sudo ansible-playbook -e "http_proxy=http://1.2.3.4:3128 https_proxy=http://1.2.3.4:3128" site.yml -K
 ```
 
 use the following syntax for authenticated proxy access:
@@ -60,7 +60,7 @@ NB: for vscode-debugging, you also need access to
 e.g. if your hasura metadata file needs to be re-created from scratch, then use the following switch:
 
 ```console
-ansible-playbook -e "api_no_metadata=yes" site.yml -K
+sudo ansible-playbook -e "api_no_metadata=yes" site.yml -K
 ```
 
 ### Parameter "install_syslog" allows disabling of separate syslog installation
@@ -69,7 +69,7 @@ Default value is install_syslog=yes but if you already have a syslog service run
 
 run installation without syslog installation:
 ```console
-ansible-playbook -e "install_syslog=no" site.yml -K
+sudo ansible-playbook -e "install_syslog=no" site.yml -K
 ```
 
 Here is a sample config you can use for configuring your already running syslog:
@@ -130,12 +130,11 @@ rsyslog config
 ### Parameter "api_docu" to install API documentation
 
 Generating a full hasura (all tables, etc. tracked) API documentation  currently requires
-- 2.3 GB additional hdd space (at least 10 GB total for test install)
+- at least 10 GB total free hdd for test install
 - a minimum of 8 GB RAM
-- 4 minutes to generate
 
 ```console
-cd firewall-orchestrator; ansible-playbook -e "api_docu=yes" site.yml -K
+sudo ansible-playbook -e "api_docu=yes" site.yml -K
 ```
 
 api docu can then be accessed at <https://server/api_schema/index.html>
@@ -150,7 +149,7 @@ The following options exist for communication to the UI:
 
 Example:
 ```console
-cd firewall-orchestrator; ansible-playbook -e "ui_comm_mode=no_ws" site.yml -K
+sudo ansible-playbook -e "ui_comm_mode=no_ws" site.yml -K
 ```
 
 ## User interface server name and aliases
@@ -159,11 +158,11 @@ To make sure that firewall orchestrator UI webserver responds to the correct DNS
 
 Example to set fwodemo.cactus.de as webserver name:
 ```console
-cd firewall-orchestrator; ansible-playbook -e "ui_server_name='fwodemo.cactus.de'" site.yml -K
+sudo ansible-playbook -e "ui_server_name='fwodemo.cactus.de'" site.yml -K
 ```
 Example to set fwodemo.cactus.de and two additional aliases as websrver names:
 ```console
-cd firewall-orchestrator; ansible-playbook -e "ui_server_name=fwodemo.cactus.de ui_server_alias=' fwo1.cactus.de fwo2.cactus.de'" site.yml -K
+sudo ansible-playbook -e "ui_server_name=fwodemo.cactus.de ui_server_alias=' fwo1.cactus.de fwo2.cactus.de'" site.yml -K
 ```
 
 ## User interface Server Alias string
@@ -172,11 +171,11 @@ To be able to configure your webserver name, you may add the following parameter
 
 Example to set fwodemo.cactus.de as websrver name:
 ```console
-cd firewall-orchestrator; ansible-playbook -e "ui_server_alias='fwodemo.cactus.de'" site.yml -K
+sudo ansible-playbook -e "ui_server_alias='fwodemo.cactus.de'" site.yml -K
 ```
 Example to set fwodemo.cactus.de and fwo2.cactus.de as websrver names:
 ```console
-cd firewall-orchestrator; ansible-playbook -e "ui_server_alias='fwodemo.cactus.de fwo2.cactus.de'" site.yml -K
+sudo ansible-playbook -e "ui_server_alias='fwodemo.cactus.de fwo2.cactus.de'" site.yml -K
 ```
 
 ## Distributed setup with multiple servers

documentation/installer/install-for-testing.md

@@ -12,7 +12,7 @@ This includes:
 Note: the relevant secrets are displayed at the very end of the installation. They can also be found in the etc/secrets directory.
 
 ```console
-ansible-playbook/ site.yml -e "testkeys=yes" -K
+sudo ansible-playbook/ site.yml -e "testkeys=yes" -K
 ```
 
 A static jwt key helps with debugging c# code in visual studio (code) - you can use a static backend (ldap & api) with these keys.
@@ -32,38 +32,38 @@ You need to
 Set debug level for extended debugging info during installation.
 
 ```console
-ansible-playbook/ site.yml -e "debug_level='2'" -K
+sudo ansible-playbook/ site.yml -e "debug_level='2'" -K
 ```
 ## Running integration tests after installation/upgrade
 
 To only run tests (for an existing installation) use tags as follows:
 
 ```console
-ansible-playbook/ site.yml --tags test -K
+sudo ansible-playbook/ site.yml --tags test -K
 ```
 
 ## Running unit tests only
 
 To only run tests (for an existing installation, can only be combined with installation_mode=upgrade) use tags as follows:
 
 ```console
-ansible-playbook site.yml --tags unittest -e "installation_mode=upgrade" -K
+sudo ansible-playbook site.yml --tags unittest -e "installation_mode=upgrade" -K
 ```
 
 ## Parameter "api_no_metadata" to prevent meta data import
 
 e.g. if your hasura metadata file needs to be re-created from scratch, then use the following switch::
 
 ```console
-ansible-playbook -e "api_no_metadata=yes" site.yml -K
+sudo ansible-playbook -e "api_no_metadata=yes" site.yml -K
 ```
 
 ## Parameter "add_demo_data" to avoid creation of sample data (i.e. in production)
 
 The following command prevents the creation of sample data in the database:
 
 ```console
-ansible-playbook -e "add_demo_data=no" site.yml -K
+sudo ansible-playbook -e "add_demo_data=no" site.yml -K
 ```
 
 note: demo/sample data can also be removed via settings menues.
@@ -73,20 +73,20 @@ note: demo/sample data can also be removed via settings menues.
 if you want to install a second ldap database "dc=example,dc=com"
 
 ```console
-cd firewall-orchestrator; ansible-playbook -e "second_ldap_db=yes" site.yml -K
+sudo ansible-playbook -e "second_ldap_db=yes" site.yml -K
 ```
 
 ### Parameter "sample_data_rate" to ramp up sample data
 
 if you want to create sample-data changes every minute set sample_data_rate to high
 
 ```console
-cd firewall-orchestrator; ansible-playbook -e "sample_data_rate=high" site.yml -K
+sudo ansible-playbook -e "sample_data_rate=high" site.yml -K
 ```
 ### Parameter "audit_user" to add an audit user to ldap db - useful for demo installation
 
 if you want to have an extra read-only audit-user called e.g. auditor1, use the following command for installation:
 
 ```console
-cd firewall-orchestrator; ansible-playbook -e "audit_user=auditor1 auditor_initial_pwd=<pwd>" site.yml -K
+sudo ansible-playbook -e "audit_user=auditor1 auditor_initial_pwd=<pwd>" site.yml -K
 ```

documentation/installer/server-install.md

@@ -24,14 +24,12 @@ git clone https://github.com/CactuseSecurity/firewall-orchestrator.git
 3) Operating specific ansible adjustments
   - Ubuntu 18.04, Debian 10: install latest ansible before firewall orchestrator installation:
 
-        cd firewall-orchestrator; ansible-playbook scripts/install-latest-ansible.yml -K
-
-  - Debian 11: install without pipelining: comment out "pipelining = True" in ansible.cfg
+        cd firewall-orchestrator; sudo ansible-playbook scripts/install-latest-ansible.yml -K
 
 4) install (on localhost)
 
 ```console
-cd firewall-orchestrator; ansible-playbook site.yml -K
+cd firewall-orchestrator; sudo ansible-playbook site.yml -K
 ```
 Note: The installation (i.e. the connection to the target machine) is only done in root context (sudo) to secure the writing of temporary files when becoming a non-priviledged user (e.g. postgres).
 

documentation/installer/server-upgrade.md

@@ -5,5 +5,5 @@ it is really simple:
 ```console
   cd firewall-orchestrator
   git pull
-  ansible-playbook site.yml -K -e "installation_mode=upgrade"
+  sudo ansible-playbook site.yml -K -e "installation_mode=upgrade"
 ```

documentation/revision-history.md

@@ -211,3 +211,6 @@ adding report template format fk and permissions
 ### 5.6.8
 - no end ip address for obj types <> range
 - fixing range display in reporting
+
+### 5.6.9
+- handle import attempts

inventory/group_vars/all.yml

@@ -1,5 +1,5 @@
 ### general settings
-product_version: "5.6.8"
+product_version: "5.6.9"
 ansible_python_interpreter: /usr/bin/python3
 ansible_ssh_common_args: '-o StrictHostKeyChecking=no'
 product_name: fworch
@@ -115,4 +115,5 @@ http_conf_dir: /etc/{{ webserver_package_name }}/sites-available/
 ################# testing #########################
 test_dir: "{{ fworch_home }}/test"
 test_fortigate_name: "fortigate{{ test_postfix }}"
+test_checkpoint_name: "checkpoint{{ test_postfix }}"
 csharp_test_start_dir: "{{ fworch_home }}/test/csharp/FWO.Test"