Merge branch 'feature/improved_findings_search' of https://github.com/red-kite-solutions/stalker into feature/319_refactor-seach-suggestions

Author: Aboisier

.github/workflows/main.yml

@@ -16,7 +16,7 @@ jobs:
   test:
     name: E2E Tests
     # The type of runner that the job will run on
-    runs-on: ubuntu-latest
+    runs-on: Linux
     timeout-minutes: 30
 
     # Steps represent a sequence of tasks that will be executed as part of the job
@@ -37,17 +37,23 @@ jobs:
           kubectl-install: true
 
       - name: Create namespace
-        run: kubectl create namespace stalker
-
-      - name: Make room in the container
+        run: kubectl create namespace stalker --dry-run=client -o yaml | kubectl apply -f -
+
+      # Deactivated while on self-hosted runner
+      # - name: Make room in the container
+      #   run: |-
+      #     rm -rf /opt/hostedtoolcache/CodeQL
+      #     rm -rf /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk
+      #     rm -rf /opt/hostedtoolcache/PyPy
+      #     rm -rf /opt/hostedtoolcache/Python
+      #     rm -rf /opt/hostedtoolcache/Ruby
+      #     rm -rf /opt/hostedtoolcache/go
+      #     rm -rf /opt/hostedtoolcache/node
+      - name: Create certificates
         run: |-
-          rm -rf /opt/hostedtoolcache/CodeQL
-          rm -rf /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk
-          rm -rf /opt/hostedtoolcache/PyPy
-          rm -rf /opt/hostedtoolcache/Python
-          rm -rf /opt/hostedtoolcache/Ruby
-          rm -rf /opt/hostedtoolcache/go
-          rm -rf /opt/hostedtoolcache/node
+          bash setup_mongo_test.sh
+          bash setup_kafka_test.sh
+          bash setup_secrets_test.sh
 
       - name: Build test containers
         run: devspace -p e2e --var="RK_ENVIRONMENT=tests" run-pipeline build -n stalker

devspace.tests.yaml

@@ -503,6 +503,9 @@ deployments:
         name: kafka
         repo: https://charts.bitnami.com/bitnami
         version: "28.0.0"
+      upgradeArgs:
+        - --timeout
+        - 1200s
       values:
         controller:
           nodeSelector: $(echo ${KAFKA_K8S_NODE_SELECTOR})
@@ -553,6 +556,7 @@ deployments:
             value: "no"
         provisioning:
           enabled: true
+          parallel: 1
           nodeSelector: $(echo ${KAFKA_K8S_NODE_SELECTOR})
           auth:
             tls:
@@ -587,15 +591,23 @@ deployments:
               config:
                 max.message.bytes: 64000
                 flush.messages: 1
+            - name: stalker.jobs.management
+              partitions: 1
+              replicationFactor: 1
+              config:
+                max.message.bytes: 64000
+                flush.messages: 1
           extraProvisioningCommands: # Setting the ACLs
             - "/opt/bitnami/kafka/bin/kafka-acls.sh --bootstrap-server $KAFKA_SERVICE --command-config $CLIENT_CONF --add --allow-principal User:jobs-manager --operation read --operation write --operation describe --topic stalker.jobs.findings"
             - "/opt/bitnami/kafka/bin/kafka-acls.sh --bootstrap-server $KAFKA_SERVICE --command-config $CLIENT_CONF --add --allow-principal User:jobs-manager --operation write --operation describe --topic stalker.jobs.requests"
             - "/opt/bitnami/kafka/bin/kafka-acls.sh --bootstrap-server $KAFKA_SERVICE --command-config $CLIENT_CONF --add --allow-principal User:jobs-manager --operation read --operation describe --topic stalker.jobs.logs"
             - "/opt/bitnami/kafka/bin/kafka-acls.sh --bootstrap-server $KAFKA_SERVICE --command-config $CLIENT_CONF --add --allow-principal User:jobs-manager --operation read --operation write --operation describe --topic stalker.jobs.models"
+            - "/opt/bitnami/kafka/bin/kafka-acls.sh --bootstrap-server $KAFKA_SERVICE --command-config $CLIENT_CONF --add --allow-principal User:jobs-manager --operation read --operation write --operation describe --topic stalker.jobs.management"
             - "/opt/bitnami/kafka/bin/kafka-acls.sh --bootstrap-server $KAFKA_SERVICE --command-config $CLIENT_CONF --add --allow-principal User:orchestrator --operation write --operation describe --topic stalker.jobs.findings"
             - "/opt/bitnami/kafka/bin/kafka-acls.sh --bootstrap-server $KAFKA_SERVICE --command-config $CLIENT_CONF --add --allow-principal User:orchestrator --operation read --operation describe --topic stalker.jobs.requests"
             - "/opt/bitnami/kafka/bin/kafka-acls.sh --bootstrap-server $KAFKA_SERVICE --command-config $CLIENT_CONF --add --allow-principal User:orchestrator --operation write --operation describe --topic stalker.jobs.logs"
             - "/opt/bitnami/kafka/bin/kafka-acls.sh --bootstrap-server $KAFKA_SERVICE --command-config $CLIENT_CONF --add --allow-principal User:orchestrator --operation read --operation describe --topic stalker.jobs.models"
+            - "/opt/bitnami/kafka/bin/kafka-acls.sh --bootstrap-server $KAFKA_SERVICE --command-config $CLIENT_CONF --add --allow-principal User:orchestrator --operation read --operation describe --topic stalker.jobs.management"
             - "/opt/bitnami/kafka/bin/kafka-acls.sh --bootstrap-server $KAFKA_SERVICE --command-config $CLIENT_CONF --add --allow-principal User:jobs-manager --operation read --operation describe --group jobs-manager"
             - "/opt/bitnami/kafka/bin/kafka-acls.sh --bootstrap-server $KAFKA_SERVICE --command-config $CLIENT_CONF --add --allow-principal User:jobs-manager --operation read --operation describe --group jobs-manager-job-logs"
             - "/opt/bitnami/kafka/bin/kafka-acls.sh --bootstrap-server $KAFKA_SERVICE --command-config $CLIENT_CONF --add --allow-principal User:orchestrator --operation read --operation describe --group stalker"

devspace.yaml

@@ -63,6 +63,23 @@ log_finding(
 )
 ```
 
+### Attaching information to a domain
+
+Adding fields and using a custom finding type will add a finding to the domain resource.
+
+```python
+from stalker_job_sdk import DomainFinding, log_finding, TextField
+
+hostname = "example.com"
+log_finding(
+    DomainFinding(
+        "MyCustomHostnameFinding", hostname, None, "Domain info", [
+          TextField("myfield", "Field Title", "Finding data")
+        ]
+    )
+)
+```
+
 ## IpFinding
 
 An ip finding creates a new host. IP addresses are in the IPv4 format.
@@ -93,9 +110,26 @@ log_finding(
 )
 ```
 
+### Attaching information to a host
+
+Adding fields and using a custom finding type will add a finding to the host resource.
+
+```python
+from stalker_job_sdk import IpFinding, log_finding, TextField
+ip = "0.0.0.0"
+mask = 16
+log_finding(
+    IpFinding(
+      "MyCustomIpFinding", ip, "New Info", [
+          TextField("myfield", "Field Title", "Finding data")
+        ]
+    )
+)
+```
+
 ## IpRangeFinding
 
-An ip range finding creates a new ip range for a project. IP addresses are in the IPv4 format.
+An ip range finding creates a new ip range for a project. IP addresses are in the IPv4 format. `IpRangeFindings` can also be used to attach information to an IP Range resource.
 
 | Field  | Type   | Description                                                                                              |
 | ------ | ------ | -------------------------------------------------------------------------------------------------------- |
@@ -116,19 +150,32 @@ Example:
 Using the python SDK, you can emit this finding with the following code:
 
 ```python
-from stalker_job_sdk import IpFinding, log_finding
+from stalker_job_sdk import IpRangeFinding, log_finding
 ip = "0.0.0.0"
 mask = 16
 log_finding(
     IpRangeFinding(
-        ip, mask
+        'IpRangeFinding', ip, mask, None, [], "IpRangeFinding"
     )
 )
 ```
 
-> You can't attach fields to an IP range as they are different than other ressources.
+### Attaching information to an IP range
 
-Which is equivalent to the following python code, but with more metadata:
+Adding fields and using a custom finding type will add a finding to the IP range resource.
+
+```python
+from stalker_job_sdk import IpRangeFinding, log_finding, TextField
+ip = "0.0.0.0"
+mask = 16
+log_finding(
+    IpRangeFinding(
+        'IpRangeFinding', ip, mask, "Finding title", [
+            TextField("myfield", "Field Title", "Finding data")
+        ]
+    )
+)
+```
 
 ## HostnameIpFinding
 
@@ -192,7 +239,7 @@ Example:
 Using the python SDK, you can emit this finding with the following code:
 
 ```python
-from stalker_job_sdk import PortFinding, log_finding
+from stalker_job_sdk import PortFinding, log_finding, TextField
 port = 80
 ip = "1.2.3.4"
 log_finding(
@@ -208,6 +255,29 @@ log_finding(
 )
 ```
 
+### Attaching information to a port
+
+Adding fields and using a custom finding type will add a finding to the port resource.
+
+```python
+from stalker_job_sdk import PortFinding, log_finding, TextField
+port = 80
+ip = "1.2.3.4"
+log_finding(
+    PortFinding(
+        "MyCustomPortFinding",
+        ip,
+        port,
+        "tcp",
+        "New port data",
+        [
+          TextField("protocol", "This is a TCP port", "tcp"), 
+          TextField("myfield", "Field Title", "Finding data")
+        ],
+    )
+)
+```
+
 ## WebsiteFinding
 
 The `WebsiteFinding` will create a website resource. Websites are made from 4 characteristics: an IP address, a domain name, a port number
@@ -269,57 +339,74 @@ log_finding(
 )
 ```
 
+### Attaching information to a website
+
+Adding fields and using a custom finding type will add a finding to the website resource.
+
+```python
+from stalker_job_sdk import WebsiteFinding, log_finding, TextField
+port = 80
+ip = "1.2.3.4"
+domain = "example.com"
+path = "/"
+ssl = False
+
+log_finding(
+    WebsiteFinding(
+        "MyCustomWebsiteFinding",
+        ip,
+        port,
+        domain,
+        path,
+        ssl,
+        "New website data",
+        [
+          TextField("myfield", "Field Title", "Finding data")
+        ],
+    )
+)
+```
+
 ## CustomFinding
 
-Dynamic findings allow jobs to attach custom data to resources.
+Custom findings attach finding field information to a resource. There are custom findings for every type of resources. When you do not specify the _type of finding_ that you are logging, you are creating a custom finding for the associated resource type.
 
-| Field        | Description                                                     |
-| ------------ | --------------------------------------------------------------- |
-| `domainName` | The domain to which to attach the custom finding                |
-| `host`       | The host to which to attach the custom finding                  |
-| `port`       | The port to which to attach the custom finding                  |
-| `fields`     | A list of [fields](#dynamic-fields) containing the finding data |
+| SDK finding class | Resources |
+| ----------------- | --------- |
+| HostnameFinding   | Domains   |
+| IpFinding         | Hosts     |
+| IpRangeFinding    | IP ranges |
+| PortFinding       | Ports     |
+| WebsiteFinding    | Websites  |
 
-Examples:
 
-```json
-{
-  "type": "CustomFinding",
-  "host": "1.2.3.4",
-  "port": 80,
-  "fields": [
-    {
-      "type": "image",
-      "data": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
-    }
-  ]
-}
-```
+Here is an example of a **custom finding** for a port with the python SDK. In this example, the port will show the custom information _This port
+runs an HTTP server_, with a text field attached to it:
 
-```json
-{
-  "type": "CustomFinding",
-  "domainName": "red-kite.io",
-  "fields": [
-    {
-      "type": "text",
-      "label": "Domain greatness level",
-      "data": "This domain is great, would recommend"
-    }
-  ]
-}
+```python
+from stalker_job_sdk import PortFinding, log_finding, TextField
+port = 80
+ip = "0.0.0.0"
+log_finding(
+    PortFinding(
+        "PortFunFact", ip, port, "tcp", "This is a fun fact about a port", [
+          TextField('myfieldkey', 'My field title', 'My field data')
+        ]
+    )
+)
 ```
 
-Here is an example of a custom finding for a port with the python SDK. In this example, the port will show the custom information _This port
-runs an HTTP server_:
+Notice how the key `PortFunFact` can be anything, how information is provided through `TextField`s and how the finding type is not provided to use the default value.
+
+To compare, here is an example of how to create a port with the `PortFinding` class, which here is **not** used as a custom finding. You will see that the key is `PortFinding`, no fields are provided, and the type is `PortFinding` as well:
 
 ```python
 from stalker_job_sdk import PortFinding, log_finding
 port = 80
 ip = "0.0.0.0"
 log_finding(
     PortFinding(
-        "PortFunFact", ip, port, "tcp", "This is a fun fact about a port"
+        "PortFinding", ip, port, "tcp", None, None, "PortFinding"
     )
 )
 ```

docs/docs/concepts/findings.md

@@ -6,13 +6,9 @@ description: Organizing your data through projects
 
 # Projects
 
-Projects are the way to organize and centralize resources. Resources are unique per project, and deleting the project will delete all its
+Projects are the way to organize and centralize [resources](./resources.md). Resources are unique per project, and deleting the project will delete all its
 resources and related information.
 
 The project's name must be unique, but it can always be changed. You can also add a logo for display purposes, but it is not mandatory.
 
-## Subnets
-
-In the case where a target owns a public subnet, you can add the different subnets in the projects page.
-
-For a subnet of `127.0.0.1/24`, you would simply add `127.0.0.1` in the _IP Address_ field, and the `/24` in the _Short Mask_ field.
+> Red Kite allows to work on multiple projects at once, but using the global project filter in the navigation bar, you can also focus your work on a single project at a time. It will pre-filter data in displays such as tables and metrics.