[Feature] Google Workspace API Support

Author: 8naama

.github/workflows/tests.yaml

@@ -19,6 +19,7 @@ jobs:
         run: PYTHONPATH=. pytest --cov-report term-missing --cov=src tests/UnitTests/test_*.py
 
   e2e-tests:
+    needs: unit-tests  # Run e2e test only if unit tests passed
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -43,4 +44,10 @@ jobs:
           AZURE_AD_TENANT_ID: ${{ secrets.AZURE_AD_TENANT_ID }}
           AZURE_AD_CLIENT_ID: ${{ secrets.AZURE_AD_CLIENT_ID }}
           AZURE_AD_SECRET_VALUE: ${{ secrets.AZURE_AD_SECRET_VALUE }}
+          GOOGLE_PRIVATE_KEY_ID: ${{ secrets.GOOGLE_PRIVATE_KEY_ID }}
+          GOOGLE_PRIVATE_KEY: ${{ secrets.GOOGLE_PRIVATE_KEY }}
+          GOOGLE_CLIENT_EMAIL: ${{ secrets.GOOGLE_CLIENT_EMAIL }}
+          GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }}
+          GOOGLE_CERT_URL: ${{ secrets.GOOGLE_CERT_URL }}
+          GOOGLE_DELEGATED_ACCOUNT: ${{ secrets.GOOGLE_DELEGATED_ACCOUNT }}
         run: python -m unittest

.gitignore

@@ -0,0 +1,5 @@
+.idea
+.venv
+__pycache__
+.coverage
+.DS_Store

README.md

@@ -254,6 +254,49 @@ For dockerhub audit logs, use type `dockerhub` with the below parameters.
 | additional_fields      | Additional custom fields to add to the logs before sending to logzio                  | Optional          | -                 |
 
 
+</details>
+<details>
+  <summary>
+    <span><a href="./src/apis/google/README.md#google-workspace-activities">Google Workspace Activity</a></span>
+  </summary>
+
+For Google Workspace Activity, use type `google_activity` with the below parameters.
+
+
+## Configuration Options
+| Parameter Name              | Description                                                                                                                                                                                                                                         | Required/Optional | Default                                 |
+|-----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|-----------------------------------------|
+| name                        | Name of the API (custom name)                                                                                                                                                                                                                       | Optional          | `Google Workspace`                      |
+| google_ws_sa_file_name      | The name of the service account credentials file. **Required unless** `google_ws_sa_file_path` is set.                                                                                                                                              | Required*         | `""`                                    |
+| google_ws_sa_file_path      | The path to the service account credentials file. **Required unless** `google_ws_sa_file_name` is set. Use this if mounting the file to a different path than the default.                                                                          | Optional*         | `./src/shared/<google_ws_sa_file_name>` |
+| google_ws_delegated_account | The email of the user for which the application is requesting delegated access                                                                                                                                                                      | Required          | -                                       |
+| application_name            | Specifies the [Google Workspace application](https://developers.google.com/workspace/admin/reports/reference/rest/v1/activities/list#applicationname) to fetch activity data from (e.g., `saml`, `user_accounts`, `login`, `admin`, `groups`, etc). | Required          | -                                       |
+| user_key                    | The unique ID of the user to fetch activity data for                                                                                                                                                                                                | Optional          | `all`                                   |
+| additional_fields           | Additional custom fields to add to the logs before sending to logzio                                                                                                                                                                                | Optional          | -                                       |
+| days_back_fetch             | The amount of days to fetch back in the first request                                                                                                                                                                                               | Optional          | 1 (day)                                 |
+| scrape_interval             | Time interval to wait between runs (unit: `minutes`)                                                                                                                                                                                                | Optional          | 1 (minute)                              |
+
+
+</details>
+<details>
+  <summary>
+    <span><a href="./src/apis/google/README.md#google-workspace-general">Google Workspace General API</a></span>
+  </summary>
+
+For structuring custom general Google Workspace API calls use type `google_workspace` API with the parameters below.
+
+| Parameter Name              | Description                                                                                                                                                                | Required/Optional | Default                                                            |
+|-----------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|--------------------------------------------------------------------|
+| name                        | Name of the API (custom name)                                                                                                                                              | Optional          | `Google Workspace`                                                 |
+| google_ws_sa_file_name      | The name of the service account credentials file. **Required unless** `google_ws_sa_file_path` is set.                                                                     | Required*         | `""`                                                               |
+| google_ws_sa_file_path      | The path to the service account credentials file. **Required unless** `google_ws_sa_file_name` is set. Use this if mounting the file to a different path than the default. | Optional*         | `./src/shared/<google_ws_sa_file_name>`                            |
+| google_ws_delegated_account | The email of the user for which the application is requesting delegated access                                                                                             | Required          | -                                                                  |
+| scopes                      | The OAuth 2.0 scopes that you might need to request to access Google APIs                                                                                                  | Optional          | `["https://www.googleapis.com/auth/admin.reports.audit.readonly"]` |
+| data_request                | Nest here any detail relevant to the data request. (Options in [General API](../general/README.md))                                                                        | Required          | -                                                                  |
+| additional_fields           | Additional custom fields to add to the logs before sending to logzio                                                                                                       | Optional          | -                                                                  |
+| days_back_fetch             | The amount of days to fetch back in the first request                                                                                                                      | Optional          | 1 (day)                                                            |
+| scrape_interval             | Time interval to wait between runs (unit: `minutes`)                                                                                                                       | Optional          | 1 (minute)                                                         |
+
 </details>
 
 
@@ -312,6 +355,11 @@ docker stop -t 30 logzio-api-fetcher
 ```
 
 ## Changelog:
+- **2.0.0**:
+  - Add Google Workspace Support
+  - Add option to configure multiple Logz.io outputs.
+  - Bug fix for Cloudflare `next_url` to be optional.
+  - **Note:** No breaking changes, major version bump is made to align with semantic versioning in future releases.
 - **0.3.1**:
   - Handle internal logger configuration in memory instead of disk
 - **0.3.0**:

e2e_tests/apis/test_e2e_google.py

@@ -0,0 +1,43 @@
+import json
+import os
+import time
+import unittest
+from os.path import abspath, dirname
+
+from e2e_tests.api_e2e_test import ApiE2ETest
+from e2e_tests.utils.config_utils import update_json_tokens
+
+
+class TestGoogleE2E(ApiE2ETest):
+    def module_specific_setup(self):
+        secrets_map = {
+            "private_key_id": "GOOGLE_PRIVATE_KEY_ID",
+            "private_key": "GOOGLE_PRIVATE_KEY",
+            "client_email": "GOOGLE_CLIENT_EMAIL",
+            "client_id": "GOOGLE_CLIENT_ID",
+            "client_x509_cert_url": "GOOGLE_CERT_URL",
+        }
+        curr_path = abspath(dirname(__file__))
+        creds_path = f"{curr_path}/testdata/google_sa_creds.json"
+        creds_path_temp = f"{curr_path}/testdata/google_sa_creds_temp.json"
+        os.environ["GOOGLE_SA_CREDS"] = creds_path_temp
+        os.environ["GOOGLE_PRIVATE_KEY"] = os.getenv("GOOGLE_PRIVATE_KEY").encode().decode("unicode_escape")
+        update_json_tokens(creds_path, secrets_map)
+
+    def test_google_data_in_logz(self):
+        secrets_map = {
+            "apis.0.google_ws_delegated_account": "GOOGLE_DELEGATED_ACCOUNT",
+            "logzio.token": "LOGZIO_SHIPPING_TOKEN",
+            "apis.0.additional_fields.type": "TEST_TYPE",
+            "apis.0.google_ws_sa_file_path": "GOOGLE_SA_CREDS"
+        }
+        curr_path = abspath(dirname(__file__))
+        config_path = f"{curr_path}/testdata/google_api_conf.yaml"
+        self.run_main_program(config_path=config_path, secrets_map=secrets_map)
+        time.sleep(120)
+        logs = self.search_logs(f"type:{self.test_type}")
+        self.assertTrue(logs)
+
+
+if __name__ == '__main__':
+    unittest.main()

e2e_tests/apis/testdata/google_api_conf.yaml

@@ -0,0 +1,14 @@
+apis:
+  - name: google login
+    type: google_activity
+    google_ws_sa_file_path: ./google_sa_creds_temp.json
+    google_ws_delegated_account: [email protected]
+    application_name: login
+    additional_fields:
+      type: google_activity_shipping_test
+    days_back_fetch: 7
+    scrape_interval: 5
+
+logzio:
+  url: https://listener.logz.io:8071
+  token: token

e2e_tests/apis/testdata/google_sa_creds.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "core-998",
+  "private_key_id": "<<GOOGLE_PRIVATE_KEY_ID>>",
+  "private_key": "<<GOOGLE_PRIVATE_KEY>>",
+  "client_email": "<<GOOGLE_CLIENT_EMAIL>>",
+  "client_id": "<<GOOGLE_CLIENT_ID>>",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "<<GOOGLE_CERT_URL>>",
+  "universe_domain": "googleapis.com"
+}

e2e_tests/utils/config_utils.py

@@ -1,3 +1,4 @@
+import json
 import os
 import yaml
 import glob
@@ -13,17 +14,13 @@ def validate_config_tokens(secrets_map):
         if os.getenv(env_var) is None:
             raise EnvironmentError(f"{env_var} environment variable is missing")
 
-
-def update_config_tokens(file_path, secrets_map):
+def _replace_key_with_env_var(secrets_map, content):
     """
-    Updates the tokens in the given file based on the provided token updates.
-    :param file_path: Path to the configuration file.
-    :param secrets_map: Dictionary of token updates.
-    :return: Path to the temporary configuration file.
+    Replaces the keys in the content with the corresponding environment variable values.
+    :param secrets_map: Dictionary of keys and corresponding environment variable.
+    :param content: The content to be updated.
+    :return: Updated content.
     """
-    with open(file_path, "r") as conf:
-        content = yaml.safe_load(conf)
-
     for key, env_var in secrets_map.items():
         value = os.getenv(env_var)
         print(f"Updating {key} with {env_var}={value}")
@@ -41,6 +38,19 @@ def update_config_tokens(file_path, secrets_map):
             d[int(keys[-1])] = value
         else:
             d[keys[-1]] = value
+    return content
+
+def update_config_tokens(file_path, secrets_map):
+    """
+    Updates the tokens in the given yaml file based on the provided token updates.
+    :param file_path: Path to the configuration file.
+    :param secrets_map: Dictionary of token updates.
+    :return: Path to the temporary configuration file.
+    """
+    with open(file_path, "r") as conf:
+        content = yaml.safe_load(conf)
+
+    content = _replace_key_with_env_var(secrets_map, content)
 
     path, ext = file_path.rsplit(".", 1)
     temp_test_path = f"{path}_temp.{ext}"
@@ -50,6 +60,25 @@ def update_config_tokens(file_path, secrets_map):
 
     return temp_test_path
 
+def update_json_tokens(file_path, secrets_map):
+    """
+    Updates the tokens in the given json file based on the provided token updates.
+    :param file_path: Path to the configuration file.
+    :param secrets_map: Dictionary of token updates.
+    :return: Path to the temporary configuration file.
+    """
+    with open(file_path, "r") as conf:
+        content = json.loads(conf.read())
+
+    content = _replace_key_with_env_var(secrets_map, content)
+
+    path, ext = file_path.rsplit(".", 1)
+    temp_test_path = f"{path}_temp.{ext}"
+
+    with open(temp_test_path, "w") as file:
+        json.dump(content, file)
+
+    return temp_test_path
 
 def delete_temp_files():
     """

requirements.txt

@@ -1,3 +1,5 @@
 pydantic~=2.10.6
 pyyaml~=6.0.2
 requests~=2.32.3
+google~=3.0.0
+google-auth~=2.38.0

src/apis/azure/AzureGraph.py

@@ -8,8 +8,8 @@
 from src.apis.general.StopPaginationSettings import StopPaginationSettings
 
 
-DATE_FROM_END_PATTERN = re.compile(r'\S+$')
-
+DATE_FROM_END_PATTERN = re.compile(r'(\S+)$')
+DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
 
 logger = logging.getLogger(__name__)
 
@@ -69,12 +69,6 @@ def send_request(self):
         data = super().send_request()
 
         # Add 1s to the time we took from the response to avoid duplicates
-        org_date = re.search(DATE_FROM_END_PATTERN, self.data_request.url).group(0)
-        try:
-            org_date = datetime.strptime(org_date, "%Y-%m-%dT%H:%M:%SZ")
-            org_date_plus_second = (org_date + timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%SZ")
-            self.data_request.url = self._replace_url_date(org_date_plus_second)
-        except ValueError:
-            logger.error(f"Failed to parse API {self.name} date in URL: {self.data_request.url}")
+        self.data_request.add_seconds_to_url_date_filter(1, DATE_FORMAT, DATE_FROM_END_PATTERN)
 
         return data

src/apis/cloudflare/Cloudflare.py

@@ -10,6 +10,7 @@
 
 DATE_FILTER_PARAMETER = "since="
 FIND_DATE_PATTERN = re.compile(r'since=(\S+?)(?:&|$)')
+DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
 
 logger = logging.getLogger(__name__)
 
@@ -61,21 +62,13 @@ def _initialize_url_date(self):
             self.url += f"?since={self._generate_start_fetch_date()}"
 
     def _generate_start_fetch_date(self):
-        return (datetime.now(UTC) - timedelta(days=self.days_back_fetch)).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+        return (datetime.now(UTC) - timedelta(days=self.days_back_fetch)).strftime(DATE_FORMAT)
 
     def send_request(self):
         data = super().send_request()
 
         # Add 1 second to a known date filter to avoid duplicates in the logs
         if DATE_FILTER_PARAMETER in self.url:
-            try:
-                org_date = re.search(FIND_DATE_PATTERN, self.url).group(1)
-                org_date_date = datetime.strptime(org_date, "%Y-%m-%dT%H:%M:%S.%fZ")
-                org_date_plus_second = (org_date_date + timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
-                self.url = self.url.replace(org_date, org_date_plus_second)
-            except IndexError:
-                logger.error(f"Failed to add 1s to the {self.name} api 'since' filter value, on url {self.url}")
-            except ValueError:
-                logger.error(f"Failed to parse API {self.name} date in URL: {self.url}")
+            self.add_seconds_to_url_date_filter(1, DATE_FORMAT, FIND_DATE_PATTERN)
 
         return data