diff --git a/external-import/google-ti-feeds/.env.sample b/external-import/google-ti-feeds/.env.sample new file mode 100644 index 0000000000..820a73cd21 --- /dev/null +++ b/external-import/google-ti-feeds/.env.sample @@ -0,0 +1,4 @@ +OPENCTI_URL='http://localhost:3000' +OPENCTI_TOKEN='d434ce02-e58e-4cac-8b4c-42bf16748e84' +CONNECTOR_ID='309d0e9b-5e80-4f8d-8371-731cd7ed567d' +GTI_API_KEY='your_gti_api_key' diff --git a/external-import/google-ti-feeds/CONTRIBUTING.md b/external-import/google-ti-feeds/CONTRIBUTING.md new file mode 100644 index 0000000000..00f2f133d8 --- /dev/null +++ b/external-import/google-ti-feeds/CONTRIBUTING.md @@ -0,0 +1,592 @@ +# Contributing to Google Threat Intelligence Feeds Connector + +Thank you for your interest in contributing to the Google Threat Intelligence (GTI) Feeds connector for OpenCTI. This document provides guidelines and explains the project structure to help you contribute effectively. + +## TL;DR: Contribution Checklist + +Use this checklist to guide your journey through implementing a new feature or entity type: + +- [ ] **Understand the Architecture** + - [ ] Review the connector flow diagram + - [ ] Understand the fetcher hierarchy and factory pattern + - [ ] Familiarize yourself with STIX 2.1 mapping concepts + +- [ ] **Implement the Data Model** + - [ ] Create a Pydantic model in `models/gti_reports/` + - [ ] Define proper validation and field types + - [ ] Add response models for API interactions + +- [ ] **Set Up Exception Handling** + - [ ] Create appropriate exception classes + - [ ] Update `__init__.py` files to expose the exceptions + - [ ] Follow the established exception hierarchy + +- [ ] **Configure the Fetcher** + - [ ] Add an `EntityFetcherConfig` in `fetchers/entity_config.py` + - [ ] Register the config in the `ENTITY_CONFIGS` dictionary + - [ ] Ensure endpoint and relationship types are correct + +- [ ] **Create the STIX Mapper** + - [ ] Implement a mapper class in `mappers/gti_reports/` + - [ ] Define the `to_stix()` method + - [ ] Map all relevant fields to STIX objects + +- [ ] **Update ConvertToSTIX** + - [ ] Add a conversion method for your entity type + - [ ] Update the main processing flow to include your entity + - [ ] Ensure relationships are properly established + +- [ ] **Testing** + - [ ] Create mock API responses + - [ ] Test fetching and processing logic + - [ ] Verify STIX conversion accuracy + - [ ] Check integration with other entity types + +- [ ] **Documentation** + - [ ] Update relevant documentation + - [ ] Add examples if necessary + +## Table of Contents +1. [Architecture Overview](#architecture-overview) +2. [Code Organization](#code-organization) +3. [Development Workflow](#development-workflow) +4. [Key Components](#key-components) +5. [Adding New Features](#adding-new-features) +6. [Best Practices](#best-practices) +7. [Error Handling](#error-handling) +8. [Testing](#testing) +9. [Complete Example: Adding Campaigns Support](#complete-example-adding-campaigns-support) + +## Architecture Overview + +The GTI Feeds connector follows a modular architecture designed to fetch, normalize, and ingest threat intelligence data from Google Threat Intelligence into OpenCTI. The connector uses asynchronous processing, handles pagination, implements comprehensive error handling, and converts data into STIX 2.1 format. + +```mermaid +graph TD + subgraph Configuration + DockerEnv[Environment Variables] + ConfigYml[config.yml] + GTIConfig[GTI Configuration] + end + + subgraph Connector Flow + Main("__main__.py") + Connector("Connector") + WorkManager("WorkManager") + FetchAll("FetchAll") + ConvertToSTIX("ConvertToSTIX") + ApiClient("ApiClient") + end + + subgraph Error Handling + ExceptionHandling["Exception Hierarchy"] + CircuitBreaker["Circuit Breaker"] + RetryStrategy["Retry Strategy"] + RateLimiter["Rate Limiter"] + end + + subgraph Data Models + GTIModels["GTI Pydantic Models"] + MapperClasses["STIX Mapper Classes"] + STIXModels["STIX Pydantic Models"] + end + + subgraph OpenCTI Integration + StateManagement["State Management"] + StixBundle["STIX Bundle"] + OpenCTIQueue["OpenCTI Queue"] + end + + DockerEnv --> Main + ConfigYml --> Main + Main --> GTIConfig + Main --> Connector + + Connector -- "1 - Initiates Work" --> WorkManager + WorkManager -- "2 - Starts Fetching" --> FetchAll + FetchAll -- "3 - Fetches Reports" --> ApiClient + FetchAll -- "4 - Fetches Related Entities" --> ApiClient + ApiClient -- "Uses" --> RetryStrategy + RetryStrategy -- "Uses" --> CircuitBreaker + RetryStrategy -- "Uses" --> RateLimiter + + FetchAll -- "5 - Returns Structured Data" --> GTIModels + WorkManager -- "6 - Passes Data to" --> ConvertToSTIX + GTIModels -- "Processed by" --> ConvertToSTIX + ConvertToSTIX -- "7 - Transforms using" --> MapperClasses + MapperClasses -- "8 - Produces" --> STIXModels + + ConvertToSTIX -- "9 - Bundles into" --> StixBundle + WorkManager -- "10 - Updates" --> StateManagement + WorkManager -- "11 - Sends to" --> OpenCTIQueue + + Connector -- "Handles" --> ExceptionHandling +``` + +## Code Organization + +The connector code is organized into several key directories: + +``` +connector/src/ +├── custom/ # GTI-specific implementation +│ ├── configs/ # Configuration classes +│ ├── exceptions/ # Specialized exception hierarchy +│ ├── fetchers/ # Data fetching components +│ ├── mappers/ # Data transformation to STIX +│ ├── models/ # Pydantic models for GTI data +│ ├── batch_processor.py # Processes batches of data +│ ├── convert_to_stix.py # Converts GTI data to STIX +│ └── fetch_all.py # Orchestrates data fetching +├── octi/ # OpenCTI integration +├── stix/ # STIX models and utilities +└── utils/ # Common utilities +``` + +## Development Workflow + +1. **Environment Setup**: Configure your development environment with Python 3.11+ +2. **Install Dependencies**: `pip install -e .[all]` +3. **Code Changes**: Make your changes following the project structure +4. **Testing**: Write and run tests for your changes +5. **Documentation**: Update relevant documentation +6. **Pull Request**: Submit a pull request with a clear description of changes + +## Key Components + +### Generic Fetcher Architecture + +The connector uses a generic, configurable architecture for fetching data from the GTI API: + +``` +FetchAll (Orchestrator) +├── ReportFetcher (Reports + Pagination) +└── EntityFetcher (Coordinates all entity types via factory) + └── FetcherFactory + ├── GenericEntityFetcher(MalwareFamilyConfig) + ├── GenericEntityFetcher(ThreatActorConfig) + ├── GenericEntityFetcher(AttackTechniqueConfig) + ├── GenericEntityFetcher(VulnerabilityConfig) + └── RelationshipFetcher (Shared by all entity fetchers) +``` + +### BaseFetcher +- Base class providing common functionality +- Manages API client and headers +- Provides logging utilities and helper methods + +### EntityFetcherConfig +- Configuration for entity-specific parameters +- Defines entity type names and relationship mappings +- Specifies API endpoint templates and response models +- Maps exception classes and display names + +### GenericEntityFetcher +- Single implementation that handles all entity types +- Configures behavior based on EntityFetcherConfig +- Implements parallel fetching with asyncio +- Handles exceptions with graceful degradation + +### FetcherFactory +- Creates configured GenericEntityFetcher instances +- Provides type-safe factory methods +- Supports bulk creation of all entity fetchers + +### EntityFetcher +- Orchestrates fetching of all entity types for reports +- Implements parallel processing across entity types +- Provides flexible entity type selection + +### ReportFetcher +- Handles report fetching with pagination +- Manages date filtering, sorting, and batch processing +- Supports report type and origin filtering + +### ConvertToSTIX (Gonna be rework soon, working PoC legacy for now.) +- Transforms GTI data into STIX 2.1 format +- Uses specialized mapper classes for different entity types +- Maintains relationships between objects +- Creates properly formatted STIX bundles + +## Adding New Features + +### Adding New Entity Types + +1. Define a new Pydantic model to map API response in `models/gti_reports/` +2. Create a new configuration in `fetchers/entity_config.py`: + +```python +NEW_ENTITY_CONFIG = EntityFetcherConfig( + entity_type="new_entities", + relationship_type="new_entities", + endpoint_template="/new_endpoint/{entity_id}", + response_model=NewEntityResponse, + exception_class=NewEntityFetchError, + display_name="new entities", + display_name_singular="new entity" +) + +# Add to registry +ENTITY_CONFIGS["new_entities"] = NEW_ENTITY_CONFIG +``` + +3. Create a new STIX mapper in `mappers/gti_reports/` +4. Update `convert_to_stix.py` to include the new entity type + +### Adding New STIX Mappings + +1. Create a new mapper class in `mappers/gti_reports/` +2. Implement the `to_stix()` method to convert GTI data to STIX +3. Update `ConvertToSTIX` class to use your new mapper + +## Best Practices + +1. **Type Safety**: Use type hints and Pydantic models consistently +2. **Error Handling**: Follow the exception hierarchy pattern +3. **Asynchronous Code**: + - Use `asyncio` for concurrent operations, if needed. Take care of race conditions + - Handle task cancellation properly + - Implement proper timeout handling +4. **Logging**: + - Use structured logging with context + - Log at appropriate levels + - Include relevant identifiers in log messages +5. **Configuration**: + - Use Pydantic for config validation + - Implement proper defaults + - Support both environment variables and config files + +## Error Handling + +The connector implements a specialized exception hierarchy: + +``` +GTIBaseError +├── GTIConfigurationError +├── GTIConvertingError +│ └── GTIEntityConversionError +│ ├── GTIReportConversionError +│ ├── GTIMalwareConversionError +│ ├── GTIActorConversionError +│ ├── GTITechniqueConversionError +│ └── GTIVulnerabilityConversionError +└── GTIFetchingError + ├── GTIApiError + │ ├── GTIReportFetchError + │ ├── GTIMalwareFetchError + │ ├── GTIActorFetchError + │ ├── GTITechniqueFetchError + │ ├── GTIVulnerabilityFetchError + │ └── GTIRelationshipFetchError + ├── GTIPaginationError + └── GTIParsingError +``` + +When adding new features: +1. Use the appropriate exception class +2. Include context information in exceptions +3. Handle exceptions at appropriate levels +4. Implement graceful degradation when possible + +## Testing + +1. **Unit Tests**: Test individual components in isolation +2. **Integration Tests**: Test component interactions +3. **Mock API Responses**: Use mock data for API testing +4. **Error Scenarios**: Test failure conditions and recovery +5. **Performance Testing**: Test with realistic data volumes + +### Testing New Entity Types + +When adding a new entity type: +1. Create mock API responses in test fixtures +2. Test the fetching process with different scenarios +3. Test the STIX conversion with various inputs +4. Verify relationships (if any) are correctly established + +## Complete Example: Adding Campaigns Support + +This example walks through the complete process of adding support for GTI campaign entities to the connector. + +### Step-by-Step Checklist + +#### 1. Create Pydantic Models + +First, create a new model file in `connector/src/custom/models/gti_reports/gti_campaign_model.py`: + +```python +"""Model representing a Google Threat Intelligence Campaign.""" + +from typing import Dict, List, Optional, Union + +from pydantic import BaseModel, Field + + +class CampaignModel(BaseModel): + """Model representing a GTI campaign.""" + + name: str = Field(..., description="Campaign's name.") + creation_date: int = Field(..., description="Creation date of the campaign (UTC timestamp).") + last_modification_date: int = Field(..., description="Date when the campaign was last updated.") + description: Optional[str] = Field(None, description="Campaign's description.") + objective: Optional[str] = Field(None, description="Campaign's objective.") + first_seen: Optional[int] = Field(None, description="First observed timestamp.") + last_seen: Optional[int] = Field(None, description="Last observed timestamp.") + private: bool = Field(False, description="Whether the campaign object is private.") + + +class GTICampaignData(BaseModel): + """Model representing data for a GTI campaign.""" + + id: str + type: str = Field("campaign") + links: Optional[Dict[str, str]] = None + attributes: Optional[CampaignModel] = None + + +class GTICampaignResponse(BaseModel): + """Model representing a response containing GTI campaign data.""" + + data: Union[GTICampaignData, List[GTICampaignData]] +``` + +#### 2. Create Exception Class + +Add a new exception class in `connector/src/custom/exceptions/fetch_errors/gti_campaign_fetch_error.py`: + +```python +"""Exception for errors when fetching campaigns from Google Threat Intelligence API.""" + +from typing import Any, Dict, Optional + +from connector.src.custom.exceptions.fetch_errors.gti_api_error import GTIApiError + + +class GTICampaignFetchError(GTIApiError): + """Exception raised when there's an error fetching campaigns from GTI API.""" + + def __init__( + self, + message: str, + campaign_id: Optional[str] = None, + endpoint: Optional[str] = None, + status_code: Optional[int] = None, + details: Optional[Dict[str, Any]] = None, + ): + """Initialize the exception. + + Args: + message: Error message + campaign_id: ID of the campaign that failed to fetch, if applicable + endpoint: API endpoint where the error occurred + status_code: HTTP status code, if available + details: Additional details about the error + + """ + error_msg = message + if campaign_id: + error_msg = f"Error fetching campaign {campaign_id}: {message}" + else: + error_msg = f"Error fetching campaigns: {message}" + + super().__init__(error_msg, status_code, endpoint, details) + self.campaign_id = campaign_id +``` + +Don't forget to update the `__init__.py` in the fetch_errors directory to include your new exception. + +#### 3. Add Entity Configuration + +Update `connector/src/custom/fetchers/entity_config.py` to add your new entity: + +```python +CAMPAIGN_CONFIG = EntityFetcherConfig( + entity_type="campaigns", + relationship_type="campaigns", + endpoint_template="/collections/{entity_id}/campaigns", # Adjust based on actual GTI API + response_model=GTICampaignResponse, + exception_class=GTICampaignFetchError, + display_name="campaigns", + display_name_singular="campaign" +) + +# Add to registry +ENTITY_CONFIGS["campaigns"] = CAMPAIGN_CONFIG +``` + +#### 4. Create STIX Mapper + +Create a new mapper in `connector/src/custom/mappers/gti_reports/gti_campaign_to_stix_campaign.py`: + +```python +"""Converts a GTI campaign to a STIX campaign object.""" + +from datetime import datetime +from typing import Dict, List, Optional + +from connector.src.custom.models.gti_reports.gti_campaign_model import ( + GTICampaignData, + CampaignModel, +) +from connector.src.stix.octi.models.campaign_model import OctiCampaignModel +from stix2.v21 import Campaign, Identity, MarkingDefinition # type: ignore + + +class GTICampaignToSTIXCampaign: + """Converts a GTI campaign to a STIX campaign object.""" + + def __init__( + self, + campaign: GTICampaignData, + organization: Identity, + tlp_marking: MarkingDefinition, + ) -> None: + """Initialize the GTICampaignToSTIXCampaign object. + + Args: + campaign: The GTI campaign data to convert. + organization: The organization identity object. + tlp_marking: The TLP marking definition. + + """ + self.campaign = campaign + self.organization = organization + self.tlp_marking = tlp_marking + + def to_stix(self) -> Campaign: + """Convert the GTI campaign to a STIX campaign object. + + Returns: + Campaign: The STIX campaign object. + + """ + if not self.campaign or not self.campaign.attributes: + raise ValueError("Campaign attributes are missing") + + attributes = self.campaign.attributes + + created = datetime.fromtimestamp(attributes.creation_date) + modified = datetime.fromtimestamp(attributes.last_modification_date) + + first_seen = None + if attributes.first_seen: + first_seen = datetime.fromtimestamp(attributes.first_seen) + + last_seen = None + if attributes.last_seen: + last_seen = datetime.fromtimestamp(attributes.last_seen) + + campaign_model = OctiCampaignModel.create( + name=attributes.name, + organization_id=self.organization.id, + marking_ids=[self.tlp_marking.id], + description=attributes.description, + objective=attributes.objective, + first_seen=first_seen, + last_seen=last_seen, + created=created, + modified=modified, + ) + + return campaign_model.to_stix2_object() +``` + +#### 5. Update ConvertToSTIX Class + +Modify `connector/src/custom/convert_to_stix.py` to add your new entity type: + +1. Add a new import: +```python +from connector.src.custom.exceptions import GTICampaignConversionError +from connector.src.custom.mappers.gti_reports.gti_campaign_to_stix_campaign import GTICampaignToSTIXCampaign +from connector.src.custom.models.gti_reports.gti_campaign_model import GTICampaignData +``` + +2. Add a conversion method: +```python +def _convert_campaign(self, campaign: GTICampaignData) -> Optional[Campaign]: + """Convert a GTI campaign to STIX format. + + Args: + campaign: GTI campaign data + + Returns: + STIX campaign object + + Raises: + GTICampaignConversionError: If there's an error converting the campaign + + """ + try: + self.logger.debug(f"Converting campaign {campaign.id} to STIX format") + + mapper = GTICampaignToSTIXCampaign( + campaign, self.organization, self.tlp_marking + ) + stix_campaign = mapper.to_stix() + + self.object_id_map[campaign.id] = stix_campaign.id + + return stix_campaign + + except Exception as e: + self.logger.error( + f"Error converting campaign {campaign.id}: {str(e)}", + meta={"error": str(e)}, + ) + campaign_name = getattr(campaign.attributes, "name", None) + raise GTICampaignConversionError(str(e), campaign.id, campaign_name) from e +``` + +3. Update the `convert_all_data` method to process campaigns: +```python +# Add to the section where other entities are processed +for campaign in related.get("campaigns", []): + try: + stix_campaign = self._convert_campaign(campaign) + if stix_campaign is not None: + self.stix_objects.append(stix_campaign) + ids_to_add.append(stix_campaign.id) + except GTICampaignConversionError as campaign_err: + self.logger.error( + f"Error processing campaign {campaign.id}: {str(campaign_err)}" + ) + continue +``` + +#### 6. Testing Checklist + +1. Create mock API responses for campaigns: + - Create a JSON fixture with sample campaign data + - Test parsing with the Pydantic model + +2. Test the entity fetcher: + - Verify campaigns are fetched correctly + - Test error handling scenarios + - Ensure pagination works as expected + +3. Test the STIX conversion: + - Verify all campaign fields are mapped correctly + - Check edge cases (missing fields, unusual values) + - Validate STIX schema compliance + +4. Test integration with reports: + - Verify relationships are correctly established + - Check that campaigns appear in report object references + +5. Run end-to-end tests with a mock server + +#### 7. Documentation Updates + +1. Update README with campaign support information +2. Document campaign fields and mapping logic +3. Add examples of campaign data to documentation + +## Questions and Support + +If you have questions or need help with your contribution, please: +1. Check existing issues on GitHub +2. Create a new issue with a clear description +3. Join the OpenCTI Slack community for direct support + +Thank you for contributing to the Google Threat Intelligence Feeds connector! diff --git a/external-import/google-ti-feeds/Dockerfile b/external-import/google-ti-feeds/Dockerfile new file mode 100644 index 0000000000..8ff1175f1b --- /dev/null +++ b/external-import/google-ti-feeds/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.12-alpine AS base +ENV CONNECTOR_TYPE=EXTERNAL_IMPORT + +RUN apk update && apk upgrade && \ + apk --no-cache add git build-base libmagic libffi-dev libxml2-dev libxslt-dev && \ + pip3 install --no-cache-dir --upgrade pip + + +FROM base AS package +# Copy the package +COPY connector /opt/connector +COPY pyproject.toml /opt/pyproject.toml + +RUN cd /opt/ && \ + pip3 install --no-cache-dir . && \ + apk del git build-base && \ + rm /opt/pyproject.toml && \ + rm -rf /opt/connector + +FROM package AS app +# Run the app +CMD ["GoogleTIFeeds"] diff --git a/external-import/google-ti-feeds/README.md b/external-import/google-ti-feeds/README.md new file mode 100644 index 0000000000..0180b50780 --- /dev/null +++ b/external-import/google-ti-feeds/README.md @@ -0,0 +1,172 @@ +# Google Threat Intelligence Connector + +--- + +## Introduction + +Google Threat Intelligence Feeds Connector ingests threat intelligence from the Google Threat Intel API and feeds it into the OpenCTI solution, focusing -for now- on STIX entities tied to report objects. +It extracts and transforms relevant data types report, location, sector, malware, intrusion-set, attack-pattern, vulnerability, and raw IOCs delivering structured, and ingest that in an intelligible way into OpenCTI. + +Most of the data is extracted from the reports, but some entities are extracted from the report's relationships. +More informations can be found in the [Google Threat Intel API documentation](https://gtidocs.virustotal.com/reference/reports). + +> This connector requires a Google Threat Intel API key to function. You can obtain one by signing up for the Google Threat Intel service.5 +> Reports Analysis are only available to users with the Google Threat Intelligence (Google TI) Enterprise or Enterprise Plus licenses.5 + +--- + +## Quick start + +Here’s a high-level overview to get the connector up and running: + +1. **Set environment variables**: + - inside `docker-compose.yml` +2. **Pull and run the connector** using Docker: +```bash + docker compose up -d +``` + +--- + +## Installation + +### Requirements + +- OpenCTI Platform version **6.6.10** or higher +- Docker & Docker Compose (for containerized deployment) +- Valid GTI API credentials (token) + +--- + +## Configurations Variables + +### OpenCTI Configuration + +Below are the required parameters you'll need to set for OpenCTI: + +| Parameter | config.yml | Docker Environment Variable | Mandatory | Description | +| --- | --- | --- | --- | --- | +| OpenCTI URL | `url` | `OPENCTI_URL` | Yes | The URL of the OpenCTI platform. | +| OpenCTI Token | `token` | `OPENCTI_TOKEN` | Yes | The API token for authenticating with OpenCTI. | + +### Connector Configuration + +Below are the required parameters you can set for running the connector: + +| Parameter | config.yml | Docker Environment Variable | Default | Mandatory | Description | +| --- | --- | --- | --- | --- | --- | +| Connector ID | `id` | `CONNECTOR_ID` | / | Yes | A unique `UUIDv4` identifier for this connector. | + +Below are the optional parameters you can set for running the connector: + +| Parameter | config.yml | Docker Environment Variable | Default | Mandatory | Description | +| --- | --- | --- | --- | --- | --- | +| Connector Name | `name` | `CONNECTOR_NAME` | Google Threat Intel Feeds | No | The name of the connector as it will appear in OpenCTI. | +| Connector Scope | `scope` | `CONNECTOR_SCOPE` | report,location,identity | No | The scope of data to import, a list of Stix Objects. | +| Connector Log Level | `log_level` | `CONNECTOR_LOG_LEVEL` | info | No | Sets the verbosity of logs. Options: `debug`, `info`, `warn`, `error`. | +| Connector Duration Period | `duration_period` | `CONNECTOR_DURATION_PERIOD` | PT2H | No | The duration period between two schedule for the connector. | +| Connector TLP Level | `tlp_level` | `CONNECTOR_TLP_LEVEL` | AMBER+STRICT | No | The TLP level for the connector. Options: `WHITE`, `GREEN`, `AMBER`, `RED`. | +| Connector Queue Threshold | `queue_threshold` | `CONNECTOR_QUEUE_THRESHOLD` | 500 | No | The threshold for the queue size before processing. | + +### GTI Configuration + +Below are the required parameters you'll need to set for Google Threat Intel: + +| Parameter | config.yml | Docker Environment Variable | Default | Mandatory | Description | +| --- | --- | --- | --- | --- | --- | +| Google Threat Intel API Key | `gti_api_key` | `GTI_API_KEY` | | Yes | The API key for Google Threat Intel. | + +Below are the optional parameters you can set for Google Threat Intel: + +| Parameter | config.yml | Docker Environment Variable | Default | Mandatory | Description | +| --- | --- | --- | --- | --- | --- | +| Google Threat Intel Import Start Date | `gti_import_start_date` | `GTI_IMPORT_START_DATE` | P1D | No | The start date for importing data from Google Threat Intel. | +| Google Threat Intel API URL | `gti_api_url` | `GTI_API_URL` | https://www.virustotal.com/api/v3 | No | The API URL for Google Threat Intel. | +| Google Threat Intel Toggle Import Reports | `gti_import_reports` | `GTI_IMPORT_REPORTS` | True | No | If set to `True`, the connector will import reports from Google Threat Intel. | +| Google Threat Intel Report Types | `gti_report_types` | `GTI_REPORT_TYPES` | All | No | The types of reports to import from Google Threat Intel. Can be a string separated by comma for multiple values. Valid values are: `All`, `Actor Profile`, `Country Profile`, `Cyber Physical Security Roundup`, `Event Coverage/Implication`, `Industry Reporting`, `Malware Profile`, `Net Assessment`, `Network Activity Reports`, `News Analysis`, `OSINT Article`, `Patch Report`, `Strategic Perspective`, `TTP Deep Dive`, `Threat Activity Alert`, `Threat Activity Report`, `Trends and Forecasting`, `Weekly Vulnerability Exploitation Report` | +| Google Threat Intel Report Origins | `gti_origins` | `GTI_ORIGINS` | All | No | The origin of the reports to import from Google Threat Intel. Can be a string separated by comma for multiple values. Valid values are: `All`, `partner`, `crowdsourced`, `google threat intelligence`. | + +> 📅 The `import_start_date` can be formatted as a time zone aware datetime or as a duration (e.g., `1970-01-01T00:00:00+03:00` for January, 1st 1970 at 3AM in Timezon +3H or `P3D` for 3 days ago relative to NOW UTC). + +## Development + +## Contributing + +Please refer to [CONTRIBUTING.md](CONTRIBUTING.md). + +### Running the Connector Locally + +The connector is designed to be run in a Docker container. However, if you want to run it locally for development purposes, you can do so by following these steps: + +1/ Clone the connectors repository: +```bash + git clone +``` + +2/ Navigate to the connector directory +```bash + cd external-import/google-ti-feeds +``` + +3/ Ensure you are using a Python 3.12 version + +4/ Install the required dependencies: +```bash +pip install -e .[all] +``` +(for legacy purposes, you can also use `pip install -r requirements.txt` that is in editable mode.) + +5a/ Set the required variables: +In your shell: +```bash + export OPENCTI_URL= + ... +``` +OR sourcing a `.env` file: +```bash + source .env +``` +OR creating a "config.yml" file at the root of the project: +```yaml + opencti: + url: + ... +``` + +6/ Run the connector: +```bash + GoogleTIFeeds +``` + or by launching the main.py: +```bash + python connector/__main__.py +``` + or by launching the module: +```bash + python -m connector +``` + +### Commit + +Note: Your commits must be signed using a GPG key. Otherwise, your Pull Request will be rejected. + +### Linting and formatting + +Added to the connectors linteing and formatting rules, this connector is developped and checked using ruff and mypy to ensure the code is type-checked and linted. +The dedicated configurations are set in the `pyproject.toml` file. +You can run the following commands to check the code: + +```bash + python -m isort . + python -m black . --check + python -m ruff check . + python -m mypy . + python -m pip_audit . +``` + +### Testing + +To run the tests, you can use the following command: +```bash + python -m pytest -svv +``` diff --git a/external-import/google-ti-feeds/config.yml.sample b/external-import/google-ti-feeds/config.yml.sample new file mode 100644 index 0000000000..f06ba62239 --- /dev/null +++ b/external-import/google-ti-feeds/config.yml.sample @@ -0,0 +1,9 @@ +opencti: + url: 'http://localhost:PORT' + token: 'ChangeMe' + +connector: + id: 'ChangeMe' + +gti: + api_key: 'ChangeMe' diff --git a/external-import/google-ti-feeds/connector/__init__.py b/external-import/google-ti-feeds/connector/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/__main__.py b/external-import/google-ti-feeds/connector/__main__.py new file mode 100644 index 0000000000..c85589a8ee --- /dev/null +++ b/external-import/google-ti-feeds/connector/__main__.py @@ -0,0 +1,39 @@ +"""Main entry point for the connector.""" + +import os +import traceback + +from connector.src.octi.connector import Connector +from dotenv import load_dotenv +from pycti import OpenCTIConnectorHelper # type: ignore + + +def main() -> None: + """Define the main function to run the connector.""" + try: + dev_mode = os.getenv("CONNECTOR_DEV_MODE", "").lower() == "true" + if dev_mode: + for k in list(os.environ): + if k.upper().startswith(("CONNECTOR_", "GTI_", "OPENCTI_")): + if k != "CONNECTOR_DEV_MODE": + del os.environ[k] + else: + load_dotenv(override=True) + + from connector.src.custom.configs.gti_config import GTIConfig + from connector.src.octi.global_config import GlobalConfig + + global_config = GlobalConfig() + global_config.add_config_class(GTIConfig) + + octi_helper = OpenCTIConnectorHelper(config=global_config.to_dict()) + + connector = Connector(global_config, octi_helper) + connector.run() + except Exception: + traceback.print_exc() + exit(1) + + +if __name__ == "__main__": + main() diff --git a/external-import/google-ti-feeds/connector/py.typed b/external-import/google-ti-feeds/connector/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/__init__.py b/external-import/google-ti-feeds/connector/src/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/__init__.py b/external-import/google-ti-feeds/connector/src/custom/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/batch_processor.py b/external-import/google-ti-feeds/connector/src/custom/batch_processor.py new file mode 100644 index 0000000000..d62ecddff8 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/batch_processor.py @@ -0,0 +1,185 @@ +"""The module will contains method to process batches of STIX objects.""" + +import logging +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from connector.src.custom.exceptions import GTIWorkProcessingError + +if TYPE_CHECKING: + from logging import Logger + + from connector.src.custom.convert_to_stix import ConvertToSTIX + from connector.src.octi.work_manager import WorkManager + +LOG_PREFIX = "[BatchProcessor]" + + +class BatchProcessor: + """The class will contains method to process batches of STIX objects.""" + + def __init__( + self, + work_manager: "WorkManager", + work_id: str, + converter: "ConvertToSTIX", + logger: Optional["Logger"] = None, + ) -> None: + """Initialize the BatchProcessor. + + Args: + work_manager: The work manager object + work_id: The ID of the current work + converter: ConvertToSTIX instance + logger: Logger for logging messages + + """ + self._work_manager = work_manager + self._work_id = work_id + self._work_manager.set_current_work_id(work_id) + self._converter = converter + self._logger = logger or logging.getLogger(__name__) + self._latest_modified_date: Optional[str] = None + self._total_stix_objects_sent = 0 + self._total_batches_processed = 0 + self._all_stix_objects: List[Any] = [] + self._state_key = "last_report_date" + + def process_batch( + self, + reports: List[Any], + related_entities: Dict[str, Dict[str, List[Any]]], + ) -> None: + """Process a batch of data by converting to STIX and sending to OpenCTI. + + Args: + reports: List of report data for this batch + related_entities: Dictionary of related entities for this batch + + Raises: + GTIWorkProcessingError: If there's an error processing the work + + """ + if not reports: + self._logger.info(f"{LOG_PREFIX} No reports in batch to process") + return + + try: + self._total_batches_processed += 1 + batch_num = self._total_batches_processed + + self._logger.info( + f"{LOG_PREFIX} Converting batch #{batch_num} ({len(reports)} reports) to STIX format" + ) + stix_objects = self._converter.convert_all_data(reports, related_entities) + self._all_stix_objects.extend(stix_objects) + + self._logger.info( + f"{LOG_PREFIX} Sending {len(stix_objects)} STIX objects to OpenCTI (batch #{batch_num})" + ) + + try: + self._work_manager.send_bundle( + work_id=self._work_id, bundle=stix_objects + ) + + self._work_manager.work_to_process(work_id=self._work_id) + new_work_id = self._work_manager.initiate_work( + name=f"Google Threat Intel Feeds - Batch #{batch_num + 1}" + ) + self._work_id = new_work_id + + self._total_stix_objects_sent += len(stix_objects) + self._logger.info( + f"{LOG_PREFIX} Successfully sent batch #{batch_num}. Total STIX objects sent: {self._total_stix_objects_sent}" + ) + + converter_latest_date = self._converter.get_latest_report_date() + if converter_latest_date: + if ( + not self._latest_modified_date + or converter_latest_date > self._latest_modified_date + ): + self._latest_modified_date = converter_latest_date + + if self._latest_modified_date: + self._logger.info( + f"{LOG_PREFIX} Updating state with latest date: {self._latest_modified_date}" + ) + self._work_manager.update_state( + state_key=self._state_key, date_str=self._latest_modified_date + ) + + except Exception as bundle_err: + raise GTIWorkProcessingError( + f"Failed to send bundle for batch #{batch_num}: {str(bundle_err)}", + self._work_id, + {"stix_objects_count": len(stix_objects)}, + ) from bundle_err + + except Exception as e: + if isinstance(e, GTIWorkProcessingError): + raise + + raise GTIWorkProcessingError( + f"Failed to process fetched data batch #{batch_num}: {str(e)}", + self._work_id, + {"batch_number": batch_num, "reports_count": len(reports)}, + ) from e + + def get_latest_modified_date(self) -> Optional[str]: + """Get the latest modified date across all processed batches. + + Returns: + The latest modification date (ISO format) if available, None otherwise + + """ + return self._latest_modified_date + + def set_latest_modified_date(self, date_str: str) -> None: + """Set the latest modified date. + + Args: + date_str: The date string in ISO format + + """ + if date_str and ( + not self._latest_modified_date or date_str > self._latest_modified_date + ): + self._latest_modified_date = date_str + + def update_final_state(self) -> None: + """Update the state with the final latest modification date after all processing is complete.""" + if self._latest_modified_date: + self._logger.info( + f"{LOG_PREFIX} Updating final state with latest date: {self._latest_modified_date}" + ) + self._work_manager.update_state( + state_key=self._state_key, date_str=self._latest_modified_date + ) + + def get_all_stix_objects(self) -> List[Any]: + """Get all STIX objects processed across all batches. + + Returns: + List of all STIX objects + + """ + return self._all_stix_objects + + def get_total_stix_objects_count(self) -> int: + """Get the total number of STIX objects sent. + + Returns: + The total number of STIX objects sent + + """ + return self._total_stix_objects_sent + + def get_total_batches_processed(self) -> int: + """Get the total number of batches processed. + + Returns: + The total number of batches processed + + """ + return self._total_batches_processed diff --git a/external-import/google-ti-feeds/connector/src/custom/configs/__init__.py b/external-import/google-ti-feeds/connector/src/custom/configs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/configs/gti_config.py b/external-import/google-ti-feeds/connector/src/custom/configs/gti_config.py new file mode 100644 index 0000000000..fe3b40fbe6 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/configs/gti_config.py @@ -0,0 +1,88 @@ +"""GTI feed connector configuration—defines environment-based settings and validators.""" + +from typing import ClassVar, List + +from connector.src.custom.exceptions.gti_configuration_error import ( + GTIConfigurationError, +) +from connector.src.octi.interfaces.base_config import BaseConfig +from pydantic import field_validator +from pydantic_settings import SettingsConfigDict + +ALLOWED_REPORT_TYPES = [ + "All", + "Actor Profile", + "Country Profile", + "Cyber Physical Security Roundup", + "Event Coverage/Implication", + "Industry Reporting", + "Malware Profile", + "Net Assessment", + "Network Activity Reports", + "News Analysis", + "OSINT Article", + "Patch Report", + "Strategic Perspective", + "TTP Deep Dive", + "Threat Activity Alert", + "Threat Activity Report", + "Trends and Forecasting", + "Weekly Vulnerability Exploitation Report", +] + +ALLOWED_ORIGINS = [ + "All", + "partner", + "crowdsourced", + "google threat intelligence", +] + + +class GTIConfig(BaseConfig): + """Configuration for the GTI part of the connector.""" + + yaml_section: ClassVar[str] = "gti" + model_config = SettingsConfigDict(env_prefix="gti_") + + api_key: str + import_start_date: str = "P1D" + api_url: str = "https://www.virustotal.com/api/v3" + import_reports: bool = True + report_types: List[str] | str = "All" + origins: List[str] | str = "All" + + @field_validator("report_types", mode="before") + @classmethod + def split_and_validate(cls, v: str) -> List[str]: + """Split and validate a comma-separated string into a list and validate its contents.""" + if isinstance(v, str): + parts = [item.strip() for item in v.split(",") if item.strip()] + + if not parts: + raise GTIConfigurationError("At least one report type must be specified.") + + invalid = set(parts) - set(ALLOWED_REPORT_TYPES) + if invalid: + raise GTIConfigurationError( + f"Invalid report types: {', '.join(invalid)}. " + f"Allowed values: {', '.join(ALLOWED_REPORT_TYPES)}." + ) + return parts + + @field_validator("origins", mode="before") + @classmethod + def split_and_validate_origins(cls, v: str) -> List[str]: + """Split and validate a comma-separated string into a list and validate its contents.""" + if isinstance(v, str): + parts = [item.strip() for item in v.split(",") if item.strip()] + + if not parts: + raise GTIConfigurationError("At least one origin must be specified.") + + invalid = set(parts) - set(ALLOWED_ORIGINS) + if invalid: + raise GTIConfigurationError( + f"Invalid origins: {', '.join(invalid)}. " + f"Allowed values: {', '.join(ALLOWED_ORIGINS)}." + ) + return parts diff --git a/external-import/google-ti-feeds/connector/src/custom/convert_to_stix.py b/external-import/google-ti-feeds/connector/src/custom/convert_to_stix.py new file mode 100644 index 0000000000..8b9aaaf69c --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/convert_to_stix.py @@ -0,0 +1,627 @@ +"""Simple converter for Google Threat Intelligence data to STIX format. + +This module provides a simpler approach to converting GTI data to STIX format +by using specialized mapper classes. +""" + +import logging +from datetime import datetime, timezone +from typing import Any, Dict, List, Literal, Optional, Union, cast + +from connector.src.custom.exceptions import ( + GTIActorConversionError, + GTIEntityConversionError, + GTIMalwareConversionError, + GTIMarkingCreationError, + GTIOrganizationCreationError, + GTIReferenceError, + GTIReportConversionError, + GTITechniqueConversionError, + GTIVulnerabilityConversionError, +) +from connector.src.custom.mappers.gti_reports.gti_attack_technique_to_stix_attack_pattern import ( + GTIAttackTechniqueToSTIXAttackPattern, +) +from connector.src.custom.mappers.gti_reports.gti_malware_family_to_stix_malware import ( + GTIMalwareFamilyToSTIXMalware, +) +from connector.src.custom.mappers.gti_reports.gti_report_relationship import ( + GTIReportRelationship, +) +from connector.src.custom.mappers.gti_reports.gti_report_to_stix_identity import ( + GTIReportToSTIXIdentity, +) +from connector.src.custom.mappers.gti_reports.gti_report_to_stix_location import ( + GTIReportToSTIXLocation, +) +from connector.src.custom.mappers.gti_reports.gti_report_to_stix_report import ( + GTIReportToSTIXReport, +) +from connector.src.custom.mappers.gti_reports.gti_report_to_stix_sector import ( + GTIReportToSTIXSector, +) +from connector.src.custom.mappers.gti_reports.gti_threat_actor_to_stix_intrusion_set import ( + GTIThreatActorToSTIXIntrusionSet, +) +from connector.src.custom.mappers.gti_reports.gti_vulnerability_to_stix_vulnerability import ( + GTIVulnerabilityToSTIXVulnerability, +) +from connector.src.custom.models.gti_reports.gti_attack_technique_model import ( + GTIAttackTechniqueData, +) +from connector.src.custom.models.gti_reports.gti_malware_family_model import ( + GTIMalwareFamilyData, +) +from connector.src.custom.models.gti_reports.gti_report_model import GTIReportData +from connector.src.custom.models.gti_reports.gti_threat_actor_model import ( + GTIThreatActorData, +) +from connector.src.custom.models.gti_reports.gti_vulnerability_model import ( + GTIVulnerabilityData, +) +from connector.src.stix.octi.models.identity_organization_model import ( + OctiOrganizationModel, +) +from connector.src.stix.octi.models.tlp_marking_model import TLPMarkingModel +from stix2.v21 import ( # type: ignore + AttackPattern, + Identity, + IntrusionSet, + Malware, + MarkingDefinition, + Report, + Vulnerability, +) + + +class ConvertToSTIX: + """A simple converter for Google Threat Intelligence data to STIX format. + + This class converts GTI data (reports, threat actors, malware families, + attack techniques, vulnerabilities) to STIX format using dedicated mapper classes. + """ + + def __init__( + self, + tlp_level: str = "amber", + logger: Optional[logging.Logger] = None, + ): + """Initialize the GTI STIX converter. + + Args: + tlp_level: TLP marking level for STIX objects + logger: Logger for logging messages + + """ + self.logger = logger or logging.getLogger(__name__) + + self.organization = self._create_organization() + self.tlp_marking = self._create_tlp_marking(tlp_level) + + self.stix_objects: List[Any] = [] + self.object_id_map: Dict[str, str] = {} + self.latest_report_date: Optional[str] = None + + def convert_all_data( + self, + reports: List[GTIReportData], + related_entities: Dict[str, Dict[str, List[Any]]], + ) -> List[Dict[str, Any]]: + """Convert all GTI data to STIX format. + + Args: + reports: List of GTI reports + related_entities: Dictionary mapping report IDs to related entities + + Returns: + List of STIX objects + + Raises: + GTIEntityConversionError: If there's an error converting an entity + + """ + try: + self.logger.info("Starting to convert GTI data to STIX format") + + self.stix_objects = [self.organization, self.tlp_marking] + self.object_id_map = {} + self.latest_report_date = None + + total_reports = len(reports) + for i, report in enumerate(reports): + report_id = report.id + progress_info = ( + f"({i + 1}/{total_reports} reports) " if total_reports > 0 else "" + ) + self.logger.info( + f"{progress_info}Converting report {report_id} to STIX format" + ) + + if not hasattr(report, "attributes") or not report.attributes: + raise GTIReportConversionError( + f"Report {report_id} has no attributes" + ) + + if ( + hasattr(report.attributes, "last_modification_date") + and report.attributes.last_modification_date + ): + report_date = str(report.attributes.last_modification_date) + if ( + not self.latest_report_date + or report_date > self.latest_report_date + ): + self.latest_report_date = report_date + + try: + report_entities = self._convert_report(report) + self.stix_objects.extend(report_entities) + except GTIReportConversionError as report_err: + self.logger.error( + f"Error converting report {report_id}: {str(report_err)}" + ) + + continue + + if report_id in related_entities: + related = related_entities[report_id] + + ids_to_add = [] + + for malware in related.get("malware_families", []): + try: + stix_malware = self._convert_malware_family(malware) + if stix_malware is not None: + self.stix_objects.append(stix_malware) + ids_to_add.append(stix_malware.id) + except GTIMalwareConversionError as malware_err: + self.logger.error( + f"Error processing malware family {malware.id}: {str(malware_err)}" + ) + continue + + for actor in related.get("threat_actors", []): + try: + stix_actor = self._convert_threat_actor(actor) + if stix_actor is not None: + self.stix_objects.append(stix_actor) + ids_to_add.append(stix_actor.id) + except GTIActorConversionError as actor_err: + self.logger.error( + f"Error processing threat actor {actor.id}: {str(actor_err)}" + ) + continue + + for technique in related.get("attack_techniques", []): + try: + stix_technique = self._convert_attack_technique(technique) + if stix_technique is not None: + self.stix_objects.append(stix_technique) + ids_to_add.append(stix_technique.id) + except GTITechniqueConversionError as technique_err: + self.logger.error( + f"Error processing attack technique {technique.id}: {str(technique_err)}" + ) + continue + + for vulnerability in related.get("vulnerabilities", []): + try: + stix_vuln = self._convert_vulnerability(vulnerability) + if stix_vuln is not None: + self.stix_objects.append(stix_vuln) + ids_to_add.append(stix_vuln.id) + except GTIVulnerabilityConversionError as vuln_err: + self.logger.error( + f"Error processing vulnerability {vulnerability.id}: {str(vuln_err)}" + ) + continue + + if ids_to_add: + try: + self._add_reference_to_report(ids_to_add, report_id) + except GTIReferenceError as ref_err: + self.logger.error( + f"Error adding references to report {report_id}: {str(ref_err)}" + ) + + self.logger.info(f"Converted {len(self.stix_objects)} STIX objects") + return self.stix_objects + + except GTIEntityConversionError: + raise + except Exception as e: + self.logger.error( # type: ignore + f"Error converting GTI data to STIX format: {str(e)}", + meta={"error": str(e)}, + ) + + if self.stix_objects: + self.logger.info( + f"Returning {len(self.stix_objects)} partially converted STIX objects" + ) + return self.stix_objects + raise GTIEntityConversionError( + f"Failed to convert GTI data: {str(e)}" + ) from e + + def _create_organization(self) -> Identity: + """Create a STIX Identity object for the organization. + + Returns: + STIX Identity object + + Raises: + GTIOrganizationCreationError: If there's an error creating the organization + + """ + try: + organization_model = OctiOrganizationModel.create( + name="Google Threat Intelligence", + description="Google Threat Intelligence provides information on the latest threats.", + contact_information="https://gtidocs.virustotal.com", + organization_type="vendor", + reliability=None, + aliases=["GTI"], + ) + return organization_model.to_stix2_object() + except Exception as e: + raise GTIOrganizationCreationError(str(e)) from e + + def _create_tlp_marking(self, tlp_level: str) -> MarkingDefinition: + """Create a TLP marking definition. + + Args: + tlp_level: TLP level (white, green, amber, red) + + Returns: + TLP marking definition + + Raises: + GTIMarkingCreationError: If there's an error creating the TLP marking + + """ + try: + normalized_level = tlp_level.lower() + + if normalized_level not in ( + "white", + "green", + "amber", + "amber+strict", + "red", + ): + normalized_level = "amber" + + tlp_literal = cast( + Literal["white", "green", "amber", "amber+strict", "red"], + normalized_level, + ) + + return TLPMarkingModel(level=tlp_literal).to_stix2_object() + except Exception as e: + raise GTIMarkingCreationError(str(e), tlp_level) from e + + def _convert_report(self, report: GTIReportData) -> List[Dict[str, Any]]: + """Convert a GTI report to STIX format. + + Args: + report: GTI report data + + Returns: + List of STIX objects + + Raises: + GTIReportConversionError: If there's an error converting the report + + """ + try: + self.logger.debug(f"Converting report {report.id} to STIX format") + result = [] + + try: + author_mapper = GTIReportToSTIXIdentity(report, self.organization) + author_identity = author_mapper.to_stix() + result.append(author_identity) + except Exception as author_err: + raise GTIReportConversionError( + f"Failed to convert report author: {str(author_err)}", + report.id, + "author_conversion", + ) from author_err + + try: + location_mapper = GTIReportToSTIXLocation( + report, self.organization, self.tlp_marking + ) + locations = location_mapper.to_stix() + result.extend(locations) + except Exception as location_err: + raise GTIReportConversionError( + f"Failed to convert report locations: {str(location_err)}", + report.id, + "location_conversion", + ) from location_err + + try: + sector_mapper = GTIReportToSTIXSector( + report, self.organization, self.tlp_marking + ) + sectors = sector_mapper.to_stix() + result.extend(sectors) + except Exception as sector_err: + raise GTIReportConversionError( + f"Failed to convert report sectors: {str(sector_err)}", + report.id, + "sector_conversion", + ) from sector_err + + try: + report_mapper = GTIReportToSTIXReport( + report, + self.organization, + self.tlp_marking, + author_identity, + sectors, + locations, + ) + report_obj = report_mapper.to_stix() + result.append(report_obj) + except Exception as report_err: + raise GTIReportConversionError( + f"Failed to convert report object: {str(report_err)}", + report.id, + "report_object_conversion", + ) from report_err + + self.object_id_map[report.id] = report_obj.id + + try: + relationship_mapper = GTIReportRelationship( + report, self.organization, self.tlp_marking, report_obj.id + ) + relationships = relationship_mapper.to_stix() + result.extend(relationships) + except Exception as rel_err: + raise GTIReportConversionError( + f"Failed to convert report relationships: {str(rel_err)}", + report.id, + "relationship_conversion", + ) from rel_err + + return result + + except GTIReportConversionError: + raise + except Exception as e: + self.logger.error( # type: ignore + f"Error converting report {report.id}: {str(e)}", meta={"error": str(e)} + ) + raise GTIReportConversionError(str(e), report.id) from e + + def _convert_malware_family( + self, malware: GTIMalwareFamilyData + ) -> Optional[Malware]: + """Convert a GTI malware family to STIX format. + + Args: + malware: GTI malware family data + + Returns: + STIX malware object + + Raises: + GTIMalwareConversionError: If there's an error converting the malware family + + """ + try: + self.logger.debug(f"Converting malware family {malware.id} to STIX format") + + mapper = GTIMalwareFamilyToSTIXMalware( + malware, self.organization, self.tlp_marking + ) + stix_malware = mapper.to_stix() + + self.object_id_map[malware.id] = stix_malware.id + + return stix_malware + + except Exception as e: + self.logger.error( # type: ignore + f"Error converting malware family {malware.id}: {str(e)}", + meta={"error": str(e)}, + ) + malware_name = getattr(malware, "name", None) + raise GTIMalwareConversionError(str(e), malware.id, malware_name) from e + + def _convert_threat_actor( + self, actor: GTIThreatActorData + ) -> Optional[IntrusionSet]: + """Convert a GTI threat actor to STIX format. + + Args: + actor: GTI threat actor data + + Returns: + STIX intrusion set object + + Raises: + GTIActorConversionError: If there's an error converting the threat actor + + """ + try: + self.logger.debug(f"Converting threat actor {actor.id} to STIX format") + + mapper = GTIThreatActorToSTIXIntrusionSet( + actor, self.organization, self.tlp_marking + ) + stix_actor = mapper.to_stix() + + self.object_id_map[actor.id] = stix_actor.id + + return stix_actor + + except Exception as e: + self.logger.error( # type: ignore + f"Error converting threat actor {actor.id}: {str(e)}", + meta={"error": str(e)}, + ) + actor_name = getattr(actor, "name", None) + raise GTIActorConversionError(str(e), actor.id, actor_name) from e + + def _convert_attack_technique( + self, technique: GTIAttackTechniqueData + ) -> Optional[AttackPattern]: + """Convert a GTI attack technique to STIX format. + + Args: + technique: GTI attack technique data + + Returns: + STIX attack pattern object + + Raises: + GTITechniqueConversionError: If there's an error converting the attack technique + + """ + try: + self.logger.debug( + f"Converting attack technique {technique.id} to STIX format" + ) + + mapper = GTIAttackTechniqueToSTIXAttackPattern( + technique, self.organization, self.tlp_marking + ) + stix_technique = mapper.to_stix() + + self.object_id_map[technique.id] = stix_technique.id + + return stix_technique + + except Exception as e: + self.logger.error( # type: ignore + f"Error converting attack technique {technique.id}: {str(e)}", + meta={"error": str(e)}, + ) + technique_name = getattr(technique, "name", None) + mitre_id = getattr(technique, "mitre_attack_id", None) + raise GTITechniqueConversionError( + str(e), technique.id, technique_name, mitre_id + ) from e + + def _convert_vulnerability( + self, vulnerability: GTIVulnerabilityData + ) -> Optional[Vulnerability]: + """Convert a GTI vulnerability to STIX format. + + Args: + vulnerability: GTI vulnerability data + + Returns: + STIX vulnerability object + + Raises: + GTIVulnerabilityConversionError: If there's an error converting the vulnerability + + """ + try: + self.logger.debug( + f"Converting vulnerability {vulnerability.id} to STIX format" + ) + + mapper = GTIVulnerabilityToSTIXVulnerability( + vulnerability, self.organization, self.tlp_marking + ) + stix_vuln = mapper.to_stix() + + self.object_id_map[vulnerability.id] = stix_vuln.id + + return stix_vuln + + except Exception as e: + self.logger.error( # type: ignore + f"Error converting vulnerability {vulnerability.id}: {str(e)}", + meta={"error": str(e)}, + ) + cve_id = getattr(vulnerability, "cve_id", None) + raise GTIVulnerabilityConversionError( + str(e), vulnerability.id, cve_id + ) from e + + def _add_reference_to_report( + self, object_id: Union[str, List[str]], report_id: str + ) -> None: + """Add reference to one or more objects in a report. + + Args: + object_id: ID of the object to reference, or a list of object IDs + report_id: ID of the report + + Raises: + GTIReferenceError: If there's an error adding the reference + + """ + try: + object_ids = [object_id] if isinstance(object_id, str) else object_id + + stix_report_id = self.object_id_map.get(report_id) + if not stix_report_id: + self.logger.warning(f"Report {report_id} not found in object_id_map") + raise GTIReferenceError( + f"Report {report_id} not found in object_id_map", + target_id=report_id, + ) + + latest_report_index: Optional[int] = None + latest_report: Optional[Report] = None + + for i, obj in enumerate(self.stix_objects): + if isinstance(obj, Report) and obj.id == stix_report_id: + latest_report_index = i + latest_report = obj + + if latest_report is None: + self.logger.warning( + f"Report with ID {stix_report_id} not found in STIX objects" + ) + raise GTIReferenceError( + f"Report with ID {stix_report_id} not found in STIX objects", + source_id=stix_report_id, + ) + + try: + updated_report = GTIReportToSTIXReport.add_object_refs( + objects_to_add=object_ids, existing_report=latest_report + ) + + if latest_report_index is not None: + self.stix_objects[latest_report_index] = updated_report + except Exception as update_err: + raise GTIReferenceError( + f"Failed to update report references: {str(update_err)}", + source_id=stix_report_id, + target_id=str(object_ids), + ) from update_err + + except GTIReferenceError: + raise + except Exception as e: + self.logger.error(f"Error adding reference to report: {str(e)}", meta={"error": str(e)}) # type: ignore + raise GTIReferenceError( + str(e), + source_id=stix_report_id if "stix_report_id" in locals() else None, + target_id=str(object_id), + ) from e + + def get_latest_report_date(self) -> Optional[str]: + """Return the latest report modification date processed. + + Returns: + ISO format string of the latest report date, or None if no reports were processed + + """ + if self.latest_report_date is None: + return None + + dt = datetime.fromtimestamp(int(self.latest_report_date), tz=timezone.utc) + + return dt.isoformat() diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/__init__.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/__init__.py new file mode 100644 index 0000000000..931a906877 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/__init__.py @@ -0,0 +1,68 @@ +"""Exception classes for the Google Threat Intelligence Feed connector.""" + +from connector.src.custom.exceptions.connector_errors import ( + GTIApiClientError, + GTIAsyncError, + GTIPartialDataProcessingError, + GTIStateManagementError, + GTIWorkProcessingError, +) +from connector.src.custom.exceptions.convert_errors import ( + GTIActorConversionError, + GTIEntityConversionError, + GTIMalwareConversionError, + GTIMarkingCreationError, + GTIOrganizationCreationError, + GTIReferenceError, + GTIReportConversionError, + GTITechniqueConversionError, + GTIVulnerabilityConversionError, +) +from connector.src.custom.exceptions.fetch_errors import ( + GTIActorFetchError, + GTIApiError, + GTIMalwareFetchError, + GTIPaginationError, + GTIParsingError, + GTIRelationshipFetchError, + GTIReportFetchError, + GTITechniqueFetchError, + GTIVulnerabilityFetchError, +) +from connector.src.custom.exceptions.gti_base_error import GTIBaseError +from connector.src.custom.exceptions.gti_configuration_error import ( + GTIConfigurationError, +) +from connector.src.custom.exceptions.gti_converting_error import GTIConvertingError +from connector.src.custom.exceptions.gti_fetching_error import GTIFetchingError + +__all__ = [ + "GTIBaseError", + "GTIConfigurationError", + "GTIConvertReportsError", + "GTIConvertingError", + "GTIFetchingError", + "GTIEntityConversionError", + "GTIOrganizationCreationError", + "GTIMarkingCreationError", + "GTIReferenceError", + "GTIReportConversionError", + "GTIMalwareConversionError", + "GTIActorConversionError", + "GTITechniqueConversionError", + "GTIVulnerabilityConversionError", + "GTIApiError", + "GTIPaginationError", + "GTIParsingError", + "GTIReportFetchError", + "GTIMalwareFetchError", + "GTIActorFetchError", + "GTITechniqueFetchError", + "GTIVulnerabilityFetchError", + "GTIRelationshipFetchError", + "GTIWorkProcessingError", + "GTIAsyncError", + "GTIStateManagementError", + "GTIPartialDataProcessingError", + "GTIApiClientError", +] diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/__init__.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/__init__.py new file mode 100644 index 0000000000..32618cf584 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/__init__.py @@ -0,0 +1,25 @@ +"""Exception classes for connector processing errors.""" + +from connector.src.custom.exceptions.connector_errors.gti_api_client_error import ( + GTIApiClientError, +) +from connector.src.custom.exceptions.connector_errors.gti_async_error import ( + GTIAsyncError, +) +from connector.src.custom.exceptions.connector_errors.gti_partial_data_processing_error import ( + GTIPartialDataProcessingError, +) +from connector.src.custom.exceptions.connector_errors.gti_state_management_error import ( + GTIStateManagementError, +) +from connector.src.custom.exceptions.connector_errors.gti_work_processing_error import ( + GTIWorkProcessingError, +) + +__all__ = [ + "GTIWorkProcessingError", + "GTIAsyncError", + "GTIStateManagementError", + "GTIPartialDataProcessingError", + "GTIApiClientError", +] diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/gti_api_client_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/gti_api_client_error.py new file mode 100644 index 0000000000..5c81f8fcbb --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/gti_api_client_error.py @@ -0,0 +1,31 @@ +"""Exception for errors related to API client setup and configuration.""" + +from typing import Any, Dict, Optional + +from connector.src.custom.exceptions.gti_base_error import GTIBaseError + + +class GTIApiClientError(GTIBaseError): + """Exception raised when there's an error setting up or using the API client.""" + + def __init__( + self, + message: str, + client_component: Optional[str] = None, + details: Optional[Dict[str, Any]] = None, + ): + """Initialize the exception. + + Args: + message: Error message + client_component: Component of the API client that failed (e.g., "retry_strategy", "http_client") + details: Additional details about the error + + """ + error_msg = message + if client_component: + error_msg = f"API client error in {client_component}: {message}" + + super().__init__(error_msg) + self.client_component = client_component + self.details = details or {} diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/gti_async_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/gti_async_error.py new file mode 100644 index 0000000000..014034aa72 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/gti_async_error.py @@ -0,0 +1,31 @@ +"""Exception for asynchronous processing errors in the connector.""" + +from typing import Any, Dict, Optional + +from connector.src.custom.exceptions.gti_base_error import GTIBaseError + + +class GTIAsyncError(GTIBaseError): + """Exception raised when there's an error in asynchronous processing.""" + + def __init__( + self, + message: str, + operation: Optional[str] = None, + details: Optional[Dict[str, Any]] = None, + ): + """Initialize the exception. + + Args: + message: Error message + operation: Name of the async operation that failed + details: Additional details about the error + + """ + error_msg = message + if operation: + error_msg = f"Async error during '{operation}' operation: {message}" + + super().__init__(error_msg) + self.operation = operation + self.details = details or {} diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/gti_partial_data_processing_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/gti_partial_data_processing_error.py new file mode 100644 index 0000000000..047171d6a4 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/gti_partial_data_processing_error.py @@ -0,0 +1,42 @@ +"""Exception for errors when processing partial data after interruption.""" + +from typing import Any, Dict, Optional + +from connector.src.custom.exceptions.connector_errors.gti_work_processing_error import ( + GTIWorkProcessingError, +) + + +class GTIPartialDataProcessingError(GTIWorkProcessingError): + """Exception raised when there's an error processing partial data after interruption.""" + + def __init__( + self, + message: str, + work_id: Optional[str] = None, + interruption_type: Optional[str] = None, + reports_count: Optional[int] = None, + details: Optional[Dict[str, Any]] = None, + ): + """Initialize the exception. + + Args: + message: Error message + work_id: ID of the work that was interrupted + interruption_type: Type of interruption (e.g., "cancellation", "exception") + reports_count: Number of reports that were fetched before interruption + details: Additional details about the error + + """ + error_msg = f"Error processing partial data: {message}" + if interruption_type: + error_msg = ( + f"Error processing partial data after {interruption_type}: {message}" + ) + + if reports_count is not None: + error_msg += f" ({reports_count} reports were fetched)" + + super().__init__(error_msg, work_id, details) + self.interruption_type = interruption_type + self.reports_count = reports_count diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/gti_state_management_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/gti_state_management_error.py new file mode 100644 index 0000000000..f7c416090c --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/gti_state_management_error.py @@ -0,0 +1,31 @@ +"""Exception for errors related to state management in the connector.""" + +from typing import Any, Dict, Optional + +from connector.src.custom.exceptions.gti_base_error import GTIBaseError + + +class GTIStateManagementError(GTIBaseError): + """Exception raised when there's an error managing connector state.""" + + def __init__( + self, + message: str, + state_key: Optional[str] = None, + details: Optional[Dict[str, Any]] = None, + ): + """Initialize the exception. + + Args: + message: Error message + state_key: State key that was being accessed or modified + details: Additional details about the error + + """ + error_msg = message + if state_key: + error_msg = f"State management error for key '{state_key}': {message}" + + super().__init__(error_msg) + self.state_key = state_key + self.details = details or {} diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/gti_work_processing_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/gti_work_processing_error.py new file mode 100644 index 0000000000..ec815910f4 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/connector_errors/gti_work_processing_error.py @@ -0,0 +1,31 @@ +"""Exception for errors when processing work in the connector.""" + +from typing import Any, Dict, Optional + +from connector.src.custom.exceptions.gti_base_error import GTIBaseError + + +class GTIWorkProcessingError(GTIBaseError): + """Exception raised when there's an error processing work in the connector.""" + + def __init__( + self, + message: str, + work_id: Optional[str] = None, + details: Optional[Dict[str, Any]] = None, + ): + """Initialize the exception. + + Args: + message: Error message + work_id: ID of the work that failed to process + details: Additional details about the error + + """ + error_msg = message + if work_id: + error_msg = f"Error processing work {work_id}: {message}" + + super().__init__(error_msg) + self.work_id = work_id + self.details = details or {} diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/__init__.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/__init__.py new file mode 100644 index 0000000000..3686aa2848 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/__init__.py @@ -0,0 +1,41 @@ +"""Exception classes for STIX conversion errors.""" + +from connector.src.custom.exceptions.convert_errors.gti_actor_conversion_error import ( + GTIActorConversionError, +) +from connector.src.custom.exceptions.convert_errors.gti_entity_conversion_error import ( + GTIEntityConversionError, +) +from connector.src.custom.exceptions.convert_errors.gti_malware_conversion_error import ( + GTIMalwareConversionError, +) +from connector.src.custom.exceptions.convert_errors.gti_marking_creation_error import ( + GTIMarkingCreationError, +) +from connector.src.custom.exceptions.convert_errors.gti_organization_creation_error import ( + GTIOrganizationCreationError, +) +from connector.src.custom.exceptions.convert_errors.gti_reference_error import ( + GTIReferenceError, +) +from connector.src.custom.exceptions.convert_errors.gti_report_conversion_error import ( + GTIReportConversionError, +) +from connector.src.custom.exceptions.convert_errors.gti_technique_conversion_error import ( + GTITechniqueConversionError, +) +from connector.src.custom.exceptions.convert_errors.gti_vulnerability_conversion_error import ( + GTIVulnerabilityConversionError, +) + +__all__ = [ + "GTIEntityConversionError", + "GTIOrganizationCreationError", + "GTIMarkingCreationError", + "GTIReferenceError", + "GTIReportConversionError", + "GTIMalwareConversionError", + "GTIActorConversionError", + "GTITechniqueConversionError", + "GTIVulnerabilityConversionError", +] diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_actor_conversion_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_actor_conversion_error.py new file mode 100644 index 0000000000..a73876f28c --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_actor_conversion_error.py @@ -0,0 +1,31 @@ +"""Exception for errors when converting GTI threat actors to STIX intrusion sets.""" + +from typing import Optional + +from connector.src.custom.exceptions.convert_errors.gti_entity_conversion_error import ( + GTIEntityConversionError, +) + + +class GTIActorConversionError(GTIEntityConversionError): + """Exception raised when there's an error converting a GTI threat actor to STIX format.""" + + def __init__( + self, + message: str, + actor_id: Optional[str] = None, + actor_name: Optional[str] = None, + ): + """Initialize the exception. + + Args: + message: Error message + actor_id: ID of the threat actor that failed to convert + actor_name: Name of the threat actor, if available + + """ + super().__init__(message, actor_id, "ThreatActor") + self.actor_name = actor_name + + if actor_name and not self.args[0].endswith(f"(name: {actor_name})"): + self.args = (f"{self.args[0]} (name: {actor_name})",) diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_entity_conversion_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_entity_conversion_error.py new file mode 100644 index 0000000000..e8c165d469 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_entity_conversion_error.py @@ -0,0 +1,40 @@ +"""Base class for entity conversion errors.""" + +from typing import Optional + +from connector.src.custom.exceptions.gti_converting_error import GTIConvertingError + + +class GTIEntityConversionError(GTIConvertingError): + """Exception raised for errors in converting entities to STIX format. + + This serves as a base class for more specific entity conversion errors. + """ + + def __init__( + self, + message: str, + entity_id: Optional[str] = None, + entity_type: Optional[str] = None, + ): + """Initialize the exception. + + Args: + message: Error message + entity_id: ID of the entity that failed to convert + entity_type: Type of the entity that failed to convert + + """ + error_msg = message + if entity_id and entity_type: + error_msg = ( + f"Error converting {entity_type} entity (ID: {entity_id}): {message}" + ) + elif entity_id: + error_msg = f"Error converting entity (ID: {entity_id}): {message}" + elif entity_type: + error_msg = f"Error converting {entity_type} entity: {message}" + + super().__init__(error_msg) + self.entity_id = entity_id + self.entity_type = entity_type diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_malware_conversion_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_malware_conversion_error.py new file mode 100644 index 0000000000..c448206d70 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_malware_conversion_error.py @@ -0,0 +1,31 @@ +"""Exception for errors when converting GTI malware families to STIX malware objects.""" + +from typing import Optional + +from connector.src.custom.exceptions.convert_errors.gti_entity_conversion_error import ( + GTIEntityConversionError, +) + + +class GTIMalwareConversionError(GTIEntityConversionError): + """Exception raised when there's an error converting a GTI malware family to STIX format.""" + + def __init__( + self, + message: str, + malware_id: Optional[str] = None, + malware_name: Optional[str] = None, + ): + """Initialize the exception. + + Args: + message: Error message + malware_id: ID of the malware family that failed to convert + malware_name: Name of the malware family, if available + + """ + super().__init__(message, malware_id, "Malware") + self.malware_name = malware_name + + if malware_name and not self.args[0].endswith(f"(name: {malware_name})"): + self.args = (f"{self.args[0]} (name: {malware_name})",) diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_marking_creation_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_marking_creation_error.py new file mode 100644 index 0000000000..2c1bbe7a54 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_marking_creation_error.py @@ -0,0 +1,24 @@ +"""Exception for errors when creating TLP marking definitions.""" + +from typing import Optional + +from connector.src.custom.exceptions.gti_converting_error import GTIConvertingError + + +class GTIMarkingCreationError(GTIConvertingError): + """Exception raised when there's an error creating the TLP marking definition.""" + + def __init__(self, message: str, tlp_level: Optional[str] = None): + """Initialize the exception. + + Args: + message: Error message + tlp_level: The TLP level that failed to be created + + """ + error_msg = f"Failed to create TLP marking: {message}" + if tlp_level: + error_msg = f"Failed to create TLP '{tlp_level}' marking: {message}" + + super().__init__(error_msg) + self.tlp_level = tlp_level diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_organization_creation_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_organization_creation_error.py new file mode 100644 index 0000000000..60620951bb --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_organization_creation_error.py @@ -0,0 +1,20 @@ +"""Exception for errors when creating organization Identity objects.""" + +from typing import Any, Dict, Optional + +from connector.src.custom.exceptions.gti_converting_error import GTIConvertingError + + +class GTIOrganizationCreationError(GTIConvertingError): + """Exception raised when there's an error creating the organization Identity object.""" + + def __init__(self, message: str, details: Optional[Dict[str, Any]] = None): + """Initialize the exception. + + Args: + message: Error message + details: Additional details about the error + + """ + super().__init__(f"Failed to create organization Identity: {message}") + self.details = details or {} diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_reference_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_reference_error.py new file mode 100644 index 0000000000..aadf03653b --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_reference_error.py @@ -0,0 +1,37 @@ +"""Exception for errors related to adding references between STIX objects.""" + +from typing import Optional + +from connector.src.custom.exceptions.gti_converting_error import GTIConvertingError + + +class GTIReferenceError(GTIConvertingError): + """Exception raised when there's an error adding a reference between STIX objects.""" + + def __init__( + self, + message: str, + source_id: Optional[str] = None, + target_id: Optional[str] = None, + ): + """Initialize the exception. + + Args: + message: Error message + source_id: ID of the source object where the reference should be added + target_id: ID of the target object being referenced + + """ + error_msg = f"Failed to add reference: {message}" + if source_id and target_id: + error_msg = ( + f"Failed to add reference from {source_id} to {target_id}: {message}" + ) + elif source_id: + error_msg = f"Failed to add reference from {source_id}: {message}" + elif target_id: + error_msg = f"Failed to add reference to {target_id}: {message}" + + super().__init__(error_msg) + self.source_id = source_id + self.target_id = target_id diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_report_conversion_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_report_conversion_error.py new file mode 100644 index 0000000000..9e819c611d --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_report_conversion_error.py @@ -0,0 +1,31 @@ +"""Exception for errors when converting GTI reports to STIX reports.""" + +from typing import Optional + +from connector.src.custom.exceptions.convert_errors.gti_entity_conversion_error import ( + GTIEntityConversionError, +) + + +class GTIReportConversionError(GTIEntityConversionError): + """Exception raised when there's an error converting a GTI report to STIX format.""" + + def __init__( + self, + message: str, + report_id: Optional[str] = None, + processing_stage: Optional[str] = None, + ): + """Initialize the exception. + + Args: + message: Error message + report_id: ID of the report that failed to convert + processing_stage: The stage of processing where the error occurred + + """ + super().__init__(message, report_id, "Report") + self.processing_stage = processing_stage + + if processing_stage: + self.args = (f"{self.args[0]} (stage: {processing_stage})",) diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_technique_conversion_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_technique_conversion_error.py new file mode 100644 index 0000000000..48118824f9 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_technique_conversion_error.py @@ -0,0 +1,40 @@ +"""Exception for errors when converting GTI attack techniques to STIX attack patterns.""" + +from typing import Optional + +from connector.src.custom.exceptions.convert_errors.gti_entity_conversion_error import ( + GTIEntityConversionError, +) + + +class GTITechniqueConversionError(GTIEntityConversionError): + """Exception raised when there's an error converting a GTI attack technique to STIX format.""" + + def __init__( + self, + message: str, + technique_id: Optional[str] = None, + technique_name: Optional[str] = None, + mitre_id: Optional[str] = None, + ): + """Initialize the exception. + + Args: + message: Error message + technique_id: ID of the attack technique that failed to convert + technique_name: Name of the attack technique, if available + mitre_id: MITRE ATT&CK ID, if available + + """ + super().__init__(message, technique_id, "AttackTechnique") + self.technique_name = technique_name + self.mitre_id = mitre_id + + details = [] + if technique_name: + details.append(f"name: {technique_name}") + if mitre_id: + details.append(f"MITRE ID: {mitre_id}") + + if details and not self.args[0].endswith(f"({', '.join(details)})"): + self.args = (f"{self.args[0]} ({', '.join(details)})",) diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_vulnerability_conversion_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_vulnerability_conversion_error.py new file mode 100644 index 0000000000..72eb6c82e8 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/convert_errors/gti_vulnerability_conversion_error.py @@ -0,0 +1,28 @@ +"""Exception for errors when converting GTI vulnerabilities to STIX vulnerability objects.""" + +from typing import Optional + +from connector.src.custom.exceptions.convert_errors.gti_entity_conversion_error import ( + GTIEntityConversionError, +) + + +class GTIVulnerabilityConversionError(GTIEntityConversionError): + """Exception raised when there's an error converting a GTI vulnerability to STIX format.""" + + def __init__( + self, message: str, vuln_id: Optional[str] = None, cve_id: Optional[str] = None + ): + """Initialize the exception. + + Args: + message: Error message + vuln_id: ID of the vulnerability that failed to convert + cve_id: CVE ID of the vulnerability, if available + + """ + super().__init__(message, vuln_id, "Vulnerability") + self.cve_id = cve_id + + if cve_id and not self.args[0].endswith(f"(CVE: {cve_id})"): + self.args = (f"{self.args[0]} (CVE: {cve_id})",) diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/__init__.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/__init__.py new file mode 100644 index 0000000000..95d450f783 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/__init__.py @@ -0,0 +1,39 @@ +"""Exception classes for data fetching errors.""" + +from connector.src.custom.exceptions.fetch_errors.gti_actor_fetch_error import ( + GTIActorFetchError, +) +from connector.src.custom.exceptions.fetch_errors.gti_api_error import GTIApiError +from connector.src.custom.exceptions.fetch_errors.gti_malware_fetch_error import ( + GTIMalwareFetchError, +) +from connector.src.custom.exceptions.fetch_errors.gti_pagination_error import ( + GTIPaginationError, +) +from connector.src.custom.exceptions.fetch_errors.gti_parsing_error import ( + GTIParsingError, +) +from connector.src.custom.exceptions.fetch_errors.gti_relationship_fetch_error import ( + GTIRelationshipFetchError, +) +from connector.src.custom.exceptions.fetch_errors.gti_report_fetch_error import ( + GTIReportFetchError, +) +from connector.src.custom.exceptions.fetch_errors.gti_technique_fetch_error import ( + GTITechniqueFetchError, +) +from connector.src.custom.exceptions.fetch_errors.gti_vulnerability_fetch_error import ( + GTIVulnerabilityFetchError, +) + +__all__ = [ + "GTIApiError", + "GTIPaginationError", + "GTIParsingError", + "GTIReportFetchError", + "GTIMalwareFetchError", + "GTIActorFetchError", + "GTITechniqueFetchError", + "GTIVulnerabilityFetchError", + "GTIRelationshipFetchError", +] diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_actor_fetch_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_actor_fetch_error.py new file mode 100644 index 0000000000..ac49a5151c --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_actor_fetch_error.py @@ -0,0 +1,36 @@ +"""Exception for errors when fetching threat actors from Google Threat Intelligence API.""" + +from typing import Any, Dict, Optional + +from connector.src.custom.exceptions.fetch_errors.gti_api_error import GTIApiError + + +class GTIActorFetchError(GTIApiError): + """Exception raised when there's an error fetching threat actors from GTI API.""" + + def __init__( + self, + message: str, + actor_id: Optional[str] = None, + endpoint: Optional[str] = None, + status_code: Optional[int] = None, + details: Optional[Dict[str, Any]] = None, + ): + """Initialize the exception. + + Args: + message: Error message + actor_id: ID of the threat actor that failed to fetch, if applicable + endpoint: API endpoint where the error occurred + status_code: HTTP status code, if available + details: Additional details about the error + + """ + error_msg = message + if actor_id: + error_msg = f"Error fetching threat actor {actor_id}: {message}" + else: + error_msg = f"Error fetching threat actors: {message}" + + super().__init__(error_msg, status_code, endpoint, details) + self.actor_id = actor_id diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_api_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_api_error.py new file mode 100644 index 0000000000..9cb35d4fd5 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_api_error.py @@ -0,0 +1,38 @@ +"""Exception for API-related errors when fetching data from Google Threat Intelligence.""" + +from typing import Any, Dict, Optional + +from connector.src.custom.exceptions.gti_fetching_error import GTIFetchingError + + +class GTIApiError(GTIFetchingError): + """Exception raised when there's an error with the Google Threat Intelligence API.""" + + def __init__( + self, + message: str, + status_code: Optional[int] = None, + endpoint: Optional[str] = None, + details: Optional[Dict[str, Any]] = None, + ): + """Initialize the exception. + + Args: + message: Error message + status_code: HTTP status code, if available + endpoint: API endpoint where the error occurred + details: Additional details about the error + + """ + error_msg = message + if endpoint and status_code: + error_msg = f"API error at {endpoint} (status {status_code}): {message}" + elif endpoint: + error_msg = f"API error at {endpoint}: {message}" + elif status_code: + error_msg = f"API error (status {status_code}): {message}" + + super().__init__(error_msg) + self.status_code = status_code + self.endpoint = endpoint + self.details = details or {} diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_malware_fetch_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_malware_fetch_error.py new file mode 100644 index 0000000000..6f0049fb18 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_malware_fetch_error.py @@ -0,0 +1,36 @@ +"""Exception for errors when fetching malware families from Google Threat Intelligence API.""" + +from typing import Any, Dict, Optional + +from connector.src.custom.exceptions.fetch_errors.gti_api_error import GTIApiError + + +class GTIMalwareFetchError(GTIApiError): + """Exception raised when there's an error fetching malware families from GTI API.""" + + def __init__( + self, + message: str, + malware_id: Optional[str] = None, + endpoint: Optional[str] = None, + status_code: Optional[int] = None, + details: Optional[Dict[str, Any]] = None, + ): + """Initialize the exception. + + Args: + message: Error message + malware_id: ID of the malware family that failed to fetch, if applicable + endpoint: API endpoint where the error occurred + status_code: HTTP status code, if available + details: Additional details about the error + + """ + error_msg = message + if malware_id: + error_msg = f"Error fetching malware family {malware_id}: {message}" + else: + error_msg = f"Error fetching malware families: {message}" + + super().__init__(error_msg, status_code, endpoint, details) + self.malware_id = malware_id diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_pagination_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_pagination_error.py new file mode 100644 index 0000000000..095cb14384 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_pagination_error.py @@ -0,0 +1,43 @@ +"""Exception for pagination-related errors when fetching data from Google Threat Intelligence.""" + +from typing import Optional + +from connector.src.custom.exceptions.fetch_errors.gti_api_error import GTIApiError + + +class GTIPaginationError(GTIApiError): + """Exception raised when there's an error with pagination while fetching data.""" + + def __init__( + self, + message: str, + endpoint: Optional[str] = None, + page: Optional[int] = None, + page_size: Optional[int] = None, + status_code: Optional[int] = None, + ): + """Initialize the exception. + + Args: + message: Error message + endpoint: API endpoint where the error occurred + page: Page number that caused the error + page_size: Size of the page requested + status_code: HTTP status code, if available + + """ + pagination_details = [] + if page is not None: + pagination_details.append(f"page={page}") + if page_size is not None: + pagination_details.append(f"size={page_size}") + + pagination_info = "" + if pagination_details: + pagination_info = f" with {', '.join(pagination_details)}" + + error_msg = f"Pagination error{pagination_info}: {message}" + + super().__init__(error_msg, status_code, endpoint) + self.page = page + self.page_size = page_size diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_parsing_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_parsing_error.py new file mode 100644 index 0000000000..585d23df6e --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_parsing_error.py @@ -0,0 +1,45 @@ +"""Exception for errors when parsing responses from Google Threat Intelligence API.""" + +from typing import Optional + +from connector.src.custom.exceptions.gti_fetching_error import GTIFetchingError + + +class GTIParsingError(GTIFetchingError): + """Exception raised when there's an error parsing API responses.""" + + def __init__( + self, + message: str, + endpoint: Optional[str] = None, + entity_type: Optional[str] = None, + data_sample: Optional[str] = None, + ): + """Initialize the exception. + + Args: + message: Error message + endpoint: API endpoint where the response was received + entity_type: Type of entity being parsed (e.g., "report", "malware") + data_sample: Sample of the data that failed to parse (truncated if large) + + """ + error_msg = f"Error parsing response: {message}" + if entity_type and endpoint: + error_msg = f"Error parsing {entity_type} data from {endpoint}: {message}" + elif entity_type: + error_msg = f"Error parsing {entity_type} data: {message}" + elif endpoint: + error_msg = f"Error parsing response from {endpoint}: {message}" + + super().__init__(error_msg) + self.endpoint = endpoint + self.entity_type = entity_type + + if data_sample and isinstance(data_sample, str): + if len(data_sample) > 200: + self.data_sample = data_sample[:200] + "..." + else: + self.data_sample = data_sample + else: + self.data_sample = "" diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_relationship_fetch_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_relationship_fetch_error.py new file mode 100644 index 0000000000..3c9e2b0eb9 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_relationship_fetch_error.py @@ -0,0 +1,43 @@ +"""Exception for errors when fetching relationships from Google Threat Intelligence API.""" + +from typing import Any, Dict, Optional + +from connector.src.custom.exceptions.fetch_errors.gti_api_error import GTIApiError + + +class GTIRelationshipFetchError(GTIApiError): + """Exception raised when there's an error fetching relationships from GTI API.""" + + def __init__( + self, + message: str, + source_id: Optional[str] = None, + relationship_type: Optional[str] = None, + endpoint: Optional[str] = None, + status_code: Optional[int] = None, + details: Optional[Dict[str, Any]] = None, + ): + """Initialize the exception. + + Args: + message: Error message + source_id: ID of the source entity for the relationships + relationship_type: Type of relationship that failed to fetch + endpoint: API endpoint where the error occurred + status_code: HTTP status code, if available + details: Additional details about the error + + """ + error_msg = message + if source_id and relationship_type: + error_msg = f"Error fetching {relationship_type} relationships for {source_id}: {message}" + elif source_id: + error_msg = f"Error fetching relationships for {source_id}: {message}" + elif relationship_type: + error_msg = f"Error fetching {relationship_type} relationships: {message}" + else: + error_msg = f"Error fetching relationships: {message}" + + super().__init__(error_msg, status_code, endpoint, details) + self.source_id = source_id + self.relationship_type = relationship_type diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_report_fetch_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_report_fetch_error.py new file mode 100644 index 0000000000..62ab8db82c --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_report_fetch_error.py @@ -0,0 +1,36 @@ +"""Exception for errors when fetching reports from Google Threat Intelligence API.""" + +from typing import Any, Dict, Optional + +from connector.src.custom.exceptions.fetch_errors.gti_api_error import GTIApiError + + +class GTIReportFetchError(GTIApiError): + """Exception raised when there's an error fetching reports from GTI API.""" + + def __init__( + self, + message: str, + report_id: Optional[str] = None, + endpoint: Optional[str] = None, + status_code: Optional[int] = None, + details: Optional[Dict[str, Any]] = None, + ): + """Initialize the exception. + + Args: + message: Error message + report_id: ID of the report that failed to fetch, if applicable + endpoint: API endpoint where the error occurred + status_code: HTTP status code, if available + details: Additional details about the error + + """ + error_msg = message + if report_id: + error_msg = f"Error fetching report {report_id}: {message}" + else: + error_msg = f"Error fetching reports: {message}" + + super().__init__(error_msg, status_code, endpoint, details) + self.report_id = report_id diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_technique_fetch_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_technique_fetch_error.py new file mode 100644 index 0000000000..dbabdfdfa7 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_technique_fetch_error.py @@ -0,0 +1,36 @@ +"""Exception for errors when fetching attack techniques from Google Threat Intelligence API.""" + +from typing import Any, Dict, Optional + +from connector.src.custom.exceptions.fetch_errors.gti_api_error import GTIApiError + + +class GTITechniqueFetchError(GTIApiError): + """Exception raised when there's an error fetching attack techniques from GTI API.""" + + def __init__( + self, + message: str, + technique_id: Optional[str] = None, + endpoint: Optional[str] = None, + status_code: Optional[int] = None, + details: Optional[Dict[str, Any]] = None, + ): + """Initialize the exception. + + Args: + message: Error message + technique_id: ID of the attack technique that failed to fetch, if applicable + endpoint: API endpoint where the error occurred + status_code: HTTP status code, if available + details: Additional details about the error + + """ + error_msg = message + if technique_id: + error_msg = f"Error fetching attack technique {technique_id}: {message}" + else: + error_msg = f"Error fetching attack techniques: {message}" + + super().__init__(error_msg, status_code, endpoint, details) + self.technique_id = technique_id diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_vulnerability_fetch_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_vulnerability_fetch_error.py new file mode 100644 index 0000000000..5285b2c61c --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/fetch_errors/gti_vulnerability_fetch_error.py @@ -0,0 +1,43 @@ +"""Exception for errors when fetching vulnerabilities from Google Threat Intelligence API.""" + +from typing import Any, Dict, Optional + +from connector.src.custom.exceptions.fetch_errors.gti_api_error import GTIApiError + + +class GTIVulnerabilityFetchError(GTIApiError): + """Exception raised when there's an error fetching vulnerabilities from GTI API.""" + + def __init__( + self, + message: str, + vulnerability_id: Optional[str] = None, + cve_id: Optional[str] = None, + endpoint: Optional[str] = None, + status_code: Optional[int] = None, + details: Optional[Dict[str, Any]] = None, + ): + """Initialize the exception. + + Args: + message: Error message + vulnerability_id: ID of the vulnerability that failed to fetch, if applicable + cve_id: CVE ID of the vulnerability, if available + endpoint: API endpoint where the error occurred + status_code: HTTP status code, if available + details: Additional details about the error + + """ + error_msg = message + if vulnerability_id and cve_id: + error_msg = f"Error fetching vulnerability {vulnerability_id} (CVE: {cve_id}): {message}" + elif vulnerability_id: + error_msg = f"Error fetching vulnerability {vulnerability_id}: {message}" + elif cve_id: + error_msg = f"Error fetching vulnerability with CVE {cve_id}: {message}" + else: + error_msg = f"Error fetching vulnerabilities: {message}" + + super().__init__(error_msg, status_code, endpoint, details) + self.vulnerability_id = vulnerability_id + self.cve_id = cve_id diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/gti_base_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/gti_base_error.py new file mode 100644 index 0000000000..5a907d1a12 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/gti_base_error.py @@ -0,0 +1,7 @@ +"""Base class for GTI exceptions.""" + + +class GTIBaseError(Exception): + """Base class for GTI exceptions.""" + + ... diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/gti_configuration_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/gti_configuration_error.py new file mode 100644 index 0000000000..e2d3a3f8f8 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/gti_configuration_error.py @@ -0,0 +1,9 @@ +"""Exception raised for errors in the configuration.""" + +from connector.src.custom.exceptions.gti_base_error import GTIBaseError + + +class GTIConfigurationError(GTIBaseError): + """Exception raised for errors in the configuration.""" + + ... diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/gti_converting_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/gti_converting_error.py new file mode 100644 index 0000000000..ddaca74d98 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/gti_converting_error.py @@ -0,0 +1,9 @@ +"""Exception raised for errors in the Converting.""" + +from connector.src.custom.exceptions.gti_base_error import GTIBaseError + + +class GTIConvertingError(GTIBaseError): + """Exception raised for errors in the Converting.""" + + ... diff --git a/external-import/google-ti-feeds/connector/src/custom/exceptions/gti_fetching_error.py b/external-import/google-ti-feeds/connector/src/custom/exceptions/gti_fetching_error.py new file mode 100644 index 0000000000..1787a90796 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/exceptions/gti_fetching_error.py @@ -0,0 +1,9 @@ +"""Exception raised for errors in the Fetching.""" + +from connector.src.custom.exceptions.gti_base_error import GTIBaseError + + +class GTIFetchingError(GTIBaseError): + """Exception raised for errors in the Fetching.""" + + ... diff --git a/external-import/google-ti-feeds/connector/src/custom/fetch_all.py b/external-import/google-ti-feeds/connector/src/custom/fetch_all.py new file mode 100644 index 0000000000..0ef0ccd716 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/fetch_all.py @@ -0,0 +1,384 @@ +"""Simple orchestrator for Google Threat Intelligence data fetching. + +This module provides an orchestrator approach to fetching data from the Google Threat Intelligence API. +It coordinates multiple specialized fetchers to fetch reports and their related entities efficiently. +""" + +import asyncio +import logging +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Tuple + +from connector.src.custom.batch_processor import BatchProcessor +from connector.src.custom.configs.gti_config import GTIConfig +from connector.src.custom.exceptions import ( + GTIApiError, + GTIFetchingError, + GTIParsingError, +) +from connector.src.custom.fetchers.entity_fetcher import EntityFetcher +from connector.src.custom.fetchers.report_fetcher import ReportFetcher +from connector.src.custom.models.gti_reports.gti_report_model import ( + GTIReportData, + GTIReportResponse, +) +from connector.src.utils.api_engine.api_client import ApiClient +from connector.src.utils.api_engine.exceptions.api_network_error import ApiNetworkError + + +class FetchAll: + """Orchestrator for Google Threat Intelligence data fetching. + + This class coordinates the fetching of reports and their related entities + using specialized fetchers for different entity types. It maintains the + same batch processing workflow but delegates the actual fetching to + specialized classes. + """ + + def __init__( + self, + gti_config: GTIConfig, + api_client: ApiClient, + state: Optional[Dict[str, str]] = None, + logger: Optional[logging.Logger] = None, + batch_processor: Optional[BatchProcessor] = None, + ): + """Initialize the GTI data orchestrator. + + Args: + gti_config: Configuration for accessing the GTI API + api_client: Client for making API requests + state: Dictionary for storing state between runs + batch_processor: Optional processor for handling batches of data + logger: Logger for logging messages + + """ + self.config = gti_config + self.api_client = api_client + self.state = state or {} + self.logger = logger or logging.getLogger(__name__) + self.batch_processor = batch_processor + + self.report_fetcher = ReportFetcher(gti_config, api_client, logger) + self.entity_fetcher = EntityFetcher(gti_config, api_client, logger) + + self.reports: List[GTIReportData] = [] + self.report_related_entities: Dict[str, Dict[str, List[Any]]] = {} + self.reports_with_complete_entities: set[str] = set() + self.current_page_reports: List[GTIReportData] = [] + self.all_stix_objects: List[Any] = [] + self.latest_modified_date: Optional[str] = None + + def _extract_endpoint_name(self, url: str) -> str: + """Extract a readable endpoint name from a URL. + + Args: + url: The URL to extract from + + Returns: + A simplified endpoint name for logging + + """ + try: + parts = url.split("/") + if len(parts) > 0: + last_part = parts[-1] + if "?" in last_part: + last_part = last_part.split("?")[0] + return last_part + return url + except Exception: + return url + + def _prepare_partial_results( + self, source: str + ) -> Tuple[List[GTIReportData], Dict[str, Dict[str, List[Any]]], Optional[str]]: + """Prepare partial results when processing is interrupted. + + Args: + source: Description of why partial results are being returned + + Returns: + Tuple containing filtered reports, related entities, and latest modified date + + """ + complete_reports = [ + report + for report in self.reports + if report.id in self.reports_with_complete_entities + ] + self.logger.info( + f"[{source}] Returning {len(complete_reports)} complete reports out of {len(self.reports)} total fetched" + ) + + complete_related_entities = { + report_id: entities + for report_id, entities in self.report_related_entities.items() + if report_id in self.reports_with_complete_entities + } + + if complete_reports: + try: + complete_reports.sort( + key=lambda x: ( + x.last_modification_date + if hasattr(x, "last_modification_date") + and x.last_modification_date + else "" + ), + reverse=True, + ) + except Exception as sort_err: + self.logger.warning(f"Could not sort reports: {str(sort_err)}") + + return complete_reports, complete_related_entities, self.latest_modified_date + + async def fetch_all_data( + self, + ) -> Tuple[List[Any], Dict[str, Dict[str, List[Any]]], Optional[str]]: + """Fetch all GTI data using a batch processing workflow. + + This method orchestrates the fetching process: + 1. Fetches reports in batches using ReportFetcher + 2. For each batch, fetches related entities using EntityFetcher + 3. Converts each batch to STIX if needed + 4. Continues to the next batch + + Returns: + Tuple containing: + - List of reports + - Dictionary mapping report IDs to their related entities + - Latest modification date (ISO format) of successfully processed reports + + Raises: + GTIFetchingError: Base class for all fetching errors + GTIApiError: If an API error occurs + GTIParsingError: If there's an error parsing API responses + ApiNetworkError: If a network connectivity issue occurs + asyncio.CancelledError: If the operation is cancelled + + """ + try: + self.logger.info("Starting to fetch GTI data using orchestrated fetchers") + + self.reports = [] + self.report_related_entities = {} + self.reports_with_complete_entities = set() + self.current_page_reports = [] + self.all_stix_objects = [] + self.latest_modified_date = None + + if not self.batch_processor: + from .convert_to_stix import ConvertToSTIX + + self.stix_converter = ConvertToSTIX( + tlp_level="amber", logger=self.logger + ) + + await self._fetch_reports_in_batches() + + self.logger.info( + f"Total reports processed: {len(self.reports_with_complete_entities)}" + ) + + self.reports.sort( + key=lambda x: ( + x.last_modification_date + if hasattr(x, "last_modification_date") and x.last_modification_date + else "" + ), + reverse=True, + ) + return self.reports, self.report_related_entities, self.latest_modified_date + + except asyncio.CancelledError: + self.logger.info("Fetch operation was cancelled") + return self._prepare_partial_results("Cancelled") + except ApiNetworkError as e: + self.logger.error(f"Network connectivity issue: {str(e)}") + GTIApiError(f"Network connectivity issue: {str(e)}", endpoint="multiple") + return self._prepare_partial_results("Network error") + except GTIFetchingError as e: + self.logger.error(f"GTI fetch error: {str(e)}") + return self._prepare_partial_results("Fetch error") + except Exception as e: + self.logger.error(f"Error fetching GTI data: {str(e)}") + GTIFetchingError(f"Unexpected error fetching GTI data: {str(e)}") + return self._prepare_partial_results("Exception") + + async def _fetch_reports_in_batches(self) -> None: + """Orchestrate the batch fetching of reports. + + This method uses the ReportFetcher to handle the pagination and + calls the processing function for each page of reports. + """ + try: + await self.report_fetcher.fetch_reports_in_batches( + state=self.state, process_func=self._process_report_page + ) + + total_reports = len( + [r for r in self.reports if r.id in self.reports_with_complete_entities] + ) + self.logger.info(f"Fetched and processed {total_reports} reports") + + except Exception as e: + self.logger.error(f"Error in batch report fetching: {str(e)}") + raise + + async def _process_report_page(self, response: GTIReportResponse) -> None: + """Process a page of report data and trigger entity fetching. + + Args: + response: The API response containing report data + + Raises: + GTIParsingError: If there's an error parsing the report data + + """ + try: + current_page_reports = await self.report_fetcher.process_report_page( + response + ) + + if not current_page_reports: + return + + fetcher_latest_date = self.report_fetcher.get_latest_modified_date() + if fetcher_latest_date: + if ( + not self.latest_modified_date + or fetcher_latest_date > self.latest_modified_date + ): + self.latest_modified_date = fetcher_latest_date + + self.current_page_reports = current_page_reports + + await self._process_current_page_reports() + + except GTIParsingError: + raise + except Exception as e: + raise GTIParsingError( + f"Failed to process report page: {str(e)}", + entity_type="report", + endpoint="/collections", + ) from e + + async def _process_current_page_reports(self) -> None: + """Process the current page of reports. + + This method implements the batch processing workflow using the EntityFetcher: + 1. Process a page of reports (typically 40 reports) + 2. Fetch all sub-entities for those reports using EntityFetcher + 3. Convert those reports and their sub-entities to STIX + 4. Move to the next page and repeat until all data is processed + """ + if not self.current_page_reports: + return + + self.logger.info( + f"Processing batch of {len(self.current_page_reports)} reports" + ) + page_start_time = datetime.now() + batch_reports = [] + batch_report_related_entities = {} + + for i, report in enumerate(self.current_page_reports): + await asyncio.sleep(0.01) + report_id = report.id + + self.logger.info( + f"Processing report {i + 1}/{len(self.current_page_reports)} in batch - ID: {report_id}" + ) + + self.report_related_entities[report_id] = { + "malware_families": [], + "threat_actors": [], + "attack_techniques": [], + "vulnerabilities": [], + } + + try: + related_entities = ( + await self.entity_fetcher.fetch_report_related_entities( + report, i + 1, len(self.current_page_reports) + ) + ) + + self.report_related_entities[report_id].update(related_entities) + batch_report_related_entities[report_id] = related_entities.copy() + + self.reports_with_complete_entities.add(report_id) + batch_reports.append(report) + + except Exception as e: + self.logger.error( + f"Failed to fetch entities for report {report_id}: {str(e)}" + ) + continue + + if self.batch_processor: + self.logger.info(f"Using batch processor for {len(batch_reports)} reports") + + if self.latest_modified_date: + self.batch_processor.set_latest_modified_date(self.latest_modified_date) + + self.batch_processor.process_batch( + reports=batch_reports, related_entities=batch_report_related_entities + ) + + latest_date = self.batch_processor.get_latest_modified_date() + if latest_date: + if ( + not self.latest_modified_date + or latest_date > self.latest_modified_date + ): + self.latest_modified_date = latest_date + else: + self.logger.info( + f"Converting batch of {len(batch_reports)} reports to STIX" + ) + stix_objects = self.stix_converter.convert_all_data( + reports=batch_reports, related_entities=batch_report_related_entities + ) + self.all_stix_objects.extend(stix_objects) + self.logger.info( + f"Generated {len(stix_objects)} STIX objects for the batch" + ) + + for report in batch_reports: + if ( + hasattr(report, "last_modification_date") + and report.last_modification_date + ): + try: + report_date = datetime.fromisoformat( + report.last_modification_date.replace("Z", "+00:00") + ) + if ( + not self.latest_modified_date + or report_date + > datetime.fromisoformat( + self.latest_modified_date.replace("Z", "+00:00") + ) + ): + self.latest_modified_date = report_date.astimezone( + timezone.utc + ).isoformat() + except ValueError: + self.logger.warning( + f"Invalid date format in report: {report.last_modification_date}" + ) + + self.reports.extend(batch_reports) + + if self.latest_modified_date: + self.logger.info( + f"Current latest modification date: {self.latest_modified_date}" + ) + + batch_elapsed = datetime.now() - page_start_time + self.logger.info(f"Completed batch processing in {batch_elapsed}") + + self.current_page_reports = [] diff --git a/external-import/google-ti-feeds/connector/src/custom/fetchers/__init__.py b/external-import/google-ti-feeds/connector/src/custom/fetchers/__init__.py new file mode 100644 index 0000000000..66fad2ef36 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/fetchers/__init__.py @@ -0,0 +1,28 @@ +"""Fetchers package for Google Threat Intelligence API. + +This package provides a generic fetcher architecture for different types of entities +from the Google Threat Intelligence API, including reports, malware families, +threat actors, attack techniques, and vulnerabilities. + +The package uses a factory pattern with configurable generic fetchers to eliminate +code duplication while maintaining flexibility and performance. +""" + +from .base_fetcher import BaseFetcher +from .entity_config import ENTITY_CONFIGS, EntityFetcherConfig +from .entity_fetcher import EntityFetcher +from .fetcher_factory import FetcherFactory +from .generic_entity_fetcher import GenericEntityFetcher +from .relationship_fetcher import RelationshipFetcher +from .report_fetcher import ReportFetcher + +__all__ = [ + "BaseFetcher", + "RelationshipFetcher", + "ReportFetcher", + "EntityFetcher", + "GenericEntityFetcher", + "EntityFetcherConfig", + "FetcherFactory", + "ENTITY_CONFIGS", +] diff --git a/external-import/google-ti-feeds/connector/src/custom/fetchers/base_fetcher.py b/external-import/google-ti-feeds/connector/src/custom/fetchers/base_fetcher.py new file mode 100644 index 0000000000..e93bcf5e01 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/fetchers/base_fetcher.py @@ -0,0 +1,137 @@ +"""Base fetcher class with common functionality for Google Threat Intelligence API. + +This module provides a base class that contains shared functionality for all +specialized fetchers, including API client management, headers, logging, and +common utility methods. +""" + +import logging +from typing import Optional + +from connector.src.custom.configs.gti_config import GTIConfig +from connector.src.utils.api_engine.api_client import ApiClient + + +class BaseFetcher: + """Base class for all GTI data fetchers. + + This class provides common functionality shared across all specialized fetchers: + - API client and headers management + - Configuration access + - Logging utilities + - Common helper methods + """ + + def __init__( + self, + gti_config: GTIConfig, + api_client: ApiClient, + logger: Optional[logging.Logger] = None, + ): + """Initialize the base fetcher. + + Args: + gti_config: Configuration for accessing the GTI API + api_client: Client for making API requests + logger: Logger for logging messages + + """ + self.config = gti_config + self.api_client = api_client + self.logger = logger or logging.getLogger(__name__) + self.headers = { + "X-Apikey": self.config.api_key, + "accept": "application/json", + } + + def _extract_endpoint_name(self, url: str) -> str: + """Extract a readable endpoint name from a URL. + + Args: + url: The URL to extract from + + Returns: + A simplified endpoint name for logging + + """ + try: + parts = url.split("/") + if len(parts) > 0: + last_part = parts[-1] + if "?" in last_part: + last_part = last_part.split("?")[0] + return last_part + return url + except Exception: + return url + + def _log_fetch_start( + self, + entity_type: str, + entity_id: Optional[str] = None, + report_id: Optional[str] = None, + ) -> None: + """Log the start of a fetch operation. + + Args: + entity_type: Type of entity being fetched + entity_id: ID of the specific entity (optional) + report_id: ID of the related report (optional) + + """ + if report_id: + self.logger.info(f"Fetching {entity_type} for report {report_id}...") + elif entity_id: + self.logger.info(f"Fetching {entity_type} {entity_id}...") + else: + self.logger.info(f"Fetching {entity_type}...") + + def _log_fetch_result( + self, entity_type: str, count: int, report_id: Optional[str] = None + ) -> None: + """Log the result of a fetch operation. + + Args: + entity_type: Type of entity that was fetched + count: Number of entities fetched + report_id: ID of the related report (optional) + + """ + if count > 0: + if report_id: + self.logger.info( + f"Fetched {count} {entity_type} for report {report_id}" + ) + else: + self.logger.info(f"Fetched {count} {entity_type}") + else: + if report_id: + self.logger.debug(f"No {entity_type} found for report {report_id}") + else: + self.logger.debug(f"No {entity_type} found") + + def _log_error( + self, + message: str, + entity_type: Optional[str] = None, + entity_id: Optional[str] = None, + error: Optional[Exception] = None, + ) -> None: + """Log an error with appropriate context. + + Args: + message: Error message + entity_type: Type of entity related to the error (optional) + entity_id: ID of the entity related to the error (optional) + error: The exception that occurred (optional) + + """ + meta = {} + if error: + meta["error"] = str(error) + if entity_id: + meta["entity_id"] = entity_id + if entity_type: + meta["entity_type"] = entity_type + + self.logger.error(message, meta=meta if meta else None) # type: ignore[call-arg] diff --git a/external-import/google-ti-feeds/connector/src/custom/fetchers/entity_config.py b/external-import/google-ti-feeds/connector/src/custom/fetchers/entity_config.py new file mode 100644 index 0000000000..520c4c34ef --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/fetchers/entity_config.py @@ -0,0 +1,91 @@ +"""Configuration classes for generic entity fetchers. + +This module defines configuration classes that specify the differences between +various entity types (malware families, threat actors, etc.) allowing for +a single generic fetcher implementation. +""" + +from dataclasses import dataclass +from typing import Any, Type + +from connector.src.custom.exceptions import ( + GTIActorFetchError, + GTIApiError, + GTIMalwareFetchError, + GTITechniqueFetchError, + GTIVulnerabilityFetchError, +) +from connector.src.custom.models.gti_reports.gti_attack_technique_model import ( + GTIAttackTechniqueResponse, +) +from connector.src.custom.models.gti_reports.gti_malware_family_model import ( + GTIMalwareFamilyResponse, +) +from connector.src.custom.models.gti_reports.gti_threat_actor_model import ( + GTIThreatActorResponse, +) +from connector.src.custom.models.gti_reports.gti_vulnerability_model import ( + GTIVulnerabilityResponse, +) + + +@dataclass +class EntityFetcherConfig: + """Configuration for a specific entity type fetcher.""" + + entity_type: str + relationship_type: str + endpoint_template: str + response_model: Type[Any] + exception_class: Type[GTIApiError] + display_name: str + display_name_singular: str + + +MALWARE_FAMILY_CONFIG = EntityFetcherConfig( + entity_type="malware_families", + relationship_type="malware_families", + endpoint_template="/collections/{entity_id}", + response_model=GTIMalwareFamilyResponse, + exception_class=GTIMalwareFetchError, + display_name="malware families", + display_name_singular="malware family", +) + +THREAT_ACTOR_CONFIG = EntityFetcherConfig( + entity_type="threat_actors", + relationship_type="threat_actors", + endpoint_template="/collections/{entity_id}", + response_model=GTIThreatActorResponse, + exception_class=GTIActorFetchError, + display_name="threat actors", + display_name_singular="threat actor", +) + +ATTACK_TECHNIQUE_CONFIG = EntityFetcherConfig( + entity_type="attack_techniques", + relationship_type="attack_techniques", + endpoint_template="/attack_techniques/{entity_id}", + response_model=GTIAttackTechniqueResponse, + exception_class=GTITechniqueFetchError, + display_name="attack techniques", + display_name_singular="attack technique", +) + +VULNERABILITY_CONFIG = EntityFetcherConfig( + entity_type="vulnerabilities", + relationship_type="vulnerabilities", + endpoint_template="/collections/{entity_id}", + response_model=GTIVulnerabilityResponse, + exception_class=GTIVulnerabilityFetchError, + display_name="vulnerabilities", + display_name_singular="vulnerability", +) + + +ENTITY_CONFIGS = { + "malware_families": MALWARE_FAMILY_CONFIG, + "threat_actors": THREAT_ACTOR_CONFIG, + "attack_techniques": ATTACK_TECHNIQUE_CONFIG, + "vulnerabilities": VULNERABILITY_CONFIG, +} diff --git a/external-import/google-ti-feeds/connector/src/custom/fetchers/entity_fetcher.py b/external-import/google-ti-feeds/connector/src/custom/fetchers/entity_fetcher.py new file mode 100644 index 0000000000..0ca97696c6 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/fetchers/entity_fetcher.py @@ -0,0 +1,188 @@ +"""Entity fetcher orchestrator for Google Threat Intelligence API. +This module provides a unified interface for fetching all types of entities +related to reports by using the factory pattern efficiently. +""" + +import asyncio +import logging +from typing import Any, Dict, List, Optional + +from connector.src.custom.configs.gti_config import GTIConfig +from connector.src.custom.exceptions import GTIRelationshipFetchError +from connector.src.custom.fetchers.base_fetcher import BaseFetcher +from connector.src.custom.fetchers.fetcher_factory import FetcherFactory +from connector.src.custom.models.gti_reports.gti_report_model import GTIReportData +from connector.src.utils.api_engine.api_client import ApiClient + + +class EntityFetcher(BaseFetcher): + """Orchestrator for fetching all types of entities related to reports.""" + + def __init__( + self, + gti_config: GTIConfig, + api_client: ApiClient, + logger: Optional[logging.Logger] = None, + ): + """Initialize the entity fetcher with all generic fetchers via factory.""" + super().__init__(gti_config, api_client, logger) + + self.entity_fetchers = FetcherFactory.create_all_entity_fetchers( + gti_config, api_client, logger + ) + + async def fetch_report_related_entities( + self, report: GTIReportData, report_index: int = 0, total_reports: int = 0 + ) -> Dict[str, List[Any]]: + """Fetch all entities related to a specific report. + + Args: + report: The report for which to fetch related entities + report_index: Current report index (for progress tracking) + total_reports: Total number of reports (for progress tracking) + + Returns: + Dictionary containing all related entities organized by type + Raises: + GTIRelationshipFetchError: If there's an error fetching related entities + + """ + report_id = report.id + progress_info = ( + f"({report_index}/{total_reports} reports) " if total_reports > 0 else "" + ) + self.logger.info( + f"{progress_info}Fetching related entities for report {report_id}..." + ) + + related_entities: Dict[str, List[Any]] = { + entity_type: [] for entity_type in self.entity_fetchers.keys() + } + try: + self.logger.info( + f"Processing {len(self.entity_fetchers)} entity types for report {report_id}" + ) + related_entities = { + entity_type: [] for entity_type in self.entity_fetchers.keys() + } + + for entity_type, fetcher in self.entity_fetchers.items(): + self.logger.info(f"Fetching {entity_type} for report {report_id}") + try: + result = await fetcher.fetch_entities(report) + related_entities[entity_type] = result + self.logger.info( + f"Success: {entity_type} for report {report_id} - got {len(result)} entities" + ) + except Exception as fetch_ex: + self.logger.error( + f"Failed to fetch {entity_type} for report {report_id}: {fetch_ex}" + ) + related_entities[entity_type] = [] + + total_entities = sum( + len(entities) for entities in related_entities.values() + ) + summary_parts = [ + f"{entity_type.replace('_', ' ')}: {len(entities)}" + for entity_type, entities in related_entities.items() + ] + summary = ", ".join(summary_parts) + self.logger.info( + f"Successfully fetched {total_entities} related entities for report {report_id} " + f"({summary})" + ) + return related_entities + except asyncio.CancelledError: + self.logger.info(f"Entity fetch cancelled for report {report_id}") + raise + except Exception as e: + self._log_error( + f"Error fetching related entities for report {report_id}: {str(e)}", + entity_type="report_entities", + entity_id=report_id, + error=e, + ) + raise GTIRelationshipFetchError( + f"Failed to fetch related entities: {str(e)}", + source_id=report_id, + relationship_type="report_entities", + ) from e + + def get_supported_entity_types(self) -> List[str]: + """Get list of supported entity types. + + Returns: + List of entity type names that this fetcher supports + + """ + return list(self.entity_fetchers.keys()) + + async def fetch_specific_entity_types( + self, report: GTIReportData, entity_types: List[str] + ) -> Dict[str, List[Any]]: + """Fetch only specific types of entities for a report. + + Args: + report: The report for which to fetch entities + entity_types: List of entity types to fetch + Returns: + Dictionary containing requested entities organized by type + Raises: + ValueError: If any requested entity type is not supported + GTIRelationshipFetchError: If there's an error fetching entities + + """ + unsupported_types = set(entity_types) - set(self.entity_fetchers.keys()) + if unsupported_types: + raise ValueError( + f"Unsupported entity types: {unsupported_types}. " + f"Supported types: {list(self.entity_fetchers.keys())}" + ) + report_id = report.id + self.logger.info( + f"Fetching specific entity types {entity_types} for report {report_id}..." + ) + + related_entities: Dict[str, List[Any]] = { + entity_type: [] for entity_type in entity_types + } + try: + for entity_type in entity_types: + try: + self.logger.info(f"Fetching {entity_type} for report {report_id}") + result = await self.entity_fetchers[entity_type].fetch_entities( + report + ) + related_entities[entity_type] = result + self.logger.info( + f"Success: {entity_type} for report {report_id} - got {len(result)} entities" + ) + except Exception as fetch_ex: + self.logger.error( + f"Failed to fetch {entity_type} for report {report_id}: {fetch_ex}" + ) + related_entities[entity_type] = [] + + total_entities = sum( + len(entities) for entities in related_entities.values() + ) + self.logger.info( + f"Fetched {total_entities} entities of requested types for report {report_id}" + ) + return related_entities + except asyncio.CancelledError: + self.logger.info(f"Specific entity fetch cancelled for report {report_id}") + raise + except Exception as e: + self._log_error( + f"Error fetching specific entity types {entity_types} for report {report_id}: {str(e)}", + entity_type="specific_entities", + entity_id=report_id, + error=e, + ) + raise GTIRelationshipFetchError( + f"Failed to fetch specific entity types: {str(e)}", + source_id=report_id, + relationship_type=f"specific_entities_{entity_types}", + ) from e diff --git a/external-import/google-ti-feeds/connector/src/custom/fetchers/fetcher_factory.py b/external-import/google-ti-feeds/connector/src/custom/fetchers/fetcher_factory.py new file mode 100644 index 0000000000..125bb63473 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/fetchers/fetcher_factory.py @@ -0,0 +1,87 @@ +"""Fetcher factory for creating entity fetchers. + +This module provides a factory class for creating entity fetchers with the +appropriate configuration, eliminating the need for separate specialized +fetcher classes. +""" + +import logging +from typing import Dict, Optional + +from connector.src.custom.configs.gti_config import GTIConfig +from connector.src.custom.fetchers.entity_config import ( + ENTITY_CONFIGS, +) +from connector.src.custom.fetchers.generic_entity_fetcher import GenericEntityFetcher +from connector.src.utils.api_engine.api_client import ApiClient + + +class FetcherFactory: + """Factory for creating entity fetchers with appropriate configuration.""" + + @classmethod + def create_entity_fetcher( + cls, + entity_type: str, + gti_config: GTIConfig, + api_client: ApiClient, + logger: Optional[logging.Logger] = None, + ) -> GenericEntityFetcher: + """Create an entity fetcher for the specified type. + + Args: + entity_type: Type of entity to create fetcher for (e.g., "malware_families") + gti_config: Configuration for accessing the GTI API + api_client: Client for making API requests + logger: Logger for logging messages + + Returns: + Configured generic entity fetcher for the specified entity type + + Raises: + ValueError: If the entity type is not supported + + """ + if entity_type not in ENTITY_CONFIGS: + available_types = ", ".join(ENTITY_CONFIGS.keys()) + raise ValueError( + f"Unsupported entity type '{entity_type}'. Available types: {available_types}" + ) + + config = ENTITY_CONFIGS[entity_type] + return GenericEntityFetcher(config, gti_config, api_client, logger) + + @classmethod + def get_available_entity_types(cls) -> list[str]: + """Get list of available entity types. + + Returns: + List of supported entity type names + + """ + return list(ENTITY_CONFIGS.keys()) + + @classmethod + def create_all_entity_fetchers( + cls, + gti_config: GTIConfig, + api_client: ApiClient, + logger: Optional[logging.Logger] = None, + ) -> Dict[str, GenericEntityFetcher]: + """Create fetchers for all supported entity types. + + Args: + gti_config: Configuration for accessing the GTI API + api_client: Client for making API requests + logger: Logger for logging messages + + Returns: + Dictionary mapping entity type names to configured fetchers + + """ + return { + entity_type: cls.create_entity_fetcher( + entity_type, gti_config, api_client, logger + ) + for entity_type in ENTITY_CONFIGS.keys() + } diff --git a/external-import/google-ti-feeds/connector/src/custom/fetchers/generic_entity_fetcher.py b/external-import/google-ti-feeds/connector/src/custom/fetchers/generic_entity_fetcher.py new file mode 100644 index 0000000000..932822eacb --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/fetchers/generic_entity_fetcher.py @@ -0,0 +1,175 @@ +"""Generic entity fetcher for Google Threat Intelligence API. +This module provides a generic implementation for fetching entities +related to reports from the Google Threat Intelligence API. It replaces +the specialized fetchers with a configurable generic approach. +""" + +import logging +from typing import Any, List, Optional + +from connector.src.custom.configs.gti_config import GTIConfig +from connector.src.custom.exceptions import GTIRelationshipFetchError +from connector.src.custom.fetchers.base_fetcher import BaseFetcher +from connector.src.custom.fetchers.entity_config import EntityFetcherConfig +from connector.src.custom.fetchers.relationship_fetcher import RelationshipFetcher +from connector.src.custom.models.gti_reports.gti_report_model import GTIReportData +from connector.src.utils.api_engine.api_client import ApiClient +from connector.src.utils.api_engine.exceptions.api_network_error import ApiNetworkError + + +class GenericEntityFetcher(BaseFetcher): + """Generic fetcher for any type of entity related to reports.""" + + def __init__( + self, + config: EntityFetcherConfig, + gti_config: GTIConfig, + api_client: ApiClient, + logger: Optional[logging.Logger] = None, + ): + """Initialize the generic entity fetcher. + + Args: + config: Configuration specifying entity type, endpoints, models, etc. + gti_config: Configuration for accessing the GTI API + api_client: Client for making API requests + logger: Logger for logging messages + + """ + super().__init__(gti_config, api_client, logger) + self.entity_config = config + self.relationship_fetcher = RelationshipFetcher(gti_config, api_client, logger) + + async def fetch_entities(self, report: GTIReportData) -> List[Any]: + """Fetch entities of the configured type related to a report. + + Args: + report: The report for which to fetch entities + Returns: + List of entity objects + Raises: + Configured exception class: If there's an error fetching entities + + """ + report_id = report.id + entities = [] + self._log_fetch_start(self.entity_config.display_name, report_id=report_id) + try: + self.logger.info( + f"Fetching {self.entity_config.entity_type} IDs with relationship_type={self.entity_config.relationship_type} for report {report_id}" + ) + entity_ids = await self.relationship_fetcher.fetch_relationship_ids( + report_id=report_id, + relationship_type=self.entity_config.relationship_type, + ) + + if entity_ids: + self.logger.info( + f"Got {len(entity_ids)} {self.entity_config.entity_type} IDs to fetch: {entity_ids}" + ) + self.logger.info( + f"Starting sequential API requests for {len(entity_ids)} {self.entity_config.entity_type} entities" + ) + for idx, entity_id in enumerate(entity_ids): + try: + result = await self._fetch_single_entity(entity_id) + if result is not None: + self.logger.debug( + f"Successfully fetched {self.entity_config.entity_type} #{idx + 1}/{len(entity_ids)}: {entity_id}" + ) + entities.append(result) + else: + self.logger.warning( + f"No data returned for {self.entity_config.display_name_singular} {entity_id}" + ) + except Exception as e: + self.logger.warning( + f"Failed to fetch {self.entity_config.display_name_singular} {entity_id}: {e}" + ) + self._log_fetch_result( + self.entity_config.display_name, len(entities), report_id + ) + self.logger.info( + f"Completed fetching {self.entity_config.entity_type} - Success: {len(entities)}/{len(entity_ids) if entity_ids else 0} entities" + ) + return entities + except GTIRelationshipFetchError as rel_err: + endpoint = f"{self.config.api_url}/collections/{report_id}/{self.entity_config.entity_type}" + error_msg = f"Failed to fetch {self.entity_config.display_name_singular} relationship IDs: {str(rel_err)}" + + try: + exception = self.entity_config.exception_class( + error_msg, endpoint=endpoint + ) + raise exception from rel_err + except TypeError: + exception = self.entity_config.exception_class(error_msg) + raise exception from rel_err + except (ApiNetworkError, self.entity_config.exception_class): + raise + except Exception as e: + endpoint = f"{self.config.api_url}/collections/{report_id}/{self.entity_config.entity_type}" + error_msg = ( + f"Unexpected error fetching {self.entity_config.display_name}: {str(e)}" + ) + + try: + exception = self.entity_config.exception_class( + error_msg, endpoint=endpoint + ) + raise exception from e + except TypeError: + exception = self.entity_config.exception_class(error_msg) + raise exception from e + + async def _fetch_single_entity(self, entity_id: str) -> Optional[Any]: + """Fetch a single entity by ID. + + Args: + entity_id: ID of the entity to fetch + Returns: + Entity data or None if fetch fails + Raises: + Configured exception class: If there's an error fetching the entity + + """ + try: + endpoint = f"{self.config.api_url}{self.entity_config.endpoint_template.format(entity_id=entity_id)}" + self.logger.debug( + f"Fetching {self.entity_config.entity_type} entity {entity_id} from {endpoint}" + ) + response = await self.api_client.call_api( + url=endpoint, + headers=self.headers, + model=self.entity_config.response_model, + timeout=60, + ) + if response and hasattr(response, "data"): + self.logger.debug( + f"Successfully fetched {self.entity_config.entity_type} data for {entity_id}" + ) + return response.data + self.logger.warning( + f"Empty or invalid response for {self.entity_config.entity_type} {entity_id}" + ) + return None + except ApiNetworkError as net_err: + error_msg = f"Network error fetching {self.entity_config.entity_type} {entity_id}: {str(net_err)}" + self.logger.error( + f"Network error at {endpoint} for {self.entity_config.entity_type} {entity_id}: {str(net_err)}" + ) + + exception = self.entity_config.exception_class(error_msg) + raise exception from net_err + except Exception as e: + error_msg = f"Error fetching {self.entity_config.display_name_singular} {entity_id}: {str(e)}" + self._log_error( + error_msg, + entity_type=self.entity_config.entity_type.rstrip("s"), + entity_id=entity_id, + error=e, + ) + self.logger.error( + f"Failed to fetch {self.entity_config.entity_type} {entity_id} from {endpoint}: {str(e)}" + ) + return None diff --git a/external-import/google-ti-feeds/connector/src/custom/fetchers/relationship_fetcher.py b/external-import/google-ti-feeds/connector/src/custom/fetchers/relationship_fetcher.py new file mode 100644 index 0000000000..e7553696b1 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/fetchers/relationship_fetcher.py @@ -0,0 +1,108 @@ +"""Relationship fetcher for Google Threat Intelligence API. + +This module provides functionality to fetch relationship IDs between reports +and their related entities (malware families, threat actors, attack techniques, vulnerabilities). +""" + +from typing import List + +from connector.src.custom.exceptions import GTIRelationshipFetchError +from connector.src.custom.fetchers.base_fetcher import BaseFetcher +from connector.src.utils.api_engine.exceptions.api_network_error import ApiNetworkError + + +class RelationshipFetcher(BaseFetcher): + """Fetcher for relationship IDs between reports and their related entities.""" + + async def fetch_relationship_ids( + self, report_id: str, relationship_type: str + ) -> List[str]: + """Fetch IDs of entities related to a report through a specific relationship. + + Args: + report_id: ID of the report + relationship_type: Type of relationship (e.g., "malware_families", "threat_actors") + + Returns: + List of entity IDs + + Raises: + GTIRelationshipFetchError: If there's an error fetching relationship IDs + + """ + entity_ids = [] + endpoint = f"{self.config.api_url}/collections/{report_id}/relationships/{relationship_type}" + + try: + self.logger.info( + f"Fetching {relationship_type} for report {report_id} from {endpoint}" + ) + + response = await self.api_client.call_api( + url=endpoint, + headers=self.headers, + params={"limit": 40}, + timeout=60, + ) + + self.logger.debug( + f"Response for {relationship_type} ({report_id}): {response}" + ) + + if response and isinstance(response, dict) and "data" in response: + data_length = ( + len(response["data"]) if isinstance(response["data"], list) else 0 + ) + self.logger.info( + f"API returned {data_length} items in data array for {relationship_type} ({report_id})" + ) + + for item in response["data"]: + if isinstance(item, dict) and "id" in item: + entity_type = item.get("type", "unknown") + entity_ids.append(item["id"]) + self.logger.debug( + f"Found entity ID: {item['id']} (type: {entity_type}) for {relationship_type} ({report_id})" + ) + + if len(entity_ids) > 0: + self.logger.info( + f"Found {len(entity_ids)} {relationship_type} for report {report_id}: {entity_ids}" + ) + return entity_ids + return [] + + except ApiNetworkError as net_err: + error_msg = f"Network error fetching {relationship_type} IDs for report {report_id}: {str(net_err)}" + self._log_error( + error_msg, + entity_type=relationship_type, + entity_id=report_id, + error=net_err, + ) + self.logger.error( + f"Network failure for {relationship_type} ({report_id}) at {endpoint}: {str(net_err)}" + ) + raise GTIRelationshipFetchError( + f"Network error: {str(net_err)}", + source_id=report_id, + relationship_type=relationship_type, + endpoint=endpoint, + ) from net_err + except Exception as e: + error_msg = f"Error fetching {relationship_type} IDs for report {report_id}: {str(e)}" + self._log_error( + error_msg, + entity_type=relationship_type, + entity_id=report_id, + error=e, + ) + self.logger.error( + f"General error for {relationship_type} ({report_id}) at {endpoint}: {str(e)}" + ) + raise GTIRelationshipFetchError( + f"Failed to fetch relationship IDs: {str(e)}", + source_id=report_id, + relationship_type=relationship_type, + endpoint=endpoint, + ) from e diff --git a/external-import/google-ti-feeds/connector/src/custom/fetchers/report_fetcher.py b/external-import/google-ti-feeds/connector/src/custom/fetchers/report_fetcher.py new file mode 100644 index 0000000000..d23ac5152b --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/fetchers/report_fetcher.py @@ -0,0 +1,346 @@ +"""Report fetcher for Google Threat Intelligence API. + +This module provides functionality to fetch reports from the Google Threat Intelligence API +with pagination support and batch processing capabilities. +""" + +import asyncio +import logging +from datetime import datetime, timezone +from typing import Any, Callable, Dict, List, Optional + +import isodate # type: ignore[import-untyped] +from connector.src.custom.configs.gti_config import GTIConfig +from connector.src.custom.exceptions import ( + GTIApiError, + GTIPaginationError, + GTIParsingError, + GTIReportFetchError, +) +from connector.src.custom.fetchers.base_fetcher import BaseFetcher +from connector.src.custom.models.gti_reports.gti_report_model import ( + GTIReportData, + GTIReportResponse, +) +from connector.src.utils.api_engine.api_client import ApiClient +from connector.src.utils.api_engine.exceptions.api_network_error import ApiNetworkError + + +class ReportFetcher(BaseFetcher): + """Fetcher for report entities with pagination support.""" + + def __init__( + self, + gti_config: GTIConfig, + api_client: ApiClient, + logger: Optional[logging.Logger] = None, + ): + """Initialize the report fetcher. + + Args: + gti_config: Configuration for accessing the GTI API + api_client: Client for making API requests + logger: Logger for logging messages + + """ + super().__init__(gti_config, api_client, logger) + self.current_page_reports: List[GTIReportData] = [] + self.latest_modified_date: Optional[str] = None + + async def fetch_reports_in_batches( + self, state: Dict[str, str], process_func: Callable[[Any], Any] + ) -> None: + """Fetch reports from the GTI API in batches and process them. + + Args: + state: Dictionary containing state information (e.g., last_report_date) + process_func: Function to process each page of reports + + Raises: + GTIReportFetchError: If there's an error fetching reports + + """ + try: + start_date_iso_8601 = self.config.import_start_date + duration = isodate.parse_duration(start_date_iso_8601) + past_date = datetime.now() - duration + start_date = past_date.strftime("%Y-%m-%dT%H:%M:%S") + + last_mod_date = state.get("last_report_date") + if last_mod_date: + start_date = datetime.fromisoformat(last_mod_date).strftime( + "%Y-%m-%dT%H:%M:%S" + ) + + filters = f"collection_type:report last_modification_date:{start_date}+" + + report_types = self.config.report_types + origins = self.config.origins + + if (not report_types or "All" in report_types) and ( + not origins or "All" in origins + ): + params = { + "filter": filters, + "limit": 40, + "order": "last_modification_date+", + } + self.logger.info( + f"Fetching all reports from GTI API (from {start_date})" + ) + await self._fetch_paginated_data( + endpoint=f"{self.config.api_url}/collections", + params=params, + model=GTIReportResponse, + process_func=process_func, + batch_process=True, + ) + else: + for report_type in report_types: + for origin in origins: + if origin == "All": + report_filter = f'{filters} report_type:"{report_type}"' + else: + report_filter = ( + f'{filters} report_type:"{report_type}" origin:{origin}' + ) + params = { + "filter": report_filter, + "limit": 40, + "order": "last_modification_date+", + } + + self.logger.info( + f"Fetching reports with type={report_type}, origin={origin} (from {start_date})" + ) + + await self._fetch_paginated_data( + endpoint=f"{self.config.api_url}/collections", + params=params, + model=GTIReportResponse, + process_func=process_func, + batch_process=True, + ) + + except ApiNetworkError as e: + raise GTIReportFetchError( + f"Network error fetching reports: {str(e)}", + endpoint=f"{self.config.api_url}/collections", + ) from e + except GTIReportFetchError: + raise + except Exception as e: + raise GTIReportFetchError( + f"Failed to fetch reports: {str(e)}", + endpoint=f"{self.config.api_url}/collections", + ) from e + + async def process_report_page( + self, response: GTIReportResponse + ) -> List[GTIReportData]: + """Process a page of report data. + + Args: + response: The API response containing report data + + Returns: + List of processed reports from the current page + + Raises: + GTIParsingError: If there's an error parsing the report data + + """ + try: + if not hasattr(response, "data") or not response.data: + self.logger.warning("Received empty response data") + return [] + + items_in_page = len(response.data) + current_page_reports = [] + + for report in response.data: + current_page_reports.append(report) + + if ( + hasattr(report, "last_modification_date") + and report.last_modification_date + ): + try: + report_date = datetime.fromisoformat( + report.last_modification_date.replace("Z", "+00:00") + ) + + if ( + not self.latest_modified_date + or report_date + > datetime.fromisoformat( + self.latest_modified_date.replace("Z", "+00:00") + ) + ): + self.latest_modified_date = report_date.astimezone( + timezone.utc + ).isoformat() + except ValueError as ve: + raise GTIParsingError( + f"Invalid date format: {report.last_modification_date}", + entity_type="report", + data_sample=report.last_modification_date, + ) from ve + + self.current_page_reports = current_page_reports + + self.logger.info(f"Processed page with {items_in_page} reports") + return current_page_reports + except GTIParsingError: + raise + except Exception as e: + raise GTIParsingError( + f"Failed to process report page: {str(e)}", + entity_type="report", + endpoint="/collections", + ) from e + + async def _fetch_paginated_data( + self, + endpoint: str, + params: Dict[str, Any], + model: Any, + process_func: Callable[[Any], Any], + batch_process: bool = False, + ) -> None: + """Fetch paginated data from the API. + + Args: + endpoint: API endpoint to fetch data from + params: Query parameters + model: Model class for response data + process_func: Function to process each page of data + batch_process: Whether to process data in batches + + Raises: + GTIPaginationError: If there's an error with pagination + GTIApiError: If there's an error with the API call + GTIParsingError: If there's an error parsing the response + + """ + current_url = endpoint + current_params = params + + page_count = 0 + start_time = datetime.now() + endpoint_name = self._extract_endpoint_name(endpoint) + self.logger.info(f"Starting paginated data fetch from {endpoint_name}") + + while current_url: + page_count += 1 + + elapsed = datetime.now() - start_time + self.logger.info( + f"Fetching page {page_count} from {endpoint_name} (elapsed: {elapsed})" + ) + + try: + response = await self.api_client.call_api( + url=current_url, + headers=self.headers, + params=current_params, + model=model, + timeout=60, + ) + + items_count = 0 + if hasattr(response, "data"): + if isinstance(response.data, list): + items_count = len(response.data) + elif isinstance(response.data, dict) and "data" in response.data: + items_count = len(response.data["data"]) + elif isinstance(response, dict) and "data" in response: + if isinstance(response["data"], list): + items_count = len(response["data"]) + + current_endpoint = self._extract_endpoint_name(current_url) + self.logger.info( + f"Processing page {page_count} with {items_count} items (endpoint: {current_endpoint})" + ) + + try: + await process_func(response) + except GTIParsingError: + raise + except Exception as proc_err: + sample_data = str(response)[:200] if str(response) else "" + raise GTIParsingError( + f"Error processing page {page_count}: {str(proc_err)}", + endpoint=current_url, + data_sample=sample_data, + ) from proc_err + + try: + if ( + hasattr(response, "links") + and hasattr(response.links, "next") + and response.links.next + ): + current_url = response.links.next + current_params = {} + else: + break + except Exception as link_err: + raise GTIPaginationError( + f"Error extracting next page link: {str(link_err)}", + endpoint=current_url, + page=page_count, + ) from link_err + + except asyncio.CancelledError: + self.logger.info(f"Pagination fetch cancelled for {current_url}") + raise + except ApiNetworkError as net_err: + self._log_error( + f"Network error fetching data from {current_url}: {str(net_err)}", + error=net_err, + ) + raise GTIApiError( + f"Network error: {str(net_err)}", endpoint=current_url + ) from net_err + except GTIApiError: + raise + except GTIPaginationError: + raise + except Exception as e: + self._log_error( + f"Error fetching data from {current_url}: {str(e)}", error=e + ) + + if ( + "page" in str(e).lower() + or "next" in str(e).lower() + or "link" in str(e).lower() + ): + raise GTIPaginationError( + f"Pagination error: {str(e)}", + endpoint=current_url, + page=page_count, + ) from e + else: + raise GTIApiError( + f"API error: {str(e)}", endpoint=current_url + ) from e + + def get_latest_modified_date(self) -> Optional[str]: + """Get the latest modification date from processed reports. + + Returns: + Latest modification date in ISO format, or None if no reports processed + + """ + return self.latest_modified_date + + def get_current_page_reports(self) -> List[GTIReportData]: + """Get the current page of reports. + + Returns: + List of reports from the current page + + """ + return self.current_page_reports diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/__init__.py b/external-import/google-ti-feeds/connector/src/custom/mappers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/__init__.py b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_attack_technique_to_stix_attack_pattern.py b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_attack_technique_to_stix_attack_pattern.py new file mode 100644 index 0000000000..d98ea58b09 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_attack_technique_to_stix_attack_pattern.py @@ -0,0 +1,199 @@ +"""Converts a GTI attack technique to a STIX attack pattern object.""" + +from datetime import datetime +from typing import Dict, List, Optional + +from connector.src.custom.models.gti_reports.gti_attack_technique_model import ( + AttackTechniqueModel, + GTIAttackTechniqueData, +) +from connector.src.stix.octi.models.attack_pattern_model import OctiAttackPatternModel +from connector.src.stix.v21.models.cdts.kill_chain_phase_model import ( + KillChainPhaseModel, +) +from stix2.v21 import AttackPattern, Identity, MarkingDefinition # type: ignore + + +class GTIAttackTechniqueToSTIXAttackPattern: + """Converts a GTI attack technique to a STIX attack pattern object.""" + + def __init__( + self, + attack_technique: GTIAttackTechniqueData, + organization: Identity, + tlp_marking: MarkingDefinition, + ) -> None: + """Initialize the GTIAttackTechniqueToSTIXAttackPattern object. + + Args: + attack_technique (GTIAttackTechniqueData): The GTI attack technique data to convert. + organization (Identity): The organization identity object. + tlp_marking (MarkingDefinition): The TLP marking definition. + + """ + self.attack_technique = attack_technique + self.organization = organization + self.tlp_marking = tlp_marking + + def to_stix(self) -> AttackPattern: + """Convert the GTI attack technique to a STIX attack pattern object. + + Returns: + AttackPattern: The STIX attack pattern object. + + """ + if not self.attack_technique or not self.attack_technique.attributes: + raise ValueError("Attack technique attributes are missing") + + attributes = self.attack_technique.attributes + + created = datetime.fromtimestamp(attributes.creation_date) + modified = datetime.fromtimestamp(attributes.last_modification_date) + + aliases = self._extract_aliases(attributes) + kill_chain_phases = self._extract_kill_chain_phases(attributes) + first_seen, last_seen = None, None + labels = self._extract_labels(attributes) + external_references = self._create_external_references(attributes) + + attack_pattern_model = OctiAttackPatternModel.create( + name=attributes.name, + mitre_id=self.attack_technique.id, + organization_id=self.organization.id, + marking_ids=[self.tlp_marking.id], + description=attributes.description, + aliases=aliases, + first_seen=first_seen, + last_seen=last_seen, + kill_chain_phases=kill_chain_phases, + labels=labels, + external_references=external_references, + created=created, + modified=modified, + ) + + return attack_pattern_model.to_stix2_object() + + def _extract_aliases(self, attributes: AttackTechniqueModel) -> Optional[List[str]]: + """Extract aliases from attack technique attributes. + + Args: + attributes: The attack technique attributes + + Returns: + Optional[List[str]]: Extracted aliases or None if no aliases exist + + """ + if not attributes: + return None + return None + + def _extract_kill_chain_phases( + self, attributes: AttackTechniqueModel + ) -> Optional[List[KillChainPhaseModel]]: + """Extract kill chain phases from attack technique attributes. + + Args: + attributes: The attack technique attributes + + Returns: + Optional[List[KillChainPhaseModel]]: Extracted kill chain phases or None if no phases exist + + """ + if not attributes: + return None + return None + + def _normalize_tactic_name(self, tactic_name: str) -> str: + """Normalize tactic name to match MITRE ATT&CK format. + + Args: + tactic_name: The tactic name to normalize + + Returns: + str: Normalized tactic name + + """ + normalized = tactic_name.lower().replace(" ", "-") + return normalized + + def _extract_labels(self, attributes: AttackTechniqueModel) -> Optional[List[str]]: + """Extract labels from attack technique attributes. + + Args: + attributes: The attack technique attributes + + Returns: + Optional[List[str]]: Extracted labels or None if no labels exist + + """ + if not attributes: + return None + labels = [] + + if ( + hasattr(attributes, "info") + and attributes.info + and hasattr(attributes.info, "x_mitre_platforms") + ): + platforms = attributes.info.x_mitre_platforms + if platforms: + for platform in platforms: + labels.append(f"platform:{platform}") + + if ( + hasattr(attributes, "info") + and attributes.info + and hasattr(attributes.info, "x_mitre_data_sources") + ): + data_sources = attributes.info.x_mitre_data_sources + if data_sources: + for source in data_sources: + labels.append(f"data-source:{source}") + + return labels if labels else None + + def _create_external_references( + self, attributes: AttackTechniqueModel + ) -> Optional[List[Dict[str, str]]]: + """Create external references from attack technique attributes. + + Args: + attributes: The attack technique attributes + + Returns: + Optional[List[Dict[str, str]]]: Created external references or None if no references exist + + """ + if not attributes: + return None + external_references = [] + + technique_id = self.attack_technique.id + + if technique_id: + mitre_reference = { + "source_name": "mitre-attack", + "external_id": technique_id, + "url": f"https://attack.mitre.org/techniques/{technique_id}/", + } + external_references.append(mitre_reference) + + if hasattr(attributes, "link") and attributes.link: + link_reference = { + "source_name": "mitre-attack", + "url": attributes.link, + } + if not any( + ref.get("url") == attributes.link for ref in external_references + ): + external_references.append(link_reference) + + if hasattr(attributes, "stix_id") and attributes.stix_id: + stix_reference = { + "source_name": "stix", + "external_id": attributes.stix_id, + } + external_references.append(stix_reference) + + return external_references if external_references else None diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_domain_to_stix_domain.py b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_domain_to_stix_domain.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_file_to_stix_file.py b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_file_to_stix_file.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_ip_address_to_stix_ipv4_address.py b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_ip_address_to_stix_ipv4_address.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_ip_address_to_stix_ipv6_address.py b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_ip_address_to_stix_ipv6_address.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_malware_family_to_stix_malware.py b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_malware_family_to_stix_malware.py new file mode 100644 index 0000000000..db5222fc51 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_malware_family_to_stix_malware.py @@ -0,0 +1,232 @@ +"""Converts a GTI malware family to a STIX malware object.""" + +from datetime import datetime +from typing import List, Optional + +from connector.src.custom.models.gti_reports.gti_malware_family_model import ( + GTIMalwareFamilyData, + MalwareFamilyModel, +) +from connector.src.stix.octi.models.malware_model import OctiMalwareModel +from connector.src.stix.v21.models.ovs.malware_type_ov_enums import MalwareTypeOV +from stix2.v21 import Identity, Malware, MarkingDefinition # type: ignore + + +class GTIMalwareFamilyToSTIXMalware: + """Converts a GTI malware family to a STIX malware object.""" + + def __init__( + self, + malware_family: GTIMalwareFamilyData, + organization: Identity, + tlp_marking: MarkingDefinition, + ) -> None: + """Initialize the GTIMalwareFamilyToSTIXMalware object. + + Args: + malware_family (GTIMalwareFamilyData): The GTI malware family data to convert. + organization (Identity): The organization identity object. + tlp_marking (MarkingDefinition): The TLP marking definition. + + """ + self.malware_family = malware_family + self.organization = organization + self.tlp_marking = tlp_marking + + def to_stix(self) -> Malware: + """Convert the GTI malware family to a STIX malware object. + + Returns: + Malware: The STIX malware object. + + """ + if ( + not hasattr(self.malware_family, "attributes") + or not self.malware_family.attributes + ): + raise ValueError("Invalid report attributes") + + attributes = self.malware_family.attributes + + created = datetime.fromtimestamp(attributes.creation_date) + modified = datetime.fromtimestamp(attributes.last_modification_date) + + aliases = self._extract_aliases(attributes) + + malware_types = self._extract_malware_types(attributes) + + first_seen, last_seen = self._extract_seen_dates(attributes) + + labels = self._extract_labels(attributes) + + malware_model = OctiMalwareModel.create( + name=attributes.name, + organization_id=self.organization.id, + marking_ids=[self.tlp_marking.id], + malware_types=malware_types, + is_family=True, + description=attributes.description, + aliases=aliases, + first_seen=first_seen, + last_seen=last_seen, + labels=labels, + created=created, + modified=modified, + ) + + return malware_model.to_stix2_object() + + def _get_timestamps( + self, attributes: MalwareFamilyModel + ) -> tuple[datetime, datetime]: + """Extract creation and modification timestamps from attributes. + + Args: + attributes: The malware family attributes + + Returns: + tuple: (created, modified) datetime objects + + """ + created = datetime.fromtimestamp(attributes.creation_date) + modified = datetime.fromtimestamp(attributes.last_modification_date) + return created, modified + + def _extract_aliases(self, attributes: MalwareFamilyModel) -> Optional[List[str]]: + """Extract aliases from malware family attributes. + + Args: + attributes: The malware family attributes + + Returns: + Optional[List[str]]: Extracted aliases or None if no aliases exist + + """ + if ( + not hasattr(attributes, "alt_names_details") + or not attributes.alt_names_details + ): + return None + + aliases = [] + for alt_name in attributes.alt_names_details: + if hasattr(alt_name, "value") and alt_name.value: + aliases.append(alt_name.value) + + return aliases if aliases else None + + def _extract_malware_types( + self, attributes: MalwareFamilyModel + ) -> List[MalwareTypeOV]: + """Extract malware types from malware family attributes. + + Args: + attributes: The malware family attributes + + Returns: + List[MalwareTypeOV]: Extracted malware types + + """ + malware_types = [] + if hasattr(attributes, "malware_roles") and attributes.malware_roles: + for role in attributes.malware_roles: + if hasattr(role, "value") and role.value: + try: + malware_type = self._map_gti_role_to_stix_type(role.value) + if malware_type: + malware_types.append(malware_type) + except ValueError: + malware_types.append(MalwareTypeOV.UNKNOWN) + + if not malware_types: + malware_types.append(MalwareTypeOV.UNKNOWN) + + return malware_types + + def _map_gti_role_to_stix_type(self, role: str) -> Optional[MalwareTypeOV]: + """Map GTI malware role to STIX malware type. + + Args: + role: The GTI malware role + + Returns: + Optional[MalwareTypeOV]: Mapped STIX malware type or None if no mapping exists + + """ + role_map = { + "Adware": MalwareTypeOV.ADWARE, + "Backdoor": MalwareTypeOV.BACKDOOR, + "Bot": MalwareTypeOV.BOT, + "Bootkit": MalwareTypeOV.BOOTKIT, + "DDoS": MalwareTypeOV.DDOS, + "Downloader": MalwareTypeOV.DOWNLOADER, + "Dropper": MalwareTypeOV.DROPPER, + "Exploit Kit": MalwareTypeOV.EXPLOIT_KIT, + "Keylogger": MalwareTypeOV.KEYLOGGER, + "Ransomware": MalwareTypeOV.RANSOMWARE, + "Remote Access Trojan": MalwareTypeOV.REMOTE_ACCESS_TROJAN, + "Resource Exploitation": MalwareTypeOV.RESOURCE_EXPLOITATION, + "Rogue Security Software": MalwareTypeOV.ROGUE_SECURITY_SOFTWARE, + "Rootkit": MalwareTypeOV.ROOTKIT, + "Screen Capture": MalwareTypeOV.SCREEN_CAPTURE, + "Spyware": MalwareTypeOV.SPYWARE, + "Trojan": MalwareTypeOV.TROJAN, + "Virus": MalwareTypeOV.VIRUS, + "Webshell": MalwareTypeOV.WEBSHELL, + "Wiper": MalwareTypeOV.WIPER, + "Worm": MalwareTypeOV.WORM, + } + + return role_map.get(role) + + def _extract_seen_dates( + self, attributes: MalwareFamilyModel + ) -> tuple[Optional[datetime], Optional[datetime]]: + """Extract first_seen and last_seen dates from malware family attributes. + + Args: + attributes: The malware family attributes + + Returns: + tuple: (first_seen, last_seen) datetime objects or None if dates don't exist + + """ + first_seen = None + if ( + hasattr(attributes, "first_seen_details") + and attributes.first_seen_details + and len(attributes.first_seen_details) > 0 + and hasattr(attributes.first_seen_details[0], "values") + ): + first_seen = attributes.first_seen_details[0].values + + last_seen = None + if ( + hasattr(attributes, "last_seen_details") + and attributes.last_seen_details + and len(attributes.last_seen_details) > 0 + and hasattr(attributes.last_seen_details[0], "values") + ): + last_seen = attributes.last_seen_details[0].values + + return first_seen, last_seen + + def _extract_labels(self, attributes: MalwareFamilyModel) -> Optional[List[str]]: + """Extract labels from malware family attributes. + + Args: + attributes: The malware family attributes + + Returns: + Optional[List[str]]: Extracted labels or None if no labels exist + + """ + if not hasattr(attributes, "tags_details") or not attributes.tags_details: + return None + + labels = [] + for tag in attributes.tags_details: + if hasattr(tag, "value") and tag.value: + labels.append(tag.value) + + return labels if labels else None diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_report_relationship.py b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_report_relationship.py new file mode 100644 index 0000000000..a3720dc6f9 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_report_relationship.py @@ -0,0 +1,92 @@ +"""Converts a GTI report into STIX relationship objects.""" + +from datetime import datetime +from typing import Any, List, Optional + +from connector.src.custom.models.gti_reports.gti_report_model import GTIReportData +from connector.src.stix.octi.models.relationship_model import OctiRelationshipModel +from stix2.v21 import Identity, MarkingDefinition, Relationship # type: ignore + + +class GTIReportRelationship: + """Converts a GTI report into STIX relationship objects.""" + + def __init__( + self, + report: GTIReportData, + organization: Identity, + tlp_marking: MarkingDefinition, + report_id: str, + ): + """Initialize the GTIReportRelationship object. + + Args: + report (GTIReportData): The GTI report data to convert. + organization (Identity): The organization identity object. + tlp_marking (MarkingDefinition): The TLP marking definition. + report_id (str): The STIX ID of the report object. + + """ + if hasattr(report, "attributes") and report.attributes is not None: + created = datetime.fromtimestamp(report.attributes.creation_date) + modified = datetime.fromtimestamp(report.attributes.last_modification_date) + else: + raise ValueError("Invalid report data") + + self.report = report + + self.organization = organization + self.tlp_marking = tlp_marking + self.report_id = report_id + self.created = created + self.modified = modified + + def create_relationship( + self, + relationship_type: str, + target_ref: str, + target_name: Optional[str] = None, + description: Optional[str] = None, + ) -> Relationship: + """Create a generic relationship from report to any target entity. + + Args: + relationship_type (str): The type of relationship (e.g., 'targets', 'indicates'). + target_ref (str): The ID of the target entity. + target_name (Optional[str]): The name of the target entity, for description purposes. + description (Optional[str]): Custom description for the relationship. + + Returns: + Relationship: The STIX relationship object. + + """ + if hasattr(self.report, "attributes") and self.report.attributes is not None: + name = self.report.attributes.name + else: + raise ValueError("Report not initialized") + + return OctiRelationshipModel.create_from_report( + relationship_type=relationship_type, + report_id=self.report_id, + target_ref=target_ref, + report_name=name, + target_name=target_name, + organization_id=self.organization.id, + marking_ids=[self.tlp_marking.id], + created=self.created, + modified=self.modified, + description=description, + ) + + def to_stix(self, **kwargs: Any) -> List[Relationship]: + """Convert the GTI report into STIX relationship objects. + + Args: + **kwargs: Additional arguments passed to the method. + + Returns: + List[Relationship]: The list of STIX relationship objects. + + """ + result: List[Relationship] = [] + return result diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_report_to_stix_identity.py b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_report_to_stix_identity.py new file mode 100644 index 0000000000..7e1f273443 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_report_to_stix_identity.py @@ -0,0 +1,43 @@ +"""Converts a GTI report to a STIX identity object.""" + +from connector.src.custom.models.gti_reports.gti_report_model import GTIReportData +from connector.src.stix.octi.models.identity_author_model import OctiIdentityAuthorModel +from stix2.v21 import Identity # type: ignore + + +class GTIReportToSTIXIdentity: + """Converts a GTI report to a STIX identity object.""" + + def __init__(self, report: GTIReportData, organization: Identity): + """Initialize the GTIReportToSTIXIdentity object. + + Args: + report (GTIReportData): The GTI report data to convert. + organization (Identity): The organization identity object. + + """ + self.report = report + self.organization = organization + + def to_stix(self) -> Identity: + """Convert the GTI report to a STIX identity object. + + Returns: + Identity: The STIX identity object. + + """ + if not hasattr(self.report, "attributes") or not self.report.attributes: + raise ValueError("Invalid report attributes") + attributes = self.report.attributes + author = "Google Threat Intelligence" + if attributes.author and len(attributes.author) > 2: + author = attributes.author + + identity = OctiIdentityAuthorModel.create( + name=author, + organization_id=self.organization.id, + ) + + identity_stix = identity.to_stix2_object() + + return identity_stix diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_report_to_stix_location.py b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_report_to_stix_location.py new file mode 100644 index 0000000000..f937a80d8e --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_report_to_stix_location.py @@ -0,0 +1,137 @@ +"""Converts a GTI report's targeted regions to STIX Location objects.""" + +from typing import List, Optional + +from connector.src.custom.models.gti_reports.gti_report_model import ( + GTIReportData, + TargetedRegion, +) +from connector.src.stix.octi.models.location_model import OctiLocationModel +from connector.src.stix.v21.models.ovs.region_ov_enums import RegionOV +from stix2.v21 import Identity, Location, MarkingDefinition # type: ignore + + +class GTIReportToSTIXLocation: + """Converts a GTI report's targeted regions to STIX Location objects.""" + + def __init__( + self, + report: GTIReportData, + organization: Identity, + tlp_marking: MarkingDefinition, + ): + """Initialize the GTIReportToSTIXLocation object. + + Args: + report (GTIReportData): The GTI report data to convert. + organization (Identity): The organization identity object. + tlp_marking (MarkingDefinition): The TLP marking definition. + + """ + self.report = report + self.organization = organization + self.tlp_marking = tlp_marking + + def to_stix(self) -> List[Location]: + """Convert the GTI report targeted regions to STIX Location objects. + + Returns: + List[Location]: The list of STIX Location objects. + + """ + result: List[Location] = [] + if not hasattr(self.report, "attributes") or not self.report.attributes: + raise ValueError("Invalid report attributes") + + targeted_regions = self.report.attributes.targeted_regions_hierarchy + if not targeted_regions: + return result + + for region_data in targeted_regions: + location = self._process_region(region_data) + if location: + result.append(location) + + return result + + def _process_region(self, region_data: TargetedRegion) -> Optional[Location]: + """Process a targeted region entry and convert to appropriate Location type. + + Args: + region_data (TargetedRegion): The targeted region data to process. + + Returns: + Optional[Location]: The STIX Location object, or None if no valid location found. + + """ + location = None + if region_data.country: + location = self._create_country(region_data) + if location is None and region_data.sub_region: + location = self._create_region(region_data, is_sub_region=True) + if location is None and region_data.region: + location = self._create_region(region_data, is_sub_region=False) + + return location + + def _create_country(self, region_data: TargetedRegion) -> Location: + """Create a LocationCountry object. + + Args: + region_data (TargetedRegion): The targeted region data containing country information. + + Returns: + Location: The STIX LocationCountry object. + + """ + if not region_data.country: + return None + + iso_code = region_data.country_iso2 + if iso_code is None: + return None + + country = OctiLocationModel.create_country( + name=region_data.country, + country_code=iso_code, + description=region_data.description, + organization_id=self.organization.id, + marking_ids=[self.tlp_marking.id], + ) + + country_stix = country.to_stix2_object() + return country_stix + + def _create_region( + self, region_data: TargetedRegion, is_sub_region: bool + ) -> Location: + """Create a LocationRegion object. + + Args: + region_data (TargetedRegion): The targeted region data containing region information. + is_sub_region (bool): Whether to use the sub_region field (True) or region field (False). + + Returns: + Location: The STIX LocationRegion object. + + """ + region_name = region_data.sub_region if is_sub_region else region_data.region + if not region_name: + return None + + try: + region_value = RegionOV(region_name.lower().replace(" ", "-")) + except ValueError: + return None + + region = OctiLocationModel.create_region( + name=region_name, + region_value=region_value, + description=region_data.description, + organization_id=self.organization.id, + marking_ids=[self.tlp_marking.id], + ) + + region_stix = region.to_stix2_object() + + return region_stix diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_report_to_stix_report.py b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_report_to_stix_report.py new file mode 100644 index 0000000000..5a12c5a2a9 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_report_to_stix_report.py @@ -0,0 +1,241 @@ +"""Converts a GTI report to a STIX report object.""" + +from datetime import datetime +from typing import List, Tuple + +from connector.src.custom.models.gti_reports.gti_report_model import ( + GTIReportData, + ReportModel, +) +from connector.src.stix.octi.models.report_model import OctiReportModel +from connector.src.stix.v21.models.cdts.external_reference_model import ( + ExternalReferenceModel, +) +from connector.src.stix.v21.models.ovs.report_type_ov_enums import ReportTypeOV +from stix2.v21 import Identity, Location, MarkingDefinition, Report # type: ignore + + +class GTIReportToSTIXReport: + """Converts a GTI report to a STIX report object.""" + + def __init__( + self, + report: GTIReportData, + organization: Identity, + tlp_marking: MarkingDefinition, + author_identity: Identity, + sectors: List[Identity], + locations: List[Location], + ) -> None: + """Initialize the GTIReportToSTIXReport object. + + Args: + report (GTIReportData): The GTI report data to convert. + organization (Identity): The organization identity object. + tlp_marking (MarkingDefinition): The TLP marking definition. + author_identity (Identity): The author identity object. + sectors (List[Identity]): The list of sector identity objects. + locations (List[Location]): The list of location objects. + + """ + self.report = report + self.organization = organization + self.tlp_marking = tlp_marking + self.author_identity = author_identity + self.sectors = sectors + self.locations = locations + + def to_stix(self) -> Report: + """Convert the GTI report to a STIX report object. + + Returns: + Report: The STIX report object. + + """ + if not hasattr(self.report, "attributes") or not self.report.attributes: + raise ValueError("Invalid GTI report data") + + attributes = self.report.attributes + + name = attributes.name + if len(name) < 2: + raise ValueError("Report name must be at least 2 characters long") + + created, modified = self._get_timestamps(attributes) + labels = self._extract_labels(attributes) + external_references = self._build_external_references(attributes) + report_type = self._determine_report_type(attributes) + object_refs = self._collect_object_refs() + + report = OctiReportModel.create( + name=name, + created=created, + modified=modified, + description=attributes.autogenerated_summary, + report_types=[report_type], + published=created, + object_refs=object_refs, + organization_id=self.author_identity.id, + marking_ids=[self.tlp_marking.id], + labels=labels, + external_references=[ + ref.model_dump(exclude_none=True) for ref in external_references + ], + content=attributes.content, + ) + + return report.to_stix2_object() + + def _get_timestamps(self, attributes: ReportModel) -> Tuple[datetime, datetime]: + """Extract creation and modification timestamps from attributes. + + Args: + attributes: The report attributes + + Returns: + tuple: (created, modified) datetime objects + + """ + created = datetime.fromtimestamp(attributes.creation_date) + modified = datetime.fromtimestamp(attributes.last_modification_date) + return created, modified + + def _extract_labels(self, attributes: ReportModel) -> List[str]: + """Extract labels from report attributes. + + Args: + attributes: The report attributes + + Returns: + list: Extracted labels + + """ + labels = [] + if attributes.intended_effects: + labels.extend(attributes.intended_effects) + if attributes.threat_scape: + labels.extend(attributes.threat_scape) + if attributes.motivations: + for motivation in attributes.motivations: + if motivation.value: + labels.append(motivation.value) + return labels + + def _build_external_references( + self, attributes: ReportModel + ) -> List[ExternalReferenceModel]: + """Build external references from report attributes. + + Args: + attributes: The report attributes + + Returns: + list: External references + + """ + external_references = [] + if attributes.link: + external_reference = ExternalReferenceModel( + source_name="Source link", + description="Source link for the Report", + url=attributes.link, + ) + external_references.append(external_reference) + + if self.report.id: + external_reference = ExternalReferenceModel( + source_name="Google Threat Intelligence Platform", + description="Google Threat Intelligence Report Link", + url=f"https://www.virustotal.com/gui/collection/{self.report.id}", + ) + external_references.append(external_reference) + return external_references + + def _determine_report_type(self, attributes: ReportModel) -> ReportTypeOV: + """Determine the report type based on attributes. + + Args: + attributes: The report attributes + + Returns: + ReportTypeOV: The determined report type + + """ + report_type = ReportTypeOV.THREAT_REPORT + if attributes.report_type: + gti_to_stix_report_type = { + "News": ReportTypeOV.THREAT_REPORT, + "Actor Profile": ReportTypeOV.THREAT_ACTOR, + "Country Profile": ReportTypeOV.IDENTITY, + "Cyber Physical Security Roundup": ReportTypeOV.THREAT_REPORT, + "Event Coverage/Implication": ReportTypeOV.THREAT_REPORT, + "Industry Reporting": ReportTypeOV.THREAT_REPORT, + "Malware Profile": ReportTypeOV.MALWARE, + "Net Assessment": ReportTypeOV.THREAT_REPORT, + "Network Activity Reports": ReportTypeOV.OBSERVED_DATA, + "News Analysis": ReportTypeOV.THREAT_REPORT, + "OSINT Article": ReportTypeOV.THREAT_REPORT, + "Patch Report": ReportTypeOV.VULNERABILITY, + "Strategic Perspective": ReportTypeOV.THREAT_REPORT, + "TTP Deep Dive": ReportTypeOV.ATTACK_PATTERN, + "Threat Activity Alert": ReportTypeOV.INDICATOR, + "Threat Activity Report": ReportTypeOV.THREAT_REPORT, + "Trends and Forecasting": ReportTypeOV.CAMPAIGN, + "Weekly Vulnerability Exploitation Report": ReportTypeOV.VULNERABILITY, + } + report_type = gti_to_stix_report_type.get( + attributes.report_type, ReportTypeOV.THREAT_REPORT + ) + return report_type + + def _collect_object_refs(self) -> list[str]: + """Collect object references from related objects. + + Returns: + list: Collected object references + + """ + object_refs = [] + for sector in self.sectors: + object_refs.append(sector.id) + + for location in self.locations: + object_refs.append(location.id) + + return object_refs + + @staticmethod + def add_object_refs(objects_to_add: List[str], existing_report: Report) -> Report: + """Add object references to an existing STIX report while preserving all report data. + + Args: + objects_to_add: Object ID(s) to add to the report's object_refs. + existing_report: The existing STIX report object to update. + + Returns: + Report: The updated STIX report with all original data preserved. + + """ + updated_refs = existing_report.get("object_refs", []) + + for obj_id in objects_to_add: + if obj_id not in updated_refs: + updated_refs.append(obj_id) + + report = OctiReportModel.create( + name=existing_report.name, + created=existing_report.created, + modified=existing_report.modified, + description=existing_report.get("description", ""), + report_types=existing_report.report_types, + published=existing_report.published, + object_refs=updated_refs, + organization_id=existing_report.created_by_ref, + marking_ids=existing_report.object_marking_refs, + labels=existing_report.get("labels", []), + external_references=existing_report.get("external_references", []), + content=existing_report.get("x_opencti_content", None), + ) + report.object_refs = updated_refs + + return report.to_stix2_object() diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_report_to_stix_sector.py b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_report_to_stix_sector.py new file mode 100644 index 0000000000..93461b6dd8 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_report_to_stix_sector.py @@ -0,0 +1,92 @@ +"""Converts a GTI report's targeted industries to STIX Identity objects as sectors.""" + +from typing import List, Optional + +from connector.src.custom.models.gti_reports.gti_report_model import ( + GTIReportData, + TargetedIndustry, +) +from connector.src.stix.octi.models.identity_sector_model import OctiIdentitySectorModel +from stix2.v21 import Identity, MarkingDefinition # type: ignore + + +class GTIReportToSTIXSector: + """Converts a GTI report's targeted industries to STIX Identity objects as sectors.""" + + def __init__( + self, + report: GTIReportData, + organization: Identity, + tlp_marking: MarkingDefinition, + ): + """Initialize the GTIReportToSTIXSector object. + + Args: + report (GTIReportData): The GTI report data to convert. + organization (Identity): The organization identity object. + tlp_marking (MarkingDefinition): The TLP marking definition. + + """ + self.report = report + self.organization = organization + self.tlp_marking = tlp_marking + + def to_stix(self) -> List[Identity]: + """Convert the GTI report targeted industries to STIX Identity objects. + + Returns: + List[Identity]: The list of STIX Identity objects representing sectors. + + """ + result: List[Identity] = [] + if not hasattr(self.report, "attributes") or not self.report.attributes: + raise ValueError("Invalid report attributes") + + targeted_industries = self.report.attributes.targeted_industries_tree + if not targeted_industries: + return result + + for industry_data in targeted_industries: + sector = self._process_industry(industry_data) + if sector: + result.append(sector) + + return result + + def _process_industry(self, industry_data: TargetedIndustry) -> Optional[Identity]: + """Process a targeted industry entry and convert to a sector Identity. + + Args: + industry_data (TargetedIndustry): The targeted industry data to process. + + Returns: + Optional[Identity]: The STIX Identity object, or None if no valid industry group found. + + """ + if not industry_data.industry_group: + return None + + return self._create_sector(industry_data) + + def _create_sector(self, industry_data: TargetedIndustry) -> Identity: + """Create a Sector Identity object. + + Args: + industry_data (TargetedIndustry): The targeted industry data containing industry group information. + + Returns: + Identity: The STIX Identity object representing a sector. + + """ + sector_name = industry_data.industry_group + + sector = OctiIdentitySectorModel.create( + name=sector_name, + description=industry_data.description, + organization_id=self.organization.id, + marking_ids=[self.tlp_marking.id], + ) + + sector_stix = sector.to_stix2_object() + + return sector_stix diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_threat_actor_to_stix_intrusion_set.py b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_threat_actor_to_stix_intrusion_set.py new file mode 100644 index 0000000000..7b242a02d2 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_threat_actor_to_stix_intrusion_set.py @@ -0,0 +1,277 @@ +"""Converts a GTI threat actor to a STIX intrusion set object.""" + +from datetime import datetime +from typing import List, Optional + +from connector.src.custom.models.gti_reports.gti_threat_actor_model import ( + GTIThreatActorData, + ThreatActorModel, +) +from connector.src.stix.octi.models.intrusion_set_model import OctiIntrusionSetModel +from connector.src.stix.v21.models.ovs.attack_motivation_ov_enums import ( + AttackMotivationOV, +) +from stix2.v21 import Identity, IntrusionSet, MarkingDefinition # type: ignore + + +class GTIThreatActorToSTIXIntrusionSet: + """Converts a GTI threat actor to a STIX intrusion set object.""" + + def __init__( + self, + threat_actor: GTIThreatActorData, + organization: Identity, + tlp_marking: MarkingDefinition, + ) -> None: + """Initialize the GTIThreatActorToSTIXIntrusionSet object. + + Args: + threat_actor (GTIThreatActorData): The GTI threat actor data to convert. + organization (Identity): The organization identity object. + tlp_marking (MarkingDefinition): The TLP marking definition. + + """ + self.threat_actor = threat_actor + self.organization = organization + self.tlp_marking = tlp_marking + + def to_stix(self) -> IntrusionSet: + """Convert the GTI threat actor to a STIX intrusion set object. + + Returns: + IntrusionSet: The STIX intrusion set object. + + """ + if ( + not hasattr(self.threat_actor, "attributes") + or not self.threat_actor.attributes + ): + raise ValueError("Invalid GTI threat actor data") + + attributes = self.threat_actor.attributes + + created = datetime.fromtimestamp(attributes.creation_date) + modified = datetime.fromtimestamp(attributes.last_modification_date) + + aliases = self._extract_aliases(attributes) + + first_seen, last_seen = self._extract_seen_dates(attributes) + + labels = self._extract_labels(attributes) + + goals = self._extract_goals(attributes) + + primary_motivation, secondary_motivations = self._extract_motivations( + attributes + ) + + resource_level = self._extract_resource_level(attributes) + name = attributes.name + description = attributes.description + + intrusion_set_model = OctiIntrusionSetModel.create( + name=name, + organization_id=self.organization.id, + marking_ids=[self.tlp_marking.id], + description=description, + aliases=aliases, + first_seen=first_seen, + last_seen=last_seen, + goals=goals, + resource_level=resource_level, + primary_motivation=primary_motivation, + secondary_motivations=secondary_motivations, + labels=labels, + created=created, + modified=modified, + ) + + return intrusion_set_model.to_stix2_object() + + def _extract_aliases(self, attributes: ThreatActorModel) -> Optional[List[str]]: + """Extract aliases from threat actor attributes. + + Args: + attributes: The threat actor attributes + + Returns: + Optional[List[str]]: Extracted aliases or None if no aliases exist + + """ + if ( + not hasattr(attributes, "alt_names_details") + or not attributes.alt_names_details + ): + return None + + aliases = [] + for alt_name in attributes.alt_names_details: + if hasattr(alt_name, "value") and alt_name.value: + aliases.append(alt_name.value) + + return aliases if aliases else None + + def _extract_seen_dates( + self, attributes: ThreatActorModel + ) -> tuple[Optional[datetime], Optional[datetime]]: + """Extract first_seen and last_seen dates from threat actor attributes. + + Args: + attributes: The threat actor attributes + + Returns: + tuple: (first_seen, last_seen) datetime objects or None if dates don't exist + + """ + first_seen = None + if ( + hasattr(attributes, "first_seen_details") + and attributes.first_seen_details + and len(attributes.first_seen_details) > 0 + and hasattr(attributes.first_seen_details[0], "value") + and attributes.first_seen_details[0].value + ): + try: + first_seen_str = attributes.first_seen_details[0].value + first_seen = datetime.strptime(first_seen_str, "%Y-%m-%dT%H:%M:%SZ") + except (ValueError, TypeError): + first_seen = None + + last_seen = None + if ( + hasattr(attributes, "last_seen_details") + and attributes.last_seen_details + and len(attributes.last_seen_details) > 0 + and hasattr(attributes.last_seen_details[0], "value") + and attributes.last_seen_details[0].value + ): + try: + last_seen_str = attributes.last_seen_details[0].value + last_seen = datetime.strptime(last_seen_str, "%Y-%m-%dT%H:%M:%SZ") + except (ValueError, TypeError): + last_seen = None + + return first_seen, last_seen + + def _extract_labels(self, attributes: ThreatActorModel) -> Optional[List[str]]: + """Extract labels from threat actor attributes. + + Args: + attributes: The threat actor attributes + + Returns: + Optional[List[str]]: Extracted labels or None if no labels exist + + """ + if not hasattr(attributes, "tags_details") or not attributes.tags_details: + return None + + labels = [] + for tag in attributes.tags_details: + if hasattr(tag, "value") and tag.value: + labels.append(tag.value) + + return labels if labels else None + + def _extract_goals(self, attributes: ThreatActorModel) -> Optional[List[str]]: + """Extract goals from threat actor attributes. + + Args: + attributes: The threat actor attributes + + Returns: + Optional[List[str]]: Extracted goals or None if no goals exist + + """ + if ( + not hasattr(attributes, "targeted_industries_tree") + or not attributes.targeted_industries_tree + ): + return None + + goals = [] + for industry in attributes.targeted_industries_tree: + if hasattr(industry, "industry_group") and industry.industry_group: + goal = f"Target {industry.industry_group} industry" + goals.append(goal) + + return goals if goals else None + + def _extract_motivations( + self, attributes: ThreatActorModel + ) -> tuple[Optional[str], Optional[List[str]]]: + """Extract primary and secondary motivations from threat actor attributes. + + Args: + attributes: The threat actor attributes + + Returns: + tuple: (primary_motivation, secondary_motivations) or (None, None) if motivations don't exist + + """ + if not hasattr(attributes, "motivations") or not attributes.motivations: + return None, None + + motivations = [] + for motivation in attributes.motivations: + if hasattr(motivation, "value") and motivation.value: + mapped_motivation = self._map_gti_motivation_to_stix_motivation( + motivation.value + ) + if mapped_motivation: + motivations.append(mapped_motivation) + else: + motivations.append(AttackMotivationOV.UNPREDICTABLE) + + if not motivations: + return None, None + + primary_motivation = motivations[0] + secondary_motivations = motivations[1:] if len(motivations) > 1 else None + + return primary_motivation, secondary_motivations + + def _map_gti_motivation_to_stix_motivation(self, motivation: str) -> Optional[str]: + """Map GTI motivation to STIX attack motivation. + + Args: + motivation: The GTI motivation + + Returns: + Optional[str]: Mapped STIX attack motivation or None if no mapping exists + + """ + motivation_map = { + "Accidental": AttackMotivationOV.ACCIDENTAL, + "Coercion": AttackMotivationOV.COERCION, + "Control": AttackMotivationOV.DOMINANCE, + "Dominance": AttackMotivationOV.DOMINANCE, + "Ideology": AttackMotivationOV.IDEOLOGY, + "Political": AttackMotivationOV.IDEOLOGY, + "Religious": AttackMotivationOV.IDEOLOGY, + "Notoriety": AttackMotivationOV.NOTORIETY, + "Fame": AttackMotivationOV.NOTORIETY, + "Corporate Espionage": AttackMotivationOV.ORGANIZATIONAL_GAIN, + "Economic": AttackMotivationOV.ORGANIZATIONAL_GAIN, + "Organizational Gain": AttackMotivationOV.ORGANIZATIONAL_GAIN, + "Financial": AttackMotivationOV.PERSONAL_GAIN, + "Personal Gain": AttackMotivationOV.PERSONAL_GAIN, + "Entertainment": AttackMotivationOV.PERSONAL_SATISFACTION, + "Personal Satisfaction": AttackMotivationOV.PERSONAL_SATISFACTION, + "Revenge": AttackMotivationOV.REVENGE, + "Unpredictable": AttackMotivationOV.UNPREDICTABLE, + } + + return motivation_map.get(motivation) + + def _extract_resource_level(self, attributes: ThreatActorModel) -> Optional[str]: + """Extract resource level from threat actor attributes. + + Args: + attributes: The threat actor attributes + + Returns: + Optional[str]: Extracted resource level or None if resource level doesn't exist + + """ + return None diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_url_to_stix_url.py b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_url_to_stix_url.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_vulnerability_to_stix_vulnerability.py b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_vulnerability_to_stix_vulnerability.py new file mode 100644 index 0000000000..7f021d2580 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/mappers/gti_reports/gti_vulnerability_to_stix_vulnerability.py @@ -0,0 +1,176 @@ +"""Converts a GTI vulnerability to a STIX vulnerability object.""" + +from datetime import datetime +from typing import Dict, List, Optional + +from connector.src.custom.models.gti_reports.gti_vulnerability_model import ( + GTIVulnerabilityData, + VulnerabilityModel, +) +from connector.src.stix.octi.models.vulnerability_model import OctiVulnerabilityModel +from stix2.v21 import Identity, MarkingDefinition, Vulnerability # type: ignore + + +class GTIVulnerabilityToSTIXVulnerability: + """Converts a GTI vulnerability to a STIX vulnerability object.""" + + def __init__( + self, + vulnerability: GTIVulnerabilityData, + organization: Identity, + tlp_marking: MarkingDefinition, + ) -> None: + """Initialize the GTIVulnerabilityToSTIXVulnerability object. + + Args: + vulnerability (GTIVulnerabilityData): The GTI vulnerability data to convert. + organization (Identity): The organization identity object. + tlp_marking (MarkingDefinition): The TLP marking definition. + + """ + self.vulnerability = vulnerability + self.organization = organization + self.tlp_marking = tlp_marking + + def to_stix(self) -> Vulnerability: + """Convert the GTI vulnerability to a STIX vulnerability object. + + Returns: + Vulnerability: The STIX vulnerability object. + + """ + if not self.vulnerability or not self.vulnerability.attributes: + raise ValueError("Vulnerability attributes are missing") + + attributes = self.vulnerability.attributes + + created = datetime.fromtimestamp(attributes.creation_date) + modified = datetime.fromtimestamp(attributes.last_modification_date) + + base_score = None + if ( + hasattr(attributes, "cvss") + and attributes.cvss + and hasattr(attributes.cvss, "cvssv3") + and attributes.cvss.cvssv3 + and hasattr(attributes.cvss.cvssv3, "base_score") + ): + base_score = attributes.cvss.cvssv3.base_score + + epss_score = None + epss_percentile = None + if hasattr(attributes, "epss") and attributes.epss: + if hasattr(attributes.epss, "score"): + epss_score = attributes.epss.score + if hasattr(attributes.epss, "percentile"): + epss_percentile = attributes.epss.percentile + + labels = self._extract_labels(attributes) + external_references = self._create_external_references(attributes) + + vulnerability_model = OctiVulnerabilityModel.create( + name=attributes.name, + cve_id=self.vulnerability.id, + organization_id=self.organization.id, + marking_ids=[self.tlp_marking.id], + description=attributes.description, + created=created, + modified=modified, + base_score=base_score, + epss_score=epss_score, + epss_percentile=epss_percentile, + labels=labels, + external_references=external_references, + ) + + return vulnerability_model.to_stix2_object() + + def _extract_labels(self, attributes: VulnerabilityModel) -> Optional[List[str]]: + """Extract labels from vulnerability attributes. + + Args: + attributes: The vulnerability attributes + + Returns: + Optional[List[str]]: Extracted labels or None if no labels exist + + """ + if not attributes: + return None + + labels = [] + + if ( + hasattr(attributes, "cvss") + and attributes.cvss + and hasattr(attributes.cvss, "cvssv3") + and attributes.cvss.cvssv3 + ): + cvss = attributes.cvss.cvssv3 + + if hasattr(cvss, "attack_vector") and cvss.attack_vector: + labels.append(f"attack-vector:{cvss.attack_vector}") + + if hasattr(cvss, "attack_complexity") and cvss.attack_complexity: + labels.append(f"attack-complexity:{cvss.attack_complexity}") + + if hasattr(cvss, "privileges_required") and cvss.privileges_required: + labels.append(f"privileges-required:{cvss.privileges_required}") + + if hasattr(cvss, "user_interaction") and cvss.user_interaction: + labels.append(f"user-interaction:{cvss.user_interaction}") + + return labels if labels else None + + def _create_external_references( + self, attributes: VulnerabilityModel + ) -> Optional[List[Dict[str, str]]]: + """Create external references from vulnerability attributes. + + Args: + attributes: The vulnerability attributes + + Returns: + Optional[List[dict]]: Created external references or None if no references exist + + """ + if not attributes: + return None + + external_references = [] + + cve_id = self.vulnerability.id + + if cve_id: + nvd_reference = { + "source_name": "nvd", + "external_id": cve_id, + "url": f"https://nvd.nist.gov/vuln/detail/{cve_id}", + } + external_references.append(nvd_reference) + + if hasattr(attributes, "link") and attributes.link: + link_reference = { + "source_name": "url", + "url": attributes.link, + } + if not any( + ref.get("url") == attributes.link for ref in external_references + ): + external_references.append(link_reference) + + if ( + hasattr(attributes, "cvss") + and attributes.cvss + and hasattr(attributes.cvss, "cvssv3") + and attributes.cvss.cvssv3 + and hasattr(attributes.cvss.cvssv3, "vector_string") + and attributes.cvss.cvssv3.vector_string + ): + cvss_reference = { + "source_name": "cvss-v3", + "description": attributes.cvss.cvssv3.vector_string, + } + external_references.append(cvss_reference) + + return external_references if external_references else None diff --git a/external-import/google-ti-feeds/connector/src/custom/models/__init__.py b/external-import/google-ti-feeds/connector/src/custom/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/__init__.py b/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_attack_technique_model.py b/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_attack_technique_model.py new file mode 100644 index 0000000000..9b9949e95a --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_attack_technique_model.py @@ -0,0 +1,102 @@ +"""Model representing a Google Threat Intelligence Attack Technique.""" + +from typing import Dict, List, Optional, Union + +from pydantic import BaseModel, Field + + +class Info(BaseModel): + """Information related to the attack technique.""" + + x_mitre_contributors: Optional[List[str]] = Field( + None, description="People and organizations who have contributed to the object." + ) + x_mitre_platforms: Optional[List[str]] = Field( + None, description="List of platforms that apply to the technique." + ) + x_mitre_is_subtechnique: Optional[bool] = Field( + None, description="If true, this technique has sub-techniques." + ) + x_mitre_permissions_required: Optional[List[str]] = Field( + None, + description="The lowest level of permissions required to perform the technique.", + ) + x_mitre_version: Optional[str] = Field( + None, description="The version of the object in format major.minor." + ) + x_mitre_data_sources: Optional[List[str]] = Field( + None, + description="Sources of information that may be used to identify the action.", + ) + x_mitre_detection: Optional[str] = Field( + None, description="Strategies for identifying if a technique has been used." + ) + x_mitre_effective_permissions: Optional[List[str]] = Field( + None, description="The level of permissions the adversary will attain." + ) + x_mitre_defense_bypassed: Optional[List[str]] = Field( + None, + description="List of defensive tools, methodologies, or processes the technique can bypass.", + ) + x_mitre_remote_support: Optional[bool] = Field( + None, + description="If true, the technique can be used to execute something on a remote system.", + ) + x_mitre_impact_type: Optional[Union[str, List[str]]] = Field( + None, + description="Denotes if the technique can be used for integrity or availability attacks.", + ) + x_mitre_system_requirements: Optional[str] = Field( + None, + description="Additional information on requirements needed for the technique.", + ) + x_mitre_tactic_type: Optional[Union[str, List[str]]] = Field( + None, description="Tactic type of the technique." + ) + x_mitre_deprecated: Optional[bool] = Field( + None, + description="Marked as deprecated. There is not a revoking technique replacing this one.", + ) + x_mitre_old_attack_id: Optional[str] = Field(None, description="Old ATT&CK ID.") + x_mitre_network_requirements: Optional[bool] = Field( + None, description="Requires network to execute the technique." + ) + + +class AttackTechniqueModel(BaseModel): + """Model representing a Google Threat Intelligence Attack Technique.""" + + info: Optional[Info] = Field(None, description="Technique's additional info.") + revoked: bool = Field( + False, description="Indicates if the technique has been revoked." + ) + name: str = Field(..., description="Technique's name.") + creation_date: int = Field( + ..., description="Creation date of the attack technique (UTC timestamp)." + ) + link: Optional[str] = Field( + None, description="URL of the technique on MITRE's website." + ) + stix_id: Optional[str] = Field(None, description="Technique's STIX ID.") + last_modification_date: int = Field( + ..., description="Date when the technique was last updated (UTC timestamp)." + ) + description: Optional[str] = Field(None, description="Technique's description.") + private: bool = Field( + False, description="Whether the attack technique object is private." + ) + + +class GTIAttackTechniqueData(BaseModel): + """Model representing data for a GTI attack technique.""" + + id: str + type: str = Field("attack_technique") + links: Optional[Dict[str, str]] = None + attributes: Optional[AttackTechniqueModel] = None + + +class GTIAttackTechniqueResponse(BaseModel): + """Model representing a response containing GTI attack technique data.""" + + data: GTIAttackTechniqueData diff --git a/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_domain_model.py b/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_domain_model.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_file_model.py b/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_file_model.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_ip_addresses_model.py b/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_ip_addresses_model.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_malware_family_model.py b/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_malware_family_model.py new file mode 100644 index 0000000000..f14b5702e7 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_malware_family_model.py @@ -0,0 +1,371 @@ +"""Model representing a Google Threat Intelligence Malware Family.""" + +from typing import Any, Dict, List, Optional, Union + +from pydantic import BaseModel, Field + + +class AggregationCommonalities(BaseModel): + """Technical commonalities among all domains, files, IP addresses, and URLs tied to the malware family.""" + + domains: Optional[Dict[str, Any]] = Field( + None, + description="Technical commonalities among all domains tied to the malware family.", + ) + files: Optional[Dict[str, Any]] = Field( + None, + description="Technical commonalities among all files tied to the malware family.", + ) + ip_addresses: Optional[Dict[str, Any]] = Field( + None, + description="Technical commonalities among all IP addresses tied to the malware family.", + ) + urls: Optional[Dict[str, Any]] = Field( + None, + description="Technical commonalities among all URLs tied to the malware family.", + ) + + +class Counters(BaseModel): + """Count of technical commonalities among all domains, files, IP addresses, and URLs tied to the malware family.""" + + attack_techniques: int = Field( + ..., + description="Number of MITRE ATT&CK techniques associated with the malware family.", + ) + domains: int = Field( + ..., description="Number of domains related to the malware family." + ) + files: int = Field( + ..., description="Number of files related to the malware family." + ) + iocs: int = Field( + ..., + description="Number of IoCs related to the malware family (files + URLs + domains + IP addresses).", + ) + ip_addresses: int = Field( + ..., description="Number of IP addresses related to the malware family." + ) + subscribers: int = Field( + ..., description="Number of users subscribed to the malware family." + ) + urls: int = Field(..., description="Number of URLs related to the malware family.") + + +class AltNameDetail(BaseModel): + """Alternative names/aliases by which the malware family could be known.""" + + confidence: str = Field( + ..., + description="Confidence on the information or the attribution of the alternative name.", + ) + description: Optional[str] = Field( + None, description="Additional information related to the alternative name." + ) + first_seen: Optional[int] = Field( + None, + description="The first time the alternative name was attributed (UTC timestamp).", + ) + last_seen: Optional[int] = Field( + None, + description="The last time the alternative name was attributed (UTC timestamp).", + ) + value: str = Field(..., description="Alternative name/alias.") + + +class Capability(BaseModel): + """Capabilities associated with malware family's files.""" + + confidence: str = Field( + ..., description="The confidence of the malware family's associated capability." + ) + description: Optional[str] = Field( + None, description="Description of the capability." + ) + first_seen: Optional[int] = Field( + None, description="First time the capability was associated (UTC timestamp)." + ) + last_seen: Optional[int] = Field( + None, description="Last time the capability was associated (UTC timestamp)." + ) + value: str = Field(..., description="Capability name.") + + +class DetectionName(BaseModel): + """External detection names associated with the malware family.""" + + confidence: str = Field( + ..., + description="The confidence of the detection name associated to the malware family.", + ) + description: Optional[str] = Field( + None, description="Descriptive information related to the detection name." + ) + first_seen: Optional[int] = Field( + None, + description="First time the detection name was associated (UTC timestamp).", + ) + last_seen: Optional[int] = Field( + None, description="Last time the detection name was associated (UTC timestamp)." + ) + value: str = Field(..., description="The detection name.") + + +class SeenDetail(BaseModel): + """Details about when the malware family was first or last seen.""" + + confidence: str = Field( + ..., description="Confidence on the information or the attribution." + ) + description: Optional[str] = Field( + None, description="Additional information about the activity." + ) + first_seen: Optional[int] = Field( + None, description="First time this date was attributed (UTC timestamp)." + ) + last_seen: Optional[int] = Field( + None, description="Last time this date was attributed (UTC timestamp)." + ) + value: str = Field( + ..., + description="Date when the observation was made (YYYY-MM-DDTHH:mm:ssZ format).", + ) + + +class MalwareRole(BaseModel): + """Malware roles associated with the malware family.""" + + confidence: str = Field( + ..., description="The confidence of the malware family's associated role." + ) + description: Optional[str] = Field( + None, description="Descriptive information related to the role." + ) + first_seen: Optional[int] = Field( + None, description="First time the role was associated (UTC timestamp)." + ) + last_seen: Optional[int] = Field( + None, description="Last time the role was associated (UTC timestamp)." + ) + value: str = Field(..., description="The malware role name.") + + +class OperatingSystem(BaseModel): + """Operating systems affected by the malware family.""" + + confidence: str = Field( + ..., description="The confidence that the OS is affected by the malware family." + ) + description: Optional[str] = Field( + None, description="Descriptive information related to the targeted OS." + ) + first_seen: Optional[int] = Field( + None, description="First time the OS was associated (UTC timestamp)." + ) + last_seen: Optional[int] = Field( + None, description="Last time the OS was associated (UTC timestamp)." + ) + value: str = Field(..., description="Operating system name.") + + +class SourceRegion(BaseModel): + """Country or region from which the malware family is known to originate.""" + + confidence: str = Field( + ..., + description="Confidence on the information related to the source region.", + ) + country: Optional[str] = Field( + None, description="Country of malware family origin." + ) + country_iso2: Optional[str] = Field( + None, description="Source country in ISO 3166 Alpha2 code format." + ) + description: Optional[str] = Field( + None, description="Additional information about the source region." + ) + first_seen: Optional[int] = Field( + None, + description="First time this source region was attributed (UTC timestamp).", + ) + last_seen: Optional[int] = Field( + None, description="Last time this source region was attributed (UTC timestamp)." + ) + region: Optional[str] = Field(None, description="Region of malware family origin.") + source: Optional[str] = Field(None, description="Information's supplier.") + sub_region: Optional[str] = Field( + None, description="Subregion of malware family origin." + ) + + +class TagDetail(BaseModel): + """Tags associated with the malware family with additional context.""" + + confidence: str = Field( + ..., description="Confidence on the tag association to the malware family." + ) + description: Optional[str] = Field( + None, description="Additional information related to the tag." + ) + first_seen: Optional[int] = Field( + None, description="First time this tag was attributed (UTC timestamp)." + ) + last_seen: Optional[int] = Field( + None, description="Last time this tag was attributed (UTC timestamp)." + ) + value: str = Field(..., description="Value of the tag.") + + +class TargetedIndustry(BaseModel): + """Industries and industry groups known to be targeted by the malware family.""" + + confidence: str = Field( + ..., description="Confidence on the industry targeted by the malware family." + ) + description: Optional[str] = Field( + None, description="Additional information related to the targeted industry." + ) + first_seen: Optional[int] = Field( + None, + description="First time this targeted industry was associated (UTC timestamp).", + ) + industry: Optional[str] = Field( + None, description="Sub-industry targeted by the malware family." + ) + industry_group: str = Field( + ..., description="Industry group targeted by the malware family." + ) + last_seen: Optional[int] = Field( + None, + description="Last time this targeted industry was associated (UTC timestamp).", + ) + source: Optional[str] = Field(None, description="Information's supplier.") + + +class TargetedRegion(BaseModel): + """Regions and countries known to be targeted by the malware family.""" + + confidence: str = Field( + ..., + description="Confidence on the malware family's targeted region association.", + ) + country: Optional[str] = Field( + None, description="Country targeted by the malware family." + ) + country_iso2: Optional[str] = Field( + None, description="Targeted country in ISO 3166 Alpha2 code format." + ) + description: Optional[str] = Field( + None, description="Additional information related to the targeted region." + ) + first_seen: Optional[int] = Field( + None, + description="First time this targeted region was associated (UTC timestamp).", + ) + last_seen: Optional[int] = Field( + None, + description="Last time this targeted region was associated (UTC timestamp).", + ) + region: Optional[str] = Field( + None, description="Region targeted by the malware family." + ) + source: Optional[str] = Field(None, description="Information's supplier.") + sub_region: Optional[str] = Field( + None, description="Sub-region targeted by the malware family." + ) + + +class MalwareFamilyModel(BaseModel): + """Model representing a GTI malware family.""" + + name: str = Field(..., description="Malware family's name.") + collection_type: Optional[str] = Field( + None, + description="Type of object; typically 'malware-family' or 'malware_family'", + ) + creation_date: int = Field( + ..., description="UTC timestamp of malware family object creation." + ) + last_modification_date: int = Field( + ..., description="UTC timestamp of last malware family update." + ) + description: Optional[str] = Field( + None, description="Description/context about the malware family." + ) + status: Optional[str] = Field( + None, + description="Status of attribute computation: PENDING_RECOMPUTE or COMPUTED.", + ) + private: bool = Field( + ..., description="Whether the malware family object is private." + ) + origin: Optional[str] = Field( + None, + description="Source of the information: Partner or Google Threat Intelligence.", + ) + + recent_activity_relative_change: Optional[float] = Field( + None, description="Ratio of recent activity change (14-day interval)." + ) + recent_activity_summary: Optional[List[int]] = Field( + None, description="Time series of IoC activity (14-day)." + ) + top_icon_md5: Optional[List[str]] = Field( + None, description="List of the 3 most frequent icons' MD5 hashes." + ) + + counters: Optional[Counters] = Field( + None, description="Counters for related indicators and metadata." + ) + aggregations: Optional[AggregationCommonalities] = Field( + None, description="Grouped common traits across related IoCs." + ) + alt_names_details: Optional[List[AltNameDetail]] = Field( + None, description="Alternative names/aliases for the malware family." + ) + capabilities: Optional[List[Capability]] = Field( + None, description="Capabilities associated with the malware family's files." + ) + detection_names: Optional[List[DetectionName]] = Field( + None, description="External detection names associated with the malware family." + ) + first_seen_details: Optional[List[SeenDetail]] = Field( + None, description="Information about when the malware family was first seen." + ) + last_seen_details: Optional[List[SeenDetail]] = Field( + None, description="Information about when the malware family was last seen." + ) + malware_roles: Optional[List[MalwareRole]] = Field( + None, description="Roles associated with the malware family." + ) + operating_systems: Optional[List[OperatingSystem]] = Field( + None, description="Operating systems affected by the malware family." + ) + source_regions_hierarchy: Optional[List[SourceRegion]] = Field( + None, description="Regions/countries of malware family origin." + ) + tags_details: Optional[List[TagDetail]] = Field( + None, description="Tags applied to the malware family, with context." + ) + targeted_industries_tree: Optional[List[TargetedIndustry]] = Field( + None, description="Industries targeted by the malware family." + ) + targeted_regions_hierarchy: Optional[List[TargetedRegion]] = Field( + None, description="Regions/countries targeted by the malware family." + ) + + +class GTIMalwareFamilyData(BaseModel): + """Model representing data for a GTI malware family.""" + + id: str + type: Optional[str] = None + links: Optional[Dict[str, str]] = None + attributes: Optional[MalwareFamilyModel] = None + context_attributes: Optional[Dict[str, Any]] = None + + +class GTIMalwareFamilyResponse(BaseModel): + """Model representing a response containing GTI malware family data.""" + + data: Union[GTIMalwareFamilyData, List[GTIMalwareFamilyData]] diff --git a/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_report_model.py b/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_report_model.py new file mode 100644 index 0000000000..e57d4977c7 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_report_model.py @@ -0,0 +1,305 @@ +"""Module containing models for GTI reports response from Google Threat Intelligence API.""" + +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Field + + +class AggregationCommonalities(BaseModel): + """Technical commonalities among all domains, files, IP addresses, and URLs tied to the report.""" + + domains: Optional[Dict[str, Any]] = Field( + None, + description="Technical commonalities among all domains tied to the report.", + ) + files: Optional[Dict[str, Any]] = Field( + None, description="Technical commonalities among all files tied to the report." + ) + ip_addresses: Optional[Dict[str, Any]] = Field( + None, + description="Technical commonalities among all IP addresses tied to the report.", + ) + urls: Optional[Dict[str, Any]] = Field( + None, description="Technical commonalities among all URLs tied to the report." + ) + + +class Counters(BaseModel): + """Count of technical commonalities among all domains, files, IP addresses, and URLs tied to the report.""" + + domains: int = Field(..., description="Number of domains related to the report.") + files: int = Field(..., description="Number of files related to the report.") + iocs: int = Field( + ..., + description="Number of IoCs related to the report (files + URLs + domains + IP addresses).", + ) + ip_addresses: int = Field( + ..., description="Number of IP addresses related to the report." + ) + subscribers: int = Field( + ..., description="Number of users subscribed to the report." + ) + urls: int = Field(..., description="Number of URLs related to the report.") + + +class Motivation(BaseModel): + """Motivations of the threat described in the report such as espionage, financial gain, etc.""" + + confidence: str = Field( + ..., + description="Confidence level on the motivation's attribution to the threat.", + ) + description: Optional[str] = Field( + None, description="Additional information about the threat's motivation." + ) + first_seen: Optional[str] = Field( + ..., description="UTC timestamp when the motivation was first seen." + ) + last_seen: Optional[str] = Field( + ..., description="UTC timestamp when the motivation was last seen." + ) + value: str = Field( + ..., description="Motivation of the threat (e.g. espionage, financial gain)." + ) + + +class SourceRegion(BaseModel): + """Country or region from which the threat described in the report is known to originate.""" + + confidence: str = Field( + ..., + description="Confidence level in the attribution of this region as a threat source.", + ) + country: Optional[str] = Field(None, description="Country of threat origin.") + country_iso2: Optional[str] = Field( + None, description="ISO 3166 Alpha2 country code." + ) + description: Optional[str] = Field( + None, description="Additional context about the source region." + ) + first_seen: Optional[str] = Field( + ..., description="UTC timestamp when the source region was first seen." + ) + last_seen: Optional[str] = Field( + ..., description="UTC timestamp when the source region was last seen." + ) + region: Optional[str] = Field(None, description="Region of threat origin.") + source: Optional[str] = Field( + None, description="Supplier of this source region information." + ) + sub_region: Optional[str] = Field(None, description="Sub-region of threat origin.") + + +class TagDetail(BaseModel): + """Tags associated with the report with some additional context.""" + + confidence: str = Field( + ..., description="Confidence in the tag's association with the report." + ) + description: Optional[str] = Field( + None, description="Additional context about the tag." + ) + first_seen: Optional[str] = Field( + ..., description="UTC timestamp when the tag was first seen." + ) + last_seen: Optional[str] = Field( + ..., description="UTC timestamp when the tag was last seen." + ) + value: str = Field(..., description="Tag value.") + + +class TargetedIndustry(BaseModel): + """Industries and industry groups known to be targeted by the threat described in the report.""" + + confidence: str = Field( + ..., description="Confidence in the attribution of the targeted industry." + ) + description: Optional[str] = Field( + None, description="Additional info about the targeted industry." + ) + first_seen: Optional[str] = Field( + ..., description="UTC timestamp when the industry was first targeted." + ) + last_seen: Optional[str] = Field( + ..., description="UTC timestamp when the industry was last seen targeted." + ) + industry: Optional[str] = Field( + ..., description="Sub-industry targeted by the threat." + ) + industry_group: str = Field( + ..., description="Industry group targeted by the threat." + ) + source: Optional[str] = Field( + None, description="Supplier of this industry targeting information." + ) + + +class TargetedRegion(BaseModel): + """Regions and countries known to be targeted by the threat described in the report.""" + + confidence: str = Field( + ..., description="Confidence in the attribution of the targeted region." + ) + country: Optional[str] = Field(None, description="Country targeted by the threat.") + country_iso2: Optional[str] = Field( + None, description="ISO 3166 Alpha2 code for the country." + ) + description: Optional[str] = Field( + None, description="Additional context on the targeted region." + ) + first_seen: Optional[str] = Field( + ..., description="UTC timestamp when the region was first targeted." + ) + last_seen: Optional[str] = Field( + ..., description="UTC timestamp when the region was last seen targeted." + ) + region: Optional[str] = Field(None, description="Region targeted by the threat.") + source: Optional[str] = Field( + None, description="Supplier of this region targeting information." + ) + sub_region: Optional[str] = Field( + None, description="Sub-region targeted by the threat." + ) + + +class Technology(BaseModel): + """Common Platform Enumeration (CPE) objects referring to the vulnerability described by the report.""" + + cpe: Optional[str] = Field(None, description="CPE standardized product identifier.") + cpe_title: Optional[str] = Field( + None, description="Human-readable vendor and technology name." + ) + technology_name: Optional[str] = Field( + None, description="Technology affected by the vulnerability." + ) + vendor: Optional[str] = Field( + None, description="Vendor affected by the vulnerability." + ) + + +class ReportModel(BaseModel): + """Model representing a GTI report.""" + + report_id: Optional[str] = Field(None, description="Identifier of the report.") + name: str = Field(..., description="Title of the report.") + author: Optional[str] = Field(None, description="Author of the report.") + collection_type: str = Field( + ..., description="Type of object; always 'report' here." + ) + creation_date: int = Field(..., description="UTC timestamp of report creation.") + last_modification_date: int = Field( + ..., description="UTC timestamp of last report update." + ) + content: Optional[str] = Field(None, description="Full report content.") + executive_summary: Optional[str] = Field( + None, description="Summary of the report's content." + ) + autogenerated_summary: Optional[str] = Field( + None, description="ML-generated summary of the report." + ) + analyst_comment: Optional[str] = Field( + None, description="Comments made by GTI analysts." + ) + report_type: Optional[str] = Field( + None, description="Type of report: News, Actor Profile, OSINT, etc." + ) + report_confidence: Optional[str] = Field( + None, description="Confidence in the report's content/source." + ) + status: Optional[str] = Field( + None, + description="Status of attribute computation: PENDING_RECOMPUTE or COMPUTED.", + ) + link: Optional[str] = Field(None, description="URL to the original report.") + + version: Optional[int] = Field(None, description="Version number of the report.") + private: bool = Field(..., description="Whether the report is private.") + origin: Optional[str] = Field( + None, + description="Source of the information: Partner, Google TI, or Crowdsourced.", + ) + + affected_systems: Optional[List[str]] = Field( + None, description="Systems affected by the threat." + ) + intended_effects: Optional[List[str]] = Field( + None, description="Intended effects of the threat." + ) + targeted_informations: Optional[List[str]] = Field( + None, description="Types of info targeted by the threat." + ) + threat_categories: Optional[List[str]] = Field( + None, description="Threat categories based on IoCs." + ) + threat_scape: Optional[List[str]] = Field( + None, description="Topic areas covered by the report." + ) + top_icon_md5: Optional[List[str]] = Field( + None, description="MD5 hashes of the most frequent favicons/icons." + ) + recent_activity_relative_change: Optional[float] = Field( + None, description="Ratio of recent activity change (14-day interval)." + ) + recent_activity_summary: Optional[List[int]] = Field( + None, description="Time series of IoC activity (14-day)." + ) + + counters: Optional[Counters] = Field( + None, description="Counters for related indicators and metadata." + ) + aggregations: Optional[AggregationCommonalities] = Field( + None, description="Grouped common traits across related IoCs." + ) + motivations: Optional[List[Motivation]] = Field( + None, description="Motivations behind the threat actor’s behavior." + ) + source_regions_hierarchy: Optional[List[SourceRegion]] = Field( + None, description="Regions/countries of threat origin." + ) + tags_details: Optional[List[TagDetail]] = Field( + None, description="Tags applied to the report, with context." + ) + targeted_industries_tree: Optional[List[TargetedIndustry]] = Field( + None, description="Industries targeted by the threat." + ) + targeted_regions_hierarchy: Optional[List[TargetedRegion]] = Field( + None, description="Regions/countries targeted by the threat." + ) + technologies: Optional[List[Technology]] = Field( + None, description="Technologies and vendors affected by vulnerabilities." + ) + + +class Links(BaseModel): + """Model representing links to related resources.""" + + self: str + next: Optional[str] = Field(None, description="Link to the next page of results.") + + +class GTIReportMeta(BaseModel): + """Model representing metadata for a GTI report.""" + + count: int + cursor: Optional[str] = Field(None, description="Cursor for pagination.") + + +class GTIReportData(BaseModel): + """Model representing data for a GTI report.""" + + id: str + type: str + links: Links + attributes: Optional[ReportModel] + context_attributes: Dict[str, Any] + + +class GTIReportResponse(BaseModel): + """Model representing a response containing GTI report data.""" + + data: List[GTIReportData] = [] + meta: Optional[GTIReportMeta] = Field( + default=None, + description="Metadata for the response. May be absent when no data is returned.", + ) + links: Links diff --git a/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_threat_actor_model.py b/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_threat_actor_model.py new file mode 100644 index 0000000000..81881d152a --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_threat_actor_model.py @@ -0,0 +1,330 @@ +"""Model representing a Google Threat Intelligence Threat Actor.""" + +from typing import Any, Dict, List, Optional, Union + +from pydantic import BaseModel, Field + + +class AggregationCommonalities(BaseModel): + """Technical commonalities among all domains, files, IP addresses, and URLs tied to the threat actor.""" + + domains: Optional[Dict[str, Any]] = Field( + None, + description="Technical commonalities among all domains tied to the threat actor.", + ) + files: Optional[Dict[str, Any]] = Field( + None, + description="Technical commonalities among all files tied to the threat actor.", + ) + ip_addresses: Optional[Dict[str, Any]] = Field( + None, + description="Technical commonalities among all IP addresses tied to the threat actor.", + ) + urls: Optional[Dict[str, Any]] = Field( + None, + description="Technical commonalities among all URLs tied to the threat actor.", + ) + + +class Counters(BaseModel): + """Count of technical commonalities among all domains, files, IP addresses, and URLs tied to the threat actor.""" + + attack_techniques: int = Field( + ..., + description="Number of MITRE ATT&CK techniques associated with the threat actor.", + ) + domains: int = Field( + ..., description="Number of domains related to the threat actor." + ) + files: int = Field(..., description="Number of files related to the threat actor.") + iocs: int = Field( + ..., + description="Number of IoCs related to the threat actor (files + URLs + domains + IP addresses).", + ) + ip_addresses: int = Field( + ..., description="Number of IP addresses related to the threat actor." + ) + subscribers: int = Field( + ..., description="Number of users subscribed to the threat actor." + ) + urls: int = Field(..., description="Number of URLs related to the threat actor.") + + +class AltNameDetail(BaseModel): + """Alternative names/aliases by which the threat actor could be known.""" + + confidence: str = Field( + ..., + description="Confidence on the information or the attribution of the alternative name.", + ) + description: Optional[str] = Field( + None, description="Additional information related to the alternative name." + ) + first_seen: Optional[int] = Field( + None, + description="The first time the alternative name was attributed (UTC timestamp).", + ) + last_seen: Optional[int] = Field( + None, + description="The last time the alternative name was attributed (UTC timestamp).", + ) + value: str = Field(..., description="Alternative name/alias.") + + +class SeenDetail(BaseModel): + """Details about when the threat actor was first or last seen.""" + + confidence: str = Field( + ..., description="Confidence on the information or the attribution." + ) + description: Optional[str] = Field( + None, description="Additional information about the activity." + ) + first_seen: Optional[int] = Field( + None, description="First time this date was attributed (UTC timestamp)." + ) + last_seen: Optional[int] = Field( + None, description="Last time this date was attributed (UTC timestamp)." + ) + value: str = Field( + ..., + description="Date when the observation was made (YYYY-MM-DDTHH:mm:ssZ format).", + ) + + +class MergedActor(BaseModel): + """Actors confirmed to be part of a larger group (current threat actor).""" + + confidence: str = Field( + ..., + description="Confidence on the information or the attribution of the merged threat actor.", + ) + description: Optional[str] = Field( + None, description="Additional information about the merged actor." + ) + first_seen: Optional[int] = Field( + None, + description="First time this merged threat actor was attributed (UTC timestamp).", + ) + last_seen: Optional[int] = Field( + None, + description="Last time this merged threat actor was attributed (UTC timestamp).", + ) + value: str = Field(..., description="Name of the merged threat actor.") + + +class Motivation(BaseModel): + """Threat actor's motivations such as espionage, financial gain, etc.""" + + confidence: str = Field( + ..., + description="Confidence on the information or the attribution of the motivation.", + ) + description: Optional[str] = Field( + None, description="Additional information about the motivation." + ) + first_seen: Optional[int] = Field( + None, + description="First time this motivation was attributed (UTC timestamp).", + ) + last_seen: Optional[int] = Field( + None, + description="Last time this motivation was attributed (UTC timestamp).", + ) + value: str = Field(..., description="Threat actor's motivation.") + + +class SourceRegion(BaseModel): + """Country or region from which the threat actor is known to originate.""" + + confidence: str = Field( + ..., + description="Confidence on the information related to the source region.", + ) + country: Optional[str] = Field(None, description="Country of threat actor origin.") + country_iso2: Optional[str] = Field( + None, description="Source country in ISO 3166 Alpha2 code format." + ) + description: Optional[str] = Field( + None, description="Additional information about the source region." + ) + first_seen: Optional[int] = Field( + None, + description="First time this source region was attributed (UTC timestamp).", + ) + last_seen: Optional[int] = Field( + None, description="Last time this source region was attributed (UTC timestamp)." + ) + region: Optional[str] = Field(None, description="Region of threat actor origin.") + source: Optional[str] = Field(None, description="Information's supplier.") + sub_region: Optional[str] = Field( + None, description="Subregion of threat actor origin." + ) + + +class TagDetail(BaseModel): + """Tags associated with the threat actor with additional context.""" + + confidence: str = Field( + ..., description="Confidence on the tag association to the threat actor." + ) + description: Optional[str] = Field( + None, description="Additional information related to the tag." + ) + first_seen: Optional[int] = Field( + None, description="First time this tag was attributed (UTC timestamp)." + ) + last_seen: Optional[int] = Field( + None, description="Last time this tag was attributed (UTC timestamp)." + ) + value: str = Field(..., description="Value of the tag.") + + +class TargetedIndustry(BaseModel): + """Industries and industry groups known to be targeted by the threat actor.""" + + confidence: str = Field( + ..., description="Confidence on the industry targeted by the threat actor." + ) + description: Optional[str] = Field( + None, description="Additional information related to the targeted industry." + ) + first_seen: Optional[int] = Field( + None, + description="First time this targeted industry was associated (UTC timestamp).", + ) + industry: Optional[str] = Field( + None, description="Sub-industry targeted by the threat actor." + ) + industry_group: str = Field( + ..., description="Industry group targeted by the threat actor." + ) + last_seen: Optional[int] = Field( + None, + description="Last time this targeted industry was associated (UTC timestamp).", + ) + source: Optional[str] = Field(None, description="Information's supplier.") + + +class TargetedRegion(BaseModel): + """Regions and countries known to be targeted by the threat actor.""" + + confidence: str = Field( + ..., + description="Confidence on the threat actor's targeted region association.", + ) + country: Optional[str] = Field( + None, description="Country targeted by the threat actor." + ) + country_iso2: Optional[str] = Field( + None, description="Targeted country in ISO 3166 Alpha2 code format." + ) + description: Optional[str] = Field( + None, description="Additional information related to the targeted region." + ) + first_seen: Optional[int] = Field( + None, + description="First time this targeted region was associated (UTC timestamp).", + ) + last_seen: Optional[int] = Field( + None, + description="Last time this targeted region was associated (UTC timestamp).", + ) + region: Optional[str] = Field( + None, description="Region targeted by the threat actor." + ) + source: Optional[str] = Field(None, description="Information's supplier.") + sub_region: Optional[str] = Field( + None, description="Sub-region targeted by the threat actor." + ) + + +class ThreatActorModel(BaseModel): + """Model representing a GTI threat actor.""" + + name: str = Field(..., description="Threat actor's name.") + collection_type: Optional[str] = Field( + None, + description="Type of object; typically 'threat_actor'.", + ) + creation_date: int = Field( + ..., description="UTC timestamp of threat actor object creation." + ) + last_modification_date: int = Field( + ..., description="UTC timestamp of last threat actor update." + ) + description: Optional[str] = Field( + None, description="Description/context about the threat actor." + ) + status: Optional[str] = Field( + None, + description="Status of attribute computation: PENDING_RECOMPUTE or COMPUTED.", + ) + private: bool = Field( + ..., description="Whether the threat actor object is private." + ) + origin: Optional[str] = Field( + None, + description="Source of the information: Partner or Google Threat Intelligence.", + ) + + recent_activity_relative_change: Optional[float] = Field( + None, description="Ratio of recent activity change (14-day interval)." + ) + recent_activity_summary: Optional[List[int]] = Field( + None, description="Time series of IoC activity (14-day)." + ) + top_icon_md5: Optional[List[str]] = Field( + None, description="List of the 3 most frequent icons' MD5 hashes." + ) + + counters: Optional[Counters] = Field( + None, description="Counters for related indicators and metadata." + ) + aggregations: Optional[AggregationCommonalities] = Field( + None, description="Grouped common traits across related IoCs." + ) + alt_names_details: Optional[List[AltNameDetail]] = Field( + None, description="Alternative names/aliases for the threat actor." + ) + first_seen_details: Optional[List[SeenDetail]] = Field( + None, description="Information about when the threat actor was first seen." + ) + last_seen_details: Optional[List[SeenDetail]] = Field( + None, description="Information about when the threat actor was last seen." + ) + merged_actors: Optional[List[MergedActor]] = Field( + None, description="Actors confirmed to be part of this threat actor group." + ) + motivations: Optional[List[Motivation]] = Field( + None, + description="Threat actor's motivations such as espionage, financial gain, etc.", + ) + source_regions_hierarchy: Optional[List[SourceRegion]] = Field( + None, description="Regions/countries of threat actor origin." + ) + tags_details: Optional[List[TagDetail]] = Field( + None, description="Tags applied to the threat actor, with context." + ) + targeted_industries_tree: Optional[List[TargetedIndustry]] = Field( + None, description="Industries targeted by the threat actor." + ) + targeted_regions_hierarchy: Optional[List[TargetedRegion]] = Field( + None, description="Regions/countries targeted by the threat actor." + ) + + +class GTIThreatActorData(BaseModel): + """Model representing data for a GTI threat actor.""" + + id: str + type: Optional[str] = None + links: Optional[Dict[str, str]] = None + attributes: Optional[ThreatActorModel] = None + context_attributes: Optional[Dict[str, Any]] = None + + +class GTIThreatActorResponse(BaseModel): + """Model representing a response containing GTI threat actor data.""" + + data: Union[GTIThreatActorData, List[GTIThreatActorData]] diff --git a/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_urls_model.py b/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_urls_model.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_vulnerability_model.py b/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_vulnerability_model.py new file mode 100644 index 0000000000..1225fbaecb --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/custom/models/gti_reports/gti_vulnerability_model.py @@ -0,0 +1,87 @@ +"""Model representing a Google Threat Intelligence Vulnerability.""" + +from typing import Dict, List, Optional, Union + +from pydantic import BaseModel, Field + + +class CvssV3(BaseModel): + """Model representing the CVSS v3 scoring data.""" + + attack_complexity: Optional[str] = Field( + None, description="Attack complexity rating." + ) + attack_vector: Optional[str] = Field(None, description="Attack vector description.") + availability_impact: Optional[str] = Field( + None, description="Impact on availability." + ) + base_score: Optional[float] = Field(None, description="CVSS v3 base score.") + confidentiality_impact: Optional[str] = Field( + None, description="Impact on confidentiality." + ) + integrity_impact: Optional[str] = Field(None, description="Impact on integrity.") + privileges_required: Optional[str] = Field( + None, description="Required privileges level." + ) + scope: Optional[str] = Field(None, description="Scope of the vulnerability.") + user_interaction: Optional[str] = Field( + None, description="Required user interaction." + ) + vector_string: Optional[str] = Field(None, description="CVSS vector string.") + + +class Cvss(BaseModel): + """Model representing the CVSS data.""" + + cvssv3: Optional[CvssV3] = Field(None, description="CVSS v3 scoring data.") + + +class Epss(BaseModel): + """Model representing the EPSS data.""" + + score: Optional[float] = Field(None, description="EPSS score.") + percentile: Optional[float] = Field(None, description="EPSS percentile.") + + +class VulnerabilityModel(BaseModel): + """Model representing a Google Threat Intelligence Vulnerability.""" + + name: str = Field(..., description="Vulnerability's name.") + creation_date: int = Field( + ..., description="Creation date of the vulnerability (UTC timestamp)." + ) + cvss: Optional[Cvss] = Field(None, description="CVSS scoring data.") + epss: Optional[Epss] = Field(None, description="EPSS data.") + link: Optional[str] = Field(None, description="URL of the vulnerability.") + last_modification_date: int = Field( + ..., description="Date when the vulnerability was last updated (UTC timestamp)." + ) + description: Optional[str] = Field(None, description="Vulnerability's description.") + private: bool = Field( + False, description="Whether the vulnerability object is private." + ) + + +class GTIVulnerabilityData(BaseModel): + """Model representing data for a GTI vulnerability.""" + + id: str + type: str = Field("vulnerability") + links: Optional[Dict[str, str]] = None + attributes: Optional[VulnerabilityModel] = None + + +class GTIVulnerabilityReference(BaseModel): + """Model representing a reference to a GTI vulnerability (without full attributes).""" + + id: str + type: str + + +class GTIVulnerabilityResponse(BaseModel): + """Model representing a response containing GTI vulnerability data.""" + + data: Union[ + GTIVulnerabilityData, + List[Union[GTIVulnerabilityData, GTIVulnerabilityReference]], + ] diff --git a/external-import/google-ti-feeds/connector/src/octi/__init__.py b/external-import/google-ti-feeds/connector/src/octi/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/octi/configs/__init__.py b/external-import/google-ti-feeds/connector/src/octi/configs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/octi/configs/connector_config.py b/external-import/google-ti-feeds/connector/src/octi/configs/connector_config.py new file mode 100644 index 0000000000..a0d9f4c68f --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/octi/configs/connector_config.py @@ -0,0 +1,36 @@ +"""Connector-specific configuration definitions for OpenCTI external imports.""" + +from typing import ClassVar, Literal, Optional + +from connector.src.octi.interfaces.base_config import BaseConfig +from pydantic_settings import SettingsConfigDict + + +class ConnectorConfig(BaseConfig): + """Configuration for the connector.""" + + yaml_section: ClassVar[str] = "connector" + model_config = SettingsConfigDict(env_prefix="connector_") + + id: str + type: Literal["EXTERNAL_IMPORT"] = "EXTERNAL_IMPORT" + name: str = "Google Threat Intel Feeds" + scope: str = "report,location,identity" + log_level: Literal["debug", "info", "warn", "error"] = "info" + duration_period: str = "PT2H" + queue_threshold: int = 500 + tlp_level: Literal[ + "WHITE", + "GREEN", + "AMBER", + "RED", + "WHITE+STRICT", + "GREEN+STRICT", + "AMBER+STRICT", + "RED+STRICT", + ] = "AMBER+STRICT" + run_and_terminate: Optional[bool] = None + send_to_queue: Optional[bool] = None + send_to_directory: Optional[bool] = None + send_to_directory_path: Optional[str] = None + send_to_directory_retention: Optional[int] = None diff --git a/external-import/google-ti-feeds/connector/src/octi/configs/octi_config.py b/external-import/google-ti-feeds/connector/src/octi/configs/octi_config.py new file mode 100644 index 0000000000..f0bdb14843 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/octi/configs/octi_config.py @@ -0,0 +1,15 @@ +"""Global OpenCTI connector configuration—common settings for all connectors.""" + +from connector.src.octi.interfaces.base_config import BaseConfig +from pydantic_settings import SettingsConfigDict + + +class OctiConfig(BaseConfig): + """Configuration for the OpenCTI platform.""" + + yaml_section = "opencti" + + model_config = SettingsConfigDict(env_prefix="opencti_") + + url: str + token: str diff --git a/external-import/google-ti-feeds/connector/src/octi/connector.py b/external-import/google-ti-feeds/connector/src/octi/connector.py new file mode 100644 index 0000000000..7dd11a8a99 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/octi/connector.py @@ -0,0 +1,464 @@ +"""Core connector as defined in the OpenCTI connector template.""" + +import asyncio +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from uuid import uuid4 + +from connector.src.custom.batch_processor import BatchProcessor +from connector.src.custom.configs.gti_config import GTIConfig +from connector.src.custom.convert_to_stix import ConvertToSTIX +from connector.src.custom.exceptions import ( + GTIApiClientError, + GTIAsyncError, + GTIEntityConversionError, + GTIPartialDataProcessingError, + GTIWorkProcessingError, +) +from connector.src.custom.fetch_all import FetchAll +from connector.src.octi.work_manager import WorkManager +from connector.src.utils.api_engine.aio_http_client import AioHttpClient +from connector.src.utils.api_engine.api_client import ApiClient +from connector.src.utils.api_engine.circuit_breaker import CircuitBreaker +from connector.src.utils.api_engine.retry_request_strategy import RetryRequestStrategy + +if TYPE_CHECKING: + from pycti import OpenCTIConnectorHelper as OctiHelper # type: ignore + + from connector.src.octi.global_config import GlobalConfig + + +LOG_PREFIX = "[Connector]" + + +class Connector: + """Specifications of the external import connector. + + This class encapsulates the main actions, expected to be run by any external import connector. + Note that the attributes defined below will be complemented per each connector type. + This type of connector aim to fetch external data to create STIX bundle and send it in a RabbitMQ queue. + The STIX bundle in the queue will be processed by the workers. + This type of connector uses the basic methods of the helper. + + --- + + Attributes + - `config (ConfigConnector())`: + Initialize the connector with necessary configuration environment variables + + - `helper (OpenCTIConnectorHelper(config))`: + This is the helper to use. + ALL connectors have to instantiate the connector helper with configurations. + Doing this will do a lot of operations behind the scene. + + - `converter_to_stix (ConnectorConverter(helper))`: + Provide methods for converting various types of input data into STIX 2.1 objects. + + --- + + Best practices + - `self.helper.api.work.initiate_work(...)` is used to initiate a new work + - `self.helper.schedule_iso()` is used to encapsulate the main process in a scheduler + - `self.helper.connector_logger.[info/debug/warning/error]` is used when logging a message + - `self.helper.stix2_create_bundle(stix_objects)` is used when creating a bundle + - `self.helper.send_stix2_bundle(stix_objects_bundle)` is used to send the bundle to RabbitMQ + - `self.helper.set_state()` is used to set state + + """ + + def __init__(self, config: "GlobalConfig", helper: "OctiHelper") -> None: + """Initialize the connector with necessary configuration environment variables + and the helper to use. + + Arguments: + + ---------- + config : GlobalConfig + Configuration object containing the connector's configuration variables. + helper : OpenCTIConnectorHelper + This is the helper from the OpenCTI client python library. + + """ + self._config = config + self._helper = helper + self._logger = self._helper.connector_logger + self.work_manager = WorkManager(self._config, self._helper, self._logger) + + def _process_callback(self) -> None: + """Connector main process to collect intelligence. + + For now, it only imports reports from Google Threat Intelligence. + But it can be extended to import other types of intelligence in the future. + """ + error_flag = False + error_message = None + try: + gti_config = self._config.get_config_class(GTIConfig) + if gti_config.import_reports: + error_message = self._process_reports() + if error_message: + error_flag = True + except (KeyboardInterrupt, SystemExit): + error_message = "Connector stopped due to user interrupt" + self._logger.info( + f"{LOG_PREFIX} {error_message}...", + {"connector_name": self._helper.connect_name}, + ) + error_flag = True + except asyncio.CancelledError: + error_message = "Operation was cancelled" + self._logger.info( + f"{LOG_PREFIX} {error_message}.", + {"connector_name": self._helper.connect_name}, + ) + error_flag = True + except GTIWorkProcessingError as work_err: + error_message = f"Work processing error: {str(work_err)}" + work_id = getattr(work_err, "work_id", self.work_manager.get_current_work_id()) + self._logger.error( + f"{LOG_PREFIX} {error_message}", + meta={ + "error": str(work_err), + "work_id": work_id, + }, + ) + error_flag = True + except Exception as err: + error_message = f"An unexpected error occurred: {str(err)}" + self._logger.error(f"{LOG_PREFIX} {error_message}", meta={"error": str(err)}) + error_flag = True + finally: + self._logger.info( + f"{LOG_PREFIX} Connector stopped...", + {"connector_name": self._helper.connect_name}, + ) + self.work_manager.process_all_remaining_works( + error_flag=error_flag, error_message=error_message + ) + self._logger.info(f"{LOG_PREFIX} All remaining works marked to process.") + + def run(self) -> None: + """Run the main process encapsulated in a scheduler. + + It allows you to schedule the process to run at a certain intervals + This specific scheduler from the pycti connector helper will also check the queue size of a connector + If `CONNECTOR_QUEUE_THRESHOLD` is set, if the connector's queue size exceeds the queue threshold, + the connector's main process will not run until the queue is ingested and reduced sufficiently, + allowing it to restart during the next scheduler check. (default is 500MB) + It requires the `duration_period` connector variable in ISO-8601 standard format + Example: `CONNECTOR_DURATION_PERIOD=PT5M` => Will run the process every 5 minutes + """ + self._helper.schedule_iso( + message_callback=self._process_callback, + duration_period=self._config.connector_config.duration_period, + ) + + def _process_reports(self) -> Optional[Any]: + """Process GTI reports and related entities. + + Returns: + str: Error message if an error occurred, None otherwise + + """ + error_flag = False + error_message = None + self._logger.info(f"{LOG_PREFIX} Starting Google Threat Intel Feeds process") + + reports: List[Dict[str, Any]] = [] + related_entities: Dict[str, Any] = {} + latest_modified_date = None + fetch_task = None + + loop = None + try: + api_client = self._setup_api_client() + work_id = self.work_manager.initiate_work(name="Google Threat Intel Feeds - Reports") + state = self.work_manager.get_state() + gti_config = self._config.get_config_class(GTIConfig) + + fetcher, converter, batch_processor = self._create_fetcher_and_converter( + api_client, state, gti_config, work_id + ) + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + try: + fetch_task = asyncio.ensure_future(fetcher.fetch_all_data(), loop=loop) + self._logger.info(f"{LOG_PREFIX} Fetching data from Google Threat Intelligence API") + + try: + reports, related_entities, latest_modified_date = loop.run_until_complete( + fetch_task + ) + self._logger.info( + f"{LOG_PREFIX} Fetched {len(reports)} reports with related entities" + ) + + if not reports: + self._logger.info(f"{LOG_PREFIX} No reports to process") + return "" + + batch_processor.update_final_state() + self._logger.info( + f"{LOG_PREFIX} Processed {batch_processor.get_total_stix_objects_count()} STIX objects in {batch_processor.get_total_batches_processed()} batches" + ) + except (KeyboardInterrupt, SystemExit): + error_flag, error_message = self._handle_keyboard_interrupt( + loop, fetch_task, work_id, converter + ) + + except asyncio.CancelledError: + self._logger.info( + f"{LOG_PREFIX} Operation was cancelled.", + meta={"operation": "cancelled"}, + ) + + error_message = self._try_process_partial_data( + fetch_task, work_id, converter, "cancellation" + ) + except GTIEntityConversionError as conversion_err: + error_message = f"Entity conversion error: {str(conversion_err)}" + self._logger.error( + f"{LOG_PREFIX} {error_message}", + meta={ + "error": str(conversion_err), + "entity_type": getattr(conversion_err, "entity_type", None), + }, + ) + error_flag = True + + partial_error = self._try_process_partial_data( + fetch_task, work_id, converter, "conversion_error" + ) + if partial_error: + error_message = f"{error_message}. Additionally: {partial_error}" + except Exception as e: + error_message = f"An error occurred while processing the work: {str(e)}" + self._logger.error( + f"{LOG_PREFIX} An error occurred while processing the work.", + meta={"error": str(e)}, + ) + error_flag = True + + partial_error = self._try_process_partial_data( + fetch_task, work_id, converter, "exception" + ) + if partial_error: + error_message = f"{error_message}. Additionally: {partial_error}" + except GTIApiClientError as api_err: + error_message = f"API client setup error: {str(api_err)}" + self._logger.error( + f"{LOG_PREFIX} {error_message}", + meta={ + "error": str(api_err), + "component": getattr(api_err, "client_component", None), + }, + ) + error_flag = True + finally: + if loop is not None: + self._cleanup_event_loop(loop) + + self.work_manager.work_to_process( + work_id=work_id, + error_flag=error_flag, + error_message=error_message if error_flag else None, + ) + return error_message + + def _setup_api_client(self) -> ApiClient: + """Set up the API client with retry strategy and circuit breaker. + + Returns: + ApiClient: Configured API client + + Raises: + GTIApiClientError: If there's an error setting up the API client + + """ + try: + http_client = AioHttpClient(default_timeout=120, logger=self._logger) + breaker = CircuitBreaker(max_failures=5, cooldown_time=60) + limiter_config = { + "key": f"gti-api-{uuid4()}", + "max_requests": 60, + "period": 60, + } + retry_strategy = RetryRequestStrategy( + http=http_client, + breaker=breaker, + limiter=limiter_config, + hooks=None, + max_retries=5, + backoff=2, + logger=self._logger, + ) + return ApiClient(strategy=retry_strategy, logger=self._logger) + except Exception as e: + raise GTIApiClientError(f"Failed to set up API client: {str(e)}", "setup") from e + + def _create_fetcher_and_converter( + self, + api_client: ApiClient, + state: Dict[str, Any], + gti_config: GTIConfig, + work_id: str, + ) -> Tuple[FetchAll, ConvertToSTIX, BatchProcessor]: + """Create fetcher and converter instances. + + Args: + api_client: The API client for fetching data + state: Current connector state + gti_config: GTI configuration + work_id: OpenCTI work ID to send data as it's processed + + Returns: + tuple: (FetchAll instance, ConvertToSTIX instance, BatchProcessor instance) + + """ + converter = ConvertToSTIX( + tlp_level=self._config.connector_config.tlp_level.lower(), + logger=self._logger, + ) + + batch_processor = BatchProcessor( + work_manager=self.work_manager, + work_id=work_id, + converter=converter, + logger=self._logger, + ) + + fetcher = FetchAll(gti_config, api_client, state, self._logger, batch_processor) + return fetcher, converter, batch_processor + + def _handle_keyboard_interrupt( + self, + loop: asyncio.AbstractEventLoop, + fetch_task: Any, + work_id: str, + converter: ConvertToSTIX, + ) -> Tuple[bool, Optional[str]]: + """Handle keyboard interrupt by processing partial data. + + Args: + loop: Event loop + fetch_task: The fetch task + work_id: Work ID + converter: STIX converter + + Returns: + tuple: (error_flag, error_message) where error_flag is a bool and error_message is a string or None + + Raises: + GTIAsyncError: If there's an error during async operation handling + + """ + error_message = "User interrupted the connector operation" + self._logger.info(f"{LOG_PREFIX} Gracefully cancelling fetch operation...") + fetch_task.cancel() + try: + try: + if fetch_task.done() and not fetch_task.exception(): + reports, related_entities, latest_modified_date = fetch_task.result() + else: + reports, related_entities, latest_modified_date = loop.run_until_complete( + asyncio.wait_for(asyncio.shield(fetch_task), 2.0) + ) + + if reports: + self._logger.info( + f"{LOG_PREFIX} Processing {len(reports)} reports fetched before cancellation" + ) + current_work_id = self.work_manager.get_current_work_id() or work_id + + cancel_batch_processor = BatchProcessor( + work_manager=self.work_manager, + work_id=current_work_id, + converter=converter, + logger=self._logger, + ) + cancel_batch_processor.process_batch(reports, related_entities) + if latest_modified_date: + cancel_batch_processor.set_latest_modified_date(latest_modified_date) + cancel_batch_processor.update_final_state() + except (asyncio.CancelledError, asyncio.TimeoutError) as err: + error_message = f"Could not retrieve partial results: {str(err)}" + self._logger.info(f"{LOG_PREFIX} {error_message}") + raise GTIAsyncError( + error_message, "interrupt_handler", {"work_id": work_id} + ) from err + except (asyncio.CancelledError, asyncio.TimeoutError) as e: + error_message = f"Async operation failed during interrupt handling: {str(e)}" + raise GTIAsyncError(error_message, "interrupt_handler") from e + except GTIAsyncError: + raise + except Exception as e: + error_message = f"Unexpected error during interrupt handling: {str(e)}" + raise GTIAsyncError(error_message, "interrupt_handler") from e + return True, error_message + + def _try_process_partial_data( + self, fetch_task: Any, work_id: str, converter: ConvertToSTIX, source: str + ) -> Optional[Any]: + """Try to process any data fetched before an error or cancellation. + + Args: + fetch_task: The fetch task that might have partial results + work_id: The work ID + converter: STIX converter + source: String describing the source of interruption ("cancellation" or "exception") + + Returns: + str: Optional error message if an error occurred + + """ + error_message = None + try: + if fetch_task and fetch_task.done() and not fetch_task.exception(): + reports, related_entities, latest_modified_date = fetch_task.result() + if reports: + current_work_id = self.work_manager.get_current_work_id() or work_id + partial_batch_processor = BatchProcessor( + work_manager=self.work_manager, + work_id=current_work_id, + converter=converter, + logger=self._logger, + ) + partial_batch_processor.process_batch(reports, related_entities) + if latest_modified_date: + partial_batch_processor.set_latest_modified_date(latest_modified_date) + partial_batch_processor.update_final_state() + except Exception as process_err: + error_message = f"Error processing partial data after {source}: {str(process_err)}" + self._logger.error( + f"{LOG_PREFIX} {error_message}", + meta={"error": str(process_err)}, + ) + reports_count = len(reports) if "reports" in locals() else None + raise GTIPartialDataProcessingError( + str(process_err), + work_id, + source, + reports_count, + {"exception_type": type(process_err).__name__}, + ) from process_err + return error_message + + def _cleanup_event_loop(self, loop: Any) -> None: + """Clean up the event loop by cancelling pending tasks and closing it. + + Args: + loop: The event loop to clean up + + """ + pending = asyncio.all_tasks(loop=loop) + for task in pending: + task.cancel() + + if pending: + try: + loop.run_until_complete(asyncio.wait(pending, timeout=1.0)) + except (asyncio.CancelledError, Exception) as e: + self._logger.debug(f"{LOG_PREFIX} Exception while cleaning up event loop: {str(e)}") + + loop.close() diff --git a/external-import/google-ti-feeds/connector/src/octi/exceptions/__init__.py b/external-import/google-ti-feeds/connector/src/octi/exceptions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/octi/exceptions/configuration_error.py b/external-import/google-ti-feeds/connector/src/octi/exceptions/configuration_error.py new file mode 100644 index 0000000000..2f373e6166 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/octi/exceptions/configuration_error.py @@ -0,0 +1,12 @@ +"""Custom exception for configuration errors in the connectors.""" + +from typing import Any + + +class ConfigurationError(Exception): + """Raised when the application configuration is invalid.""" + + def __init__(self, message: str, *, errors: Any = None): + """Initialize the ConfigurationError with a message and original traceback.""" + super().__init__(message) + self.errors = errors diff --git a/external-import/google-ti-feeds/connector/src/octi/global_config.py b/external-import/google-ti-feeds/connector/src/octi/global_config.py new file mode 100644 index 0000000000..32c815d78a --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/octi/global_config.py @@ -0,0 +1,90 @@ +"""Handles the global configuration for the connector.""" + +from typing import TYPE_CHECKING, Any, Dict, Type + +from connector.src.octi.configs.connector_config import ConnectorConfig +from connector.src.octi.configs.octi_config import OctiConfig +from connector.src.octi.exceptions.configuration_error import ConfigurationError +from pydantic_core import ValidationError + +if TYPE_CHECKING: + from connector.src.octi.interfaces.base_config import BaseConfig + + +class GlobalConfig: + """Global configuration for the connector.""" + + def __init__(self) -> None: + """Initialize the global configuration.""" + self.instanciate_configs: Dict[str, Any] = {} + try: + self.octi_config = OctiConfig() + except ValidationError as e: + raise ConfigurationError( + "Error loading the OpenCTI configuration", errors=e.errors + ) from e + try: + self.connector_config = ConnectorConfig() + except ValidationError as e: + raise ConfigurationError( + "Error loading the connector configuration", errors=e.errors + ) from e + + self.instanciate_configs.update( + { + "opencti": ( + self.octi_config.model_dump(exclude_none=True), + self.octi_config, + ) + } + ) + self.instanciate_configs.update( + { + "connector": ( + self.connector_config.model_dump(exclude_none=True), + self.connector_config, + ) + } + ) + + self.to_dict() + + def add_config_class(self, config_class: Type["BaseConfig"]) -> None: + """Add a configuration class to the global configuration.""" + try: + config_instance = config_class() + except ValidationError as e: + raise ConfigurationError( + f"Error loading the {config_class.__name__} configuration", + errors=e.errors, + ) from e + self.instanciate_configs.update( + { + config_class.yaml_section.lower(): ( + config_instance.model_dump(exclude_none=True), + config_instance, + ) + } + ) + + self.to_dict() + + def get_config_class(self, config_class: Type["BaseConfig"]) -> Any: + """Get a configuration class from the global configuration.""" + config_name = config_class.yaml_section.lower() + if config_name in self.instanciate_configs: + return self.instanciate_configs[config_name][1] + else: + raise ConfigurationError( + f"Configuration class {config_name} not found in global configuration." + ) + + def to_dict(self) -> Dict[str, Dict[str, Any]]: + """Convert the configuration to a dictionary.""" + dicc: Dict[str, Dict[str, Any]] = {} + for config_name, tuples in self.instanciate_configs.items(): + dicc[config_name] = {} + for key, value in tuples[0].items(): + dicc[config_name].update({key: value}) + + return dicc diff --git a/external-import/google-ti-feeds/connector/src/octi/interfaces/__init__.py b/external-import/google-ti-feeds/connector/src/octi/interfaces/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/octi/interfaces/base_config.py b/external-import/google-ti-feeds/connector/src/octi/interfaces/base_config.py new file mode 100644 index 0000000000..685b7f4271 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/octi/interfaces/base_config.py @@ -0,0 +1,69 @@ +"""Base configuration class for all connector configs.""" + +import os +from abc import ABC +from pathlib import Path +from typing import ClassVar, Tuple, Type + +import yaml +from pydantic_settings import BaseSettings, PydanticBaseSettingsSource + +SettingsSource = PydanticBaseSettingsSource +SettingsSources = Tuple[SettingsSource, ...] + +EMPTY_SOURCES = ( # type: ignore + lambda: {}, + lambda: {}, + lambda: {}, + lambda: {}, +) + + +class BaseConfig(ABC, BaseSettings): + """Base configuration class for all connector configs.""" + + yaml_section: ClassVar[str] + + @classmethod + def settings_customise_sources( + cls: Type["BaseConfig"], + settings_cls: Type[BaseSettings], + init_settings: SettingsSource, + env_settings: SettingsSource, + dotenv_settings: SettingsSource, + file_secret_settings: SettingsSource, + ) -> SettingsSources: + """Customise the settings sources so that in dev mode we only load from config.yml. + + Parameters + ---------- + settings_cls + The Pydantic settings class being initialized. + init_settings + The default initializer source (usually the class defaults). + env_settings + The environment‐variable source. + dotenv_settings + The `.env`‐file source. + file_secret_settings + Secrets from mounted files. + + Returns + ------- + A tuple of callables producing dicts for Pydantic to merge. + + """ + if os.getenv("CONNECTOR_DEV_MODE", "").lower() == "true": + path = Path("config.yml") + if not path.exists(): + raise FileNotFoundError("Config.yml not found.") + + raw = yaml.safe_load(path.read_text()) + data = raw.get(cls.yaml_section) + + if not data: + return EMPTY_SOURCES # type: ignore + + return (lambda: data,) # type: ignore + + return (init_settings, env_settings, dotenv_settings, file_secret_settings) diff --git a/external-import/google-ti-feeds/connector/src/octi/work_manager.py b/external-import/google-ti-feeds/connector/src/octi/work_manager.py new file mode 100644 index 0000000000..7780a2d4c3 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/octi/work_manager.py @@ -0,0 +1,196 @@ +"""The module will contains method to manage OpenCTI Works related tasks.""" + +import logging +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Dict, Optional + +if TYPE_CHECKING: + from logging import Logger + + from connector.src.octi.global_config import GlobalConfig + from pycti import OpenCTIConnectorHelper as OctiHelper # type: ignore + +LOG_PREFIX = "[Work Manager]" + + +class WorkManager: + """The class will contains method to manage OpenCTI Works related tasks.""" + + def __init__( + self, + config: "GlobalConfig", + helper: "OctiHelper", + logger: Optional["Logger"] = None, + ) -> None: + """Initialize the WorkManager class. + + Args: + config (dict): The configuration dictionary. + helper (Helper): The helper object. + logger (logging.Logger): The logger object. + + """ + self._config = config + self._helper = helper + self._logger = logger or logging.getLogger(__name__) + self._current_work_id: Optional[str] = None + + def get_state(self) -> Dict[str, Any]: + """Get the current state dict of the Connector. + + Returns: + dict: The current state of the Connector. + + """ + self._helper.force_ping() + return self._helper.get_state() or {} + + def _is_valid_iso_format(self, date_string: str) -> bool: + """Check if a string is a valid ISO format date. + + Args: + date_string (str): The date string to check + + Returns: + bool: True if valid ISO format, False otherwise + + """ + try: + datetime.fromisoformat(date_string.replace("Z", "+00:00")) + return True + except ValueError: + return False + + def get_current_work_id(self) -> Optional[str]: + """Get the current work ID. + + Returns: + Optional[str]: The current work ID or None if no work is currently active. + + """ + return self._current_work_id + + def set_current_work_id(self, work_id: str) -> None: + """Set the current work ID. + + Args: + work_id (str): The work ID to set as current. + + """ + self._current_work_id = work_id + self._logger.info(f"{LOG_PREFIX} Current work ID set to {work_id}") + + def update_state( + self, state_key: str, date_str: str = "", error_flag: bool = False + ) -> None: + """Update the state of the connector. + + Args: + error_flag (bool): Whether the work finished in error. + state_key (str): The key of the state to update. + date_str (str, optional): The date string. Defaults to "". + + """ + if not error_flag: + current_state = self.get_state() + now = datetime.now(timezone.utc).isoformat() + if date_str != "" and isinstance(date_str, str): + if self._is_valid_iso_format(date_str): + now = date_str + elif "T" not in date_str or not ("+" in date_str or "Z" in date_str): + parsed_date = datetime.fromisoformat( + date_str.replace("Z", "+00:00") + ) + now = parsed_date.isoformat() + current_state[state_key] = now + self._helper.set_state(state=current_state) + self._helper.force_ping() + self._logger.info(f"{LOG_PREFIX} Updated state for {state_key} to {now}") + + def initiate_work(self, name: str, work_counter: Optional[int] = None) -> str: + """Initiate a new work for the Connector. + + Args: + name (str): The name of the work. + work_counter (Optional[int]): The counter for the work. + + Returns: + str: The ID of the initiated work. + + """ + if work_counter is not None: + name = f"{name} #({work_counter})" + work_id: str = self._helper.api.work.initiate_work( + self._helper.connect_id, name + ) + self._current_work_id = work_id + self._logger.info(f"{LOG_PREFIX} Initiated work {work_id} for {name}") + return work_id + + def work_to_process( + self, + work_id: str, + error_flag: bool = False, + error_message: Optional[str] = None, + ) -> None: + """Work to process. + + Args: + work_id (str): The ID of the work. + error_flag (bool): Whether the work finished in error. + error_message (Optional[str]): Specific error message to report. Defaults to None. + + """ + message = "Connector's work finished gracefully" + if error_flag and error_message: + message = f"Error: {error_message}" + + self._helper.api.work.to_processed( + work_id=work_id, + message=message, + in_error=error_flag, + ) + if self._current_work_id == work_id: + self._current_work_id = None + self._logger.info(f"{LOG_PREFIX} Work {work_id} marked to be processed") + + def process_all_remaining_works( + self, error_flag: bool = False, error_message: Optional[str] = None + ) -> None: + """Process all remaining works and update the state. + + Args: + error_flag (bool): Whether the work finished in error. + error_message (Optional[str]): Specific error message to report. Defaults to None. + + """ + works = self._helper.api.work.get_connector_works( + connector_id=self._helper.connect_id + ) + for work in works: + if work["status"] != "complete": + self.work_to_process( + work_id=work["id"], + error_flag=error_flag, + error_message=error_message, + ) + self._current_work_id = None + self._logger.info(f"{LOG_PREFIX} All remaining works marked to be process.") + + def send_bundle(self, work_id: str, bundle: Any) -> None: + """Send a bundle to OpenCTI. + + Args: + work_id (str): The ID of the work. + bundle (dict): The bundle to send. + + """ + bundle_json = self._helper.stix2_create_bundle(bundle) + bundles_sent = self._helper.send_stix2_bundle( + bundle=bundle_json, + work_id=work_id, + ) + self._logger.info( + f"{LOG_PREFIX} STIX objects sent to OpenCTI queue.", + {"bundles_sent": str(len(bundles_sent))}, + ) diff --git a/external-import/google-ti-feeds/connector/src/stix/__init__.py b/external-import/google-ti-feeds/connector/src/stix/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/stix/octi/__init__.py b/external-import/google-ti-feeds/connector/src/stix/octi/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/stix/octi/models/__init__.py b/external-import/google-ti-feeds/connector/src/stix/octi/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/stix/octi/models/attack_pattern_model.py b/external-import/google-ti-feeds/connector/src/stix/octi/models/attack_pattern_model.py new file mode 100644 index 0000000000..d7a4a225a1 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/octi/models/attack_pattern_model.py @@ -0,0 +1,72 @@ +"""The module contains the OctiAttackPatternModel class, which represents an OpenCTI Attack Pattern.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +from connector.src.stix.v21.models.cdts.kill_chain_phase_model import ( + KillChainPhaseModel, +) +from connector.src.stix.v21.models.sdos.attack_pattern_model import AttackPatternModel + + +class OctiAttackPatternModel: + """Model for creating OpenCTI Attack Pattern objects.""" + + @staticmethod + def create( + name: str, + mitre_id: str, + organization_id: str, + marking_ids: list[str], + description: Optional[str] = None, + aliases: Optional[List[str]] = None, + first_seen: Optional[datetime] = None, + last_seen: Optional[datetime] = None, + kill_chain_phases: Optional[List[KillChainPhaseModel]] = None, + labels: Optional[List[str]] = None, + external_references: Optional[List[Dict[str, Any]]] = None, + **kwargs: Any, + ) -> AttackPatternModel: + """Create an Attack Pattern model. + + Args: + name: The name of the attack pattern + mitre_id: MITRE ATT&CK ID for the attack pattern + organization_id: The ID of the organization that created this attack pattern + marking_ids: List of marking definition IDs to apply to the attack pattern + description: Description of the attack pattern + aliases: Alternative names for the attack pattern + first_seen: First time the attack pattern was observed + last_seen: Last time the attack pattern was observed + kill_chain_phases: Kill chain phases associated with the attack pattern + labels: Labels to apply to the attack pattern + external_references: External references related to the attack pattern + **kwargs: Additional arguments to pass to AttackPatternModel + + Returns: + AttackPatternModel: The created attack pattern model + + """ + custom_properties = {} + custom_properties["x_mitre_id"] = mitre_id + + data = { + "type": "attack-pattern", + "spec_version": "2.1", + "custom_properties": custom_properties, + "created": kwargs.pop("created", datetime.now()), + "modified": kwargs.pop("modified", datetime.now()), + "name": name, + "description": description, + "aliases": aliases, + "first_seen": first_seen, + "last_seen": last_seen, + "kill_chain_phases": kill_chain_phases, + "labels": labels, + "external_references": external_references, + "created_by_ref": organization_id, + "object_marking_refs": marking_ids, + **kwargs, + } + + return AttackPatternModel(**data) diff --git a/external-import/google-ti-feeds/connector/src/stix/octi/models/identity_author_model.py b/external-import/google-ti-feeds/connector/src/stix/octi/models/identity_author_model.py new file mode 100644 index 0000000000..506e3bcdd2 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/octi/models/identity_author_model.py @@ -0,0 +1,39 @@ +"""The module contains the OctiIdentityAuthorModel class, which represents an OpenCTI Author Identity.""" + +from datetime import datetime +from typing import Any + +from connector.src.stix.v21.models.ovs.identity_class_ov_enums import IdentityClassOV +from connector.src.stix.v21.models.sdos.identity_model import IdentityModel + + +class OctiIdentityAuthorModel: + """Model for creating OpenCTI Author Identity objects.""" + + @staticmethod + def create(name: str, organization_id: str, **kwargs: Any) -> IdentityModel: + """Create an Author Identity model. + + Args: + name: The name of the author + organization_id: The ID of the organization that created this author identity + **kwargs: Additional arguments to pass to IdentityModel + + Returns: + IdentityModel: The created identity model + + """ + identity_class = IdentityClassOV.ORGANIZATION + + data = { + "type": "identity", + "spec_version": "2.1", + "created": kwargs.pop("created", datetime.now()), + "modified": kwargs.pop("modified", datetime.now()), + "name": name, + "identity_class": identity_class, + "created_by_ref": organization_id, + **kwargs, + } + + return IdentityModel(**data) diff --git a/external-import/google-ti-feeds/connector/src/stix/octi/models/identity_organization_model.py b/external-import/google-ti-feeds/connector/src/stix/octi/models/identity_organization_model.py new file mode 100644 index 0000000000..676998ac54 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/octi/models/identity_organization_model.py @@ -0,0 +1,63 @@ +"""The module contains the OctiOrganizationModel class, which represents an OpenCTI Organization Identity.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +from connector.src.stix.v21.models.ovs.identity_class_ov_enums import IdentityClassOV +from connector.src.stix.v21.models.sdos.identity_model import IdentityModel + + +class OctiOrganizationModel: + """Model for creating OpenCTI Organization Identity objects.""" + + @staticmethod + def create( + name: str, + description: Optional[str] = None, + contact_information: Optional[str] = None, + organization_type: Optional[str] = None, + reliability: Optional[str] = None, + aliases: Optional[List[str]] = None, + **kwargs: Any, + ) -> IdentityModel: + """Create an Organization Identity model with OpenCTI custom properties. + + Args: + name: The name of the organization + description: Description of the organization + contact_information: Contact details for the organization + organization_type: OpenCTI organization type (e.g., 'vendor') + reliability: OpenCTI reliability level + aliases: List of alternative names for the organization + **kwargs: Additional arguments to pass to IdentityModel + + Returns: + IdentityModel: The created identity model which can be converted to STIX using to_stix2_object() + + """ + custom_properties: Dict[str, Any] = {} + if organization_type: + custom_properties["x_opencti_organization_type"] = organization_type + + custom_properties["x_opencti_reliability"] = reliability + + if aliases: + custom_properties["x_opencti_aliases"] = aliases + + existing_custom = kwargs.pop("custom_properties", {}) + custom_properties.update(existing_custom) + + data = { + "type": "identity", + "spec_version": "2.1", + "created": kwargs.pop("created", datetime.now()), + "modified": kwargs.pop("modified", datetime.now()), + "name": name, + "description": description, + "contact_information": contact_information, + "identity_class": IdentityClassOV.ORGANIZATION, + "custom_properties": custom_properties, + **kwargs, + } + + return IdentityModel(**data) diff --git a/external-import/google-ti-feeds/connector/src/stix/octi/models/identity_sector_model.py b/external-import/google-ti-feeds/connector/src/stix/octi/models/identity_sector_model.py new file mode 100644 index 0000000000..0ac65d2f0c --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/octi/models/identity_sector_model.py @@ -0,0 +1,47 @@ +"""The module contains the OctiIdentitySectorModel class, which represents an OpenCTI Identity Sector.""" + +from datetime import datetime +from typing import Any, Optional + +from connector.src.stix.v21.models.ovs.identity_class_ov_enums import IdentityClassOV +from connector.src.stix.v21.models.sdos.identity_model import IdentityModel + + +class OctiIdentitySectorModel: + """Model for creating OpenCTI Identity Sector objects.""" + + @staticmethod + def create( + name: str, + organization_id: str, + marking_ids: list[str], + description: Optional[str] = None, + **kwargs: Any, + ) -> IdentityModel: + """Create an Identity Sector model. + + Args: + name: The name of the sector + organization_id: The ID of the organization that created this sector + marking_ids: List of marking definition IDs to apply to the sector + description: Description of the sector + **kwargs: Additional arguments to pass to IdentityModel + + Returns: + IdentityModel: The created identity model + + """ + data = { + "type": "identity", + "spec_version": "2.1", + "created": kwargs.pop("created", datetime.now()), + "modified": kwargs.pop("modified", datetime.now()), + "name": name, + "description": description, + "identity_class": IdentityClassOV.CLASS_, + "created_by_ref": organization_id, + "object_marking_refs": marking_ids, + **kwargs, + } + + return IdentityModel(**data) diff --git a/external-import/google-ti-feeds/connector/src/stix/octi/models/intrusion_set_model.py b/external-import/google-ti-feeds/connector/src/stix/octi/models/intrusion_set_model.py new file mode 100644 index 0000000000..d10333fd18 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/octi/models/intrusion_set_model.py @@ -0,0 +1,69 @@ +"""The module contains the OctiIntrusionSetModel class, which represents an OpenCTI Intrusion Set.""" + +from datetime import datetime +from typing import Any, List, Optional + +from connector.src.stix.v21.models.sdos.intrusion_set_model import IntrusionSetModel + + +class OctiIntrusionSetModel: + """Model for creating OpenCTI Intrusion Set objects.""" + + @staticmethod + def create( + name: str, + organization_id: str, + marking_ids: list[str], + description: Optional[str] = None, + aliases: Optional[List[str]] = None, + first_seen: Optional[datetime] = None, + last_seen: Optional[datetime] = None, + goals: Optional[List[str]] = None, + resource_level: Optional[str] = None, + primary_motivation: Optional[str] = None, + secondary_motivations: Optional[List[str]] = None, + labels: Optional[List[str]] = None, + **kwargs: Any, + ) -> IntrusionSetModel: + """Create an Intrusion Set model. + + Args: + name: The name of the intrusion set + organization_id: The ID of the organization that created this intrusion set + marking_ids: List of marking definition IDs to apply to the intrusion set + description: Description of the intrusion set + aliases: Alternative names for the intrusion set + first_seen: First time the intrusion set was observed + last_seen: Last time the intrusion set was observed + goals: High-level goals of the intrusion set + resource_level: Resource level of the intrusion set + primary_motivation: Primary motivation of the intrusion set + secondary_motivations: Secondary motivations of the intrusion set + labels: Labels to apply to the intrusion set + **kwargs: Additional arguments to pass to IntrusionSetModel + + Returns: + IntrusionSetModel: The created intrusion set model + + """ + data = { + "type": "intrusion-set", + "spec_version": "2.1", + "created": kwargs.pop("created", datetime.now()), + "modified": kwargs.pop("modified", datetime.now()), + "name": name, + "description": description, + "aliases": aliases, + "first_seen": first_seen, + "last_seen": last_seen, + "goals": goals, + "resource_level": resource_level, + "primary_motivation": primary_motivation, + "secondary_motivations": secondary_motivations, + "labels": labels, + "created_by_ref": organization_id, + "object_marking_refs": marking_ids, + **kwargs, + } + + return IntrusionSetModel(**data) diff --git a/external-import/google-ti-feeds/connector/src/stix/octi/models/location_model.py b/external-import/google-ti-feeds/connector/src/stix/octi/models/location_model.py new file mode 100644 index 0000000000..cac865ad9b --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/octi/models/location_model.py @@ -0,0 +1,95 @@ +"""The module contains the OctiLocationModel class, which represents an OpenCTI Location.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +from connector.src.stix.v21.models.ovs.region_ov_enums import RegionOV +from connector.src.stix.v21.models.sdos.location_model import LocationModel + + +class OctiLocationModel: + """Model for creating OpenCTI Location objects.""" + + @staticmethod + def create_country( + name: str, + country_code: str, + organization_id: str, + marking_ids: List[str], + description: Optional[str] = None, + **kwargs: Any, + ) -> LocationModel: + """Create a Country Location model with OpenCTI custom properties. + + Args: + name: The name of the country + country_code: The ISO 3166-1 alpha-2 country code + organization_id: The ID of the organization that created this location + marking_ids: List of marking definition IDs to apply to the location + description: Description of the location + **kwargs: Additional arguments to pass to LocationModel + + Returns: + LocationModel: The created location model + + """ + custom_properties: Dict[str, Any] = kwargs.pop("custom_properties", {}) + custom_properties["x_opencti_location_type"] = "Country" + + data = { + "type": "location", + "spec_version": "2.1", + "created": kwargs.pop("created", datetime.now()), + "modified": kwargs.pop("modified", datetime.now()), + "name": name, + "description": description, + "country": country_code, + "created_by_ref": organization_id, + "object_marking_refs": marking_ids, + "custom_properties": custom_properties, + **kwargs, + } + + return LocationModel(**data) + + @staticmethod + def create_region( + name: str, + region_value: RegionOV, + organization_id: str, + marking_ids: List[str], + description: Optional[str] = None, + **kwargs: Any, + ) -> LocationModel: + """Create a Region Location model with OpenCTI custom properties. + + Args: + name: The name of the region + region_value: The region value from RegionOV enum + organization_id: The ID of the organization that created this location + marking_ids: List of marking definition IDs to apply to the location + description: Description of the location + **kwargs: Additional arguments to pass to LocationModel + + Returns: + LocationModel: The created location model + + """ + custom_properties: Dict[str, Any] = kwargs.pop("custom_properties", {}) + custom_properties["x_opencti_location_type"] = "Region" + + data = { + "type": "location", + "spec_version": "2.1", + "created": kwargs.pop("created", datetime.now()), + "modified": kwargs.pop("modified", datetime.now()), + "name": name, + "description": description, + "region": region_value, + "created_by_ref": organization_id, + "object_marking_refs": marking_ids, + "custom_properties": custom_properties, + **kwargs, + } + + return LocationModel(**data) diff --git a/external-import/google-ti-feeds/connector/src/stix/octi/models/malware_model.py b/external-import/google-ti-feeds/connector/src/stix/octi/models/malware_model.py new file mode 100644 index 0000000000..fd87aaf07e --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/octi/models/malware_model.py @@ -0,0 +1,70 @@ +"""The module contains the OctiMalwareModel class, which represents an OpenCTI Malware.""" + +from datetime import datetime +from typing import Any, List, Optional + +from connector.src.stix.v21.models.cdts.kill_chain_phase_model import ( + KillChainPhaseModel, +) +from connector.src.stix.v21.models.ovs.malware_type_ov_enums import MalwareTypeOV +from connector.src.stix.v21.models.sdos.malware_model import MalwareModel + + +class OctiMalwareModel: + """Model for creating OpenCTI Malware objects.""" + + @staticmethod + def create( + name: str, + organization_id: str, + marking_ids: list[str], + malware_types: List[MalwareTypeOV], + is_family: bool = True, + description: Optional[str] = None, + aliases: Optional[List[str]] = None, + first_seen: Optional[datetime] = None, + last_seen: Optional[datetime] = None, + kill_chain_phases: Optional[List[KillChainPhaseModel]] = None, + labels: Optional[List[str]] = None, + **kwargs: Any, + ) -> MalwareModel: + """Create a Malware model. + + Args: + name: The name of the malware + organization_id: The ID of the organization that created this malware + marking_ids: List of marking definition IDs to apply to the malware + malware_types: List of malware types + is_family: Whether this is a malware family (True) or instance (False) + description: Description of the malware + aliases: Alternative names for the malware + first_seen: First time the malware was observed + last_seen: Last time the malware was observed + kill_chain_phases: Kill chain phases associated with the malware + labels: Labels to apply to the malware + **kwargs: Additional arguments to pass to MalwareModel + + Returns: + MalwareModel: The created malware model + + """ + data = { + "type": "malware", + "spec_version": "2.1", + "created": kwargs.pop("created", datetime.now()), + "modified": kwargs.pop("modified", datetime.now()), + "name": name, + "description": description, + "malware_types": malware_types, + "is_family": is_family, + "aliases": aliases, + "first_seen": first_seen, + "last_seen": last_seen, + "kill_chain_phases": kill_chain_phases, + "labels": labels, + "created_by_ref": organization_id, + "object_marking_refs": marking_ids, + **kwargs, + } + + return MalwareModel(**data) diff --git a/external-import/google-ti-feeds/connector/src/stix/octi/models/relationship_model.py b/external-import/google-ti-feeds/connector/src/stix/octi/models/relationship_model.py new file mode 100644 index 0000000000..c5411f0620 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/octi/models/relationship_model.py @@ -0,0 +1,109 @@ +"""The module contains the OctiRelationshipModel class, which represents an OpenCTI Relationship.""" + +from datetime import datetime +from typing import Any, List, Optional + +from connector.src.stix.v21.models.sros.relationship_model import RelationshipModel + + +class OctiRelationshipModel: + """Model for creating OpenCTI Relationship objects.""" + + @staticmethod + def create( + relationship_type: str, + source_ref: str, + target_ref: str, + organization_id: str, + marking_ids: List[str], + created: datetime, + modified: datetime, + description: Optional[str] = None, + **kwargs: Any, + ) -> RelationshipModel: + """Create a Relationship model with OpenCTI custom properties. + + Args: + relationship_type: The type of relationship (e.g., 'targets', 'indicates') + source_ref: The ID of the source entity + target_ref: The ID of the target entity + organization_id: The ID of the organization that created this relationship + marking_ids: List of marking definition IDs to apply to the relationship + created: When the relationship was created + modified: When the relationship was last modified + description: Description of the relationship + **kwargs: Additional arguments to pass to RelationshipModel + + Returns: + RelationshipModel: The created relationship model + + """ + data = { + "type": "relationship", + "spec_version": "2.1", + "created": created, + "modified": modified, + "relationship_type": relationship_type, + "source_ref": source_ref, + "target_ref": target_ref, + "description": description, + "created_by_ref": organization_id, + "object_marking_refs": marking_ids, + **kwargs, + } + + return RelationshipModel(**data) + + @staticmethod + def create_from_report( + relationship_type: str, + report_id: str, + target_ref: str, + organization_id: str, + marking_ids: List[str], + created: datetime, + modified: datetime, + report_name: Optional[str] = None, + target_name: Optional[str] = None, + **kwargs: Any, + ) -> RelationshipModel: + """Create a Relationship from a report to another entity. + + Args: + relationship_type: The type of relationship (e.g., 'targets', 'indicates') + report_id: The ID of the report + target_ref: The ID of the target entity + organization_id: The ID of the organization that created this relationship + marking_ids: List of marking definition IDs to apply to the relationship + created: When the relationship was created + modified: When the relationship was last modified + report_name: The name of the report, for description purposes + target_name: The name of the target entity, for description purposes + **kwargs: Additional arguments to pass to RelationshipModel + + Returns: + RelationshipModel: The created relationship model + + """ + description = kwargs.pop("description", None) + if description is None: + if report_name and target_name: + description = ( + f"Report '{report_name}' {relationship_type} '{target_name}'" + ) + elif report_name: + description = f"Report '{report_name}' {relationship_type} entity" + else: + description = f"Report {relationship_type} entity" + + return OctiRelationshipModel.create( + relationship_type=relationship_type, + source_ref=report_id, + target_ref=target_ref, + organization_id=organization_id, + marking_ids=marking_ids, + created=created, + modified=modified, + description=description, + **kwargs, + ) diff --git a/external-import/google-ti-feeds/connector/src/stix/octi/models/report_model.py b/external-import/google-ti-feeds/connector/src/stix/octi/models/report_model.py new file mode 100644 index 0000000000..07e64cc294 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/octi/models/report_model.py @@ -0,0 +1,84 @@ +"""The module contains the OctiReportModel class, which represents an OpenCTI Report.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +from connector.src.stix.v21.models.ovs.report_type_ov_enums import ReportTypeOV +from connector.src.stix.v21.models.sdos.report_model import ReportModel + + +class OctiReportModel: + """Model for creating OpenCTI Report objects.""" + + @staticmethod + def create( + name: str, + created: datetime, + modified: datetime, + organization_id: str, + marking_ids: List[str], + description: Optional[str] = None, + report_types: Optional[List[ReportTypeOV]] = None, + published: Optional[datetime] = None, + object_refs: Optional[List[str]] = None, + labels: Optional[List[str]] = None, + external_references: Optional[List[Dict[str, Any]]] = None, + content: Optional[str] = None, + **kwargs: Any, + ) -> ReportModel: + """Create a Report model with OpenCTI custom properties. + + Args: + name: The name of the report + created: When the report was created + modified: When the report was last modified + organization_id: The ID of the organization that created this report + marking_ids: List of marking definition IDs to apply to the report + description: Description of the report + report_types: List of report types + published: When the report was published (defaults to created if not provided) + object_refs: List of referenced object IDs + labels: List of labels for this report + external_references: List of external references + content: The full content of the report + **kwargs: Additional arguments to pass to ReportModel + + Returns: + ReportModel: The created report model + + """ + if published is None: + published = created + + if report_types is None: + report_types = [ReportTypeOV.THREAT_REPORT] + + if object_refs is None: + object_refs = [] + + if labels is None: + labels = [] + + custom_properties = kwargs.pop("custom_properties", {}) + if content: + custom_properties["x_opencti_content"] = content + + data = { + "type": "report", + "spec_version": "2.1", + "created": created, + "modified": modified, + "name": name, + "description": description, + "report_types": report_types, + "published": published, + "object_refs": object_refs, + "created_by_ref": organization_id, + "object_marking_refs": marking_ids, + "labels": labels, + "external_references": external_references, + "custom_properties": custom_properties, + **kwargs, + } + + return ReportModel(**data) diff --git a/external-import/google-ti-feeds/connector/src/stix/octi/models/tlp_marking_model.py b/external-import/google-ti-feeds/connector/src/stix/octi/models/tlp_marking_model.py new file mode 100644 index 0000000000..95614154ef --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/octi/models/tlp_marking_model.py @@ -0,0 +1,35 @@ +"""The module defines the MarkingDefinitionModel class, which represents a marking definition in STIX 2.1 format.""" + +from typing import Literal + +import pycti # type: ignore +import stix2 # type: ignore +from pydantic import BaseModel, Field + + +class TLPMarkingModel(BaseModel): + """Model representing a marking definition in STIX 2.1 format.""" + + level: Literal["white", "green", "amber", "amber+strict", "red"] = Field( + ..., + description="The level of the marking.", + ) + + def to_stix2_object(self) -> stix2.v21.MarkingDefinition: + """Make stix object.""" + mapping = { + "white": stix2.TLP_WHITE, + "green": stix2.TLP_GREEN, + "amber": stix2.TLP_AMBER, + "amber+strict": stix2.MarkingDefinition( + id=pycti.MarkingDefinition.generate_id("TLP", "TLP:AMBER+STRICT"), + definition_type="statement", + definition={"statement": "custom"}, + custom_properties=dict( # noqa: C408 # No literal dict for maintainability + x_opencti_definition_type="TLP", + x_opencti_definition="TLP:AMBER+STRICT", + ), + ), + "red": stix2.TLP_RED, + } + return mapping[self.level] diff --git a/external-import/google-ti-feeds/connector/src/stix/octi/models/vulnerability_model.py b/external-import/google-ti-feeds/connector/src/stix/octi/models/vulnerability_model.py new file mode 100644 index 0000000000..24685c84cc --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/octi/models/vulnerability_model.py @@ -0,0 +1,73 @@ +"""The module contains the OctiVulnerabilityModel class, which represents an OpenCTI Vulnerability.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +from connector.src.stix.v21.models.sdos.vulnerability_model import VulnerabilityModel + + +class OctiVulnerabilityModel: + """Model for creating OpenCTI Vulnerability objects.""" + + @staticmethod + def create( + name: str, + cve_id: str, + organization_id: str, + marking_ids: list[str], + description: Optional[str] = None, + created: Optional[datetime] = None, + modified: Optional[datetime] = None, + base_score: Optional[float] = None, + epss_score: Optional[float] = None, + epss_percentile: Optional[float] = None, + labels: Optional[List[str]] = None, + external_references: Optional[List[Dict[str, Any]]] = None, + **kwargs: Any, + ) -> VulnerabilityModel: + """Create a Vulnerability model. + + Args: + name: The name of the vulnerability + cve_id: CVE ID for the vulnerability + organization_id: The ID of the organization that created this vulnerability + marking_ids: List of marking definition IDs to apply to the vulnerability + description: Description of the vulnerability + created: Time the vulnerability was created + modified: Time the vulnerability was last modified + base_score: CVSS base score of the vulnerability + epss_score: EPSS score of the vulnerability + epss_percentile: EPSS percentile of the vulnerability + labels: Labels to apply to the vulnerability + external_references: External references related to the vulnerability + **kwargs: Additional arguments to pass to VulnerabilityModel + + Returns: + VulnerabilityModel: The created vulnerability model + + """ + custom_properties = {} + + if base_score is not None: + custom_properties["x_opencti_base_score"] = base_score + if epss_score is not None: + custom_properties["x_opencti_epss_score"] = epss_score + if epss_percentile is not None: + custom_properties["x_opencti_epss_percentile"] = epss_percentile + + data = { + "type": "vulnerability", + "spec_version": "2.1", + "custom_properties": custom_properties, + "created": created or kwargs.pop("created", datetime.now()), + "modified": modified or kwargs.pop("modified", datetime.now()), + "name": name, + "description": description, + "labels": labels, + "external_references": external_references, + "created_by_ref": organization_id, + "object_marking_refs": marking_ids, + **kwargs, + } + + return VulnerabilityModel(**data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/__init__.py b/external-import/google-ti-feeds/connector/src/stix/v21/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/__init__.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/cdts/__init__.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/cdts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/cdts/external_reference_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/cdts/external_reference_model.py new file mode 100644 index 0000000000..db0cb62780 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/cdts/external_reference_model.py @@ -0,0 +1,33 @@ +"""The module defines the ExternalReferenceModel class, which represents an external reference in STIX 2.1 format.""" + +from typing import Dict, Optional + +from connector.src.stix.v21.models.ovs.hashing_algorithm_ov_enums import ( + HashAlgorithmOV, +) +from pydantic import BaseModel, Field + + +class ExternalReferenceModel(BaseModel): + """Model representing an external reference in STIX 2.1 format.""" + + source_name: str = Field( + ..., description="The name of the source that defines the reference." + ) + description: Optional[str] = Field( + default=None, + description="A human-readable description of the external reference.", + ) + url: Optional[str] = Field( + default=None, description="A URL pointing to an external resource." + ) + hashes: Optional[Dict[HashAlgorithmOV, str]] = Field( + default=None, + description="A dictionary of hashes for the content referenced by the URL. Keys must be valid hash algorithms.", + ) + external_id: Optional[str] = Field( + default=None, + description="An external identifier for the referenced content.", + ) + + model_config = {"use_enum_values": True} diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/cdts/kill_chain_phase_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/cdts/kill_chain_phase_model.py new file mode 100644 index 0000000000..d172e2456d --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/cdts/kill_chain_phase_model.py @@ -0,0 +1,27 @@ +"""The module contains the KillChainPhaseModel class, which represents a kill chain phase in STIX 2.1 format.""" + +from pydantic import BaseModel, Field, model_validator + + +class KillChainPhaseModel(BaseModel): + """Model representing a Kill Chain Phase in STIX 2.1 format.""" + + kill_chain_name: str = Field( + ..., + description="The name of the kill chain. SHOULD be lowercase with hyphens.", + ) + phase_name: str = Field( + ..., + description="The phase name within the kill chain. SHOULD be lowercase with hyphens.", + ) + + @model_validator(mode="after") + def validate_formatting(self, values): # type: ignore + """Ensure that kill_chain_name and phase_name are lowercase and use hyphens.""" + for field in ["kill_chain_name", "phase_name"]: + val = getattr(values, field) + if not val.islower() or " " in val or "_" in val: + raise ValueError( + f"{field} must be lowercase and use hyphens instead of spaces or underscores: got '{val}'" + ) + return values diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/__init__.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/account_type_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/account_type_ov_enums.py new file mode 100644 index 0000000000..ff0b8dba05 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/account_type_ov_enums.py @@ -0,0 +1,19 @@ +"""The module defines an the stix2.1 account-type-ov.""" + +from enum import Enum + + +class AccountTypeOV(str, Enum): + """Account Type OV Enum.""" + + FACEBOOK = "facebook" + LDAP = "ldap" + NIS = "nis" + OPENID = "openid" + RADIUS = "radius" + SKYPE = "skype" + TACACS = "tacacs" + TWITTER = "twitter" + UNIX = "unix" + WINDOWS_LOCAL = "windows-local" + WINDOWS_DOMAIN = "windows-domain" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/attack_motivation_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/attack_motivation_ov_enums.py new file mode 100644 index 0000000000..0175c522d0 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/attack_motivation_ov_enums.py @@ -0,0 +1,18 @@ +"""The module contains the AttackMotivationOV enum class, which defines various attack motivations.""" + +from enum import Enum + + +class AttackMotivationOV(str, Enum): + """Attack Motivation OV Enum.""" + + ACCIDENTAL = "accidental" + COERCION = "coercion" + DOMINANCE = "dominance" + IDEOLOGY = "ideology" + NOTORIETY = "notoriety" + ORGANIZATIONAL_GAIN = "organizational-gain" + PERSONAL_GAIN = "personal-gain" + PERSONAL_SATISFACTION = "personal-satisfaction" + REVENGE = "revenge" + UNPREDICTABLE = "unpredictable" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/attack_resource_level_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/attack_resource_level_ov_enums.py new file mode 100644 index 0000000000..dbc65f26c1 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/attack_resource_level_ov_enums.py @@ -0,0 +1,14 @@ +"""The module contains the AttackResourceLevelOV enum class, which is used to represent different levels of attack resources in the context of threat intelligence.""" + +from enum import Enum + + +class AttackResourceLevelOV(str, Enum): + """Attack Resource Level OV Enum.""" + + INDIVIDUAL = "individual" + CLUB = "club" + CONTEST = "contest" + TEAM = "team" + ORGANIZATION = "organization" + GOVERNMENT = "government" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/course_of_action_type_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/course_of_action_type_ov_enums.py new file mode 100644 index 0000000000..a785c24539 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/course_of_action_type_ov_enums.py @@ -0,0 +1,12 @@ +"""The module defines the Course of Action Type Enumeration.""" + +from enum import Enum + + +class CourseOfActionTypeOV(str, Enum): + """Course of Action Type Enumeration.""" + + TEXTUAL_PLAIN = "textual:text/plain" + TEXTUAL_HTML = "textual:text/html" + TEXTUAL_MD = "textual:text/md" + TEXTUAL_PDF = "textual:pdf" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/encryption_algorithm_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/encryption_algorithm_ov_enums.py new file mode 100644 index 0000000000..49c2099ff3 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/encryption_algorithm_ov_enums.py @@ -0,0 +1,11 @@ +"""The file contains the EncryptionAlgorithmOV enum class for OpenAI API encryption algorithms.""" + +from enum import Enum + + +class EncryptionAlgorithmOV(str, Enum): + """Encryption Algorithm Enumeration.""" + + AES_256_GCM = "AES-256-GCM" + CHACHA20_POLY1305 = "ChaCha20-Poly1305" + MIME_TYPE_INDICATED = "mime-type-indicated" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/grouping_context_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/grouping_context_ov_enums.py new file mode 100644 index 0000000000..c2ff25fad4 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/grouping_context_ov_enums.py @@ -0,0 +1,11 @@ +"""The module contains the GroupingContextOV enum class.""" + +from enum import Enum + + +class GroupingContextOV(str, Enum): + """Grouping Context Enumeration.""" + + SUSPICIOUS_ACTIVITY = "suspicious-activity" + MALWARE_ANALYSIS = "malware-analysis" + UNSPECIFIED = "unspecified" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/hashing_algorithm_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/hashing_algorithm_ov_enums.py new file mode 100644 index 0000000000..947ffbdb14 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/hashing_algorithm_ov_enums.py @@ -0,0 +1,15 @@ +"""The module defines an enumeration for various hashing algorithms.""" + +from enum import Enum + + +class HashAlgorithmOV(str, Enum): + """Hash Algorithm Enumeration.""" + + MD5 = "MD5" + SHA1 = "SHA-1" + SHA256 = "SHA-256" + SHA512 = "SHA-512" + SHA3_256 = "SHA3-256" + SHA3_512 = "SHA3-512" + SSDEEP = "SSDEEP" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/identity_class_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/identity_class_ov_enums.py new file mode 100644 index 0000000000..bc54ffa351 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/identity_class_ov_enums.py @@ -0,0 +1,14 @@ +"""The module contains the IdentityClassOV enum class.""" + +from enum import Enum + + +class IdentityClassOV(str, Enum): + """Identity Class Enumeration.""" + + INDIVIDUAL = "individual" + GROUP = "group" + SYSTEM = "system" + ORGANIZATION = "organization" + CLASS_ = "class" + UNKNOWN = "unknown" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/implementation_language_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/implementation_language_ov_enums.py new file mode 100644 index 0000000000..759eca933c --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/implementation_language_ov_enums.py @@ -0,0 +1,29 @@ +"""The module contains the ImplementationLanguageOV enum class.""" + +from enum import Enum + + +class ImplementationLanguageOV(str, Enum): + """Implementation Language Enumeration.""" + + APPLESCRIPT = "applescript" + BASH = "bash" + C = "c" + CPP = "c++" + CSHARP = "c#" + GO = "go" + JAVA = "java" + JAVASCRIPT = "javascript" + LUA = "lua" + OBJECTIVE_C = "objective-c" + PERL = "perl" + PHP = "php" + POWERSHELL = "powershell" + PYTHON = "python" + RUBY = "ruby" + SCALA = "scala" + SWIFT = "swift" + TYPESCRIPT = "typescript" + VISUAL_BASIC = "visual-basic" + X86_32 = "x86-32" + X86_64 = "x86-64" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/indicator_type_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/indicator_type_ov_enums.py new file mode 100644 index 0000000000..c46ef84d4b --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/indicator_type_ov_enums.py @@ -0,0 +1,15 @@ +"""The module contains the IndicatorTypeOV enum class.""" + +from enum import Enum + + +class IndicatorTypeOV(str, Enum): + """Indicator Type Enumeration.""" + + ANOMALOUS_ACTIVITY = "anomalous-activity" + ANONYMIZATION = "anonymization" + BENIGN = "benign" + COMPROMISED = "compromised" + MALICIOUS_ACTIVITY = "malicious-activity" + ATTRIBUTION = "attribution" + UNKNOWN = "unknown" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/industry_sector_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/industry_sector_ov_enums.py new file mode 100644 index 0000000000..e6f481edd1 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/industry_sector_ov_enums.py @@ -0,0 +1,35 @@ +"""The module defines the IndustrySectorOV enumeration for various industry sectors.""" + +from enum import Enum + + +class IndustrySectorOV(str, Enum): + """Industry Sector Enumeration.""" + + AGRICULTURE = "agriculture" + AEROSPACE = "aerospace" + AUTOMOTIVE = "automotive" + COMMUNICATIONS = "communications" + CONSTRUCTION = "construction" + DEFENCE = "defence" + EDUCATION = "education" + ENERGY = "energy" + ENTERTAINMENT = "entertainment" + FINANCIAL_SERVICES = "financial-services" + GOVERNMENT_NATIONAL = "government-national" + GOVERNMENT_REGIONAL = "government-regional" + GOVERNMENT_LOCAL = "government-local" + GOVERNMENT_PUBLIC_SERVICES = "government-public-services" + HEALTHCARE = "healthcare" + HOSPITALITY_LEISURE = "hospitality-leisure" + INFRASTRUCTURE = "infrastructure" + INSURANCE = "insurance" + MANUFACTURING = "manufacturing" + MINING = "mining" + NON_PROFIT = "non-profit" + PHARMACEUTICALS = "pharmaceuticals" + RETAIL = "retail" + TECHNOLOGY = "technology" + TELECOMMUNICATIONS = "telecommunications" + TRANSPORTATION = "transportation" + UTILITIES = "utilities" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/infrastructure_type_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/infrastructure_type_ov_enums.py new file mode 100644 index 0000000000..4200e83888 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/infrastructure_type_ov_enums.py @@ -0,0 +1,19 @@ +"""The module contains the InfrastructureTypeOV enum class.""" + +from enum import Enum + + +class InfrastructureTypeOV(str, Enum): + """Infrastructure Type Enumeration.""" + + AMPLIFICATION = "amplification" + ANONYMIZATION = "anonymization" + BOTNET = "botnet" + COMMAND_AND_CONTROL = "command-and-control" + EXFILTRATION = "exfiltration" + HOSTING_MALWARE = "hosting-malware" + HOSTING_TARGET_LISTS = "hosting-target-lists" + PHISHING = "phishing" + RECONNAISSANCE = "reconnaissance" + STAGING = "staging" + UNDEFINED = "undefined" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/malware_av_result_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/malware_av_result_ov_enums.py new file mode 100644 index 0000000000..c3c2437a83 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/malware_av_result_ov_enums.py @@ -0,0 +1,12 @@ +"""The module contains the MalwareAVResultOV enum class.""" + +from enum import Enum + + +class MalwareAVResultOV(str, Enum): + """Malware AV Result Enumeration.""" + + MALICIOUS = "malicious" + SUSPICIOUS = "suspicious" + BENIGN = "benign" + UNKNOWN = "unknown" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/malware_capabilities_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/malware_capabilities_ov_enums.py new file mode 100644 index 0000000000..fc7ac32c62 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/malware_capabilities_ov_enums.py @@ -0,0 +1,45 @@ +"""The module contains the MalwareCapabilitiesOV enumeration class.""" + +from enum import Enum + + +class MalwareCapabilitiesOV(str, Enum): + """Malware Capabilities Enumeration.""" + + ACCESSES_REMOTE_MACHINES = "accesses-remote-machines" + ANTI_DEBUGGING = "anti-debugging" + ANTI_DISASSEMBLY = "anti-disassembly" + ANTI_EMULATION = "anti-emulation" + ANTI_MEMORY_FORENSICS = "anti-memory-forensics" + ANTI_SANDBOX = "anti-sandbox" + ANTI_VM = "anti-vm" + CAPTURES_INPUT_PERIPHERALS = "captures-input-peripherals" + CAPTURES_OUTPUT_PERIPHERALS = "captures-output-peripherals" + CAPTURES_SYSTEM_STATE_DATA = "captures-system-state-data" + CLEANS_TRACES_OF_INFECTION = "cleans-traces-of-infection" + COMMITS_FRAUD = "commits-fraud" + COMMUNICATES_WITH_C2 = "communicates-with-c2" + COMPROMISES_DATA_AVAILABILITY = "compromises-data-availability" + COMPROMISES_DATA_INTEGRITY = "compromises-data-integrity" + COMPROMISES_SYSTEM_AVAILABILITY = "compromises-system-availability" + CONTROLS_LOCAL_MACHINE = "controls-local-machine" + DEGRADES_SECURITY_SOFTWARE = "degrades-security-software" + DEGRADES_SYSTEM_UPDATES = "degrades-system-updates" + DETERMINES_C2_SERVER = "determines-c2-server" + EMAILS_SPAM = "emails-spam" + ESCALATES_PRIVILEGES = "escalates-privileges" + EVADES_AV = "evades-av" + EXFILTRATES_DATA = "exfiltrates-data" + FINGERPRINTS_HOST = "fingerprints-host" + HIDES_ARTIFACTS = "hides-artifacts" + HIDES_EXECUTING_CODE = "hides-executing-code" + INFECTS_FILES = "infects-files" + INFECTS_REMOTE_MACHINES = "infects-remote-machines" + INSTALLS_OTHER_COMPONENTS = "installs-other-components" + PERSISTS_AFTER_SYSTEM_REBOOT = "persists-after-system-reboot" + PREVENTS_ARTIFACT_ACCESS = "prevents-artifact-access" + PREVENTS_ARTIFACT_DELETION = "prevents-artifact-deletion" + PROBES_NETWORK_ENVIRONMENT = "probes-network-environment" + SELF_MODIFIES = "self-modifies" + STEALS_AUTHENTICATION_CREDENTIALS = "steals-authentication-credentials" + VIOLATES_SYSTEM_OPERATIONAL_INTEGRITY = "violates-system-operational-integrity" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/malware_type_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/malware_type_ov_enums.py new file mode 100644 index 0000000000..78a1b679e6 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/malware_type_ov_enums.py @@ -0,0 +1,30 @@ +"""The module contains the MalwareTypeOV enumeration class.""" + +from enum import Enum + + +class MalwareTypeOV(str, Enum): + """Malware Type Enumeration.""" + + ADWARE = "adware" + BACKDOOR = "backdoor" + BOT = "bot" + BOOTKIT = "bootkit" + DDOS = "ddos" + DOWNLOADER = "downloader" + DROPPER = "dropper" + EXPLOIT_KIT = "exploit-kit" + KEYLOGGER = "keylogger" + RANSOMWARE = "ransomware" + REMOTE_ACCESS_TROJAN = "remote-access-trojan" + RESOURCE_EXPLOITATION = "resource-exploitation" + ROGUE_SECURITY_SOFTWARE = "rogue-security-software" + ROOTKIT = "rootkit" + SCREEN_CAPTURE = "screen-capture" + SPYWARE = "spyware" + TROJAN = "trojan" + UNKNOWN = "unknown" + VIRUS = "virus" + WEBSHELL = "webshell" + WIPER = "wiper" + WORM = "worm" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/network_socket_address_family_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/network_socket_address_family_ov_enums.py new file mode 100644 index 0000000000..2d0abe2841 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/network_socket_address_family_ov_enums.py @@ -0,0 +1,16 @@ +"""The module defines the NetworkSocketAddressFamilyOV enum class.""" + +from enum import Enum + + +class NetworkSocketAddressFamilyOV(str, Enum): + """Network Socket Address Family Enumeration.""" + + AF_UNSPEC = "AF_UNSPEC" + AF_INET = "AF_INET" + AF_IPX = "AF_IPX" + AF_APPLETALK = "AF_APPLETALK" + AF_NETBIOS = "AF_NETBIOS" + AF_INET6 = "AF_INET6" + AF_IRDA = "AF_IRDA" + AF_BTH = "AF_BTH" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/network_socket_type_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/network_socket_type_ov_enums.py new file mode 100644 index 0000000000..33d84c46b7 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/network_socket_type_ov_enums.py @@ -0,0 +1,13 @@ +"""The module defines an enumeration for network socket types.""" + +from enum import Enum + + +class NetworkSocketTypeOV(str, Enum): + """Network Socket Type Enumeration.""" + + SOCK_STREAM = "SOCK_STREAM" + SOCK_DGRAM = "SOCK_DGRAM" + SOCK_RAW = "SOCK_RAW" + SOCK_RDM = "SOCK_RDM" + SOCK_SEQPACKET = "SOCK_SEQPACKET" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/opinion_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/opinion_ov_enums.py new file mode 100644 index 0000000000..e89340328e --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/opinion_ov_enums.py @@ -0,0 +1,13 @@ +"""The module contains the OpinionOV enum class, which defines the possible opinions for a given statement.""" + +from enum import Enum + + +class OpinionOV(str, Enum): + """Opinion Enumeration.""" + + STRONGLY_DISAGREE = "strongly-disagree" + DISAGREE = "disagree" + NEUTRAL = "neutral" + AGREE = "agree" + STRONGLY_AGREE = "strongly-agree" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/processor_architecture_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/processor_architecture_ov_enums.py new file mode 100644 index 0000000000..51160e020b --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/processor_architecture_ov_enums.py @@ -0,0 +1,16 @@ +"""The module defines the ProcessorArchitectureOV enumeration.""" + +from enum import Enum + + +class ProcessorArchitectureOV(str, Enum): + """Processor Architecture Enumeration.""" + + ALPHA = "alpha" + ARM = "arm" + IA_64 = "ia-64" + MIPS = "mips" + POWERPC = "powerpc" + SPARC = "sparc" + X86 = "x86" + X86_64 = "x86-64" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/region_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/region_ov_enums.py new file mode 100644 index 0000000000..d04c9683bf --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/region_ov_enums.py @@ -0,0 +1,41 @@ +"""The module contains the RegionOV enum class, which defines various geographical regions.""" + +from enum import Enum + + +class RegionOV(str, Enum): + """Region Enumeration.""" + + AFRICA = "africa" + EASTERN_AFRICA = "eastern-africa" + MIDDLE_AFRICA = "middle-africa" + NORTHERN_AFRICA = "northern-africa" + SOUTHERN_AFRICA = "southern-africa" + WESTERN_AFRICA = "western-africa" + + AMERICAS = "americas" + LATIN_AMERICA_CARIBBEAN = "latin-america-caribbean" + SOUTH_AMERICA = "south-america" + CARIBBEAN = "caribbean" + CENTRAL_AMERICA = "central-america" + NORTHERN_AMERICA = "northern-america" + + ASIA = "asia" + CENTRAL_ASIA = "central-asia" + EASTERN_ASIA = "eastern-asia" + SOUTHERN_ASIA = "southern-asia" + SOUTH_EASTERN_ASIA = "south-eastern-asia" + WESTERN_ASIA = "western-asia" + + EUROPE = "europe" + EASTERN_EUROPE = "eastern-europe" + NORTHERN_EUROPE = "northern-europe" + SOUTHERN_EUROPE = "southern-europe" + WESTERN_EUROPE = "western-europe" + + OCEANIA = "oceania" + ANTARCTICA = "antarctica" + AUSTRALIA_NEW_ZEALAND = "australia-new-zealand" + MELANESIA = "melanesia" + MICRONESIA = "micronesia" + POLYNESIA = "polynesia" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/report_type_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/report_type_ov_enums.py new file mode 100644 index 0000000000..0fa8a71065 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/report_type_ov_enums.py @@ -0,0 +1,19 @@ +"""The module contains the ReportTypeOV enum class for OpenVAS report types.""" + +from enum import Enum + + +class ReportTypeOV(str, Enum): + """Report Type Enumeration.""" + + ATTACK_PATTERN = "attack-pattern" + CAMPAIGN = "campaign" + IDENTITY = "identity" + INDICATOR = "indicator" + INTRUSION_SET = "intrusion-set" + MALWARE = "malware" + OBSERVED_DATA = "observed-data" + THREAT_ACTOR = "threat-actor" + THREAT_REPORT = "threat-report" + TOOL = "tool" + VULNERABILITY = "vulnerability" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/threat_actor_role_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/threat_actor_role_ov_enums.py new file mode 100644 index 0000000000..17027bb44d --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/threat_actor_role_ov_enums.py @@ -0,0 +1,15 @@ +"""The module contains the Threat Actor Role OV Enums.""" + +from enum import Enum + + +class ThreatActorRoleOV(str, Enum): + """Threat Actor Role Enumeration.""" + + AGENT = "agent" + DIRECTOR = "director" + INDEPENDENT = "independent" + INFRASTRUCTURE_ARCHITECT = "infrastructure-architect" + INFRASTRUCTURE_OPERATOR = "infrastructure-operator" + MALWARE_AUTHOR = "malware-author" + SPONSOR = "sponsor" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/threat_actor_sophistication_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/threat_actor_sophistication_ov_enums.py new file mode 100644 index 0000000000..c10d6596ba --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/threat_actor_sophistication_ov_enums.py @@ -0,0 +1,15 @@ +"""The module defines the ThreatActorSophisticationOV enum class.""" + +from enum import Enum + + +class ThreatActorSophisticationOV(str, Enum): + """Threat Actor Sophistication Enumeration.""" + + NONE = "none" + MINIMAL = "minimal" + INTERMEDIATE = "intermediate" + ADVANCED = "advanced" + EXPERT = "expert" + INNOVATOR = "innovator" + STRATEGIC = "strategic" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/threat_actor_type_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/threat_actor_type_ov_enums.py new file mode 100644 index 0000000000..bab7ff90d1 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/threat_actor_type_ov_enums.py @@ -0,0 +1,20 @@ +"""The module contains the ThreatActorTypeOV enum class.""" + +from enum import Enum + + +class ThreatActorTypeOV(str, Enum): + """Threat Actor Type Enumeration.""" + + ACTIVIST = "activist" + COMPETITOR = "competitor" + CRIME_SYNDICATE = "crime-syndicate" + CRIMINAL = "criminal" + HACKER = "hacker" + INSIDER_ACCIDENTAL = "insider-accidental" + INSIDER_DISGRUNTLED = "insider-disgruntled" + NATION_STATE = "nation-state" + SENSATIONALIST = "sensationalist" + SPY = "spy" + TERRORIST = "terrorist" + UNKNOWN = "unknown" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/tool_type_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/tool_type_ov_enums.py new file mode 100644 index 0000000000..6657bacfeb --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/tool_type_ov_enums.py @@ -0,0 +1,16 @@ +"""The module contains the ToolTypeOV enum class for representing various tool types in an OV context.""" + +from enum import Enum + + +class ToolTypeOV(str, Enum): + """Tool Type OV Enum.""" + + DENIAL_OF_SERVICE = "denial-of-service" + EXPLOITATION = "exploitation" + INFORMATION_GATHERING = "information-gathering" + NETWORK_CAPTURE = "network-capture" + CREDENTIAL_EXPLOITATION = "credential-exploitation" + REMOTE_ACCESS = "remote-access" + VULNERABILITY_SCANNING = "vulnerability-scanning" + UNKNOWN = "unknown" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_integrity_level_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_integrity_level_ov_enums.py new file mode 100644 index 0000000000..663b75a958 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_integrity_level_ov_enums.py @@ -0,0 +1,12 @@ +"""The module contains the WindowsIntegrityLevelOV enum class.""" + +from enum import Enum + + +class WindowsIntegrityLevelOV(str, Enum): + """Windows Integrity Level Enumeration.""" + + LOW = "low" + MEDIUM = "medium" + HIGH = "high" + SYSTEM = "system" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_pe_binary_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_pe_binary_ov_enums.py new file mode 100644 index 0000000000..078594962a --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_pe_binary_ov_enums.py @@ -0,0 +1,11 @@ +"""The module contains the Windows PE Binary Type OV Enums.""" + +from enum import Enum + + +class WindowsPEBinaryTypeOV(str, Enum): + """Windows PE Binary Type Enumeration.""" + + DLL = "dll" + EXE = "exe" + SYS = "sys" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_registry_datatype_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_registry_datatype_ov_enums.py new file mode 100644 index 0000000000..67c7618f3e --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_registry_datatype_ov_enums.py @@ -0,0 +1,22 @@ +"""The module defines an enumeration for Windows Registry data types.""" + +from enum import Enum + + +class WindowsRegistryDatatypeOV(str, Enum): + """Windows Registry Data Type Enumeration.""" + + REG_NONE = "REG_NONE" + REG_SZ = "REG_SZ" + REG_EXPAND_SZ = "REG_EXPAND_SZ" + REG_BINARY = "REG_BINARY" + REG_DWORD = "REG_DWORD" + REG_DWORD_BIG_ENDIAN = "REG_DWORD_BIG_ENDIAN" + REG_DWORD_LITTLE_ENDIAN = "REG_DWORD_LITTLE_ENDIAN" + REG_LINK = "REG_LINK" + REG_MULTI_SZ = "REG_MULTI_SZ" + REG_RESOURCE_LIST = "REG_RESOURCE_LIST" + REG_FULL_RESOURCE_DESCRIPTION = "REG_FULL_RESOURCE_DESCRIPTION" + REG_RESOURCE_REQUIREMENTS_LIST = "REG_RESOURCE_REQUIREMENTS_LIST" + REG_QWORD = "REG_QWORD" + REG_INVALID_TYPE = "REG_INVALID_TYPE" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_service_start_type_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_service_start_type_ov_enums.py new file mode 100644 index 0000000000..c7312a8dfa --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_service_start_type_ov_enums.py @@ -0,0 +1,13 @@ +"""The module defines the WindowsServiceStartTypeOV enum class.""" + +from enum import Enum + + +class WindowsServiceStartTypeOV(str, Enum): + """Windows Service Start Type Enumeration.""" + + SERVICE_AUTO_START = "SERVICE_AUTO_START" + SERVICE_BOOT_START = "SERVICE_BOOT_START" + SERVICE_DEMAND_START = "SERVICE_DEMAND_START" + SERVICE_DISABLED = "SERVICE_DISABLED" + SERVICE_SYSTEM_ALERT = "SERVICE_SYSTEM_ALERT" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_service_status_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_service_status_ov_enums.py new file mode 100644 index 0000000000..7fb030dc76 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_service_status_ov_enums.py @@ -0,0 +1,15 @@ +"""The module contains the WindowsServiceStatusOV enum class.""" + +from enum import Enum + + +class WindowsServiceStatusOV(str, Enum): + """Windows Service Status Enumeration.""" + + SERVICE_CONTINUE_PENDING = "SERVICE_CONTINUE_PENDING" + SERVICE_PAUSE_PENDING = "SERVICE_PAUSE_PENDING" + SERVICE_PAUSED = "SERVICE_PAUSED" + SERVICE_RUNNING = "SERVICE_RUNNING" + SERVICE_START_PENDING = "SERVICE_START_PENDING" + SERVICE_STOP_PENDING = "SERVICE_STOP_PENDING" + SERVICE_STOPPED = "SERVICE_STOPPED" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_service_type_ov_enums.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_service_type_ov_enums.py new file mode 100644 index 0000000000..4263f32f93 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/ovs/windows_service_type_ov_enums.py @@ -0,0 +1,12 @@ +"""The module defines an enumeration for Windows service types.""" + +from enum import Enum + + +class WindowsServiceTypeOV(str, Enum): + """Windows Service Type Enumeration.""" + + SERVICE_KERNEL_DRIVER = "SERVICE_KERNEL_DRIVER" + SERVICE_FILE_SYSTEM_DRIVER = "SERVICE_FILE_SYSTEM_DRIVER" + SERVICE_WIN32_OWN_PROCESS = "SERVICE_WIN32_OWN_PROCESS" + SERVICE_WIN32_SHARE_PROCESS = "SERVICE_WIN32_SHARE_PROCESS" diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/__init__.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/artifact_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/artifact_model.py new file mode 100644 index 0000000000..3cacc9616a --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/artifact_model.py @@ -0,0 +1,56 @@ +"""The module defines the ArtifactModel class, which represents a STIX 2.1 Artifact object.""" + +from typing import Dict, Optional + +from connector.src.stix.v21.models.ovs.encryption_algorithm_ov_enums import ( + EncryptionAlgorithmOV, +) +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field, model_validator +from stix2.v21 import Artifact, _STIXBase21 # type: ignore + + +class ArtifactModel(BaseSCOModel): + """Model representing an Artifact in STIX 2.1 format.""" + + mime_type: Optional[str] = Field( + default=None, + description="IANA media type of the artifact. SHOULD follow the IANA media type registry format if possible.", + ) + payload_bin: Optional[bytes] = Field( + default=None, + description="Base64-encoded binary data of the artifact. MUST NOT be used if 'url' is present.", + ) + url: Optional[str] = Field( + default=None, + description="URL to the artifact content. MUST NOT be used if 'payload_bin' is present.", + ) + hashes: Optional[Dict[str, str]] = Field( + default=None, + description="Dictionary of hashes for the artifact. MUST be present if 'url' is used. Keys MUST come from hash-algorithm-ov.", + ) + encryption_algorithm: Optional[EncryptionAlgorithmOV] = Field( + default=None, + description="Encryption algorithm used on the payload or URL content. MUST come from encryption-algorithm-enum.", + ) + decryption_key: Optional[str] = Field( + default=None, + description="Decryption key for encrypted content. MUST NOT be present unless 'encryption_algorithm' is set.", + ) + + @model_validator(mode="after") + def validate_artifact_logic(self) -> "ArtifactModel": + """Validate the ArtifactModel instance.""" + if self.payload_bin and self.url: + raise ValueError("Only one of 'payload_bin' or 'url' may be set—not both.") + if self.url and not self.hashes: + raise ValueError("'hashes' MUST be provided when 'url' is set.") + if self.decryption_key and not self.encryption_algorithm: + raise ValueError( + "'decryption_key' MUST NOT be set unless 'encryption_algorithm' is also set." + ) + return self + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return Artifact(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/as_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/as_model.py new file mode 100644 index 0000000000..992d281663 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/as_model.py @@ -0,0 +1,27 @@ +"""The module defines a model for an Autonomous System (AS) in STIX 2.1 format.""" + +from typing import Optional + +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field +from stix2.v21 import AutonomousSystem, _STIXBase21 # type: ignore + + +class AutonomousSystemModel(BaseSCOModel): + """Model representing an Autonomous System in STIX 2.1 format.""" + + number: int = Field( + ..., + description="The assigned Autonomous System Number (ASN). Typically assigned by a Regional Internet Registry (RIR).", + ) + name: Optional[str] = Field( + default=None, description="The name of the AS, if known." + ) + rir: Optional[str] = Field( + default=None, + description="Name of the RIR that assigned the ASN (e.g., ARIN, RIPE, APNIC).", + ) + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return AutonomousSystem(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/directory_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/directory_model.py new file mode 100644 index 0000000000..b9fe1f2d0f --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/directory_model.py @@ -0,0 +1,40 @@ +"""The module defines the DirectoryModel class, which represents a STIX 2.1 Directory object.""" + +from datetime import datetime +from typing import List, Optional + +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field +from stix2.v21 import Directory, _STIXBase21 # type: ignore + + +class DirectoryModel(BaseSCOModel): + """Model representing a Directory in STIX 2.1 format.""" + + path: str = Field( + ..., + description="The observed path to the directory on the file system.", + ) + path_enc: Optional[str] = Field( + default=None, + description="Character encoding of the path if it's non-Unicode. MUST use IANA character set registry name.", + ) + ctime: Optional[datetime] = Field( + default=None, description="Timestamp when the directory was created." + ) + mtime: Optional[datetime] = Field( + default=None, + description="Timestamp when the directory was last modified.", + ) + atime: Optional[datetime] = Field( + default=None, + description="Timestamp when the directory was last accessed.", + ) + contains_refs: Optional[List[str]] = Field( + default=None, + description="List of identifiers referring to SCOs of type 'file' or 'directory' contained within this directory.", + ) + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return Directory(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/domain_name_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/domain_name_model.py new file mode 100644 index 0000000000..f70d275d33 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/domain_name_model.py @@ -0,0 +1,24 @@ +"""The module defines the DomainNameModel class, which represents a STIX 2.1 Domain Name object.""" + +from typing import List, Optional + +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field +from stix2.v21 import DomainName, _STIXBase21 # type: ignore + + +class DomainNameModel(BaseSCOModel): + """Model representing a Domain Name in STIX 2.1 format.""" + + value: str = Field( + ..., + description="The domain name, which MUST conform to RFC1034 and RFC5890.", + ) + resolves_to_refs: Optional[List[str]] = Field( + default=None, + description="(Deprecated) List of references to SCOs of type 'ipv4-addr', 'ipv6-addr', or 'domain-name' that this domain resolves to.", + ) + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return DomainName(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/email_address_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/email_address_model.py new file mode 100644 index 0000000000..a155d54591 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/email_address_model.py @@ -0,0 +1,28 @@ +"""The module defines the EmailAddressModel class, which represents a STIX 2.1 Email Address object.""" + +from typing import Optional + +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field +from stix2.v21 import EmailAddress, _STIXBase21 # type: ignore + + +class EmailAddressModel(BaseSCOModel): + """Model representing an Email Address in STIX 2.1 format.""" + + value: str = Field( + ..., + description="The email address, conforming to addr-spec from RFC5322 (e.g., jane.smith@example.com). MUST NOT include a display name.", + ) + display_name: Optional[str] = Field( + default=None, + description="The human-readable display name for this address, per RFC5322 (e.g., Jane Smith).", + ) + belongs_to_ref: Optional[str] = Field( + default=None, + description="Reference to a user-account SCO object that this email address belongs to. MUST point to type 'user-account'.", + ) + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return EmailAddress(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/email_message_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/email_message_model.py new file mode 100644 index 0000000000..c070264e06 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/email_message_model.py @@ -0,0 +1,71 @@ +"""The module defines a model for an Email Message in STIX 2.1 format.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field +from stix2.v21 import EmailMessage, _STIXBase21 # type: ignore + + +class EmailMessageModel(BaseSCOModel): + """Model representing an Email Message in STIX 2.1 format.""" + + is_multipart: bool = Field( + ..., description="True if the email contains multiple MIME parts." + ) + date: Optional[datetime] = Field( + default=None, description="Date/time the email was sent." + ) + content_type: Optional[str] = Field( + default=None, description="Value of the Content-Type header." + ) + from_ref: Optional[str] = Field( + default=None, + description="STIX ID of the 'From' sender (type: email-address).", + ) + sender_ref: Optional[str] = Field( + default=None, + description="STIX ID of the 'Sender' (transmitter agent).", + ) + to_refs: Optional[List[str]] = Field( + default=None, + description="STIX IDs of the To: recipients (type: email-address).", + ) + cc_refs: Optional[List[str]] = Field( + default=None, + description="STIX IDs of the CC: recipients (type: email-address).", + ) + bcc_refs: Optional[List[str]] = Field( + default=None, + description="STIX IDs of the BCC: recipients (type: email-address).", + ) + message_id: Optional[str] = Field( + default=None, description="Message-ID header field value." + ) + subject: Optional[str] = Field( + default=None, description="Subject of the email message." + ) + received_lines: Optional[List[str]] = Field( + default=None, description="List of Received header fields (in order)." + ) + additional_header_fields: Optional[Dict[str, str]] = Field( + default=None, + description="Other header fields (not explicitly modeled); keys preserved as case-sensitive field names.", + ) + body: Optional[str] = Field( + default=None, + description="Email body content (must NOT be used if is_multipart is true).", + ) + body_multipart: Optional[List[Dict[str, Any]]] = Field( + default=None, + description="List of MIME parts for the body (must NOT be used if is_multipart is false).", + ) + raw_email_ref: Optional[str] = Field( + default=None, + description="Reference to the full raw email (type: artifact).", + ) + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return EmailMessage(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/file_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/file_model.py new file mode 100644 index 0000000000..dacc36b168 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/file_model.py @@ -0,0 +1,86 @@ +"""The module defines a FileModel class that represents a file in STIX 2.1 format.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field, model_validator +from stix2.v21 import File, _STIXBase21 # type: ignore + + +class FileModel(BaseSCOModel): + """FileModel class represents a file in STIX 2.1 format.""" + + extensions: Optional[Dict[str, Dict[str, Any]]] = Field( + default=None, + description="Dictionary of file extensions (e.g., ntfs-ext, pdf-ext, archive-ext). Keys MUST be extension names.", + ) + + hashes: Optional[Dict[str, str]] = Field( + default=None, + description="Dictionary of hash algorithm names and hash values. Keys MUST come from hash-algorithm-ov.", + ) + + size: Optional[int] = Field( + default=None, + ge=0, + description="Size of the file in bytes. MUST NOT be negative.", + ) + name: Optional[str] = Field( + default=None, description="Name of the file as observed." + ) + name_enc: Optional[str] = Field( + default=None, + description="Character encoding used for the file name, per IANA character set registry.", + ) + + magic_number_hex: Optional[str] = Field( + default=None, + description="Hexadecimal magic number associated with the file format.", + ) + mime_type: Optional[str] = Field( + default=None, + description="MIME type of the file. SHOULD follow IANA media type registry.", + ) + + ctime: Optional[datetime] = Field( + default=None, description="Timestamp when the file was created." + ) + mtime: Optional[datetime] = Field( + default=None, description="Timestamp when the file was last modified." + ) + atime: Optional[datetime] = Field( + default=None, description="Timestamp when the file was last accessed." + ) + + parent_directory_ref: Optional[str] = Field( + default=None, + description="Reference to a directory SCO representing this file's parent. MUST be of type 'directory'.", + ) + contains_refs: Optional[List[str]] = Field( + default=None, + description="List of references to other SCOs contained within the file (e.g., embedded IPs, appended files).", + ) + content_ref: Optional[str] = Field( + default=None, + description="Reference to an Artifact object representing this file's content.", + ) + + @model_validator(mode="after") + def validate_cross_refs(self) -> "FileModel": + """Validate the cross-references in the FileModel instance.""" + if self.parent_directory_ref and not self.parent_directory_ref.startswith( + "directory--" + ): + raise ValueError( + "'parent_directory_ref' must reference an object of type 'directory'." + ) + if self.content_ref and not self.content_ref.startswith("artifact--"): + raise ValueError( + "'content_ref' must reference an object of type 'artifact'." + ) + return self + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return File(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/ipv4_address_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/ipv4_address_model.py new file mode 100644 index 0000000000..6bda4d14b1 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/ipv4_address_model.py @@ -0,0 +1,30 @@ +"""The module defines the IPv4AddressModel class, which represents a STIX 2.1 IPv4 Address object.""" + +from typing import List, Optional + +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field +from stix2.v21 import IPv4Address, _STIXBase21 # type: ignore + + +class IPv4AddressModel(BaseSCOModel): + """Model representing an IPv4 Address in STIX 2.1 format.""" + + value: str = Field( + ..., + description="IPv4 address or CIDR block (e.g., '192.168.1.1' or '10.0.0.0/24'). MUST conform to CIDR notation.", + ) + + resolves_to_refs: Optional[List[str]] = Field( + default=None, + description="(Deprecated) List of MAC address object references this IP resolves to. MUST be of type 'mac-addr'.", + ) + + belongs_to_refs: Optional[List[str]] = Field( + default=None, + description="(Deprecated) List of autonomous-system object references this IP belongs to. MUST be of type 'autonomous-system'.", + ) + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return IPv4Address(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/ipv6_address_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/ipv6_address_model.py new file mode 100644 index 0000000000..453e868324 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/ipv6_address_model.py @@ -0,0 +1,30 @@ +"""The module defines the IPv6AddressModel class, which represents a STIX 2.1 IPv6 Address object.""" + +from typing import List, Optional + +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field +from stix2.v21 import IPv6Address, _STIXBase21 # type: ignore + + +class IPv6AddressModel(BaseSCOModel): + """Model representing an IPv6 Address in STIX 2.1 format.""" + + value: str = Field( + ..., + description="One or more IPv6 addresses expressed in CIDR notation (e.g., '2001:db8::1/64'). /128 MAY be omitted for single addresses.", + ) + + resolves_to_refs: Optional[List[str]] = Field( + default=None, + description="(Deprecated) References to MAC address objects this IPv6 resolves to. MUST be of type 'mac-addr'.", + ) + + belongs_to_refs: Optional[List[str]] = Field( + default=None, + description="(Deprecated) References to autonomous system objects this IPv6 belongs to. MUST be of type 'autonomous-system'.", + ) + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return IPv6Address(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/mac_address_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/mac_address_model.py new file mode 100644 index 0000000000..2451dfbf60 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/mac_address_model.py @@ -0,0 +1,31 @@ +"""The module defines the MACAddressModel class, which represents a STIX 2.1 MAC Address object.""" + +import re + +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field, field_validator +from stix2.v21 import MACAddress, _STIXBase21 # type: ignore + + +class MACAddressModel(BaseSCOModel): + """Model representing a MAC Address in STIX 2.1 format.""" + + value: str = Field( + ..., + description="A single colon-delimited, lowercase MAC-48 address with leading zeros (e.g., 00:00:ab:cd:ef:01).", + ) + + @field_validator("value") + @classmethod + def validate_mac_format(cls, v: str) -> str: + """Validate the MAC address format.""" + pattern = r"^([0-9a-f]{2}:){5}[0-9a-f]{2}$" + if not re.fullmatch(pattern, v): + raise ValueError( + "MAC address must be colon-delimited, lowercase, and include leading zeros (e.g., 00:00:ab:cd:ef:01)." + ) + return v + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return MACAddress(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/mutex_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/mutex_model.py new file mode 100644 index 0000000000..ecbfa85b2a --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/mutex_model.py @@ -0,0 +1,15 @@ +"""The module defines the MutexModel class, which represents a STIX 2.1 Mutex object.""" + +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field +from stix2.v21 import Mutex, _STIXBase21 # type: ignore + + +class MutexModel(BaseSCOModel): + """Model representing a Mutex in STIX 2.1 format.""" + + name: str = Field(..., description="The name of the mutex object as observed.") + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return Mutex(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/network_traffic_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/network_traffic_model.py new file mode 100644 index 0000000000..fd329e2786 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/network_traffic_model.py @@ -0,0 +1,113 @@ +"""The module defines the NetworkTrafficModel class, which represents a STIX 2.1 Network Traffic object.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional, Union + +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field, model_validator +from stix2.v21 import NetworkTraffic, _STIXBase21 # type: ignore + + +class NetworkTrafficModel(BaseSCOModel): + """Model representing a Network Traffic in STIX 2.1 format.""" + + extensions: Optional[Dict[str, Dict[str, Any]]] = Field( + default=None, + description="Dictionary of supported extensions (e.g., http-request-ext, tcp-ext, etc.).", + ) + + start: Optional[datetime] = Field( + default=None, description="Timestamp when the network traffic began." + ) + end: Optional[datetime] = Field( + default=None, + description="Timestamp when the network traffic ended. MUST be > start if both are present.", + ) + is_active: Optional[bool] = Field( + default=None, + description="Indicates whether the network traffic is still ongoing. If true, 'end' MUST NOT be present.", + ) + + src_ref: Optional[str] = Field( + default=None, + description="Source of the traffic. MUST reference ipv4-addr, ipv6-addr, mac-addr, or domain-name.", + ) + dst_ref: Optional[str] = Field( + default=None, + description="Destination of the traffic. MUST reference ipv4-addr, ipv6-addr, mac-addr, or domain-name.", + ) + + src_port: Optional[int] = Field( + default=None, + ge=0, + le=65535, + description="Source port number (0–65535).", + ) + dst_port: Optional[int] = Field( + default=None, + ge=0, + le=65535, + description="Destination port number (0–65535).", + ) + + protocols: List[str] = Field( + ..., + description="List of protocols used in the traffic, from outer to inner layers. SHOULD align with IANA service names.", + ) + + src_byte_count: Optional[int] = Field( + default=None, + ge=0, + description="Number of bytes sent from source to destination.", + ) + dst_byte_count: Optional[int] = Field( + default=None, + ge=0, + description="Number of bytes sent from destination to source.", + ) + src_packets: Optional[int] = Field( + default=None, + ge=0, + description="Number of packets sent from source to destination.", + ) + dst_packets: Optional[int] = Field( + default=None, + ge=0, + description="Number of packets sent from destination to source.", + ) + + ipfix: Optional[Dict[str, Union[str, int]]] = Field( + default=None, + description="IP Flow Information Export data. Keys are case-sensitive IPFIX element names, values are string/int.", + ) + + src_payload_ref: Optional[str] = Field( + default=None, + description="Reference to an Artifact object containing source payload bytes.", + ) + dst_payload_ref: Optional[str] = Field( + default=None, + description="Reference to an Artifact object containing destination payload bytes.", + ) + + encapsulates_refs: Optional[List[str]] = Field( + default=None, + description="References to other network-traffic objects encapsulated by this one.", + ) + encapsulated_by_ref: Optional[str] = Field( + default=None, + description="Reference to a network-traffic object that encapsulates this one.", + ) + + @model_validator(mode="after") + def validate_timestamps_and_state(self) -> "NetworkTrafficModel": + """Validate the timestamps and state of the NetworkTrafficModel instance.""" + if self.start and self.end and self.end < self.start: + raise ValueError("'end' must be later than 'start'.") + if self.is_active and self.end is not None: + raise ValueError("'end' must not be present if 'is_active' is True.") + return self + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return NetworkTraffic(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/process_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/process_model.py new file mode 100644 index 0000000000..7786ec98ec --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/process_model.py @@ -0,0 +1,69 @@ +"""The module defines the ProcessModel class, which represents a STIX 2.1 Process object.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field +from stix2.v21 import Process, _STIXBase21 # type: ignore + + +class ProcessModel(BaseSCOModel): + """Model representing a Process in STIX 2.1 format.""" + + extensions: Optional[Dict[str, Dict[str, Any]]] = Field( + default=None, + description="Dictionary of supported process extensions (e.g., windows-process-ext, windows-service-ext).", + ) + + is_hidden: Optional[bool] = Field( + default=None, + description="Indicates whether the process is hidden from userland/system tools.", + ) + pid: Optional[int] = Field( + default=None, ge=0, description="Process ID (PID) of the process." + ) + created_time: Optional[datetime] = Field( + default=None, description="Timestamp of when the process was created." + ) + cwd: Optional[str] = Field( + default=None, description="Current working directory of the process." + ) + command_line: Optional[str] = Field( + default=None, + description="Full command line used to launch the process (including executable and arguments).", + ) + + environment_variables: Optional[Dict[str, str]] = Field( + default=None, + description="Dictionary of environment variables (case-sensitive). Keys are variable names, values are their string contents.", + ) + + opened_connection_refs: Optional[List[str]] = Field( + default=None, + description="List of references to network-traffic objects opened by this process. MUST be of type 'network-traffic'.", + ) + + creator_user_ref: Optional[str] = Field( + default=None, + description="Reference to a user-account object representing the user that created the process. MUST be of type 'user-account'.", + ) + + image_ref: Optional[str] = Field( + default=None, + description="Reference to a file object representing the executable binary run by this process. MUST be of type 'file'.", + ) + + parent_ref: Optional[str] = Field( + default=None, + description="Reference to the parent process (if any). MUST be of type 'process'.", + ) + + child_refs: Optional[List[str]] = Field( + default=None, + description="References to child processes spawned by this one. MUST be of type 'process'.", + ) + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return Process(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/sco_common_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/sco_common_model.py new file mode 100644 index 0000000000..394f273403 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/sco_common_model.py @@ -0,0 +1,49 @@ +"""The module defines the BaseSCOModel class, which serves as a base model for all STIX Cyber Observable (SCO) objects.""" + +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Field +from stix2.v21 import _STIXBase21 # type: ignore + + +class SCORequiredModel(BaseModel): + """Required fields for all STIX Cyber Observable (SCO) objects.""" + + type: str = Field( + ..., + description="The object type. MUST match the specific SCO type being defined.", + ) + id: str = Field(..., description="The unique STIX identifier for this SCO object.") + + +class SCOOptionalModel(BaseModel): + """Optional fields for all STIX Cyber Observable (SCO) objects.""" + + spec_version: Optional[str] = Field( + default=None, + description="The STIX specification version, typically '2.1'.", + ) + object_marking_refs: Optional[List[str]] = Field( + default=None, + description="List of marking-definition IDs applied to this object.", + ) + granular_markings: Optional[List[Dict[str, Any]]] = Field( + default=None, + description="List of granular markings on specific fields.", + ) + defanged: Optional[bool] = Field( + default=None, + description="Whether the object has been defanged to prevent accidental execution.", + ) + extensions: Optional[Dict[str, Any]] = Field( + default=None, + description="Custom STIX extensions applied to this object.", + ) + + +class BaseSCOModel(SCORequiredModel, SCOOptionalModel): + """Base class for all STIX Cyber Observable (SCO) objects.""" + + def to_stix2_object(self) -> _STIXBase21: + """Convert to a STIX 2.1 object.""" + raise NotImplementedError("Subclasses must implement this method.") diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/software_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/software_model.py new file mode 100644 index 0000000000..902f1cde69 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/software_model.py @@ -0,0 +1,29 @@ +"""The module defines the SoftwareModel class, which represents a STIX 2.1 Software object.""" + +from typing import List, Optional + +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field +from stix2.v21 import Software, _STIXBase21 # type: ignore + + +class SoftwareModel(BaseSCOModel): + """Model representing a Software in STIX 2.1 format.""" + + name: str = Field(..., description="The name of the software.") + cpe: Optional[str] = Field( + default=None, + description="CPE v2.3 entry for the software from the official NVD CPE Dictionary.", + ) + languages: Optional[List[str]] = Field( + default=None, + description="List of supported languages (ISO 639-2 codes).", + ) + vendor: Optional[str] = Field( + default=None, description="The name of the software vendor." + ) + version: Optional[str] = Field(default=None, description="Version of the software.") + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return Software(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/url_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/url_model.py new file mode 100644 index 0000000000..58597e1ea4 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/url_model.py @@ -0,0 +1,18 @@ +"""The module defines the URLModel class, which represents a STIX 2.1 URL object.""" + +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field +from stix2.v21 import URL, _STIXBase21 # type: ignore + + +class URLModel(BaseSCOModel): + """Model representing a URL in STIX 2.1 format.""" + + value: str = Field( + ..., + description="The URL value, which MUST conform to RFC3986 (Uniform Resource Locator).", + ) + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return URL(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/user_account_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/user_account_model.py new file mode 100644 index 0000000000..b3d8a667bc --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/user_account_model.py @@ -0,0 +1,76 @@ +"""The module provides a model for representing User Account objects in STIX 2.1 format.""" + +from datetime import datetime +from typing import Dict, Optional, Union + +from connector.src.stix.v21.models.ovs.account_type_ov_enums import AccountTypeOV +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import Field +from stix2.v21 import UserAccount, _STIXBase21 # type: ignore + + +class UserAccountModel(BaseSCOModel): + """Model representing a User Account in STIX 2.1 format.""" + + extensions: Optional[Dict[str, Dict[str, Union[str, int, bool]]]] = Field( + default=None, + description="Dictionary of extensions. Keys are extension names like 'unix-account-ext', values are the corresponding content.", + ) + + user_id: Optional[str] = Field( + default=None, + description="System-unique identifier for the user account (e.g., UID, GUID, email).", + ) + credential: Optional[str] = Field( + default=None, + description="Cleartext credential (ONLY for malware analysis use cases; avoid sharing PII).", + ) + account_login: Optional[str] = Field( + default=None, + description="Login string entered by the user (e.g., 'root').", + ) + account_type: Optional[AccountTypeOV] = Field( + default=None, + description="Open vocabulary value representing the account type. SHOULD come from account-type-ov.", + ) + display_name: Optional[str] = Field( + default=None, + description="Human-friendly display name (e.g., GECOS field on Unix).", + ) + + is_service_account: Optional[bool] = Field( + default=None, + description="True if account is tied to a system service/daemon.", + ) + is_privileged: Optional[bool] = Field( + default=None, + description="True if account has elevated (e.g., admin/root) privileges.", + ) + can_escalate_privs: Optional[bool] = Field( + default=None, + description="True if account can escalate to elevated privileges (e.g., sudo access).", + ) + is_disabled: Optional[bool] = Field( + default=None, description="True if the account is currently disabled." + ) + + account_created: Optional[datetime] = Field( + default=None, description="Timestamp when the account was created." + ) + account_expires: Optional[datetime] = Field( + default=None, description="Timestamp when the account will expire." + ) + credential_last_changed: Optional[datetime] = Field( + default=None, + description="Timestamp when the account's credential was last changed.", + ) + account_first_login: Optional[datetime] = Field( + default=None, description="Timestamp of the account's first login." + ) + account_last_login: Optional[datetime] = Field( + default=None, description="Timestamp of the account's last login." + ) + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return UserAccount(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/windows_registry_key_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/windows_registry_key_model.py new file mode 100644 index 0000000000..173d4102a2 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/windows_registry_key_model.py @@ -0,0 +1,71 @@ +"""The module defines the WindowsRegistryKeyModel class, which represents a STIX 2.1 Windows Registry Key object.""" + +from datetime import datetime +from typing import List, Optional + +from connector.src.stix.v21.models.ovs.windows_registry_datatype_ov_enums import ( + WindowsRegistryDatatypeOV, +) +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import BaseModel, Field, model_validator +from stix2.v21 import WindowsRegistryKey, _STIXBase21 # type: ignore + + +class WindowsRegistryValueModel(BaseModel): + """Model representing a Windows Registry Value.""" + + name: Optional[str] = Field( + default=None, + description="Name of the registry value. Use empty string for default value.", + ) + data: Optional[str] = Field( + default=None, description="String data stored in the registry value." + ) + data_type: Optional[WindowsRegistryDatatypeOV] = Field( + default=None, + description="The data type of the registry value (e.g., REG_SZ, REG_DWORD).", + ) + + model_config = {"use_enum_values": True} + + @model_validator(mode="after") + def at_least_one_field_required(self) -> "WindowsRegistryValueModel": + """Ensure at least one of name, data, or data_type is set.""" + if self.name is None and self.data is None and self.data_type is None: + raise ValueError( + "At least one of 'name', 'data', or 'data_type' must be set for WindowsRegistryValueModel." + ) + return self + + +class WindowsRegistryKeyModel(BaseSCOModel): + """Model representing a Windows Registry Key in STIX 2.1 format.""" + + key: Optional[str] = Field( + default=None, + description="Full registry key path including hive (e.g., HKEY_LOCAL_MACHINE\\System\\Foo).", + ) + values: Optional[List[WindowsRegistryValueModel]] = Field( + default=None, + description="List of values found under the registry key. Each must include name, data, and data_type.", + ) + + modified_time: Optional[datetime] = Field( + default=None, + description="Timestamp when the registry key was last modified.", + ) + creator_user_ref: Optional[str] = Field( + default=None, + description="Reference to the user-account object that created this key.", + ) + number_of_subkeys: Optional[int] = Field( + default=None, ge=0, description="Number of subkeys under this key." + ) + + model_config = { + "use_enum_values": True, + } + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return WindowsRegistryKey(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/x509_certificate_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/x509_certificate_model.py new file mode 100644 index 0000000000..09109f6d66 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/scos/x509_certificate_model.py @@ -0,0 +1,135 @@ +"""The module defines the X509CertificateModel class, which represents a STIX 2.1 X.509 Certificate object.""" + +from datetime import datetime +from typing import Dict, Optional + +from connector.src.stix.v21.models.scos.sco_common_model import BaseSCOModel +from pydantic import BaseModel, Field +from stix2.v21 import X509Certificate, _STIXBase21 # type: ignore + + +class X509V3ExtensionsTypeModel(BaseModel): + """Model representing X.509 v3 extensions.""" + + basic_constraints: Optional[str] = Field( + default=None, + description="Specifies if the certificate is a CA (e.g., CA:TRUE, pathlen:0). OID: 2.5.29.19", + ) + name_constraints: Optional[str] = Field( + default=None, + description="Namespace for subject names in cert path. OID: 2.5.29.30", + ) + policy_constraints: Optional[str] = Field( + default=None, + description="Constraints on path validation for CA certs. OID: 2.5.29.36", + ) + key_usage: Optional[str] = Field( + default=None, description="Permitted key usages. OID: 2.5.29.15" + ) + extended_key_usage: Optional[str] = Field( + default=None, + description="Purposes for which the public key may be used. OID: 2.5.29.37", + ) + subject_key_identifier: Optional[str] = Field( + default=None, + description="Identifier for the subject public key. OID: 2.5.29.14", + ) + authority_key_identifier: Optional[str] = Field( + default=None, + description="Identifier for the signing authority public key. OID: 2.5.29.35", + ) + subject_alternative_name: Optional[str] = Field( + default=None, + description="Additional subject identities. OID: 2.5.29.17", + ) + issuer_alternative_name: Optional[str] = Field( + default=None, + description="Additional issuer identities. OID: 2.5.29.18", + ) + subject_directory_attributes: Optional[str] = Field( + default=None, + description="Identification attributes of the subject. OID: 2.5.29.9", + ) + crl_distribution_points: Optional[str] = Field( + default=None, description="How CRL info is obtained. OID: 2.5.29.31" + ) + inhibit_any_policy: Optional[str] = Field( + default=None, + description="Max certs before 'anyPolicy' is blocked. OID: 2.5.29.54", + ) + private_key_usage_period_not_before: Optional[datetime] = Field( + default=None, + description="Start of private key usage period, if different from cert validity.", + ) + private_key_usage_period_not_after: Optional[datetime] = Field( + default=None, + description="End of private key usage period, if different from cert validity.", + ) + certificate_policies: Optional[str] = Field( + default=None, + description="One or more policy OIDs and optional qualifiers. OID: 2.5.29.32", + ) + policy_mappings: Optional[str] = Field( + default=None, + description="Pairs of issuer/subject policy OIDs. OID: 2.5.29.33", + ) + + +class X509CertificateModel(BaseSCOModel): + """Model representing an X.509 Certificate in STIX 2.1 format.""" + + is_self_signed: Optional[bool] = Field( + default=None, description="True if the certificate is self-signed." + ) + hashes: Optional[Dict[str, str]] = Field( + default=None, + description="Hashes for the full certificate content. Keys MUST follow hash-algorithm-ov.", + ) + + version: Optional[str] = Field( + default=None, description="Version of the encoded certificate." + ) + serial_number: Optional[str] = Field( + default=None, + description="Unique identifier for the cert as issued by the CA.", + ) + signature_algorithm: Optional[str] = Field( + default=None, description="Algorithm used to sign the certificate." + ) + + issuer: Optional[str] = Field( + default=None, + description="Name of the Certificate Authority that issued this certificate.", + ) + validity_not_before: Optional[datetime] = Field( + default=None, description="Start of certificate validity period." + ) + validity_not_after: Optional[datetime] = Field( + default=None, description="End of certificate validity period." + ) + + subject: Optional[str] = Field( + default=None, + description="Subject name—the entity the certificate is issued to.", + ) + subject_public_key_algorithm: Optional[str] = Field( + default=None, + description="Algorithm for encrypting data to the subject.", + ) + subject_public_key_modulus: Optional[str] = Field( + default=None, + description="RSA modulus portion of the subject's public key.", + ) + subject_public_key_exponent: Optional[int] = Field( + default=None, + description="RSA exponent portion of the subject's public key.", + ) + + x509_v3_extensions: Optional[X509V3ExtensionsTypeModel] = Field( + default=None, + description="Standard X.509 v3 extensions as key-value pairs (e.g., BasicConstraints, SubjectAltName).", + ) + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + return X509Certificate(**self.model_dump(exclude_none=True)) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/__init__.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/attack_pattern_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/attack_pattern_model.py new file mode 100644 index 0000000000..a0391dfcca --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/attack_pattern_model.py @@ -0,0 +1,54 @@ +"""The module contains the AttackPatternModel class, which represents an attack pattern in STIX 2.1 format.""" + +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.cdts.kill_chain_phase_model import ( + KillChainPhaseModel, +) +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import AttackPattern, _STIXBase21 # type: ignore + + +class AttackPatternModel(BaseSDOModel): + """Model representing an Attack Pattern in STIX 2.1 format.""" + + name: str = Field(..., description="A name used to identify the Attack Pattern.") + description: Optional[str] = Field( + default=None, + description="A description that provides more details and context about the Attack Pattern, potentially including its purpose and its key characteristics.", + ) + aliases: Optional[List[str]] = Field( + default=None, + description="Alternative names used to identify this Attack Pattern.", + ) + kill_chain_phases: Optional[List[KillChainPhaseModel]] = Field( + default=None, + description="The list of Kill Chain Phases for which this Attack Pattern is used.", + ) + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = AttackPatternModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "name" in data: + x_mitre_id = data.get("custom_properties", {}).get("x_mitre_id", None) + data["id"] = pycti.AttackPattern.generate_id( + name=data["name"], x_mitre_id=x_mitre_id + ) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = AttackPatternModel._generate_id(data=data) + data.pop("id") + + return AttackPattern(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/campaign_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/campaign_model.py new file mode 100644 index 0000000000..6dc3124ed0 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/campaign_model.py @@ -0,0 +1,57 @@ +"""The module contains the CampaignModel class, which represents a STIX 2.1 Campaign object.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import Campaign, _STIXBase21 # type: ignore + + +class CampaignModel(BaseSDOModel): + """Model representing a Campaign in STIX 2.1 format.""" + + name: str = Field(..., description="A name used to identify the Campaign.") + description: Optional[str] = Field( + default=None, + description="A description that provides more details and context about the Campaign, potentially including its purpose and its key characteristics.", + ) + aliases: Optional[List[str]] = Field( + default=None, + description="Alternative names used to identify this Campaign.", + ) + first_seen: Optional[datetime] = Field( + default=None, + description="The time that this Campaign was first seen. May be updated if earlier sightings are received.", + ) + last_seen: Optional[datetime] = Field( + default=None, + description="The time that this Campaign was last seen. Must be >= first_seen. May be updated with newer sighting data.", + ) + objective: Optional[str] = Field( + default=None, + description="Defines the Campaign’s primary goal, objective, desired outcome, or intended effect — what the Threat Actor or Intrusion Set hopes to accomplish.", + ) + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = CampaignModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "name" in data: + data["id"] = pycti.Campaign.generate_id(name=data["name"]) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = CampaignModel._generate_id(data=data) + data.pop("id") + + return Campaign(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/course_of_action_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/course_of_action_model.py new file mode 100644 index 0000000000..39b962acbc --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/course_of_action_model.py @@ -0,0 +1,74 @@ +"""The module defines the CourseOfActionModel class, which represents a course of action in STIX 2.1 format.""" + +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.cdts.external_reference_model import ( + ExternalReferenceModel, +) +from connector.src.stix.v21.models.ovs.course_of_action_type_ov_enums import ( + CourseOfActionTypeOV, +) +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Base64Bytes, Field, model_validator +from stix2.v21 import CourseOfAction, _STIXBase21 # type: ignore + + +class CourseOfActionModel(BaseSDOModel): + """Model representing a Course of Action in STIX 2.1 format.""" + + name: str = Field(..., description="A name used to identify the Course of Action.") + description: Optional[str] = Field( + default=None, + description="Context for the Course of Action, possibly including intent and characteristics. May contain prose.", + ) + action_type: Optional[CourseOfActionTypeOV] = Field( + default=None, + description="Open vocabulary describing the action type (e.g., textual:text/plain). Should use course-of-action-type-ov.", + ) + os_execution_envs: Optional[List[str]] = Field( + default=None, + description="Recommended OS environments for execution. Preferably CPE v2.3 from NVD. Can include custom values.", + ) + action_bin: Optional[Base64Bytes] = Field( + default=None, + description="Base64-encoded binary representing the Course of Action. MUST NOT be set with action_reference.", + ) + action_reference: Optional[ExternalReferenceModel] = Field( + default=None, + description="External reference to an action. MUST NOT be set if action_bin is present.", + ) + + @model_validator(mode="after") + def validate_action_exclusivity(self, model): # type: ignore + """Ensure that only one of action_bin or action_reference is set.""" + if model.action_bin and model.action_reference: + raise ValueError( + "Only one of 'action_bin' or 'action_reference' may be set, not both." + ) + return model + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = CourseOfActionModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "name" in data: + x_mitre_id = data.get("custom_properties", {}).get("x_mitre_id", None) + data["id"] = pycti.CourseOfAction.generate_id( + name=data["name"], x_mitre_id=x_mitre_id + ) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = CourseOfActionModel._generate_id(data=data) + data.pop("id") + + return CourseOfAction(id=pycti_id, allow_custom=True, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/grouping_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/grouping_model.py new file mode 100644 index 0000000000..b1dd0ac15f --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/grouping_model.py @@ -0,0 +1,58 @@ +"""The module contains the GroupingModel class, which represents a STIX 2.1 Grouping object.""" + +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.ovs.grouping_context_ov_enums import ( + GroupingContextOV, +) +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import Grouping, _STIXBase21 # type: ignore + + +class GroupingModel(BaseSDOModel): + """Model representing a Grouping in STIX 2.1 format.""" + + name: Optional[str] = Field( + default=None, description="A name used to identify the Grouping." + ) + description: Optional[str] = Field( + default=None, + description="A description that provides more details and context about the Grouping, potentially including its purpose and key characteristics.", + ) + context: GroupingContextOV = Field( + ..., + description="Short descriptor of the context shared by the content in this Grouping. SHOULD come from the grouping-context-ov vocabulary.", + ) + object_refs: List[str] = Field( + ..., + description="List of STIX Object identifiers referred to by this Grouping.", + ) + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = GroupingModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "name" in data: + name = data.get("name", None) + context = data.get("context", None) + created = data.get("created", None) + data["id"] = pycti.Grouping.generate_id( + name=name, context=context, created=created + ) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = GroupingModel._generate_id(data=data) + data.pop("id") + + return Grouping(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/identity_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/identity_model.py new file mode 100644 index 0000000000..aa929ee082 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/identity_model.py @@ -0,0 +1,69 @@ +"""The module contains the IdentityModel class, which represents a STIX 2.1 Identity object.""" + +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.ovs.identity_class_ov_enums import ( + IdentityClassOV, +) +from connector.src.stix.v21.models.ovs.industry_sector_ov_enums import ( + IndustrySectorOV, +) +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import Identity, _STIXBase21 # type: ignore + + +class IdentityModel(BaseSDOModel): + """Model representing an Identity in STIX 2.1 format.""" + + name: str = Field( + ..., + description="The name of this Identity. SHOULD be the canonical name when referring to a specific entity.", + ) + description: Optional[str] = Field( + default=None, + description="More details and context about the Identity, including its purpose and characteristics.", + ) + roles: Optional[List[str]] = Field( + default=None, + description="The roles this Identity performs (e.g., CEO, Domain Admins, Doctors). No open vocabulary yet defined.", + ) + identity_class: IdentityClassOV = Field( + ..., + description="The type of entity described by this Identity. SHOULD come from the identity-class-ov vocabulary.", + ) + sectors: Optional[List[IndustrySectorOV]] = Field( + default=None, + description="Industry sectors this Identity belongs to. SHOULD come from the industry-sector-ov vocabulary.", + ) + contact_information: Optional[str] = Field( + default=None, + description="Contact details for this Identity (email, phone, etc.). No defined format.", + ) + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = IdentityModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "name" in data: + name = data.get("name", None) + identity_class = data.get("identity_class", None) + data["id"] = pycti.Identity.generate_id( + name=name, identity_class=identity_class + ) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = IdentityModel._generate_id(data=data) + data.pop("id") + + return Identity(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/indicator_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/indicator_model.py new file mode 100644 index 0000000000..42f7a42c97 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/indicator_model.py @@ -0,0 +1,79 @@ +"""The module contains the IndicatorModel class, which represents a STIX 2.1 Indicator object.""" + +from datetime import datetime +from typing import Any, Dict, List, Literal, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.cdts.kill_chain_phase_model import ( + KillChainPhaseModel, +) +from connector.src.stix.v21.models.ovs.indicator_type_ov_enums import ( + IndicatorTypeOV, +) +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import Indicator, _STIXBase21 # type: ignore + + +class IndicatorModel(BaseSDOModel): + """Model representing an Indicator in STIX 2.1 format.""" + + name: Optional[str] = Field( + default=None, + description="A name used to identify the Indicator. Helps analysts and tools understand its purpose.", + ) + description: Optional[str] = Field( + default=None, + description="Details and context about the Indicator's intent, behavior, and usage.", + ) + indicator_types: List[IndicatorTypeOV] = Field( + ..., + description="Open vocabulary categorizing the type of Indicator. SHOULD come from the indicator-type-ov vocabulary.", + ) + pattern: str = Field( + ..., + description="The detection pattern expressed using the STIX Pattern specification (section 9).", + ) + pattern_type: Optional[Literal["stix", "snort", "yara"]] = Field( + ..., + description="The type of pattern used (e.g., stix, snort, yara). Open vocabulary.", + ) + pattern_version: Optional[str] = Field( + default=None, + description="Version of the pattern used. If no spec version exists, use the build or code version.", + ) + valid_from: datetime = Field( + ..., + description="Timestamp when the Indicator becomes valid for detecting behavior.", + ) + valid_until: Optional[datetime] = Field( + default=None, + description="Timestamp when this Indicator is no longer considered valid. MUST be > valid_from if set.", + ) + kill_chain_phases: Optional[List[KillChainPhaseModel]] = Field( + default=None, + description="Kill chain phases to which this Indicator corresponds.", + ) + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = IndicatorModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "pattern" in data: + pattern = data.get("pattern", None) + data["id"] = pycti.Indicator.generate_id(pattern=pattern) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = IndicatorModel._generate_id(data=data) + data.pop("id") + + return Indicator(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/infrastructure_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/infrastructure_model.py new file mode 100644 index 0000000000..6d4b1c03c5 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/infrastructure_model.py @@ -0,0 +1,70 @@ +"""The module defines the InfrastructureModel class, which represents a STIX 2.1 Infrastructure object.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.cdts.kill_chain_phase_model import ( + KillChainPhaseModel, +) +from connector.src.stix.v21.models.ovs.infrastructure_type_ov_enums import ( + InfrastructureTypeOV, +) +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import Infrastructure, _STIXBase21 # type: ignore + + +class InfrastructureModel(BaseSDOModel): + """Model representing an Infrastructure in STIX 2.1 format.""" + + name: str = Field( + ..., + description="A name or characterizing text used to identify the Infrastructure.", + ) + description: Optional[str] = Field( + default=None, + description="More details and context about the Infrastructure—purpose, use, relationships, and key characteristics.", + ) + infrastructure_types: List[InfrastructureTypeOV] = Field( + ..., + description="Open vocabulary describing the type(s) of Infrastructure. SHOULD come from the infrastructure-type-ov vocabulary.", + ) + aliases: Optional[List[str]] = Field( + default=None, + description="Alternative names used to identify this Infrastructure.", + ) + kill_chain_phases: Optional[List[KillChainPhaseModel]] = Field( + default=None, + description="Kill Chain Phases for which this Infrastructure is used.", + ) + first_seen: Optional[datetime] = Field( + default=None, + description="Timestamp when this Infrastructure was first observed performing malicious activity.", + ) + last_seen: Optional[datetime] = Field( + default=None, + description="Timestamp when this Infrastructure was last observed. MUST be >= first_seen if both are present.", + ) + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = InfrastructureModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "name" in data: + data["id"] = pycti.Infrastructure.generate_id(name=data["name"]) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = InfrastructureModel._generate_id(data=data) + data.pop("id") + + return Infrastructure(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/intrusion_set_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/intrusion_set_model.py new file mode 100644 index 0000000000..e5bbdd48e1 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/intrusion_set_model.py @@ -0,0 +1,75 @@ +"""The module defines the IntrusionSetModel class, which represents a STIX 2.1 Intrusion Set object.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.ovs.attack_motivation_ov_enums import ( + AttackMotivationOV, +) +from connector.src.stix.v21.models.ovs.attack_resource_level_ov_enums import ( + AttackResourceLevelOV, +) +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import IntrusionSet, _STIXBase21 # type: ignore + + +class IntrusionSetModel(BaseSDOModel): + """Model representing an Intrusion Set in STIX 2.1 format.""" + + name: str = Field(..., description="A name used to identify this Intrusion Set.") + description: Optional[str] = Field( + default=None, + description="Details and context about the Intrusion Set, including its purpose and key characteristics.", + ) + aliases: Optional[List[str]] = Field( + default=None, + description="Alternative names used to identify this Intrusion Set.", + ) + first_seen: Optional[datetime] = Field( + default=None, + description="Timestamp when this Intrusion Set was first seen. May be updated with earlier sightings.", + ) + last_seen: Optional[datetime] = Field( + default=None, + description="Timestamp when this Intrusion Set was last seen. MUST be >= first_seen if both are set.", + ) + goals: Optional[List[str]] = Field( + default=None, + description="High-level goals of this Intrusion Set—what they're trying to achieve.", + ) + resource_level: Optional[AttackResourceLevelOV] = Field( + default=None, + description="Organizational level at which this Intrusion Set operates. SHOULD come from the attack-resource-level-ov vocabulary.", + ) + primary_motivation: Optional[AttackMotivationOV] = Field( + default=None, + description="Primary motivation behind this Intrusion Set. SHOULD come from the attack-motivation-ov vocabulary.", + ) + secondary_motivations: Optional[List[AttackMotivationOV]] = Field( + default=None, + description="Secondary motivations behind this Intrusion Set. SHOULD come from the attack-motivation-ov vocabulary.", + ) + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = IntrusionSetModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "name" in data: + data["id"] = pycti.IntrusionSet.generate_id(name=data["name"]) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = IntrusionSetModel._generate_id(data=data) + data.pop("id") + + return IntrusionSet(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/location_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/location_model.py new file mode 100644 index 0000000000..f36569868e --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/location_model.py @@ -0,0 +1,90 @@ +"""The module defines the LocationModel class, which represents a STIX 2.1 Location object.""" + +from typing import Any, Dict, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.ovs.region_ov_enums import RegionOV +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import Location, _STIXBase21 # type: ignore + + +class LocationModel(BaseSDOModel): + """Model representing a Location in STIX 2.1 format.""" + + name: Optional[str] = Field( + default=None, description="A name used to identify the Location." + ) + description: Optional[str] = Field( + default=None, description="A textual description of the Location." + ) + + latitude: Optional[float] = Field( + default=None, + description="Latitude in decimal degrees. Must be between -90.0 and 90.0. Required if longitude is present.", + ge=-90.0, + le=90.0, + ) + longitude: Optional[float] = Field( + default=None, + description="Longitude in decimal degrees. Must be between -180.0 and 180.0. Required if latitude is present.", + ge=-180.0, + le=180.0, + ) + precision: Optional[float] = Field( + default=None, + description="Precision in meters. If present, latitude and longitude MUST also be present.", + ) + + region: Optional[RegionOV] = Field( + default=None, description="Region for this location (from region-ov)." + ) + country: Optional[str] = Field( + default=None, description="Country code in ISO 3166-1 ALPHA-2 format." + ) + administrative_area: Optional[str] = Field( + default=None, description="Sub-national area (state, province, etc)." + ) + city: Optional[str] = Field( + default=None, description="The city that this Location describes." + ) + street_address: Optional[str] = Field( + default=None, description="The full street address for the Location." + ) + postal_code: Optional[str] = Field( + default=None, description="Postal code for the Location." + ) + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = LocationModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "name" in data: + name = data.get("name", None) + x_opencti_location_type = data.get("custom_properties", {}).get( + "x_opencti_location_type", None + ) + latitude = data.get("latitude", None) + longitude = data.get("longitude", None) + + data["id"] = pycti.Location.generate_id( + name=name, + x_opencti_location_type=x_opencti_location_type, + latitude=latitude, + longitude=longitude, + ) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = LocationModel._generate_id(data=data) + data.pop("id") + + return Location(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/malware_analysis_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/malware_analysis_model.py new file mode 100644 index 0000000000..26e79e100f --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/malware_analysis_model.py @@ -0,0 +1,99 @@ +"""The module defines a Pydantic model for the STIX 2.1 Malware Analysis object.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import MalwareAnalysis, _STIXBase21 # type: ignore + + +class MalwareAnalysisModel(BaseSDOModel): + """Model representing a Malware Analysis in STIX 2.1 format.""" + + product: str = Field( + ..., + description="Name of the analysis engine or product used. SHOULD be lowercase and dash-separated. Use 'anonymized' if the name must be redacted.", + ) + version: Optional[str] = Field( + default=None, description="Version of the analysis product used." + ) + host_vm_ref: Optional[str] = Field( + default=None, + description="Identifier for the virtual machine (SCO software object) used in dynamic analysis.", + ) + operating_system_ref: Optional[str] = Field( + default=None, + description="Identifier for the OS (SCO software object) used in dynamic analysis.", + ) + installed_software_refs: Optional[List[str]] = Field( + default=None, + description="Identifiers for any non-standard software installed on the analysis OS.", + ) + configuration_version: Optional[str] = Field( + default=None, + description="Named configuration version of the product's analysis setup.", + ) + modules: Optional[List[str]] = Field( + default=None, + description="List of specific analysis modules enabled during the analysis run.", + ) + analysis_engine_version: Optional[str] = Field( + default=None, description="Version of the analysis engine used." + ) + analysis_definition_version: Optional[str] = Field( + default=None, + description="Version of definitions (AV or other) used by the analysis engine.", + ) + submitted: Optional[datetime] = Field( + default=None, + description="Timestamp when the malware was submitted for analysis.", + ) + analysis_started: Optional[datetime] = Field( + default=None, description="Timestamp when the analysis began." + ) + analysis_ended: Optional[datetime] = Field( + default=None, description="Timestamp when the analysis ended." + ) + analysis_sco_refs: Optional[List[str]] = Field( + default=None, + description="References to SCOs captured during the analysis.", + ) + + @model_validator(mode="after") + def validate_timestamps(self) -> "MalwareAnalysisModel": + """Validate the timestamps in the model.""" + if self.analysis_started and self.analysis_ended: + if self.analysis_ended < self.analysis_started: + raise ValueError( + "'analysis_ended' must be greater than or equal to 'analysis_started'." + ) + return self + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = MalwareAnalysisModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "name" in data: + name = data.get("name", None) + product = data.get("product", None) + submitted = data.get("submitted", None) + data["id"] = pycti.MalwareAnalysis.generate_id( + result_name=name, product=product, submitted=submitted + ) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = MalwareAnalysisModel._generate_id(data=data) + data.pop("id") + + return MalwareAnalysis(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/malware_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/malware_model.py new file mode 100644 index 0000000000..89b06382d1 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/malware_model.py @@ -0,0 +1,111 @@ +"""The module defines a Pydantic model for the STIX 2.1 Malware Analysis object.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.cdts.kill_chain_phase_model import ( + KillChainPhaseModel, +) +from connector.src.stix.v21.models.ovs.implementation_language_ov_enums import ( + ImplementationLanguageOV, +) +from connector.src.stix.v21.models.ovs.malware_capabilities_ov_enums import ( + MalwareCapabilitiesOV, +) +from connector.src.stix.v21.models.ovs.malware_type_ov_enums import MalwareTypeOV +from connector.src.stix.v21.models.ovs.processor_architecture_ov_enums import ( + ProcessorArchitectureOV, +) +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import Malware, _STIXBase21 # type: ignore + + +class MalwareModel(BaseSDOModel): + """Model representing a Malware in STIX 2.1 format.""" + + name: Optional[str] = Field( + default=None, + description="A name used to identify the malware instance or family. MUST be defined for a family. If unavailable for an instance, SHA-256 or filename MAY be used.", + ) + description: Optional[str] = Field( + default=None, + description="Detailed context about the malware instance or family—its purpose and key characteristics.", + ) + malware_types: List[MalwareTypeOV] = Field( + ..., + description="Open vocabulary categorizing the malware. SHOULD come from malware-type-ov.", + ) + is_family: bool = Field( + ..., + description="True if this is a malware family, False if it's a specific instance.", + ) + aliases: Optional[List[str]] = Field( + default=None, + description="Alternative names used to identify this malware or malware family.", + ) + kill_chain_phases: Optional[List[KillChainPhaseModel]] = Field( + default=None, + description="Kill Chain Phases in which this malware can be used.", + ) + first_seen: Optional[datetime] = Field( + default=None, + description="Earliest time this malware was observed. May be updated with earlier sightings.", + ) + last_seen: Optional[datetime] = Field( + default=None, + description="Most recent time this malware was observed. MUST be >= first_seen if both are set.", + ) + os_execution_envs: Optional[List[str]] = Field( + default=None, + description="Operating systems this malware runs on. SHOULD use CPE v2.3 format.", + ) + architecture_execution_envs: Optional[List[ProcessorArchitectureOV]] = Field( + default=None, + description="Processor architectures this malware can execute on. SHOULD come from processor-architecture-ov.", + ) + implementation_languages: Optional[List[ImplementationLanguageOV]] = Field( + default=None, + description="Programming languages used to implement this malware. SHOULD come from implementation-language-ov.", + ) + capabilities: Optional[List[MalwareCapabilitiesOV]] = Field( + default=None, + description="Identified capabilities of the malware. SHOULD come from malware-capabilities-ov.", + ) + sample_refs: Optional[List[str]] = Field( + default=None, + description="List of identifiers for SCO file or artifact samples linked to this malware.", + ) + + @model_validator(mode="after") + def validate_timestamps(self) -> "MalwareModel": + """Ensure that 'last_seen' is greater than or equal to 'first_seen'.""" + if self.first_seen and self.last_seen: + if self.last_seen < self.first_seen: + raise ValueError( + "'last_seen' must be greater than or equal to 'first_seen'." + ) + return self + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = MalwareModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "name" in data: + data["id"] = pycti.Malware.generate_id(name=data["name"]) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = MalwareModel._generate_id(data=data) + data.pop("id") + + return Malware(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/note_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/note_model.py new file mode 100644 index 0000000000..230fa55283 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/note_model.py @@ -0,0 +1,48 @@ +"""The module defines the NoteModel class, which represents a STIX 2.1 Note object.""" + +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import Note, _STIXBase21 # type: ignore + + +class NoteModel(BaseSDOModel): + """Model representing a Note in STIX 2.1 format.""" + + abstract: Optional[str] = Field( + default=None, description="A brief summary of the note content." + ) + content: str = Field(..., description="The main content of the note.") + authors: Optional[List[str]] = Field( + default=None, + description="Names of the author(s) of this note (e.g., the analyst(s) who wrote it).", + ) + object_refs: List[str] = Field( + ..., description="STIX Object identifiers this note applies to." + ) + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = NoteModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "created" in data: + created = data.get("created", None) + content = data.get("content", None) + data["id"] = pycti.Note.generate_id(created=created, content=content) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = NoteModel._generate_id(data=data) + data.pop("id") + + return Note(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/observed_data_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/observed_data_model.py new file mode 100644 index 0000000000..6217d2eb10 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/observed_data_model.py @@ -0,0 +1,73 @@ +"""The module defines the ObservedDataModel class, which represents a STIX 2.1 Observed Data object.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import ObservedData, _STIXBase21 # type: ignore + + +class ObservedDataModel(BaseSDOModel): + """Model representing an Observed Data in STIX 2.1 format.""" + + first_observed: datetime = Field( + ..., description="Start time of the observation window." + ) + last_observed: datetime = Field( + ..., + description="End time of the observation window. MUST be >= first_observed.", + ) + number_observed: int = Field( + ..., + ge=1, + le=999_999_999, + description="Number of times the data was observed. MUST be an integer between 1 and 999,999,999 inclusive.", + ) + objects: Optional[Dict[str, Dict[str, Any]]] = Field( + default=None, + description="(Deprecated) Dictionary of SCOs observed. MUST NOT be present if object_refs is set. Will be removed in future STIX versions.", + ) + object_refs: Optional[List[str]] = Field( + default=None, + description="List of references to SCOs/SROs observed. MUST NOT be set if 'objects' is present.", + ) + + @model_validator(mode="after") + def validate_observed_data(self) -> "ObservedDataModel": + """Validate the ObservedDataModel instance.""" + if self.last_observed < self.first_observed: + raise ValueError( + "'last_observed' must be greater than or equal to 'first_observed'." + ) + if self.objects and self.object_refs: + raise ValueError( + "Only one of 'objects' or 'object_refs' may be set—not both." + ) + if not self.objects and not self.object_refs: + raise ValueError("At least one of 'objects' or 'object_refs' must be set.") + return self + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = ObservedDataModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "object_refs" in data: + object_ids = data.get("object_refs", []) + data["id"] = pycti.ObservedData.generate_id(object_ids=object_ids) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = ObservedDataModel._generate_id(data=data) + data.pop("id") + + return ObservedData(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/opinion_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/opinion_model.py new file mode 100644 index 0000000000..88ea97ec5e --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/opinion_model.py @@ -0,0 +1,54 @@ +"""The module defines the OpinionModel class, which represents a STIX 2.1 Opinion object.""" + +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.ovs.opinion_ov_enums import OpinionOV +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import Opinion, _STIXBase21 # type: ignore + + +class OpinionModel(BaseSDOModel): + """Model representing an Opinion in STIX 2.1 format.""" + + explanation: Optional[str] = Field( + default=None, + description="Explanation for the Opinion, including reasoning and any supporting evidence.", + ) + authors: Optional[List[str]] = Field( + default=None, + description="List of authors (e.g., analysts) who created this Opinion.", + ) + opinion: OpinionOV = Field( + ..., + description="The producer's opinion about the object(s). MUST be a value from the opinion-enum.", + ) + object_refs: List[str] = Field( + ..., + description="STIX Object identifiers that this Opinion applies to.", + ) + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = OpinionModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "created" in data: + created = data.get("created", None) + opinion = data.get("opinion", None) + data["id"] = pycti.Opinion.generate_id(created=created, opinion=opinion) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = OpinionModel._generate_id(data=data) + data.pop("id") + + return Opinion(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/report_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/report_model.py new file mode 100644 index 0000000000..9b60de8eb6 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/report_model.py @@ -0,0 +1,62 @@ +"""The module defines a ReportModel class that represents a STIX 2.1 Report object.""" + +from collections import OrderedDict +from datetime import datetime +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.ovs.report_type_ov_enums import ReportTypeOV +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.properties import ListProperty, ReferenceProperty # type: ignore +from stix2.v21 import Report, _STIXBase21 # type: ignore + + +class ReportModel(BaseSDOModel): + """Model representing a Report in STIX 2.1 format.""" + + name: str = Field(..., description="A name used to identify the Report.") + description: Optional[str] = Field( + default=None, + description="More details and context about the Report—its purpose and key characteristics.", + ) + report_types: List[ReportTypeOV] = Field( + ..., + description="Open vocabulary defining the primary subject(s) of this report. SHOULD use report-type-ov.", + ) + published: datetime = Field( + ..., description="The official publication date of this Report." + ) + object_refs: List[str] = Field( + ..., + description="List of STIX Object identifiers referenced in this Report.", + ) + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = ReportModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "name" in data: + name = data.get("name", None) + published = data.get("published", None) + data["id"] = pycti.Report.generate_id(name=name, published=published) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + Report._properties = OrderedDict(Report._properties) + Report._properties["object_refs"] = ListProperty( + ReferenceProperty(valid_types=["SCO", "SDO", "SRO"], spec_version="2.1"), + required=False, + ) + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = ReportModel._generate_id(data=data) + data.pop("id") + + return Report(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/sdo_common_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/sdo_common_model.py new file mode 100644 index 0000000000..0b9ba20b64 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/sdo_common_model.py @@ -0,0 +1,96 @@ +"""The module defines the base model for STIX Domain Objects (SDOs) in STIX 2.1 format.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +from connector.src.stix.v21.models.cdts.external_reference_model import ( + ExternalReferenceModel, +) +from pydantic import BaseModel, Field, model_validator +from stix2.v21 import _STIXBase21 # type: ignore + + +class SDORequiredModel(BaseModel): + """Required fields for all STIX Domain Objects (SDOs).""" + + type: str = Field(..., description="The object type, must match the SDO type.") + spec_version: str = Field( + ..., description="The STIX specification version, e.g., '2.1'." + ) + id: str = Field(..., description="The unique STIX identifier for this object.") + created: datetime = Field( + ..., description="Timestamp when this object was created." + ) + modified: datetime = Field( + ..., description="Timestamp when this object was last modified." + ) + + +class SDOOptionalModel(BaseModel): + """Optional fields for all STIX Domain Objects (SDOs).""" + + created_by_ref: Optional[str] = Field( + default=None, + description="Reference to the identity that created the object.", + ) + revoked: Optional[bool] = Field( + default=None, + description="Indicates whether this object has been revoked.", + ) + labels: Optional[List[str]] = Field( + default=None, description="User-defined labels for this object." + ) + confidence: Optional[int] = Field( + default=None, + description="Level of confidence in the accuracy of this object (0–100).", + ) + lang: Optional[str] = Field( + default=None, description="Language code used for this object." + ) + external_references: Optional[List[ExternalReferenceModel]] = Field( + default=None, + description="List of external references relevant to this object.", + ) + object_marking_refs: Optional[List[str]] = Field( + default=None, + description="List of marking-definition IDs that apply to this object.", + ) + granular_markings: Optional[List[Any]] = Field( + default=None, + description="Granular markings on specific object fields.", + ) + extensions: Optional[dict[str, Any]] = Field( + default=None, + description="Custom STIX extensions applied to this object.", + ) + custom_properties: Optional[dict[str, Any]] = Field( + default=None, + description="Custom properties that are not part of the STIX specification.", + ) + + +class BaseSDOModel(SDORequiredModel, SDOOptionalModel): + """Base model for all SDOs (STIX Domain Objects).""" + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided. + + This base implementation doesn't generate an ID. Subclasses must override + this method to provide type-specific ID generation logic using pycti. + + The generated ID will replace any existing ID to ensure consistency. + """ + return data + + def to_stix2_object(self) -> _STIXBase21: + """Convert the Pydantic model to a STIX 2.1 object. + + This base implementation doesn't create a concrete object. + Subclasses must implement this method with type-specific logic. + + Important: The implementation should always generate a new ID using pycti + regardless of whether an ID was provided in the model to ensure consistency. + """ + raise NotImplementedError("Subclasses must implement this method.") diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/threat_actor_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/threat_actor_model.py new file mode 100644 index 0000000000..b452ae38d3 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/threat_actor_model.py @@ -0,0 +1,120 @@ +"""The module defines a Threat Actor model for STIX 2.1, including validation and serialization methods.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.ovs.attack_motivation_ov_enums import ( + AttackMotivationOV, +) +from connector.src.stix.v21.models.ovs.attack_resource_level_ov_enums import ( + AttackResourceLevelOV, +) +from connector.src.stix.v21.models.ovs.threat_actor_role_ov_enums import ( + ThreatActorRoleOV, +) +from connector.src.stix.v21.models.ovs.threat_actor_sophistication_ov_enums import ( + ThreatActorSophisticationOV, +) +from connector.src.stix.v21.models.ovs.threat_actor_type_ov_enums import ( + ThreatActorTypeOV, +) +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import ThreatActor, _STIXBase21 # type: ignore + + +class ThreatActorModel(BaseSDOModel): + """Model representing a Threat Actor in STIX 2.1 format.""" + + name: str = Field( + ..., description="A name used to identify this Threat Actor or group." + ) + description: Optional[str] = Field( + default=None, + description="Context and characteristics of the Threat Actor—who they are, how they operate, and why.", + ) + + threat_actor_types: List[ThreatActorTypeOV] = Field( + ..., + description="Open vocab describing the type(s) of this Threat Actor. SHOULD come from threat-actor-type-ov.", + ) + aliases: Optional[List[str]] = Field( + default=None, + description="Other names believed to refer to the same Threat Actor.", + ) + + first_seen: Optional[datetime] = Field( + default=None, + description="Time this Threat Actor was first seen. May be updated with earlier sightings.", + ) + last_seen: Optional[datetime] = Field( + default=None, + description="Time this Threat Actor was last seen. MUST be >= first_seen if both are set.", + ) + + roles: Optional[List[ThreatActorRoleOV]] = Field( + default=None, + description="Roles this Threat Actor plays. SHOULD come from threat-actor-role-ov.", + ) + goals: Optional[List[str]] = Field( + default=None, + description="High-level goals of the Threat Actor (e.g., steal credit card numbers, exfiltrate data).", + ) + + sophistication: Optional[ThreatActorSophisticationOV] = Field( + default=None, + description="Level of knowledge, training, or expertise. SHOULD come from threat-actor-sophistication-ov.", + ) + resource_level: Optional[AttackResourceLevelOV] = Field( + default=None, + description="Organizational level this Threat Actor operates at. SHOULD come from attack-resource-level-ov.", + ) + + primary_motivation: Optional[AttackMotivationOV] = Field( + default=None, + description="Primary reason driving this Threat Actor. SHOULD come from attack-motivation-ov.", + ) + secondary_motivations: Optional[List[AttackMotivationOV]] = Field( + default=None, + description="Additional motivations influencing this Threat Actor. SHOULD come from attack-motivation-ov.", + ) + personal_motivations: Optional[List[AttackMotivationOV]] = Field( + default=None, + description="Personal (non-organizational) motivations behind actions. SHOULD come from attack-motivation-ov.", + ) + + @model_validator(mode="after") + def validate_seen_window(self) -> "ThreatActorModel": + """Ensure 'last_seen' is greater than or equal to 'first_seen'.""" + if self.first_seen and self.last_seen and self.last_seen < self.first_seen: + raise ValueError( + "'last_seen' must be greater than or equal to 'first_seen'." + ) + return self + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = ThreatActorModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "name" in data: + name = data.get("name", None) + opencti_type = data.get("custom_properties", {}).get("opencti_type", None) + data["id"] = pycti.ThreatActor.generate_id( + name=name, opencti_type=opencti_type + ) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = ThreatActorModel._generate_id(data=data) + data.pop("id") + + return ThreatActor(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/tool_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/tool_model.py new file mode 100644 index 0000000000..ad8350c50d --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/tool_model.py @@ -0,0 +1,59 @@ +"""The module defines the ToolModel class, which represents a STIX 2.1 Tool object.""" + +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.cdts.kill_chain_phase_model import ( + KillChainPhaseModel, +) +from connector.src.stix.v21.models.ovs.tool_type_ov_enums import ToolTypeOV +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import Tool, _STIXBase21 # type: ignore + + +class ToolModel(BaseSDOModel): + """Model representing a Tool in STIX 2.1 format.""" + + name: str = Field(..., description="The name used to identify the Tool.") + description: Optional[str] = Field( + default=None, + description="Details about the Tool's purpose, use, and characteristics.", + ) + tool_types: List[ToolTypeOV] = Field( + ..., + description="Open vocabulary of tool types. SHOULD come from tool-type-ov.", + ) + aliases: Optional[List[str]] = Field( + default=None, + description="Alternative names used to identify this Tool.", + ) + kill_chain_phases: Optional[List[KillChainPhaseModel]] = Field( + default=None, + description="Kill Chain Phases where this Tool can be used.", + ) + tool_version: Optional[str] = Field( + default=None, description="Version identifier of the Tool." + ) + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = ToolModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "name" in data: + data["id"] = pycti.Tool.generate_id(name=data["name"]) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = ToolModel._generate_id(data=data) + data.pop("id") + + return Tool(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/vulnerability_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/vulnerability_model.py new file mode 100644 index 0000000000..2f145b20df --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sdos/vulnerability_model.py @@ -0,0 +1,40 @@ +"""The module defines the VulnerabilityModel class, which represents a STIX 2.1 Vulnerability object.""" + +from typing import Any, Dict, Optional + +import pycti # type: ignore +from connector.src.stix.v21.models.sdos.sdo_common_model import BaseSDOModel +from pydantic import Field, model_validator +from stix2.v21 import Vulnerability, _STIXBase21 # type: ignore + + +class VulnerabilityModel(BaseSDOModel): + """Model representing a Vulnerability in STIX 2.1 format.""" + + name: str = Field(..., description="A name used to identify the Vulnerability.") + description: Optional[str] = Field( + default=None, + description="Context and details about the Vulnerability, including purpose and key characteristics.", + ) + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = VulnerabilityModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "name" in data: + data["id"] = pycti.Vulnerability.generate_id(name=data["name"]) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = VulnerabilityModel._generate_id(data=data) + data.pop("id") + + return Vulnerability(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/stix/v21/models/sros/relationship_model.py b/external-import/google-ti-feeds/connector/src/stix/v21/models/sros/relationship_model.py new file mode 100644 index 0000000000..1500975ff1 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/stix/v21/models/sros/relationship_model.py @@ -0,0 +1,129 @@ +"""The module defines a RelationshipModel class that represents a STIX 2.1 Relationship object.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +import pycti # type: ignore +from pydantic import BaseModel, Field, model_validator +from stix2.v21 import Relationship, _STIXBase21 # type: ignore + + +class RelationshipModel(BaseModel): + """Model representing a Relationship in STIX 2.1 format.""" + + type: str = Field( + "relationship", + description="The type of this object, which MUST be 'relationship'.", + ) + spec_version: str = Field( + "2.1", + description="The version of the STIX specification used to represent this object.", + ) + id: str = Field(..., description="The identifier of this object.") + created: datetime = Field( + ..., description="The time at which this object was created." + ) + modified: datetime = Field( + ..., description="The time at which this object was last modified." + ) + + relationship_type: str = Field( + ..., + description="The name used to identify the type of relationship, e.g., 'indicates' or 'mitigates'.", + ) + source_ref: str = Field( + ..., + description="The ID of the source (from) object in the relationship.", + ) + target_ref: str = Field( + ..., + description="The ID of the target (to) object in the relationship.", + ) + + description: Optional[str] = Field( + default=None, + description="A description that provides more details and context about the Relationship.", + ) + start_time: Optional[datetime] = Field( + default=None, + description="When the relationship began or was in effect.", + ) + stop_time: Optional[datetime] = Field( + default=None, + description="When the relationship ended or was no longer in effect.", + ) + + created_by_ref: Optional[str] = Field( + default=None, + description="Reference to the identity that created the object.", + ) + revoked: Optional[bool] = Field( + default=None, + description="Indicates whether this object has been revoked.", + ) + labels: Optional[List[str]] = Field( + default=None, description="User-defined labels for this object." + ) + confidence: Optional[int] = Field( + default=None, + description="Level of confidence in the accuracy of this object (0–100).", + ) + lang: Optional[str] = Field( + default=None, description="Language code used for this object." + ) + external_references: Optional[List[Dict[str, Any]]] = Field( + default=None, + description="List of external references relevant to this object.", + ) + object_marking_refs: Optional[List[str]] = Field( + default=None, + description="List of marking-definition IDs that apply to this object.", + ) + granular_markings: Optional[List[Dict[str, Any]]] = Field( + default=None, + description="Granular markings on specific object fields.", + ) + extensions: Optional[Dict[str, Dict[str, Any]]] = Field( + default=None, + description="Custom STIX extensions applied to this object.", + ) + custom_properties: Optional[Dict[str, Any]] = Field( + default=None, + description="Custom properties that are not part of the STIX specification.", + ) + + model_config = {"extra": "forbid"} + + @model_validator(mode="before") + @classmethod + def generate_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Generate ID regardless of whether one is provided.""" + data["id"] = RelationshipModel._generate_id(data=data) + return data + + @classmethod + def _generate_id(cls, data: Dict[str, Any]) -> Any: + """Generate ID regardless of whether one is provided.""" + if isinstance(data, dict) and "relationship_type" in data: + relationship_type = data.get("relationship_type", None) + source_ref = data.get("source_ref", None) + target_ref = data.get("target_ref", None) + start_time = data.get("start_time", None) + stop_time = data.get("stop_time", None) + + data["id"] = pycti.Relationship.generate_id( + relationship_type=relationship_type, + source_ref=source_ref, + target_ref=target_ref, + start_time=start_time, + stop_time=stop_time, + ) + return data["id"] + + def to_stix2_object(self) -> _STIXBase21: + """Convert the model to a STIX 2.1 object.""" + data = self.model_dump(exclude={"id"}, exclude_none=True) + pycti_id = RelationshipModel._generate_id(data=data) + data.pop("id") + + return Relationship(id=pycti_id, **data) diff --git a/external-import/google-ti-feeds/connector/src/utils/__init__.py b/external-import/google-ti-feeds/connector/src/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/__init__.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/aio_http_client.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/aio_http_client.py new file mode 100644 index 0000000000..1138f0af9f --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/aio_http_client.py @@ -0,0 +1,171 @@ +"""AioHttpClient class for making HTTP requests using aiohttp.""" + +import logging +from asyncio import TimeoutError +from typing import TYPE_CHECKING, Any, Dict, Optional + +from aiohttp import ( + ClientConnectorError, + ClientError, + ClientSession, + ClientTimeout, + ServerConnectionError, + ServerDisconnectedError, +) + +from .exceptions.api_error import ApiError +from .exceptions.api_http_error import ApiHttpError +from .exceptions.api_network_error import ApiNetworkError +from .exceptions.api_timeout_error import ApiTimeoutError +from .interfaces.base_http_client import BaseHttpClient + +if TYPE_CHECKING: + from logging import Logger + +LOG_PREFIX = "[API AioHttp]" + +NETWORK_ERROR_TYPES = ( + ClientConnectorError, + ServerDisconnectedError, + ServerConnectionError, + ConnectionError, + ConnectionRefusedError, + ConnectionAbortedError, + ConnectionResetError, +) + +NETWORK_ERROR_INDICATORS = [ + "network location cannot be reached", + "connection refused", + "cannot connect to host", + "eof", + "connection reset", + "connection timeout", + "network is unreachable", + "no route to host", + "socket.gaierror", + "dns lookup failed", +] + + +class AioHttpClient(BaseHttpClient): + """AioHttpClient class for making HTTP requests using aiohttp. + + This class provides an asynchronous HTTP client implementation using aiohttp. + It allows making HTTP requests with various parameters such as method, URL, headers, parameters, data, and JSON payload. + The client supports setting a default timeout for requests and handles exceptions like API errors, HTTP errors, and timeouts. + """ + + def __init__( + self, default_timeout: int = 60, logger: Optional["Logger"] = None + ) -> None: + """Initialize the AioHttpClient with a default timeout and an optional logger.""" + self.default_timeout = default_timeout + self._logger = logger or logging.getLogger(__name__) + + def _is_network_error(self, error: Exception) -> bool: + """Check if an exception represents a network connectivity issue. + + Args: + error: The exception to check + + Returns: + bool: True if the exception indicates a network issue, False otherwise + + """ + if isinstance(error, NETWORK_ERROR_TYPES): + return True + + error_message = str(error).lower() + return any(indicator in error_message for indicator in NETWORK_ERROR_INDICATORS) + + async def request( + self, + method: str, + url: str, + headers: Optional[Dict[str, str]] = None, + params: Optional[Dict[str, Any]] = None, + data: Optional[Dict[str, Any]] = None, + json_payload: Optional[Dict[str, Any]] = None, + timeout: Optional[int] = None, + ) -> Any: + """Make an asynchronous HTTP request using aiohttp. + + Args: + method (str): The HTTP method to use. + url (str): The URL to send the request to. + headers (Optional[Dict[str, str]], optional): The headers to include in the request. Defaults to None. + params (Optional[Dict[str, Any]], optional): The query parameters to include in the request. Defaults to None. + data (Optional[Dict[str, Any]], optional): The data to include in the request body. Defaults to None. + json_payload (Optional[Dict[str, Any]], optional): The JSON data to include in the request body. Defaults to None. + timeout (Optional[int], optional): The timeout in seconds for the request. Defaults to None. + + Returns: + Dict[str, Any]: The JSON response from the server. + + Raises: + ApiTimeoutError: If the request times out. + ApiHttpError: If the server returns an HTTP error. + ApiError: If an unexpected error occurs. + + """ + actual_timeout = ClientTimeout(total=timeout or self.default_timeout) + self._logger.debug( + f"{LOG_PREFIX} Making {method} request to {url} with timeout {actual_timeout.total}s. " + f"Params: {params is not None}, JSON: {json_payload is not None}" + ) + try: + async with ClientSession(timeout=actual_timeout) as session: + async with session.request( + method=method, + url=url, + headers=headers, + params=params, + data=data, + json=json_payload, + ) as response: + self._logger.debug( + f"{LOG_PREFIX} Received response with status {response.status} for {method} {url}" + ) + if response.status >= 400: + response_text = await response.text() + self._logger.error( # type: ignore[call-arg] + f"{LOG_PREFIX} HTTP Error {response.status} for {method} {url}: {response_text}", + meta={"error": response_text}, + ) + raise ApiHttpError(response.status, response_text) + return await response.json() + except TimeoutError as e: + self._logger.error( # type: ignore[call-arg] + f"{LOG_PREFIX} Request to {url} timed out after {actual_timeout.total}s: {e}", + meta={"error": str(e)}, + ) + raise ApiTimeoutError("Request timed out") from e + except ClientError as e: + if self._is_network_error(e): + self._logger.error( # type: ignore[call-arg] + f"{LOG_PREFIX} Network connectivity issue for {method} {url}: {str(e)}", + meta={"error": str(e), "is_network_error": True}, + ) + raise ApiNetworkError(f"Network connectivity issue: {str(e)}") from e + else: + self._logger.error( # type: ignore[call-arg] + f"{LOG_PREFIX} ClientError for {url}: {e}", + meta={"error": str(e)}, + ) + raise ApiHttpError(0, str(e)) from e + except ApiHttpError: + raise + except Exception as e: + if self._is_network_error(e): + self._logger.error( # type: ignore[call-arg] + f"{LOG_PREFIX} Network connectivity issue for {method} {url}: {str(e)}", + meta={"error": str(e), "is_network_error": True}, + ) + raise ApiNetworkError(f"Network connectivity issue: {str(e)}") from e + + self._logger.error( # type: ignore[call-arg] + f"{LOG_PREFIX} Unexpected error during request to {url}: {e}", + meta={"error": str(e)}, + ) + raise ApiError(f"Unexpected error: {str(e)}") from e diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/api_client.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/api_client.py new file mode 100644 index 0000000000..396089dbd9 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/api_client.py @@ -0,0 +1,114 @@ +"""API Client will orchestrate API calls.""" + +import logging +from typing import TYPE_CHECKING, Any, Dict, Optional, Type + +from pydantic import BaseModel + +from .api_request_model import ApiRequestModel +from .exceptions.api_error import ApiError +from .exceptions.api_http_error import ApiHttpError +from .exceptions.api_network_error import ApiNetworkError +from .exceptions.api_ratelimit_error import ApiRateLimitError +from .exceptions.api_timeout_error import ApiTimeoutError +from .exceptions.api_validation_error import ApiValidationError +from .interfaces.base_request_strategy import BaseRequestStrategy + +if TYPE_CHECKING: + from logging import Logger + +LOG_PREFIX = "[API Client]" + + +class ApiClient: + """Orchestrates API calls.""" + + def __init__( + self, strategy: BaseRequestStrategy, logger: Optional["Logger"] = None + ) -> None: + """Initialize the API client with a request strategy.""" + self.strategy = strategy + self._logger = logger or logging.getLogger(__name__) + + async def call_api( + self, + url: str, + method: str = "GET", + headers: Optional[Dict[str, Any]] = None, + params: Optional[Dict[str, Any]] = None, + data: Optional[Dict[str, Any]] = None, + json_payload: Optional[Dict[str, Any]] = None, + response_key: Optional[str] = None, + model: Optional[Type[BaseModel]] = None, + timeout: Optional[float] = None, + ) -> Any: + """Call the API using the provided strategy. + + Args: + url (str): The URL to call. + method (str): The HTTP method to use. + headers (Optional[Dict[str, Any]]): The headers to include in the request. + params (Optional[Dict[str, Any]]): The query parameters to include in the request. + data (Optional[Dict[str, Any]]): The data to include in the request. + json_payload (Optional[Dict[str, Any]]): The JSON data to include in the request. + response_key (Optional[str]): The key to extract from the response. + model (Optional[Type[BaseModel]]): The model to deserialize the response into. + timeout (Optional[float]): The timeout for the request. + + Returns: + Any: The response from the API. + + Raises: + ApiError: If the API call fails. + + """ + self._logger.debug( + f"{LOG_PREFIX} Preparing to call API: {method} {url} (Model: {model.__name__ if model else 'No'}, " + f"ResponseKey: {response_key}, Timeout: {timeout})" + ) + try: + api_request = ApiRequestModel( + url=url, + method=method, + headers=headers, + params=params, + data=data, + json_payload=json_payload, + response_key=response_key, + model=model, + timeout=timeout, + ) + response = await self.strategy.execute(api_request) + self._logger.debug(f"{LOG_PREFIX} API call to {method} {url} successful.") + return response + except ( + ApiTimeoutError, + ApiRateLimitError, + ApiHttpError, + ApiNetworkError, + ApiValidationError, + ) as known_api_err: + error_type = type(known_api_err).__name__ + error_prefix = ( + "Network connectivity issue" + if isinstance(known_api_err, ApiNetworkError) + else "Known API error" + ) + + self._logger.error( # type: ignore[call-arg] + f"{LOG_PREFIX} {error_prefix} during call_api for {method} {url}: {error_type} - {known_api_err}", + meta={"error": str(known_api_err), "error_type": error_type}, + ) + raise known_api_err + except ApiError as api_err: + self._logger.error( # type: ignore[call-arg] + f"{LOG_PREFIX} API error during call_api for {method} {url}: {api_err}", + meta={"error": str(api_err)}, + ) + raise api_err + except Exception as e: + self._logger.error( # type: ignore[call-arg] + f"{LOG_PREFIX} Unexpected failure in call_api for {method} {url}: {e}", + meta={"error": str(e)}, + ) + raise ApiError(f"Failed to call API {method} {url}: {e}") from e diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/api_request_model.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/api_request_model.py new file mode 100644 index 0000000000..5be22d68fc --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/api_request_model.py @@ -0,0 +1,21 @@ +"""API Request Model.""" + +from typing import Any, Dict, Optional, Type + +from pydantic import BaseModel + +from .interfaces.base_request_model import BaseRequestModel + + +class ApiRequestModel(BaseRequestModel): + """API Request Model.""" + + url: str + method: str = "GET" + headers: Optional[Dict[str, Any]] = None + params: Optional[Dict[str, Any]] = None + data: Optional[Dict[str, Any]] = None + json_payload: Optional[Dict[str, Any]] = None + response_key: Optional[str] = None + model: Optional[Type[BaseModel]] = None + timeout: Optional[int] = None diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/circuit_breaker.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/circuit_breaker.py new file mode 100644 index 0000000000..caea69032d --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/circuit_breaker.py @@ -0,0 +1,43 @@ +"""Simple CircuitBreaker.""" + +import time + +from .interfaces.base_circuit_breaker import BaseCircuitBreaker + + +class CircuitBreaker(BaseCircuitBreaker): + """Simple CircuitBreaker.""" + + def __init__(self, max_failures: int = 5, cooldown_time: int = 60) -> None: + """Initialize the CircuitBreaker with max_failures and cooldown_time.""" + self.max_failures = max_failures + self.cooldown_time = cooldown_time + self.failure_count = 0 + self.last_failure_time = 0.0 + + def is_open(self) -> bool: + """Check if the CircuitBreaker is open. + + Returns: + bool: True if the CircuitBreaker is open, False otherwise. + + """ + now = time.time() + if ( + self.failure_count >= self.max_failures + and now - self.last_failure_time < self.cooldown_time + ): + return True + if now - self.last_failure_time >= self.cooldown_time: + self.reset() + return False + + def record_failure(self) -> None: + """Record a failure and update the last failure time.""" + self.failure_count += 1 + self.last_failure_time = time.time() + + def reset(self) -> None: + """Reset the CircuitBreaker.""" + self.failure_count = 0 + self.last_failure_time = 0.0 diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/__init__.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_circuit_open_error.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_circuit_open_error.py new file mode 100644 index 0000000000..304f9128b9 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_circuit_open_error.py @@ -0,0 +1,9 @@ +"""Base class for circuit open errors.""" + +from .api_error import ApiError + + +class ApiCircuitOpenError(ApiError): + """Raised when the circuit is open.""" + + pass diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_error.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_error.py new file mode 100644 index 0000000000..6bae5745ba --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_error.py @@ -0,0 +1,7 @@ +"""Base class for API errors.""" + + +class ApiError(Exception): + """Base class for All API errors.""" + + pass diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_http_error.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_http_error.py new file mode 100644 index 0000000000..54ed0a6e44 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_http_error.py @@ -0,0 +1,12 @@ +"""Base class for HTTP API errors.""" + +from .api_error import ApiError + + +class ApiHttpError(ApiError): + """Base class for HTTP API errors in status code >= 400.""" + + def __init__(self, status_code: int, message: str): + """Initialize the error with status code and message.""" + self.status_code = status_code + super().__init__(f"HTTP {status_code}: {message}") diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_network_error.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_network_error.py new file mode 100644 index 0000000000..5bc8b253ef --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_network_error.py @@ -0,0 +1,20 @@ +"""API network error exception.""" + +from .api_error import ApiError + + +class ApiNetworkError(ApiError): + """Exception raised specifically for network connectivity issues. + + This class is used to differentiate network connectivity issues (like DNS resolution failures, + connection refused, etc.) from other types of API errors. + """ + + def __init__(self, message: str): + """Initialize the network error with a message. + + Args: + message (str): Descriptive message about the network error + + """ + super().__init__(f"Network connectivity error: {message}") diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_ratelimit_error.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_ratelimit_error.py new file mode 100644 index 0000000000..ebb6f9bac4 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_ratelimit_error.py @@ -0,0 +1,9 @@ +"""Base class for rate limit errors.""" + +from .api_error import ApiError + + +class ApiRateLimitError(ApiError): + """Raises when rate limit is exceeded.""" + + pass diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_timeout_error.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_timeout_error.py new file mode 100644 index 0000000000..256c0e1858 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_timeout_error.py @@ -0,0 +1,9 @@ +"""Base class for timeout errors.""" + +from .api_error import ApiError + + +class ApiTimeoutError(ApiError): + """Raises when request times out.""" + + pass diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_validation_error.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_validation_error.py new file mode 100644 index 0000000000..390de0ac28 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/exceptions/api_validation_error.py @@ -0,0 +1,9 @@ +"""Base class for validation errors.""" + +from .api_error import ApiError + + +class ApiValidationError(ApiError): + """Raised when the response cannot be validated or parsed.""" + + pass diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/__init__.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_circuit_breaker.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_circuit_breaker.py new file mode 100644 index 0000000000..339edf1c19 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_circuit_breaker.py @@ -0,0 +1,30 @@ +"""Base circuit breaker interface.""" + +from abc import ABC, abstractmethod + + +class BaseCircuitBreaker(ABC): + """Base circuit breaker interface. + + This interface defines the basic methods for a circuit breaker. + """ + + max_failures = 5 + cooldown_time = 60 + failure_count = 0 + last_failure_time = 0.0 + + @abstractmethod + def is_open(self) -> bool: + """Check if the circuit breaker is open.""" + raise NotImplementedError("Subclass must implement this method.") + + @abstractmethod + def record_failure(self) -> None: + """Record a failure attempt.""" + raise NotImplementedError("Subclass must implement this method.") + + @abstractmethod + def reset(self) -> None: + """Reset the circuit breaker.""" + raise NotImplementedError("Subclass must implement this method.") diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_http_client.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_http_client.py new file mode 100644 index 0000000000..1b7e13c513 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_http_client.py @@ -0,0 +1,42 @@ +"""BaseHttpClient Interfaces.""" + +from abc import ABC, abstractmethod +from typing import Any, Dict, Optional + + +class BaseHttpClient(ABC): + """BaseHttpClient Interfaces. + + This class defines the interface for a base HTTP client. + + It provides an abstract base class for implementing HTTP clients. + Subclasses must implement the `request` method to perform HTTP requests. + """ + + @abstractmethod + async def request( + self, + method: str, + url: str, + headers: Optional[Dict[str, str]] = None, + params: Optional[Dict[str, Any]] = None, + data: Optional[Dict[str, Any]] = None, + json_payload: Optional[Dict[str, Any]] = None, + timeout: Optional[int] = None, + ) -> Dict[str, Any]: + """Perform an HTTP request and return the parsed JSON. + + Args: + method (str): The HTTP method to use. + url (str): The URL to send the request to. + headers (Optional[Dict[str, str]], optional): The headers to include in the request. Defaults to None. + params (Optional[Dict[str, Any]], optional): The query parameters to include in the request. Defaults to None. + data (Optional[Dict[str, Any]], optional): The data to include in the request body. Defaults to None. + json_payload (Optional[Dict[str, Any]], optional): The JSON data to include in the request body. Defaults to None. + timeout (Optional[int], optional): The timeout in seconds for the request. Defaults to None. + + Returns: + Dict[str, Any]: The JSON response from the server. + + """ + raise NotImplementedError("Subclass must implement this method.") diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_rate_limiter.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_rate_limiter.py new file mode 100644 index 0000000000..c510e7268e --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_rate_limiter.py @@ -0,0 +1,17 @@ +"""BaseRateLimiter Interfaces.""" + +from abc import ABC, abstractmethod + + +class BaseRateLimiter(ABC): + """BaseRateLimiter Interfaces + This class defines the interface for rate limiters. + It provides an abstract method for acquiring a token from the rate limiter. + """ + + @abstractmethod + async def acquire(self) -> None: + """Acquire a token from the rate limiter. + This method should be implemented by subclasses to acquire a token from the rate limiter. + """ + raise NotImplementedError("Subclasses must implement this method") diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_request_hook.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_request_hook.py new file mode 100644 index 0000000000..83651f73ce --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_request_hook.py @@ -0,0 +1,25 @@ +"""BaseRequestHook interface.""" + +from abc import ABC, abstractmethod +from typing import Any + +from .base_request_model import BaseRequestModel + + +class BaseRequestHook(ABC): + """BaseRequestHook interface. + This class defines the interface for request hooks. + + This class provides a base implementation for request hooks. + One before the request is sent and after the response is received. + """ + + @abstractmethod + async def before(self, request: BaseRequestModel) -> None: + """Call before the request is sent.""" + raise NotImplementedError("Subclasses must implement this method") + + @abstractmethod + async def after(self, request: BaseRequestModel, response: Any) -> None: + """Call after the response is received.""" + raise NotImplementedError("Subclasses must implement this method") diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_request_model.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_request_model.py new file mode 100644 index 0000000000..a8d2b297ab --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_request_model.py @@ -0,0 +1,11 @@ +"""Base request model for API engine.""" + +from abc import ABC + +from pydantic import BaseModel + + +class BaseRequestModel(ABC, BaseModel): + """Base request model for API engine.""" + + pass diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_request_strategy.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_request_strategy.py new file mode 100644 index 0000000000..dd69432e49 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/interfaces/base_request_strategy.py @@ -0,0 +1,15 @@ +"""RequestStrategy interface.""" + +from abc import ABC, abstractmethod +from typing import Any + +from .base_request_model import BaseRequestModel + + +class BaseRequestStrategy(ABC): + """Base class for request strategies.""" + + @abstractmethod + async def execute(self, request: BaseRequestModel) -> Any: + """Execute the request strategy.""" + raise NotImplementedError("Subclasses must implement this method") diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/rate_limiter.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/rate_limiter.py new file mode 100644 index 0000000000..2baeef5540 --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/rate_limiter.py @@ -0,0 +1,57 @@ +"""Rate limiter module.""" + +import asyncio +import time +from collections import deque +from typing import Dict + +from .interfaces.base_rate_limiter import BaseRateLimiter + + +class TokenBucketRateLimiter(BaseRateLimiter): + """Token bucket rate limiter implementation.""" + + def __init__(self, max_requests: int, period: int) -> None: + """Initialize the token bucket rate limiter.""" + self.max_requests = max_requests + self.period = period + self.timestamps: deque[float] = deque() + self.lock = asyncio.Lock() + + async def acquire(self) -> None: + """Acquire a token from the rate limiter.""" + async with self.lock: + now = time.time() + while self.timestamps and now - self.timestamps[0] > self.period: + self.timestamps.popleft() + + if len(self.timestamps) >= self.max_requests: + sleep = self.period - (now - self.timestamps[0]) + await asyncio.sleep(sleep) + + self.timestamps.append(time.time()) + + +class RateLimiterRegistry: + """Rate limiter registry implementation.""" + + _store: Dict[str, BaseRateLimiter] = {} + _lock = asyncio.Lock() + + @classmethod + async def get(cls, key: str, max_requests: int, period: int) -> BaseRateLimiter: + """Get a rate limiter from the registry. + + Args: + key: The key to use for the rate limiter. + max_requests: The maximum number of requests allowed. + period: The period in seconds for the rate limiter. + + Returns: + The rate limiter instance. + + """ + async with cls._lock: + if key not in cls._store: + cls._store[key] = TokenBucketRateLimiter(max_requests, period) + return cls._store[key] diff --git a/external-import/google-ti-feeds/connector/src/utils/api_engine/retry_request_strategy.py b/external-import/google-ti-feeds/connector/src/utils/api_engine/retry_request_strategy.py new file mode 100644 index 0000000000..e66fd1a6fc --- /dev/null +++ b/external-import/google-ti-feeds/connector/src/utils/api_engine/retry_request_strategy.py @@ -0,0 +1,412 @@ +"""Base retry request strategy with hooks handling.""" + +import asyncio +import logging +import time +from typing import TYPE_CHECKING, Any, Dict, Optional, Union + +from .api_request_model import ApiRequestModel +from .exceptions.api_circuit_open_error import ApiCircuitOpenError +from .exceptions.api_error import ApiError +from .exceptions.api_http_error import ApiHttpError +from .exceptions.api_network_error import ApiNetworkError +from .exceptions.api_ratelimit_error import ApiRateLimitError +from .exceptions.api_timeout_error import ApiTimeoutError +from .exceptions.api_validation_error import ApiValidationError +from .interfaces.base_circuit_breaker import BaseCircuitBreaker +from .interfaces.base_http_client import BaseHttpClient +from .interfaces.base_rate_limiter import BaseRateLimiter +from .interfaces.base_request_hook import BaseRequestHook +from .interfaces.base_request_model import BaseRequestModel +from .interfaces.base_request_strategy import BaseRequestStrategy +from .rate_limiter import RateLimiterRegistry + +if TYPE_CHECKING: + from logging import Logger + +LOG_PREFIX = "[API Retry Strategy]" + + +class RetryRequestStrategy(BaseRequestStrategy): + """Strategy that retries failed requests intelligently.""" + + def __init__( + self, + http: BaseHttpClient, + breaker: BaseCircuitBreaker, + limiter: Optional[Union[BaseRateLimiter, Dict[str, Any]]] = None, + hooks: Optional[list[BaseRequestHook]] = None, + max_retries: int = 5, + backoff: int = 2, + logger: Optional["Logger"] = None, + ) -> None: + """Initialize the retry request strategy. + + Args: + http: The HTTP client to use. + breaker: The circuit breaker to use. + limiter: The rate limiter to use or a dictionary of rate limiter configuration. + if a dictionary is provided, it will be used to initialize the rate limiter with the following keys: + - key: The rate limiter key. + - max_requests: The maximum number of requests allowed within the time window. + - period: The time window in seconds. + hooks: The request hooks to use. + max_retries: The maximum number of retries. + backoff: The backoff factor. + logger: The logger to use. + + Raises: + Valueor: If max_retries is less than 0. + ValueError: If backoff is less than 1. + + """ + self.http = http + self.breaker = breaker + self.hooks = hooks or [] + self.max_retries = max_retries + self.backoff = backoff + self._logger = logger or logging.getLogger(__name__) + + self._limiter_config = None + self.limiter = None + + if isinstance(limiter, dict): + self._limiter_config = limiter + else: + self.limiter = limiter + self._initialized = False + + async def _initialize(self) -> None: + """Initialize the retry strategy.""" + if self._initialized: + return + + if self._limiter_config and not self.limiter: + required_keys = ["key", "max_requests", "period"] + if not all(key in self._limiter_config for key in required_keys): + missing_keys = [ + key for key in required_keys if key not in self._limiter_config + ] + self._logger.warning( + f"{LOG_PREFIX} Missing required keys in limiter config: {missing_keys}" + ) + raise ValueError( + f"Missing required keys in limiter config: {missing_keys}" + ) + + self.limiter = await RateLimiterRegistry.get( + key=self._limiter_config["key"], + max_requests=self._limiter_config["max_requests"], + period=self._limiter_config["period"], + ) + self._logger.debug( + f"{LOG_PREFIX} Rate limiter initialized with key {self._limiter_config['key']}", + f"max_requests={self._limiter_config['max_requests']}, period={self._limiter_config['period']}", + ) + + self._initialized = True + + async def _perform_single_attempt(self, request: BaseRequestModel) -> Any: + """Perform a single request attempt and process response.""" + try: + await self._initialize() + + if self.limiter: + await self.limiter.acquire() + self._logger.debug( + f"{LOG_PREFIX} Rate limiter token acquired for {self.api_req.url}" + ) + + for hook in self.hooks: + await hook.before(self.api_req) + + self._logger.debug( + f"{LOG_PREFIX} Attempting {self.api_req.method} to {self.api_req.url} (Headers: {self.api_req.headers is not None}, " + f"Params: {self.api_req.params is not None}, JSON: {self.api_req.json_payload is not None})" + ) + + raw = await self.http.request( + self.api_req.method, + self.api_req.url, + headers=self.api_req.headers, + params=self.api_req.params, + json_payload=self.api_req.json_payload, + timeout=self.api_req.timeout, + ) + self._logger.debug( + f"{LOG_PREFIX} Raw response received from {self.api_req.url}" + ) + + for hook in self.hooks: + await hook.after(self.api_req, raw) + + data = ( + raw.get(self.api_req.response_key) if self.api_req.response_key else raw + ) + + if self.api_req.model: + try: + return self.api_req.model.model_validate(data) + except Exception as validation_err: + self._logger.error( # type: ignore[call-arg] + f"{LOG_PREFIX} Response validation failed: {validation_err}", + meta={"error": str(validation_err)}, + ) + raise ApiValidationError( + f"Response validation failed: {str(validation_err)}" + ) from validation_err + self._logger.debug( + f"{LOG_PREFIX} Successfully processed request to {self.api_req.url}" + ) + return data + except ( + ApiTimeoutError, + ApiRateLimitError, + ApiHttpError, + ApiValidationError, + ApiCircuitOpenError, + ) as known_api_err: + self._logger.warning( + f"{LOG_PREFIX} Known API error during attempt for {self.api_req.url}: {type(known_api_err).__name__} - {known_api_err}" + ) + raise known_api_err + except ApiError as other_api_err: + self._logger.warning( + f"{LOG_PREFIX} API error during attempt for {self.api_req.url}: {other_api_err}" + ) + raise other_api_err + except Exception as generic_err: + self._logger.error( # type: ignore[call-arg] + f"{LOG_PREFIX} Unexpected error during single attempt for {self.api_req.url}", + meta={"error": str(generic_err)}, + ) + raise ApiError( + f"{LOG_PREFIX} Unexpected error occurred during request processing: {str(generic_err)}" + ) from generic_err + + async def execute(self, request: BaseRequestModel) -> Any: + """Execute the request with retry and rate limiting. + + Args: + request: The request to execute. + + Returns: + The response from the request. + + Raises: + ApiCircuitOpenError: If the circuit breaker is open. + ApiRateLimitExceededError: If the rate limit is exceeded. + ApiNetworkError: If there's a persistent network connectivity issue. + ApiError: If the request fails. + + """ + await self._validate_request(request) + + current_backoff_delay = self.backoff + last_exception: Optional[Exception] = None + network_error_count = 0 + max_network_errors = min(self.max_retries, 3) + + for attempt in range(self.max_retries + 1): + try: + self._logger.debug( + f"{LOG_PREFIX} Executing request attempt {attempt + 1}/{self.max_retries + 1} for {self.api_req.url}" + ) + return await self._perform_single_attempt(request) + except ApiNetworkError as e: + last_exception = e + network_error_count += 1 + current_backoff_delay = await self._handle_network_error( + e, + attempt, + network_error_count, + max_network_errors, + current_backoff_delay, + ) + except (ApiTimeoutError, ApiRateLimitError, ApiHttpError) as e: + last_exception = e + network_error_count = 0 + await self._handle_api_error(e, attempt, current_backoff_delay) + current_backoff_delay *= 2 + except ApiCircuitOpenError: + self._logger.info( + f"{LOG_PREFIX} Circuit breaker open, waiting before retry..." + ) + await self._wait_for_circuit_to_close() + attempt -= 1 + except ApiError as e: + await self._handle_unrecoverable_error(e) + raise e + + return await self._handle_max_retries_exceeded(last_exception) + + async def _validate_request(self, request: BaseRequestModel) -> None: + """Validate the request and check circuit breaker status. + + Args: + request: The request to validate. + + Raises: + TypeError: If the request is not an ApiRequestModel. + ApiCircuitOpenError: If the circuit breaker is open. + + """ + if not isinstance(request, ApiRequestModel): + self._logger.error( + f"{LOG_PREFIX} RetryRequestStrategy only supports ApiRequestModel", # type: ignore[call-arg] + meta={"error": "RetryRequestStrategy only supports ApiRequestModel"}, + ) + raise TypeError("RetryRequestStrategy only supports ApiRequestModel") + + self.api_req: ApiRequestModel = request + + if self.breaker.is_open(): + self._logger.warning( + f"{LOG_PREFIX} Circuit breaker is open. Request to {self.api_req.url} blocked." + ) + await self._wait_for_circuit_to_close() + + async def _handle_network_error( + self, + error: ApiNetworkError, + attempt: int, + network_error_count: int, + max_network_errors: int, + current_backoff_delay: int, + ) -> int: + """Handle network errors with appropriate backoff and retries. + + Args: + error: The network error that occurred. + attempt: The current attempt number. + network_error_count: The count of consecutive network errors. + max_network_errors: The maximum allowed consecutive network errors. + current_backoff_delay: The current backoff delay. + + Returns: + The updated backoff delay. + + Raises: + ApiNetworkError: If max network errors are exceeded. + + """ + self._logger.warning( + f"{LOG_PREFIX} Network connectivity issue on attempt {attempt + 1}/{self.max_retries + 1} for {self.api_req.url}: {error}" + ) + self.breaker.record_failure() + + if network_error_count >= max_network_errors: + self._logger.error( # type: ignore[call-arg] + f"{LOG_PREFIX} Persistent network connectivity issues detected after {network_error_count} consecutive failures for {self.api_req.url}.", + meta={"error": str(error)}, + ) + raise ApiNetworkError( + f"Persistent network connectivity issues: {error}" + ) from error + + if attempt < self.max_retries: + backoff_time = current_backoff_delay * (1.5 ** (network_error_count - 1)) + self._logger.info( + f"{LOG_PREFIX} Network error detected. Backing off for {backoff_time:.2f} seconds before retry..." + ) + await asyncio.sleep(backoff_time) + return int(backoff_time) + + return current_backoff_delay + + async def _handle_api_error( + self, + error: Union[ApiTimeoutError, ApiRateLimitError, ApiHttpError], + attempt: int, + current_backoff_delay: int, + ) -> None: + """Handle API errors with appropriate logging and retries. + + Args: + error: The API error that occurred. + attempt: The current attempt number. + current_backoff_delay: The current backoff delay. + + Raises: + ApiHttpError: If the error is a non-retryable HTTP error. + ApiError: If the maximum number of retries is reached. + + """ + self._logger.warning( + f"{LOG_PREFIX} Attempt {attempt + 1}/{self.max_retries + 1} for {self.api_req.url} failed with {type(error).__name__}: {error}." + ) + if not (isinstance(error, ApiHttpError) and error.status_code == 404): + self.breaker.record_failure() + self._logger.error( # type: ignore[call-arg] + f"{LOG_PREFIX} Failure recorded for circuit breaker due to error on {self.api_req.url}.", + meta={"error": str(error)}, + ) + else: + self._logger.info( + f"{LOG_PREFIX} 404 error detected for {self.api_req.url} - Not counting toward circuit breaker failures." + ) + + if isinstance(error, ApiHttpError) and error.status_code < 500: + self._logger.error( # type: ignore[call-arg] + f"{LOG_PREFIX} Non-retryable HTTP error {error.status_code} for {self.api_req.url}. Not retrying.", + meta={"error": str(error)}, + ) + raise + + self._logger.info( + f"{LOG_PREFIX} Retrying request to {self.api_req.url} in {current_backoff_delay:.2f} seconds..." + ) + await asyncio.sleep(current_backoff_delay) + + async def _wait_for_circuit_to_close(self) -> None: + """Wait until the circuit breaker is closed. + + This method waits for the cooldown period and then resets the circuit breaker. + """ + if not self.breaker.is_open(): + return + + cooldown_time = self.breaker.cooldown_time + self._logger.info( + f"{LOG_PREFIX} Circuit breaker is open. Waiting {cooldown_time} seconds before retry..." + ) + await asyncio.sleep(cooldown_time) + + if hasattr(self.breaker, "last_failure_time"): + now = time.time() + if now - self.breaker.last_failure_time >= cooldown_time: + self._logger.info( + f"{LOG_PREFIX} Cooldown period completed, resetting circuit breaker." + ) + self.breaker.reset() + + async def _handle_unrecoverable_error(self, error: ApiError) -> None: + """Handle unrecoverable API errors. + + Args: + error: The unrecoverable API error. + + """ + self._logger.error( # type: ignore[call-arg] + f"{LOG_PREFIX} Unrecoverable API error for {self.api_req.url}: {type(error).__name__} - {error}", + meta={"error": str(error)}, + ) + + async def _handle_max_retries_exceeded( + self, last_exception: Optional[Exception] + ) -> None: + """Handle the case when maximum retries are exceeded. + + Args: + last_exception: The last exception that occurred. + + Raises: + Exception: Re-raises the last exception if there was one. + ApiError: If max retries are exceeded without a specific exception. + + """ + if last_exception: + raise last_exception + raise ApiError( + f"{LOG_PREFIX} Max retries ({self.max_retries}) exceeded for {self.api_req.url} without a successful response." + ) diff --git a/external-import/google-ti-feeds/docker-compose.yml b/external-import/google-ti-feeds/docker-compose.yml new file mode 100644 index 0000000000..7b4db77686 --- /dev/null +++ b/external-import/google-ti-feeds/docker-compose.yml @@ -0,0 +1,16 @@ +services: + connector-google-ti-feeds: + image: opencti/connector-google-ti-feeds:6.6.11 + environment: + - OPENCTI_URL= + - OPENCTI_TOKEN= + - CONNECTOR_ID= + - GTI_API_KEY= + restart: unless-stopped + networks: + - docker_default + +networks: + default: + external: true + name: docker_default diff --git a/external-import/google-ti-feeds/pyproject.toml b/external-import/google-ti-feeds/pyproject.toml new file mode 100644 index 0000000000..97f22c5b56 --- /dev/null +++ b/external-import/google-ti-feeds/pyproject.toml @@ -0,0 +1,144 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "GoogleTIFeeds" +description = "External Import connector to retrieve feeds from GoogleTI." +readme = "README.md" +dynamic = ["version"] +classifiers = [ + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] + +requires-python = ">=3.12, <3.13" + +dependencies = [ + "PyYAML >=6.0.2, < 7", + "aiohttp >=3.11.18, <4", + "pycti >=6.6.11, <7", + "pydantic >=2.11.4, <3", + "pydantic_settings >=2.9.1, <3", + "python-dotenv >=1.0.1, <2", + "stix2 >=3.0.1, <4", + "aiohttp >=3.11.13, <4", + "isodate >=0.7.2, <0.8", +] + +[project.optional-dependencies] +test = [ + "pytest >=8.1.1, <9", + "pytest-asyncio >=0.16, <1", + "pytest-order >=1.3.0, <2", +] +dev = [ + "black >=25.1, <26", # Code formatter + "isort >=6, <7", # Import sorter + "ruff >=0.7.2, <1", # linter + "mypy >=1.13.0, <2", # Type validator + "pip_audit >=2, <3", # Security checker + "pre-commit >=4.1.0, <5", # Git hooks + "pynvim >=0.5.2, <1", # Neovim python client + "flake8 >=7.1.1, <8", # Linter + "types-PyYAML", # stubs for untyped module +] +all = ["GoogleTIFeeds[test,dev]"] + +[project.scripts] +GoogleTIFeeds = "connector.__main__:main" + +[tool.setuptools.packages.find] +where = ["."] + + +[tool.pytest.ini_options] +testpaths = ["./tests"] +asyncio_default_fixture_loop_scope = "function" +asyncio_mode = "auto" + +[tool.isort] +profile = "black" +line_length = 88 + +[tool.black] +target-version = ['py312'] +line_length = 88 + +[tool.ruff] +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +target-version = "py312" + +[tool.ruff.lint] +# Never enforce `I001` (unsorted import). Already handle with isort +# Never enforce `E501` (line length violations). Already handle with black +# Never enforce `F821` (Undefined name `null`). incorrect issue with notebook +# Never enforce `D213` (Multi-line docstring summary should start at the second line) conflict with our docstring convention +# Never enforce `D211` (NoBlankLinesBeforeClass)` +# Never enforce `G004` (logging-f-string) Logging statement uses f-string +# Never enforce `TRY003`() Avoid specifying long messages outside the exception class not useful +# Never enforce `D104` (Missing docstring in public package) +# Never enforce `D407` (Missing dashed underline after section) +# Never enforce `D408` (Section underline should be in the line following the section’s name) +# Never enforce `D409` (Section underline should match the length of its name) +ignore = [ + "I001", + "D203", + "E501", + "F821", + "D205", + "D213", + "D211", + "G004", + "TRY003", + "D104", + "D407", + "D408", + "D409", +] +select = ["E", "F", "W", "D", "G", "T", "B", "C", "N", "I", "S"] + +[tool.mypy] +strict = true +exclude = [ + '^tests', + '^docs', + '^build', + '^dist', + '^venv', + '^site-packages', + '^__pypackages__', + '^.venv', +] +plugins = ["pydantic.mypy"] diff --git a/external-import/google-ti-feeds/pytest.ini b/external-import/google-ti-feeds/pytest.ini new file mode 100644 index 0000000000..76ad34c2d7 --- /dev/null +++ b/external-import/google-ti-feeds/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +asyncio_mode = strict +asyncio_default_fixture_loop_scope = function diff --git a/external-import/google-ti-feeds/requirements.txt b/external-import/google-ti-feeds/requirements.txt new file mode 100644 index 0000000000..961f63dbcf --- /dev/null +++ b/external-import/google-ti-feeds/requirements.txt @@ -0,0 +1 @@ +-e .[all] diff --git a/external-import/google-ti-feeds/tests/__init__.py b/external-import/google-ti-feeds/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/tests/conftest.py b/external-import/google-ti-feeds/tests/conftest.py new file mode 100644 index 0000000000..d5941948ee --- /dev/null +++ b/external-import/google-ti-feeds/tests/conftest.py @@ -0,0 +1,91 @@ +"""Conftest file for Pytest fixtures.""" + +import logging +import sys +import types +from typing import TYPE_CHECKING, Any +from unittest.mock import patch + +from pytest import fixture + +if TYPE_CHECKING: + from os import _Environ + + +@fixture(autouse=True) +def disable_dotenv() -> Any: + """Fixture to disable dotenv loading for tests.""" + fake_dotenv = types.ModuleType("dotenv") + fake_dotenv.load_dotenv = lambda *args, **kwargs: None + + sys.modules["dotenv"] = fake_dotenv + yield + sys.modules.pop("dotenv", None) + + +def mock_env_vars(os_environ: "_Environ[str]", wanted_env: dict[str, str]) -> Any: + """Fixture to mock environment variables dynamically and clean up after.""" + mock_env = patch.dict(os_environ, wanted_env) + mock_env.start() + + return mock_env + + +@fixture(autouse=True) +def disable_config_yml() -> Any: + """Fixture to disable config.yml for tests by stubbing yml_settings → {}.""" + + def fake_settings_customise_sources( + cls, + settings_cls, + init_settings, + env_settings, + dotenv_settings, + file_secret_settings, + ): + def yml_settings() -> dict: + return {} + + return (yml_settings, env_settings, dotenv_settings, file_secret_settings) + + patcher = patch( + "connector.src.octi.interfaces.base_config.BaseConfig.settings_customise_sources", + new=classmethod(fake_settings_customise_sources), + ) + patcher.start() + + yield patcher + + patcher.stop() + + +@fixture(autouse=True) +def mock_opencti_api_client() -> Any: + """Fixture to mock OpenCTI API calls and clean up after.""" + mock_api = patch("requests.Session.post") + mock_healthcheck = patch( + "pycti.api.opencti_api_client.OpenCTIApiClient.health_check" + ) + mock_query = patch("pycti.api.opencti_api_client.OpenCTIApiClient.query") + + mock_api.start() + mock_healthcheck.start() + mock_query.start() + + yield mock_api, mock_healthcheck, mock_query + + mock_api.stop() + mock_healthcheck.stop() + mock_query.stop() + + +@fixture(autouse=True) +def patch_logger_error(monkeypatch): + """Patch to drop the meta for testing purposes.""" + orig_error = logging.Logger.error + + def fake_error(self, msg, *args, **kwargs): + kwargs.pop("meta", None) + return orig_error(self, msg, *args, **kwargs) + + monkeypatch.setattr(logging.Logger, "error", fake_error) diff --git a/external-import/google-ti-feeds/tests/custom/__init__.py b/external-import/google-ti-feeds/tests/custom/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/tests/custom/api_engine/test_api_engine.py b/external-import/google-ti-feeds/tests/custom/api_engine/test_api_engine.py new file mode 100644 index 0000000000..36bfbac650 --- /dev/null +++ b/external-import/google-ti-feeds/tests/custom/api_engine/test_api_engine.py @@ -0,0 +1,368 @@ +"""Module to test the API engine components.""" + +from typing import Any +from unittest.mock import AsyncMock + +import pytest +import pytest_asyncio +from connector.src.utils.api_engine.aio_http_client import AioHttpClient +from connector.src.utils.api_engine.api_client import ApiClient +from connector.src.utils.api_engine.circuit_breaker import CircuitBreaker +from connector.src.utils.api_engine.exceptions.api_error import ApiError +from connector.src.utils.api_engine.exceptions.api_timeout_error import ApiTimeoutError +from connector.src.utils.api_engine.rate_limiter import RateLimiterRegistry +from connector.src.utils.api_engine.retry_request_strategy import RetryRequestStrategy +from pydantic import BaseModel + +# ===================== +# Fixtures +# ===================== + + +@pytest.fixture +def mock_aiohttp_client() -> AsyncMock: + """Fixture for a mocked AioHttpClient that proxies .request → .get/.post.""" + mock = AsyncMock(spec=AioHttpClient) + + mock.get = AsyncMock(return_value={"success": True, "data": {"key": "value"}}) + mock.post = AsyncMock(return_value={"success": True, "id": "123"}) + + async def _request( + method, + url, + headers=None, + params=None, + data=None, + json_payload=None, + ssl=None, + timeout=None, + ): + m = method.upper() + if m == "GET": + return await mock.get( + url, + params=params, + headers=headers, + data=data, + json_payload=json_payload, + ssl=ssl, + timeout=timeout, + ) + elif m == "POST": + return await mock.post( + url, + params=params, + headers=headers, + data=data, + json_payload=json_payload, + ssl=ssl, + timeout=timeout, + ) + else: + raise NotImplementedError(f"Test fixture doesn’t handle HTTP {method!r}") + + mock.request = AsyncMock(side_effect=_request) + return mock + + +@pytest.fixture +def circuit_breaker() -> CircuitBreaker: + """Fixture for a CircuitBreaker.""" + return CircuitBreaker(max_failures=3, cooldown_time=10) + + +@pytest_asyncio.fixture +async def rate_limiter(): + """Fixture for a RateLimiter.""" + return await RateLimiterRegistry.get("test_api", max_requests=5, period=1) + + +@pytest.fixture +def retry_strategy( + mock_aiohttp_client: AsyncMock, circuit_breaker: CircuitBreaker, rate_limiter +) -> RetryRequestStrategy: + """Fixture for a RetryRequestStrategy.""" + return RetryRequestStrategy( + http=mock_aiohttp_client, + breaker=circuit_breaker, + limiter=rate_limiter, + max_retries=2, + backoff=2, + ) + + +@pytest.fixture +def api_client(retry_strategy: RetryRequestStrategy) -> ApiClient: + """Fixture for a ApiClient.""" + return ApiClient(retry_strategy) + + +class SimpleModel(BaseModel): + """SimpleModel is a Pydantic model for testing.""" + + key: str + + +class GitHubInfo(BaseModel): + """GitHubInfo is a Pydantic model for testing GitHub API responses.""" + + current_user_url: str + user_url: str + emails_url: str + + +# ===================== +# Test Data Fixtures (if needed) +# ===================== + + +@pytest.fixture( + params=[ + { + "url": "https://api.test.com/data", + "response_data": {"key": "test_value"}, + "model": SimpleModel, + }, + { + "url": "https://api.github.com", + "response_data": { + "current_user_url": "url1", + "user_url": "url2", + "emails_url": "url3", + }, + "model": GitHubInfo, + }, + ] +) +def successful_get_scenario(request): + """Fixture for successful GET request scenarios.""" + return request.param + + +@pytest.fixture( + params=[ + { + "url": "https://api.test.com/notfound", + "status_code": 404, + "error_message": "Not Found", + }, + ] +) +def failed_get_scenario(request): + """Fixture for failed GET request scenarios.""" + return request.param + + +# ===================== +# Test Cases +# ===================== + + +@pytest.mark.asyncio +async def test_api_client_successful_get_no_model( + api_client: ApiClient, + mock_aiohttp_client: AsyncMock, +): + """Test successful GET request without a Pydantic model.""" + # Given + url = "https://api.test.com/data" + expected_response_data = {"key": "value", "another_key": "another_value"} + _given_mock_response(mock_aiohttp_client, method="get", data=expected_response_data) + + # When + response, exception = await _when_api_get_called(api_client, url) + + # Then + _then_response_is_successful(response, expected_response_data) + mock_aiohttp_client.request.assert_awaited_once_with( + "GET", url, headers=None, params=None, json_payload=None, timeout=None + ) + + +@pytest.mark.asyncio +async def test_api_client_successful_get_with_model( + api_client: ApiClient, + mock_aiohttp_client: AsyncMock, + successful_get_scenario, +): + """Test successful GET request with a Pydantic model.""" + # Given + url = successful_get_scenario["url"] + response_data = successful_get_scenario["response_data"] + model = successful_get_scenario["model"] + _given_mock_response(mock_aiohttp_client, method="get", data=response_data) + + # When + response, exception = await _when_api_get_called(api_client, url, model=model) + + # Then + _then_response_is_successful(response, response_data, model_type=model) + mock_aiohttp_client.request.assert_awaited_once_with( + "GET", url, headers=None, params=None, json_payload=None, timeout=None + ) + + +@pytest.mark.asyncio +async def test_api_client_get_http_error( + api_client: ApiClient, + mock_aiohttp_client: AsyncMock, + failed_get_scenario, +): + """Test GET request that results in an HTTP error (e.g., 404, 500).""" + # Given + url = failed_get_scenario["url"] + status_code = failed_get_scenario["status_code"] + underlying_exception = Exception(f"Simulated HTTP {status_code} error") + _given_mock_response( + mock_aiohttp_client, method="get", raise_exception=underlying_exception + ) + + # When + response, exception = await _when_api_get_called(api_client, url) + + # Then + expected_message = f"simulated http {status_code} error" + _then_api_exception_is_raised(exception, expected_message_part=expected_message) + + +@pytest.mark.asyncio +async def test_retry_strategy_exhausts_retries( + api_client: ApiClient, mock_aiohttp_client: AsyncMock +): + """Test that the retry strategy gives up after exhausting retries.""" + # Given + url = "https://api.test.com/persistent-error" + persistent_error = ApiTimeoutError("Simulated persistent error") + mock_aiohttp_client.get.side_effect = persistent_error + max_retries = api_client.strategy.max_retries + + # When + response, exception = await _when_api_get_called(api_client, url) + + # Then + _then_api_exception_is_raised( + exception, expected_message_part="simulated persistent error" + ) + assert mock_aiohttp_client.request.await_count == max_retries + 1 # noqa: S101 + + +@pytest.mark.asyncio +async def test_circuit_breaker_opens_after_failures( + mock_aiohttp_client: AsyncMock, + circuit_breaker: CircuitBreaker, + rate_limiter, +): + """Test that the circuit breaker opens after consecutive failures.""" + # Given + strategy = RetryRequestStrategy( + http=mock_aiohttp_client, + breaker=circuit_breaker, + limiter=rate_limiter, + max_retries=0, + ) + client = ApiClient(strategy) + + url = "https://api.test.com/break-me" + failure_exception = ApiTimeoutError("Failure to trigger breaker") + mock_aiohttp_client.get.side_effect = failure_exception + + # When & Then + for _ in range(circuit_breaker.max_failures): + try: + await client.call_api(url) + except ApiError: + pass + + _then_circuit_breaker_is_open(circuit_breaker) + + with pytest.raises(ApiError): + await client.call_api(url) + + assert ( # noqa: S101 + mock_aiohttp_client.get.await_count == circuit_breaker.max_failures + 1 + ) + + +# ===================== +# GWT Gherkin-style functions +# ===================== + + +# --- GIVEN --- +def _given_mock_response( + mock_client: AsyncMock, + method: str = "get", + data: dict | None = None, + status: int = 200, + raise_exception: Exception | None = None, +): + """Set up the mock client's specified method to return data or raise an exception.""" + if raise_exception: + getattr(mock_client, method).side_effect = raise_exception + else: + mock_response = AsyncMock() + mock_response.json = AsyncMock(return_value=data) + mock_response.status = status + if status >= 200 and status < 300: + getattr(mock_client, method).return_value = data + else: + http_error = Exception(f"HTTP Error {status}") + if hasattr(getattr(mock_client, method), "side_effect"): + getattr(mock_client, method).side_effect = http_error + else: + mock_actual_client_response = AsyncMock() + mock_actual_client_response.status = status + mock_actual_client_response.json = AsyncMock(return_value=data) + pass + + +# --- WHEN --- +async def _when_api_get_called( + client: ApiClient, url: str, model: BaseModel | None = None +): + """Call the get method of the ApiClient.""" + try: + return await client.call_api(url, model=model), None + except ApiError as e: + return None, e + except Exception as e: + return None, e + + +# --- THEN --- +def _then_response_is_successful( + response: Any, expected_data: dict, model_type: BaseModel | None = None +): + """Assert that the API response is successful and matches expected data.""" + assert response is not None # noqa: S101 + if model_type: + assert isinstance(response, model_type) # noqa: S101 + assert response.model_dump() == expected_data # noqa: S101 + else: + assert response == expected_data # noqa: S101 + + +def _then_api_exception_is_raised( + exception: ApiError | None, + expected_message_part: str | None = None, + expected_status_code: int | None = None, +): + """Assert that an ApiError was raised.""" + assert exception is not None # noqa: S101 + assert isinstance(exception, ApiError) # noqa: S101 + if expected_message_part: + assert expected_message_part in str(exception).lower() # noqa: S101 + if expected_status_code and hasattr(exception, "status_code"): + assert exception.status_code == expected_status_code # noqa: S101 + + +def _then_circuit_breaker_is_open(breaker: CircuitBreaker): + """Assert that the circuit breaker is open.""" + assert breaker.is_open() is True # noqa: S101 + + +def _then_rate_limiter_called( + limiter_mock: AsyncMock, +): + """Assert that the rate limiter was called.""" + limiter_mock.acquire.assert_called() diff --git a/external-import/google-ti-feeds/tests/custom/configs/test_gti_config.py b/external-import/google-ti-feeds/tests/custom/configs/test_gti_config.py new file mode 100644 index 0000000000..91aed305a5 --- /dev/null +++ b/external-import/google-ti-feeds/tests/custom/configs/test_gti_config.py @@ -0,0 +1,323 @@ +"""Module to test the OpenCTI connector GTI configuration loading and instantiation.""" + +from os import environ as os_environ +from typing import Any, Dict +from unittest.mock import patch +from uuid import uuid4 + +import pytest +from connector.src.custom.configs.gti_config import GTIConfig +from connector.src.custom.exceptions.gti_configuration_error import ( + GTIConfigurationError, +) +from connector.src.octi.connector import Connector +from connector.src.octi.exceptions.configuration_error import ConfigurationError +from connector.src.octi.global_config import GlobalConfig +from pycti import OpenCTIConnectorHelper # type: ignore +from tests.conftest import mock_env_vars + +# ===================== +# Fixtures +# ===================== + + +@pytest.fixture( + params=[ + { + "opencti_url": "http://localhost:8080", + "opencti_token": f"{uuid4()}", + "connector_id": f"{uuid4()}", + "gti_api_key": f"{uuid4()}", + }, + ] +) +def min_required_config(request) -> dict[str, str]: # type: ignore + """Fixture for minimum required configuration.""" + return { + "OPENCTI_URL": request.param["opencti_url"], + "OPENCTI_TOKEN": request.param["opencti_token"], + "CONNECTOR_ID": request.param["connector_id"], + "GTI_API_KEY": request.param["gti_api_key"], + } + + +@pytest.fixture( + params=[ + { + "gti_import_start_date": "P3D", + "gti_api_url": "https://api.gti.com", + "gti_import_reports": "False", + "gti_report_types": "Actor Profile", + "gti_origins": "google threat intelligence", + }, + { + "gti_import_start_date": "P20D", + "gti_api_url": "https://api2.gti.com", + "gti_import_reports": "True", + "gti_report_types": "Patch Report,TTP Deep Dive", + "gti_origins": "google threat intelligence,partner", + }, + ] +) +def all_optional_config(request) -> dict[str, str]: + """Fixture for all optional configuration.""" + return { + "GTI_IMPORT_START_DATE": request.param["gti_import_start_date"], + "GTI_API_URL": request.param["gti_api_url"], + "GTI_IMPORT_REPORTS": request.param["gti_import_reports"], + "GTI_REPORT_TYPES": request.param["gti_report_types"], + "GTI_ORIGINS": request.param["gti_origins"], + } + + +@pytest.fixture( + params=[ + {"gti_import_start_date": "P1D"}, + {"gti_api_url": "https://www.virustotal.com/api/v3"}, + {"gti_import_reports": "True"}, + {"gti_report_types": "All"}, + {"gti_origins": "All"}, + ] +) +def all_defaulted_config(request) -> dict[str, str]: + """Fixture for all defaulted configuration.""" + opt = request.param + key, value = next(iter(opt.items())) + return {key.upper(): value} + + +@pytest.fixture( + params=[ + {"gti_report_types": "All"}, + {"gti_report_types": "Actor Profile"}, + {"gti_report_types": "Country Profile"}, + {"gti_report_types": "Cyber Physical Security Roundup"}, + {"gti_report_types": "Event Coverage/Implication"}, + {"gti_report_types": "Industry Reporting"}, + {"gti_report_types": "Malware Profile"}, + {"gti_report_types": "Net Assessment"}, + {"gti_report_types": "Network Activity Reports"}, + {"gti_report_types": "News Analysis"}, + {"gti_report_types": "OSINT Article"}, + {"gti_report_types": "Patch Report"}, + {"gti_report_types": "Strategic Perspective"}, + {"gti_report_types": "TTP Deep Dive"}, + {"gti_report_types": "Threat Activity Alert"}, + {"gti_report_types": "Actor Profile,Country Profile"}, + ] +) +def valid_gti_report_types(request) -> dict[str, str]: + """Fixture for valid GTI report types.""" + return {"GTI_REPORT_TYPES": request.param["gti_report_types"]} + + +@pytest.fixture( + params=[ + {"gti_report_types": "invalid report type"}, + {"gti_report_types": "Actor Profile,Invalid Report Type"}, + {"gti_report_types": "Country Profile,Invalid Report Type"}, + {"gti_report_types": "Cyber Physical Security Roundup,Invalid Report Type"}, + ] +) +def invalid_gti_report_types(request) -> dict[str, str]: + """Fixture for invalid GTI report types.""" + return {"GTI_REPORT_TYPES": request.param["gti_report_types"]} + + +@pytest.fixture( + params=[ + {"gti_origins": "All"}, + {"gti_origins": "google threat intelligence"}, + {"gti_origins": "partner"}, + {"gti_origins": "crowdsourced"}, + {"gti_origins": "google threat intelligence,partner"}, + ] +) +def valid_gti_origins(request) -> dict[str, str]: + """Fixture for valid GTI origin.""" + return {"GTI_ORIGINS": request.param["gti_origins"]} + + +@pytest.fixture( + params=[ + {"gti_origins": "invalid origin"}, + {"gti_origins": "google threat intelligence,partner,other"}, + ] +) +def invalid_gti_origins(request) -> dict[str, str]: + """Fixture for invalid GTI origin.""" + return {"GTI_ORIGINS": request.param["gti_origins"]} + + +# ===================== +# Test Cases +# ===================== + + +# Scenario: Create a connector with minimum required configuration for GTI +def test_gti_connector_min_required_config( # type: ignore + capfd, min_required_config: Dict[str, str] +) -> None: + """Test GTI connector with minimum required configuration.""" + # Given a minimum required configuration for GTI + mock_env = _given_setup_env_vars(min_required_config) + # When the connector is created + connector, _ = _when_connector_created() + # Then the connector should be created successfully + _then_connector_created_successfully( + capfd, mock_env, connector, min_required_config + ) + + +# Scenario: Create a connector with all optional configuration for GTI +def test_gti_connector_all_optional_config( # type: ignore + capfd, min_required_config: Dict[str, str], all_optional_config: Dict[str, str] +) -> None: + """Test GTI connector with all optional configuration.""" + data = {**min_required_config, **all_optional_config} + # Given a minimum required configuration for GTI and all optional configuration + mock_env = _given_setup_env_vars(data) + # When the connector is created + connector, _ = _when_connector_created() + # Then the connector should be created successfully + _then_connector_created_successfully(capfd, mock_env, connector, data) + + +# Scenario: Ensure that all defaulted configuration values are set correctly +def test_gti_connector_all_defaulted_config( # type: ignore + capfd, min_required_config: Dict[str, str], all_defaulted_config: Dict[str, str] +) -> None: + """Test GTI connector with all defaulted configuration.""" + # Given a minimum required configuration for GTI and all defaulted configuration + mock_env = _given_setup_env_vars(min_required_config) + # When the connector is created + connector, _ = _when_connector_created() + # Then the connector should be created successfully and all defaulted values should be set correctly + data = {**min_required_config, **all_defaulted_config} + _then_connector_created_successfully(capfd, mock_env, connector, data) + + +# Scenario: Create a connector with valid GTI report types +def test_gti_connector_valid_gti_report_types( # type: ignore + capfd, min_required_config: Dict[str, str], valid_gti_report_types: Dict[str, str] +) -> None: + """Test GTI connector with valid GTI report types.""" + # Given a minimum required configuration for GTI and valid GTI report types + data = {**min_required_config, **valid_gti_report_types} + mock_env = _given_setup_env_vars(data) + # When the connector is created + connector, _ = _when_connector_created() + # Then the connector should be created successfully + _then_connector_created_successfully(capfd, mock_env, connector, data) + + +# Scenario: Create a connector with invalid GTI report types +def test_gti_connector_invalid_gti_report_types( # type: ignore + min_required_config: Dict[str, str], invalid_gti_report_types: Dict[str, str] +) -> None: + """Test GTI connector with invalid GTI report types.""" + # Given a minimum required configuration for GTI and invalid GTI report types + data = {**min_required_config, **invalid_gti_report_types} + mock_env = _given_setup_env_vars(data) + # When the connector is created + connector, config_ex = _when_connector_created() + # Then the connector should raise a ConfigurationError + _then_connector_configuration_exception(mock_env, connector, config_ex) + + +# Scenario: Create a connector with valid GTI origins +def test_gti_connector_valid_gti_origins( # type: ignore + capfd, min_required_config: Dict[str, str], valid_gti_origins: Dict[str, str] +) -> None: + """Test GTI connector with valid GTI origins.""" + # Given a minimum required configuration for GTI and valid GTI origins + data = {**min_required_config, **valid_gti_origins} + mock_env = _given_setup_env_vars(data) + # When the connector is created + connector, _ = _when_connector_created() + # Then the connector should be created successfully + _then_connector_created_successfully(capfd, mock_env, connector, data) + + +# Scenario: Create a connector with invalid GTI origins +def test_gti_connector_invalid_gti_origins( # type: ignore + min_required_config: Dict[str, str], invalid_gti_origins: Dict[str, str] +) -> None: + """Test GTI connector with invalid GTI origins.""" + # Given a minimum required configuration for GTI and invalid GTI origins + data = {**min_required_config, **invalid_gti_origins} + mock_env = _given_setup_env_vars(data) + # When the connector is created + connector, config_ex = _when_connector_created() + # Then the connector should raise a ConfigurationError + _then_connector_configuration_exception(mock_env, connector, config_ex) + + +# ===================== +# GWT Gherkin-style functions +# ===================== + + +# Given setup environment variables +def _given_setup_env_vars(data: dict[str, str]) -> Any: + """Set up the environment variables for the test.""" + mock_env = mock_env_vars(os_environ, data) + return mock_env + + +# When the connector is created +def _when_connector_created() -> tuple[Any, Any]: + """Create the connector.""" + try: + global_config = GlobalConfig() + global_config.add_config_class(GTIConfig) + except (ConfigurationError, GTIConfigurationError) as config_ex: + return None, config_ex + + octi_helper = OpenCTIConnectorHelper(config=global_config.to_dict()) + connector = Connector(global_config, octi_helper) + + with patch("pycti.OpenCTIConnectorHelper.schedule_iso"): + connector.run() + + return connector, None + + +# Then the connector should be created successfully +def _then_connector_created_successfully(capfd, mock_env, connector, data) -> None: # type: ignore + """Check if the connector was created successfully.""" + assert connector is not None # noqa: S101 + + for key, value in data.items(): + if key.startswith("OPENCTI_"): + config_key = key[len("OPENCTI_") :].lower() + assert ( # noqa: S101 + getattr(connector._config.octi_config, config_key) + ) == value + elif key.startswith("GTI_"): + config_key = key[len("GTI_") :].lower() + gti_config = connector._config.get_config_class(GTIConfig) + val = getattr(gti_config, config_key) + if type(val) is list: + val = ",".join(val) + assert str(val) == value # noqa: S101 + + log_records = capfd.readouterr() + if connector._config.connector_config.log_level in ["info", "debug"]: + registered_message = f'"name": "{connector._config.connector_config.name}", "message": "Connector registered with ID", "attributes": {{"id": "{connector._config.connector_config.id}"}}' + assert registered_message in log_records.err # noqa: S101 + + mock_env.stop() + + +# Then the connector config should raise a custom ConfigurationException +def _then_connector_configuration_exception( # type: ignore + mock_env, connector, config_ex +) -> None: + """Check if the connector config raises a custom ConfigurationException.""" + assert connector is None # noqa: S101 + assert isinstance(config_ex, ConfigurationError) or isinstance( # noqa: S101 + config_ex, GTIConfigurationError + ) + + mock_env.stop() diff --git a/external-import/google-ti-feeds/tests/octi/__init__.py b/external-import/google-ti-feeds/tests/octi/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/external-import/google-ti-feeds/tests/octi/configs/test_connector_config.py b/external-import/google-ti-feeds/tests/octi/configs/test_connector_config.py new file mode 100644 index 0000000000..d5091a283d --- /dev/null +++ b/external-import/google-ti-feeds/tests/octi/configs/test_connector_config.py @@ -0,0 +1,296 @@ +"""Module to test the OpenCTI connector configuration loading and instantiation.""" + +from os import environ as os_environ +from typing import Any, Dict +from unittest.mock import patch +from uuid import uuid4 + +import pytest +from connector.src.octi.connector import Connector +from connector.src.octi.exceptions.configuration_error import ConfigurationError +from connector.src.octi.global_config import GlobalConfig +from pycti import OpenCTIConnectorHelper # type: ignore +from tests.conftest import mock_env_vars + +# ===================== +# Fixtures +# ===================== + + +@pytest.fixture( + params=[ + { + "opencti_url": "http://localhost:8080", + "opencti_token": f"{uuid4()}", + "connector_id": f"{uuid4()}", + } + ] +) +def min_required_config(request) -> dict[str, str]: # type: ignore + """Fixture for minimum required configuration.""" + return { + "OPENCTI_URL": request.param["opencti_url"], + "OPENCTI_TOKEN": request.param["opencti_token"], + "CONNECTOR_ID": request.param["connector_id"], + } + + +@pytest.fixture( + params=[ + { + "connector_duration_period": "PT1H", + "connector_log_level": "info", + "connector_name": "Connector Test", + "connector_scope": "identity", + "connector_queue_threshold": "500", + "connector_tlp_level": "AMBER+STRICT", + }, + { + "connector_duration_period": "PT5H", + "connector_log_level": "debug", + "connector_name": "Connector Test2", + "connector_scope": "vulnerability", + "connector_queue_threshold": "5000", + "connector_tlp_level": "WHITE", + }, + ] +) +def all_optional_config(request) -> dict[str, str]: # type: ignore + """Fixture for all optional configuration.""" + return { + "CONNECTOR_DURATION_PERIOD": request.param["connector_duration_period"], + "CONNECTOR_LOG_LEVEL": request.param["connector_log_level"], + "CONNECTOR_NAME": request.param["connector_name"], + "CONNECTOR_SCOPE": request.param["connector_scope"], + "CONNECTOR_QUEUE_THRESHOLD": request.param["connector_queue_threshold"], + "CONNECTOR_TLP_LEVEL": request.param["connector_tlp_level"], + } + + +@pytest.fixture( + params=[ + {"connector_duration_period": "PT2H"}, + {"connector_log_level": "info"}, + {"connector_name": "Google Threat Intel Feeds"}, + {"connector_scope": "report,location,identity"}, + {"connector_queue_threshold": "500"}, + {"connector_tlp_level": "AMBER+STRICT"}, + ] +) +def all_defaulted_config(request) -> dict[str, str]: # type: ignore + """Fixture for all defaulted configuration.""" + opt = request.param + key, value = next(iter(opt.items())) + return {key.upper(): value} + + +@pytest.fixture( + params=[ + {"log_level": "info"}, + {"log_level": "debug"}, + {"log_level": "error"}, + {"log_level": "warn"}, + ] +) +def valid_log_level_config(request) -> dict[str, str]: # type: ignore + """Fixture for valid log level configuration.""" + return {"CONNECTOR_LOG_LEVEL": request.param["log_level"]} + + +@pytest.fixture( + params=[ + {"log_level": "not_a_log_level"}, + {"log_level": "still_not_a_log_level"}, + ] +) +def invalid_log_level_config(request) -> dict[str, str]: # type: ignore + """Fixture for invalid log level configuration.""" + return {"CONNECTOR_LOG_LEVEL": request.param["log_level"]} + + +@pytest.fixture( + params=[ + {"connector_type": "EXTERNAL_IMPORT"}, + ] +) +def valid_connector_type_config(request) -> dict[str, str]: # type: ignore + """Fixture for valid connector type configuration.""" + return {"CONNECTOR_TYPE": request.param["connector_type"]} + + +@pytest.fixture( + params=[ + {"connector_type": "not_a_connector_type"}, + {"connector_type": "still_not_a_connector_type"}, + ] +) +def invalid_connector_type_config(request) -> dict[str, str]: # type: ignore + """Fixture for invalid connector type configuration.""" + return {"CONNECTOR_TYPE": request.param["connector_type"]} + + +# ===================== +# Test Cases +# ===================== + + +# Scenario: Create a connector with the minimum required configuration. +def test_connector_config_min_required( # type: ignore + capfd, min_required_config: Dict[str, str] +) -> None: + """Test for the connector with the minimum required configuration.""" + # Given a minimum required configuration are provided + mock_env = _given_setup_env_vars(min_required_config) + # When the connector is created + connector, _ = _when_connector_created() + # Then the connector should be created successfully + _then_connector_created_successfully( + capfd, mock_env, connector, min_required_config + ) + + +# Scenario: Create a connector with all optional configuration. +def test_connector_config_all_optional( # type: ignore + capfd, min_required_config, all_optional_config +) -> None: + """Test for the connector with all optional configuration.""" + data = {**min_required_config, **all_optional_config} + # Given a minimum required configuration and all optional configuration are provided + mock_env = _given_setup_env_vars(data) + # When the connector is created + connector, _ = _when_connector_created() + # Then the connector should be created successfully + _then_connector_created_successfully(capfd, mock_env, connector, data) + + +# Scenario: Ensure that all defaulted values are set correctly. +def test_connector_config_all_defaulted(capfd, min_required_config, all_defaulted_config) -> None: # type: ignore + """Test for the connector to check all of the defaulted values.""" + # Given a minimum required configuration + mock_env = _given_setup_env_vars(min_required_config) + # When the connector is created + connector, _ = _when_connector_created() + # Then the connector should be created successfully and optional values should be defaulted + data = {**min_required_config, **all_defaulted_config} + _then_connector_created_successfully(capfd, mock_env, connector, data) + + +# Scenario: Test for the connector with all valid log level values. +def test_connector_config_valid_log_level( # type: ignore + capfd, min_required_config, valid_log_level_config +) -> None: + """Test for the connector all valid log level values.""" + # Given a minimum required configuration and all valid log level configuration are provided. + data = {**min_required_config, **valid_log_level_config} + mock_env = _given_setup_env_vars(data) + # When the connector is created + connector, _ = _when_connector_created() + # Then the connector should be created successfully + _then_connector_created_successfully(capfd, mock_env, connector, data) + + +# Scenario: Test for the connector for invalid log level values. +def test_connector_config_invalid_log_level( # type: ignore + min_required_config, invalid_log_level_config +) -> None: + """Test for the connector for invalid log level values.""" + # Given a minimum required configuration and invalid log level configuration are provided. + data = {**min_required_config, **invalid_log_level_config} + mock_env = _given_setup_env_vars(data) + # When the connector is created + connector, config_ex = _when_connector_created() + # Then the connector config should raise a custom ConfigurationException + _then_connector_configuration_exception(mock_env, connector, config_ex) + + +# Scenario: Test for the connector with all valid connector type values. +def test_connector_config_valid_connector_type( # type: ignore + capfd, min_required_config, valid_connector_type_config +) -> None: + """Test for the connector for all valid connector type values.""" + # Given a minimum required configuration and all valid connector type configuration are provided. + data = {**min_required_config, **valid_connector_type_config} + mock_env = _given_setup_env_vars(data) + # When the connector is created + connector, _ = _when_connector_created() + # Then the connector should be created successfully + _then_connector_created_successfully(capfd, mock_env, connector, data) + + +# Scenario: Test for the connector for invalid connector type values. +def test_connector_config_invalid_connector_type( # type: ignore + min_required_config, invalid_connector_type_config +) -> None: + """Test for the connector for invalid connector type values.""" + # Given a minimum required configuration and invalid connector type configuration are provided. + data = {**min_required_config, **invalid_connector_type_config} + mock_env = _given_setup_env_vars(data) + # When the connector is created + connector, config_ex = _when_connector_created() + # Then the connector config should raise a custom ConfigurationException + _then_connector_configuration_exception(mock_env, connector, config_ex) + + +# ===================== +# GWT Gherkin-style functions +# ===================== + + +# Given setup environment variables +def _given_setup_env_vars(data: dict[str, str]) -> Any: + """Set up the environment variables for the test.""" + mock_env = mock_env_vars(os_environ, data) + return mock_env + + +# When the connector is created +def _when_connector_created() -> tuple[Any, Any]: + """Create the connector.""" + try: + global_config = GlobalConfig() + except ConfigurationError as config_ex: + return None, config_ex + + octi_helper = OpenCTIConnectorHelper(config=global_config.to_dict()) + connector = Connector(global_config, octi_helper) + + with patch("pycti.OpenCTIConnectorHelper.schedule_iso"): + connector.run() + + return connector, None + + +# Then the connector should be created successfully +def _then_connector_created_successfully(capfd, mock_env, connector, data) -> None: # type: ignore + """Check if the connector was created successfully.""" + assert connector is not None # noqa: S101 + + for key, value in data.items(): + if key.startswith("OPENCTI_"): + config_key = key[len("OPENCTI_") :].lower() + assert ( # noqa: S101 + getattr(connector._config.octi_config, config_key) + ) == value + elif key.startswith("CONNECTOR_"): + config_key = key[len("CONNECTOR_") :].lower() + assert ( # noqa: S101 + str(getattr(connector._config.connector_config, config_key)) == value + ) + + log_records = capfd.readouterr() + if connector._config.connector_config.log_level in ["info", "debug"]: + registered_message = f'"name": "{connector._config.connector_config.name}", "message": "Connector registered with ID", "attributes": {{"id": "{connector._config.connector_config.id}"}}' + assert registered_message in log_records.err # noqa: S101 + + mock_env.stop() + + +# Then the connector config should raise a custom ConfigurationException +def _then_connector_configuration_exception( # type: ignore + mock_env, connector, config_ex +) -> None: + """Check if the connector config raises a custom ConfigurationException.""" + assert connector is None # noqa: S101 + assert isinstance(config_ex, ConfigurationError) # noqa: S101 + + mock_env.stop() diff --git a/external-import/google-ti-feeds/tests/stix/__init__.py b/external-import/google-ti-feeds/tests/stix/__init__.py new file mode 100644 index 0000000000..e69de29bb2