Added profiles, refactored services

Author: krystianbajno

README.md

@@ -12,13 +12,14 @@ A powerful, modular, and extensible vulnerability assessment and vulnerability i
 - **Enrichment Capabilities**: Enhance CVE details with severity metrics, reference URLs, available exploits, and mitigations, enabling you to produce actionable intelligence.
 - **Caching and Optimization**: Uses smart caching, and reads straight from compressed archives to minimize API requests and optimize performance, enabling you to use it in air-gapped networks.
 - **Flexible Configuration**: Enable or disable providers and enrichment sources as needed.
+- **Profiles**: Create profiles, for example "offline" that uses only offline resources, "normal" that uses online resources, "stable" that does not update automatically, and more.
 - **CLI Simplicity**: Intuitive command-line interface for streamlined operations.
 - **Reports**: Create vulnerability reports with ease.
 
 # Use Case Scenarios
 
+- Look up vulnerabilities for a specified product, version, identifier, and more.
 - Research vulnerabilities and associated metadata.
-- Product vulnerability background checking.
 - Automate vulnerability triaging and reporting.
 - Gain insights for security monitoring and proactive threat mitigation.
 
@@ -28,13 +29,22 @@ A powerful, modular, and extensible vulnerability assessment and vulnerability i
 ```bash
 pip3 install -r requirements.txt
 
+On first run, the offline dataset will be downloaded automatically. The default profile is stable, you can change it in config.yaml. The stable profile does not auto-update when cache duration passes, so it is manual work to run --autoupdate or --reload. Each provider has different cache duration. In order to use online providers and update automatically, use "normal" profile.
+
+In order to use only local resources, use offline profile.
+
 python3 cveseeker.py <keywords>
 python3 cveseeker.py windows smbv1
 python3 cveseeker.py windows remote code execution
 python3 cveseeker.py cve-2024
+python3 cveseeker.py cve-2024 --reload # Override force re-download the dataset on next run
+python3 cveseeker.py cve-2024 --autoupdate # Override allow auto-updating the dataset on next run
+python3 cveseeker.py cve-2024 --no-autoupdate # Override do not allow updating the dataset on next run
+python3 cveseeker.py cve-2024 --offline # offline mode - do not update, do not use online providers on next run. same as --profile offline
+python3 cveseeker.py cve-2024 --profile [normal, stable, offline, debug, ...] # select a profile. modify profiles.yaml to add more. Profiles modify config.
 python3 cveseeker.py cve-2024 --max-per-provider 2000 # max results per provider, default 100
 python3 cveseeker.py cve-2024 --report # generate CSV, JSON and HTML report
-python3 cveseeker.py cve-2024 --critical --high --medium --low # include critical, high, medium, and low severities
+python3 cveseeker.py cve-2024 --critical --high --medium --low # include critical, high, medium, or low severities
 ```
 
 # Sources

config.yaml

@@ -1,3 +1,6 @@
+default_profile: stable
+cache_dir: "dataset"
+
 providers:
   NistAPI: false
   NistCachedAPI: true
@@ -17,4 +20,7 @@ enrichment:
     cisa_kev: true
     github_poc: false
     github_poc_cached: true
-    nist_cached: true
+    nist_cached: true
+
+reload: false
+autoupdate: true

cveseeker.py

@@ -1,16 +1,23 @@
 import argparse
+from services.config.config import configure_on_first_run, load_config, update_config
+from services.profile import profile_guard
+from services.profile.profile import load_profiles
 from providers.search_provider import SearchProvider
 from services.vulnerability_intelligence.reports.vulnerability_intelligence_report_service import VulnerabilityIntelligenceReportService
 from services.vulnerability_intelligence.printers.vulnerability_intelligence_printer import VulnerabilityIntelligencePrinter
 from terminal import logo
+from terminal.cli import print_configuration, print_wrong_profile
 
 def main():
+    config = load_config('config.yaml')
+    profiles = load_profiles("profiles.yaml")
+    
     parser = argparse.ArgumentParser(description="Search for vulnerabilities using keywords.")
 
     parser.add_argument(
         'keywords', 
         nargs='+',
-        help='List of keywords to search for (e.g., Windows RCE)'
+        help='List of keywords to search for (e.g., Microsoft Windows Remote Code Execution)'
     )
 
     parser.add_argument(
@@ -27,25 +34,78 @@ def main():
         help="Generate CSV report"
     )
 
-    parser.add_argument('--low', action='store_true', help='Include low severity vulnerabilities')
-    parser.add_argument('--medium', action='store_true', help='Include medium severity vulnerabilities')
-    parser.add_argument('--high', action='store_true', help='Include high severity vulnerabilities')
-    parser.add_argument('--critical', action='store_true', help='Include critical severity vulnerabilities')
+    parser.add_argument(
+        '--reload',
+        action="store_true",
+        default=False,
+        help="Override - Force reload, update, download the dataset."
+    )
+    
+    parser.add_argument(
+        '--no-autoupdate',
+        action="store_true",
+        default=False,
+        help="Override - Do not allow auto-updating the dataset on next run."
+    )
+    
+    parser.add_argument(
+        '--autoupdate',
+        action="store_true",
+        default=False,
+        help="Override - Allow auto-updating the dataset on next run."
+    )
 
     parser.add_argument(
-        '--playwright',
+        '--offline',
         action="store_true",
         default=False,
-        help="Utilize Playwright to use more sources (does nothing at the moment)"
+        help="Offline mode - do not update, do not use online providers. Same as --profile offline"
     )
 
+    parser.add_argument(
+        '--profile',
+        type=str,
+        help="Max results per provider",
+        default=config.get("default_profile")
+    )
+    
+    parser.add_argument('--low', action='store_true', help='Include low severity vulnerabilities')
+    parser.add_argument('--medium', action='store_true', help='Include medium severity vulnerabilities')
+    parser.add_argument('--high', action='store_true', help='Include high severity vulnerabilities')
+    parser.add_argument('--critical', action='store_true', help='Include critical severity vulnerabilities')
+    
     logo.print_logo()
 
     args = parser.parse_args()
 
     keywords = args.keywords
+
+    profilename = args.profile
+    
+    if args.offline:
+        profilename = "offline"
+        
+    profile = profiles.get(profilename)
 
-    search_provider = SearchProvider(playwright_enabled=args.playwright, config_file='config.yaml')
+    if not profile:
+        print_wrong_profile(profiles)
+        exit(1)
+        
+    profile_guard.enforce_profile(config, profile)
+    
+    if args.reload:
+        update_config(config, {"reload": True})
+    
+    if args.autoupdate:
+        update_config(config, {"autoupdate": True})
+        
+    if args.no_autoupdate:
+        update_config(config, {"autoupdate": False})
+        
+    configure_on_first_run(config)
+    print_configuration(profilename, config)
+    
+    search_provider = SearchProvider(config)
     search_service = search_provider.make_service_api()
 
     desired_severities = [
@@ -65,6 +125,5 @@ def main():
         VulnerabilityIntelligenceReportService.generate_json_report(results, filename_json)
         VulnerabilityIntelligenceReportService.generate_html_report(results, " ".join(keywords), filename_html)
 
-    
 if __name__ == "__main__":
     main()

profiles.yaml

@@ -0,0 +1,99 @@
+offline:
+  providers:
+    NistAPI: false
+    NistCachedAPI: true
+    PacketStormSecurityAPI: false
+    OpenCVEAPI: false
+    ExploitDBAPI: false
+    GitHubAdvisoryAPI: false
+    VulnersAPI: false
+    CISAKEVAPI: true
+    RAPID7: false
+
+  enrichment:
+      sources:
+        vulners: false
+        trickest_cve_github: false
+        trickest_cve_github_cached: true
+        cisa_kev: true
+        github_poc: false
+        github_poc_cached: true
+        nist_cached: true
+
+  autoupdate: false
+  reload: false
+
+normal:
+  providers:
+    NistAPI: false
+    NistCachedAPI: true
+    PacketStormSecurityAPI: true
+    OpenCVEAPI: true
+    ExploitDBAPI: true
+    GitHubAdvisoryAPI: true
+    VulnersAPI: false
+    CISAKEVAPI: true
+    RAPID7: true
+
+  enrichment:
+    sources:
+      vulners: true
+      trickest_cve_github: false
+      trickest_cve_github_cached: true
+      cisa_kev: true
+      github_poc: false
+      github_poc_cached: true
+      nist_cached: true
+
+  reload: false
+  autoupdate: true
+
+stable:
+  providers:
+    NistAPI: false
+    NistCachedAPI: true
+    PacketStormSecurityAPI: true
+    OpenCVEAPI: true
+    ExploitDBAPI: true
+    GitHubAdvisoryAPI: true
+    VulnersAPI: false
+    CISAKEVAPI: true
+    RAPID7: true
+
+  enrichment:
+    sources:
+      vulners: true
+      trickest_cve_github: false
+      trickest_cve_github_cached: true
+      cisa_kev: true
+      github_poc: false
+      github_poc_cached: true
+      nist_cached: true
+
+  reload: false
+  autoupdate: false
+
+debug:
+  providers:
+    NistAPI: false
+    NistCachedAPI: true
+    PacketStormSecurityAPI: false
+    OpenCVEAPI: false
+    ExploitDBAPI: false
+    GitHubAdvisoryAPI: false
+    VulnersAPI: false
+    CISAKEVAPI: false
+    RAPID7: false
+
+  enrichment:
+    sources:
+      vulners: false
+      trickest_cve_github: false
+      trickest_cve_github_cached: false
+      cisa_kev: false
+      github_poc: false
+      github_poc_cached: false
+      nist_cached: false
+
+  reload: false
+  autoupdate: false

providers/search_provider.py

@@ -10,18 +10,15 @@
 from services.api.sources.rapid7 import RAPID7
 from services.api.sources.vulners import VulnersAPI
 
-from typing import Dict
-
 from services.cache.cache_manager import CacheManager
 from services.search.engine.progress_factory import ProgressManagerFactory
 from services.search.search_manager import SearchManager
-from services.search.engine.progress_manager import ProgressManager
+from terminal.cli import print_greyed_out
 
 class SearchProvider:
-    def __init__(self, playwright_enabled=False, config_file='config.yaml'):
+    def __init__(self, config):
         self.search_service: SearchManager = None
-        self.playwright_enabled = playwright_enabled
-        self.config_file = config_file
+        self.config = config
 
         self.provider_registry = {
             'NistAPI': NistAPI,
@@ -41,7 +38,7 @@ def make_service_api(self) -> SearchManager:
         return self.search_service
 
     def boot(self):
-        config = self.load_config()
+        config = self.config
         providers_config = config.get('providers', {})
         enrichment_config = config.get("enrichment", {})
 
@@ -54,32 +51,16 @@ def boot(self):
                 provider_class = self.provider_registry.get(provider_name)
                 if provider_class:
                     if provider_name in [
-                            'NistCachedAPI', 
-                            'CISAKEVAPI'
-                        ]:
-                        providers.append(provider_class(cache_manager))
+                        'NistCachedAPI', 
+                        'CISAKEVAPI'
+                    ]:
+                        providers.append(provider_class(config, cache_manager))
                     else:
-                        providers.append(provider_class())
+                        providers.append(provider_class(config))
                 else:
                     print(f"[!] Provider '{provider_name}' not found in registry.")
             else:
-                print(f"[-] Provider '{provider_name}' is disabled in configuration.")
-
-        if self.playwright_enabled:
-            playwright_providers = []
-            providers.extend(playwright_providers)
+                print_greyed_out(f"[-] Provider '{provider_name}' is disabled in configuration.")
 
         progress_manager_factory = ProgressManagerFactory()
         self.search_service = SearchManager(providers, enrichment_config, progress_manager_factory=progress_manager_factory, cache_manager=cache_manager)
-        
-    def load_config(self) -> Dict:
-        try:
-            with open(self.config_file, 'r') as f:
-                config = yaml.safe_load(f)
-                return config
-        except FileNotFoundError:
-            print(f"[!] Config file '{self.config_file}' not found. Using default settings.")
-            return {}
-        except yaml.YAMLError as exc:
-            print(f"[!] Error parsing config file: {exc}")
-            return {}