Resolve merge conflicts

Author: Janson Bunce

backend/src/xfd_django/xfd_api/api_methods/cve.py

@@ -1,8 +1,15 @@
 """Cve API."""
+# Standard Python Libraries
+import datetime
+from typing import Optional
 
 # Third-Party Libraries
-from fastapi import HTTPException
-from xfd_mini_dl.models import Cve
+from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator
+from django.db.models import Q
+from fastapi import HTTPException, status
+from xfd_mini_dl.models import Cve as CveModel
+
+from ..auth import is_global_write_admin
 
 
 def get_cves_by_id(cve_id):
@@ -13,7 +20,7 @@ def get_cves_by_id(cve_id):
         object: a single Cve object.
     """
     try:
-        cve = Cve.objects.get(id=cve_id)
+        cve = CveModel.objects.get(id=cve_id)
         return cve
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
@@ -27,7 +34,56 @@ def get_cves_by_name(cve_name):
         object: a single Cpe object.
     """
     try:
-        cve = Cve.objects.get(name=cve_name)
+        cve = CveModel.objects.get(name=cve_name)
         return cve
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
+
+
+async def get_all_cves(
+    current_user,
+    *,
+    page: int = 1,
+    per_page: int = 100,
+    since_timestamp: Optional[datetime.datetime] = None,
+) -> tuple[int, list[CveModel]]:
+    """
+    Return (total_pages, list_of_CveModel) for the given filters.
+
+    Raise HTTPException(403) if the user is not an admin, or HTTPException(500) on DB errors.
+    """
+    if not is_global_write_admin(current_user):
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Unauthorized access.",
+        )
+
+    try:
+        # 1) base queryset
+        qs = CveModel.objects.all()
+
+        # 2) optional date filter
+        if since_timestamp is not None:
+            qs = qs.filter(Q(modified_at__gte=since_timestamp))
+
+        # 3) deterministic ordering
+        qs = qs.order_by("modified_at", "id")
+
+        # 4) paginate
+        paginator = Paginator(qs, per_page)
+        try:
+            page_obj = paginator.page(page)
+            objects = list(page_obj.object_list)
+        except PageNotAnInteger:
+            page_obj = paginator.page(1)
+            objects = list(page_obj.object_list)
+        except EmptyPage:
+            objects = []
+
+        return paginator.num_pages, objects
+
+    except Exception as e:
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"DB error: {e}",
+        )

backend/src/xfd_django/xfd_api/management/commands/sync_cves.py

@@ -0,0 +1,15 @@
+"""Get CVEs from FastAPI endpoint and sync to local DB."""
+# Third-Party Libraries
+from django.core.management.base import BaseCommand
+from xfd_api.tasks.nist_lz_sync import handler
+
+
+class Command(BaseCommand):
+    """Sync CVEs from FastAPI endpoint to local DB."""
+
+    help = "Fetch CVEs from the FastAPI endpoint and sync to local DB"
+
+    def handle(self, *args, **options):
+        """Handle the command."""
+        result = handler()
+        self.stdout.write(str(result))

backend/src/xfd_django/xfd_api/schema_models/cve.py

@@ -44,3 +44,10 @@ class Config:
         """Config."""
 
         from_attributes = True
+
+
+class GetAllCvesResponse(BaseModel):
+    """GetAllCvesResponse schema."""
+
+    status: str
+    payload: List[Cve]

backend/src/xfd_django/xfd_api/schema_models/scan.py

@@ -253,6 +253,14 @@ class GenericMessageResponseModel(BaseModel):
         global_scan=True,
         description="Update CVE data using the NIST API",
     ),
+    "nist_lz_sync": ScanSchema(
+        type="fargate",
+        is_passive=True,
+        global_scan=True,
+        cpu="1024",
+        memory="8192",
+        description="Pull in NIST cve data from commercial mdl",
+    ),
     "portscanner": ScanSchema(
         type="fargate",
         is_passive=False,
@@ -376,10 +384,11 @@ class GenericMessageResponseModel(BaseModel):
     "xpanse_alert_pull": ScanSchema(
         type="fargate",
         is_passive=True,
-        global_scan=True,
+        global_scan=False,
         cpu="1024",
         memory="8192",
         description="Pull in Xpanse alert data from Xpanse API",
+        max_concurrent_tasks=3,
     ),
     "xpanse_org_sync": ScanSchema(
         type="fargate",

backend/src/xfd_django/xfd_api/tasks/ecs_client.py

@@ -58,7 +58,7 @@ def run_command(self, command_options):
                     container = self.docker.containers.run(
                         "crossfeed-worker",
                         name=container_name,
-                        network_mode="xfd_backend",
+                        network_mode="backend",
                         mem_limit="4g",
                         environment={
                             "CROSSFEED_COMMAND_OPTIONS": json.dumps(command_options),
@@ -106,6 +106,7 @@ def run_command(self, command_options):
                             "SERVICE_QUEUE_URL": os.getenv("QUEUE_URL", ""),
                             "DMZ_SYNC_ENDPOINT": os.getenv("DMZ_SYNC_ENDPOINT", ""),
                             "DMZ_API_KEY": os.getenv("DMZ_API_KEY", ""),
+                            "QUEUE_URL": os.getenv("QUEUE_URL", ""),
                             "XPANSE_ORG_SYNC_BUCKET_NAME": os.getenv(
                                 "XPANSE_ORG_SYNC_BUCKET_NAME"
                             ),

backend/src/xfd_django/xfd_api/tasks/nist_lz_sync.py

@@ -0,0 +1,180 @@
+"""CVE Sync scan hitting new FastAPI endpoint."""
+# --- Standard Libraries ---
+# Standard Python Libraries
+from datetime import datetime, timezone
+import hashlib
+import json
+import logging
+import os
+from urllib.parse import urljoin
+
+# Third-Party Libraries
+# --- Third-Party Libraries ---
+import django
+import requests
+
+# --- Django setup ---
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xfd_django.settings")
+os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
+django.setup()
+
+# Third-Party Libraries
+from xfd_api.helpers.date_time_helpers import calculate_days_back
+
+# --- Your CVE model import ---
+from xfd_mini_dl.models import Cve as CveModel
+from xfd_mini_dl.models import DataSource
+
+# --- Constants & Logging ---
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+LOGGER = logging.getLogger(__name__)
+
+SALT = os.getenv("CHECKSUM_SALT", "default_salt")
+HEADERS = {
+    "X-API-KEY": os.getenv("DMZ_API_KEY"),
+    "Content-Type": "application/json",
+}
+
+# e.g. “https://api.staging-cd.crossfeed.cyber.dhs.gov/sync”
+base_url = os.getenv("DMZ_SYNC_ENDPOINT", "").rstrip("/")
+
+if base_url.endswith("/sync"):
+    # drop “/sync” and append “/cves”
+    CVE_API_URL = base_url.rsplit("/", 1)[0] + "/dmz_sync/cves"
+else:
+    # fallback: join “/cves” onto whatever they provided
+    CVE_API_URL = urljoin(base_url + "/", "dmz_sync/cves")
+
+
+def validate_response_checksum(response):
+    """Validate the checksum from the API."""
+    try:
+        data = response.json()
+        received = response.headers.get("X-Salted-Checksum")
+        if not received:
+            LOGGER.warning("No checksum header")
+            return False
+
+        serialized = json.dumps(data, default=str, sort_keys=True)
+        calc = hashlib.sha256((SALT + serialized).encode()).hexdigest()
+        return received == calc
+    except Exception as e:
+        LOGGER.error("Error validating checksum: %s", e)
+        return False
+
+
+def save_cves_to_db(cve_list):
+    """
+    Upsert each CVE dict into the local DB.
+
+    Matches your corrected Cve model with ArrayFields.
+    """
+    for item in cve_list:
+        # parse ISO timestamps
+        try:
+            pub = datetime.datetime.fromisoformat(item["published_at"])
+        except Exception:
+            pub = None
+        try:
+            mod = datetime.datetime.fromisoformat(item["modified_at"])
+        except Exception:
+            mod = None
+
+        defaults = {
+            "name": item.get("name"),
+            "published_at": pub,
+            "modified_at": mod,
+            "status": item.get("status"),
+            "description": item.get("description"),
+            # CVSS v2
+            "cvss_v2_source": item.get("cvss_v2_source"),
+            "cvss_v2_type": item.get("cvss_v2_type"),
+            "cvss_v2_version": item.get("cvss_v2_version"),
+            "cvss_v2_vector_string": item.get("cvss_v2_vector_string"),
+            "cvss_v2_base_score": item.get("cvss_v2_base_score"),
+            "cvss_v2_base_severity": item.get("cvss_v2_base_severity"),
+            "cvss_v2_exploitability_score": item.get("cvss_v2_exploitability_score"),
+            "cvss_v2_impact_score": item.get("cvss_v2_impact_score"),
+            # CVSS v3
+            "cvss_v3_source": item.get("cvss_v3_source"),
+            "cvss_v3_type": item.get("cvss_v3_type"),
+            "cvss_v3_version": item.get("cvss_v3_version"),
+            "cvss_v3_vector_string": item.get("cvss_v3_vector_string"),
+            "cvss_v3_base_score": item.get("cvss_v3_base_score"),
+            "cvss_v3_base_severity": item.get("cvss_v3_base_severity"),
+            "cvss_v3_exploitability_score": item.get("cvss_v3_exploitability_score"),
+            "cvss_v3_impact_score": item.get("cvss_v3_impact_score"),
+            # CVSS v4
+            "cvss_v4_source": item.get("cvss_v4_source"),
+            "cvss_v4_type": item.get("cvss_v4_type"),
+            "cvss_v4_version": item.get("cvss_v4_version"),
+            "cvss_v4_vector_string": item.get("cvss_v4_vector_string"),
+            "cvss_v4_base_score": item.get("cvss_v4_base_score"),
+            "cvss_v4_base_severity": item.get("cvss_v4_base_severity"),
+            "cvss_v4_exploitability_score": item.get("cvss_v4_exploitability_score"),
+            "cvss_v4_impact_score": item.get("cvss_v4_impact_score"),
+            # ArrayFields
+            "weaknesses": item.get("weaknesses"),
+            "reference_urls": item.get("reference_urls"),
+            "cpe_list": item.get("cpe_list"),
+            # dve_score left untouched (payload doesn’t include it)
+        }
+
+        try:
+            CveModel.objects.update_or_create(id=item["id"], defaults=defaults)
+        except Exception as e:
+            LOGGER.error("Error saving CVE %s: %s", item["id"], e)
+
+
+def handler(command_options=None):
+    """Fetch all CVEs and save them locally."""
+    nist_source, _ = DataSource.objects.get_or_create(
+        name="Nist",
+        defaults={
+            "description": "Nist cve capture",
+            "last_run": timezone.now().date(),
+        },
+    )
+
+    since_date = calculate_days_back(15)
+
+    try:
+        LOGGER.info("Starting CVE sync…")
+        page = 1
+        per_page = 200
+        done = False
+
+        while not done:
+            payload = {
+                "page": page,
+                "page_size": per_page,
+                "since_date": since_date,
+            }
+
+            resp = requests.post(CVE_API_URL, headers=HEADERS, json=payload, timeout=60)
+            resp.raise_for_status()
+
+            if not validate_response_checksum(resp):
+                LOGGER.error("Checksum mismatch!")
+                return {"statusCode": 500, "body": "Checksum mismatch"}
+
+            body = resp.json()
+            if body.get("status") != "ok":
+                LOGGER.error("API returned bad status: %s", body)
+                return {"statusCode": 500, "body": "Bad status"}
+            total_pages = resp.get("total_pages", 1)
+            current_page = resp.get("current_page", 1)
+            payload = body.get("payload", [])
+            LOGGER.info("Fetched %s CVEs", len(payload))
+            save_cves_to_db(payload)
+            if current_page >= total_pages:
+                done = True
+            else:
+                page += 1
+        LOGGER.info("CVE sync completed successfully")
+
+        return {"statusCode": 200, "body": "Shodan sync completed successfully."}
+
+    except Exception as e:
+        LOGGER.error("Sync error: %s", e)
+        return {"statusCode": 500, "body": str(e)}

backend/src/xfd_django/xfd_api/tasks/syncdb_helpers.py

@@ -621,7 +621,11 @@ def create_api_key_for_user(user):
     )
 
     # Print the raw key for debugging or manual testing
-    print("Created API key for user {}: {}".format(user.email, key))
+    print(
+        "Created API key for user, keep this and enter at .env file CF_API_KEY {}: {}".format(
+            user.email, key
+        )
+    )
 
 
 def generate_random_name():