Merge Develop into Integration

backend/.pylintrc

@@ -13,6 +13,7 @@ max-line-length=120
 
 [MESSAGES CONTROL]
 disable=C0103,  # Conforms to snake_case
+        C0209,  # Formatting a regular string which could be an f-string (consider-using-f-string)
         C0301,  # Line too long
         C0415,  # Import outside top-level (consider refactoring)
         C0302,  # Too many lines in module

backend/Dockerfile.pe

@@ -7,7 +7,7 @@ COPY ./package* ./
 
 COPY src ./src
 
-RUN apt update && apt install git zlib1g-dev
+RUN apt update && apt install -y git zlib1g-dev
 
 RUN apt-get update && apt-get install -y jq
 
@@ -28,7 +28,7 @@ RUN ./aws/install
 # Sync the latest from cf-staging branch
 RUN git clone -b crossfeed-SQS https://github.com/cisagov/ATC-Framework.git && \
     cd ATC-Framework && \
-    git checkout 694dc68487ab24b625f157fb5d79f675c7ec1467 && \
+    git checkout 563f59e8b67ca153ab5564d697433ca0d8db451a && \
     pip install .
 
 RUN python -m spacy download en_core_web_lg

backend/Makefile

@@ -1,5 +1,6 @@
 .PHONY: syncdb
 .PHONY: populate-cache
+.PHONY: pytest
 
 # Synchronize and populate the database
 syncdb:
@@ -8,6 +9,18 @@ syncdb:
 syncdb-populate:
 	docker compose exec backend python manage.py syncdb --populate
 
+syncdb-dangerously-force:
+	docker compose exec backend python manage.py syncdb --dangerouslyforce
+
+
+# Synchronize and populate the database
+syncmdl:
+	docker compose exec backend python manage.py syncmdl
+
+syncmdl-dangerously-force:
+	docker compose exec backend python manage.py syncmdl --dangerouslyforce
+
+
 # Pytest
 # i.e. make pytest FILE=xfd_api/tests/test_domain.py
 pytest:

backend/env.yml

@@ -6,12 +6,15 @@ dev:
   REGION: us-east-1
 
 staging-cd:
+  IS_DMZ: 1
   REGION: us-east-1
   ENDPOINT_TYPE: REGIONAL
   COGNITO_URL: https://cognito-idp.us-east-1.amazonaws.com
   BACKEND_DOMAIN: https://api.staging-cd.crossfeed.cyber.dhs.gov
   EMAIL_REGION: us-east-1
   DJANGO_KEY: ${ssm:/crossfeed/staging/DJANGO_KEY}
+  PYTHONPATH: src/xfd_django
+  DJANGO_SETTINGS_MODULE: xfd_django.settings
   DB_DIALECT: postgres
   DB_PORT: 5432
   DB_HOST: ${ssm:/crossfeed/staging/DATABASE_HOST}
@@ -22,6 +25,8 @@ staging-cd:
   MDL_USERNAME: ${ssm:/crossfeed/staging/MDL_USERNAME}
   MDL_PASSWORD: ${ssm:/crossfeed/staging/MDL_PASSWORD}
   MDL_NAME: ${ssm:/crossfeed/staging/MDL_NAME}
+  POSTGRES_SCAN_USER: ${ssm:/crossfeed/staging/POSTGRES_SCAN_USER}
+  POSTGRES_SCAN_PASSWORD: ${ssm:/crossfeed/staging/POSTGRES_SCAN_PASSWORD}
   MI_ACCOUNT_NAME: ${ssm:/readysetcyber/staging/MI_ACCOUNT_NAME}
   MI_PASSWORD: ${ssm:/readysetcyber/staging/MI_ACCOUNT_PASSWORD}
   PE_DB_NAME: ${ssm:/crossfeed/staging/PE_DB_NAME}
@@ -76,6 +81,7 @@ staging-cd:
   REACT_APP_COGNITO_CALLBACK_URL: https://staging-cd.crossfeed.cyber.dhs.gov/okta-callback
 
 integration:
+  IS_DMZ: 1
   REGION: us-east-1
   ENDPOINT_TYPE: REGIONAL
   COGNITO_URL: https://cognito-idp.us-east-1.amazonaws.com
@@ -93,6 +99,8 @@ integration:
   MDL_USERNAME: ${ssm:/crossfeed/integration/MDL_USERNAME}
   MDL_PASSWORD: ${ssm:/crossfeed/integration/MDL_PASSWORD}
   MDL_NAME: ${ssm:/crossfeed/integration/MDL_NAME}
+  POSTGRES_SCAN_USER: ${ssm:/crossfeed/integration/POSTGRES_SCAN_USER}
+  POSTGRES_SCAN_PASSWORD: ${ssm:/crossfeed/integration/POSTGRES_SCAN_PASSWORD}
   MI_ACCOUNT_NAME: ${ssm:/readysetcyber/integration/MI_ACCOUNT_NAME}
   MI_PASSWORD: ${ssm:/readysetcyber/integration/MI_ACCOUNT_PASSWORD}
   JWT_SECRET: ${ssm:/crossfeed/integration/APP_JWT_SECRET}
@@ -140,6 +148,7 @@ integration:
   ELASTICACHE_ENDPOINT: ${ssm:/crossfeed/integration/ELASTICACHE_ENDPOINT}
 
 staging:
+  IS_DMZ: 0
   REGION: us-gov-east-1
   ENDPOINT_TYPE: PRIVATE
   COGNITO_URL: https://cognito-idp.us-gov-west-1.amazonaws.com
@@ -155,6 +164,8 @@ staging:
   DB_NAME: ${ssm:/crossfeed/staging/DATABASE_NAME}
   DB_USERNAME: ${ssm:/crossfeed/staging/DATABASE_USER}
   DB_PASSWORD: ${ssm:/crossfeed/staging/DATABASE_PASSWORD}
+  POSTGRES_SCAN_USER: ${ssm:/crossfeed/staging/POSTGRES_SCAN_USER}
+  POSTGRES_SCAN_PASSWORD: ${ssm:/crossfeed/staging/POSTGRES_SCAN_PASSWORD}
   JWT_SECRET: ${ssm:/crossfeed/staging/APP_JWT_SECRET}
   JWT_ALGORITHM: ${ssm:/crossfeed/staging/JWT_ALGORITHM}
   JWT_TIMEOUT_HOURS: ${ssm:/crossfeed/staging/JWT_TIMEOUT_HOURS}
@@ -202,13 +213,16 @@ staging:
   ELASTICACHE_ENDPOINT: ${ssm:/crossfeed/staging/ELASTICACHE_ENDPOINT}
 
 prod:
+  IS_DMZ: 0
   REGION: us-gov-east-1
   ENDPOINT_TYPE: PRIVATE
   COGNITO_URL: https://cognito-idp.us-gov-west-1.amazonaws.com
   BACKEND_DOMAIN: https://api.crossfeed.cyber.dhs.gov
   EMAIL_REGION: us-gov-west-1
   LZ_PROXY_URL: ${ssm:/crossfeed/prod/LZ_PROXY_URL}
   DJANGO_KEY: ${ssm:/crossfeed/prod/DJANGO_KEY}
+  PYTHONPATH: src/xfd_django
+  DJANGO_SETTINGS_MODULE: xfd_django.settings
   DB_DIALECT: postgres
   DB_PORT: 5432
   DB_HOST: ${ssm:/crossfeed/prod/DATABASE_HOST}
@@ -219,6 +233,8 @@ prod:
   MDL_PASSWORD: ${ssm:/crossfeed/prod/MDL_PASSWORD}
   DJANGO_SECRET: ${ssm:/crossfeed/prod/DJANGO_SECRECT}
   MDL_NAME: ${ssm:/crossfeed/prod/MDL_NAME}
+  POSTGRES_SCAN_USER: ${ssm:/crossfeed/prod/POSTGRES_SCAN_USER}
+  POSTGRES_SCAN_PASSWORD: ${ssm:/crossfeed/prod/POSTGRES_SCAN_PASSWORD}
   JWT_SECRET: ${ssm:/crossfeed/prod/APP_JWT_SECRET}
   JWT_ALGORITHM: ${ssm:/crossfeed/prod/JWT_ALGORITHM}
   JWT_TIMEOUT_HOURS: ${ssm:/crossfeed/prod/JWT_TIMEOUT_HOURS}

backend/requirements.txt

@@ -17,6 +17,7 @@ click==8.1.7
 cryptography==38.0.0
 dill==0.3.9
 Django==5.1.4
+django-netfields==1.3.2
 dnspython==2.7.0
 docker==7.1.0
 elasticsearch==7.9.0

backend/scripts/populateCountiesCities/cities.py

@@ -117,7 +117,7 @@ def pull_cities():
                         )
                     time.sleep(1)
                 except Exception as e:
-                    print(f"Error: {e}")
+                    print("Error: {}".format(e))
                     pass
 
         df = pd.DataFrame(holding_pen, columns=["State", "County", "City", "URL"])

backend/scripts/populateCountiesCities/counties.py

@@ -62,7 +62,7 @@ def pull_counties():
                 }
             )
         except Exception as e:
-            print(f"Error: {e}")
+            print("Error: {}".format(e))
             pass
 
         time.sleep(1)

backend/serverless.yml

@@ -59,7 +59,9 @@ custom:
       Condition:
         IpAddress:
           aws:SourceIp:
-            - ${file(env.yml):${self:provider.stage}.DMZ_CIDR, ''}
+            'Fn::Split':
+              - ','
+              - ${file(env.yml):${self:provider.stage}.DMZ_CIDR, ''}
 
   # Conditional logic for GovCloud vs
   # non-GovCloud (Private endpoints require a VPC Endpoint)

backend/src/api/app.ts

@@ -444,17 +444,17 @@ app.use(
     // API Gateway isn't able to proxy fonts properly -- so we're using a CDN instead.
     if (req.path === '/plugins/Morpheus/fonts/matomo.woff2') {
       return res.redirect(
-        'https://cdn.jsdelivr.net/gh/matomo-org/matomo@3.14.1/plugins/Morpheus/fonts/matomo.woff2'
+        'https://cdn.jsdelivr.net/gh/matomo-org/matomo@5.2.1/plugins/Morpheus/fonts/matomo.woff2'
       );
     }
     if (req.path === '/plugins/Morpheus/fonts/matomo.woff') {
       return res.redirect(
-        'https://cdn.jsdelivr.net/gh/matomo-org/matomo@3.14.1/plugins/Morpheus/fonts/matomo.woff'
+        'https://cdn.jsdelivr.net/gh/matomo-org/matomo@5.2.1/plugins/Morpheus/fonts/matomo.woff'
       );
     }
     if (req.path === '/plugins/Morpheus/fonts/matomo.ttf') {
       return res.redirect(
-        'https://cdn.jsdelivr.net/gh/matomo-org/matomo@3.14.1/plugins/Morpheus/fonts/matomo.ttf'
+        'https://cdn.jsdelivr.net/gh/matomo-org/matomo@5.2.1/plugins/Morpheus/fonts/matomo.ttf'
       );
     }
     // Only allow global admins to access all other paths.

backend/src/api/users.ts

@@ -899,7 +899,9 @@ export const updateV2 = wrapHandler(async (event) => {
   }
 
   // Check if authorizer's region matches the user's
-  if (!matchesUserRegion(event, user.regionId)) return Unauthorized;
+  // Allows new users to select their state without a region on initial account creation
+  if (user.invitePending === false && !matchesUserRegion(event, user.regionId))
+    return Unauthorized;
 
   if (body.state) {
     body.regionId = REGION_STATE_MAP[body.state];

backend/src/api/vulnerabilities.ts

@@ -124,11 +124,12 @@ class VulnerabilitySearch {
     if (this.filters?.severity) {
       if (this.filters.severity === 'N/A') {
         qs.andWhere(
-          "vulnerability.severity IS NULL OR vulnerability.severity = ''"
+          "vulnerability.severity IS NULL OR vulnerability.severity = '' OR vulnerability.severity ILIKE 'N/A' OR vulnerability.severity ILIKE 'NULL'"
         );
       } else if (this.filters.severity === 'Other') {
         qs.andWhere(
-          `vulnerability.severity NOT ILIKE 'N/A' AND
+          `vulnerability.severity NOT ILIKE 'NULL' AND
+          vulnerability.severity NOT ILIKE 'N/A' AND
          vulnerability.severity NOT ILIKE 'Low' AND
          vulnerability.severity NOT ILIKE 'Medium' AND
          vulnerability.severity NOT ILIKE 'High' AND

backend/src/tasks/functions.yml

@@ -12,6 +12,10 @@ syncdb:
   timeout: 900
   handler: src/xfd_django/xfd_api/tasks/run_syncdb.handler
 
+syncmdl:
+  timeout: 900
+  handler: src/xfd_django/xfd_api/tasks/run_syncmdl.handler
+
 bastion:
   timeout: 900
   handler: src/xfd_django/xfd_api/tasks/bastion.handler

backend/src/xfd_django/xfd_api/api_methods/auth.py

@@ -8,20 +8,20 @@
 
 async def handle_okta_callback(request):
     """POST API LOGIC."""
-    print(f"Request from /auth/okta-callback: {str(request)}")
+    print("Request from /auth/okta-callback: {}".format(str(request)))
     body = await request.json()
-    print(f"Request json from callback: {str(request)}")
-    print(f"Request json from callback: {body}")
-    print(f"Body type: {type(body)}")
+    print("Request json from callback: {}".format(str(request)))
+    print("Request json from callback: {}".format(body))
+    print("Body type: {}".format(type(body)))
     code = body.get("code")
-    print(f"Code: {code}")
+    print("Code: {}".format(code))
     if not code:
         return HTTPException(
             status_code=status.HTTP_400_BAD_REQUEST,
             detail="Code not found in request body",
         )
     jwt_data = await get_jwt_from_code(code)
-    print(f"JWT Data: {jwt_data}")
+    print("JWT Data: {}".format(jwt_data))
     if jwt_data is None:
         raise HTTPException(
             status_code=status.HTTP_400_BAD_REQUEST,

backend/src/xfd_django/xfd_api/api_methods/domain.py

@@ -167,7 +167,7 @@ def export_domains(domain_search: DomainSearch, current_user):
                     for product in service.products.all():
                         if product.name:
                             product_entry = (
-                                f"{product.name} {product.version}"
+                                "{} {}".format(product.name, product.version)
                                 if product.version
                                 else product.name
                             )
@@ -221,5 +221,5 @@ def export_domains(domain_search: DomainSearch, current_user):
 
     except Exception as e:
         # Log the exception for debugging (optional)
-        print(f"Error exporting domains: {e}")
+        print("Error exporting domains: {}".format(e))
         raise HTTPException(status_code=500, detail=str(e))

backend/src/xfd_django/xfd_api/api_methods/organization.py

@@ -250,7 +250,7 @@ def get_organization(organization_id, current_user):
         raise http_exc
 
     except Exception as e:
-        print(f"An error occurred: {e}")
+        print("An error occurred: {}".format(e))
         raise HTTPException(status_code=500, detail=str(e))
 
 
@@ -715,7 +715,7 @@ def delete_organization(org_id: str, current_user):
         # Return success response
         return {
             "status": "success",
-            "message": f"Organization {org_id} has been deleted successfully.",
+            "message": "Organization {} has been deleted successfully.".format(org_id),
         }
 
     except HTTPException as http_exc:
@@ -1064,7 +1064,7 @@ def search_organizations_task(search_body, current_user: User):
         # Use match_all if searchTerm is empty
         if search_body.searchTerm.strip():
             query_body["query"]["bool"]["must"].append(
-                {"wildcard": {"name": f"*{search_body.searchTerm}*"}}
+                {"wildcard": {"name": "*{}*".format(search_body.searchTerm)}}
             )
         else:
             query_body["query"]["bool"]["must"].append({"match_all": {}})
@@ -1076,7 +1076,7 @@ def search_organizations_task(search_body, current_user: User):
             )
 
         # Log the query for debugging
-        print(f"Query body: {query_body}")
+        print("Query body: {}".format(query_body))
 
         # Execute the search
         search_results = client.search_organizations(query_body)

backend/src/xfd_django/xfd_api/api_methods/proxy.py

@@ -32,13 +32,13 @@ async def proxy_request(
     if cookie_name:
         cookies = manipulate_cookie(request, cookie_name)
         if cookies:
-            headers["Cookie"] = f"{cookie_name}={cookies[cookie_name]}"
+            headers["Cookie"] = "{}={}".format(cookie_name, cookies[cookie_name])
 
     # Make the request to the target URL
     async with httpx.AsyncClient() as client:
         proxy_response = await client.request(
             method=request.method,
-            url=f"{target_url}/{path}",
+            url="{}/{}".format(target_url, path),
             headers=headers,
             params=request.query_params,
             content=await request.body(),

backend/src/xfd_django/xfd_api/api_methods/saved_search.py

@@ -235,7 +235,7 @@ def delete_saved_search(saved_search_id, user):
         return JsonResponse(
             {
                 "status": "success",
-                "message": f"Saved search id:{saved_search_id} deleted.",
+                "message": "Saved search id:{} deleted.".format(saved_search_id),
             }
         )
     except User.DoesNotExist:

backend/src/xfd_django/xfd_api/api_methods/scan.py

@@ -270,7 +270,10 @@ def delete_scan(scan_id: str, current_user):
 
         scan.delete()
 
-        return {"status": "success", "message": f"Scan {scan_id} deleted successfully."}
+        return {
+            "status": "success",
+            "message": "Scan {} deleted successfully.".format(scan_id),
+        }
 
     except HTTPException as http_exc:
         raise http_exc
@@ -297,7 +300,7 @@ def run_scan(scan_id: str, current_user):
         scan.save()
         return {
             "status": "success",
-            "message": f"Scan {scan_id} set to manualRunPending.",
+            "message": "Scan {} set to manualRunPending.".format(scan_id),
         }
 
     except HTTPException as http_exc:
@@ -320,7 +323,7 @@ async def invoke_scheduler(current_user):
         lambda_client = LambdaClient()
 
         # Form the lambda function name using environment variable
-        lambda_function_name = f"{os.getenv('SLS_LAMBDA_PREFIX')}-scheduler"
+        lambda_function_name = "{}-scheduler".format(os.getenv("SLS_LAMBDA_PREFIX"))
         print(lambda_function_name)
 
         # Run the Lambda command

backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py

@@ -37,9 +37,9 @@ def list_scan_tasks(search_data: Optional[ScanTaskSearch], current_user):
 
         # Determine the correct ordering based on the 'order' field
         ordering_field = (
-            f"-{search_data.sort}"
+            "-{}".format(search_data.sort)
             if search_data.order and search_data.order.upper() == "DESC"
-            else search_data.sort
+            else "{}".format(search_data.sort)
         )
 
         # Construct query based on filters
@@ -71,7 +71,7 @@ def list_scan_tasks(search_data: Optional[ScanTaskSearch], current_user):
         for task in qs:
             # Ensure scan is not None before accessing its properties
             if task.scan is None:
-                print(f"Warning: ScanTask {task.id} has no scan associated.")
+                print("Warning: ScanTask {} has no scan associated.".format(task.id))
                 scan_data = None
             else:
                 scan_data = {
@@ -177,7 +177,7 @@ def kill_scan_task(scan_task_id, current_user):
         utc_now = datetime.now(timezone.utc)
         scan_task.status = "failed"
         scan_task.finishedAt = utc_now
-        scan_task.output = f"Manually stopped at {utc_now.isoformat()}"
+        scan_task.output = "Manually stopped at {}".format(utc_now.isoformat())
         scan_task.save()
 
         return {"statusCode": 200, "message": "ScanTask successfully marked as failed."}