Add bloom checking endpoints

core/logger.py

@@ -94,6 +94,8 @@ def format(self, record):
 class LogFilter(logging.Filter):
     no_log_endpoints = {
         "/api/v2/system/config",
+        "/api/v2/bloom/search",
+        "/api/v2/bloom/search/raw",
     }
 
     sensitive_endpoint_prefixes = (

core/web/apiv2/bloom.py

@@ -0,0 +1,86 @@
+import requests
+from fastapi import APIRouter, Depends, HTTPException, Request
+from pydantic import BaseModel, ConfigDict
+
+from core.config.config import yeti_config
+
+
+class BloomSearchRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    values: list[str]
+
+
+class BloomHit(BaseModel):
+    value: str
+    hits: list[str]
+
+
+# API endpoints
+router = APIRouter()
+
+BLOOMCHECK_ENDPOINT = yeti_config.get("bloom", "bloomcheck_endpoint")
+
+def check_bloomcheck_endpoint():
+    """Ensure BLOOMCHECK_ENDPOINT is set."""
+    if not BLOOMCHECK_ENDPOINT:
+        raise HTTPException(
+            status_code=503,
+            detail="bloomcheck endpoint not set in config",
+        )
+    return BLOOMCHECK_ENDPOINT
+
+@router.post("/search")
+def search(
+    httpreq: Request,
+    request: BloomSearchRequest,
+    bloomcheck_endpoint: str = Depends(check_bloomcheck_endpoint),
+) -> list[BloomHit]:
+    """Checks the bloomcheck microservice for hits."""
+    try:
+        response = requests.post(
+            f"{bloomcheck_endpoint}/check",
+            json={"values": request.values, "filters": []},
+        )
+    except requests.ConnectionError as e:
+        raise HTTPException(
+            status_code=503,
+            detail=f"Error connecting to bloomcheck: {e}",
+        )
+    if response.status_code != 200:
+        raise HTTPException(
+            status_code=response.status_code,
+            detail=f"Error fetching bloomcheck: {response.text}",
+        )
+
+    data = response.json()
+    if not data:
+        return []
+    return [BloomHit(**hit) for hit in data]
+
+
+@router.post("/search/raw")
+async def search_raw(
+    httpreq: Request, bloomcheck_endpoint: str = Depends(check_bloomcheck_endpoint)
+) -> list[BloomHit]:
+    """Checks the bloomcheck microservice for hits."""
+    values = await httpreq.body()
+    try:
+        response = requests.post(
+            f"{bloomcheck_endpoint}/check/raw",
+            data=values,
+        )
+    except requests.ConnectionError as e:
+        raise HTTPException(
+            status_code=503,
+            detail=f"Error connecting to bloomcheck: {e}",
+        )
+    if response.status_code != 200:
+        raise HTTPException(
+            status_code=response.status_code,
+            detail=f"Error fetching bloomcheck: {response.text}",
+        )
+
+    data = response.json()
+    if not data:
+        return []
+    return [BloomHit(**hit) for hit in data]

core/web/webapp.py

@@ -10,6 +10,7 @@
 from core.web.apiv2 import (
     audit,
     auth,
+    bloom,
     dfiq,
     entities,
     graph,
@@ -83,6 +84,14 @@
     tags=["graph"],
     dependencies=[Depends(auth.get_current_active_user)],
 )
+
+api_router.include_router(
+    bloom.router,
+    prefix="/bloom",
+    tags=["bloom"],
+    dependencies=[Depends(auth.get_current_active_user)],
+)
+
 api_router.include_router(
     templates.router,
     prefix="/templates",

tests/apiv2/bloom.py

@@ -0,0 +1,77 @@
+import unittest
+from unittest import mock
+
+import requests
+from fastapi.testclient import TestClient
+
+from core.schemas.user import UserSensitive
+from core.web import webapp
+
+client = TestClient(webapp.app)
+
+
+class IndicatorTest(unittest.TestCase):
+    def setUp(self) -> None:
+        user = UserSensitive(username="test", password="test", enabled=True).save()
+        apikey = user.create_api_key("default")
+        token_data = client.post(
+            "/api/v2/auth/api-token", headers={"x-yeti-apikey": apikey}
+        ).json()
+        client.headers = {"Authorization": "Bearer " + token_data["access_token"]}
+
+    @mock.patch("core.web.apiv2.bloom.requests.post")
+    def testConnectionError(self, mock_post) -> None:
+        mock_post.side_effect = requests.ConnectionError("Connection error")
+
+        response = client.post(
+            "/api/v2/bloom/search",
+            json={
+                "values": ["test"],
+            },
+        )
+        data = response.json()
+        self.assertEqual(response.status_code, 503, data)
+        self.assertIn("Error connecting to bloomcheck", data["detail"])
+
+    @mock.patch("core.web.apiv2.bloom.requests.post")
+    def testBloomCall(self, mock_post) -> None:
+        mock_response = mock.Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = [{"value": "test", "hits": ["fltr"]}]
+        mock_post.return_value = mock_response
+
+        response = client.post(
+            "/api/v2/bloom/search",
+            json={
+                "values": ["test"],
+            },
+        )
+
+        data = response.json()
+        self.assertEqual(response.status_code, 200, data)
+        self.assertEqual(len(data), 1)
+        self.assertEqual(data[0]["value"], "test")
+
+        mock_post.assert_called_once_with(
+            "http://bloomcheck:8100/check",
+            json={"values": ["test"], "filters": []},
+        )
+
+    @mock.patch("core.web.apiv2.bloom.requests.post")
+    def testBloomCallRaw(self, mock_post) -> None:
+        mock_response = mock.Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = [{"value": "test", "hits": ["fltr"]}]
+        mock_post.return_value = mock_response
+        test_body = b"test1\ntest2\ntest3\n"
+        response = client.post("/api/v2/bloom/search/raw", data=test_body)
+
+        data = response.json()
+        self.assertEqual(response.status_code, 200, data)
+        self.assertEqual(len(data), 1)
+        self.assertEqual(data[0]["value"], "test")
+
+        mock_post.assert_called_once_with(
+            "http://bloomcheck:8100/check/raw",
+            data=b"test1\ntest2\ntest3\n",
+        )

yeti.conf.sample

@@ -96,6 +96,15 @@ default_acls = All users
 # database = 0
 # tls = ok
 
+[bloom]
+
+##
+## Use these settings if you want to use the bloomcheck service
+##
+
+bloomcheck_endpoint = http://bloomcheck:8100
+
+
 [events]
 # Define in MiB the maximum allocated memory for events queue
 memory_limit = 64