Feat/linux semantic

.env.example

@@ -38,6 +38,20 @@ APP_CLEANUP_TOKEN_RETENTION=7d
 APP_CLEANUP_LOGIN_ATTEMPT_RETENTION=30d
 APP_CLEANUP_BATCH_SIZE=1000
 
+# Semantic/hybrid search. The URL points to an external HTTP service exposing GET /health and POST /embed.
+# For Docker Compose, set this to a URL reachable from the app container; do not use localhost for a host service on Linux.
+APP_SEMANTIC_SEARCH_ENABLED=true
+APP_EMBEDDING_SERVICE_URL=http://localhost:8001
+APP_EMBEDDING_MODEL=BAAI/bge-m3
+APP_EMBEDDING_MODEL_VERSION=1.3.5
+APP_EMBEDDING_BATCH_SIZE=16
+APP_SEMANTIC_MIN_SIMILARITY=0.45
+APP_SEMANTIC_CANDIDATE_LIMIT=100
+APP_SEMANTIC_MAX_CANDIDATE_LIMIT=1000
+APP_HYBRID_RRF_K=60
+APP_EMBEDDING_CONNECT_TIMEOUT=10s
+APP_EMBEDDING_READ_TIMEOUT=120s
+
 CLOUDINARY_CLOUD_NAME=
 CLOUDINARY_API_KEY=
 CLOUDINARY_API_SECRET=

docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   postgres:
-    image: postgres:16-alpine
+    image: pgvector/pgvector:pg16
     container_name: ahadith-postgres
     environment:
       POSTGRES_DB: ${LOCAL_POSTGRES_DB:-ahadith}
@@ -25,13 +25,10 @@ services:
       context: .
       dockerfile: Dockerfile
     container_name: ahadith-app
-    depends_on:
-      postgres:
-        condition: service_healthy
     environment:
-      SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/${LOCAL_POSTGRES_DB:-ahadith}
-      SPRING_DATASOURCE_USERNAME: ${LOCAL_POSTGRES_USERNAME:-postgres}
-      SPRING_DATASOURCE_PASSWORD: ${LOCAL_POSTGRES_PASSWORD:-postgres}
+      SPRING_DATASOURCE_URL: ${SPRING_DATASOURCE_URL:?SPRING_DATASOURCE_URL is required}
+      SPRING_DATASOURCE_USERNAME: ${SPRING_DATASOURCE_USERNAME:?SPRING_DATASOURCE_USERNAME is required}
+      SPRING_DATASOURCE_PASSWORD: ${SPRING_DATASOURCE_PASSWORD:?SPRING_DATASOURCE_PASSWORD is required}
       SPRING_PROFILES_ACTIVE: ${SPRING_PROFILES_ACTIVE:-prod}
       PORT: ${PORT:-8080}
       JWT_SECRET: ${JWT_SECRET:?JWT_SECRET is required}
@@ -45,6 +42,17 @@ services:
       CLOUDINARY_CLOUD_NAME: ${CLOUDINARY_CLOUD_NAME:?CLOUDINARY_CLOUD_NAME is required}
       CLOUDINARY_API_KEY: ${CLOUDINARY_API_KEY:?CLOUDINARY_API_KEY is required}
       CLOUDINARY_API_SECRET: ${CLOUDINARY_API_SECRET:?CLOUDINARY_API_SECRET is required}
+      APP_SEMANTIC_SEARCH_ENABLED: ${APP_SEMANTIC_SEARCH_ENABLED:-true}
+      APP_EMBEDDING_SERVICE_URL: ${APP_EMBEDDING_SERVICE_URL:?APP_EMBEDDING_SERVICE_URL must point to an external embedding HTTP service}
+      APP_EMBEDDING_MODEL: ${APP_EMBEDDING_MODEL:-BAAI/bge-m3}
+      APP_EMBEDDING_MODEL_VERSION: ${APP_EMBEDDING_MODEL_VERSION:-1.3.5}
+      APP_EMBEDDING_BATCH_SIZE: ${APP_EMBEDDING_BATCH_SIZE:-16}
+      APP_SEMANTIC_MIN_SIMILARITY: ${APP_SEMANTIC_MIN_SIMILARITY:-0.45}
+      APP_SEMANTIC_CANDIDATE_LIMIT: ${APP_SEMANTIC_CANDIDATE_LIMIT:-100}
+      APP_SEMANTIC_MAX_CANDIDATE_LIMIT: ${APP_SEMANTIC_MAX_CANDIDATE_LIMIT:-1000}
+      APP_HYBRID_RRF_K: ${APP_HYBRID_RRF_K:-60}
+      APP_EMBEDDING_CONNECT_TIMEOUT: ${APP_EMBEDDING_CONNECT_TIMEOUT:-10s}
+      APP_EMBEDDING_READ_TIMEOUT: ${APP_EMBEDDING_READ_TIMEOUT:-120s}
     ports:
       - "8080:8080"
     healthcheck:

src/main/java/com/jamil/ahadith/features/hadith/service/HadithService.java

@@ -32,8 +32,11 @@
 import lombok.AllArgsConstructor;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
+import org.springframework.context.ApplicationEventPublisher;
+import com.jamil.ahadith.features.search.semantic.event.HadithTextChangedEvent;
 
 import java.util.List;
+import java.util.Objects;
 import java.util.UUID;
 
 @Transactional
@@ -50,6 +53,7 @@ public class HadithService {
     private final ExplainingRepository explainingRepository;
     private final AuditEventPublisher auditEventPublisher;
     private final HadithSearchService hadithSearchService;
+    private final ApplicationEventPublisher eventPublisher;
 
     public HadithResponseDto getHadithById(UUID id) {
         return hadithRepository.findById(id)
@@ -64,6 +68,7 @@ public HadithResponseDto createHadith(HadithRequestDto request) {
         hadith = hadithRepository.saveAndFlush(hadith);
         entityManager.refresh(hadith);
         auditEventPublisher.publishCreate("ahadith", hadith.getId(), AuditData.snapshot(hadith));
+        eventPublisher.publishEvent(new HadithTextChangedEvent(hadith.getId(), hadith.getText()));
         return toResponseWithFullSubValid(hadith);
     }
 
@@ -76,12 +81,16 @@ public HadithResponseDto updateHadith(UUID id, HadithUpdateDto request) {
         }
 
         var oldData = AuditData.snapshot(hadith);
+        String previousText = hadith.getText();
         hadithMapper.updateEntity(request, hadith);
         applyUpdateRelations(request, hadith);
         currentUserService.getCurrentUser().ifPresent(hadith::setUpdatedBy);
         var savedHadith = hadithRepository.saveAndFlush(hadith);
         entityManager.refresh(savedHadith);
         auditEventPublisher.publishUpdate("ahadith", savedHadith.getId(), oldData, AuditData.snapshot(savedHadith));
+        if (!Objects.equals(previousText, savedHadith.getText())) {
+            eventPublisher.publishEvent(new HadithTextChangedEvent(savedHadith.getId(), savedHadith.getText()));
+        }
         return toResponseWithFullSubValid(savedHadith);
     }
 

src/main/java/com/jamil/ahadith/features/search/dto/request/SearchMode.java

@@ -2,5 +2,7 @@
 
 public enum SearchMode {
     EXACT,
-    FLEXIBLE
+    FLEXIBLE,
+    SEMANTIC,
+    HYBRID
 }

src/main/java/com/jamil/ahadith/features/search/semantic/EmbeddingServiceHealthIndicator.java

@@ -0,0 +1,33 @@
+package com.jamil.ahadith.features.search.semantic;
+
+import com.jamil.ahadith.features.search.semantic.client.EmbeddingClient;
+import com.jamil.ahadith.features.search.semantic.config.SemanticSearchProperties;
+import lombok.RequiredArgsConstructor;
+import org.springframework.boot.health.contributor.Health;
+import org.springframework.boot.health.contributor.HealthIndicator;
+import org.springframework.stereotype.Component;
+
+@Component("embeddingService")
+@RequiredArgsConstructor
+public class EmbeddingServiceHealthIndicator implements HealthIndicator {
+    private final EmbeddingClient client;
+    private final SemanticSearchProperties properties;
+
+    @Override
+    public Health health() {
+        if (!properties.isEnabled()) {
+            return Health.up().withDetail("enabled", false).build();
+        }
+        try {
+            var response = client.health();
+            return Health.up()
+                    .withDetail("model", response.model())
+                    .withDetail("modelVersion", response.modelVersion())
+                    .withDetail("dimension", response.dimension())
+                    .withDetail("device", response.device())
+                    .build();
+        } catch (RuntimeException ex) {
+            return Health.down(ex).build();
+        }
+    }
+}

src/main/java/com/jamil/ahadith/features/search/semantic/client/EmbeddingClient.java

@@ -0,0 +1,77 @@
+package com.jamil.ahadith.features.search.semantic.client;
+
+import com.jamil.ahadith.features.search.semantic.config.SemanticSearchProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.client.SimpleClientHttpRequestFactory;
+import org.springframework.stereotype.Component;
+import org.springframework.web.client.RestClient;
+import org.springframework.web.client.RestClientException;
+
+import java.util.List;
+import java.util.Map;
+
+@Component
+public class EmbeddingClient {
+    public static final int DIMENSION = 1024;
+
+    private final RestClient restClient;
+
+    @Autowired
+    public EmbeddingClient(SemanticSearchProperties properties) {
+        SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
+        requestFactory.setConnectTimeout(properties.getConnectTimeout());
+        requestFactory.setReadTimeout(properties.getReadTimeout());
+        this.restClient = RestClient.builder()
+                .baseUrl(properties.getServiceUrl())
+                .requestFactory(requestFactory)
+                .build();
+    }
+
+    EmbeddingClient(RestClient restClient) {
+        this.restClient = restClient;
+    }
+
+    public EmbeddingResponse embed(List<String> texts) {
+        if (texts == null || texts.isEmpty()) {
+            throw new IllegalArgumentException("At least one text is required");
+        }
+        try {
+            EmbeddingResponse response = restClient.post()
+                    .uri("/embed")
+                    .body(Map.of("texts", texts))
+                    .retrieve()
+                    .body(EmbeddingResponse.class);
+            validate(response, texts.size());
+            return response;
+        } catch (EmbeddingServiceException ex) {
+            throw ex;
+        } catch (RestClientException ex) {
+            throw new EmbeddingServiceException("Embedding service is unavailable", ex);
+        }
+    }
+
+    public EmbeddingHealthResponse health() {
+        try {
+            EmbeddingHealthResponse response = restClient.get()
+                    .uri("/health")
+                    .retrieve()
+                    .body(EmbeddingHealthResponse.class);
+            if (response == null || !"UP".equalsIgnoreCase(response.status()) || response.dimension() != DIMENSION) {
+                throw new EmbeddingServiceException("Embedding service returned an invalid health response");
+            }
+            return response;
+        } catch (EmbeddingServiceException ex) {
+            throw ex;
+        } catch (RestClientException ex) {
+            throw new EmbeddingServiceException("Embedding service is unavailable", ex);
+        }
+    }
+
+    private void validate(EmbeddingResponse response, int expectedCount) {
+        if (response == null || response.dimension() != DIMENSION || response.embeddings() == null
+                || response.embeddings().size() != expectedCount
+                || response.embeddings().stream().anyMatch(vector -> vector == null || vector.size() != DIMENSION)) {
+            throw new EmbeddingServiceException("Embedding service returned an invalid response");
+        }
+    }
+}

src/main/java/com/jamil/ahadith/features/search/semantic/client/EmbeddingHealthResponse.java

@@ -0,0 +1,10 @@
+package com.jamil.ahadith.features.search.semantic.client;
+
+public record EmbeddingHealthResponse(
+        String status,
+        String model,
+        String modelVersion,
+        int dimension,
+        String device
+) {
+}

src/main/java/com/jamil/ahadith/features/search/semantic/client/EmbeddingResponse.java

@@ -0,0 +1,11 @@
+package com.jamil.ahadith.features.search.semantic.client;
+
+import java.util.List;
+
+public record EmbeddingResponse(
+        String model,
+        String modelVersion,
+        int dimension,
+        List<List<Double>> embeddings
+) {
+}

src/main/java/com/jamil/ahadith/features/search/semantic/client/EmbeddingServiceException.java

@@ -0,0 +1,11 @@
+package com.jamil.ahadith.features.search.semantic.client;
+
+public class EmbeddingServiceException extends RuntimeException {
+    public EmbeddingServiceException(String message) {
+        super(message);
+    }
+
+    public EmbeddingServiceException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

src/main/java/com/jamil/ahadith/features/search/semantic/config/SemanticSearchProperties.java

@@ -0,0 +1,24 @@
+package com.jamil.ahadith.features.search.semantic.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+import java.time.Duration;
+
+@Getter
+@Setter
+@ConfigurationProperties(prefix = "app.semantic-search")
+public class SemanticSearchProperties {
+    private boolean enabled;
+    private String serviceUrl = "http://localhost:8001";
+    private String model = "BAAI/bge-m3";
+    private String modelVersion = "1.3.5";
+    private int batchSize = 16;
+    private double minSimilarity = 0.45;
+    private int candidateLimit = 100;
+    private int maxCandidateLimit = 1000;
+    private int hybridRrfK = 60;
+    private Duration connectTimeout = Duration.ofSeconds(10);
+    private Duration readTimeout = Duration.ofSeconds(120);
+}

src/main/java/com/jamil/ahadith/features/search/semantic/controller/AdminEmbeddingController.java

@@ -0,0 +1,28 @@
+package com.jamil.ahadith.features.search.semantic.controller;
+
+import com.jamil.ahadith.features.search.semantic.repository.EmbeddingStatus;
+import com.jamil.ahadith.features.search.semantic.service.HadithEmbeddingService;
+import com.jamil.ahadith.features.search.semantic.service.ReindexResult;
+import lombok.RequiredArgsConstructor;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequiredArgsConstructor
+@RequestMapping("/api/v1/admin/hadith-embeddings")
+public class AdminEmbeddingController {
+    private final HadithEmbeddingService service;
+
+    @GetMapping("/status")
+    public EmbeddingStatus status() {
+        return service.status();
+    }
+
+    @PostMapping("/reindex")
+    public ReindexResult reindex(@RequestParam(defaultValue = "false") boolean force) {
+        return service.reindex(force);
+    }
+}

src/main/java/com/jamil/ahadith/features/search/semantic/event/HadithEmbeddingListener.java

@@ -0,0 +1,27 @@
+package com.jamil.ahadith.features.search.semantic.event;
+
+import com.jamil.ahadith.features.search.semantic.service.HadithEmbeddingService;
+import lombok.RequiredArgsConstructor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.event.TransactionPhase;
+import org.springframework.transaction.event.TransactionalEventListener;
+
+@Component
+@RequiredArgsConstructor
+public class HadithEmbeddingListener {
+    private static final Logger log = LoggerFactory.getLogger(HadithEmbeddingListener.class);
+    private final HadithEmbeddingService embeddingService;
+
+    @Async
+    @TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT)
+    public void onHadithTextChanged(HadithTextChangedEvent event) {
+        try {
+            embeddingService.embedHadith(event.hadithId(), event.text());
+        } catch (RuntimeException ex) {
+            log.warn("Failed to generate Hadith embedding after commit hadithId={}", event.hadithId(), ex);
+        }
+    }
+}

src/main/java/com/jamil/ahadith/features/search/semantic/event/HadithTextChangedEvent.java

@@ -0,0 +1,6 @@
+package com.jamil.ahadith.features.search.semantic.event;
+
+import java.util.UUID;
+
+public record HadithTextChangedEvent(UUID hadithId, String text) {
+}

src/main/java/com/jamil/ahadith/features/search/semantic/repository/EmbeddingStatus.java

@@ -0,0 +1,4 @@
+package com.jamil.ahadith.features.search.semantic.repository;
+
+public record EmbeddingStatus(long totalHadiths, long current, long missing, long stale) {
+}