Merge pull request #4409 from makr-code/copilot/create-training-folder

Add training materials from presentation and examples

Author: makr-code

include/api/themisdb_grpc_service.h

@@ -31,6 +31,13 @@ class RocksDBWrapper;
 class TransactionManager;
 } // namespace themis
 
+// Interface forward declarations (always available without proto stubs)
+namespace themis {
+class IQueryEngine;
+class IVectorIndex;
+using IQueryEnginePtr = std::shared_ptr<IQueryEngine>;
+} // namespace themis
+
 namespace themis {
 namespace api {
 
@@ -45,7 +52,7 @@ namespace api {
  *
  * Lifecycle – register with GrpcApiServer before calling start():
  * @code
- *   ThemisDBGrpcService svc(db, txn_mgr);
+ *   ThemisDBGrpcService svc(db, txn_mgr, aql_engine, vector_index);
  *   grpc_api_server.registerService(svc.service());  // nullptr when stubs absent
  *   grpc_api_server.start();
  * @endcode
@@ -56,10 +63,16 @@ namespace api {
  *   - AQL           : ExecuteAQL, StreamAQL (server-side streaming)
  *   - Vector search : VectorSearch, FilteredVectorSearch, HybridSearch, FullTextSearch
  *   - Health        : HealthCheck
+ *
+ * When an AQL engine and/or vector index are provided via the extended
+ * constructor the corresponding RPC stubs delegate to them rather than
+ * returning UNIMPLEMENTED.  See ThemisDBGrpcServiceFactory for a fluent
+ * builder that wires all components together.
  */
 class ThemisDBGrpcService {
 public:
     /**
+     * @brief Construct with storage only (AQL and vector search return UNIMPLEMENTED).
      * @param db       Storage backend (must outlive this object).
      * @param txn_mgr  Transaction manager (must outlive this object).
      */
@@ -68,6 +81,24 @@ class ThemisDBGrpcService {
         std::shared_ptr<TransactionManager> txn_mgr
     );
 
+    /**
+     * @brief Construct with all components wired in.
+     *
+     * AQL engine enables ExecuteAQL, StreamAQL, HybridSearch, and FullTextSearch.
+     * Vector index enables VectorSearch and FilteredVectorSearch.
+     *
+     * @param db           Storage backend (must outlive this object).
+     * @param txn_mgr      Transaction manager (must outlive this object).
+     * @param aql_engine   Query engine for AQL RPCs (may be nullptr).
+     * @param vector_index Vector similarity index (may be nullptr).
+     */
+    ThemisDBGrpcService(
+        std::shared_ptr<RocksDBWrapper>          db,
+        std::shared_ptr<TransactionManager>      txn_mgr,
+        std::shared_ptr<themis::IQueryEngine>    aql_engine,
+        std::shared_ptr<themis::IVectorIndex>    vector_index
+    );
+
     ~ThemisDBGrpcService();
 
     /**
@@ -80,11 +111,16 @@ class ThemisDBGrpcService {
     void* service();
 
 private:
-    std::shared_ptr<RocksDBWrapper>     db_;
-    std::shared_ptr<TransactionManager> txn_mgr_;
+    std::shared_ptr<RocksDBWrapper>             db_;
+    std::shared_ptr<TransactionManager>         txn_mgr_;
+    std::shared_ptr<themis::IQueryEngine>       aql_engine_;
+    std::shared_ptr<themis::IVectorIndex>       vector_index_;
 
     class Impl;
     std::unique_ptr<Impl> impl_;
+
+    /// Internal helper used by both constructors.
+    void buildImpl();
 };
 
 } // namespace api

include/api/themisdb_grpc_service_factory.h

@@ -0,0 +1,88 @@
+#pragma once
+
+#include "api/themisdb_grpc_service.h"
+#include <memory>
+
+// Forward declarations
+namespace themis {
+class RocksDBWrapper;
+class TransactionManager;
+class IQueryEngine;
+class IVectorIndex;
+} // namespace themis
+
+namespace themis {
+namespace api {
+
+/**
+ * @brief Fluent factory for building a fully-wired ThemisDBGrpcService.
+ *
+ * Usage:
+ * @code
+ *   auto svc = ThemisDBGrpcServiceFactory{}
+ *                  .withDb(my_db)
+ *                  .withTxnMgr(my_txn_mgr)
+ *                  .withQueryEngine(my_aql_engine)
+ *                  .withVectorIndex(my_vector_index)
+ *                  .build();
+ *   grpc_server.registerService(svc->service());
+ * @endcode
+ *
+ * All components are optional.  When a component is not provided the
+ * corresponding RPC stubs return grpc::StatusCode::UNIMPLEMENTED.
+ *
+ * This factory is the recommended way to assemble a ThemisDBGrpcService
+ * because it makes the dependency set explicit and allows incremental wiring
+ * as components become available.
+ */
+class ThemisDBGrpcServiceFactory {
+public:
+    ThemisDBGrpcServiceFactory() = default;
+
+    /// Set the storage backend.
+    ThemisDBGrpcServiceFactory& withDb(
+        std::shared_ptr<RocksDBWrapper> db) {
+        db_ = std::move(db);
+        return *this;
+    }
+
+    /// Set the transaction manager.
+    ThemisDBGrpcServiceFactory& withTxnMgr(
+        std::shared_ptr<TransactionManager> txn_mgr) {
+        txn_mgr_ = std::move(txn_mgr);
+        return *this;
+    }
+
+    /// Set the AQL engine (enables ExecuteAQL, StreamAQL, HybridSearch, FullTextSearch).
+    ThemisDBGrpcServiceFactory& withQueryEngine(
+        std::shared_ptr<themis::IQueryEngine> engine) {
+        aql_engine_ = std::move(engine);
+        return *this;
+    }
+
+    /// Set the vector index (enables VectorSearch and FilteredVectorSearch).
+    ThemisDBGrpcServiceFactory& withVectorIndex(
+        std::shared_ptr<themis::IVectorIndex> index) {
+        vector_index_ = std::move(index);
+        return *this;
+    }
+
+    /// Build and return the configured ThemisDBGrpcService.
+    std::unique_ptr<ThemisDBGrpcService> build() const {
+        return std::make_unique<ThemisDBGrpcService>(
+            db_,
+            txn_mgr_,
+            aql_engine_,
+            vector_index_
+        );
+    }
+
+private:
+    std::shared_ptr<RocksDBWrapper>         db_;
+    std::shared_ptr<TransactionManager>     txn_mgr_;
+    std::shared_ptr<themis::IQueryEngine>   aql_engine_;
+    std::shared_ptr<themis::IVectorIndex>   vector_index_;
+};
+
+} // namespace api
+} // namespace themis

include/query/query_federation.h

@@ -23,11 +23,14 @@
 #pragma once
 
 #include "sharding/shard_router.h"
+#include "sharding/urn_resolver.h"
 #include "sharding/sharding_manager.h"
 #include "query/query_optimizer.h"
 #include <string>
 #include <vector>
 #include <memory>
+#include <optional>
+#include <atomic>
 #include <nlohmann/json.hpp>
 
 namespace themis::query {
@@ -223,6 +226,18 @@ class QueryFederation {
      * @return Query metadata (tables, predicates, etc.)
      */
     struct QueryMetadata {
+        // ── Shard-key predicate ──────────────────────────────────────────────
+        // Populated by analyzeQuery() when it detects a _key == <value> or
+        // _key >= <min> AND _key <= <max> predicate, enabling partition pruning.
+        struct ShardKeyPredicate {
+            enum class Kind { POINT, RANGE };
+            Kind kind;
+            std::string collection;
+            std::string key_value;   // used when kind == POINT
+            std::string key_min;     // used when kind == RANGE
+            std::string key_max;     // used when kind == RANGE
+        };
+
         std::vector<std::string> tables;
         std::vector<std::string> predicates;
         std::vector<std::string> projections;
@@ -231,6 +246,9 @@ class QueryFederation {
         std::optional<std::string> order_by;
         std::optional<uint64_t> limit;
         std::optional<uint64_t> offset;
+
+        // Shard-key routing hint (nullopt → no routing hint, use scatter-gather)
+        std::optional<ShardKeyPredicate> shard_key_predicate;
 
         // Extended for adaptive capability-based routing
         std::string query_text;               // Original query text

include/sharding/consistent_hash.h

@@ -121,6 +121,28 @@ class ConsistentHashRing {
      * @return List of unique shard IDs
      */
     std::vector<std::string> getAllShards() const;
+
+    /**
+     * Get all shards responsible for any key whose hash lies in [hash_start, hash_end].
+     *
+     * Performs a clockwise ring walk from hash_start to hash_end, collecting every
+     * distinct shard ID encountered.  When hash_start > hash_end the range wraps
+     * around the ring and all shards are returned.
+     *
+     * @param hash_start  Lower bound of the hash range (inclusive)
+     * @param hash_end    Upper bound of the hash range (inclusive)
+     * @return Ordered, deduplicated list of shard IDs (may be all shards)
+     */
+    std::vector<std::string> getShardsInRange(uint64_t hash_start, uint64_t hash_end) const;
+
+    /**
+     * Hash an arbitrary string key using the internal FNV-1a + mix function.
+     * Exposed so callers can compute consistent hash positions without duplicating
+     * the hash logic.
+     * @param key  String to hash
+     * @return 64-bit hash value
+     */
+    uint64_t hashKey(const std::string& key) const { return hash(key); }
 
     /**
      * Calculate balance factor (standard deviation of virtual nodes per shard)

include/sharding/shard_router.h

@@ -159,7 +159,30 @@ class ShardRouter {
      * @param query Query to execute
      * @return Merged results from all shards
      */
-    std::vector<ShardResult> scatterGather(const std::string& query);
+    virtual std::vector<ShardResult> scatterGather(const std::string& query);
+
+    /**
+     * Execute a query on a specific subset of shards.
+     *
+     * Behaves identically to scatterGather but only contacts the shards
+     * whose IDs appear in @p shard_ids.  Unknown or unhealthy shard IDs are
+     * skipped with a WARN log rather than causing an error.
+     *
+     * @param query     AQL query string
+     * @param shard_ids Shard identifiers to target
+     * @return Results from the targeted shards (success + failure entries)
+     */
+    virtual std::vector<ShardResult> executeOnShards(
+        const std::string& query,
+        const std::vector<std::string>& shard_ids
+    );
+
+    /**
+     * Access the URN resolver to perform key-based shard lookups.
+     * @return Reference to the resolver
+     */
+    URNResolver& getResolver() { return *resolver_; }
+    const URNResolver& getResolver() const { return *resolver_; }
 
     /**
      * Execute cross-shard join (simplified two-phase approach)

include/sharding/urn_resolver.h

@@ -89,6 +89,36 @@ class URNResolver {
      * @return Shard ID string, or empty if ring is empty
      */
     std::string getShardId(const URN& urn) const;
+
+    /**
+     * Get the single shard responsible for an arbitrary partition key.
+     *
+     * The key is hashed with the same FNV-1a + mix64 function used by the
+     * ring, then looked up with a clockwise walk.
+     *
+     * @param collection  Collection name (for logging / future topology hints)
+     * @param key         Partition key value (e.g. document _key)
+     * @return Shard ID, or empty string if the ring is empty
+     */
+    std::string getShardForKey(const std::string& collection, const std::string& key) const;
+
+    /**
+     * Get all shards that could own keys in the range [min_key, max_key].
+     *
+     * Hashes both endpoints and collects every shard whose virtual-node
+     * token falls within that hash range (clockwise walk).  For wrap-around
+     * ranges every shard is returned.
+     *
+     * @param collection  Collection name
+     * @param min_key     Inclusive lower bound of the key range
+     * @param max_key     Inclusive upper bound of the key range
+     * @return Deduplicated list of shard IDs; never empty if the ring is not empty
+     */
+    std::vector<std::string> getShardsForKeyRange(
+        const std::string& collection,
+        const std::string& min_key,
+        const std::string& max_key
+    ) const;
 
     /**
      * Get all Shards in cluster