feat(coprocessor): add healthcheck to tx-sender

.github/workflows/coprocessor-docker-build-tx-sender.yml

@@ -0,0 +1,54 @@
+name: Docker - Fhevm Coprocessor Transaction Sender
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  release:
+    types:
+      - published
+
+concurrency:
+  group: fhevm-coprocessor-tx-sender-${{ github.ref_name }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+jobs:
+  check-changes:
+    permissions:
+      actions: 'read'
+      contents: 'read'
+      pull-requests: 'read'
+    runs-on: ubuntu-latest
+    outputs:
+      changes-coprocessor-tx-sender: ${{ steps.filter.outputs.coprocessor-tx-sender }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36  # v3.0.2
+        id: filter
+        with:
+          filters: |
+            coprocessor-tx-sender:
+              - .github/workflows/coprocessor-docker-build-tx-sender.yml
+              - coprocessor/fhevm-engine/transaction-sender/**
+              - coprocessor/fhevm-engine/Cargo.toml
+              - coprocessor/fhevm-engine/Cargo.lock
+  docker-fhevm-coprocessor:
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.changes-coprocessor-tx-sender == 'true' || github.event_name == 'release' }}
+    uses: zama-ai/ci-templates/.github/workflows/docker_common.yml@44333c96991d6747e0bef6a3308bfd98b20390f8 # main
+    secrets:
+      GHCR_ACTION_TOKEN: ${{ secrets.BLOCKCHAIN_ACTIONS_TOKEN }}
+      GRAVITON_BUILDER_SSH_PRIVATE_KEY: ${{ secrets.GRAVITON_BUILDER_SSH_PRIVATE_KEY }}
+    permissions:
+      contents: "read"
+      id-token: "write"
+      packages: "write"
+    with:
+      working-directory: "."
+      docker-context: "."
+      push_image: true
+      image-name: "fhevm/coprocessor/tx-sender"
+      generate-dev-image: false
+      docker-file: "coprocessor/fhevm-engine/transaction-sender/Dockerfile"
+      arm-build: true

coprocessor/fhevm-engine/transaction-sender/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "transaction-sender"
-version = "0.6.1"
+version = "0.7.0"
 authors.workspace = true
 edition.workspace = true
 license.workspace = true
@@ -19,7 +19,11 @@ tokio = { workspace = true }
 tokio-util = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
-hex = { workspace = true }
+# Health check related additions
+axum = { workspace = true }
+tower-http = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
 humantime = { workspace = true }
 
 # crates.io dependencies

coprocessor/fhevm-engine/transaction-sender/Dockerfile

@@ -0,0 +1,34 @@
+# Stage 1: Build Transaction Sender
+FROM ghcr.io/zama-ai/fhevm/gci/rust-glibc:1.85.0 AS builder
+
+USER root
+
+WORKDIR /app
+
+COPY coprocessor/fhevm-engine ./coprocessor/fhevm-engine
+COPY coprocessor/proto ./coprocessor/proto
+
+WORKDIR /app/coprocessor/fhevm-engine
+
+# Build transaction_sender binary
+RUN cargo fetch && \
+    SQLX_OFFLINE=true cargo build --release -p transaction-sender
+
+# Stage 3: Runtime image
+FROM cgr.dev/chainguard/glibc-dynamic:latest AS runtime
+
+COPY --from=builder /lib/ /lib/
+COPY --from=builder /bin/ /bin/
+COPY --from=builder /usr/lib/ /usr/lib/
+COPY --from=builder /usr/bin/ /usr/bin/
+COPY --from=builder /etc/group /etc/group
+COPY --from=builder /etc/passwd /etc/passwd
+COPY --from=builder /etc/ssl/certs/ /etc/ssl/certs/
+COPY --from=builder --chown=fhevm:fhevm /home/fhevm /home/fhevm
+COPY --from=builder --chown=fhevm:fhevm /app /app
+
+COPY --from=builder --chown=fhevm:fhevm /app/coprocessor/fhevm-engine/target/release/transaction_sender /usr/local/bin/transaction_sender
+
+USER fhevm:fhevm
+
+CMD ["/usr/local/bin/transaction_sender"]

coprocessor/fhevm-engine/transaction-sender/src/bin/transaction_sender.rs

@@ -11,8 +11,9 @@ use aws_config::BehaviorVersion;
 use clap::{Parser, ValueEnum};
 use tokio::signal::unix::{signal, SignalKind};
 use tokio_util::sync::CancellationToken;
+use tracing::{error, info}; // Add the missing tracing macros
 use transaction_sender::{
-    get_chain_id, make_abstract_signer, AbstractSigner, ConfigSettings,
+    get_chain_id, http_server::HttpServer, make_abstract_signer, AbstractSigner, ConfigSettings,
     FillersWithoutNonceManagement, NonceManagedProvider, TransactionSender,
 };
 
@@ -104,6 +105,13 @@ struct Conf {
 
     #[arg(long, default_value = "4s", value_parser = parse_duration)]
     provider_retry_interval: Duration,
+
+    /// HTTP server port for health checks
+    #[arg(long, default_value_t = 8080)]
+    health_check_port: u16,
+
+    #[arg(long, default_value = "4s", value_parser = parse_duration)]
+    health_check_timeout: Duration,
 }
 
 fn install_signal_handlers(cancel_token: CancellationToken) -> anyhow::Result<()> {
@@ -168,36 +176,64 @@ async fn main() -> anyhow::Result<()> {
             .await?,
         Some(wallet.default_signer().address()),
     );
-    let sender = TransactionSender::new(
+    let config = ConfigSettings {
+        database_url,
+        database_pool_size: conf.database_pool_size,
+        verify_proof_resp_db_channel: conf.verify_proof_resp_database_channel,
+        add_ciphertexts_db_channel: conf.add_ciphertexts_database_channel,
+        allow_handle_db_channel: conf.allow_handle_database_channel,
+        verify_proof_resp_batch_limit: conf.verify_proof_resp_batch_limit,
+        verify_proof_resp_max_retries: conf.verify_proof_resp_max_retries,
+        verify_proof_remove_after_max_retries: conf.verify_proof_remove_after_max_retries,
+        add_ciphertexts_batch_limit: conf.add_ciphertexts_batch_limit,
+        db_polling_interval_secs: conf.database_polling_interval_secs,
+        error_sleep_initial_secs: conf.error_sleep_initial_secs,
+        error_sleep_max_secs: conf.error_sleep_max_secs,
+        add_ciphertexts_max_retries: conf.add_ciphertexts_max_retries,
+        allow_handle_batch_limit: conf.allow_handle_batch_limit,
+        allow_handle_max_retries: conf.allow_handle_max_retries,
+        txn_receipt_timeout_secs: conf.txn_receipt_timeout_secs,
+        required_txn_confirmations: conf.required_txn_confirmations,
+        review_after_unlimited_retries: conf.review_after_unlimited_retries,
+        health_check_port: conf.health_check_port,
+        health_check_timeout: conf.health_check_timeout,
+    };
+    let transaction_sender = TransactionSender::new(
         conf.input_verification_address,
         conf.ciphertext_commits_address,
         conf.multichain_acl_address,
         abstract_signer,
         provider,
         cancel_token.clone(),
-        ConfigSettings {
-            database_url,
-            database_pool_size: conf.database_pool_size,
-            verify_proof_resp_db_channel: conf.verify_proof_resp_database_channel,
-            add_ciphertexts_db_channel: conf.add_ciphertexts_database_channel,
-            allow_handle_db_channel: conf.allow_handle_database_channel,
-            verify_proof_resp_batch_limit: conf.verify_proof_resp_batch_limit,
-            verify_proof_resp_max_retries: conf.verify_proof_resp_max_retries,
-            verify_proof_remove_after_max_retries: conf.verify_proof_remove_after_max_retries,
-            add_ciphertexts_batch_limit: conf.add_ciphertexts_batch_limit,
-            db_polling_interval_secs: conf.database_polling_interval_secs,
-            error_sleep_initial_secs: conf.error_sleep_initial_secs,
-            error_sleep_max_secs: conf.error_sleep_max_secs,
-            add_ciphertexts_max_retries: conf.add_ciphertexts_max_retries,
-            allow_handle_batch_limit: conf.allow_handle_batch_limit,
-            allow_handle_max_retries: conf.allow_handle_max_retries,
-            txn_receipt_timeout_secs: conf.txn_receipt_timeout_secs,
-            required_txn_confirmations: conf.required_txn_confirmations,
-            review_after_unlimited_retries: conf.review_after_unlimited_retries,
-        },
+        config,
         None,
     )
     .await?;
-    install_signal_handlers(cancel_token)?;
-    sender.run().await
+
+    // Wrap the TransactionSender in an Arc
+    let transaction_sender = std::sync::Arc::new(transaction_sender);
+
+    // Create HTTP server with the Arc-wrapped sender
+    let http_server = HttpServer::new(
+        transaction_sender.clone(),
+        conf.health_check_port,
+        cancel_token.clone(),
+    );
+
+    // Run both services concurrently
+    let (sender_result, http_result) = tokio::join!(transaction_sender.run(), http_server.start());
+
+    // Check results
+    if let Err(e) = sender_result {
+        error!("Transaction sender error: {}", e);
+        return Err(e);
+    }
+
+    if let Err(e) = http_result {
+        error!("HTTP server error: {}", e);
+        return Err(e);
+    }
+
+    info!("Transaction sender and HTTP server stopped gracefully");
+    Ok(())
 }

coprocessor/fhevm-engine/transaction-sender/src/config.rs

@@ -0,0 +1,64 @@
+use std::time::Duration;
+
+#[derive(Clone, Debug)]
+pub struct ConfigSettings {
+    pub database_url: String,
+    pub database_pool_size: u32,
+
+    pub verify_proof_resp_db_channel: String,
+    pub add_ciphertexts_db_channel: String,
+    pub allow_handle_db_channel: String,
+
+    pub verify_proof_resp_batch_limit: u32,
+    pub verify_proof_resp_max_retries: u32,
+    pub verify_proof_remove_after_max_retries: bool,
+
+    pub add_ciphertexts_batch_limit: u32,
+    pub add_ciphertexts_max_retries: u32,
+
+    pub allow_handle_batch_limit: u32,
+    pub allow_handle_max_retries: u32,
+
+    pub db_polling_interval_secs: u16,
+
+    pub error_sleep_initial_secs: u16,
+    pub error_sleep_max_secs: u16,
+
+    pub txn_receipt_timeout_secs: u16,
+
+    pub required_txn_confirmations: u16,
+
+    pub review_after_unlimited_retries: u16,
+
+    // Health check related settings
+    pub health_check_port: u16,
+    pub health_check_timeout: Duration,
+}
+
+impl Default for ConfigSettings {
+    fn default() -> Self {
+        Self {
+            database_url: std::env::var("DATABASE_URL")
+                .unwrap_or("postgres://postgres:postgres@localhost/coprocessor".to_owned()),
+            database_pool_size: 10,
+            verify_proof_resp_db_channel: "verify_proof_responses".to_owned(),
+            add_ciphertexts_db_channel: "add_ciphertexts".to_owned(),
+            allow_handle_db_channel: "event_allowed_handle".to_owned(),
+            verify_proof_resp_batch_limit: 128,
+            verify_proof_resp_max_retries: 3,
+            verify_proof_remove_after_max_retries: true,
+            db_polling_interval_secs: 5,
+            error_sleep_initial_secs: 1,
+            error_sleep_max_secs: 16,
+            add_ciphertexts_batch_limit: 10,
+            add_ciphertexts_max_retries: 15,
+            allow_handle_batch_limit: 10,
+            allow_handle_max_retries: 10,
+            txn_receipt_timeout_secs: 10,
+            required_txn_confirmations: 0,
+            review_after_unlimited_retries: 30,
+            health_check_port: 8080,
+            health_check_timeout: Duration::from_secs(4),
+        }
+    }
+}