puller container now working

Author: aronchick

utility_containers/event-puller/.env.example

@@ -5,15 +5,21 @@ AWS_ACCESS_KEY_ID=your-access-key-id
 AWS_SECRET_ACCESS_KEY=your-secret-access-key
 
 # Advanced SQS Configuration (Optional - defaults shown)
-# POLL_INTERVAL=100ms      # Time between SQS polls
-# MAX_MESSAGES=10          # Maximum messages to process in one batch
-# NUM_WORKERS=5            # Number of concurrent workers
-# WS_BATCH_SIZE=100        # Number of messages to batch for websocket
+POLL_INTERVAL=100ms             # Time between SQS polls (default: 100ms)
+MAX_MESSAGES=10                 # Maximum messages per poll (default: 10, max: 10)
+NUM_WORKERS=5                   # Number of concurrent SQS polling workers (default: 5)
+WS_BATCH_SIZE=100               # WebSocket message batch size (default: 100)
+BUFFER_SIZE=1000                # Message channel buffer size (default: 1000)
+MAX_RETRY_ATTEMPTS=5            # Maximum retry attempts for SQS operations (default: 5)
+INITIAL_RETRY_DELAY=100ms       # Initial retry delay for exponential backoff (default: 100ms)
+MAX_RETRY_DELAY=5s              # Maximum retry delay (default: 5s)
+SQS_VISIBILITY_TIMEOUT=30       # Visibility timeout for received messages (default: 30 seconds)
+SQS_WAIT_TIME=0                 # SQS long polling wait time (default: 0, max: 20 seconds)
 
 # Azure Cosmos DB Configuration (Optional)
-# COSMOS_ENDPOINT=https://your-cosmos-account.documents.azure.com:443/
-# COSMOS_KEY=your-cosmos-key
-# COSMOS_DATABASE=your-database-name
-# COSMOS_CONTAINER=your-container-name
-# COSMOS_BATCH_SIZE=100    # Number of messages to batch before writing to Cosmos
-# AZURE_REGION=eastus      # Used for partitioning data in Cosmos DB
+COSMOS_ENDPOINT=https://your-cosmos-account.documents.azure.com:443/
+COSMOS_KEY=your-cosmos-key
+COSMOS_DATABASE=your-database-name
+COSMOS_CONTAINER=your-container-name
+COSMOS_BATCH_SIZE=100          # Number of messages to batch before writing to Cosmos
+AZURE_REGION=eastus            # Used for partitioning data in Cosmos DB

utility_containers/event-puller/Dockerfile

@@ -1,7 +1,39 @@
-FROM alpine:3.19
+# Build stage
+FROM --platform=$BUILDPLATFORM golang:1.23 AS builder
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /build
+
+# Copy go mod and sum files
+COPY go.mod go.sum ./
+
+# Download dependencies
+RUN go mod download
+
+# Copy source code
+COPY . .
+
+# Build the binary
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
+RUN echo "Building for $TARGETPLATFORM on $BUILDPLATFORM" && \
+    case "$TARGETPLATFORM" in \
+    linux/amd64) \
+    CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o event-puller ;; \
+    linux/arm64) \
+    CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w" -o event-puller ;; \
+    esac
+
+# Final stage
+FROM --platform=$TARGETPLATFORM alpine:3.19
 
 # Install necessary dependencies
-RUN apk --no-cache add ca-certificates tzdata
+RUN apk --no-cache add ca-certificates tzdata file
 
 # Create a non-root user and group
 RUN addgroup -g 1000 appuser && \
@@ -10,15 +42,19 @@ RUN addgroup -g 1000 appuser && \
 # Set working directory
 WORKDIR /app
 
-# Copy the pre-built binary
-COPY bin/event-puller .
+# Copy the binary from builder
+COPY --from=builder /build/event-puller .
 
 # Copy static files - maintain backward compatibility
 COPY static/ ./static/
 
 # Copy the Next.js dashboard if it exists
 COPY dashboard/out/ ./dashboard/out/
 
+# Copy and set up the entrypoint script
+COPY entrypoint.sh .
+RUN chmod +x entrypoint.sh
+
 # Set ownership
 RUN chown -R appuser:appuser /app
 
@@ -28,5 +64,5 @@ USER appuser
 # Expose the web UI port
 EXPOSE 8080
 
-# Run the application
-CMD ["./event-puller"]
+# Set the entrypoint
+ENTRYPOINT ["/app/entrypoint.sh"]

utility_containers/event-puller/README.md

@@ -42,15 +42,15 @@ go build -o bin/event-puller
 
 ```bash
 # Pull the latest image
-docker pull bacalhauproject/event-puller:latest
+docker pull bacalhau-project/event-puller:latest
 
 # Run with your environment variables
 docker run -p 8080:8080 \
   -e AWS_ACCESS_KEY_ID=your_key \
   -e AWS_SECRET_ACCESS_KEY=your_secret \
   -e AWS_REGION=your_region \
   -e SQS_QUEUE_URL=your_queue_url \
-  bacalhauproject/event-puller:latest
+  bacalhau-project/event-puller:latest
 ```
 
 ## ⚙️ Configuration
@@ -66,6 +66,18 @@ AWS_SECRET_ACCESS_KEY=your_aws_secret_key
 AWS_REGION=us-east-1
 SQS_QUEUE_URL=https://sqs.us-east-1.amazonaws.com/123456789012/your-queue
 
+# SQS Configuration Options
+POLL_INTERVAL=100ms             # Time between SQS polls (default: 100ms)
+MAX_MESSAGES=10                 # Maximum messages per poll (default: 10, max: 10)
+NUM_WORKERS=5                   # Number of concurrent SQS polling workers (default: 5)
+WS_BATCH_SIZE=100               # WebSocket message batch size (default: 100)
+BUFFER_SIZE=1000                # Message channel buffer size (default: 1000)
+MAX_RETRY_ATTEMPTS=5            # Maximum retry attempts for SQS operations (default: 5)
+INITIAL_RETRY_DELAY=100ms       # Initial retry delay for exponential backoff (default: 100ms)
+MAX_RETRY_DELAY=5s              # Maximum retry delay (default: 5s)
+SQS_VISIBILITY_TIMEOUT=30       # Visibility timeout for received messages (default: 30 seconds)
+SQS_WAIT_TIME=0                 # SQS long polling wait time (default: 0, max: 20 seconds)
+
 # Optional Cosmos DB Variables
 COSMOS_ENDPOINT=https://your-account.documents.azure.com:443/
 COSMOS_KEY=your_cosmos_key
@@ -104,9 +116,9 @@ npm run build
 
 **Solution**:
 - Ensure AWS credentials have `sqs:ReceiveMessage`, `sqs:DeleteMessage`, and `sqs:GetQueueAttributes` permissions
-- Increase `NUM_WORKERS` (line 35 in main.go) for higher throughput
-- Add exponential backoff for transient errors
-- Implement better error handling and retry logic
+- Increase `NUM_WORKERS` environment variable for higher throughput
+- Adjust `POLL_INTERVAL` or `SQS_WAIT_TIME` for better polling behavior 
+- Adjust `MAX_RETRY_ATTEMPTS`, `INITIAL_RETRY_DELAY`, and `MAX_RETRY_DELAY` for better error handling
 
 ### Cosmos DB Integration Issues
 

utility_containers/event-puller/bin/event-puller

@@ -32,8 +32,8 @@ mkdir -p bin
 TIMESTAMP=$(date +%Y%m%d%H%M)
 
 # Registry configuration
-REGISTRY="docker.io"    
-ORGANIZATION="bacalhauproject"      
+REGISTRY="ghcr.io"    
+ORGANIZATION="bacalhau-project"      
 IMAGE_NAME="event-puller"
 TAG="${REGISTRY}/${ORGANIZATION}/${IMAGE_NAME}:${TIMESTAMP}"
 

utility_containers/event-puller/build.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+
+echo "🔍 Starting entrypoint script..."
+
+# System information
+echo "📊 System Information:"
+echo "  Architecture: $(uname -m)"
+echo "  OS: $(uname -s)"
+echo "  Kernel: $(uname -r)"
+echo "  CPU Info: $(cat /proc/cpuinfo | grep "model name" | head -n1 | cut -d':' -f2 || echo "Not available")"
+
+# Check if we're in the right directory
+echo "📂 Current directory: $(pwd)"
+echo "📂 Directory contents:"
+ls -la
+
+# Check if the binary exists and is executable
+if [ ! -f "/app/event-puller" ]; then
+    echo "❌ Error: Binary not found at /app/event-puller"
+    exit 1
+fi
+
+if [ ! -x "/app/event-puller" ]; then
+    echo "❌ Error: Binary at /app/event-puller is not executable"
+    exit 1
+fi
+
+# Check binary architecture
+echo "🔍 Binary information:"
+file /app/event-puller
+
+# Check if ENV_FILE is set
+if [ -z "$ENV_FILE" ]; then
+    echo "❌ Error: ENV_FILE environment variable is not set"
+    echo "Please set ENV_FILE to the path of your environment file"
+    echo "Example: Set ENV_FILE=/app/.env when running the container"
+    exit 1
+fi
+
+echo "📄 ENV_FILE is set to: $ENV_FILE"
+
+# Check if the env file exists
+if [ ! -f "$ENV_FILE" ]; then
+    echo "❌ Error: Environment file not found at $ENV_FILE"
+    echo "Please ensure the environment file exists and is mounted correctly"
+    echo "The environment file should be mounted to the path specified in ENV_FILE"
+    exit 1
+fi
+
+echo "✅ Environment file found at $ENV_FILE"
+echo "📄 Environment file contents (excluding sensitive data):"
+grep -v "KEY\|SECRET\|PASSWORD" "$ENV_FILE" || true
+
+# Display all environment variables (excluding sensitive data)
+echo "🌍 Environment variables (excluding sensitive data):"
+env | grep -v "KEY\|SECRET\|PASSWORD" || true
+
+# Run the binary with additional debugging
+echo "🚀 Starting event-puller..."
+exec /app/event-puller 

utility_containers/event-puller/entrypoint.sh

@@ -3,17 +3,18 @@ module github.com/bacalhauproject/bacalhau-examples/utility_containers/event-pul
 go 1.23.3
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.1
 	github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.3.0
 	github.com/aws/aws-sdk-go v1.55.5
 	github.com/charmbracelet/bubbles v0.20.0
 	github.com/charmbracelet/bubbletea v1.2.4
 	github.com/gorilla/websocket v1.5.3
 	github.com/joho/godotenv v1.5.1
+	github.com/mattn/go-isatty v0.0.20
 )
 
 require (
 	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/charmbracelet/lipgloss v1.0.0 // indirect
@@ -22,7 +23,6 @@ require (
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect