Sync from private

Author: jiahuei

.env.example

@@ -1,32 +1,65 @@
 # External API keys
-OPENAI_API_KEY=
-ANTHROPIC_API_KEY=
-COHERE_API_KEY=
-TOGETHER_API_KEY=
-HYPERBOLIC_API_KEY=
-CEREBRAS_API_KEY=
-SAMBANOVA_API_KEY=
-DEEPSEEK_API_KEY=
-CUSTOM_API_KEY=
+OWL_ANTHROPIC_API_KEY=
+OWL_AZURE_API_KEY=
+OWL_AZURE_AI_API_KEY=
+OWL_BEDROCK_API_KEY=
+OWL_CEREBRAS_API_KEY=
+OWL_COHERE_API_KEY=
+OWL_DEEPSEEK_API_KEY=
+OWL_ELLM_API_KEY=
+OWL_GEMINI_API_KEY=
+OWL_GROQ_API_KEY=
+OWL_HYPERBOLIC_API_KEY=
+OWL_JINA_AI_API_KEY=
+OWL_OPENAI_API_KEY=
+OWL_OPENROUTER_API_KEY=
+OWL_SAGEMAKER_API_KEY=
+OWL_SAMBANOVA_API_KEY=
+OWL_TOGETHER_AI_API_KEY=
+OWL_VERTEX_AI_API_KEY=
+OWL_VOYAGE_API_KEY=
+OWL_STRIPE_API_KEY=
+OWL_STRIPE_PUBLISHABLE_KEY_LIVE=
+OWL_STRIPE_PUBLISHABLE_KEY_TEST=
+OWL_STRIPE_WEBHOOK_SECRET_LIVE=
+OWL_STRIPE_WEBHOOK_SECRET_TEST=
+OWL_AUTH0_API_KEY=
 
-# Service URLs
-DOCIO_URL=http://docio:6979/api/docio
-UNSTRUCTUREDIO_URL=http://unstructuredio:6989
-JAMAI_API_BASE=http://owl:6969/api
+# CI
+JAMAI_TOKEN=
+JAMAI_API_BASE=http://localhost:6969/api
 
-# Frontend config
-JAMAI_URL=http://owl:6969
-PUBLIC_JAMAI_URL=
-PUBLIC_IS_SPA=false
-CHECK_ORIGIN=false
+# Service connection (dev)
+# OWL_DB_PATH=postgresql+psycopg://owlpguser:owlpgpassword@localhost:5432/jamaibase_owl
+# OWL_CLICKHOUSE_HOST=localhost
+# OWL_CLICKHOUSE_PORT=8123
+# OWL_OPENTELEMETRY_HOST=localhost
+# OWL_OPENTELEMETRY_PORT=4317
+# OWL_REDIS_HOST=localhost
+# OWL_REDIS_PORT=6379
+# OWL_VICTORIA_METRICS_HOST=localhost
+# OWL_VICTORIA_LOGS_HOST=localhost
+# OWL_CODE_EXECUTOR_ENDPOINT=http://localhost:5569
+# OWL_DOCLING_URL=http://localhost:5001
+# OWL_TEST_LLM_API_BASE=http://localhost:6970/v1
+# OWL_S3_ENDPOINT=http://localhost:9000
+# OWL_FILE_PROXY_URL=website-url
 
 # Configuration
 OWL_PORT=6969
 OWL_WORKERS=3
-DOCIO_WORKERS=1
-DOCIO_DEVICE=cpu
-EMBEDDING_MODEL=BAAI/bge-small-en-v1.5
-RERANKER_MODEL=mixedbread-ai/mxbai-rerank-xsmall-v1
 OWL_CONCURRENT_ROWS_BATCH_SIZE=5
 OWL_CONCURRENT_COLS_BATCH_SIZE=5
 OWL_MAX_WRITE_BATCH_SIZE=1000
+PB_MAX_CLIENT_CONN=500
+PB_MAX_CLIENT_CONN=80
+PG_MAX_CONNECTIONS=100
+
+# Frontend config
+HOST=localhost
+ORIGIN=http://localhost:4000
+AUTH_SECRET="changeme"
+OWL_URL=http://owl:6969
+PUBLIC_JAMAI_URL=
+PUBLIC_IS_SPA=false
+CHECK_ORIGIN=false

.gitattributes

@@ -33,12 +33,18 @@
 *.db binary
 *.doc binary
 *.docx binary
+*.gif binary
 *.gz binary
+*.heic* binary
+*.heif* binary
 *.jar binary
 *.jpeg binary
 *.jpg binary
+*.mov binary
+*.mp* binary
 *.npy binary
 *.npz binary
+*.parquet binary
 *.pcd binary
 *.pdf binary
 *.pkl binary
@@ -47,13 +53,16 @@
 *.pptx binary
 *.pth binary
 *.so binary
+*.ttf binary
+*.webp binary
 *.xls binary
 *.xlsx binary
 *.zip binary
 
+# Track with LFS
+# *.pth filter=lfs diff=lfs merge=lfs -text
+# *.parquet filter=lfs diff=lfs merge=lfs -text
+
 # These files should not be processed by Linguist for language detection on GitHub.com
 *.p linguist-detectable=false
 *.gz linguist-detectable=false
-
-# Track with Git LFS
-*.parquet filter=lfs diff=lfs merge=lfs -text

.github/workflows/cd.yml

@@ -0,0 +1,51 @@
+name: CD - Build and Push Docker Images
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build_and_push:
+    name: Build and Push Docker Images
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Prepare HCL file
+        run: |
+          # Convert repository owner to lowercase
+          REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
+
+          # Get short SHA
+          SHORT_SHA=$(echo ${{ github.sha }} | cut -c1-7)
+
+          # Replace variables in HCL file
+          sed -i "s|AZURE_STORAGE_ACCOUNT_URL|${{ secrets.AZURE_STORAGE_ACCOUNT_URL }}|g" docker/compose.bake.hcl
+          sed -i "s|AZURE_STORAGE_ACCESS_KEY|${{ secrets.AZURE_STORAGE_ACCESS_KEY }}|g" docker/compose.bake.hcl
+
+          # Add tags to HCL file
+          sed -i "/target \"owl\"/a \  tags = [\"ghcr.io/${REPO_OWNER}/jamaibase-owl:latest\", \"ghcr.io/${REPO_OWNER}/jamaibase-owl:${SHORT_SHA}\"]" docker/compose.bake.hcl
+          sed -i "/target \"jambu\"/a \  tags = [\"ghcr.io/${REPO_OWNER}/jamaibase-jambu:latest\", \"ghcr.io/${REPO_OWNER}/jamaibase-jambu:${SHORT_SHA}\"]" docker/compose.bake.hcl
+
+      - name: Build and Push Images
+        run: |
+          # Build and push JamaiBase image with both latest and commit hash
+          docker buildx bake --file docker/compose.bake.hcl --push