[owl] Bug fix: String column should not have file extension validation (#869)

Backend - owl (API server)

- Bug fix: String column should be able to take in arbitrary strings without URL validation

CI

- Change concurrency limit to be at job level
- Mirror all images to GitHub Registry
- Increase Docker timeouts

Author: jiahuei

.github/workflows/ci.yml

@@ -13,12 +13,6 @@ on:
     tags:
       - "v*"
 
-# Cancel in-progress CI jobs if there is a new push
-# https://stackoverflow.com/a/72408109
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
 jobs:
   check_changes:
     name: Check for changes
@@ -58,6 +52,9 @@ jobs:
         jamai-mode: ["oss"]
         test-group: [group1, group2, group3, group4]
     timeout-minutes: 90
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ matrix.jamai-mode }}-${{ matrix.test-group }}
+      cancel-in-progress: true
 
     steps:
       - name: Checkout code
@@ -148,14 +145,22 @@ jobs:
           OWL_STRIPE_WEBHOOK_SECRET_TEST: ${{ secrets.OWL_STRIPE_WEBHOOK_SECRET_TEST }}
           OWL_STRIPE_PUBLISHABLE_KEY_TEST: ${{ secrets.OWL_STRIPE_PUBLISHABLE_KEY_TEST }}
 
+      - name: Configure Docker timeout
+        run: |
+          mkdir -p ~/.docker
+          echo '{"max-concurrent-downloads": 3, "max-download-attempts": 5}' > ~/.docker/config.json
+
       - name: Launch services
         id: launch_services
         timeout-minutes: 20
         if: always()
-        run: docker compose -p jm -f docker/compose.ci.yml up --quiet-pull -d --wait
+        run: |
+          docker compose -p jm -f docker/compose.ci.yml up --quiet-pull -d --wait
         env:
           COMPOSE_DOCKER_CLI_BUILD: 1
           DOCKER_BUILDKIT: 1
+          COMPOSE_HTTP_TIMEOUT: 30
+          DOCKER_CLIENT_TIMEOUT: 30
 
       - name: Inspect owl logs if failed to launch
         timeout-minutes: 1

clients/python/src/jamaibase/client.py

@@ -3296,7 +3296,7 @@ async def export_table_data(
         Args:
             table_type (str): Table type.
             table_id (str): ID or name of the table to be exported.
-            delimiter (str, optional): The delimiter of the file: can be "," or "\\t". Defaults to ",".
+            delimiter (str, optional): The delimiter of the content can be "," or "\\t". Defaults to ",".
             columns (list[str], optional): A list of columns to be exported. Defaults to None (export all columns).
 
         Returns:
@@ -6130,7 +6130,7 @@ def export_table_data(
         Args:
             table_type (str): Table type.
             table_id (str): ID or name of the table to be exported.
-            delimiter (str, optional): The delimiter of the file: can be "," or "\\t". Defaults to ",".
+            delimiter (str, optional): The delimiter of the content can be "," or "\\t". Defaults to ",".
             columns (list[str], optional): A list of columns to be exported. Defaults to None (export all columns).
 
         Returns:

clients/python/src/jamaibase/types/gen_table.py

@@ -647,7 +647,7 @@ class TableDataImportRequest(BaseModel):
     # ] = None
     delimiter: Annotated[
         Literal[",", "\t"],
-        Field(description='The delimiter of the file: can be "," or "\\t". Defaults to ",".'),
+        Field(description='The delimiter of the content can be "," or "\\t". Defaults to ",".'),
     ] = ","
 
 

docker/compose.base.yml

@@ -19,15 +19,15 @@ services:
       - jamai
 
   otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.113.0
+    image: ghcr.io/embeddedllm/otel/opentelemetry-collector-contrib:0.113.0
     command: ["--config=/etc/otelcol/config.yaml"]
     volumes:
       - ./otel_configs/otel-collector-config.yaml:/etc/otelcol/config.yaml
     networks:
       - jamai
 
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.124.0
+    image: ghcr.io/embeddedllm/victoriametrics/victoria-metrics:v1.124.0
     command:
       - "--selfScrapeInterval=15s"
       - "--retentionPeriod=100y"
@@ -43,7 +43,7 @@ services:
       start_period: 30s
 
   vmauth:
-    image: victoriametrics/vmauth:v1.124.0
+    image: ghcr.io/embeddedllm/victoriametrics/vmauth:v1.124.0
     command:
       - "--auth.config=/etc/config.yml"
     volumes:
@@ -58,7 +58,7 @@ services:
       start_period: 30s
 
   victorialogs:
-    image: victoriametrics/victoria-logs:v1.28.0
+    image: ghcr.io/embeddedllm/victoriametrics/victoria-logs:v1.28.0
     command:
       - "--retentionPeriod=100y"
     volumes:
@@ -73,7 +73,7 @@ services:
       start_period: 30s
 
   vmagent:
-    image: victoriametrics/vmagent:v1.124.0
+    image: ghcr.io/embeddedllm/victoriametrics/vmagent:v1.124.0
     depends_on:
       victoriametrics:
         condition: service_healthy
@@ -97,7 +97,7 @@ services:
       - jamai
 
   clickhouse:
-    image: clickhouse:24.10.2.80
+    image: ghcr.io/embeddedllm/clickhouse:24.10.2.80
     volumes:
       - ../docker_data/ch_data:/var/lib/clickhouse/
       - ../docker_data/ch_logs:/var/log/clickhouse-server/
@@ -157,7 +157,7 @@ services:
       start_period: 30s
 
   pgbouncer:
-    image: edoburu/pgbouncer:v1.24.0-p0
+    image: ghcr.io/embeddedllm/edoburu/pgbouncer:v1.24.0-p0
     depends_on:
       postgresql:
         condition: service_healthy
@@ -187,7 +187,7 @@ services:
       - jamai
 
   minio:
-    image: minio/minio:RELEASE.2025-05-24T17-08-30Z
+    image: ghcr.io/embeddedllm/minio/minio:RELEASE.2025-05-24T17-08-30Z
     entrypoint: /bin/sh -c " minio server /data --console-address ':9001' & until (mc config host add myminio http://localhost:9000 $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD}) do echo '...waiting...' && sleep 1; done; mc mb myminio/file; wait "
     environment:
       MINIO_ROOT_USER: minioadmin

scripts/mirror_docker_image.py

@@ -0,0 +1,169 @@
+"""
+Docker Image Mirror Tool
+Mirrors selected Docker images from source to destination registry using `skopeo`.
+"""
+
+import json
+import subprocess
+import sys
+
+
+def run_command(cmd, capture_output=True):
+    """Run a shell command and return output."""
+    try:
+        if capture_output:
+            result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+            return result.stdout.strip()
+        else:
+            return subprocess.run(cmd, check=True)
+    except subprocess.CalledProcessError as e:
+        print(f"Error: {e.stderr if hasattr(e, 'stderr') else str(e)}")
+        return None
+    except FileNotFoundError:
+        print("Error: 'skopeo' not found. Please install it first.")
+        sys.exit(1)
+
+
+def check_skopeo_installed() -> bool:
+    """Check if skopeo is installed."""
+    try:
+        subprocess.run(["skopeo", "--version"], capture_output=True, check=True)
+        return True
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return False
+
+
+def list_tags(image_path) -> list[str] | None:
+    """List all tags for a given image."""
+    print(f"\nFetching tags from {image_path}...")
+
+    output = run_command(["skopeo", "list-tags", f"docker://{image_path}"])
+    if not output:
+        return None
+
+    try:
+        data = json.loads(output)
+        return [t for t in data.get("Tags", []) if "sha256" not in t]
+    except json.JSONDecodeError as e:
+        print(f"Error parsing tags: {e}")
+        return None
+
+
+def copy_image(source, destination, tag):
+    """Copy a single image tag using skopeo."""
+    if ":" in source:
+        source = source.split(":")[0]
+    if ":" in destination:
+        destination = destination.split(":")[0]
+    source_full = f"docker://{source}:{tag}"
+    dest_full = f"docker://{destination}:{tag}"
+
+    print(f"\nCopying {tag}...")
+    print(f"  From: {source_full}")
+    print(f"  To:   {dest_full}")
+
+    cmd = ["skopeo", "copy", source_full, dest_full]
+    return run_command(cmd, capture_output=False) is not None
+
+
+def main():
+    print("=" * 60)
+    print("Docker Image Mirror Tool")
+    print("=" * 60)
+
+    # Check if skopeo is installed
+    if not check_skopeo_installed():
+        print("\n❌ Error: skopeo is not installed.")
+        print("\nInstallation instructions:")
+        print("  Ubuntu/Debian: sudo apt-get install skopeo")
+        print("  macOS: brew install skopeo")
+        print("  Fedora/RHEL: sudo dnf install skopeo")
+        sys.exit(1)
+
+    print("✅ skopeo is installed")
+
+    # Get source and destination
+    source_image = input("\nEnter source image path (tag is optional): ").strip()
+    if not source_image:
+        print("Error: Source image cannot be empty")
+        sys.exit(1)
+
+    destination_image = input("Enter destination image path (exclude tag): ").strip()
+    if not destination_image:
+        destination_image = f"ghcr.io/embeddedllm/{source_image}"
+        print(f'Defaulting to "{destination_image}"')
+
+    # List available tags
+    if ":" in source_image:
+        src_tag = source_image.split(":")[-1]
+        if ":" in destination_image and src_tag != destination_image.split(":")[-1]:
+            print("Error: When specifying tags in source and destination, they must match")
+            sys.exit(1)
+        tags = [src_tag]
+    else:
+        tags = list_tags(source_image)
+        if not tags:
+            print("Error: No tags found")
+            sys.exit(1)
+    if len(tags) == 1:
+        selected_tags = tags
+    else:
+        print(f"\nAvailable tags ({len(tags)} total):")
+        for tag in tags:
+            print(f"  - {tag}")
+
+        # Get user selection
+        print("\nEnter tags to mirror (comma-separated):")
+        print("Example: latest,v1.0.0,v1.1.0")
+        selection = input("Tags: ").strip()
+
+        if not selection:
+            print("Error: No tags selected")
+            sys.exit(1)
+
+        selected_tags = [tag.strip() for tag in selection.split(",")]
+
+        # Validate selected tags
+        invalid_tags = [tag for tag in selected_tags if tag not in tags]
+        if invalid_tags:
+            print(f"\nError: Invalid tags: {', '.join(invalid_tags)}")
+            sys.exit(1)
+
+    # Confirm
+    print(f"\nWill mirror {len(selected_tags)} tag(s):")
+    for tag in selected_tags:
+        print(f"  - {tag}")
+
+    confirm = input("\nProceed? (Y/n): ").strip().lower()
+    if confirm not in ("", "y"):
+        print("Cancelled")
+        sys.exit(0)
+
+    # Copy images
+    print("\nStarting mirror process...")
+    success_count = 0
+    failed_tags = []
+
+    for idx, tag in enumerate(selected_tags, 1):
+        print(f"\n[{idx}/{len(selected_tags)}]")
+        if copy_image(source_image, destination_image, tag):
+            success_count += 1
+            print(f"✅ Success: {tag}")
+        else:
+            failed_tags.append(tag)
+            print(f"❌ Failed: {tag}")
+
+    # Summary
+    print("\n" + "=" * 60)
+    print(f"Summary: {success_count}/{len(selected_tags)} successful")
+    if failed_tags:
+        print(f"Failed tags: {', '.join(failed_tags)}")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        print("\n\nCancelled by user")
+        sys.exit(1)

services/api/src/owl/types/__init__.py

@@ -1,6 +1,5 @@
 from datetime import datetime
 from enum import StrEnum
-from os.path import splitext
 from typing import Annotated, Any, Generic, Literal, Self, Type, TypeVar
 
 import pandas as pd
@@ -498,28 +497,11 @@ class ColumnRenameRequest(t.ColumnRenameRequest):
 )
 
 
-def check_data(value: Any) -> Any:
-    if isinstance(value, str) and (value.startswith("s3://") or value.startswith("file://")):
-        extension = splitext(value)[1].lower()
-        if extension not in ALLOWED_FILE_EXTENSIONS:
-            raise ValueError(
-                "Unsupported file type. Make sure the file belongs to "
-                "one of the following formats: \n"
-                f"[Image File Types]: \n{IMAGE_FILE_EXTENSIONS} \n"
-                f"[Audio File Types]: \n{AUDIO_FILE_EXTENSIONS} \n"
-                f"[Document File Types]: \n{DOCUMENT_FILE_EXTENSIONS}"
-            )
-    return value
-
-
-CellValue = Annotated[Any, AfterValidator(check_data)]
-
-
 class RowAdd(BaseModel):
     table_id: str = Field(
         description="Table name or ID.",
     )
-    data: dict[str, CellValue] = Field(
+    data: dict[str, Any] = Field(
         description="Mapping of column names to its value.",
     )
     stream: bool = Field(
@@ -533,7 +515,7 @@ class RowAdd(BaseModel):
 
 
 class MultiRowAddRequest(t.MultiRowAddRequest):
-    data: list[dict[str, CellValue]] = Field(
+    data: list[dict[str, Any]] = Field(
         min_length=1,
         description=(
             "List of mapping of column names to its value. "
@@ -544,7 +526,7 @@ class MultiRowAddRequest(t.MultiRowAddRequest):
 
 
 class MultiRowAddRequestWithLimit(MultiRowAddRequest):
-    data: list[dict[str, CellValue]] = Field(
+    data: list[dict[str, Any]] = Field(
         min_length=1,
         max_length=100,
         description=(
@@ -556,22 +538,22 @@ class MultiRowAddRequestWithLimit(MultiRowAddRequest):
 
 
 class MultiRowUpdateRequest(t.MultiRowUpdateRequest):
-    data: dict[str, dict[str, CellValue]] = Field(
+    data: dict[str, dict[str, Any]] = Field(
         min_length=1,
         description="Mapping of row IDs to row data, where each row data is a mapping of column names to its value.",
     )
 
 
 class MultiRowUpdateRequestWithLimit(MultiRowUpdateRequest):
-    data: dict[str, dict[str, CellValue]] = Field(
+    data: dict[str, dict[str, Any]] = Field(
         min_length=1,
         max_length=100,
         description="Mapping of row IDs to row data, where each row data is a mapping of column names to its value.",
     )
 
 
 class RowUpdateRequest(t.RowUpdateRequest):
-    data: dict[str, CellValue] = Field(
+    data: dict[str, Any] = Field(
         description="Mapping of column names to its value.",
     )