arcane-stream-microsoft-synapse-link/docker-compose.yaml at main · SneaksAndData/arcane-stream-microsoft-synapse-link · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
version: '3.3'

services:
  minio:
    container_name: minio
    hostname: minio-test
    network_mode: host
    image: quay.io/minio/minio
    restart: always
    command:
      - server
      - /data
      - "--console-address"
      - ":9001"
    healthcheck:
      test: [ "CMD", "curl", "-f", "http://localhost:9001" ]
      interval: 30s
      timeout: 10s
      retries: 5
  prepare_buckets:
    container_name: minio-setup
    image: quay.io/minio/minio
    network_mode: host
    depends_on:
      minio:
        condition: service_healthy
    entrypoint:
      - "/bin/sh"
      - "-c"
      - |
        mc alias set e2e "http://localhost:9000" minioadmin minioadmin
        mc admin info e2e
        mc mb e2e/tmp && mc mb e2e/lakehouse

  azurite:
    image: mcr.microsoft.com/azure-storage/azurite
    restart: always
    network_mode: host
    command:
      - "azurite-blob"
      - "--blobHost"
      - "localhost"
      - "--blobPort"
      - "10001"
      - "--skipApiVersionCheck"
  trino:
    depends_on:
      lakekeeper:
        condition: service_healthy
    network_mode: host
    image: "trinodb/trino:474"
    volumes:
      - ./integration-tests.properties:/etc/trino/catalog/iceberg.properties
  lakekeeper:
    image: quay.io/lakekeeper/catalog:v0.9.2
    network_mode: host
    environment:
#      - LAKEKEEPER__PG_ENCRYPTION_KEY=This-is-NOT-Secure!
      - LAKEKEEPER__LISTEN_PORT=20001
      - LAKEKEEPER__METRICS_PORT=20002
      - LAKEKEEPER__BASE_URI=http://localhost:20001
      - ICEBERG_REST__BASE_URI=http://localhost:20001
      - LAKEKEEPER__PG_DATABASE_URL_READ=postgresql://postgres:postgres@localhost:5432/postgres
      - LAKEKEEPER__PG_DATABASE_URL_WRITE=postgresql://postgres:postgres@localhost:5432/postgres
      - RUST_LOG=trace,axum=trace,sqlx=trace,iceberg-catalog=trace
    command: [ "serve" ]
    healthcheck:
      test: [ "CMD", "/home/nonroot/lakekeeper", "healthcheck" ]
      interval: 1s
      timeout: 10s
      retries: 30
    depends_on:
      lakekeeper_migrate:
        condition: service_completed_successfully
      lakekeeper_db:
        condition: service_healthy
      minio:
        condition: service_healthy
  lakekeeper_migrate:
    image: quay.io/lakekeeper/catalog:v0.9.2
    network_mode: host
    environment:
#      - LAKEKEEPER__PG_ENCRYPTION_KEY=This-is-NOT-Secure!
      - LAKEKEEPER__PG_DATABASE_URL_READ=postgresql://postgres:postgres@localhost:5432/postgres
      - LAKEKEEPER__PG_DATABASE_URL_WRITE=postgresql://postgres:postgres@localhost:5432/postgres
      - RUST_LOG=info
    restart: "no"
    command: [ "migrate" ]
    depends_on:
      lakekeeper_db:
        condition: service_healthy
  lakekeeper_prepare:
    image: python:3.11-slim-bookworm
    network_mode: host
    environment:
      CATALOG_HOST: localhost
      CATALOG_WAREHOUSE: demo
    depends_on:
      lakekeeper:
        condition: service_healthy
    restart: "no"
    volumes:
      - ./bootstrap-lk.py:/bootstrap-lk.py
    command: [ "/bin/sh", "-c", "pip install requests && python /bootstrap-lk.py"  ]
  lakekeeper_db:
    image: postgres:16.3-bullseye
    network_mode: host
    restart: always
    environment:
      - POSTGRES_USERNAME=postgres
      - POSTGRES_PASSWORD=postgres
      - POSTGRES_HOST_AUTH_METHOD=trust
      - POSTGRES_DB=postgres
      - PGHOST=localhost
      - PGPORT=5432
    healthcheck:
      test: [ "CMD-SHELL", "pg_isready -U postgres -p 5432 -d postgres" ]
      interval: 2s
      timeout: 30s
      retries: 8