kgo: share-group test hardening, txn error recovery, misc fixes #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: lint and test | |
| on: | |
| push: | |
| branches: ["master"] | |
| paths-ignore: | |
| - '**/*.md' | |
| pull_request: | |
| branches: ["*"] | |
| paths-ignore: | |
| - '**/*.md' | |
| concurrency: | |
| group: ${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| golangci: | |
| if: github.repository == 'twmb/franz-go' | |
| runs-on: ubuntu-latest | |
| name: "golangci-lint" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: 'stable' | |
| - run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest | |
| # kfake library code is constrained to the published franz-go dep (the | |
| # test-kfake job verifies that with `go test` against the pinned | |
| # version). kfake's *test* files legitimately exercise HEAD kgo APIs | |
| # (Ack, ShareGroup, FlushAcks); golangci-lint typechecks tests too, | |
| # so we need a workspace to resolve those. | |
| - run: go work init . pkg/kadm pkg/kfake pkg/kmsg pkg/sr pkg/sasl/kerberos | |
| - run: ./lint.sh | |
| test-kfake: | |
| if: github.repository == 'twmb/franz-go' | |
| runs-on: ubuntu-latest | |
| name: "test kfake" | |
| container: golang:latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: cd pkg/kfake && go work init . ../.. ../kadm ../kmsg && go test ./... && go test -race ./... | |
| test-sr: | |
| if: github.repository == 'twmb/franz-go' | |
| runs-on: ubuntu-latest | |
| name: "test sr" | |
| container: golang:latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: cd pkg/sr && go work init ../.. . && go test ./... && go test -race ./... | |
| integration-test-kfake: | |
| if: github.repository == 'twmb/franz-go' | |
| runs-on: ubuntu-latest | |
| name: "integration test kfake ${{ matrix.version }}${{ matrix.data-dir && ' (persist)' || '' }}" | |
| container: golang:latest | |
| strategy: | |
| matrix: | |
| include: | |
| - version: "0.11.0" | |
| data-dir: "" | |
| - version: "1.1" | |
| data-dir: "/tmp/kfake_data" | |
| - version: "2.8" | |
| data-dir: "" | |
| - version: "3.9" | |
| data-dir: "/tmp/kfake_data" | |
| - version: "4.1" | |
| data-dir: "" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Start kfake and run tests | |
| run: | | |
| (cd pkg/kfake && go work init . ../.. ../kadm ../kmsg && go run main.go --as-version ${{ matrix.version }} ${{ matrix.data-dir && format('--data-dir {0}', matrix.data-dir) || '' }} -c group.consumer.heartbeat.interval.ms=1000 -l debug) > /tmp/kfake.log 2>&1 & | |
| sleep 2 | |
| go test -timeout 5m ./... | |
| env: | |
| KGO_TEST_RF: 1 | |
| KGO_SEEDS: 127.0.0.1:9092 | |
| - name: Upload kfake logs | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: kfake-logs-${{ matrix.version }}${{ matrix.data-dir && '-persist' || '' }} | |
| path: /tmp/kfake.log | |
| integration-test-kafka: | |
| if: github.repository == 'twmb/franz-go' | |
| runs-on: ubuntu-latest | |
| name: "integration test kafka" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: 'stable' | |
| - name: Create Kafka configuration files | |
| run: | | |
| mkdir -p kafka-config | |
| cat > kafka-config/jaas.conf << 'EOF' | |
| KafkaServer { | |
| org.apache.kafka.common.security.scram.ScramLoginModule required; | |
| }; | |
| EOF | |
| cat > docker-compose.yml << 'EOF' | |
| services: | |
| kafka: | |
| image: apache/kafka:latest | |
| hostname: kafka | |
| ports: | |
| - "9092:9092" | |
| - "9094:9094" | |
| environment: | |
| KAFKA_NODE_ID: 1 | |
| CLUSTER_ID: MkU3OEVBNTcwNTJENDM1Tk | |
| KAFKA_PROCESS_ROLES: controller,broker | |
| KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER | |
| KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,SASL_PLAINTEXT://:9094 | |
| KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT | |
| KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 | |
| KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092,SASL_PLAINTEXT://127.0.0.1:9094 | |
| KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 | |
| KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 | |
| KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 | |
| KAFKA_GROUP_COORDINATOR_REBALANCE_PROTOCOLS: classic,consumer,share | |
| KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 1 | |
| KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 1 | |
| KAFKA_SHARE_COORDINATOR_STATE_TOPIC_NUM_PARTITIONS: 5 | |
| KAFKA_SASL_ENABLED_MECHANISMS: SCRAM-SHA-256 | |
| KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf" | |
| volumes: | |
| - ./kafka-config/jaas.conf:/etc/kafka/jaas.conf:ro | |
| healthcheck: | |
| test: ["CMD", "/opt/kafka/bin/kafka-broker-api-versions.sh", "--bootstrap-server", "localhost:9092"] | |
| interval: 5s | |
| timeout: 5s | |
| retries: 20 | |
| EOF | |
| - name: Start Kafka | |
| run: docker compose up -d --wait | |
| - name: Create SCRAM user | |
| run: | | |
| docker compose exec -T kafka /opt/kafka/bin/kafka-configs.sh \ | |
| --bootstrap-server localhost:9092 \ | |
| --alter --add-config 'SCRAM-SHA-256=[iterations=4096,password=testpass]' \ | |
| --entity-type users --entity-name testuser | |
| - run: go test -timeout 5m ./... | |
| env: | |
| KGO_TEST_RF: 1 | |
| KGO_SEEDS: 127.0.0.1:9092 | |
| KGO_TEST_STABLE_FETCH: true | |
| KGO_TEST_SCRAM_SEEDS: 127.0.0.1:9094 | |
| KGO_TEST_SCRAM: testuser:testpass | |
| - name: Cleanup | |
| if: always() | |
| run: docker compose down -v | |
| integration-test-kafka-38: | |
| if: github.repository == 'twmb/franz-go' | |
| runs-on: ubuntu-latest | |
| name: "integration test kafka 3.8" | |
| container: golang:latest | |
| services: | |
| zookeeper: | |
| image: bitnamilegacy/zookeeper:latest | |
| ports: | |
| - 2181:2181 | |
| env: | |
| ALLOW_ANONYMOUS_LOGIN: yes | |
| kafka: | |
| image: bitnamilegacy/kafka:3.8 | |
| ports: | |
| - 9092:9092 | |
| env: | |
| KAFKA_BROKER_ID: 1 | |
| KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 | |
| KAFKA_CFG_LISTENERS: PLAINTEXT://:9092 | |
| KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 | |
| ALLOW_PLAINTEXT_LISTENER: yes | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: go test -timeout 5m ./... | |
| env: | |
| KGO_TEST_RF: 1 | |
| KGO_SEEDS: kafka:9092 | |
| KGO_TEST_STABLE_FETCH: true | |
| integration-test-redpanda: | |
| if: github.repository == 'twmb/franz-go' | |
| runs-on: ubuntu-latest | |
| name: "integration test redpanda" | |
| container: golang:latest | |
| services: | |
| redpanda: | |
| image: redpandadata/redpanda | |
| ports: | |
| - 9092:9092 | |
| env: | |
| REDPANDA_ADVERTISE_KAFKA_ADDRESS: redpanda:9092 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: go test -timeout 5m ./... | |
| env: | |
| KGO_TEST_RF: 1 | |
| KGO_SEEDS: redpanda:9092 |