Add protobuf 6 support

Author: vgvoleg

.github/workflows/style.yaml

@@ -2,10 +2,12 @@ name: Style checks
 
 on:
   push:
+    branches:
+      - main
   pull_request:
 
 jobs:
-  build:
+  lint:
     runs-on: ubuntu-latest
     strategy:
       max-parallel: 4

.github/workflows/tests.yaml

@@ -1,4 +1,4 @@
-name: Functional tests
+name: Tests
 
 on:
   push:
@@ -7,26 +7,18 @@ on:
   pull_request:
 
 jobs:
-  build:
+  integration:
     runs-on: ubuntu-latest
 
     concurrency:
-      group: unit-${{ github.ref }}-${{ matrix.environment }}-${{ matrix.python-version }}-${{ matrix.folder }}
+      group: integration-${{ github.ref }}-${{ matrix.python-version }}-${{ matrix.protobuf-version }}
       cancel-in-progress: true
 
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9]
-        environment: [py, py-tls, py-proto4, py-tls-proto4, py-proto3, py-tls-proto3]
-        folder: [ydb, tests]
-        exclude:
-          - environment: py-tls
-            folder: ydb
-          - environment: py-tls-proto4
-            folder: ydb
-          - environment: py-tls-proto3
-            folder: ydb
+        python-version: ["3.8", "3.9", "3.10"]
+        protobuf-version: [proto3, proto4, proto5, proto6]
 
     steps:
     - uses: actions/checkout@v1
@@ -41,4 +33,31 @@ jobs:
         pip install tox==4.2.6
 
     - name: Run unit tests
-      run: tox -e ${{ matrix.environment }} -- ${{ matrix.folder }}
+      run: tox -e py-${{ matrix.protobuf-version }} -- tests
+
+  unit:
+    runs-on: ubuntu-latest
+
+    concurrency:
+      group: unit-${{ github.ref }}-${{ matrix.python-version }}
+      cancel-in-progress: true
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install tox
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox==4.2.6
+
+    - name: Run unit tests
+      run: tox -e py -- ydb

Makefile

@@ -10,4 +10,8 @@ protobuf-4:
 
 protobuf-5:
 	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env-5 --build-arg PY_PROTOBUF_VER=5.26.1 --build-arg PROTOC_VER=26.1
-	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env-5 python generate_protoc.py --target-version=v5
+	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env-5 python generate_protoc.py --target-version=v5
+
+protobuf-6:
+	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env-6 --build-arg GRPCIO_VER=1.76.0 --build-arg PY_PROTOBUF_VER=6.30.2 --build-arg PROTOC_VER=30.2
+	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env-6 python generate_protoc.py --target-version=v6

docker-compose-tls.yml

@@ -1,11 +1,14 @@
-version: "3.3"
+version: "3.9"
 services:
   ydb:
     image: ydbplatform/local-ydb:trunk
     restart: always
     ports:
       - 2136:2136
+      - 2135:2135
     hostname: localhost
+    volumes:
+    - ./ydb_certs:/ydb_certs
     environment:
       - YDB_ENABLE_COLUMN_TABLES=true
       - YDB_USE_IN_MEMORY_PDISKS=true

docker-compose.yml

@@ -16,7 +16,7 @@ jsonschema==3.2.0
 packaging==21.0
 paramiko==2.10.1
 pluggy==0.13.1
-protobuf>=3.13.0,<6.0.0
+protobuf>=3.13.0,<7.0.0
 py==1.10.0
 pycparser==2.20
 PyNaCl==1.4.0

test-requirements.txt

@@ -12,3 +12,4 @@ def test_connect_secure(secure_endpoint, database):
     ) as driver:
         driver.wait(timeout=10)
         driver.scheme_client.make_directory("/local/test")
+        driver.scheme_client.remove_directory("/local/test")

tests/ssl/test_ssl.py

@@ -78,6 +78,9 @@ def test_tx_snapshot_ro(driver_sync, database):
     )
     assert data[0].rows == [{"value": 2}]
 
+    with suppress(ydb.issues.SchemeError):
+        session.drop_table(tb_name)
+
 
 def test_split_transactions_deny_split(driver_sync, table_name):
     with ydb.SessionPool(driver_sync, 1) as pool:

tests/table/test_tx.py

@@ -12,84 +12,50 @@ setenv =
 deps =
     -r{toxinidir}/test-requirements.txt
 
-[testenv:dev-proto5]
+[testenv:py]
 commands =
+    pytest -v {posargs}
 deps =
     -r{toxinidir}/test-requirements.txt
-    protobuf<6.0.0
+    protobuf<7.0.0
 
-[testenv:dev-proto4]
+[testenv:py-proto6]
 commands =
+    pytest -v {posargs}
 deps =
     -r{toxinidir}/test-requirements.txt
-    protobuf<5.0.0
+    protobuf<7.0.0
 
-[testenv:dev-proto3]
+[testenv:py-proto5]
 commands =
-deps =
-    -r{toxinidir}/test-requirements.txt
-    protobuf<4.0.0
-
-[testenv:py]
-commands =
-    pytest -v -m "not tls" {posargs}
+    pytest -v {posargs}
 deps =
     -r{toxinidir}/test-requirements.txt
     protobuf<6.0.0
 
 [testenv:py-proto4]
 commands =
-    pytest -v -m "not tls" {posargs}
-deps =
-    -r{toxinidir}/test-requirements.txt
-    protobuf<5.0.0
-
-[testenv:py-cov-proto4]
-commands =
-    pytest -v -m "not tls" \
-        --cov-report html:cov_html --cov=ydb \
-        {posargs}
+    pytest -v {posargs}
 deps =
     -r{toxinidir}/test-requirements.txt
     protobuf<5.0.0
 
 [testenv:py-proto3]
 commands =
-    pytest -v -m "not tls" {posargs}
-deps =
-    -r{toxinidir}/test-requirements.txt
-    protobuf<4.0.0
-
-[testenv:py-tls]
-commands =
-    pytest -v -m tls --docker-compose-file=docker-compose-tls.yml {posargs}
-deps =
-    -r{toxinidir}/test-requirements.txt
-    protobuf<6.0.0
-
-[testenv:py-tls-proto4]
-commands =
-    pytest -v -m tls --docker-compose-file=docker-compose-tls.yml {posargs}
-deps =
-    -r{toxinidir}/test-requirements.txt
-    protobuf<5.0.0
-
-[testenv:py-tls-proto3]
-commands =
-    pytest -v -m tls --docker-compose-file=docker-compose-tls.yml {posargs}
+    pytest -v {posargs}
 deps =
     -r{toxinidir}/test-requirements.txt
     protobuf<4.0.0
 
 [testenv:black-format]
 skip_install = true
 commands =
-    black ydb examples tests --extend-exclude "ydb/_grpc/v3|ydb/_grpc/v4|ydb/_grpc/v5"
+    black ydb examples tests --extend-exclude "ydb/_grpc/v3|ydb/_grpc/v4|ydb/_grpc/v5|ydb/_grpc/v6"
 
 [testenv:black]
 skip_install = true
 commands =
-    black --diff --check ydb examples tests --extend-exclude "ydb/_grpc/v3|ydb/_grpc/v4|ydb/_grpc/v5"
+    black --diff --check ydb examples tests --extend-exclude "ydb/_grpc/v3|ydb/_grpc/v4|ydb/_grpc/v5|ydb/_grpc/v6"
 
 [testenv:pylint]
 deps = pylint

tox.ini

@@ -45,11 +45,20 @@
 
         sys.modules["ydb._grpc.common.protos"] = sys.modules["ydb._grpc.v4.protos"]
 
-    else:
+    elif protobuf_version < Version("6.0"):
         from ydb._grpc.v5 import *  # noqa
 
         sys.modules["ydb._grpc.common"] = sys.modules["ydb._grpc.v5"]
 
         from ydb._grpc.v5 import protos  # noqa
 
         sys.modules["ydb._grpc.common.protos"] = sys.modules["ydb._grpc.v5.protos"]
+
+    else:
+        from ydb._grpc.v6 import *  # noqa
+
+        sys.modules["ydb._grpc.common"] = sys.modules["ydb._grpc.v6"]
+
+        from ydb._grpc.v6 import protos  # noqa
+
+        sys.modules["ydb._grpc.common.protos"] = sys.modules["ydb._grpc.v6.protos"]