Add datasource benchmark tests

Tests run on PR are regression tests - each datasource's performance is
compared with base reference.

Because memory datasource operates on memory only, datasources run was
limited to only smallest file (261M), excluding the 7.5G one. To support
that list of read/write files was moved to fixture.

Manually workflow gives more freedom. There is an option to run
stream/memory datasource against our default one to see how different
datasources performance compares.

Author: achaikou

.github/actions/benchmark/action.yaml

@@ -7,6 +7,9 @@ inputs:
   compare:
     description: results comparison string
     default: --benchmark-compare=0001 --benchmark-compare-fail=min:25% --benchmark-group-by=fullname --color=yes
+  datasource:
+    description: datasource to run on
+    default: default
 
 runs:
   using: "composite"
@@ -76,11 +79,29 @@ runs:
         ls -lh file-small.sgy
 
     - name: Run benchmark tests
+      if: ${{ inputs.datasource == 'default' }}
       shell: bash
       working-directory: python
       run: |
         pytest test/benchmarks.py -rP --benchmark-sort=name --benchmark-autosave ${{ inputs.compare }}
 
+    - name: Run benchmark tests on datasource
+      if: ${{ inputs.datasource != 'default' }}
+      shell: bash
+      working-directory: python
+      run: |
+        pytest \
+          test/benchmarks.py::test_read_speed \
+          test/benchmarks.py::test_cube_speed \
+          test/benchmarks.py::test_write_file \
+          -rP \
+          --datasource ${{ inputs.datasource }} \
+          --readf file-small.sgy \
+          --writef file-small.sgy \
+          --benchmark-sort=name \
+          --benchmark-autosave \
+          ${{ inputs.compare }}
+
     - name: Remove build artifacts
       shell: bash
       run: |

.github/workflows/benchmarks.yaml

@@ -10,11 +10,45 @@ on:
       commit_ref:
         description: "optional: compare to specific ref"
         required: false
+      run_on_default:
+        type: boolean
+        description: Run on default file
+        default: true
+      run_on_stream:
+        type: boolean
+        description: Run on pyfile stream
+        default: false
+      run_on_memory:
+        type: boolean
+        description: Run on memory datasource
+        default: false
+      compare_to_self:
+        description: "True: compare to older version of the same datasource. False: compare to standard C-file."
+        default: true
+        type: boolean
 
 jobs:
   benchmarks:
-    name: Benchmark scripts
+    name: Benchmark script on ${{ matrix.datasource }} datasource
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      # matrix set up to combine running manually and on PR
+      matrix:
+        datasource: [default, pyfile, nativememory]
+        run_on_default:
+          - ${{ github.event.inputs.run_on_default }}
+        run_on_stream:
+          - ${{ github.event.inputs.run_on_stream }}
+        run_on_memory:
+          - ${{ github.event.inputs.run_on_memory }}
+        exclude:
+          - datasource: default
+            run_on_default: "false"
+          - datasource: pyfile
+            run_on_stream: "false"
+          - datasource: nativememory
+            run_on_memory: "false"
 
     steps:
       - uses: actions/checkout@v4
@@ -30,19 +64,35 @@ jobs:
           python3 -m pip install -r requirements-dev.txt
 
       - name: Set default benchmark reference
-        run: echo "BENCHMARK_REF=142e45a2b941a1b603723c38882b193db1c1d968" >> $GITHUB_ENV
+        run: |
+          if [ "${{ matrix.datasource }}" = "default" ]; then
+            echo "BENCHMARK_REF=142e45a2b941a1b603723c38882b193db1c1d968" >> $GITHUB_ENV
+          else
+            echo "BENCHMARK_REF=2b3be755d8e1bd536275a9cbcf16c06717b30bd0" >> $GITHUB_ENV
+          fi
 
       - name: Set benchmark reference if defined by workflow_dispatch
         if: github.event.inputs.commit_ref != ''
         run: echo "BENCHMARK_REF=${{ github.event.inputs.commit_ref }}" >> $GITHUB_ENV
 
+      - name: Set base datasource for comparison
+        run: |
+          if [ "${{ github.event.inputs.compare_to_self }}" = "false" ]; then
+            echo "BASE_DATASOURCE=default" >> $GITHUB_ENV
+          else
+            # also the case when compare_to_self = "" because run is trigger by PR.
+            echo "BASE_DATASOURCE=${{ matrix.datasource }}" >> $GITHUB_ENV
+          fi
+
       - name: Benchmark old commit
         uses: "./.github/actions/benchmark"
         with:
           ref: ${{ env.BENCHMARK_REF }}
+          datasource: ${{ env.BASE_DATASOURCE }}
           compare: ""
 
       - name: Benchmark current commit and compare
         uses: "./.github/actions/benchmark"
         with:
           ref: ${{ github.sha }}
+          datasource: ${{ matrix.datasource }}

python/test/benchmarks.py

@@ -7,18 +7,6 @@
 import numpy as np
 
 
-# Files are expected to be present at the run location
-# Files size should be compatible with retrieved lines
-read_files = [
-    'file.sgy',
-    'file-small.sgy',
-]
-
-write_files = [
-    'file-small.sgy',
-]
-
-
 def run(filepath, mmap, func, mode="r"):
     with segyio.open(filepath, mode=mode) as f:
         if mmap:
@@ -38,9 +26,13 @@ def run_in_memory(memory_buffer, func):
 
 def run_with(filepath, mode, mmap, func, make_datasource):
     filepath, memory_buffer, stream = make_datasource(filepath, mode)
-    if stream and "open_with" in dir(segyio):
+    if stream:
+        if "open_with" not in dir(segyio):
+            raise RuntimeError("segyio.open_with is not available")
         run_with_stream(stream, func)
-    elif memory_buffer and "open_from_memory" in dir(segyio):
+    elif memory_buffer:
+        if "open_from_memory" not in dir(segyio):
+            raise RuntimeError("segyio.open_from_memory is not available")
         run_in_memory(memory_buffer, func)
     else:
         run(filepath, mmap, func, mode)
@@ -162,29 +154,25 @@ def create(output_file):
 
 
 @pytest.mark.benchmark(group="nommap")
-@pytest.mark.parametrize("read_file", read_files)
 @pytest.mark.parametrize("func", operations)
 def test_read_speed(make_datasource, benchmark, read_file, func):
     benchmark(run_with, read_file, "rb", False, func, make_datasource)
 
 
 @pytest.mark.benchmark(group="with mmap")
-@pytest.mark.parametrize("read_file", read_files)
 @pytest.mark.parametrize("func", operations)
 def test_mmap_read_speed(benchmark, read_file, func):
     benchmark(run, read_file, True, func)
 
 
 @pytest.mark.benchmark(group="cube")
-@pytest.mark.parametrize("read_file", read_files)
 def test_cube_speed(make_datasource, benchmark, read_file):
     benchmark.pedantic(
         run_with, rounds=5, args=[read_file, "rb", False, cube, make_datasource]
     )
 
 
 @pytest.mark.benchmark(group="write")
-@pytest.mark.parametrize("write_file", write_files)
 @pytest.mark.parametrize("func", write_operations)
 def test_write_file(make_datasource, benchmark, write_file, func):
     # note that original file will get overwritten

python/test/conftest.py

@@ -159,3 +159,34 @@ def pytest_addoption(parser):
         default="default",
         help="Known datasource in testing. Options: pyfile, pymemory, nativememory."
     )
+    parser.addoption(
+        "--readf",
+        action="append",
+        default=[],
+        help=("Files used for benchmark read tests. "
+              "Files size should be compatible with retrieved lines. "
+              "Option may be used multiple times.")
+    )
+
+    parser.addoption(
+        "--writef",
+        action="append",
+        default=[],
+        help=("Files used for benchmark write tests (files will get modified)."
+              "File size should be compatible with modified lines."
+              "Option may be be used multiple times.")
+    )
+
+def pytest_generate_tests(metafunc):
+    # default files are expected to be present at the run location if options are not overridden
+    if "read_file" in metafunc.fixturenames:
+        read_files = metafunc.config.getoption("readf")
+        if not read_files:
+            read_files = ['file.sgy', 'file-small.sgy']
+        metafunc.parametrize("read_file", read_files)
+
+    if "write_file" in metafunc.fixturenames:
+        write_files = metafunc.config.getoption("writef")
+        if not write_files:
+            write_files = ['file-small.sgy']
+        metafunc.parametrize("write_file", write_files)