Add datasource benchmark tests

.github/actions/benchmark/action.yaml

@@ -7,6 +7,9 @@ inputs:
   compare:
     description: results comparison string
     default: --benchmark-compare=0001 --benchmark-compare-fail=min:25% --benchmark-group-by=fullname --color=yes
+  datasource:
+    description: datasource to run on
+    default: default
 
 runs:
   using: "composite"
@@ -76,11 +79,29 @@ runs:
         ls -lh file-small.sgy
 
     - name: Run benchmark tests
+      if: ${{ inputs.datasource == 'default' }}
       shell: bash
       working-directory: python
       run: |
         pytest test/benchmarks.py -rP --benchmark-sort=name --benchmark-autosave ${{ inputs.compare }}
 
+    - name: Run benchmark tests on datasource
+      if: ${{ inputs.datasource != 'default' }}
+      shell: bash
+      working-directory: python
+      run: |
+        pytest \
+          test/benchmarks.py::test_read_speed \
+          test/benchmarks.py::test_cube_speed \
+          test/benchmarks.py::test_write_file \
+          -rP \
+          --datasource ${{ inputs.datasource }} \
+          --readf file-small.sgy \
+          --writef file-small.sgy \
+          --benchmark-sort=name \
+          --benchmark-autosave \
+          ${{ inputs.compare }}
+
     - name: Remove build artifacts
       shell: bash
       run: |

.github/workflows/benchmarks.yaml

@@ -10,11 +10,45 @@ on:
       commit_ref:
         description: "optional: compare to specific ref"
         required: false
+      run_on_default:
+        type: boolean
+        description: Run on default file
+        default: true
+      run_on_stream:
+        type: boolean
+        description: Run on pyfile stream
+        default: false
+      run_on_memory:
+        type: boolean
+        description: Run on memory datasource
+        default: false
+      compare_to_self:
+        description: "True: compare to older version of the same datasource. False: compare to standard C-file."
+        default: true
+        type: boolean
 
 jobs:
   benchmarks:
-    name: Benchmark scripts
+    name: Benchmark script on ${{ matrix.datasource }} datasource
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      # matrix set up to combine running manually and on PR
+      matrix:
+        datasource: [default, pyfile, nativememory]
+        run_on_default:
+          - ${{ github.event.inputs.run_on_default }}
+        run_on_stream:
+          - ${{ github.event.inputs.run_on_stream }}
+        run_on_memory:
+          - ${{ github.event.inputs.run_on_memory }}
+        exclude:
+          - datasource: default
+            run_on_default: "false"
+          - datasource: pyfile
+            run_on_stream: "false"
+          - datasource: nativememory
+            run_on_memory: "false"
 
     steps:
       - uses: actions/checkout@v4
@@ -30,19 +64,35 @@ jobs:
           python3 -m pip install -r requirements-dev.txt
 
       - name: Set default benchmark reference
-        run: echo "BENCHMARK_REF=142e45a2b941a1b603723c38882b193db1c1d968" >> $GITHUB_ENV
+        run: |
+          if [ "${{ matrix.datasource }}" = "default" ]; then
+            echo "BENCHMARK_REF=142e45a2b941a1b603723c38882b193db1c1d968" >> $GITHUB_ENV
+          else
+            echo "BENCHMARK_REF=2b3be755d8e1bd536275a9cbcf16c06717b30bd0" >> $GITHUB_ENV
+          fi
 
       - name: Set benchmark reference if defined by workflow_dispatch
         if: github.event.inputs.commit_ref != ''
         run: echo "BENCHMARK_REF=${{ github.event.inputs.commit_ref }}" >> $GITHUB_ENV
 
+      - name: Set base datasource for comparison
+        run: |
+          if [ "${{ github.event.inputs.compare_to_self }}" = "false" ]; then
+            echo "BASE_DATASOURCE=default" >> $GITHUB_ENV
+          else
+            # also the case when compare_to_self = "" because run is trigger by PR.
+            echo "BASE_DATASOURCE=${{ matrix.datasource }}" >> $GITHUB_ENV
+          fi
+
       - name: Benchmark old commit
         uses: "./.github/actions/benchmark"
         with:
           ref: ${{ env.BENCHMARK_REF }}
+          datasource: ${{ env.BASE_DATASOURCE }}
           compare: ""
 
       - name: Benchmark current commit and compare
         uses: "./.github/actions/benchmark"
         with:
           ref: ${{ github.sha }}
+          datasource: ${{ matrix.datasource }}

python/test/benchmarks.py

@@ -7,18 +7,6 @@
 import numpy as np
 
 
-# Files are expected to be present at the run location
-# Files size should be compatible with retrieved lines
-read_files = [
-    'file.sgy',
-    'file-small.sgy',
-]
-
-write_files = [
-    'file-small.sgy',
-]
-
-
 def run(filepath, mmap, func, mode="r"):
     with segyio.open(filepath, mode=mode) as f:
         if mmap:
@@ -38,9 +26,13 @@ def run_in_memory(memory_buffer, func):
 
 def run_with(filepath, mode, mmap, func, make_datasource):
     filepath, memory_buffer, stream = make_datasource(filepath, mode)
-    if stream and "open_with" in dir(segyio):
+    if stream:
+        if "open_with" not in dir(segyio):
+            raise RuntimeError("segyio.open_with is not available")
         run_with_stream(stream, func)
-    elif memory_buffer and "open_from_memory" in dir(segyio):
+    elif memory_buffer:
+        if "open_from_memory" not in dir(segyio):
+            raise RuntimeError("segyio.open_from_memory is not available")
         run_in_memory(memory_buffer, func)
     else:
         run(filepath, mmap, func, mode)
@@ -162,29 +154,25 @@ def create(output_file):
 
 
 @pytest.mark.benchmark(group="nommap")
-@pytest.mark.parametrize("read_file", read_files)
 @pytest.mark.parametrize("func", operations)
 def test_read_speed(make_datasource, benchmark, read_file, func):
     benchmark(run_with, read_file, "rb", False, func, make_datasource)
 
 
 @pytest.mark.benchmark(group="with mmap")
-@pytest.mark.parametrize("read_file", read_files)
 @pytest.mark.parametrize("func", operations)
 def test_mmap_read_speed(benchmark, read_file, func):
     benchmark(run, read_file, True, func)
 
 
 @pytest.mark.benchmark(group="cube")
-@pytest.mark.parametrize("read_file", read_files)
 def test_cube_speed(make_datasource, benchmark, read_file):
     benchmark.pedantic(
         run_with, rounds=5, args=[read_file, "rb", False, cube, make_datasource]
     )
 
 
 @pytest.mark.benchmark(group="write")
-@pytest.mark.parametrize("write_file", write_files)
 @pytest.mark.parametrize("func", write_operations)
 def test_write_file(make_datasource, benchmark, write_file, func):
     # note that original file will get overwritten

python/test/conftest.py

@@ -159,3 +159,34 @@ def pytest_addoption(parser):
         default="default",
         help="Known datasource in testing. Options: pyfile, pymemory, nativememory."
     )
+    parser.addoption(
+        "--readf",
+        action="append",
+        default=[],
+        help=("Files used for benchmark read tests. "
+              "Files size should be compatible with retrieved lines. "
+              "Option may be used multiple times.")
+    )
+
+    parser.addoption(
+        "--writef",
+        action="append",
+        default=[],
+        help=("Files used for benchmark write tests (files will get modified)."
+              "File size should be compatible with modified lines."
+              "Option may be be used multiple times.")
+    )
+
+def pytest_generate_tests(metafunc):
+    # default files are expected to be present at the run location if options are not overridden
+    if "read_file" in metafunc.fixturenames:
+        read_files = metafunc.config.getoption("readf")
+        if not read_files:
+            read_files = ['file.sgy', 'file-small.sgy']
+        metafunc.parametrize("read_file", read_files)
+
+    if "write_file" in metafunc.fixturenames:
+        write_files = metafunc.config.getoption("writef")
+        if not write_files:
+            write_files = ['file-small.sgy']
+        metafunc.parametrize("write_file", write_files)