diff --git a/.github/actions/benchmark/action.yaml b/.github/actions/benchmark/action.yaml index 8a32fc80..54529f7e 100644 --- a/.github/actions/benchmark/action.yaml +++ b/.github/actions/benchmark/action.yaml @@ -7,6 +7,9 @@ inputs: compare: description: results comparison string default: --benchmark-compare=0001 --benchmark-compare-fail=min:25% --benchmark-group-by=fullname --color=yes + datasource: + description: datasource to run on + default: default runs: using: "composite" @@ -49,6 +52,7 @@ runs: python -c "import segyio; import inspect; print('after: segyio loaded from ' + inspect.getfile(segyio))" - name: Make segy file + if: ${{ inputs.datasource == 'default' }} shell: bash working-directory: python run: | @@ -57,6 +61,7 @@ runs: else echo "file.sgy already exists" fi + ls -lh file.sgy - name: Make segy small file shell: bash @@ -67,19 +72,31 @@ runs: else echo "file-small.sgy already exists" fi + ls -lh file-small.sgy - - name: Print filesize + - name: Run benchmark tests + if: ${{ inputs.datasource == 'default' }} shell: bash working-directory: python run: | - ls -lh file.sgy - ls -lh file-small.sgy + pytest test/benchmarks.py -rP --benchmark-sort=name --benchmark-autosave ${{ inputs.compare }} - - name: Run benchmark tests + - name: Run benchmark tests on datasource + if: ${{ inputs.datasource != 'default' }} shell: bash working-directory: python run: | - pytest test/benchmarks.py -rP --benchmark-sort=name --benchmark-autosave ${{ inputs.compare }} + pytest \ + test/benchmarks.py::test_read_speed \ + test/benchmarks.py::test_cube_speed \ + test/benchmarks.py::test_write_file \ + -rP \ + --datasource ${{ inputs.datasource }} \ + --readf file-small.sgy \ + --writef file-small.sgy \ + --benchmark-sort=name \ + --benchmark-autosave \ + ${{ inputs.compare }} - name: Remove build artifacts shell: bash diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml index 2c1b9744..13042fb0 100644 --- a/.github/workflows/benchmarks.yaml +++ b/.github/workflows/benchmarks.yaml @@ -10,11 +10,45 @@ on: commit_ref: description: "optional: compare to specific ref" required: false + run_on_default: + type: boolean + description: Run on default file + default: true + run_on_stream: + type: boolean + description: Run on pyfile stream + default: false + run_on_memory: + type: boolean + description: Run on memory datasource + default: false + compare_to_self: + description: "True: compare to older version of the same datasource. False: compare to standard C-file." + default: true + type: boolean jobs: benchmarks: - name: Benchmark scripts + name: Benchmark script on ${{ matrix.datasource }} datasource runs-on: ubuntu-latest + strategy: + fail-fast: false + # matrix set up to combine running manually and on PR + matrix: + datasource: [default, pyfile, nativememory] + run_on_default: + - ${{ github.event.inputs.run_on_default }} + run_on_stream: + - ${{ github.event.inputs.run_on_stream }} + run_on_memory: + - ${{ github.event.inputs.run_on_memory }} + exclude: + - datasource: default + run_on_default: "false" + - datasource: pyfile + run_on_stream: "false" + - datasource: nativememory + run_on_memory: "false" steps: - uses: actions/checkout@v4 @@ -30,19 +64,35 @@ jobs: python3 -m pip install -r requirements-dev.txt - name: Set default benchmark reference - run: echo "BENCHMARK_REF=142e45a2b941a1b603723c38882b193db1c1d968" >> $GITHUB_ENV + run: | + if [ "${{ matrix.datasource }}" = "default" ]; then + echo "BENCHMARK_REF=142e45a2b941a1b603723c38882b193db1c1d968" >> $GITHUB_ENV + else + echo "BENCHMARK_REF=2b3be755d8e1bd536275a9cbcf16c06717b30bd0" >> $GITHUB_ENV + fi - name: Set benchmark reference if defined by workflow_dispatch if: github.event.inputs.commit_ref != '' run: echo "BENCHMARK_REF=${{ github.event.inputs.commit_ref }}" >> $GITHUB_ENV + - name: Set base datasource for comparison + run: | + if [ "${{ github.event.inputs.compare_to_self }}" = "false" ]; then + echo "BASE_DATASOURCE=default" >> $GITHUB_ENV + else + # also the case when compare_to_self = "" because run is trigger by PR. + echo "BASE_DATASOURCE=${{ matrix.datasource }}" >> $GITHUB_ENV + fi + - name: Benchmark old commit uses: "./.github/actions/benchmark" with: ref: ${{ env.BENCHMARK_REF }} + datasource: ${{ env.BASE_DATASOURCE }} compare: "" - name: Benchmark current commit and compare uses: "./.github/actions/benchmark" with: ref: ${{ github.sha }} + datasource: ${{ matrix.datasource }} diff --git a/python/test/benchmarks.py b/python/test/benchmarks.py index f310627a..f6bead41 100644 --- a/python/test/benchmarks.py +++ b/python/test/benchmarks.py @@ -7,18 +7,6 @@ import numpy as np -# Files are expected to be present at the run location -# Files size should be compatible with retrieved lines -read_files = [ - 'file.sgy', - 'file-small.sgy', -] - -write_files = [ - 'file-small.sgy', -] - - def run(filepath, mmap, func, mode="r"): with segyio.open(filepath, mode=mode) as f: if mmap: @@ -38,9 +26,13 @@ def run_in_memory(memory_buffer, func): def run_with(filepath, mode, mmap, func, make_datasource): filepath, memory_buffer, stream = make_datasource(filepath, mode) - if stream and "open_with" in dir(segyio): + if stream: + if "open_with" not in dir(segyio): + raise RuntimeError("segyio.open_with is not available") run_with_stream(stream, func) - elif memory_buffer and "open_from_memory" in dir(segyio): + elif memory_buffer: + if "open_from_memory" not in dir(segyio): + raise RuntimeError("segyio.open_from_memory is not available") run_in_memory(memory_buffer, func) else: run(filepath, mmap, func, mode) @@ -162,21 +154,18 @@ def create(output_file): @pytest.mark.benchmark(group="nommap") -@pytest.mark.parametrize("read_file", read_files) @pytest.mark.parametrize("func", operations) def test_read_speed(make_datasource, benchmark, read_file, func): benchmark(run_with, read_file, "rb", False, func, make_datasource) @pytest.mark.benchmark(group="with mmap") -@pytest.mark.parametrize("read_file", read_files) @pytest.mark.parametrize("func", operations) def test_mmap_read_speed(benchmark, read_file, func): benchmark(run, read_file, True, func) @pytest.mark.benchmark(group="cube") -@pytest.mark.parametrize("read_file", read_files) def test_cube_speed(make_datasource, benchmark, read_file): benchmark.pedantic( run_with, rounds=5, args=[read_file, "rb", False, cube, make_datasource] @@ -184,7 +173,6 @@ def test_cube_speed(make_datasource, benchmark, read_file): @pytest.mark.benchmark(group="write") -@pytest.mark.parametrize("write_file", write_files) @pytest.mark.parametrize("func", write_operations) def test_write_file(make_datasource, benchmark, write_file, func): # note that original file will get overwritten diff --git a/python/test/conftest.py b/python/test/conftest.py index e30af8e4..822a1d0b 100644 --- a/python/test/conftest.py +++ b/python/test/conftest.py @@ -159,3 +159,34 @@ def pytest_addoption(parser): default="default", help="Known datasource in testing. Options: pyfile, pymemory, nativememory." ) + parser.addoption( + "--readf", + action="append", + default=[], + help=("Files used for benchmark read tests. " + "Files size should be compatible with retrieved lines. " + "Option may be used multiple times.") + ) + + parser.addoption( + "--writef", + action="append", + default=[], + help=("Files used for benchmark write tests (files will get modified)." + "File size should be compatible with modified lines." + "Option may be be used multiple times.") + ) + +def pytest_generate_tests(metafunc): + # default files are expected to be present at the run location if options are not overridden + if "read_file" in metafunc.fixturenames: + read_files = metafunc.config.getoption("readf") + if not read_files: + read_files = ['file.sgy', 'file-small.sgy'] + metafunc.parametrize("read_file", read_files) + + if "write_file" in metafunc.fixturenames: + write_files = metafunc.config.getoption("writef") + if not write_files: + write_files = ['file-small.sgy'] + metafunc.parametrize("write_file", write_files)