diff --git a/conftest.py b/conftest.py index 9cfdb055a..deffc6e59 100644 --- a/conftest.py +++ b/conftest.py @@ -129,6 +129,13 @@ def pytest_addoption(parser: pytest.Parser) -> None: help="Maximum amount of data written to a volume." " Accepts sizes like '1GiB', '2.5TiB', or symbolic values 'VHD_MAX', 'QCOW2_MAX'." ) + parser.addoption( + "--write-volume-align", + action="store", + default="1", + help="Block size to align span positions to when writing in volumes." + " Accepts sizes like '512', '4KiB', '1MiB'. A value of 1 is equivalent to no alignment." + ) def pytest_configure(config: pytest.Config) -> None: global_config.ignore_ssh_banner = config.getoption('--ignore-ssh-banner') @@ -141,6 +148,9 @@ def pytest_configure(config: pytest.Config) -> None: write_volume_cap = config.getoption('--write-volume-cap') assert write_volume_cap is not None global_config.write_volume_cap = parse_size(write_volume_cap) + write_volume_align = config.getoption('--write-volume-align') + assert write_volume_align is not None + global_config.write_volume_align = parse_size(write_volume_align) def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: if "vm_ref" in metafunc.fixturenames: diff --git a/lib/config.py b/lib/config.py index 0f9a76514..3c0752a87 100644 --- a/lib/config.py +++ b/lib/config.py @@ -4,6 +4,7 @@ ssh_output_max_lines = 20 volume_size = 1 * GiB write_volume_cap = 2 * GiB +write_volume_align = 1 def sr_device_config(datakey: str, *, required: list[str] = []) -> dict[str, str]: import data # import here to avoid depending on this user file for collecting tests diff --git a/tests/storage/storage.py b/tests/storage/storage.py index 0cd46fcec..611915516 100644 --- a/tests/storage/storage.py +++ b/tests/storage/storage.py @@ -369,6 +369,64 @@ def validate(self, vm: VM, dev: str) -> None: expected_flags = f'--expected-checksum {self.checksum}' if self.checksum is not None else '' randstream(vm, f'validate {expected_flags} --position {self.position} --size {self.size} {dev}') +def compute_span_layout(dev_size: int, total_size: int, num_spans: int, block_size: int) -> list[tuple[int, int]]: + """ + Compute the positions and sizes of spans across a device. + + Spans are distributed such that the first span starts at 0 and the last + span ends at dev_size. Positions are computed right-to-left: each span's + start is rounded UP to the nearest multiple of align so the span sits + entirely within its slot. Bytes skipped by rounding are carried leftward + to the previous span, which absorbs them in its budget. + + When total_size equals dev_size, spans are contiguous and cover the full + device with no gaps. When total_size is less than dev_size, spans are + evenly spread across the device with gaps between them. + + Args: + dev_size: Total device size in bytes + total_size: Total number of bytes to write across all spans. + Use dev_size for full device coverage, or a smaller value + to leave gaps between spans. + num_spans: Number of spans to create + block_size: Block size in bytes to align span starts to. + All span starts are rounded up to the nearest multiple of align. + Bytes skipped by the rounding are carried to the previous span. + + Returns: + List of (position, size) tuples, one per span. + Spans are guaranteed to: + - Not overlap + - Have first span starting at position 0 + - Have last span ending at dev_size + - Sum of span sizes equals total_size + """ + per_span = total_size // num_spans + result: list[tuple[int, int]] = [] + # Seed the remainder into the last span so it is distributed via carry + carry = total_size % num_spans + end = dev_size + for i in range(num_spans - 1, -1, -1): + budget = per_span + carry + if i == 0: + # First span always starts at 0; any leftover space before the + # next span becomes a gap (partial coverage only) + start = 0 + size = min(budget, end) + else: + # Round start UP: this may shrink the span relative to budget; + # the trimmed bytes are carried left to the previous span + ideal_start = end - budget + start = ((ideal_start + block_size - 1) // block_size) * block_size + size = end - start + carry = budget - size + result.append((start, size)) + end = start + result.reverse() + assert sum(s for _, s in result) == total_size + return result + + def partially_populate_device(vm: VM, dev_path: str, dev_size: int, num_spans: int = 3, skip_spans: list[int] = []) \ -> list[StreamSpan]: """ @@ -402,55 +460,25 @@ def partially_populate_device(vm: VM, dev_path: str, dev_size: int, num_spans: i skip_spans: List of span indices to skip (no data generated). Skipped spans still exist in returned list with checksum=None. (default: []) + Span alignment is controlled by the --write-volume-align pytest option. Returns: List of StreamSpan objects representing the generated spans. - Spans are guaranteed to: - - Not overlap - - Have first span at position 0 - - Have last span ending at dev_size - - Be evenly distributed across device """ logging.info(f"Generate {dev_path} content") - stream_size = min(dev_size, config.write_volume_cap) // num_spans + total_size = min(dev_size, config.write_volume_cap) # Validate skip_spans assert all(0 <= i < num_spans for i in skip_spans), \ f"Invalid span index in skip_spans: must be 0 <= i < {num_spans}" + layout = compute_span_layout(dev_size, total_size, num_spans, config.write_volume_align) spans: list[StreamSpan] = [] - - # Calculate positions for regularly distributed spans - # First span always at position 0, last span always ends at dev_size - if num_spans == 1: - positions = [0] - elif num_spans == 2: - positions = [0, dev_size - stream_size] - else: - # For 3+ spans: distribute evenly across available space - # Available space is dev_size - size (last span can't extend past dev_size) - available_space = dev_size - stream_size - positions = [0] # First span always at 0 - # Distribute middle spans evenly - for i in range(1, num_spans - 1): - position = (available_space * i) // (num_spans - 1) - positions.append(position) - positions.append(dev_size - stream_size) # Last span always ends at dev_size - - # Generate spans with consistency checks - prev_end = -1 - for i, position in enumerate(positions): - # Assert no overlap - assert position >= prev_end, f"Span {i} at position {position} overlaps with previous span ending at {prev_end}" - span = StreamSpan(position=position, size=stream_size) + for i, (position, size) in enumerate(layout): + span = StreamSpan(position=position, size=size) if i not in skip_spans: span.generate(vm, dev_path, seed=1000 + i) spans.append(span) - prev_end = position + stream_size - - # Final assert: last span must not extend past dev_size - assert spans[-1].position + spans[-1].size <= dev_size, \ - f"Last span extends past device: position={spans[-1].position}, size={spans[-1].size}, dev_size={dev_size}" return spans diff --git a/tests/unit/test_spans.py b/tests/unit/test_spans.py new file mode 100644 index 000000000..bbe08ffd9 --- /dev/null +++ b/tests/unit/test_spans.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +import pytest + +from lib.common import GiB, KiB, TiB +from tests.storage.storage import compute_span_layout + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def spans_cover_device(layout: list[tuple[int, int]], dev_size: int) -> bool: + """Return True if the spans cover [0, dev_size) with no gaps.""" + cursor = 0 + for position, size in layout: + if position != cursor: + return False + cursor += size + return cursor == dev_size + + +def no_overlaps(layout: list[tuple[int, int]]) -> bool: + prev_end = 0 + for i, (position, size) in enumerate(layout): + if i > 0 and position < prev_end: + return False + prev_end = position + size + return True + + +# --------------------------------------------------------------------------- +# Basic structural guarantees +# --------------------------------------------------------------------------- + +def test_first_span_starts_at_zero() -> None: + layout = compute_span_layout(dev_size=100, total_size=90, num_spans=3, block_size=1) + assert layout[0][0] == 0 + + +def test_last_span_ends_at_dev_size() -> None: + layout = compute_span_layout(dev_size=100, total_size=90, num_spans=3, block_size=1) + position, size = layout[-1] + assert position + size == 100 + + +def test_no_overlaps_no_alignment() -> None: + layout = compute_span_layout(dev_size=1000, total_size=500, num_spans=5, block_size=1) + assert no_overlaps(layout) + + +def test_no_overlaps_with_alignment() -> None: + layout = compute_span_layout(dev_size=1000, total_size=500, num_spans=5, block_size=64) + assert no_overlaps(layout) + + +def test_correct_number_of_spans() -> None: + for n in (1, 2, 3, 4, 5): + layout = compute_span_layout(dev_size=1000, total_size=1000, num_spans=n, block_size=1) + assert len(layout) == n + + +# --------------------------------------------------------------------------- +# num_spans=1 +# --------------------------------------------------------------------------- + +def test_single_span_covers_full_device() -> None: + layout = compute_span_layout(dev_size=512, total_size=512, num_spans=1, block_size=1) + assert layout == [(0, 512)] + + +def test_single_span_with_alignment() -> None: + layout = compute_span_layout(dev_size=512, total_size=512, num_spans=1, block_size=512) + assert layout == [(0, 512)] + + +# --------------------------------------------------------------------------- +# num_spans=2 +# --------------------------------------------------------------------------- + +def test_two_spans_full_coverage_no_alignment() -> None: + # total_size=100=dev_size: positions at 0 and 50, sizes 50+50=100 + layout = compute_span_layout(dev_size=100, total_size=100, num_spans=2, block_size=1) + assert layout[0] == (0, 50) + assert layout[1] == (50, 50) + + +def test_two_spans_partial_no_alignment() -> None: + # total_size=60: last span anchored at end of device, first span writes + # its budget from position 0; gap between them + layout = compute_span_layout(dev_size=100, total_size=60, num_spans=2, block_size=1) + assert layout[0] == (0, 30) + assert layout[1] == (70, 30) + + +def test_two_spans_aligned_position() -> None: + # total_size=100, position[1] = 100*1//2 = 50, aligned down to 0 (50 % 64 = 50 → (50//64)*64=0) + # Actually 50 // 64 = 0, so position[1] = 0... that would overlap. + # Use align=32: position[1] = (50//32)*32 = 32 + layout = compute_span_layout(dev_size=100, total_size=100, num_spans=2, block_size=32) + assert layout[0][0] == 0 + assert layout[1][0] % 32 == 0 + pos, size = layout[1] + assert pos + size == 100 + + +# --------------------------------------------------------------------------- +# num_spans=3, no alignment, full coverage +# --------------------------------------------------------------------------- + +def test_three_spans_no_alignment_full_coverage_divisible() -> None: + # dev_size divisible by num_spans: even split + layout = compute_span_layout(dev_size=90, total_size=90, num_spans=3, block_size=1) + assert layout == [(0, 30), (30, 30), (60, 30)] + + +def test_three_spans_no_alignment_full_coverage_indivisible() -> None: + # dev_size=100 not divisible by 3: positions at 0, 33, 66 — no gap + layout = compute_span_layout(dev_size=100, total_size=100, num_spans=3, block_size=1) + positions = [p for p, _ in layout] + assert positions == [0, 33, 66] + assert spans_cover_device(layout, 100) + + +def test_three_spans_no_alignment_partial() -> None: + # total_size=90 < dev_size=100: spans are spread across the device with + # gaps between them. Each span writes total_size//num_spans = 30 bytes. + # The last span is anchored at the end of the device. + layout = compute_span_layout(dev_size=100, total_size=90, num_spans=3, block_size=1) + assert layout[0] == (0, 30) + assert layout[1] == (40, 30) + pos, size = layout[2] + assert pos + size == 100 + + +# --------------------------------------------------------------------------- +# Full device coverage with alignment +# --------------------------------------------------------------------------- + +def test_full_coverage_three_spans_aligned() -> None: + layout = compute_span_layout(dev_size=100, total_size=100, num_spans=3, block_size=7) + assert spans_cover_device(layout, 100) + + +def test_full_coverage_two_spans_aligned() -> None: + layout = compute_span_layout(dev_size=100, total_size=100, num_spans=2, block_size=7) + assert spans_cover_device(layout, 100) + + +def test_full_coverage_four_spans_aligned() -> None: + layout = compute_span_layout(dev_size=1000, total_size=1000, num_spans=4, block_size=64) + assert spans_cover_device(layout, 1000) + + +def test_full_coverage_large_alignment() -> None: + # align=512, dev_size not a multiple of 512 + layout = compute_span_layout(dev_size=10007, total_size=10007, num_spans=3, block_size=512) + assert spans_cover_device(layout, 10007) + + +# --------------------------------------------------------------------------- +# Carry logic: trimmed bytes propagate forward +# --------------------------------------------------------------------------- + +def test_carry_prevents_gap_before_last_span() -> None: + # dev_size=100, align=7, num_spans=3, total_size=100 + # Without carry: alignment trimming of early spans could leave a gap + # before the last span. With carry the trimmed bytes are passed forward. + layout = compute_span_layout(dev_size=100, total_size=100, num_spans=3, block_size=7) + assert spans_cover_device(layout, 100) + + +def test_carry_accumulates_across_multiple_spans() -> None: + # total_size=dev_size=1000, num_spans=5, align=64: + # positions (align=64): 0, 192, 384, 576, 768 + # per_span=200; span[0] trimmed to 192 (+carry 8), span[1] gets 208 but + # slot=192 so trimmed again (+carry 16), etc. Carry accumulates and ensures + # full coverage with no gap before the last span. + layout = compute_span_layout(dev_size=1000, total_size=1000, num_spans=5, block_size=64) + assert spans_cover_device(layout, 1000) + + +def test_carry_no_effect_when_not_needed() -> None: + # Perfectly divisible, align=1: no trimming, no carry, all spans equal size + layout = compute_span_layout(dev_size=300, total_size=300, num_spans=3, block_size=1) + assert layout == [(0, 100), (100, 100), (200, 100)] + + +# --------------------------------------------------------------------------- +# Partial coverage (total_size < dev_size): gaps are expected +# --------------------------------------------------------------------------- + +def test_partial_total_size_non_last_spans_are_smaller() -> None: + # With total_size < dev_size, span sizes are capped by total_size//num_spans. + # The last span ends at dev_size with gaps preceding it. + layout = compute_span_layout(dev_size=100, total_size=30, num_spans=3, block_size=1) + per_span = 30 // 3 + for _, size in layout[:-1]: + assert size <= per_span + pos, size = layout[-1] + assert pos + size == 100 + + +def test_last_span_always_ends_at_dev_size_even_when_partial() -> None: + layout = compute_span_layout(dev_size=100, total_size=30, num_spans=3, block_size=1) + position, size = layout[-1] + assert position + size == 100 + + +# --------------------------------------------------------------------------- +# All positions are aligned +# --------------------------------------------------------------------------- + +def test_all_positions_are_multiples_of_align() -> None: + align = 512 + layout = compute_span_layout(dev_size=10000, total_size=10000, num_spans=4, block_size=align) + for position, _ in layout: + assert position % align == 0, f"position {position} is not a multiple of {align}" + + +def test_all_positions_aligned_with_non_power_of_two() -> None: + align = 7 + layout = compute_span_layout(dev_size=1000, total_size=1000, num_spans=5, block_size=align) + for position, _ in layout: + assert position % align == 0, f"position {position} is not a multiple of {align}" + + +# --------------------------------------------------------------------------- +# Edge cases +# --------------------------------------------------------------------------- + +def test_align_equals_dev_size() -> None: + layout = compute_span_layout(dev_size=1024, total_size=1024, num_spans=1, block_size=1024) + assert layout == [(0, 1024)] + + +def test_large_realistic_disk_full_coverage() -> None: + # Simulate a 2TiB disk, 4KiB alignment, 3 spans, full coverage + dev_size = 2 * TiB + layout = compute_span_layout(dev_size=dev_size, total_size=dev_size, num_spans=3, block_size=4 * KiB) + assert spans_cover_device(layout, dev_size) + for position, _ in layout: + assert position % (4 * KiB) == 0 + + +def test_large_realistic_disk_partial_coverage() -> None: + # Simulate a 2TiB disk with write_volume_cap=2GiB + dev_size = 2 * TiB + total_size = 2 * GiB + layout = compute_span_layout(dev_size=dev_size, total_size=total_size, num_spans=3, block_size=4 * KiB) + assert layout[0][0] == 0 + pos, size = layout[-1] + assert pos + size == dev_size + assert no_overlaps(layout) + for position, _ in layout: + assert position % (4 * KiB) == 0