From 40e94cd98b7854d4f63f3a32332d5d1b8da7804c Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:51:12 -0500 Subject: [PATCH 01/10] Zigpy packet capture interface --- zigpy_cli/helpers.py | 77 ++++++++++++++++++++++++++++++++++++++++++++ zigpy_cli/radio.py | 54 +++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 zigpy_cli/helpers.py diff --git a/zigpy_cli/helpers.py b/zigpy_cli/helpers.py new file mode 100644 index 0000000..ca578d5 --- /dev/null +++ b/zigpy_cli/helpers.py @@ -0,0 +1,77 @@ +import struct + +import zigpy.types as t + + +class PcapWriter: + """Class responsible to write in PCAP format.""" + + def __init__(self, file): + """Initialize PCAP file and write global header.""" + self.file = file + + def write_header(self): + self.file.write( + struct.pack(" None: + """Write a packet with its header and TLV metadata.""" + timestamp_sec = int(packet.timestamp.timestamp()) + timestamp_usec = int(packet.timestamp.microsecond) + + sub_tlvs = b"" + + # RSSI + sub_tlvs += ( + t.uint16_t(1).serialize() + + t.uint16_t(4).serialize() + + t.Single(packet.rssi).serialize() + ) + + # LQI + sub_tlvs += ( + t.uint16_t(10).serialize() + + t.uint16_t(1).serialize() + + t.uint8_t(packet.lqi).serialize() + + b"\x00\x00\x00" + ) + + # Channel Assignment + sub_tlvs += ( + t.uint16_t(3).serialize() + + t.uint16_t(3).serialize() + + t.uint16_t(packet.channel).serialize() + + t.uint8_t(0).serialize() # page 0 + + b"\x00" + ) + + # FCS type + sub_tlvs += ( + t.uint16_t(0).serialize() + + t.uint16_t(1).serialize() + + t.uint8_t(1).serialize() # FCS type 1 + + b"\x00\x00\x00" + ) + + tlvs = b"" + + # TAP header: version:u8, reserved: u8, length: u16 + tlvs += struct.pack("": # Surely there's a better way? + output.flush() From cc3045d9409b52948226f4a844a09eddce6923aa Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:54:06 -0500 Subject: [PATCH 02/10] Include `CHANNELS_LIST` --- zigpy_cli/common.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/zigpy_cli/common.py b/zigpy_cli/common.py index 9cd9fc3..ba4bfd5 100644 --- a/zigpy_cli/common.py +++ b/zigpy_cli/common.py @@ -1,4 +1,5 @@ import click +from zigpy.types import Channels class HexOrDecIntParamType(click.ParamType): @@ -17,4 +18,18 @@ def convert(self, value, param, ctx): self.fail(f"{value!r} is not a valid integer", param, ctx) +class ChannelsType(click.ParamType): + name = "channels" + + def convert(self, value, param, ctx): + if isinstance(value, Channels): + return value + + try: + return Channels.from_channel_list(map(int, value.split(","))) + except ValueError: + self.fail(f"{value!r} is not a valid channel list", param, ctx) + + HEX_OR_DEC_INT = HexOrDecIntParamType() +CHANNELS_LIST = ChannelsType() From 1cd54a5038c3a1d484e7b6b446a9df5375095d22 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:54:40 -0500 Subject: [PATCH 03/10] Rename `-h` to `-p` --- zigpy_cli/radio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zigpy_cli/radio.py b/zigpy_cli/radio.py index 833c8a9..5b74129 100644 --- a/zigpy_cli/radio.py +++ b/zigpy_cli/radio.py @@ -249,7 +249,7 @@ async def change_channel(app, channel): type=CHANNELS_LIST, default=zigpy.types.Channels.ALL_CHANNELS, ) -@click.option("-h", "--channel-hop-period", type=int, default=5) +@click.option("-p", "--channel-hop-period", type=float, default=5.0) @click.option("-o", "--output", type=click.File("wb"), required=True) @click_coroutine async def packet_capture(app, randomize, channels, channel_hop_period, output): From ab2976f299fb85ef57457d1302664a4d89f82bc8 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:57:38 -0500 Subject: [PATCH 04/10] Rename `randomize` to `--channel-hop-randomly` --- zigpy_cli/radio.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/zigpy_cli/radio.py b/zigpy_cli/radio.py index 5b74129..64ba107 100644 --- a/zigpy_cli/radio.py +++ b/zigpy_cli/radio.py @@ -242,7 +242,7 @@ async def change_channel(app, channel): @radio.command() @click.pass_obj -@click.option("-r", "--randomize", is_flag=True, type=bool, default=False) +@click.option("-r", "--channel-hop-randomly", is_flag=True, type=bool, default=False) @click.option( "-c", "--channels", @@ -252,8 +252,10 @@ async def change_channel(app, channel): @click.option("-p", "--channel-hop-period", type=float, default=5.0) @click.option("-o", "--output", type=click.File("wb"), required=True) @click_coroutine -async def packet_capture(app, randomize, channels, channel_hop_period, output): - if not randomize: +async def packet_capture( + app, channel_hop_randomly, channels, channel_hop_period, output +): + if not channel_hop_randomly: channels_iter = itertools.cycle(channels) else: From e29c3dddcb419774587e5c3cff31043481a9bb6e Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 22 Jan 2025 16:06:21 -0500 Subject: [PATCH 05/10] Use the simplified interface --- zigpy_cli/radio.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/zigpy_cli/radio.py b/zigpy_cli/radio.py index 64ba107..3f47093 100644 --- a/zigpy_cli/radio.py +++ b/zigpy_cli/radio.py @@ -261,7 +261,7 @@ async def packet_capture( def channels_iter_func(): while True: - yield random.choice(channels) + yield random.choice(tuple(channels)) channels_iter = channels_iter_func() @@ -270,23 +270,24 @@ def channels_iter_func(): await app.connect() - async with app.packet_capture(channel=next(channels_iter)) as capture: - async with asyncio.TaskGroup() as tg: + writer = PcapWriter(output) + writer.write_header() - async def channel_hopper(): - for channel in channels_iter: - await asyncio.sleep(channel_hop_period) - LOGGER.debug("Changing channel to %s", channel) - await capture.change_channel(channel) + async with asyncio.TaskGroup() as tg: + channel_hopper_task = None - tg.create_task(channel_hopper()) + async def channel_hopper(): + for channel in channels_iter: + await asyncio.sleep(channel_hop_period) + LOGGER.debug("Changing channel to %s", channel) + await app.packet_capture_change_channel(channel) - writer = PcapWriter(output) - writer.write_header() + async for packet in app.packet_capture(channel=next(channels_iter)): + if channel_hopper_task is None: + channel_hopper_task = tg.create_task(channel_hopper()) - async for packet in capture: - LOGGER.debug("Got a packet %s", packet) - writer.write_packet(packet) + LOGGER.debug("Got a packet %s", packet) + writer.write_packet(packet) - if output.name == "": # Surely there's a better way? - output.flush() + if output.name == "": # Surely there's a better way? + output.flush() From 0f1fe7c4ca5a1fa1f880962e7caba897d969f68f Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 22 Jan 2025 17:48:51 -0500 Subject: [PATCH 06/10] Add a new command for concurrent packet capture --- zigpy_cli/pcap.py | 29 +++++++++++++++++++++++++++++ zigpy_cli/radio.py | 35 ++++++++++++++++++++++++++++++----- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/zigpy_cli/pcap.py b/zigpy_cli/pcap.py index 33685bc..6ed1802 100644 --- a/zigpy_cli/pcap.py +++ b/zigpy_cli/pcap.py @@ -1,14 +1,20 @@ from __future__ import annotations +import datetime +import json import logging +import sys import click +import zigpy.types as t from scapy.config import conf as scapy_conf from scapy.layers.dot15d4 import Dot15d4 # NOQA: F401 from scapy.utils import PcapReader, PcapWriter from zigpy_cli.cli import cli +from .helpers import PcapWriter as ZigpyPcapWriter + scapy_conf.dot15d4_protocol = "zigbee" LOGGER = logging.getLogger(__name__) @@ -29,3 +35,26 @@ def fix_fcs(input, output): for packet in reader: packet.fcs = None writer.write(packet) + + +@pcap.command() +@click.option("-o", "--output", type=click.File("wb"), required=True) +def interleave_combine(output): + if output.name == "": + output = sys.stdout.buffer.raw + + writer = ZigpyPcapWriter(output) + writer.write_header() + + while True: + line = sys.stdin.readline() + data = json.loads(line) + packet = t.CapturedPacket( + timestamp=datetime.datetime.fromisoformat(data["timestamp"]), + rssi=data["rssi"], + lqi=data["lqi"], + channel=data["channel"], + data=bytes.fromhex(data["data"]), + ) + + writer.write_packet(packet) diff --git a/zigpy_cli/radio.py b/zigpy_cli/radio.py index 3f47093..481878e 100644 --- a/zigpy_cli/radio.py +++ b/zigpy_cli/radio.py @@ -8,6 +8,7 @@ import json import logging import random +import sys import click import zigpy.state @@ -251,10 +252,19 @@ async def change_channel(app, channel): ) @click.option("-p", "--channel-hop-period", type=float, default=5.0) @click.option("-o", "--output", type=click.File("wb"), required=True) +@click.option("--interleave", is_flag=True, type=bool, default=False) @click_coroutine async def packet_capture( - app, channel_hop_randomly, channels, channel_hop_period, output + app, + channel_hop_randomly, + channels, + channel_hop_period, + output, + interleave, ): + if output.name == "" and not interleave: + output = sys.stdout.buffer.raw + if not channel_hop_randomly: channels_iter = itertools.cycle(channels) else: @@ -270,8 +280,9 @@ def channels_iter_func(): await app.connect() - writer = PcapWriter(output) - writer.write_header() + if not interleave: + writer = PcapWriter(output) + writer.write_header() async with asyncio.TaskGroup() as tg: channel_hopper_task = None @@ -287,7 +298,21 @@ async def channel_hopper(): channel_hopper_task = tg.create_task(channel_hopper()) LOGGER.debug("Got a packet %s", packet) - writer.write_packet(packet) - if output.name == "": # Surely there's a better way? + if not interleave: + writer.write_packet(packet) + else: + # To do line interleaving, encode the packets as JSON + output.write( + json.dumps( + { + "timestamp": packet.timestamp.isoformat(), + "rssi": packet.rssi, + "lqi": packet.lqi, + "channel": packet.channel, + "data": packet.data.hex(), + } + ).encode("ascii") + + b"\n" + ) output.flush() From 84d03785b00d8565a54256b5e4d2188df285e825 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 22 Jan 2025 19:32:16 -0500 Subject: [PATCH 07/10] Bump bellows --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ebac912..f66f782 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "coloredlogs", "scapy", "zigpy>=0.55.0", - "bellows>=0.35.1", + "bellows>=0.43.0", "zigpy-deconz>=0.21.0", "zigpy-xbee>=0.18.0", "zigpy-zboss>=1.1.0", From 4667c9c31f7565ffcc8ba642d63747f5404a9d37 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 22 Jan 2025 19:36:38 -0500 Subject: [PATCH 08/10] Fix CI --- .github/workflows/ci.yml | 211 ++------------------------ .github/workflows/publish-to-pypi.yml | 26 +--- 2 files changed, 16 insertions(+), 221 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54d22aa..2b41392 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,208 +1,17 @@ name: CI -# yamllint disable-line rule:truthy on: push: pull_request: ~ -env: - CACHE_VERSION: 1 - DEFAULT_PYTHON: 3.8 - PRE_COMMIT_HOME: ~/.cache/pre-commit - jobs: - # Separate job to pre-populate the base dependency cache - # This prevent upcoming jobs to do the same individually - prepare-base: - name: Prepare base dependencies - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.8, 3.9, "3.10", "3.11"] - steps: - - name: Check out code from GitHub - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - id: python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Restore base Python virtual environment - id: cache-venv - uses: actions/cache@v3 - with: - path: venv - key: ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{ steps.python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }} - restore-keys: ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{ steps.python.outputs.python-version }}- - - name: Create Python virtual environment - if: steps.cache-venv.outputs.cache-hit != 'true' - run: | - python -m venv venv - . venv/bin/activate - pip install -U pip setuptools pre-commit - pip install -e '.[testing]' - - pre-commit: - name: Prepare pre-commit environment - runs-on: ubuntu-latest - needs: prepare-base - steps: - - name: Check out code from GitHub - uses: actions/checkout@v3 - - name: Set up Python ${{ env.DEFAULT_PYTHON }} - uses: actions/setup-python@v4 - id: python - with: - python-version: ${{ env.DEFAULT_PYTHON }} - - name: Restore base Python virtual environment - id: cache-venv - uses: actions/cache@v3 - with: - path: venv - key: ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{ steps.python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }} - - name: Fail job if Python cache restore failed - if: steps.cache-venv.outputs.cache-hit != 'true' - run: | - echo "Failed to restore Python virtual environment from cache" - exit 1 - - name: Restore pre-commit environment from cache - id: cache-precommit - uses: actions/cache@v3 - with: - path: ${{ env.PRE_COMMIT_HOME }} - key: ${{ env.CACHE_VERSION}}-${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} - restore-keys: ${{ env.CACHE_VERSION}}-${{ runner.os }}-pre-commit- - - name: Install pre-commit dependencies - if: steps.cache-precommit.outputs.cache-hit != 'true' - run: | - . venv/bin/activate - pre-commit install-hooks - - lint-pre-commit: - name: Check pre-commit - runs-on: ubuntu-latest - needs: pre-commit - steps: - - name: Check out code from GitHub - uses: actions/checkout@v3 - - name: Set up Python ${{ env.DEFAULT_PYTHON }} - uses: actions/setup-python@v4 - id: python - with: - python-version: ${{ env.DEFAULT_PYTHON }} - - name: Restore base Python virtual environment - id: cache-venv - uses: actions/cache@v3 - with: - path: venv - key: ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{ steps.python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }} - - name: Fail job if Python cache restore failed - if: steps.cache-venv.outputs.cache-hit != 'true' - run: | - echo "Failed to restore Python virtual environment from cache" - exit 1 - - name: Restore pre-commit environment from cache - id: cache-precommit - uses: actions/cache@v3 - with: - path: ${{ env.PRE_COMMIT_HOME }} - key: ${{ env.CACHE_VERSION}}-${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} - - name: Fail job if cache restore failed - if: steps.cache-venv.outputs.cache-hit != 'true' - run: | - echo "Failed to restore Python virtual environment from cache" - exit 1 - - name: Run pre-commit - run: | - . venv/bin/activate - pre-commit run --all-files --show-diff-on-failure - - pytest: - runs-on: ubuntu-latest - needs: prepare-base - strategy: - matrix: - python-version: [3.8, 3.9, "3.10", "3.11"] - name: >- - Run tests Python ${{ matrix.python-version }} - steps: - - name: Check out code from GitHub - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - id: python - with: - python-version: ${{ matrix.python-version }} - - name: Restore base Python virtual environment - id: cache-venv - uses: actions/cache@v3 - with: - path: venv - key: ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{ steps.python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }} - - name: Fail job if Python cache restore failed - if: steps.cache-venv.outputs.cache-hit != 'true' - run: | - echo "Failed to restore Python virtual environment from cache" - exit 1 - - name: Register Python problem matcher - run: | - echo "::add-matcher::.github/workflows/matchers/python.json" - - name: Install Pytest Annotation plugin - run: | - . venv/bin/activate - # Ideally this should be part of our dependencies - # However this plugin is fairly new and doesn't run correctly - # on a non-GitHub environment. - pip install pytest-github-actions-annotate-failures - - name: Run pytest - run: | - . venv/bin/activate - pytest \ - -qq \ - --timeout=20 \ - --durations=10 \ - --cov zigpy_cli \ - --cov-config pyproject.toml \ - -o console_output_style=count \ - -p no:sugar \ - tests - - name: Upload coverage artifact - uses: actions/upload-artifact@v3 - with: - name: coverage-${{ matrix.python-version }} - path: .coverage - - - coverage: - name: Process test coverage - runs-on: ubuntu-latest - needs: pytest - steps: - - name: Check out code from GitHub - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - id: python - with: - python-version: ${{ env.DEFAULT_PYTHON }} - - name: Restore base Python virtual environment - id: cache-venv - uses: actions/cache@v3 - with: - path: venv - key: ${{ env.CACHE_VERSION}}-${{ runner.os }}-base-venv-${{ steps.python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }} - - name: Fail job if Python cache restore failed - if: steps.cache-venv.outputs.cache-hit != 'true' - run: | - echo "Failed to restore Python virtual environment from cache" - exit 1 - - name: Download all coverage artifacts - uses: actions/download-artifact@v3 - - name: Combine coverage results - run: | - . venv/bin/activate - coverage combine coverage*/.coverage* - coverage report - coverage xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + shared-ci: + uses: zigpy/workflows/.github/workflows/ci.yml@main + with: + CODE_FOLDER: zigpy + CACHE_VERSION: 3 + PRE_COMMIT_CACHE_PATH: ~/.cache/pre-commit + PYTHON_VERSION_DEFAULT: 3.9.15 + MINIMUM_COVERAGE_PERCENTAGE: 1 + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index c2bbc1b..3d931cf 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -1,26 +1,12 @@ -name: Publish distributions to PyPI and TestPyPI +name: Publish distributions to PyPI + on: release: types: - published jobs: - build-and-publish: - name: Build and publish distributions to PyPI and TestPyPI - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.8 - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Install wheel - run: >- - pip install wheel build - - name: Build - run: >- - python3 -m build - - name: Publish distribution to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} + shared-build-and-publish: + uses: zigpy/workflows/.github/workflows/publish-to-pypi.yml@main + secrets: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} From e144024691da5f49bb5acf57269c54163ba6548f Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 22 Jan 2025 19:40:09 -0500 Subject: [PATCH 09/10] Add `requirements_test.txt` --- requirements_test.txt | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 requirements_test.txt diff --git a/requirements_test.txt b/requirements_test.txt new file mode 100644 index 0000000..2913163 --- /dev/null +++ b/requirements_test.txt @@ -0,0 +1,5 @@ +coverage[toml] +pytest +pytest-asyncio +pytest-cov +pytest-timeout From 87def1084464c7057168fbe45ad9ba77d5b5d184 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 22 Jan 2025 19:44:02 -0500 Subject: [PATCH 10/10] Update code folder in CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b41392..83eb24b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ jobs: shared-ci: uses: zigpy/workflows/.github/workflows/ci.yml@main with: - CODE_FOLDER: zigpy + CODE_FOLDER: zigpy_cli CACHE_VERSION: 3 PRE_COMMIT_CACHE_PATH: ~/.cache/pre-commit PYTHON_VERSION_DEFAULT: 3.9.15