From fd601f3c3e3f8ce7e40d2adc7ae87ee17afeb5eb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Jun 2026 12:01:54 +0000 Subject: [PATCH 01/13] Add pcap dissection comparison corpus test --- test/scapy/layers/dissection_corpus.uts | 113 ++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 test/scapy/layers/dissection_corpus.uts diff --git a/test/scapy/layers/dissection_corpus.uts b/test/scapy/layers/dissection_corpus.uts new file mode 100644 index 00000000000..9b335ac96d6 --- /dev/null +++ b/test/scapy/layers/dissection_corpus.uts @@ -0,0 +1,113 @@ +% Dissection corpus verification tests + +############ +############ ++ Dissection corpus +~ tshark pcaps + += Compare Scapy and tshark dissections from pcap corpus + +import os + + +def _normalize_scapy_value(value): + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf8", errors="replace") + return str(value) + + +def _compare_tcp_flags(tshark_value, scapy_value): + if tshark_value == "" and scapy_value is None: + return True + return int(tshark_value) == int(scapy_value) + + +def _get_scapy_field(packet, scapy_field): + layer_name, field_name = scapy_field.split(".", 1) + layer = globals()[layer_name] + if layer not in packet: + return None + return getattr(packet[layer], field_name) + + +def _extract_tshark_rows(pcap_path, mapping): + args = ["-T", "fields", "-E", "separator=\\t", "-E", "occurrence=f"] + for tshark_field in mapping: + args.extend(["-e", tshark_field]) + output = tcpdump( + pcap_path, + prog=conf.prog.tshark, + getfd=True, + args=args, + dump=True, + wait=True, + ) + lines = output.decode("utf8").splitlines() + rows = [line.split("\t") for line in lines] + return rows + + +def _compare_pcap_dissection(pcap_file, mapping): + pcap_path = scapy_path("/" + pcap_file) + assert os.path.exists(pcap_path) + + packets = rdpcap(pcap_path) + tshark_rows = _extract_tshark_rows(pcap_path, mapping) + + assert len(packets) == len(tshark_rows), ( + "Packet count mismatch for %s: scapy=%d tshark=%d" % ( + pcap_file, + len(packets), + len(tshark_rows), + ) + ) + + tshark_fields = list(mapping) + for index, (packet, tshark_row) in enumerate(zip(packets, tshark_rows), 1): + assert len(tshark_row) == len(tshark_fields), ( + "Field count mismatch for %s packet #%d" % (pcap_file, index) + ) + for position, tshark_field in enumerate(tshark_fields): + entry = mapping[tshark_field] + compare = entry.get("compare") + if compare is None: + compare = lambda tshark_val, scapy_val: ( + tshark_val == _normalize_scapy_value(scapy_val) + ) + scapy_value = _get_scapy_field(packet, entry["scapy"]) + tshark_value = tshark_row[position] + assert compare(tshark_value, scapy_value), ( + "Mismatch in %s packet #%d for %s/%s: tshark=%r scapy=%r" % ( + pcap_file, + index, + tshark_field, + entry["scapy"], + tshark_value, + scapy_value, + ) + ) + + +DISSECTION_CORPUS = { + "test/pcaps/http_content_length.pcap": { + "ip.src": {"scapy": "IP.src"}, + "ip.dst": {"scapy": "IP.dst"}, + "ip.ttl": {"scapy": "IP.ttl"}, + "tcp.srcport": {"scapy": "TCP.sport"}, + "tcp.dstport": {"scapy": "TCP.dport"}, + "tcp.flags": {"scapy": "TCP.flags", "compare": _compare_tcp_flags}, + }, + "test/pcaps/netflowv9.pcap": { + "ip.src": {"scapy": "IP.src"}, + "ip.dst": {"scapy": "IP.dst"}, + "ip.ttl": {"scapy": "IP.ttl"}, + "udp.srcport": {"scapy": "UDP.sport"}, + "udp.dstport": {"scapy": "UDP.dport"}, + }, +} + + +for pcap_file, mapping in DISSECTION_CORPUS.items(): + _compare_pcap_dissection(pcap_file, mapping) From 2f402c7e4e1fb0d5d9fd777f23e317f0c48f7c0d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Jun 2026 12:02:44 +0000 Subject: [PATCH 02/13] Refine dissection corpus comparison helpers --- test/scapy/layers/dissection_corpus.uts | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/test/scapy/layers/dissection_corpus.uts b/test/scapy/layers/dissection_corpus.uts index 9b335ac96d6..20536cb05ba 100644 --- a/test/scapy/layers/dissection_corpus.uts +++ b/test/scapy/layers/dissection_corpus.uts @@ -14,7 +14,7 @@ def _normalize_scapy_value(value): if value is None: return "" if isinstance(value, bytes): - return value.decode("utf8", errors="replace") + return value.decode("utf-8", errors="replace") return str(value) @@ -24,9 +24,20 @@ def _compare_tcp_flags(tshark_value, scapy_value): return int(tshark_value) == int(scapy_value) +def _default_compare(tshark_value, scapy_value): + return tshark_value == _normalize_scapy_value(scapy_value) + + +LAYER_BY_NAME = { + "IP": IP, + "TCP": TCP, + "UDP": UDP, +} + + def _get_scapy_field(packet, scapy_field): layer_name, field_name = scapy_field.split(".", 1) - layer = globals()[layer_name] + layer = LAYER_BY_NAME[layer_name] if layer not in packet: return None return getattr(packet[layer], field_name) @@ -44,7 +55,7 @@ def _extract_tshark_rows(pcap_path, mapping): dump=True, wait=True, ) - lines = output.decode("utf8").splitlines() + lines = output.decode("utf-8").splitlines() rows = [line.split("\t") for line in lines] return rows @@ -71,11 +82,7 @@ def _compare_pcap_dissection(pcap_file, mapping): ) for position, tshark_field in enumerate(tshark_fields): entry = mapping[tshark_field] - compare = entry.get("compare") - if compare is None: - compare = lambda tshark_val, scapy_val: ( - tshark_val == _normalize_scapy_value(scapy_val) - ) + compare = entry.get("compare", _default_compare) scapy_value = _get_scapy_field(packet, entry["scapy"]) tshark_value = tshark_row[position] assert compare(tshark_value, scapy_value), ( From 55a9059ad351e27b570a6853bef25ebacec5c0d9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Jun 2026 12:06:53 +0000 Subject: [PATCH 03/13] Finalize pcap dissection corpus UTScapy test --- test/scapy/layers/dissection_corpus.uts | 98 +++++++++++++++---------- 1 file changed, 60 insertions(+), 38 deletions(-) diff --git a/test/scapy/layers/dissection_corpus.uts b/test/scapy/layers/dissection_corpus.uts index 20536cb05ba..a018a495780 100644 --- a/test/scapy/layers/dissection_corpus.uts +++ b/test/scapy/layers/dissection_corpus.uts @@ -19,9 +19,14 @@ def _normalize_scapy_value(value): def _compare_tcp_flags(tshark_value, scapy_value): - if tshark_value == "" and scapy_value is None: - return True - return int(tshark_value) == int(scapy_value) + if tshark_value == "": + return scapy_value is None + if scapy_value is None: + return False + try: + return int(tshark_value) == int(scapy_value) + except (TypeError, ValueError): + return False def _default_compare(tshark_value, scapy_value): @@ -36,15 +41,30 @@ LAYER_BY_NAME = { def _get_scapy_field(packet, scapy_field): + assert scapy_field.count(".") == 1, ( + f"Invalid scapy_field format: {scapy_field!r}. " + "Expected format: LayerName.field_name (exactly one dot)" + ) layer_name, field_name = scapy_field.split(".", 1) + assert layer_name and field_name, ( + f"Invalid scapy field mapping: {scapy_field!r}. " + "Layer and field names must be non-empty" + ) + assert layer_name in LAYER_BY_NAME, ( + f"Unsupported layer in mapping: {layer_name!r}. " + f"Supported layers: {list(LAYER_BY_NAME.keys())!r}" + ) layer = LAYER_BY_NAME[layer_name] if layer not in packet: return None + assert hasattr(packet[layer], field_name), ( + f"Field {field_name!r} does not exist on layer {layer_name!r}" + ) return getattr(packet[layer], field_name) def _extract_tshark_rows(pcap_path, mapping): - args = ["-T", "fields", "-E", "separator=\\t", "-E", "occurrence=f"] + args = ["-T", "fields", "-E", "separator=\t", "-E", "occurrence=f"] for tshark_field in mapping: args.extend(["-e", tshark_field]) output = tcpdump( @@ -68,53 +88,55 @@ def _compare_pcap_dissection(pcap_file, mapping): tshark_rows = _extract_tshark_rows(pcap_path, mapping) assert len(packets) == len(tshark_rows), ( - "Packet count mismatch for %s: scapy=%d tshark=%d" % ( - pcap_file, - len(packets), - len(tshark_rows), - ) + f"Packet count mismatch for {pcap_file}: " + f"scapy={len(packets)} tshark={len(tshark_rows)}" ) tshark_fields = list(mapping) - for index, (packet, tshark_row) in enumerate(zip(packets, tshark_rows), 1): + for packet_number, (packet, tshark_row) in enumerate( + zip(packets, tshark_rows), 1 + ): assert len(tshark_row) == len(tshark_fields), ( - "Field count mismatch for %s packet #%d" % (pcap_file, index) + f"Field count mismatch for {pcap_file} packet #{packet_number}" ) - for position, tshark_field in enumerate(tshark_fields): + for field_idx, tshark_field in enumerate(tshark_fields): entry = mapping[tshark_field] compare = entry.get("compare", _default_compare) scapy_value = _get_scapy_field(packet, entry["scapy"]) - tshark_value = tshark_row[position] + tshark_value = tshark_row[field_idx] assert compare(tshark_value, scapy_value), ( - "Mismatch in %s packet #%d for %s/%s: tshark=%r scapy=%r" % ( - pcap_file, - index, - tshark_field, - entry["scapy"], - tshark_value, - scapy_value, - ) + f"Mismatch in {pcap_file} packet #{packet_number} for " + f"{tshark_field}/{entry['scapy']}: " + f"tshark={tshark_value!r} scapy={scapy_value!r}" ) -DISSECTION_CORPUS = { - "test/pcaps/http_content_length.pcap": { - "ip.src": {"scapy": "IP.src"}, - "ip.dst": {"scapy": "IP.dst"}, - "ip.ttl": {"scapy": "IP.ttl"}, - "tcp.srcport": {"scapy": "TCP.sport"}, - "tcp.dstport": {"scapy": "TCP.dport"}, - "tcp.flags": {"scapy": "TCP.flags", "compare": _compare_tcp_flags}, +dissection_corpus = [ + { + "name": "http_content_length", + "pcap": "test/pcaps/http_content_length.pcap", + "mapping": { + "ip.src": {"scapy": "IP.src"}, + "ip.dst": {"scapy": "IP.dst"}, + "ip.ttl": {"scapy": "IP.ttl"}, + "tcp.srcport": {"scapy": "TCP.sport"}, + "tcp.dstport": {"scapy": "TCP.dport"}, + "tcp.flags": {"scapy": "TCP.flags", "compare": _compare_tcp_flags}, + }, }, - "test/pcaps/netflowv9.pcap": { - "ip.src": {"scapy": "IP.src"}, - "ip.dst": {"scapy": "IP.dst"}, - "ip.ttl": {"scapy": "IP.ttl"}, - "udp.srcport": {"scapy": "UDP.sport"}, - "udp.dstport": {"scapy": "UDP.dport"}, + { + "name": "netflowv9", + "pcap": "test/pcaps/netflowv9.pcap", + "mapping": { + "ip.src": {"scapy": "IP.src"}, + "ip.dst": {"scapy": "IP.dst"}, + "ip.ttl": {"scapy": "IP.ttl"}, + "udp.srcport": {"scapy": "UDP.sport"}, + "udp.dstport": {"scapy": "UDP.dport"}, + }, }, -} +] -for pcap_file, mapping in DISSECTION_CORPUS.items(): - _compare_pcap_dissection(pcap_file, mapping) +for corpus_entry in dissection_corpus: + _compare_pcap_dissection(corpus_entry["pcap"], corpus_entry["mapping"]) From e47a9d003b53d3c4f22cb0914d68d0aa7d1ee859 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Jun 2026 12:28:49 +0000 Subject: [PATCH 04/13] Add more dissection corpus pcap examples --- test/scapy/layers/dissection_corpus.uts | 35 +++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/scapy/layers/dissection_corpus.uts b/test/scapy/layers/dissection_corpus.uts index a018a495780..4c8222dae16 100644 --- a/test/scapy/layers/dissection_corpus.uts +++ b/test/scapy/layers/dissection_corpus.uts @@ -135,6 +135,41 @@ dissection_corpus = [ "udp.dstport": {"scapy": "UDP.dport"}, }, }, + { + "name": "http_compressed", + "pcap": "test/pcaps/http_compressed.pcap", + "mapping": { + "ip.src": {"scapy": "IP.src"}, + "ip.dst": {"scapy": "IP.dst"}, + "ip.ttl": {"scapy": "IP.ttl"}, + "tcp.srcport": {"scapy": "TCP.sport"}, + "tcp.dstport": {"scapy": "TCP.dport"}, + "tcp.flags": {"scapy": "TCP.flags", "compare": _compare_tcp_flags}, + }, + }, + { + "name": "ssh_ed25519", + "pcap": "test/pcaps/ssh_ed25519.pcap", + "mapping": { + "ip.src": {"scapy": "IP.src"}, + "ip.dst": {"scapy": "IP.dst"}, + "ip.ttl": {"scapy": "IP.ttl"}, + "tcp.srcport": {"scapy": "TCP.sport"}, + "tcp.dstport": {"scapy": "TCP.dport"}, + "tcp.flags": {"scapy": "TCP.flags", "compare": _compare_tcp_flags}, + }, + }, + { + "name": "ipfix", + "pcap": "test/pcaps/ipfix.pcap", + "mapping": { + "ip.src": {"scapy": "IP.src"}, + "ip.dst": {"scapy": "IP.dst"}, + "ip.ttl": {"scapy": "IP.ttl"}, + "udp.srcport": {"scapy": "UDP.sport"}, + "udp.dstport": {"scapy": "UDP.dport"}, + }, + }, ] From 5f43bc89daf96da7339fe413b90ee272c92be17c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:41:36 +0000 Subject: [PATCH 05/13] Expand dissection corpus field coverage --- test/scapy/layers/dissection_corpus.uts | 137 ++++++++++++++++++++---- 1 file changed, 119 insertions(+), 18 deletions(-) diff --git a/test/scapy/layers/dissection_corpus.uts b/test/scapy/layers/dissection_corpus.uts index 4c8222dae16..85a6eb8dd5b 100644 --- a/test/scapy/layers/dissection_corpus.uts +++ b/test/scapy/layers/dissection_corpus.uts @@ -19,14 +19,38 @@ def _normalize_scapy_value(value): def _compare_tcp_flags(tshark_value, scapy_value): + return _compare_int_field(tshark_value, scapy_value) + + +def _parse_int_value(value): + if value is None: + return None + if isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + if isinstance(value, str): + value = value.strip() + if value == "": + return None + try: + return int(value, 0) + except ValueError: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _compare_int_field(tshark_value, scapy_value): if tshark_value == "": return scapy_value is None if scapy_value is None: return False - try: - return int(tshark_value) == int(scapy_value) - except (TypeError, ValueError): + tshark_int = _parse_int_value(tshark_value) + scapy_int = _parse_int_value(scapy_value) + if tshark_int is None or scapy_int is None: return False + return tshark_int == scapy_int def _default_compare(tshark_value, scapy_value): @@ -118,9 +142,28 @@ dissection_corpus = [ "mapping": { "ip.src": {"scapy": "IP.src"}, "ip.dst": {"scapy": "IP.dst"}, - "ip.ttl": {"scapy": "IP.ttl"}, - "tcp.srcport": {"scapy": "TCP.sport"}, - "tcp.dstport": {"scapy": "TCP.dport"}, + "ip.version": {"scapy": "IP.version", "compare": _compare_int_field}, + "ip.dsfield": {"scapy": "IP.tos", "compare": _compare_int_field}, + "ip.len": {"scapy": "IP.len", "compare": _compare_int_field}, + "ip.id": {"scapy": "IP.id", "compare": _compare_int_field}, + "ip.flags": {"scapy": "IP.flags", "compare": _compare_int_field}, + "ip.frag_offset": {"scapy": "IP.frag", "compare": _compare_int_field}, + "ip.ttl": {"scapy": "IP.ttl", "compare": _compare_int_field}, + "ip.proto": {"scapy": "IP.proto", "compare": _compare_int_field}, + "ip.checksum": {"scapy": "IP.chksum", "compare": _compare_int_field}, + "tcp.srcport": {"scapy": "TCP.sport", "compare": _compare_int_field}, + "tcp.dstport": {"scapy": "TCP.dport", "compare": _compare_int_field}, + "tcp.seq": {"scapy": "TCP.seq", "compare": _compare_int_field}, + "tcp.ack": {"scapy": "TCP.ack", "compare": _compare_int_field}, + "tcp.window_size_value": { + "scapy": "TCP.window", + "compare": _compare_int_field, + }, + "tcp.checksum": {"scapy": "TCP.chksum", "compare": _compare_int_field}, + "tcp.urgent_pointer": { + "scapy": "TCP.urgptr", + "compare": _compare_int_field, + }, "tcp.flags": {"scapy": "TCP.flags", "compare": _compare_tcp_flags}, }, }, @@ -130,9 +173,19 @@ dissection_corpus = [ "mapping": { "ip.src": {"scapy": "IP.src"}, "ip.dst": {"scapy": "IP.dst"}, - "ip.ttl": {"scapy": "IP.ttl"}, - "udp.srcport": {"scapy": "UDP.sport"}, - "udp.dstport": {"scapy": "UDP.dport"}, + "ip.version": {"scapy": "IP.version", "compare": _compare_int_field}, + "ip.dsfield": {"scapy": "IP.tos", "compare": _compare_int_field}, + "ip.len": {"scapy": "IP.len", "compare": _compare_int_field}, + "ip.id": {"scapy": "IP.id", "compare": _compare_int_field}, + "ip.flags": {"scapy": "IP.flags", "compare": _compare_int_field}, + "ip.frag_offset": {"scapy": "IP.frag", "compare": _compare_int_field}, + "ip.ttl": {"scapy": "IP.ttl", "compare": _compare_int_field}, + "ip.proto": {"scapy": "IP.proto", "compare": _compare_int_field}, + "ip.checksum": {"scapy": "IP.chksum", "compare": _compare_int_field}, + "udp.srcport": {"scapy": "UDP.sport", "compare": _compare_int_field}, + "udp.dstport": {"scapy": "UDP.dport", "compare": _compare_int_field}, + "udp.length": {"scapy": "UDP.len", "compare": _compare_int_field}, + "udp.checksum": {"scapy": "UDP.chksum", "compare": _compare_int_field}, }, }, { @@ -141,9 +194,28 @@ dissection_corpus = [ "mapping": { "ip.src": {"scapy": "IP.src"}, "ip.dst": {"scapy": "IP.dst"}, - "ip.ttl": {"scapy": "IP.ttl"}, - "tcp.srcport": {"scapy": "TCP.sport"}, - "tcp.dstport": {"scapy": "TCP.dport"}, + "ip.version": {"scapy": "IP.version", "compare": _compare_int_field}, + "ip.dsfield": {"scapy": "IP.tos", "compare": _compare_int_field}, + "ip.len": {"scapy": "IP.len", "compare": _compare_int_field}, + "ip.id": {"scapy": "IP.id", "compare": _compare_int_field}, + "ip.flags": {"scapy": "IP.flags", "compare": _compare_int_field}, + "ip.frag_offset": {"scapy": "IP.frag", "compare": _compare_int_field}, + "ip.ttl": {"scapy": "IP.ttl", "compare": _compare_int_field}, + "ip.proto": {"scapy": "IP.proto", "compare": _compare_int_field}, + "ip.checksum": {"scapy": "IP.chksum", "compare": _compare_int_field}, + "tcp.srcport": {"scapy": "TCP.sport", "compare": _compare_int_field}, + "tcp.dstport": {"scapy": "TCP.dport", "compare": _compare_int_field}, + "tcp.seq": {"scapy": "TCP.seq", "compare": _compare_int_field}, + "tcp.ack": {"scapy": "TCP.ack", "compare": _compare_int_field}, + "tcp.window_size_value": { + "scapy": "TCP.window", + "compare": _compare_int_field, + }, + "tcp.checksum": {"scapy": "TCP.chksum", "compare": _compare_int_field}, + "tcp.urgent_pointer": { + "scapy": "TCP.urgptr", + "compare": _compare_int_field, + }, "tcp.flags": {"scapy": "TCP.flags", "compare": _compare_tcp_flags}, }, }, @@ -153,9 +225,28 @@ dissection_corpus = [ "mapping": { "ip.src": {"scapy": "IP.src"}, "ip.dst": {"scapy": "IP.dst"}, - "ip.ttl": {"scapy": "IP.ttl"}, - "tcp.srcport": {"scapy": "TCP.sport"}, - "tcp.dstport": {"scapy": "TCP.dport"}, + "ip.version": {"scapy": "IP.version", "compare": _compare_int_field}, + "ip.dsfield": {"scapy": "IP.tos", "compare": _compare_int_field}, + "ip.len": {"scapy": "IP.len", "compare": _compare_int_field}, + "ip.id": {"scapy": "IP.id", "compare": _compare_int_field}, + "ip.flags": {"scapy": "IP.flags", "compare": _compare_int_field}, + "ip.frag_offset": {"scapy": "IP.frag", "compare": _compare_int_field}, + "ip.ttl": {"scapy": "IP.ttl", "compare": _compare_int_field}, + "ip.proto": {"scapy": "IP.proto", "compare": _compare_int_field}, + "ip.checksum": {"scapy": "IP.chksum", "compare": _compare_int_field}, + "tcp.srcport": {"scapy": "TCP.sport", "compare": _compare_int_field}, + "tcp.dstport": {"scapy": "TCP.dport", "compare": _compare_int_field}, + "tcp.seq": {"scapy": "TCP.seq", "compare": _compare_int_field}, + "tcp.ack": {"scapy": "TCP.ack", "compare": _compare_int_field}, + "tcp.window_size_value": { + "scapy": "TCP.window", + "compare": _compare_int_field, + }, + "tcp.checksum": {"scapy": "TCP.chksum", "compare": _compare_int_field}, + "tcp.urgent_pointer": { + "scapy": "TCP.urgptr", + "compare": _compare_int_field, + }, "tcp.flags": {"scapy": "TCP.flags", "compare": _compare_tcp_flags}, }, }, @@ -165,9 +256,19 @@ dissection_corpus = [ "mapping": { "ip.src": {"scapy": "IP.src"}, "ip.dst": {"scapy": "IP.dst"}, - "ip.ttl": {"scapy": "IP.ttl"}, - "udp.srcport": {"scapy": "UDP.sport"}, - "udp.dstport": {"scapy": "UDP.dport"}, + "ip.version": {"scapy": "IP.version", "compare": _compare_int_field}, + "ip.dsfield": {"scapy": "IP.tos", "compare": _compare_int_field}, + "ip.len": {"scapy": "IP.len", "compare": _compare_int_field}, + "ip.id": {"scapy": "IP.id", "compare": _compare_int_field}, + "ip.flags": {"scapy": "IP.flags", "compare": _compare_int_field}, + "ip.frag_offset": {"scapy": "IP.frag", "compare": _compare_int_field}, + "ip.ttl": {"scapy": "IP.ttl", "compare": _compare_int_field}, + "ip.proto": {"scapy": "IP.proto", "compare": _compare_int_field}, + "ip.checksum": {"scapy": "IP.chksum", "compare": _compare_int_field}, + "udp.srcport": {"scapy": "UDP.sport", "compare": _compare_int_field}, + "udp.dstport": {"scapy": "UDP.dport", "compare": _compare_int_field}, + "udp.length": {"scapy": "UDP.len", "compare": _compare_int_field}, + "udp.checksum": {"scapy": "UDP.chksum", "compare": _compare_int_field}, }, }, ] From 4a4b75379bb5c6bf8488089586f7661a5b6ec928 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:44:20 +0000 Subject: [PATCH 06/13] Harden integer parsing in corpus comparison --- test/scapy/layers/dissection_corpus.uts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/scapy/layers/dissection_corpus.uts b/test/scapy/layers/dissection_corpus.uts index 85a6eb8dd5b..4fb87ad6d66 100644 --- a/test/scapy/layers/dissection_corpus.uts +++ b/test/scapy/layers/dissection_corpus.uts @@ -32,7 +32,9 @@ def _parse_int_value(value): if value == "": return None try: - return int(value, 0) + if value.lower().startswith(("0x", "+0x", "-0x")): + return int(value, 16) + return int(value, 10) except ValueError: return None try: From c93fa4a0f11694c2d00f77d62d18004f6143a6bd Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:46:58 +0000 Subject: [PATCH 07/13] Polish numeric parser style --- test/scapy/layers/dissection_corpus.uts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/scapy/layers/dissection_corpus.uts b/test/scapy/layers/dissection_corpus.uts index 4fb87ad6d66..d2efdc77466 100644 --- a/test/scapy/layers/dissection_corpus.uts +++ b/test/scapy/layers/dissection_corpus.uts @@ -34,7 +34,7 @@ def _parse_int_value(value): try: if value.lower().startswith(("0x", "+0x", "-0x")): return int(value, 16) - return int(value, 10) + return int(value) except ValueError: return None try: From 3f34f8eb493a0ab49d7c399c27da071452d07706 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 04:44:55 +0000 Subject: [PATCH 08/13] Support enum-aware comparator lists in dissection corpus --- test/scapy/layers/dissection_corpus.uts | 112 ++++++++++++++++++++---- 1 file changed, 96 insertions(+), 16 deletions(-) diff --git a/test/scapy/layers/dissection_corpus.uts b/test/scapy/layers/dissection_corpus.uts index d2efdc77466..271875aa319 100644 --- a/test/scapy/layers/dissection_corpus.uts +++ b/test/scapy/layers/dissection_corpus.uts @@ -11,6 +11,8 @@ import os def _normalize_scapy_value(value): + if isinstance(value, tuple): + value = value[0] if value is None: return "" if isinstance(value, bytes): @@ -23,6 +25,8 @@ def _compare_tcp_flags(tshark_value, scapy_value): def _parse_int_value(value): + if isinstance(value, tuple): + value = value[0] if value is None: return None if isinstance(value, bytes): @@ -59,6 +63,37 @@ def _default_compare(tshark_value, scapy_value): return tshark_value == _normalize_scapy_value(scapy_value) +def _normalize_enum_text(value): + if value is None: + return "" + if isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + return "".join(ch.lower() for ch in str(value) if ch.isalnum()) + + +def _compare_enum_field(tshark_value, scapy_value): + if isinstance(scapy_value, tuple): + scapy_value = scapy_value[1] + tshark_enum = _normalize_enum_text(tshark_value) + scapy_enum = _normalize_enum_text(scapy_value) + if not tshark_enum or not scapy_enum: + return False + return tshark_enum == scapy_enum + + +def _get_compare_functions(entry): + compare = entry.get("compare", _default_compare) + if not isinstance(compare, (list, tuple)): + compare = [compare] + assert compare, "At least one compare function must be provided" + for compare_func in compare: + assert callable(compare_func), ( + f"Invalid compare function in mapping for {entry['scapy']!r}: " + f"{compare_func!r}" + ) + return compare + + LAYER_BY_NAME = { "IP": IP, "TCP": TCP, @@ -86,7 +121,10 @@ def _get_scapy_field(packet, scapy_field): assert hasattr(packet[layer], field_name), ( f"Field {field_name!r} does not exist on layer {layer_name!r}" ) - return getattr(packet[layer], field_name) + return ( + getattr(packet[layer], field_name), + packet.sprintf(f"%{layer_name}.{field_name}%"), + ) def _extract_tshark_rows(pcap_path, mapping): @@ -127,10 +165,13 @@ def _compare_pcap_dissection(pcap_file, mapping): ) for field_idx, tshark_field in enumerate(tshark_fields): entry = mapping[tshark_field] - compare = entry.get("compare", _default_compare) + compare_functions = _get_compare_functions(entry) scapy_value = _get_scapy_field(packet, entry["scapy"]) tshark_value = tshark_row[field_idx] - assert compare(tshark_value, scapy_value), ( + assert any( + compare(tshark_value, scapy_value) + for compare in compare_functions + ), ( f"Mismatch in {pcap_file} packet #{packet_number} for " f"{tshark_field}/{entry['scapy']}: " f"tshark={tshark_value!r} scapy={scapy_value!r}" @@ -148,10 +189,16 @@ dissection_corpus = [ "ip.dsfield": {"scapy": "IP.tos", "compare": _compare_int_field}, "ip.len": {"scapy": "IP.len", "compare": _compare_int_field}, "ip.id": {"scapy": "IP.id", "compare": _compare_int_field}, - "ip.flags": {"scapy": "IP.flags", "compare": _compare_int_field}, + "ip.flags": { + "scapy": "IP.flags", + "compare": [_compare_int_field, _compare_enum_field], + }, "ip.frag_offset": {"scapy": "IP.frag", "compare": _compare_int_field}, "ip.ttl": {"scapy": "IP.ttl", "compare": _compare_int_field}, - "ip.proto": {"scapy": "IP.proto", "compare": _compare_int_field}, + "ip.proto": { + "scapy": "IP.proto", + "compare": [_compare_int_field, _compare_enum_field], + }, "ip.checksum": {"scapy": "IP.chksum", "compare": _compare_int_field}, "tcp.srcport": {"scapy": "TCP.sport", "compare": _compare_int_field}, "tcp.dstport": {"scapy": "TCP.dport", "compare": _compare_int_field}, @@ -166,7 +213,10 @@ dissection_corpus = [ "scapy": "TCP.urgptr", "compare": _compare_int_field, }, - "tcp.flags": {"scapy": "TCP.flags", "compare": _compare_tcp_flags}, + "tcp.flags": { + "scapy": "TCP.flags", + "compare": [_compare_tcp_flags, _compare_enum_field], + }, }, }, { @@ -179,10 +229,16 @@ dissection_corpus = [ "ip.dsfield": {"scapy": "IP.tos", "compare": _compare_int_field}, "ip.len": {"scapy": "IP.len", "compare": _compare_int_field}, "ip.id": {"scapy": "IP.id", "compare": _compare_int_field}, - "ip.flags": {"scapy": "IP.flags", "compare": _compare_int_field}, + "ip.flags": { + "scapy": "IP.flags", + "compare": [_compare_int_field, _compare_enum_field], + }, "ip.frag_offset": {"scapy": "IP.frag", "compare": _compare_int_field}, "ip.ttl": {"scapy": "IP.ttl", "compare": _compare_int_field}, - "ip.proto": {"scapy": "IP.proto", "compare": _compare_int_field}, + "ip.proto": { + "scapy": "IP.proto", + "compare": [_compare_int_field, _compare_enum_field], + }, "ip.checksum": {"scapy": "IP.chksum", "compare": _compare_int_field}, "udp.srcport": {"scapy": "UDP.sport", "compare": _compare_int_field}, "udp.dstport": {"scapy": "UDP.dport", "compare": _compare_int_field}, @@ -200,10 +256,16 @@ dissection_corpus = [ "ip.dsfield": {"scapy": "IP.tos", "compare": _compare_int_field}, "ip.len": {"scapy": "IP.len", "compare": _compare_int_field}, "ip.id": {"scapy": "IP.id", "compare": _compare_int_field}, - "ip.flags": {"scapy": "IP.flags", "compare": _compare_int_field}, + "ip.flags": { + "scapy": "IP.flags", + "compare": [_compare_int_field, _compare_enum_field], + }, "ip.frag_offset": {"scapy": "IP.frag", "compare": _compare_int_field}, "ip.ttl": {"scapy": "IP.ttl", "compare": _compare_int_field}, - "ip.proto": {"scapy": "IP.proto", "compare": _compare_int_field}, + "ip.proto": { + "scapy": "IP.proto", + "compare": [_compare_int_field, _compare_enum_field], + }, "ip.checksum": {"scapy": "IP.chksum", "compare": _compare_int_field}, "tcp.srcport": {"scapy": "TCP.sport", "compare": _compare_int_field}, "tcp.dstport": {"scapy": "TCP.dport", "compare": _compare_int_field}, @@ -218,7 +280,10 @@ dissection_corpus = [ "scapy": "TCP.urgptr", "compare": _compare_int_field, }, - "tcp.flags": {"scapy": "TCP.flags", "compare": _compare_tcp_flags}, + "tcp.flags": { + "scapy": "TCP.flags", + "compare": [_compare_tcp_flags, _compare_enum_field], + }, }, }, { @@ -231,10 +296,16 @@ dissection_corpus = [ "ip.dsfield": {"scapy": "IP.tos", "compare": _compare_int_field}, "ip.len": {"scapy": "IP.len", "compare": _compare_int_field}, "ip.id": {"scapy": "IP.id", "compare": _compare_int_field}, - "ip.flags": {"scapy": "IP.flags", "compare": _compare_int_field}, + "ip.flags": { + "scapy": "IP.flags", + "compare": [_compare_int_field, _compare_enum_field], + }, "ip.frag_offset": {"scapy": "IP.frag", "compare": _compare_int_field}, "ip.ttl": {"scapy": "IP.ttl", "compare": _compare_int_field}, - "ip.proto": {"scapy": "IP.proto", "compare": _compare_int_field}, + "ip.proto": { + "scapy": "IP.proto", + "compare": [_compare_int_field, _compare_enum_field], + }, "ip.checksum": {"scapy": "IP.chksum", "compare": _compare_int_field}, "tcp.srcport": {"scapy": "TCP.sport", "compare": _compare_int_field}, "tcp.dstport": {"scapy": "TCP.dport", "compare": _compare_int_field}, @@ -249,7 +320,10 @@ dissection_corpus = [ "scapy": "TCP.urgptr", "compare": _compare_int_field, }, - "tcp.flags": {"scapy": "TCP.flags", "compare": _compare_tcp_flags}, + "tcp.flags": { + "scapy": "TCP.flags", + "compare": [_compare_tcp_flags, _compare_enum_field], + }, }, }, { @@ -262,10 +336,16 @@ dissection_corpus = [ "ip.dsfield": {"scapy": "IP.tos", "compare": _compare_int_field}, "ip.len": {"scapy": "IP.len", "compare": _compare_int_field}, "ip.id": {"scapy": "IP.id", "compare": _compare_int_field}, - "ip.flags": {"scapy": "IP.flags", "compare": _compare_int_field}, + "ip.flags": { + "scapy": "IP.flags", + "compare": [_compare_int_field, _compare_enum_field], + }, "ip.frag_offset": {"scapy": "IP.frag", "compare": _compare_int_field}, "ip.ttl": {"scapy": "IP.ttl", "compare": _compare_int_field}, - "ip.proto": {"scapy": "IP.proto", "compare": _compare_int_field}, + "ip.proto": { + "scapy": "IP.proto", + "compare": [_compare_int_field, _compare_enum_field], + }, "ip.checksum": {"scapy": "IP.chksum", "compare": _compare_int_field}, "udp.srcport": {"scapy": "UDP.sport", "compare": _compare_int_field}, "udp.dstport": {"scapy": "UDP.dport", "compare": _compare_int_field}, From 472d912b860585d59b1e56ed8bc9ff8b39fa6551 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 04:47:39 +0000 Subject: [PATCH 09/13] Refine comparator helper structure and enum handling messages --- test/scapy/layers/dissection_corpus.uts | 57 ++++++++++++++++--------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/test/scapy/layers/dissection_corpus.uts b/test/scapy/layers/dissection_corpus.uts index 271875aa319..0d73ad55c91 100644 --- a/test/scapy/layers/dissection_corpus.uts +++ b/test/scapy/layers/dissection_corpus.uts @@ -11,8 +11,7 @@ import os def _normalize_scapy_value(value): - if isinstance(value, tuple): - value = value[0] + value = _get_scapy_raw_value(value) if value is None: return "" if isinstance(value, bytes): @@ -25,8 +24,7 @@ def _compare_tcp_flags(tshark_value, scapy_value): def _parse_int_value(value): - if isinstance(value, tuple): - value = value[0] + value = _get_scapy_raw_value(value) if value is None: return None if isinstance(value, bytes): @@ -63,7 +61,7 @@ def _default_compare(tshark_value, scapy_value): return tshark_value == _normalize_scapy_value(scapy_value) -def _normalize_enum_text(value): +def _normalize_alnum_lower(value): if value is None: return "" if isinstance(value, bytes): @@ -72,26 +70,45 @@ def _normalize_enum_text(value): def _compare_enum_field(tshark_value, scapy_value): - if isinstance(scapy_value, tuple): - scapy_value = scapy_value[1] - tshark_enum = _normalize_enum_text(tshark_value) - scapy_enum = _normalize_enum_text(scapy_value) + scapy_value = _get_scapy_display_value(scapy_value) + tshark_enum = _normalize_alnum_lower(tshark_value) + scapy_enum = _normalize_alnum_lower(scapy_value) if not tshark_enum or not scapy_enum: return False return tshark_enum == scapy_enum -def _get_compare_functions(entry): - compare = entry.get("compare", _default_compare) - if not isinstance(compare, (list, tuple)): - compare = [compare] - assert compare, "At least one compare function must be provided" - for compare_func in compare: +def _get_scapy_raw_value(scapy_value): + if isinstance(scapy_value, tuple): + return scapy_value[0] + return scapy_value + + +def _get_scapy_display_value(scapy_value): + if isinstance(scapy_value, tuple): + return scapy_value[1] + return scapy_value + + +def _get_compare_functions(tshark_field, entry): + compare = entry.get("compare") + if compare is None: + compare_functions = [_default_compare] + elif isinstance(compare, (list, tuple)): + assert compare, ( + f"At least one compare function must be provided for " + f"{tshark_field!r} ({entry.get('scapy', 'unknown')!r})" + ) + compare_functions = compare + else: + compare_functions = [compare] + for compare_func in compare_functions: assert callable(compare_func), ( - f"Invalid compare function in mapping for {entry['scapy']!r}: " + f"Invalid compare function in mapping for " + f"{tshark_field!r} ({entry.get('scapy', 'unknown')!r}): " f"{compare_func!r}" ) - return compare + return compare_functions LAYER_BY_NAME = { @@ -165,12 +182,12 @@ def _compare_pcap_dissection(pcap_file, mapping): ) for field_idx, tshark_field in enumerate(tshark_fields): entry = mapping[tshark_field] - compare_functions = _get_compare_functions(entry) + compare_functions = _get_compare_functions(tshark_field, entry) scapy_value = _get_scapy_field(packet, entry["scapy"]) tshark_value = tshark_row[field_idx] assert any( - compare(tshark_value, scapy_value) - for compare in compare_functions + compare_func(tshark_value, scapy_value) + for compare_func in compare_functions ), ( f"Mismatch in {pcap_file} packet #{packet_number} for " f"{tshark_field}/{entry['scapy']}: " From 8e3844e00a3f1a02419e5452a4f237a741ae43d7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 05:28:22 +0000 Subject: [PATCH 10/13] Extract dissection corpus helpers into typed module --- test/scapy/layers/dissection_corpus.uts | 190 +-------------- .../scapy/layers/dissection_corpus_helpers.py | 216 ++++++++++++++++++ 2 files changed, 220 insertions(+), 186 deletions(-) create mode 100644 test/scapy/layers/dissection_corpus_helpers.py diff --git a/test/scapy/layers/dissection_corpus.uts b/test/scapy/layers/dissection_corpus.uts index 0d73ad55c91..91c528ab3b8 100644 --- a/test/scapy/layers/dissection_corpus.uts +++ b/test/scapy/layers/dissection_corpus.uts @@ -7,192 +7,10 @@ = Compare Scapy and tshark dissections from pcap corpus -import os - - -def _normalize_scapy_value(value): - value = _get_scapy_raw_value(value) - if value is None: - return "" - if isinstance(value, bytes): - return value.decode("utf-8", errors="replace") - return str(value) - - -def _compare_tcp_flags(tshark_value, scapy_value): - return _compare_int_field(tshark_value, scapy_value) - - -def _parse_int_value(value): - value = _get_scapy_raw_value(value) - if value is None: - return None - if isinstance(value, bytes): - value = value.decode("utf-8", errors="replace") - if isinstance(value, str): - value = value.strip() - if value == "": - return None - try: - if value.lower().startswith(("0x", "+0x", "-0x")): - return int(value, 16) - return int(value) - except ValueError: - return None - try: - return int(value) - except (TypeError, ValueError): - return None - - -def _compare_int_field(tshark_value, scapy_value): - if tshark_value == "": - return scapy_value is None - if scapy_value is None: - return False - tshark_int = _parse_int_value(tshark_value) - scapy_int = _parse_int_value(scapy_value) - if tshark_int is None or scapy_int is None: - return False - return tshark_int == scapy_int - - -def _default_compare(tshark_value, scapy_value): - return tshark_value == _normalize_scapy_value(scapy_value) - - -def _normalize_alnum_lower(value): - if value is None: - return "" - if isinstance(value, bytes): - value = value.decode("utf-8", errors="replace") - return "".join(ch.lower() for ch in str(value) if ch.isalnum()) - - -def _compare_enum_field(tshark_value, scapy_value): - scapy_value = _get_scapy_display_value(scapy_value) - tshark_enum = _normalize_alnum_lower(tshark_value) - scapy_enum = _normalize_alnum_lower(scapy_value) - if not tshark_enum or not scapy_enum: - return False - return tshark_enum == scapy_enum - - -def _get_scapy_raw_value(scapy_value): - if isinstance(scapy_value, tuple): - return scapy_value[0] - return scapy_value - - -def _get_scapy_display_value(scapy_value): - if isinstance(scapy_value, tuple): - return scapy_value[1] - return scapy_value - - -def _get_compare_functions(tshark_field, entry): - compare = entry.get("compare") - if compare is None: - compare_functions = [_default_compare] - elif isinstance(compare, (list, tuple)): - assert compare, ( - f"At least one compare function must be provided for " - f"{tshark_field!r} ({entry.get('scapy', 'unknown')!r})" - ) - compare_functions = compare - else: - compare_functions = [compare] - for compare_func in compare_functions: - assert callable(compare_func), ( - f"Invalid compare function in mapping for " - f"{tshark_field!r} ({entry.get('scapy', 'unknown')!r}): " - f"{compare_func!r}" - ) - return compare_functions - - -LAYER_BY_NAME = { - "IP": IP, - "TCP": TCP, - "UDP": UDP, -} - - -def _get_scapy_field(packet, scapy_field): - assert scapy_field.count(".") == 1, ( - f"Invalid scapy_field format: {scapy_field!r}. " - "Expected format: LayerName.field_name (exactly one dot)" - ) - layer_name, field_name = scapy_field.split(".", 1) - assert layer_name and field_name, ( - f"Invalid scapy field mapping: {scapy_field!r}. " - "Layer and field names must be non-empty" - ) - assert layer_name in LAYER_BY_NAME, ( - f"Unsupported layer in mapping: {layer_name!r}. " - f"Supported layers: {list(LAYER_BY_NAME.keys())!r}" - ) - layer = LAYER_BY_NAME[layer_name] - if layer not in packet: - return None - assert hasattr(packet[layer], field_name), ( - f"Field {field_name!r} does not exist on layer {layer_name!r}" - ) - return ( - getattr(packet[layer], field_name), - packet.sprintf(f"%{layer_name}.{field_name}%"), - ) - - -def _extract_tshark_rows(pcap_path, mapping): - args = ["-T", "fields", "-E", "separator=\t", "-E", "occurrence=f"] - for tshark_field in mapping: - args.extend(["-e", tshark_field]) - output = tcpdump( - pcap_path, - prog=conf.prog.tshark, - getfd=True, - args=args, - dump=True, - wait=True, - ) - lines = output.decode("utf-8").splitlines() - rows = [line.split("\t") for line in lines] - return rows - - -def _compare_pcap_dissection(pcap_file, mapping): - pcap_path = scapy_path("/" + pcap_file) - assert os.path.exists(pcap_path) - - packets = rdpcap(pcap_path) - tshark_rows = _extract_tshark_rows(pcap_path, mapping) - - assert len(packets) == len(tshark_rows), ( - f"Packet count mismatch for {pcap_file}: " - f"scapy={len(packets)} tshark={len(tshark_rows)}" - ) - - tshark_fields = list(mapping) - for packet_number, (packet, tshark_row) in enumerate( - zip(packets, tshark_rows), 1 - ): - assert len(tshark_row) == len(tshark_fields), ( - f"Field count mismatch for {pcap_file} packet #{packet_number}" - ) - for field_idx, tshark_field in enumerate(tshark_fields): - entry = mapping[tshark_field] - compare_functions = _get_compare_functions(tshark_field, entry) - scapy_value = _get_scapy_field(packet, entry["scapy"]) - tshark_value = tshark_row[field_idx] - assert any( - compare_func(tshark_value, scapy_value) - for compare_func in compare_functions - ), ( - f"Mismatch in {pcap_file} packet #{packet_number} for " - f"{tshark_field}/{entry['scapy']}: " - f"tshark={tshark_value!r} scapy={scapy_value!r}" - ) +from test.scapy.layers.dissection_corpus_helpers import _compare_enum_field +from test.scapy.layers.dissection_corpus_helpers import _compare_int_field +from test.scapy.layers.dissection_corpus_helpers import _compare_pcap_dissection +from test.scapy.layers.dissection_corpus_helpers import _compare_tcp_flags dissection_corpus = [ diff --git a/test/scapy/layers/dissection_corpus_helpers.py b/test/scapy/layers/dissection_corpus_helpers.py new file mode 100644 index 00000000000..51ae1b2dc8c --- /dev/null +++ b/test/scapy/layers/dissection_corpus_helpers.py @@ -0,0 +1,216 @@ +import os +from typing import Any +from typing import Callable +from typing import Dict +from typing import List +from typing import Mapping +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import Union + +from scapy.config import conf +from scapy.layers.inet import IP +from scapy.layers.inet import TCP +from scapy.layers.inet import UDP +from scapy.packet import Packet +from scapy.tools.UTscapy import scapy_path +from scapy.utils import rdpcap +from scapy.utils import tcpdump + +ScapyFieldValue = Optional[Tuple[Any, str]] +CompareFunction = Callable[[str, ScapyFieldValue], bool] +CompareEntry = Union[CompareFunction, Sequence[CompareFunction]] +DissectionMapEntry = Dict[str, Any] +DissectionMapping = Mapping[str, DissectionMapEntry] + +LAYER_BY_NAME = { + "IP": IP, + "TCP": TCP, + "UDP": UDP, +} + + +def _get_scapy_raw_value(scapy_value: ScapyFieldValue) -> Any: + if isinstance(scapy_value, tuple): + return scapy_value[0] + return scapy_value + + +def _get_scapy_display_value(scapy_value: ScapyFieldValue) -> Any: + if isinstance(scapy_value, tuple): + return scapy_value[1] + return scapy_value + + +def _normalize_scapy_value(value: ScapyFieldValue) -> str: + value = _get_scapy_raw_value(value) + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return str(value) + + +def _parse_int_value(value: ScapyFieldValue) -> Optional[int]: + value = _get_scapy_raw_value(value) + if value is None: + return None + if isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + if isinstance(value, str): + value = value.strip() + if value == "": + return None + try: + if value.lower().startswith(("0x", "+0x", "-0x")): + return int(value, 16) + return int(value) + except ValueError: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _compare_int_field(tshark_value: str, scapy_value: ScapyFieldValue) -> bool: + if tshark_value == "": + return scapy_value is None + if scapy_value is None: + return False + tshark_int = _parse_int_value(tshark_value) + scapy_int = _parse_int_value(scapy_value) + if tshark_int is None or scapy_int is None: + return False + return tshark_int == scapy_int + + +def _compare_tcp_flags(tshark_value: str, scapy_value: ScapyFieldValue) -> bool: + return _compare_int_field(tshark_value, scapy_value) + + +def _default_compare(tshark_value: str, scapy_value: ScapyFieldValue) -> bool: + return tshark_value == _normalize_scapy_value(scapy_value) + + +def _normalize_alnum_lower(value: Any) -> str: + if value is None: + return "" + if isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + return "".join(ch.lower() for ch in str(value) if ch.isalnum()) + + +def _compare_enum_field(tshark_value: str, scapy_value: ScapyFieldValue) -> bool: + scapy_value = _get_scapy_display_value(scapy_value) + tshark_enum = _normalize_alnum_lower(tshark_value) + scapy_enum = _normalize_alnum_lower(scapy_value) + if not tshark_enum or not scapy_enum: + return False + return tshark_enum == scapy_enum + + +def _get_compare_functions( + tshark_field: str, entry: DissectionMapEntry +) -> List[CompareFunction]: + compare = entry.get("compare") + compare_functions: Sequence[CompareFunction] + if compare is None: + compare_functions = [_default_compare] + elif isinstance(compare, (list, tuple)): + assert compare, ( + f"At least one compare function must be provided for " + f"{tshark_field!r} ({entry.get('scapy', 'unknown')!r})" + ) + compare_functions = compare + else: + compare_functions = [compare] + for compare_func in compare_functions: + assert callable(compare_func), ( + f"Invalid compare function in mapping for " + f"{tshark_field!r} ({entry.get('scapy', 'unknown')!r}): " + f"{compare_func!r}" + ) + return list(compare_functions) + + +def _get_scapy_field(packet: Packet, scapy_field: str) -> ScapyFieldValue: + assert scapy_field.count(".") == 1, ( + f"Invalid scapy_field format: {scapy_field!r}. " + "Expected format: LayerName.field_name (exactly one dot)" + ) + layer_name, field_name = scapy_field.split(".", 1) + assert layer_name and field_name, ( + f"Invalid scapy field mapping: {scapy_field!r}. " + "Layer and field names must be non-empty" + ) + assert layer_name in LAYER_BY_NAME, ( + f"Unsupported layer in mapping: {layer_name!r}. " + f"Supported layers: {list(LAYER_BY_NAME.keys())!r}" + ) + layer = LAYER_BY_NAME[layer_name] + if layer not in packet: + return None + assert hasattr(packet[layer], field_name), ( + f"Field {field_name!r} does not exist on layer {layer_name!r}" + ) + return ( + getattr(packet[layer], field_name), + packet.sprintf("%%%s.%s%%" % (layer_name, field_name)), + ) + + +def _extract_tshark_rows( + pcap_path: str, mapping: DissectionMapping +) -> List[List[str]]: + args = ["-T", "fields", "-E", "separator=\t", "-E", "occurrence=f"] + for tshark_field in mapping: + args.extend(["-e", tshark_field]) + output = tcpdump( + pcap_path, + prog=conf.prog.tshark, + getfd=True, + args=args, + dump=True, + wait=True, + ) + lines = output.decode("utf-8").splitlines() + rows = [line.split("\t") for line in lines] + return rows + + +def _compare_pcap_dissection( + pcap_file: str, mapping: DissectionMapping +) -> None: + pcap_path = scapy_path("/" + pcap_file) + assert os.path.exists(pcap_path) + + packets = rdpcap(pcap_path) + tshark_rows = _extract_tshark_rows(pcap_path, mapping) + + assert len(packets) == len(tshark_rows), ( + f"Packet count mismatch for {pcap_file}: " + f"scapy={len(packets)} tshark={len(tshark_rows)}" + ) + + tshark_fields = list(mapping) + for packet_number, (packet, tshark_row) in enumerate( + zip(packets, tshark_rows), 1 + ): + assert len(tshark_row) == len(tshark_fields), ( + f"Field count mismatch for {pcap_file} packet #{packet_number}" + ) + for field_idx, tshark_field in enumerate(tshark_fields): + entry = mapping[tshark_field] + compare_functions = _get_compare_functions(tshark_field, entry) + scapy_value = _get_scapy_field(packet, entry["scapy"]) + tshark_value = tshark_row[field_idx] + assert any( + compare_func(tshark_value, scapy_value) + for compare_func in compare_functions + ), ( + f"Mismatch in {pcap_file} packet #{packet_number} for " + f"{tshark_field}/{entry['scapy']}: " + f"tshark={tshark_value!r} scapy={scapy_value!r}" + ) From c8d5b090ea96bb508225a83300b00e052dbe4027 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 05:28:54 +0000 Subject: [PATCH 11/13] Use f-string in dissection helper sprintf --- test/scapy/layers/dissection_corpus_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/scapy/layers/dissection_corpus_helpers.py b/test/scapy/layers/dissection_corpus_helpers.py index 51ae1b2dc8c..2aab27034fa 100644 --- a/test/scapy/layers/dissection_corpus_helpers.py +++ b/test/scapy/layers/dissection_corpus_helpers.py @@ -157,7 +157,7 @@ def _get_scapy_field(packet: Packet, scapy_field: str) -> ScapyFieldValue: ) return ( getattr(packet[layer], field_name), - packet.sprintf("%%%s.%s%%" % (layer_name, field_name)), + packet.sprintf(f"%{layer_name}.{field_name}%"), ) From d44b5668d740a647141c588a52b923b6890708ff Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 12:46:46 +0000 Subject: [PATCH 12/13] Treat Copilot bot commits as reminder-only in AI trailer check --- .config/ci/check_commits.sh | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.config/ci/check_commits.sh b/.config/ci/check_commits.sh index f5efc667c48..4166c3b3680 100755 --- a/.config/ci/check_commits.sh +++ b/.config/ci/check_commits.sh @@ -13,8 +13,24 @@ if [ -z "$commits" ]; then fi missing=0 +copilot_missing=0 for c in $commits; do if ! git log -1 --format=%B "$c" | grep -qi '^AI-Assisted:'; then + mapfile -t author_data < <(git log -1 --format='%an%n%ae' "$c") + author_name="${author_data[0]}" + author_email="${author_data[1]}" + is_copilot_commit=0 + if [[ "$author_name" == "copilot-swe-agent[bot]" ]]; then + is_copilot_commit=1 + elif [[ "$author_email" =~ \+Copilot@users\.noreply\.github\.com$ ]]; then + is_copilot_commit=1 + fi + + if [ $is_copilot_commit -eq 1 ]; then + echo -e "REMINDER: Commit \033[0;33m$c\033[0m (Copilot bot) is missing the 'AI-Assisted: yes|no [tool(s)]' trailer." + copilot_missing=1 + continue + fi echo -e "ERROR: Commit \033[0;33m$c\033[0m is missing the 'AI-Assisted: yes|no [tool(s)]' trailer." missing=1 else @@ -28,6 +44,10 @@ if [ $missing -eq 1 ]; then echo "See the contribution guide at: https://github.com/secdev/scapy/blob/master/CONTRIBUTING.md" exit 1 else - echo "All checked commits include the AI-Assisted trailer." + if [ $copilot_missing -eq 1 ]; then + echo "AI-Assisted trailer missing only in Copilot bot commits (reminder-only)." + else + echo "All checked commits include the AI-Assisted trailer." + fi exit 0 fi From c7e416844f84825c990dceab7258423913b01cbb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 6 Jun 2026 05:07:20 +0000 Subject: [PATCH 13/13] Use raw TCP sequence/ack fields in dissection corpus mappings --- test/scapy/layers/dissection_corpus.uts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/scapy/layers/dissection_corpus.uts b/test/scapy/layers/dissection_corpus.uts index 91c528ab3b8..d3a97e00eee 100644 --- a/test/scapy/layers/dissection_corpus.uts +++ b/test/scapy/layers/dissection_corpus.uts @@ -37,8 +37,8 @@ dissection_corpus = [ "ip.checksum": {"scapy": "IP.chksum", "compare": _compare_int_field}, "tcp.srcport": {"scapy": "TCP.sport", "compare": _compare_int_field}, "tcp.dstport": {"scapy": "TCP.dport", "compare": _compare_int_field}, - "tcp.seq": {"scapy": "TCP.seq", "compare": _compare_int_field}, - "tcp.ack": {"scapy": "TCP.ack", "compare": _compare_int_field}, + "tcp.seq_raw": {"scapy": "TCP.seq", "compare": _compare_int_field}, + "tcp.ack_raw": {"scapy": "TCP.ack", "compare": _compare_int_field}, "tcp.window_size_value": { "scapy": "TCP.window", "compare": _compare_int_field, @@ -104,8 +104,8 @@ dissection_corpus = [ "ip.checksum": {"scapy": "IP.chksum", "compare": _compare_int_field}, "tcp.srcport": {"scapy": "TCP.sport", "compare": _compare_int_field}, "tcp.dstport": {"scapy": "TCP.dport", "compare": _compare_int_field}, - "tcp.seq": {"scapy": "TCP.seq", "compare": _compare_int_field}, - "tcp.ack": {"scapy": "TCP.ack", "compare": _compare_int_field}, + "tcp.seq_raw": {"scapy": "TCP.seq", "compare": _compare_int_field}, + "tcp.ack_raw": {"scapy": "TCP.ack", "compare": _compare_int_field}, "tcp.window_size_value": { "scapy": "TCP.window", "compare": _compare_int_field, @@ -144,8 +144,8 @@ dissection_corpus = [ "ip.checksum": {"scapy": "IP.chksum", "compare": _compare_int_field}, "tcp.srcport": {"scapy": "TCP.sport", "compare": _compare_int_field}, "tcp.dstport": {"scapy": "TCP.dport", "compare": _compare_int_field}, - "tcp.seq": {"scapy": "TCP.seq", "compare": _compare_int_field}, - "tcp.ack": {"scapy": "TCP.ack", "compare": _compare_int_field}, + "tcp.seq_raw": {"scapy": "TCP.seq", "compare": _compare_int_field}, + "tcp.ack_raw": {"scapy": "TCP.ack", "compare": _compare_int_field}, "tcp.window_size_value": { "scapy": "TCP.window", "compare": _compare_int_field,