Skip to content

Commit a86da7a

Browse files
Merge branch 'vlad/native-heap-gotter-freeprobe-pr-e' into vlad/native-heap-ci-opt-in
2 parents af9a77f + 7d2117f commit a86da7a

10 files changed

Lines changed: 154 additions & 90 deletions

File tree

ddtrace/internal/writer/writer.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,14 @@
99
from typing import Any
1010
from typing import Callable
1111
from typing import Optional
12+
from typing import Sequence
1213
from typing import TextIO
1314

1415
from ddtrace.internal.dist_computing.utils import in_ray_job
1516
from ddtrace.internal.hostname import get_hostname
1617
import ddtrace.internal.native as native
1718
from ddtrace.internal.native import AgentResponse
19+
from ddtrace.internal.native._native import SpanData
1820
from ddtrace.internal.native_runtime import get_native_runtime
1921
from ddtrace.internal.runtime import get_runtime_id
2022
from ddtrace.internal.settings import env
@@ -60,7 +62,6 @@
6062

6163

6264
if TYPE_CHECKING: # pragma: no cover
63-
from ddtrace._trace.span import Span # noqa:F401
6465
from ddtrace.vendor.dogstatsd import DogStatsd
6566

6667
from .utils.http import ConnectionType # noqa:F401
@@ -129,7 +130,7 @@ def stop(self, timeout: Optional[float] = None) -> None:
129130
pass
130131

131132
@abc.abstractmethod
132-
def write(self, spans: Optional[list["Span"]] = None) -> None:
133+
def write(self, spans: Optional[Sequence[SpanData]] = None) -> None:
133134
pass
134135

135136
@abc.abstractmethod
@@ -161,7 +162,7 @@ def recreate(
161162
def stop(self, timeout: Optional[float] = None) -> None:
162163
return
163164

164-
def write(self, spans: Optional[list["Span"]] = None) -> None:
165+
def write(self, spans: Optional[Sequence[SpanData]] = None) -> None:
165166
if not spans:
166167
return
167168
encoded = self.encoder.encode_traces([spans])
@@ -407,7 +408,7 @@ def write(self, spans=None):
407408
if self._sync_mode:
408409
self.flush_queue()
409410

410-
def _write_with_client(self, client: WriterClientBase, spans: Optional[list["Span"]] = None) -> None:
411+
def _write_with_client(self, client: WriterClientBase, spans: Optional[Sequence[SpanData]] = None) -> None:
411412
if spans is None:
412413
return
413414

@@ -1087,13 +1088,13 @@ def _send_payload(self, payload: bytes, count: int, client: WriterClientBase):
10871088
)
10881089
)
10891090

1090-
def write(self, spans: Optional[list["Span"]] = None) -> None:
1091+
def write(self, spans: Optional[Sequence[SpanData]] = None) -> None:
10911092
for client in self._clients:
10921093
self._write_with_client(client, spans=spans)
10931094
if self._sync_mode:
10941095
self.flush_queue()
10951096

1096-
def _write_with_client(self, client: WriterClientBase, spans: Optional[list["Span"]] = None) -> None:
1097+
def _write_with_client(self, client: WriterClientBase, spans: Optional[Sequence[SpanData]] = None) -> None:
10971098
if spans is None:
10981099
return
10991100

riotfile.py

Lines changed: 1 addition & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
376376
"pip": "<25",
377377
},
378378
env={
379+
"BROWSER": "true", # Prevent webbrowser tests from launching the host browser.
379380
"_DD_IAST_PATCH_MODULES": "benchmarks.,tests.appsec.",
380381
"DD_IAST_REQUEST_SAMPLING": "100",
381382
"DD_IAST_DEDUPLICATION_ENABLED": "false",
@@ -3734,15 +3735,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
37343735
"protobuf": latest,
37353736
},
37363737
),
3737-
# safe_memcpy fast-copy path (process_vm_readv is the default)
3738-
Venv(
3739-
env={
3740-
"_DD_PROFILING_STACK_FAST_COPY": "1",
3741-
},
3742-
pkgs={
3743-
"protobuf": latest,
3744-
},
3745-
),
37463738
],
37473739
),
37483740
# Python 3.10
@@ -3776,15 +3768,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
37763768
"protobuf": latest,
37773769
},
37783770
),
3779-
# safe_memcpy fast-copy path (process_vm_readv is the default)
3780-
Venv(
3781-
env={
3782-
"_DD_PROFILING_STACK_FAST_COPY": "1",
3783-
},
3784-
pkgs={
3785-
"protobuf": latest,
3786-
},
3787-
),
37883771
],
37893772
),
37903773
# Python >= 3.11 (excluding 3.14)
@@ -3818,15 +3801,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
38183801
"protobuf": latest,
38193802
},
38203803
),
3821-
# safe_memcpy fast-copy path (process_vm_readv is the default)
3822-
Venv(
3823-
env={
3824-
"_DD_PROFILING_STACK_FAST_COPY": "1",
3825-
},
3826-
pkgs={
3827-
"protobuf": latest,
3828-
},
3829-
),
38303804
],
38313805
),
38323806
# Python 3.14 - protobuf 4.22.0 is not compatible (TypeError: Metaclasses with custom tp_new)
@@ -3861,15 +3835,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
38613835
"protobuf": latest,
38623836
},
38633837
),
3864-
# safe_memcpy fast-copy path (process_vm_readv is the default)
3865-
Venv(
3866-
env={
3867-
"_DD_PROFILING_STACK_FAST_COPY": "1",
3868-
},
3869-
pkgs={
3870-
"protobuf": latest,
3871-
},
3872-
),
38733838
],
38743839
),
38753840
Venv(

tests/appsec/appsec_utils.py

Lines changed: 57 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,50 @@
2222
FILE_PATH = Path(__file__).resolve().parent
2323

2424

25+
def _port_is_available(port: int) -> bool:
26+
"""Whether a server could bind the port right now.
27+
28+
Binding is the question that matters, since it is what the next server does. Probing with
29+
connect() instead reports a port as free once a bound server's listen backlog fills, and
30+
opens real connections to a live server.
31+
"""
32+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
33+
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
34+
try:
35+
sock.bind(("0.0.0.0", int(port)))
36+
return True
37+
except OSError:
38+
return False
39+
40+
41+
def _wait_for_port_release(port: int, timeout: float = 10.0) -> bool:
42+
"""Wait until the port can be bound again, returning False if it never can.
43+
44+
Server teardown is best effort and gunicorn workers can outlive it, so without this the
45+
next test to use the same port fails to bind. 31 tests share port 8050.
46+
"""
47+
deadline = time.monotonic() + timeout
48+
while time.monotonic() < deadline:
49+
if _port_is_available(port):
50+
return True
51+
time.sleep(0.1)
52+
return _port_is_available(port)
53+
54+
55+
def _server_diagnostics(server_process, port: int, cmd: list) -> str:
56+
"""Facts that are not in the captured server output but explain most startup failures."""
57+
if isinstance(server_process, multiprocessing.Process):
58+
exit_code = server_process.exitcode
59+
else:
60+
exit_code = server_process.poll()
61+
return (
62+
f"port={port} port_still_bound={not _port_is_available(port)} pid={server_process.pid} "
63+
f"exit_code={exit_code} (None means it was still running, so it was too slow rather "
64+
f"than dead; a non-zero code with the port bound means another server still holds it)\n"
65+
f"command={cmd}"
66+
)
67+
68+
2569
@contextmanager
2670
def gunicorn_flask_server(
2771
use_ddtrace_cmd: bool = True,
@@ -335,6 +379,10 @@ def appsec_application_server(
335379
if preexec is not None:
336380
subprocess_kwargs["preexec_fn"] = preexec # type: ignore[assignment]
337381

382+
# A previous test's server may still hold the port, which would make this one fail to bind.
383+
if not _wait_for_port_release(port):
384+
print(f"WARNING: port {port} was still bound when starting the server")
385+
338386
if use_multiprocess:
339387
# Run the server command by replacing the child Python process with the target binary (exec),
340388
# ensuring signals/termination behave like the subprocess.Popen path.
@@ -374,17 +422,13 @@ def appsec_application_server(
374422
print("Server started")
375423
except RetryError:
376424
raise AssertionError(
377-
"Server failed to start, see stdout and stderr logs"
378-
"\n=== Captured STDOUT ===\n%s=== End of captured STDOUT ==="
379-
"\n=== Captured STDERR ===\n%s=== End of captured STDERR ==="
380-
% (getattr(server_process, "stdout", None), getattr(server_process, "stderr", None))
425+
"Server failed to start; its output is in the captured stdout/stderr above.\n"
426+
+ _server_diagnostics(server_process, port, cmd)
381427
)
382428
except Exception:
383429
raise AssertionError(
384-
"Server FAILED, see stdout and stderr logs"
385-
"\n=== Captured STDOUT ===\n%s=== End of captured STDOUT ==="
386-
"\n=== Captured STDERR ===\n%s=== End of captured STDERR ==="
387-
% (getattr(server_process, "stdout", None), getattr(server_process, "stderr", None))
430+
"Server FAILED; its output is in the captured stdout/stderr above.\n"
431+
+ _server_diagnostics(server_process, port, cmd)
388432
)
389433

390434
# If we run a Gunicorn application, we want to get the child's pid, see test_flask_remoteconfig.py
@@ -399,9 +443,8 @@ def appsec_application_server(
399443
pass
400444
except Exception:
401445
raise AssertionError(
402-
"\n=== Captured STDOUT ===\n%s=== End of captured STDOUT ==="
403-
"\n=== Captured STDERR ===\n%s=== End of captured STDERR ==="
404-
% (getattr(server_process, "stdout", None), getattr(server_process, "stderr", None))
446+
"Server shutdown request failed; its output is in the captured stdout/stderr above.\n"
447+
+ _server_diagnostics(server_process, port, cmd)
405448
)
406449
finally:
407450
try:
@@ -433,7 +476,9 @@ def appsec_application_server(
433476
assert "Return value is tainted" in stderr_output
434477
assert "Tainted arguments:" in stderr_output
435478
finally:
436-
pass
479+
# Do not hand the port to the next test while a worker still holds it.
480+
if not _wait_for_port_release(port):
481+
print(f"WARNING: port {port} still bound after server teardown")
437482

438483

439484
def _mp_target(_cmd: list[str], _env: dict) -> None:

tests/appsec/contrib_appsec/test_django.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import importlib
22
import os
3+
from pathlib import Path
4+
import shutil
35

46
import django
57
from django.conf import settings
@@ -15,6 +17,25 @@
1517

1618
_FLAT_URLCONF = "tests.appsec.contrib_appsec.django_app.urls"
1719
_SUBAPP_URLCONF = "tests.appsec.contrib_appsec.django_app.urls_subapps"
20+
_DATABASE_TEMPLATE = Path(__file__).with_name("db.sqlite3")
21+
22+
23+
@pytest.fixture(scope="module", autouse=True)
24+
def isolated_database(tmp_path_factory):
25+
"""Use a worker-local copy of the Django database template."""
26+
database_path = tmp_path_factory.mktemp("appsec-django") / "db.sqlite3"
27+
shutil.copyfile(_DATABASE_TEMPLATE, database_path)
28+
29+
os.environ["DJANGO_SETTINGS_MODULE"] = "tests.appsec.contrib_appsec.django_app.settings"
30+
original_database_name = settings.DATABASES["default"]["NAME"]
31+
settings.DATABASES["default"]["NAME"] = str(database_path)
32+
try:
33+
yield
34+
finally:
35+
from django.db import connections
36+
37+
connections.close_all()
38+
settings.DATABASES["default"]["NAME"] = original_database_name
1839

1940

2041
class _Test_Django_Base:

tests/appsec/iast/fixtures/taint_sinks/path_traversal.py

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import pickle
99
import shutil
1010
import tarfile
11+
import tempfile
1112
from zipfile import ZipFile
1213

1314

@@ -58,8 +59,9 @@ def path_os_remove(origin_string):
5859

5960
def path_os_rename(origin_string):
6061
try:
61-
# label path_os_rename
62-
os.rename(origin_string, "test.txt")
62+
with tempfile.TemporaryDirectory() as tmp_dir:
63+
# label path_os_rename
64+
os.rename(origin_string, os.path.join(tmp_dir, "test.txt"))
6365
except Exception:
6466
pass
6567

@@ -90,24 +92,27 @@ def path_os_listdir(origin_string):
9092

9193
def path_shutil_copy(origin_string):
9294
try:
93-
# label path_shutil_copy
94-
shutil.copy(origin_string, "not_exists.txt2")
95+
with tempfile.TemporaryDirectory() as tmp_dir:
96+
# label path_shutil_copy
97+
shutil.copy(origin_string, os.path.join(tmp_dir, "copied.txt"))
9598
except Exception:
9699
pass
97100

98101

99102
def path_shutil_copytree(origin_string):
100103
try:
101-
# label path_shutil_copytree
102-
shutil.copytree(origin_string, "not_exists.txt2")
104+
with tempfile.TemporaryDirectory() as tmp_dir:
105+
# label path_shutil_copytree
106+
shutil.copytree(origin_string, os.path.join(tmp_dir, "copied"))
103107
except Exception:
104108
pass
105109

106110

107111
def path_shutil_move(origin_string):
108112
try:
109-
# label path_shutil_move
110-
shutil.move(origin_string, "not_exists.txt2")
113+
with tempfile.TemporaryDirectory() as tmp_dir:
114+
# label path_shutil_move
115+
shutil.move(origin_string, os.path.join(tmp_dir, "moved.txt"))
111116
except Exception:
112117
pass
113118

tests/appsec/iast/test_product_inspect_regression.py

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
import sys
1919

20-
from tests.utils import override_env
20+
from tests.utils import override_global_config
2121

2222

2323
def sample_function_with_many_params(
@@ -84,16 +84,9 @@ def test_iast_post_preload_does_not_drop_inspect(self):
8484
inspect_id_before = id(sys.modules.get("inspect"))
8585
inspect_module_before = sys.modules.get("inspect")
8686

87-
# Call post_preload directly (this is what happens in production)
88-
# We need to ensure IAST is enabled for post_preload to run its logic
89-
with override_env({"DD_IAST_ENABLED": "true"}):
90-
# Force reload of asm_config to pick up the environment variable
91-
import importlib
92-
93-
from ddtrace.internal.settings import asm
94-
95-
importlib.reload(asm)
96-
87+
# Enabled in place: reloading the asm module would rebind its config to a new object,
88+
# leaving every module that already imported the old one reading stale settings.
89+
with override_global_config(dict(_iast_enabled=True)):
9790
from ddtrace.internal.iast import product
9891

9992
# Call post_preload - this should NOT drop inspect

0 commit comments

Comments
 (0)