diff --git a/newsfragments/5133.bugfix.rst b/newsfragments/5133.bugfix.rst
new file mode 100644
index 0000000000..869b7fb517
--- /dev/null
+++ b/newsfragments/5133.bugfix.rst
@@ -0,0 +1,2 @@
+Fixed ``build_py`` to avoid copying extension source files from ``SOURCES.txt``
+into ``build/lib`` as package data.
diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py
index 3c7c2d1bd6..893994501c 100644
--- a/setuptools/command/build_py.py
+++ b/setuptools/command/build_py.py
@@ -182,6 +182,7 @@ def analyze_manifest(self) -> None:
         for package in self.packages or ():
             # Locate package source directory
             src_dirs[assert_relative(self.get_package_dir(package))] = package
+        extension_files = self._get_extension_source_files()
 
         if (
             self.existing_egg_info_dir
@@ -197,7 +198,12 @@ def analyze_manifest(self) -> None:
             files = ei_cmd.filelist.files
 
         check = _IncludePackageDataAbuse()
-        for path in self._filter_build_files(files, egg_info_dir):
+        files = (
+            path
+            for path in self._filter_build_files(files, egg_info_dir)
+            if self._path_norm(path) not in extension_files
+        )
+        for path in files:
             d, f = os.path.split(assert_relative(path))
             prev = None
             oldf = f
@@ -215,6 +221,18 @@ def analyze_manifest(self) -> None:
                         check.warn(importable)
                 self.manifest_files.setdefault(src_dirs[d], []).append(path)
 
+    def _get_extension_source_files(self) -> set[str]:
+        """Return extension source files that are build inputs."""
+        return {
+            self._path_norm(path)
+            for ext in self.distribution.ext_modules or ()
+            for path in getattr(ext, "sources", None) or ()
+        }
+
+    @staticmethod
+    def _path_norm(path: StrPath) -> str:
+        return os.path.normcase(os.path.abspath(os.fspath(path)))
+
     def _filter_build_files(
         self, files: Iterable[str], egg_info: StrPath
     ) -> Iterator[str]:
diff --git a/setuptools/tests/test_build_py.py b/setuptools/tests/test_build_py.py
index 78848f7182..49b4096d15 100644
--- a/setuptools/tests/test_build_py.py
+++ b/setuptools/tests/test_build_py.py
@@ -10,6 +10,7 @@
 
 from setuptools import SetuptoolsDeprecationWarning
 from setuptools.dist import Distribution
+from setuptools.extension import Extension
 
 from .textwrap import DALS
 
@@ -258,6 +259,48 @@ def test_existing_egg_info(tmpdir_cwd, monkeypatch):
     assert example in outputs
 
 
+def test_extension_sources_are_not_package_data(tmpdir_cwd):
+    """
+    Extension source files belong in sdists, but build_py should not copy them
+    to build/lib as package data.
+    """
+    jaraco.path.build({
+        "src": {
+            "foo": {
+                "__init__.py": "",
+                "resource.txt": "",
+                "extension": {"bar.cpp": ""},
+            },
+        },
+    })
+    egg_info_dir = Path("src/foo.egg-info")
+    egg_info_dir.mkdir()
+    (egg_info_dir / "SOURCES.txt").write_text(
+        f"{os.path.join('src', 'foo', 'resource.txt')}\n"
+        f"{os.path.join('src', 'foo', 'extension', 'bar.cpp')}\n",
+        encoding="utf-8",
+    )
+    dist = Distribution({
+        "name": "foo",
+        "version": "1",
+        "script_name": "%build_py-test%",
+        "packages": ["foo"],
+        "package_dir": {"": "src"},
+        "include_package_data": True,
+        "ext_modules": [Extension("foo.extension", ["src/foo/extension/bar.cpp"])],
+    })
+
+    build_py = dist.get_command_obj("build_py")
+    build_py.finalize_options()
+    build_py.existing_egg_info_dir = egg_info_dir
+    build_py.run()
+
+    outputs = get_outputs(build_py)
+    assert "foo/__init__.py" in outputs
+    assert "foo/resource.txt" in outputs
+    assert "foo/extension/bar.cpp" not in outputs
+
+
 EXAMPLE_ARBITRARY_MAPPING = {
     "pyproject.toml": DALS(
         """