Add JSX support built on the JS backend (#21206)

This is a small change for the JS backend, as the tree-sitter parser
already supports JSX.
 
This is a fairly big decision w.r.t how the js backend will evolve with
typescript and other javascript transpile formats. Since many tools that
support js also supports other kinds of files, the `JSRuntime...` will
be used to refer to the files that can be treated as a source code file
for some node-ish runtime who either processes the AST or executes
actual programmes.

Clarifiying examples:

1. A linter that can process any js-esque file should accept
JSRuntimeSourceField
2. All test sources should use JSRuntimeDependencies
3. Files that can cross import (post bundling) different transpile
formats should use JSRuntimeDependencies and JSRuntimeSourceField. Think
typescript, JSX.
4. A checker that can only process typed vue-templates should not accept
JSRuntimeSourceField.

This PR together with #21176
essentially gives pants "react" support.

---------

Co-authored-by: riisi <[email protected]>

Author: tobni

.gitignore

@@ -68,3 +68,4 @@ GTAGS
 /.venv
 .tool-versions
 TAGS
+node_modules

docs/notes/2.23.x.md

@@ -195,6 +195,9 @@ Pants now applies dependency inference according to the most permissive "bundler
 [jsconfig.json](https://code.visualstudio.com/docs/languages/jsconfig), when a jsconfig.json is
 part of your javascript workspace.
 
+Pants now ships with experimental JSX support, including Prettier formatting and JS testing as part of the
+JS backend.
+
 #### Shell
 
 The `tailor` goal now has independent options for tailoring `shell_sources` and `shunit2_tests` targets. The option was split from `tailor` into [`tailor_sources`](https://www.pantsbuild.org/2.23/reference/subsystems/shell-setup#tailor_sources) and [`tailor_shunit2_tests`](https://www.pantsbuild.org/2.23/reference/subsystems/shell-setup#tailor_shunit2_tests).

src/python/pants/backend/experimental/javascript/register.py

@@ -16,6 +16,13 @@
     JSTestsGeneratorTarget,
     JSTestTarget,
 )
+from pants.backend.jsx.goals import tailor as jsx_tailor
+from pants.backend.jsx.target_types import (
+    JSXSourcesGeneratorTarget,
+    JSXSourceTarget,
+    JSXTestsGeneratorTarget,
+    JSXTestTarget,
+)
 from pants.build_graph.build_file_aliases import BuildFileAliases
 from pants.engine.rules import Rule
 from pants.engine.target import Target
@@ -31,6 +38,7 @@ def rules() -> Iterable[Rule | UnionRule]:
         *run_rules(),
         *test.rules(),
         *export.rules(),
+        *jsx_tailor.rules(),
     )
 
 
@@ -40,6 +48,10 @@ def target_types() -> Iterable[type[Target]]:
         JSSourcesGeneratorTarget,
         JSTestTarget,
         JSTestsGeneratorTarget,
+        JSXSourceTarget,
+        JSXSourcesGeneratorTarget,
+        JSXTestTarget,
+        JSXTestsGeneratorTarget,
         *package_json.target_types(),
     )
 

src/python/pants/backend/javascript/dependency_inference/rules.py

@@ -25,9 +25,10 @@
 from pants.backend.javascript.subsystems.nodejs_infer import NodeJSInfer
 from pants.backend.javascript.target_types import (
     JS_FILE_EXTENSIONS,
-    JSDependenciesField,
-    JSSourceField,
+    JSRuntimeDependenciesField,
+    JSRuntimeSourceField,
 )
+from pants.backend.jsx.target_types import JSX_FILE_EXTENSIONS
 from pants.backend.typescript import tsconfig
 from pants.backend.typescript.tsconfig import ParentTSConfigRequest, TSConfig, find_parent_ts_config
 from pants.build_graph.address import Address
@@ -74,10 +75,10 @@ class InferNodePackageDependenciesRequest(InferDependenciesRequest):
 
 @dataclass(frozen=True)
 class JSSourceInferenceFieldSet(FieldSet):
-    required_fields = (JSSourceField, JSDependenciesField)
+    required_fields = (JSRuntimeSourceField, JSRuntimeDependenciesField)
 
-    source: JSSourceField
-    dependencies: JSDependenciesField
+    source: JSRuntimeSourceField
+    dependencies: JSRuntimeDependenciesField
 
 
 class InferJSDependenciesRequest(InferDependenciesRequest):
@@ -96,7 +97,9 @@ async def infer_node_package_dependencies(
     )
     candidate_js_files = await Get(Owners, OwnersRequest(tuple(entry_points.globs_from_root())))
     js_targets = await Get(Targets, Addresses(candidate_js_files))
-    return InferredDependencies(tgt.address for tgt in js_targets if tgt.has_field(JSSourceField))
+    return InferredDependencies(
+        tgt.address for tgt in js_targets if tgt.has_field(JSRuntimeSourceField)
+    )
 
 
 class NodePackageCandidateMap(FrozenDict[str, Address]):
@@ -155,7 +158,8 @@ async def _prepare_inference_metadata(address: Address, file_path: str) -> Infer
 
 
 def _add_extensions(file_imports: frozenset[str]) -> PathGlobs:
-    extensions = JS_FILE_EXTENSIONS + tuple(f"/index{ext}" for ext in JS_FILE_EXTENSIONS)
+    file_extensions = (*JS_FILE_EXTENSIONS, *JSX_FILE_EXTENSIONS)
+    extensions = file_extensions + tuple(f"/index{ext}" for ext in file_extensions)
     return PathGlobs(
         string
         for file_import in file_imports
@@ -227,12 +231,12 @@ async def infer_js_source_dependencies(
     request: InferJSDependenciesRequest,
     nodejs_infer: NodeJSInfer,
 ) -> InferredDependencies:
-    source: JSSourceField = request.field_set.source
+    source: JSRuntimeSourceField = request.field_set.source
     if not nodejs_infer.imports:
         return InferredDependencies(())
 
     sources = await Get(
-        HydratedSources, HydrateSourcesRequest(source, for_sources_types=[JSSourceField])
+        HydratedSources, HydrateSourcesRequest(source, for_sources_types=[JSRuntimeSourceField])
     )
     metadata = await _prepare_inference_metadata(request.field_set.address, source.file_path)
 

src/python/pants/backend/javascript/goals/test.py

@@ -22,13 +22,7 @@
     OwningNodePackageRequest,
 )
 from pants.backend.javascript.subsystems.nodejstest import NodeJSTest
-from pants.backend.javascript.target_types import (
-    JSSourceField,
-    JSTestBatchCompatibilityTagField,
-    JSTestExtraEnvVarsField,
-    JSTestSourceField,
-    JSTestTimeoutField,
-)
+from pants.backend.javascript.target_types import JSRuntimeSourceField, JSTestRuntimeSourceField
 from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
 from pants.build_graph.address import Address
 from pants.core.goals.test import (
@@ -37,10 +31,13 @@
     CoverageReports,
     FilesystemCoverageReport,
     TestExtraEnv,
+    TestExtraEnvVarsField,
     TestFieldSet,
     TestRequest,
     TestResult,
+    TestsBatchCompatibilityTagField,
     TestSubsystem,
+    TestTimeoutField,
 )
 from pants.core.target_types import AssetSourceField
 from pants.core.util_rules import source_files
@@ -88,13 +85,13 @@ class JSCoverageDataCollection(CoverageDataCollection[JSCoverageData]):
 
 @dataclass(frozen=True)
 class JSTestFieldSet(TestFieldSet):
-    required_fields = (JSTestSourceField,)
+    required_fields = (JSTestRuntimeSourceField,)
 
-    batch_compatibility_tag: JSTestBatchCompatibilityTagField
-    source: JSTestSourceField
+    batch_compatibility_tag: TestsBatchCompatibilityTagField
+    source: JSTestRuntimeSourceField
     dependencies: Dependencies
-    timeout: JSTestTimeoutField
-    extra_env_vars: JSTestExtraEnvVarsField
+    timeout: TestTimeoutField
+    extra_env_vars: TestExtraEnvVarsField
 
 
 class JSTestRequest(TestRequest):
@@ -174,7 +171,7 @@ async def run_javascript_tests(
         SourceFilesRequest(
             (tgt.get(SourcesField) for tgt in transitive_tgts.closure),
             enable_codegen=True,
-            for_sources_types=[JSSourceField, AssetSourceField],
+            for_sources_types=[JSRuntimeSourceField, AssetSourceField],
         ),
     )
     merged_digest = await Get(Digest, MergeDigests([sources.snapshot.digest, installation.digest]))

src/python/pants/backend/javascript/install_node_package.py

@@ -20,7 +20,7 @@
 )
 from pants.backend.javascript.package_manager import PackageManager
 from pants.backend.javascript.subsystems import nodejs
-from pants.backend.javascript.target_types import JSSourceField
+from pants.backend.javascript.target_types import JSRuntimeSourceField
 from pants.build_graph.address import Address
 from pants.core.target_types import FileSourceField, ResourceSourceField
 from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
@@ -71,7 +71,7 @@ async def _get_relevant_source_files(
         SourceFilesRequest(
             sources,
             for_sources_types=(PackageJsonSourceField, FileSourceField)
-            + ((ResourceSourceField, JSSourceField) if with_js else ()),
+            + ((ResourceSourceField, JSRuntimeSourceField) if with_js else ()),
             enable_codegen=True,
         ),
     )

src/python/pants/backend/javascript/lint/prettier/rules.py

@@ -11,7 +11,7 @@
 from pants.backend.javascript.lint.prettier.subsystem import Prettier
 from pants.backend.javascript.subsystems import nodejs_tool
 from pants.backend.javascript.subsystems.nodejs_tool import NodeJSToolRequest
-from pants.backend.javascript.target_types import JSSourceField
+from pants.backend.javascript.target_types import JSRuntimeSourceField
 from pants.core.goals.fmt import FmtResult, FmtTargetsRequest
 from pants.core.util_rules.config_files import ConfigFiles, ConfigFilesRequest
 from pants.core.util_rules.partitions import PartitionerType
@@ -28,9 +28,9 @@
 
 @dataclass(frozen=True)
 class PrettierFmtFieldSet(FieldSet):
-    required_fields = (JSSourceField,)
+    required_fields = (JSRuntimeSourceField,)
 
-    sources: JSSourceField
+    sources: JSRuntimeSourceField
 
 
 class PrettierFmtRequest(FmtTargetsRequest):

src/python/pants/backend/javascript/target_types.py

@@ -25,11 +25,23 @@
 JS_TEST_FILE_EXTENSIONS = tuple(f"*.test{ext}" for ext in JS_FILE_EXTENSIONS)
 
 
-class JSDependenciesField(Dependencies):
+class JSRuntimeDependenciesField(Dependencies):
+    """Dependencies of a target that is javascript at runtime."""
+
+
+class JSDependenciesField(JSRuntimeDependenciesField):
     pass
 
 
-class JSSourceField(SingleSourceField):
+class JSRuntimeSourceField(SingleSourceField):
+    """A source that is javascript at runtime."""
+
+
+class JSTestRuntimeSourceField(SingleSourceField):
+    """A source that is runnable by javascript test-runners at runtime."""
+
+
+class JSSourceField(JSRuntimeSourceField):
     expected_file_extensions = JS_FILE_EXTENSIONS
 
 
@@ -86,7 +98,7 @@ class JSTestDependenciesField(JSDependenciesField):
     pass
 
 
-class JSTestSourceField(JSSourceField):
+class JSTestSourceField(JSSourceField, JSTestRuntimeSourceField):
     expected_file_extensions = JS_FILE_EXTENSIONS
 
 

src/python/pants/backend/jsx/BUILD

@@ -0,0 +1,11 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+# NOTE: Sources restricted from the default for python_sources due to conflict with
+#   - //:all-__init__.py-files
+#   - //src/python/pants/backend/jsx/__init__.py:../../../../../all-__init__.py-files
+python_sources(
+    sources=[
+        "target_types.py",
+    ],
+)

src/python/pants/backend/jsx/__init__.py

@@ -0,0 +1,6 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()
+
+python_tests(name="tests")

src/python/pants/backend/jsx/goals/BUILD

@@ -0,0 +1,62 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import dataclasses
+from dataclasses import dataclass
+from typing import Iterable
+
+from pants.backend.jsx.target_types import (
+    JSX_FILE_EXTENSIONS,
+    JSXSourcesGeneratorTarget,
+    JSXTestsGeneratorSourcesField,
+    JSXTestsGeneratorTarget,
+)
+from pants.core.goals.tailor import (
+    AllOwnedSources,
+    PutativeTarget,
+    PutativeTargets,
+    PutativeTargetsRequest,
+)
+from pants.core.util_rules.ownership import get_unowned_files_for_globs
+from pants.core.util_rules.source_files import classify_files_for_sources_and_tests
+from pants.engine.rules import Rule, collect_rules, rule
+from pants.engine.unions import UnionRule
+from pants.util.dirutil import group_by_dir
+from pants.util.logging import LogLevel
+
+
+@dataclass(frozen=True)
+class PutativeJSXTargetsRequest(PutativeTargetsRequest):
+    pass
+
+
+@rule(level=LogLevel.DEBUG, desc="Determine candidate JSX targets to create")
+async def find_putative_jsx_targets(
+    req: PutativeJSXTargetsRequest, all_owned_sources: AllOwnedSources
+) -> PutativeTargets:
+    unowned_jsx_files = await get_unowned_files_for_globs(
+        req, all_owned_sources, (f"*{ext}" for ext in JSX_FILE_EXTENSIONS)
+    )
+    classified_unowned_js_files = classify_files_for_sources_and_tests(
+        paths=unowned_jsx_files,
+        test_file_glob=JSXTestsGeneratorSourcesField.default,
+        sources_generator=JSXSourcesGeneratorTarget,
+        tests_generator=JSXTestsGeneratorTarget,
+    )
+
+    return PutativeTargets(
+        PutativeTarget.for_target_type(
+            tgt_type, path=dirname, name=name, triggering_sources=sorted(filenames)
+        )
+        for tgt_type, paths, name in (dataclasses.astuple(f) for f in classified_unowned_js_files)
+        for dirname, filenames in group_by_dir(paths).items()
+    )
+
+
+def rules() -> Iterable[Rule | UnionRule]:
+    return (
+        *collect_rules(),
+        UnionRule(PutativeTargetsRequest, PutativeJSXTargetsRequest),
+    )