Merge branch 'main' into yarn-v2-immutable-install

Author: kevinobruno

docs/docs/using-pants/project-introspection.mdx

@@ -127,6 +127,72 @@ To include the original target itself, use `--closed`:
 helloworld/main.py:lib
 ```
 
+## Export dependency graph
+
+Both `dependencies` and `dependents` goals have the `--format` option allowing you to export data in multiple formats.
+Exporting information about the dependencies and dependents in JSON format will produce the
+[adjacency list](https://en.wikipedia.org/wiki/Adjacency_list) of your dependency graph:
+
+```bash
+$ pants dependencies --format=json \
+  helloworld/greet/greeting.py \
+  helloworld/translator/translator_test.py
+
+{
+    "helloworld/greet/greeting.py:lib": [
+        "//:reqs#setuptools",
+        "//:reqs#types-setuptools",
+        "helloworld/greet:translations",
+        "helloworld/translator/translator.py:lib"
+    ],
+    "helloworld/translator/translator_test.py:tests": [
+        "//:reqs#pytest",
+        "helloworld/translator/translator.py:lib"
+    ]
+}
+```
+
+This has various applications, and you could analyze, visualize, and process the data further. Sometimes, a fairly
+straightforward `jq` query would suffice, but for anything more complex, it may make sense to write a small program
+to process the exported graph. For instance, you could:
+
+* find tests with most transitive dependencies
+
+```bash
+$ pants dependencies --filter-target-type=python_test --format=json :: \
+  | jq -r 'to_entries[] | "\(.key)\t\(.value | length)"' \
+  | sort -k2 -n
+```
+
+* find resources that only a few other targets depend on
+
+```bash
+$ pants dependents --filter-target-type=resource --format=json :: \
+  | jq -r 'to_entries[] | select(.value | length < 2)'
+```
+
+* find files within the `src/` directory that transitively lead to the most tests
+
+```python
+# depgraph.py
+import json
+
+with open("data.json") as fh:
+    data = json.load(fh)
+
+for source, dependents in data.items():
+    print(source, len([d for d in dependents if d.startswith("tests/")]))
+```
+
+```bash
+$ pants dependents --transitive --format=json src:: > data.json
+$ python3 depgraph.py | sort -k2 -n
+```
+
+For more sophisticated graph querying, you may want to look into graph libraries such as [`networkx`](https://networkx.org/).
+In a larger repository, it may make sense to track the health of the dependency graph and use the output
+of the graph export to identify parts of your codebase that would benefit from refactoring.
+
 ## `filedeps` - find which files a target owns
 
 `filedeps` outputs all of the files belonging to a target, based on its `sources` field.

src/python/pants/backend/awslambda/python/target_types.py

@@ -191,7 +191,7 @@ class PythonAWSLambda(_AWSLambdaBaseTarget):
         f"""
         A self-contained Python function suitable for uploading to AWS Lambda.
 
-        See {doc_url('awslambda-python')}.
+        See {doc_url('docs/python/integrations/aws-lambda')}.
         """
     )
 
@@ -207,7 +207,7 @@ class PythonAWSLambdaLayer(_AWSLambdaBaseTarget):
         f"""
         A Python layer suitable for uploading to AWS Lambda.
 
-        See {doc_url('awslambda-python')}.
+        See {doc_url('docs/python/integrations/aws-lambda')}.
         """
     )
 

src/python/pants/backend/codegen/protobuf/python/python_protobuf_subsystem.py

@@ -35,7 +35,7 @@ class PythonProtobufSubsystem(Subsystem):
         f"""
         Options related to the Protobuf Python backend.
 
-        See {doc_url('protobuf-python')}.
+        See {doc_url('docs/python/integrations/protobuf-and-grpc')}.
         """
     )
 

src/python/pants/backend/codegen/protobuf/target_types.py

@@ -66,8 +66,8 @@ class ProtobufSourceTarget(Target):
         A single Protobuf file used to generate various languages.
 
         See language-specific docs:
-            Python: {doc_url('protobuf-python')}
-            Go: {doc_url('protobuf-go')}
+            Python: {doc_url('docs/python/integrations/protobuf-and-grpc')}
+            Go: {doc_url('docs/go/integrations/protobuf')}
         """
     )
 

src/python/pants/backend/codegen/thrift/target_types.py

@@ -72,7 +72,7 @@ class ThriftSourceTarget(Target):
         A single Thrift file used to generate various languages.
 
         See language-specific docs:
-            Python: {doc_url('thrift-python')}
+            Python: {doc_url('docs/python/integrations/thrift')}
         """
     )
 

src/python/pants/backend/codegen/utils.py

@@ -54,7 +54,7 @@ def find_python_runtime_library_or_raise_error(
                 No `python_requirement` target was found with the module `{runtime_library_module}`
                 in your project{for_resolve_str}, so the Python code generated from the target
                 {codegen_address} will not work properly. See
-                {doc_url('python-third-party-dependencies')} for how to add a requirement, such as
+                {doc_url('docs/python/overview/third-party-dependencies')} for how to add a requirement, such as
                 adding to requirements.txt. Usually you will want to use the
                 `{recommended_requirement_name}` project at {recommended_requirement_url}.
 

src/python/pants/backend/docker/target_types.py

@@ -149,7 +149,7 @@ class DockerImageTagsField(StringSequenceField):
 
         {_interpolation_help.format(kind="tag")}
 
-        See {doc_url('tagging-docker-images')}.
+        See {doc_url('docs/docker/tagging-docker-images')}.
         """
     )
 

src/python/pants/backend/google_cloud_function/python/target_types.py

@@ -125,7 +125,7 @@ class PythonGoogleCloudFunction(Target):
         f"""
         A self-contained Python function suitable for uploading to Google Cloud Function.
 
-        See {doc_url('google-cloud-function-python')}.
+        See {doc_url('docs/python/integrations/google-cloud-functions')}.
         """
     )
 

src/python/pants/backend/python/dependency_inference/rules.py

@@ -346,7 +346,7 @@ async def _handle_unowned_imports(
 
         If you do not expect an import to be inferrable, add `# pants: no-infer-dep` to the
         import line. Otherwise, see
-        {doc_url('troubleshooting#import-errors-and-missing-dependencies')} for common problems.
+        {doc_url('docs/using-pants/troubleshooting-common-issues#import-errors-and-missing-dependencies')} for common problems.
         """
     )
     if unowned_dependency_behavior is UnownedDependencyUsage.LogWarning:

src/python/pants/backend/python/goals/pytest_runner.py

@@ -243,7 +243,7 @@ async def validate_pytest_cov_included(_pytest: PyTest):
                 `{_pytest.install_from_resolve}` used to install pytest is missing
                 `pytest-cov`, which is needed to collect coverage data.
 
-                See {doc_url("python-test-goal#pytest-version-and-plugins")} for details
+                See {doc_url("docs/python/goals/test#pytest-version-and-plugins")} for details
                 on how to set up a custom resolve for use by pytest.
                 """
             )

src/python/pants/backend/python/goals/repl.py

@@ -54,7 +54,7 @@ def maybe_get_resolve(t: Target) -> str | None:
         raise NoCompatibleResolveException.bad_input_roots(
             root_targets,
             maybe_get_resolve=maybe_get_resolve,
-            doc_url_slug="python-third-party-dependencies#multiple-lockfiles",
+            doc_url_slug="docs/python/overview/lockfiles#multiple-lockfiles",
             workaround=softwrap(
                 f"""
                 To work around this, choose which resolve you want to use from above. Then, run

src/python/pants/backend/python/lint/flake8/subsystem.py

@@ -108,7 +108,7 @@ class Flake8(PythonToolBase):
             example, if your plugin is at `build-support/flake8/custom_plugin.py`, add
             `'build-support/flake8'` to `[source].root_patterns` in `pants.toml`. This is
             necessary for Pants to know how to tell Flake8 to discover your plugin. See
-            {doc_url('source-roots')}
+            {doc_url('docs/using-pants/key-concepts/source-roots')}
 
             You must also set `[flake8:local-plugins]` in your Flake8 config file.
 
@@ -123,7 +123,7 @@ class Flake8(PythonToolBase):
             directory or a subdirectory.
 
             To instead load third-party plugins, add them to a custom resolve alongside
-            flake8 itself, as described in {doc_url("python-lockfiles#lockfiles-for-tools")}.
+            flake8 itself, as described in {doc_url("docs/python/overview/lockfiles#lockfiles-for-tools")}.
             """
         ),
     )

src/python/pants/backend/python/lint/pylint/subsystem.py

@@ -106,7 +106,7 @@ class Pylint(PythonToolBase):
             example, if your plugin is at `build-support/pylint/custom_plugin.py`, add
             `'build-support/pylint'` to `[source].root_patterns` in `pants.toml`. This is
             necessary for Pants to know how to tell Pylint to discover your plugin. See
-            {doc_url('source-roots')}
+            {doc_url('docs/using-pants/key-concepts/source-roots')}
 
             You must also set `load-plugins=$module_name` in your Pylint config file.
 
@@ -115,7 +115,7 @@ class Pylint(PythonToolBase):
             directory or a subdirectory.
 
             To instead load third-party plugins, add them to a custom resolve alongside
-            pylint itself, as described in {doc_url("python-lockfiles#lockfiles-for-tools")}.
+            pylint itself, as described in {doc_url("docs/python/overview/lockfiles#lockfiles-for-tools")}.
             """
         ),
     )

src/python/pants/backend/python/packaging/pyoxidizer/rules.py

@@ -133,7 +133,7 @@ async def package_pyoxidizer_binary(
                 The `{PyOxidizerTarget.alias}` target {field_set.address} must include
                 in its `dependencies` field at least one `python_distribution` target that produces a
                 `.whl` file. For example, if using `{GenerateSetupField.alias}=True`, then make sure
-                `{WheelField.alias}=True`. See {doc_url('python-distributions')}.
+                `{WheelField.alias}=True`. See {doc_url('docs/python/overview/building-distributions')}.
                 """
             )
         )

src/python/pants/backend/python/packaging/pyoxidizer/target_types.py

@@ -78,14 +78,14 @@ class PyOxidizerDependenciesField(Dependencies):
 
         The distribution(s) must generate at least one wheel file. For example, if using
         `{GenerateSetupField.alias}=True`, then make sure `{WheelField.alias}=True`. See
-        {doc_url('python-distributions')}.
+        {doc_url('docs/python/overview/building-distributions')}.
 
         Usually, you only need to specify a single `python_distribution`. However, if
         that distribution depends on another first-party distribution in your repository, you
         must specify that dependency too, otherwise PyOxidizer would try installing the
         distribution from PyPI. Note that a `python_distribution` target might depend on
         another `python_distribution` target even if it is not included in its own `dependencies`
-        field, as explained at {doc_url('python-distributions')}; if code from one distribution
+        field, as explained at {doc_url('docs/python/overview/building-distributions')}; if code from one distribution
         imports code from another distribution, then there is a dependency and you must
         include both `python_distribution` targets in the `dependencies` field of this
         `pyoxidizer_binary` target.
@@ -146,7 +146,7 @@ class PyOxidizerTarget(Target):
         A single-file Python executable with a Python interpreter embedded, built via PyOxidizer.
 
         To use this target, first create a `python_distribution` target with the code you want
-        included in your binary, per {doc_url('python-distributions')}. Then add this
+        included in your binary, per {doc_url('docs/python/overview/building-distributions')}. Then add this
         `python_distribution` target to the `dependencies` field. See the `help` for
         `dependencies` for more information.
 

src/python/pants/backend/python/subsystems/python_tool_base.py

@@ -58,7 +58,7 @@ class PythonToolRequirementsBase(Subsystem):
             If specified, install the tool using the lockfile for this named resolve.
 
             This resolve must be defined in `[python].resolves`, as described in
-            {doc_url("python-third-party-dependencies#user-lockfiles")}.
+            {doc_url("docs/python/overview/third-party-dependencies#user-lockfiles")}.
 
             The resolve's entire lockfile will be installed, unless specific requirements are
             listed via the `requirements` option, in which case only those requirements

src/python/pants/backend/python/subsystems/repos.py

@@ -78,19 +78,19 @@ class PythonRepos(Subsystem):
 
             This feature is intended to be used with `[python-repos].find_links`, rather than PEP
             440 direct reference requirements (see
-            {doc_url("python-third-party-dependencies#local-requirements")}.
+            {doc_url("docs/python/overview/third-party-dependencies#local-requirements")}.
             `[python-repos].find_links` must be configured to a valid absolute path for the
             current machine.
 
             Tip: you can avoid each user needing to manually configure this option and
             `[python-repos].find_links` by using a common file location, along with Pants's
-            interpolation support ({doc_url('options#config-file-interpolation')}. For example,
+            interpolation support ({doc_url('docs/using-pants/key-concepts/options#config-file-interpolation')}. For example,
             in `pants.toml`, you could set both options to `%(buildroot)s/python_wheels`
             to point to the directory `python_wheels` in the root of
             your repository; or, use the path `%(env.HOME)s/pants_wheels` for the path
             `~/pants_wheels`. If you are not able to use a common path like this, then we
             recommend setting that each user set these options via a `.pants.rc` file
-            ({doc_url('options#pantsrc-file')}.
+            ({doc_url('docs/using-pants/key-concepts/options#pantsrc-file')}.
 
             Note: Only takes effect if using Pex lockfiles, i.e. using the
             `generate-lockfiles` goal.

src/python/pants/backend/python/subsystems/setup.py

@@ -106,7 +106,7 @@ def interpreter_constraints(self) -> tuple[str, ...]:
                     if different parts of your codebase run against different python interpreter
                     versions in a single repo.
 
-                    See {doc_url("python-interpreter-compatibility")} for details.
+                    See {doc_url("docs/python/overview/interpreter-compatibility")} for details.
                     """
                 ),
             )
@@ -126,7 +126,7 @@ def interpreter_constraints(self) -> tuple[str, ...]:
             interpreter constraints, update `[python].interpreter_constraints`, the
             `interpreter_constraints` field, and relevant tool options like
             `[isort].interpreter_constraints` to tell Pants which interpreters your code
-            actually uses. See {doc_url('python-interpreter-compatibility')}.
+            actually uses. See {doc_url('docs/python/overview/interpreter-compatibility')}.
 
             All elements must be the minor and major Python version, e.g. `'2.7'` or `'3.10'`. Do
             not include the patch version.
@@ -187,7 +187,7 @@ def interpreter_constraints(self) -> tuple[str, ...]:
                 resolve with the `resolve` field.
 
             If a target can work with multiple resolves, you can either use the `parametrize`
-            mechanism or manually create a distinct target per resolve. See {doc_url("targets")}
+            mechanism or manually create a distinct target per resolve. See {doc_url("docs/using-pants/key-concepts/targets-and-build-files")}
             for information about `parametrize`.
 
             For example:
@@ -231,7 +231,7 @@ def interpreter_constraints(self) -> tuple[str, ...]:
             Use this version of Pip for resolving requirements and generating lockfiles.
 
             The value used here must be one of the Pip versions supported by the underlying PEX
-            version. See {doc_url("pex")} for details.
+            version. See {doc_url("docs/python/overview/pex")} for details.
 
             N.B.: The `latest` value selects the latest of the choices listed by PEX which is not
             necessarily the latest Pip version released on PyPI.

src/python/pants/backend/python/subsystems/twine.py

@@ -78,7 +78,7 @@ class TwineSubsystem(PythonToolBase):
 
             This option cannot be overridden via environment targets, so if you need a different
             value than what the rest of your organization is using, override the value via an
-            environment variable, CLI argument, or `.pants.rc` file. See {doc_url('options')}.
+            environment variable, CLI argument, or `.pants.rc` file. See {doc_url('docs/using-pants/key-concepts/options')}.
             """
         ),
     )