fix: use destination path as working directory while rendering

While rendering a subproject, the working directory was the directory
Copier was invoked from, so filesystem-related Jinja filters and
extensions (e.g. `fileglob`) operated relative to it. Change the
working directory to the destination path while rendering, creating it
eagerly (except in pretend mode) so this also applies when copying into
a not-yet-existing folder.

External data is now loaded from the absolute destination path, because
it may be loaded lazily while rendering, when the working directory is
the destination path and a relative `dst_path` would no longer resolve
correctly.

Fixes copier-org#1708

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: nikzart

copier/_main.py

@@ -9,7 +9,7 @@
 import sys
 import warnings
 from collections.abc import Callable, Iterable, Mapping, Sequence
-from contextlib import suppress
+from contextlib import nullcontext, suppress
 from contextvars import ContextVar
 from dataclasses import field, replace
 from filecmp import dircmp
@@ -332,9 +332,13 @@ def _render(path: str) -> str:
                 return self._render_string(path)
 
         def _load_external_data(path: str) -> Any:
+            # NOTE: Use the absolute destination path instead of `self.dst_path`,
+            # which may be relative to the original working directory, because
+            # data may be loaded lazily while rendering, when the working
+            # directory is the destination path.
             if (
                 not (
-                    (self.dst_path / (path := _render(path)))
+                    (self.subproject.local_abspath / (path := _render(path)))
                     .resolve()
                     .is_relative_to(self.subproject.local_abspath)
                 )
@@ -348,7 +352,9 @@ def _load_external_data(path: str) -> Any:
                         "  - API: `trust=True`"
                     ),
                 )
-            return load_answersfile_data(self.dst_path, path, warn_on_missing=True)
+            return load_answersfile_data(
+                self.subproject.local_abspath, path, warn_on_missing=True
+            )
 
         # Given those values are lazily rendered on 1st access then cached
         # the phase value is irrelevant and could be misleading.
@@ -777,47 +783,58 @@ def _render_template(self) -> None:
         """Render the template in the subproject root."""
         follow_symlinks = not self.template.preserve_symlinks
         dst_root = self.dst_path.resolve()
-        for src in scantree(str(self.template_copy_root), follow_symlinks):
-            src_abspath = Path(src.path)
-            # If the source is a symlink, we are not preserving symlinks, and the
-            # symlink target is outside the template root, this means that we are
-            # copying a file/directory from outside the template, which is
-            # forbidden, so raise an error.
-            if (
-                src_abspath.is_symlink()
-                and not self.template.preserve_symlinks
-                and not (src_abspath.resolve()).is_relative_to(
-                    self.template.local_abspath
-                )
-            ):
-                raise ForbiddenPathError(
-                    path=src_abspath.relative_to(self.template_copy_root)
+        if not self.pretend:
+            dst_root.mkdir(parents=True, exist_ok=True)
+        # Render with the subproject root as the working directory, so that
+        # filesystem-related Jinja filters and extensions (e.g. `fileglob`)
+        # operate relative to the destination path. In pretend mode, a missing
+        # destination folder is not created, and the working directory is left
+        # unchanged.
+        with local.cwd(dst_root) if dst_root.is_dir() else nullcontext():
+            for src in scantree(str(self.template_copy_root), follow_symlinks):
+                src_abspath = Path(src.path)
+                # If the source is a symlink, we are not preserving symlinks, and the
+                # symlink target is outside the template root, this means that we are
+                # copying a file/directory from outside the template, which is
+                # forbidden, so raise an error.
+                if (
+                    src_abspath.is_symlink()
+                    and not self.template.preserve_symlinks
+                    and not (src_abspath.resolve()).is_relative_to(
+                        self.template.local_abspath
+                    )
+                ):
+                    raise ForbiddenPathError(
+                        path=src_abspath.relative_to(self.template_copy_root)
+                    )
+                src_relpath = Path(src_abspath).relative_to(self.template.local_abspath)
+                dst_relpaths_ctxs = self._render_path(
+                    Path(src_abspath).relative_to(self.template_copy_root)
                 )
-            src_relpath = Path(src_abspath).relative_to(self.template.local_abspath)
-            dst_relpaths_ctxs = self._render_path(
-                Path(src_abspath).relative_to(self.template_copy_root)
-            )
-            for dst_relpath, ctx in dst_relpaths_ctxs:
-                dst_abspath = dst_root / dst_relpath
-                if dst_abspath.is_symlink() and self.template.preserve_symlinks:
-                    # If destination path is a symlink, it can safely point outside the
-                    # subproject dir, while still itself existing within the subproject.
-                    # (So long as nothing is templated into it (if it is a directory),
-                    # which would be caught by that path's own check.)
-                    # Therefore avoid resolving the symlink itself:
-                    dst_realpath = dst_abspath.parent.resolve() / dst_abspath.name
-                else:
-                    dst_realpath = dst_abspath.resolve()
-                if not dst_realpath.is_relative_to(dst_root):
-                    raise ForbiddenPathError(path=dst_relpath)
-                if self.match_exclude(dst_relpath):
-                    continue
-                if src.is_symlink() and self.template.preserve_symlinks:
-                    self._render_symlink(src_relpath, dst_relpath)
-                elif src.is_dir(follow_symlinks=follow_symlinks):
-                    self._render_folder(dst_relpath)
-                else:
-                    self._render_file(src_relpath, dst_relpath, extra_context=ctx or {})
+                for dst_relpath, ctx in dst_relpaths_ctxs:
+                    dst_abspath = dst_root / dst_relpath
+                    if dst_abspath.is_symlink() and self.template.preserve_symlinks:
+                        # If destination path is a symlink, it can safely point
+                        # outside the subproject dir, while still itself existing
+                        # within the subproject. (So long as nothing is templated
+                        # into it (if it is a directory), which would be caught by
+                        # that path's own check.) Therefore avoid resolving the
+                        # symlink itself:
+                        dst_realpath = dst_abspath.parent.resolve() / dst_abspath.name
+                    else:
+                        dst_realpath = dst_abspath.resolve()
+                    if not dst_realpath.is_relative_to(dst_root):
+                        raise ForbiddenPathError(path=dst_relpath)
+                    if self.match_exclude(dst_relpath):
+                        continue
+                    if src.is_symlink() and self.template.preserve_symlinks:
+                        self._render_symlink(src_relpath, dst_relpath)
+                    elif src.is_dir(follow_symlinks=follow_symlinks):
+                        self._render_folder(dst_relpath)
+                    else:
+                        self._render_file(
+                            src_relpath, dst_relpath, extra_context=ctx or {}
+                        )
 
     def _render_file(  # noqa: C901
         self,

docs/configuring.md

@@ -1213,6 +1213,15 @@ The following extensions are _always_ loaded:
 You don't need to tell your template users to install these extensions: Copier depends
 on them, so they are always installed when Copier is installed.
 
+!!! note
+
+    While rendering the subproject, the current working directory is the destination
+    path, so filesystem-related filters, functions and tags provided by Jinja
+    extensions (e.g., the `fileglob` filter) operate relative to the destination path.
+    Keep in mind that Copier generates files in no particular order, so the result of
+    filesystem operations may depend on whether the files being queried were already
+    generated.
+
 !!! warning
 
     Including an extension allows Copier to execute uncontrolled code, thus making the

tests/test_answersfile.py

@@ -7,6 +7,7 @@
 
 import pytest
 import yaml
+from plumbum import local
 
 import copier
 from copier._user_data import load_answersfile_data
@@ -549,3 +550,30 @@ def test_external_data_path_outside_destination_root_is_unsafe_on_update(
 
     with expected:
         copier.run_update(project, defaults=True, overwrite=True, unsafe=unsafe)
+
+
+def test_external_data_lazily_loaded_while_rendering(
+    tmp_path_factory: pytest.TempPathFactory,
+) -> None:
+    """External data loads while rendering with a relative destination path.
+
+    While rendering, the working directory is the destination path
+    (https://github.com/copier-org/copier/issues/1708), and external data may
+    be loaded lazily on first access while rendering a file.
+    """
+    src, invocation = map(tmp_path_factory.mktemp, ("src", "invocation"))
+    build_file_tree(
+        {
+            (src / "copier.yml"): (
+                """\
+                _external_data:
+                    other: other-data.yml
+                """
+            ),
+            (src / "out.txt.jinja"): "{{ _external_data.other.key }}",
+            (invocation / "dst" / "other-data.yml"): "key: value",
+        }
+    )
+    with local.cwd(invocation):
+        copier.run_copy(str(src), "dst", defaults=True, overwrite=True)
+    assert (invocation / "dst" / "out.txt").read_text() == "value"

tests/test_copy.py

@@ -383,6 +383,57 @@ def test_pretend_option(tmp_path: Path) -> None:
     assert not (tmp_path / "pyproject.toml").exists()
 
 
+def test_pretend_option_missing_dst(tmp_path_factory: pytest.TempPathFactory) -> None:
+    src = tmp_path_factory.mktemp("src")
+    build_file_tree({(src / "a.txt"): ""})
+    dst = tmp_path_factory.mktemp("dst_parent") / "dst"
+    copier.run_copy(str(src), dst, pretend=True)
+    assert not dst.exists()
+
+
+def test_render_cwd_is_destination(tmp_path_factory: pytest.TempPathFactory) -> None:
+    """Filesystem-related Jinja filters operate relative to the destination.
+
+    https://github.com/copier-org/copier/issues/1708
+    """
+    src, dst, invocation = map(tmp_path_factory.mktemp, ("src", "dst", "invocation"))
+    build_file_tree(
+        {
+            (src / "fileglob.txt.jinja"): "{{ '*.txt' | fileglob | sort }}",
+            (dst / "a.txt"): "",
+            (dst / "b.txt"): "",
+            (invocation / "decoy.txt"): "",
+        }
+    )
+    with local.cwd(invocation):
+        cwd_before = Path.cwd()
+        copier.run_copy(str(src), dst, defaults=True, overwrite=True)
+        assert Path.cwd() == cwd_before
+    assert (dst / "fileglob.txt").read_text() == "['a.txt', 'b.txt']"
+
+
+def test_render_cwd_is_destination_relative_dst(
+    tmp_path_factory: pytest.TempPathFactory,
+) -> None:
+    """A relative destination path works while rendering in the destination.
+
+    https://github.com/copier-org/copier/issues/1708
+    """
+    src, invocation = map(tmp_path_factory.mktemp, ("src", "invocation"))
+    build_file_tree(
+        {
+            (src / "fileglob.txt.jinja"): "{{ '*.txt' | fileglob | sort }}",
+            (invocation / "dst" / "a.txt"): "",
+            (invocation / "decoy.txt"): "",
+        }
+    )
+    with local.cwd(invocation):
+        cwd_before = Path.cwd()
+        copier.run_copy(str(src), "dst", defaults=True, overwrite=True)
+        assert Path.cwd() == cwd_before
+    assert (invocation / "dst" / "fileglob.txt").read_text() == "['a.txt']"
+
+
 @pytest.mark.parametrize("generate", [True, False])
 def test_empty_dir(tmp_path_factory: pytest.TempPathFactory, generate: bool) -> None:
     src, dst = map(tmp_path_factory.mktemp, ("src", "dst"))

tests/test_updatediff.py

@@ -578,6 +578,39 @@ def test_update_from_tagged_to_head(tmp_path_factory: pytest.TempPathFactory) ->
     assert load_answersfile_data(dst)["_commit"] == f"v1-1-g{sha}"
 
 
+def test_update_render_cwd_is_destination(
+    tmp_path_factory: pytest.TempPathFactory,
+) -> None:
+    """While updating, the working directory during rendering is the destination.
+
+    https://github.com/copier-org/copier/issues/1708
+    """
+    src, dst, invocation = map(tmp_path_factory.mktemp, ("src", "dst", "invocation"))
+    with local.cwd(src):
+        build_file_tree(
+            {
+                "{{ _copier_conf.answers_file }}.jinja": "{{ _copier_answers|to_nice_yaml }}",
+                "version.txt": "1",
+            }
+        )
+        git_init("v1")
+        git("tag", "v1")
+    run_copy(str(src), dst, defaults=True, overwrite=True)
+    build_file_tree({(dst / "a.txt"): "", (invocation / "decoy.txt"): ""})
+    with local.cwd(dst):
+        git_init("copied")
+    with local.cwd(src):
+        build_file_tree({"fileglob.txt.jinja": "{{ '*.txt' | fileglob | sort }}"})
+        git("add", ".")
+        git("commit", "-m2")
+        git("tag", "v2")
+    with local.cwd(invocation):
+        cwd_before = Path.cwd()
+        run_update(dst, defaults=True, overwrite=True)
+        assert Path.cwd() == cwd_before
+    assert (dst / "fileglob.txt").read_text() == "['a.txt', 'version.txt']"
+
+
 @pytest.mark.parametrize("subproject_path", [Path(), Path("subproject")])
 @pytest.mark.parametrize("skip_pattern", ["skip_me", "/skip_me"])
 def test_skip_update(