Use Typst's JSON dep file (with outputs)

.github/workflows/pytest.yaml

@@ -43,7 +43,7 @@ jobs:
       - name: Install typst
         uses: typst-community/setup-typst@v4
         with:
-          typst-version: 0.13
+          typst-version: 0.14-rc2
       - name: Install oldest versions of supported dependencies
         if: ${{ matrix.python-version == '3.11'}}
         run: pip install -r .github/requirements-old.txt

docs/changelog.md

@@ -15,6 +15,7 @@ and this project adheres to [Effort-based Versioning](https://jacobtomlinson.dev
 ### Changed
 
 - Rename `wrap_git` action to `wrap-git` for consistency with other StepUp actions.
+- Update `compile_typst` to work with Typst 0.14.
 
 ## [3.1.4][] - 2025-10-18 {: v3.1.4 }
 

stepup/reprep/api.py

@@ -281,22 +281,16 @@ def compile_typst(
 
     !!! warning
 
-        This feature will only work well with typst 0.13 or later.
+        This feature only works with typst 0.14.
 
         Support for typst in StepUp RepRep is experimental.
         Expect breaking changes in future releases.
         Some limitations include:
 
-        - SVG figures with references to external bitmaps are not processed correctly.
-          These bitmaps are not rendered, neither are they included in the dep file.
-          For this problem, a workaround was suggested here:
-          https://github.com/typst/typst/issues/5335
-        - When the typst compiler detects an error in the input, it doesn't write the dep file.
-          This means that StepUp cannot reschedule it, even if that would fix the problem.
-          If it would know which files are used, it would see which ones are outdated,
-          rebuild them and then retry the typst command.
-          For more details, see:
-          https://github.com/typst/typst/issues/5886
+        - SVG figures with references to external SVG figures are not processed correctly.
+          (Referenced bitmaps are handled correctly.)
+          These referenced SVG figures are not rendered, neither are they included in the dep file.
+          See: https://github.com/typst/typst/issues/6858
 
     Parameters
     ----------
@@ -352,7 +346,7 @@ def compile_typst(
         dest = subs(dest)
     if not path_typ.endswith(".typ"):
         raise ValueError(f"The input of the typst command must end with .typ, got {path_typ}.")
-    path_out = make_path_out(path_typ, dest, ".pdf", [".svg", ".png"])
+    path_out = make_path_out(path_typ, dest, ".pdf", [".svg", ".png", ".html"])
 
     stem = path_typ[:-4]
     args = ["compile-typst"]
@@ -365,7 +359,7 @@ def compile_typst(
         paths_out.append(path_out)
     if keep_deps or string_to_bool(getenv("REPREP_KEEP_TYPST_DEPS", "0")):
         args.append("--keep-deps")
-        paths_out.append(f"{stem}.dep")
+        paths_out.append(f"{stem}.dep.json")
     if inventory is None:
         inventory = string_to_bool(getenv("REPREP_TYPST_INVENTORY", "0"))
     if inventory is True:

stepup/reprep/compile_typst.py

@@ -22,13 +22,14 @@
 This wrapper extracts relevant information from a typst build
 to inform StepUp of input files used or needed.
 
-This is tested with Typst 0.12.
-A current limitation of typst is that it will fail after the first missing file it requires,
+This is tested with Typst 0.14.
+A limitation of typst is that it will fail after the first missing file it requires,
 making it inefficient to plan ahead and build all missing required inputs early.
 """
 
 import argparse
 import contextlib
+import json
 import shlex
 import sys
 
@@ -49,8 +50,8 @@ def main(argv: list[str] | None = None, work_thread: WorkThread | None = None):
 
     if not args.path_typ.endswith(".typ"):
         raise ValueError("The Typst source must have extension .typ")
-    if not (args.path_out is None or args.path_out.suffix in (".pdf", ".png", ".svg")):
-        raise ValueError("The Typst output must be a PDF, PNG, or SVG file.")
+    if not (args.path_out is None or args.path_out.suffix in (".pdf", ".png", ".svg", ".html")):
+        raise ValueError("The Typst output must be a PDF, PNG, SVG, or HTML file.")
 
     # Get Typst executable and prepare some arguments that
     if args.typst is None:
@@ -67,6 +68,8 @@ def main(argv: list[str] | None = None, work_thread: WorkThread | None = None):
         if resolution is None:
             resolution = int(getenv("REPREP_TYPST_RESOLUTION", "144"))
         typargs.append(f"--ppi={resolution}")
+    elif args.path_out.suffix == ".html":
+        typargs.append("--features=html")
     for keyval in args.sysinp:
         typargs.append("--input")
         typargs.append(keyval)
@@ -77,36 +80,29 @@ def main(argv: list[str] | None = None, work_thread: WorkThread | None = None):
     with contextlib.ExitStack() as stack:
         if args.keep_deps:
             # Remove any existing make-deps output from a previous run.
-            path_dep = Path(args.path_typ[:-4] + ".dep")
+            path_dep = Path(args.path_typ[:-4] + ".dep.json")
             path_dep.remove_p()
         else:
             # Use a temporary file for the make-deps output.
-            path_dep = stack.enter_context(TempDir()) / "typst.dep"
-        typargs.extend(["--make-deps", path_dep])
+            path_dep = stack.enter_context(TempDir()) / "typst.dep.json"
+        typargs.extend(["--deps", path_dep, "--deps-format", "json"])
 
         # Run typst compile
         returncode, stdout, stderr = work_thread.runsh(shlex.join(typargs))
         print(stdout)
-        # Get existing input files from the dependency file and amend.
-        # Note that the deps file does not escape colons in paths,
-        # so the code below assumes one never uses colons in paths.
-        inp_paths = []
+        # Assume there is a single output file, which is the one specified.
+        # This is not correct when there are multiple outputs, e.g. as with SVG and PNG outputs.
+        # Get required input files from the dependency file.
         if path_dep.is_file():
-            out_paths = []
             with open(path_dep) as fh:
-                dep_out, dep_inp = fh.read().split(":", 1)
-                out_paths.extend(shlex.split(dep_out))
-                inp_paths.extend(shlex.split(dep_inp))
+                depinfo = json.load(fh)
+            inp_paths = depinfo["inputs"]
+            out_paths = depinfo["outputs"]
         else:
             print(f"Dependency file not created: {path_dep}.", file=sys.stderr)
-            out_paths = [args.path_out]
+            out_paths = []
+            inp_paths = []
 
-    # Look for missing input files in the standard error stream and amend them.
-    if returncode != 0:
-        lead = "error: file not found (searched at "
-        inp_paths.extend(
-            line[len(lead) : -1] for line in stderr.splitlines() if line.startswith(lead)
-        )
     sys.stderr.write(stderr)
     inp_paths = filter_dependencies(inp_paths)
     amend(inp=inp_paths)

tests/examples/compile_typst_args/expected_graph.txt

@@ -9,7 +9,7 @@ file:./
             supplies   file:lorem.pdf
             supplies   file:lorem.typ
             supplies   file:plan.py
-            supplies   step:compile-typst lorem.typ -- --pages 2
+            supplies   step:compile-typst lorem.typ -- --pages 2 --no-pdf-tags
             supplies   step:runpy ./plan.py
 
 file:plan.py
@@ -27,15 +27,15 @@ step:runpy ./plan.py
             consumes   file:./
             consumes   file:plan.py
              creates   file:lorem.typ
-             creates   step:compile-typst lorem.typ -- --pages 2
+             creates   step:compile-typst lorem.typ -- --pages 2 --no-pdf-tags
 
 file:lorem.typ
                state = STATIC
           created by   step:runpy ./plan.py
             consumes   file:./
-            supplies   step:compile-typst lorem.typ -- --pages 2
+            supplies   step:compile-typst lorem.typ -- --pages 2 --no-pdf-tags
 
-step:compile-typst lorem.typ -- --pages 2
+step:compile-typst lorem.typ -- --pages 2 --no-pdf-tags
                state = SUCCEEDED
              env_var = REPREP_TYPST [amended]
                      = STEPUP_PATH_FILTER [amended]
@@ -47,6 +47,6 @@ step:compile-typst lorem.typ -- --pages 2
 
 file:lorem.pdf
                state = BUILT
-          created by   step:compile-typst lorem.typ -- --pages 2
+          created by   step:compile-typst lorem.typ -- --pages 2 --no-pdf-tags
             consumes   file:./
-            consumes   step:compile-typst lorem.typ -- --pages 2
+            consumes   step:compile-typst lorem.typ -- --pages 2 --no-pdf-tags

tests/examples/compile_typst_args/expected_stdout.txt

@@ -3,8 +3,8 @@
         0/1 |      PHASE │ run
         0/1 |      START │ runpy ./plan.py
         1/2 |    SUCCESS │ runpy ./plan.py
-        1/2 |      START │ compile-typst lorem.typ -- --pages 2
-        2/2 |    SUCCESS │ compile-typst lorem.typ -- --pages 2
+        1/2 |      START │ compile-typst lorem.typ -- --pages 2 --no-pdf-tags
+        2/2 |    SUCCESS │ compile-typst lorem.typ -- --pages 2 --no-pdf-tags
         2/2 |   DIRECTOR │ Trying to delete 0 outdated output(s)
         2/2 |      PHASE │ watch
         2/2 |   DIRECTOR │ Stopping workers

tests/examples/compile_typst_args/plan.py

@@ -3,4 +3,4 @@
 from stepup.reprep.api import compile_typst
 
 static("lorem.typ")
-compile_typst("lorem.typ", typst_args=["--pages", "2"])
+compile_typst("lorem.typ", typst_args=["--pages", "2", "--no-pdf-tags"])

tests/examples/compile_typst_dep/.gitignore

@@ -1,7 +1,7 @@
 .stepup
 current*
 document.pdf
-document.dep
+document.dep.json
 image.jpg
 document1.pdf
 reproducibility_inventory.txt

tests/examples/compile_typst_dep/expected_graph.txt

@@ -6,7 +6,7 @@ root:
 file:./
                state = STATIC
           created by   root:
-            supplies   file:document.dep
+            supplies   file:document.dep.json
             supplies   file:document.pdf
             supplies   file:document.typ
             supplies   file:image.jpg
@@ -55,12 +55,12 @@ step:compile-typst --keep-deps document.typ
             consumes   file:./
             consumes   file:document.typ
             consumes   file:image.jpg [amended]
-             creates   file:document.dep
+             creates   file:document.dep.json
              creates   file:document.pdf
-            supplies   file:document.dep
+            supplies   file:document.dep.json
             supplies   file:document.pdf
 
-file:document.dep
+file:document.dep.json
                state = BUILT
           created by   step:compile-typst --keep-deps document.typ
             consumes   file:./

tests/examples/compile_typst_dep/main.sh

@@ -26,7 +26,7 @@ wait
 # Check files that are expected to be present and/or missing.
 [[ -f plan.py ]] || exit 1
 [[ -f document.pdf ]] || exit 1
-[[ -f document.dep ]] || exit 1
+[[ -f document.dep.json ]] || exit 1
 [[ -f image.jpg ]] || exit 1
 [[ -f document1.pdf ]] || exit 1
 [[ -f reproducibility_inventory.txt ]] || exit 1

tests/examples/compile_typst_dep_error/.gitignore

@@ -1,4 +1,5 @@
 .stepup
 current*
 document.pdf
+document.dep.json
 data.yaml

tests/examples/compile_typst_dep_error/expected_graph.txt

@@ -7,7 +7,7 @@ file:./
                state = STATIC
           created by   root:
             supplies   file:data.yaml
-            supplies   file:document.dep
+            supplies   file:document.dep.json
             supplies   file:document.pdf
             supplies   file:document.typ
             supplies   file:plan.py
@@ -40,26 +40,27 @@ file:document.typ
             supplies   step:compile-typst --keep-deps document.typ
 
 step:compile-typst --keep-deps document.typ
-               state = FAILED
+               state = SUCCEEDED
              env_var = REPREP_TYPST [amended]
                      = REPREP_TYPST_ARGS [amended]
                      = STEPUP_PATH_FILTER [amended]
           created by   step:runpy ./plan.py
             consumes   file:./
+            consumes   file:data.yaml [amended]
             consumes   file:document.typ
-             creates   file:document.dep
+             creates   file:document.dep.json
              creates   file:document.pdf
-            supplies   file:document.dep
+            supplies   file:document.dep.json
             supplies   file:document.pdf
 
-file:document.dep
-               state = AWAITED
+file:document.dep.json
+               state = BUILT
           created by   step:compile-typst --keep-deps document.typ
             consumes   file:./
             consumes   step:compile-typst --keep-deps document.typ
 
 file:document.pdf
-               state = AWAITED
+               state = BUILT
           created by   step:compile-typst --keep-deps document.typ
             consumes   file:./
             consumes   step:compile-typst --keep-deps document.typ
@@ -76,3 +77,4 @@ file:data.yaml
           created by   step:runsh echo 'fixed: new' > data.yaml
             consumes   file:./
             consumes   step:runsh echo 'fixed: new' > data.yaml
+            supplies   step:compile-typst --keep-deps document.typ

tests/examples/compile_typst_dep_error/expected_stdout.txt

@@ -4,21 +4,17 @@
         0/1 |      START │ runpy ./plan.py
         1/3 |    SUCCESS │ runpy ./plan.py
         1/3 |      START │ compile-typst --keep-deps document.typ
-        1/2 |       FAIL │ compile-typst --keep-deps document.typ
-────────────────────────────────── Step info ───────────────────────────────────
-Command               stepup act -- compile-typst --keep-deps document.typ
-Return code           1
-───────────────────────── Expected outputs not created ─────────────────────────
-document.dep
-document.pdf
-──────────────────────────────── Standard error ────────────────────────────────
-(stripped)
+        1/3 | RESCHEDULE │ compile-typst --keep-deps document.typ
+──────────────── Rescheduling due to unavailable amended inputs ────────────────
+Missing inputs and/or required directories:
+data.yaml
 ────────────────────────────────────────────────────────────────────────────────
-        1/2 |      START │ runsh echo 'fixed: new' > data.yaml
-        2/2 |    SUCCESS │ runsh echo 'fixed: new' > data.yaml
-        2/2 |    WARNING │ 1 step(s) failed.
-        2/2 |    WARNING │ Skipping file cleanup due to incomplete build
-        2/2 |    WARNING │ Check logs: .stepup/fail.log .stepup/warning.log
-        2/2 |      PHASE │ watch
-        2/2 |   DIRECTOR │ Stopping workers
-        2/2 |   DIRECTOR │ See you!
+        1/3 |      START │ runsh echo 'fixed: new' > data.yaml
+        2/3 |    SUCCESS │ runsh echo 'fixed: new' > data.yaml
+        2/3 |      START │ compile-typst --keep-deps document.typ
+        3/3 |    SUCCESS │ compile-typst --keep-deps document.typ
+        3/3 |   DIRECTOR │ Trying to delete 0 outdated output(s)
+        3/3 |    WARNING │ Check logs: .stepup/warning.log
+        3/3 |      PHASE │ watch
+        3/3 |   DIRECTOR │ Stopping workers
+        3/3 |   DIRECTOR │ See you!

tests/examples/compile_typst_dep_error/main.sh

@@ -22,4 +22,6 @@ set +e; wait -fn $PID; RETURNCODE=$?; set -e
 # Check files that are expected to be present and/or missing.
 [[ -f plan.py ]] || exit 1
 [[ -f document.pdf ]] || exit 1
+[[ -f document.dep.json ]] || exit 1
+grep data.yaml document.dep.json
 [[ -f data.yaml ]] || exit 1

tests/examples/compile_typst_html/.gitignore

@@ -0,0 +1,4 @@
+.stepup
+current*
+lorem.html
+lorem.dep.json

tests/examples/compile_typst_html/README.txt

@@ -0,0 +1 @@
+A simple example of compiling a typst document to HTML.