From 48cea185ac76de8028d2db34a91ba2f0705622de Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 4 Oct 2024 12:05:21 +0200 Subject: [PATCH 01/78] prepare for mypy 1.12 --- cwltool/context.py | 3 +- mypy-stubs/black/__init__.pyi | 26 ----- mypy-stubs/mistune.pyi | 197 ---------------------------------- setup.py | 2 +- 4 files changed, 3 insertions(+), 225 deletions(-) delete mode 100644 mypy-stubs/black/__init__.pyi delete mode 100644 mypy-stubs/mistune.pyi diff --git a/cwltool/context.py b/cwltool/context.py index 1e82ecc4a..283936d61 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -31,6 +31,7 @@ from .utils import DEFAULT_TMP_PREFIX, CWLObjectType, HasReqsHints, ResolverType if TYPE_CHECKING: + from _typeshed import SupportsWrite from cwl_utils.parser.cwl_v1_2 import LoadingOptions from .builder import Builder @@ -199,7 +200,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.default_stdout: Optional[Union[IO[bytes], TextIO]] = None self.default_stderr: Optional[Union[IO[bytes], TextIO]] = None self.validate_only: bool = False - self.validate_stdout: Optional[Union[IO[bytes], TextIO, IO[str]]] = None + self.validate_stdout: Optional["SupportsWrite[str]"] = None super().__init__(kwargs) if self.tmp_outdir_prefix == "": self.tmp_outdir_prefix = self.tmpdir_prefix diff --git a/mypy-stubs/black/__init__.pyi b/mypy-stubs/black/__init__.pyi deleted file mode 100644 index f741ef771..000000000 --- a/mypy-stubs/black/__init__.pyi +++ /dev/null @@ -1,26 +0,0 @@ -import asyncio -from concurrent.futures import Executor -from enum import Enum -from pathlib import Path -from typing import ( - Any, - Iterator, - List, - MutableMapping, - Optional, - Pattern, - Set, - Sized, - Tuple, - Union, -) - -from black.mode import Mode as Mode -from black.mode import TargetVersion as TargetVersion - -FileContent = str -Encoding = str -NewLine = str -FileMode = Mode - -def format_str(src_contents: str, mode: Mode) -> FileContent: ... diff --git a/mypy-stubs/mistune.pyi b/mypy-stubs/mistune.pyi deleted file mode 100644 index 3778c9195..000000000 --- a/mypy-stubs/mistune.pyi +++ /dev/null @@ -1,197 +0,0 @@ -__author__ = "Aleksandr Slepchenkov" -__email__ = "Sl.aleksandr28@gmail.com" - -from typing import ( - Any, - Dict, - Iterable, - List, - Match, - Optional, - Pattern, - Sequence, - Tuple, - Type, -) - -Tokens = List[Dict[str, Any]] -# There are too much levels of optional unions of lists of text in cell and align 385 and 396 lines in mistune - -def escape(text: str, quote: bool = ..., smart_amp: bool = ...) -> str: ... - -class BlockGrammar: - def_links: Pattern[str] - def_footnotes: Pattern[str] - newline: Pattern[str] - block_code: Pattern[str] - fences: Pattern[str] - hrule: Pattern[str] - heading: Pattern[str] - lheading: Pattern[str] - block_quote: Pattern[str] - list_block: Pattern[str] - list_item: Pattern[str] - list_bullet: Pattern[str] - paragraph: Pattern[str] - block_html: Pattern[str] - table: Pattern[str] - nptable: Pattern[str] - text: Pattern[str] - -class BlockLexer: - grammar_class: Type[BlockGrammar] - default_rules: List[str] - list_rules: Tuple[str] - footnote_rules: Tuple[str] - tokens: Tokens - def_links: Dict[str, Dict[str, str]] - def_footnotes: Dict[str, int] - rules = ... # type: BlockGrammar - def __init__(self, rules: Optional[BlockGrammar] = ..., **kwargs: Any) -> None: ... - def __call__(self, text: str, rules: Optional[Sequence[str]] = ...) -> Tokens: ... - def parse(self, text: str, rules: Optional[Sequence[str]] = ...) -> Tokens: ... - def parse_newline(self, m: Match[str]) -> None: ... - def parse_block_code(self, m: Match[str]) -> None: ... - def parse_fences(self, m: Match[str]) -> None: ... - def parse_heading(self, m: Match[str]) -> None: ... - def parse_lheading(self, m: Match[str]) -> None: ... - def parse_hrule(self, m: Match[str]) -> None: ... - def parse_list_block(self, m: Match[str]) -> None: ... - def parse_block_quote(self, m: Match[str]) -> None: ... - def parse_def_links(self, m: Match[str]) -> None: ... - def parse_def_footnotes(self, m: Match[str]) -> None: ... - def parse_table(self, m: Match[str]) -> None: ... - def parse_nptable(self, m: Match[str]) -> None: ... - def parse_block_html(self, m: Match[str]) -> None: ... - def parse_paragraph(self, m: Match[str]) -> None: ... - def parse_text(self, m: Match[str]) -> None: ... - -class InlineGrammar: - escape: Pattern[str] - inline_html: Pattern[str] - autolink: Pattern[str] - link: Pattern[str] - reflink: Pattern[str] - nolink: Pattern[str] - url: Pattern[str] - double_emphasis: Pattern[str] - emphasis: Pattern[str] - code: Pattern[str] - linebreak: Pattern[str] - strikethrough: Pattern[str] - footnote: Pattern[str] - text: Pattern[str] - def hard_wrap(self) -> None: ... - -class InlineLexer: - grammar_class: Type[InlineGrammar] - default_rules: List[str] - inline_html_rules: List[str] - renderer: Renderer - links: Dict[str, Dict[str, str]] - footnotes: Dict[str, int] - footnote_index: int - _in_link: bool - _in_footnote: bool - _parse_inline_html: bool - rules: InlineGrammar - def __init__( - self, renderer: Renderer, rules: Optional[InlineGrammar] = ..., **kwargs: Any - ) -> None: ... - def __call__(self, text: str, rules: Optional[Sequence[str]] = ...) -> str: ... - def setup( - self, - links: Optional[Dict[str, Dict[str, str]]], - footnotes: Optional[Dict[str, int]], - ) -> None: ... - line_match: Match[str] - line_started: bool - def output(self, text: str, rules: Optional[Sequence[str]] = ...) -> str: ... - def output_escape(self, m: Match[str]) -> str: ... - def output_autolink(self, m: Match[str]) -> str: ... - def output_url(self, m: Match[str]) -> str: ... - def output_inline_html(self, m: Match[str]) -> str: ... - def output_footnote(self, m: Match[str]) -> Optional[str]: ... - def output_link(self, m: Match[str]) -> str: ... - def output_reflink(self, m: Match[str]) -> Optional[str]: ... - def output_nolink(self, m: Match[str]) -> Optional[str]: ... - def output_double_emphasis(self, m: Match[str]) -> str: ... - def output_emphasis(self, m: Match[str]) -> str: ... - def output_code(self, m: Match[str]) -> str: ... - def output_linebreak(self, m: Match[str]) -> str: ... - def output_strikethrough(self, m: Match[str]) -> str: ... - def output_text(self, m: Match[str]) -> str: ... - -class Renderer: - options: Dict[str, str] - def __init__(self, **kwargs: Any) -> None: ... - def placeholder(self) -> str: ... - def block_code( - self, code: str, lang: Any = ... - ) -> str: ... # It seems that lang should be string, however other types are valid as well - def block_quote(self, text: str) -> str: ... - def block_html(self, html: str) -> str: ... - def header(self, text: str, level: int, raw: Optional[str] = ...) -> str: ... - def hrule(self) -> str: ... - def list( - self, body: Any, ordered: bool = ... - ) -> str: ... # body - same reason as for lang above, and for other Any in this class - def list_item(self, text: Any) -> str: ... - def paragraph(self, text: str) -> str: ... - def table(self, header: Any, body: Any) -> str: ... - def table_row(self, content: Any) -> str: ... - def table_cell(self, content: Any, **flags: Dict[str, Any]) -> str: ... - def double_emphasis(self, text: Any) -> str: ... - def emphasis(self, text: Any) -> str: ... - def codespan(self, text: str) -> str: ... - def linebreak(self) -> str: ... - def strikethrough(self, text: Any) -> str: ... - def text(self, text: Any) -> str: ... - def escape(self, text: Any) -> str: ... - def autolink(self, link: Any, is_email: bool = ...) -> str: ... - def link(self, link: Any, title: Any, text: Any) -> str: ... - def image(self, src: Any, title: Any, text: Any) -> str: ... - def inline_html(self, html: Any) -> str: ... - def newline(self) -> str: ... - def footnote_ref(self, key: Any, index: int) -> str: ... - def footnote_item(self, key: Any, text: str) -> str: ... - def footnotes(self, text: Any) -> str: ... - -class Markdown: - renderer = ... # type: Renderer - inline = ... # type: InlineLexer - block = ... # type: BlockLexer - footnotes = ... # type: List[Dict[str, Any]] - tokens = ... # type: Tokens - def __init__( - self, - renderer: Optional[Renderer] = ..., - inline: Optional[InlineLexer] = ..., - block: Optional[BlockLexer] = ..., - **kwargs: Any, - ) -> None: ... - def __call__(self, text: str) -> str: ... - def render(self, text: str) -> str: ... - def parse(self, text: str) -> str: ... - token = ... # type: Dict[str, Any] - def pop(self) -> Optional[Dict[str, Any]]: ... - def peek(self) -> Optional[Dict[str, Any]]: ... - def output(self, text: str, rules: Optional[Sequence[str]] = ...) -> str: ... - def tok(self) -> str: ... - def tok_text(self) -> str: ... - def output_newline(self) -> str: ... - def output_hrule(self) -> str: ... - def output_heading(self) -> str: ... - def output_code(self) -> str: ... - def output_table(self) -> str: ... - def output_block_quote(self) -> str: ... - def output_list(self) -> str: ... - def output_list_item(self) -> str: ... - def output_loose_item(self) -> str: ... - def output_footnote(self) -> str: ... - def output_close_html(self) -> str: ... - def output_open_html(self) -> str: ... - def output_paragraph(self) -> str: ... - def output_text(self) -> str: ... - -def markdown(text: str, escape: bool = ..., **kwargs: Any) -> str: ... diff --git a/setup.py b/setup.py index 9980276e5..5ddb526c1 100644 --- a/setup.py +++ b/setup.py @@ -134,7 +134,7 @@ "importlib_resources>=1.4;python_version<'3.9'", "coloredlogs", "pydot >= 1.4.1, <3", - "argcomplete", + "argcomplete >= 1.12.0", "pyparsing != 3.0.2", # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 "cwl-utils >= 0.32", "spython >= 0.3.0", From 75e073cf7f54660cf5e0f5fec3b9054ce4da8dfa Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Fri, 4 Oct 2024 14:47:53 +0200 Subject: [PATCH 02/78] remove usage of ancient shellescape backport of Stdlib's shlex (#2041) --- cwltool/command_line_tool.py | 4 ++-- cwltool/job.py | 4 ++-- mypy-stubs/shellescape/__init__.pyi | 5 ----- mypy-stubs/shellescape/main.pyi | 5 ----- requirements.txt | 1 - setup.py | 1 - 6 files changed, 4 insertions(+), 16 deletions(-) delete mode 100644 mypy-stubs/shellescape/__init__.pyi delete mode 100644 mypy-stubs/shellescape/main.pyi diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 7a4e8ff71..eb0b1a4f5 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -7,6 +7,7 @@ import logging import os import re +import shlex import shutil import threading import urllib @@ -31,7 +32,6 @@ cast, ) -import shellescape from mypy_extensions import mypyc_attr from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import Schema @@ -1164,7 +1164,7 @@ def register_reader(f: CWLObjectType) -> None: for b in builder.bindings: arg = builder.generate_arg(b) if b.get("shellQuote", True): - arg = [shellescape.quote(a) for a in aslist(arg)] + arg = [shlex.quote(a) for a in aslist(arg)] cmd.extend(aslist(arg)) j.command_line = ["/bin/sh", "-c", " ".join(cmd)] else: diff --git a/cwltool/job.py b/cwltool/job.py index 817cb04c0..1731a5350 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -5,6 +5,7 @@ import math import os import re +import shlex import shutil import signal import stat @@ -35,7 +36,6 @@ ) import psutil -import shellescape from prov.model import PROV from schema_salad.sourceline import SourceLine from schema_salad.utils import json_dump, json_dumps @@ -271,7 +271,7 @@ def _execute( self.outdir, " \\\n ".join( [ - shellescape.quote(str(arg)) if shouldquote(str(arg)) else str(arg) + shlex.quote(str(arg)) if shouldquote(str(arg)) else str(arg) for arg in (runtime + self.command_line) ] ), diff --git a/mypy-stubs/shellescape/__init__.pyi b/mypy-stubs/shellescape/__init__.pyi deleted file mode 100644 index 621241e5e..000000000 --- a/mypy-stubs/shellescape/__init__.pyi +++ /dev/null @@ -1,5 +0,0 @@ -# Stubs for shellescape (Python 2) -# -# NOTE: This dynamically typed stub was automatically generated by stubgen. - -from .main import quote as quote diff --git a/mypy-stubs/shellescape/main.pyi b/mypy-stubs/shellescape/main.pyi deleted file mode 100644 index 69eade63e..000000000 --- a/mypy-stubs/shellescape/main.pyi +++ /dev/null @@ -1,5 +0,0 @@ -# Stubs for shellescape.main (Python 2) - -from typing import AnyStr - -def quote(s: AnyStr) -> AnyStr: ... diff --git a/requirements.txt b/requirements.txt index df82a6f43..b1fc2207d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ requests>=2.6.1 ruamel.yaml>=0.16.0,<0.19 rdflib>=4.2.2,<7.1 -shellescape>=3.4.1,<3.9 schema-salad>=8.7,<9 prov==1.5.1 mypy-extensions diff --git a/setup.py b/setup.py index 5ddb526c1..9ab4f240e 100644 --- a/setup.py +++ b/setup.py @@ -126,7 +126,6 @@ # https://github.com/ionrock/cachecontrol/issues/137 "ruamel.yaml >= 0.16, < 0.19", "rdflib >= 4.2.2, < 7.1.0", - "shellescape >= 3.4.1, < 3.9", "schema-salad >= 8.7, < 9", "prov == 1.5.1", "mypy-extensions", From 04fd4264493fb6689d832967e72430188e60de00 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 4 Oct 2024 17:00:54 +0200 Subject: [PATCH 03/78] setuptools is not a install requirement --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 9ab4f240e..fa39a378b 100644 --- a/setup.py +++ b/setup.py @@ -121,7 +121,6 @@ package_dir={"cwltool.tests": "tests"}, include_package_data=True, install_requires=[ - "setuptools", "requests >= 2.6.1", # >= 2.6.1 to workaround # https://github.com/ionrock/cachecontrol/issues/137 "ruamel.yaml >= 0.16, < 0.19", From 0056ca52742cf7fa0c9556f511e59e778d445bd8 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 4 Oct 2024 16:35:37 +0200 Subject: [PATCH 04/78] gh-actions: fail-fast false --- .github/workflows/ci-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 1530ab92b..7de5ad329 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -165,6 +165,7 @@ jobs: runs-on: ubuntu-22.04 strategy: + fail-fast: false matrix: cwl-version: [v1.0, v1.1, v1.2] container: [docker, singularity, podman] From 18b8fdf7d425d8e7d8986e08904ef09492798cf6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 07:23:48 +0000 Subject: [PATCH 05/78] Bump sphinx-rtd-theme from 2.0.0 to 3.0.0 Bumps [sphinx-rtd-theme](https://github.com/readthedocs/sphinx_rtd_theme) from 2.0.0 to 3.0.0. - [Changelog](https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/changelog.rst) - [Commits](https://github.com/readthedocs/sphinx_rtd_theme/compare/2.0.0...3.0.0) --- updated-dependencies: - dependency-name: sphinx-rtd-theme dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 40a2eefbf..875c44720 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ sphinx >= 2.2 -sphinx-rtd-theme==2.0.0 +sphinx-rtd-theme==3.0.0 sphinx-autoapi sphinx-autodoc-typehints sphinxcontrib-autoprogram From 6970186d3f47a3998f4f1b88caf89a4b05f3d302 Mon Sep 17 00:00:00 2001 From: Iacopo Colonnelli Date: Tue, 8 Oct 2024 15:16:43 +0200 Subject: [PATCH 06/78] Handle spurious `ReceiveScatterOutput` callbacks (#2051) This commit fixes #2003 by handling spurious, repeated callbacks of the `receive_scatter_output` method of the `ReceiveScatterOutput` class. The reason of multiple awakenings has not been investigated deeply, though. In the future, a thorough examination of the `MultithreadedJobExecutor` logic may be necessary. --------- Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- cwltool/workflow_job.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 2e69ca70c..d144128e6 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -10,6 +10,7 @@ MutableMapping, MutableSequence, Optional, + Set, Sized, Tuple, Union, @@ -88,12 +89,17 @@ def __init__( ) -> None: """Initialize.""" self.dest = dest - self.completed = 0 + self._completed: Set[int] = set() self.processStatus = "success" self.total = total self.output_callback = output_callback self.steps: List[Optional[JobsGeneratorType]] = [] + @property + def completed(self) -> int: + """The number of completed internal jobs.""" + return len(self._completed) + def receive_scatter_output(self, index: int, jobout: CWLObjectType, processStatus: str) -> None: """Record the results of a scatter operation.""" for key, val in jobout.items(): @@ -108,10 +114,11 @@ def receive_scatter_output(self, index: int, jobout: CWLObjectType, processStatu if self.processStatus != "permanentFail": self.processStatus = processStatus - self.completed += 1 + if index not in self._completed: + self._completed.add(index) - if self.completed == self.total: - self.output_callback(self.dest, self.processStatus) + if self.completed == self.total: + self.output_callback(self.dest, self.processStatus) def setTotal( self, From 83000ae041c4e516ed326274d8cf315861d49aa6 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 8 Oct 2024 17:58:30 +0200 Subject: [PATCH 07/78] prepare for future mypy release and enable --local-partial-types now --- mypy.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy.ini b/mypy.ini index bac992869..02545dce5 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,6 +5,7 @@ show_column_numbers = true show_error_codes = true pretty = true warn_unreachable = True +local_partial_types = true [mypy-galaxy.tool_util.*] ignore_missing_imports = True From c6ad93a7003c4dc05cefe8d5667f2d9830002c21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:47:41 +0000 Subject: [PATCH 08/78] Bump sphinx-rtd-theme from 3.0.0 to 3.0.1 Bumps [sphinx-rtd-theme](https://github.com/readthedocs/sphinx_rtd_theme) from 3.0.0 to 3.0.1. - [Changelog](https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/changelog.rst) - [Commits](https://github.com/readthedocs/sphinx_rtd_theme/compare/3.0.0...3.0.1) --- updated-dependencies: - dependency-name: sphinx-rtd-theme dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 875c44720..d614584fc 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ sphinx >= 2.2 -sphinx-rtd-theme==3.0.0 +sphinx-rtd-theme==3.0.1 sphinx-autoapi sphinx-autodoc-typehints sphinxcontrib-autoprogram From 82f37d5bed7587690356ae90e71ddca8c653ff78 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 07:51:43 +0000 Subject: [PATCH 09/78] Update black requirement from ~=24.8 to ~=24.10 Updates the requirements on [black](https://github.com/psf/black) to permit the latest version. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/24.8.0...24.10.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index b0d7944eb..3dc9139cb 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,3 +1,3 @@ flake8-bugbear<24.9 -black~=24.8 +black~=24.10 codespell From 1a2c939865e9bc67a4b6217d649dc3255ca11261 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Tue, 8 Oct 2024 10:05:28 +0200 Subject: [PATCH 10/78] Update lint-requirements.txt --- lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index 3dc9139cb..50dc69060 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,3 +1,3 @@ flake8-bugbear<24.9 -black~=24.10 +black=24.* codespell From 83def6a48c0489cfbc13706d44fc60bcb9621533 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Tue, 8 Oct 2024 10:06:49 +0200 Subject: [PATCH 11/78] Update lint-requirements.txt --- lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index 50dc69060..5af76cd93 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,3 +1,3 @@ flake8-bugbear<24.9 -black=24.* +black==24.* codespell From 05af6c1357c327b3146e9f5da40e7c0aa3e6d976 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 4 Oct 2024 14:46:47 +0200 Subject: [PATCH 12/78] Drop Python 3.8 --- .github/workflows/ci-tests.yml | 2 +- Makefile | 4 +- cwltool/argparser.py | 33 ++++------ cwltool/builder.py | 60 +++++++---------- cwltool/checker.py | 72 +++++++++----------- cwltool/command_line_tool.py | 80 ++++++++++------------- cwltool/context.py | 34 ++++------ cwltool/cuda.py | 3 +- cwltool/cwlprov/__init__.py | 14 ++-- cwltool/cwlprov/provenance_profile.py | 37 ++++------- cwltool/cwlprov/ro.py | 53 ++++++--------- cwltool/cwlprov/writablebagfile.py | 5 +- cwltool/cwlrdf.py | 5 +- cwltool/cwlviewer.py | 5 +- cwltool/docker.py | 39 +++++------ cwltool/docker_id.py | 14 ++-- cwltool/env_to_stdout.py | 3 +- cwltool/executors.py | 39 +++++------ cwltool/factory.py | 4 +- cwltool/flatten.py | 4 +- cwltool/job.py | 54 ++++++--------- cwltool/load_tool.py | 39 +++++------ cwltool/main.py | 65 ++++++++---------- cwltool/mpi.py | 11 ++-- cwltool/mutation.py | 4 +- cwltool/pack.py | 47 ++++++------- cwltool/pathmapper.py | 27 +++----- cwltool/process.py | 78 +++++++++------------- cwltool/procgenerator.py | 6 +- cwltool/run_job.py | 6 +- cwltool/secrets.py | 7 +- cwltool/singularity.py | 31 ++++----- cwltool/software_requirements.py | 19 ++---- cwltool/stdfsaccess.py | 6 +- cwltool/subgraph.py | 37 ++++------- cwltool/udocker.py | 4 +- cwltool/update.py | 40 +++++------- cwltool/utils.py | 49 ++++++-------- cwltool/validate_js.py | 25 +++---- cwltool/workflow.py | 22 ++----- cwltool/workflow_job.py | 43 +++++------- mypy-requirements.txt | 1 + pyproject.toml | 2 +- setup.py | 4 +- tests/cwl-conformance/cwltool-conftest.py | 6 +- tests/test_dependencies.py | 6 +- tests/test_environment.py | 7 +- tests/test_examples.py | 22 +++---- tests/test_fetch.py | 4 +- tests/test_http_input.py | 8 +-- tests/test_js_sandbox.py | 6 +- tests/test_loop.py | 2 +- tests/test_loop_ext.py | 2 +- tests/test_mpi.py | 17 ++--- tests/test_override.py | 5 +- tests/test_pack.py | 3 +- tests/test_path_checks.py | 4 +- tests/test_pathmapper.py | 6 +- tests/test_provenance.py | 3 +- tests/test_relocate.py | 7 +- tests/test_secrets.py | 10 +-- tests/test_tmpdir.py | 4 +- tests/test_toolargparse.py | 4 +- tests/util.py | 11 ++-- tox.ini | 45 +++++++------ 65 files changed, 541 insertions(+), 778 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 7de5ad329..2095b53b6 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -32,7 +32,7 @@ jobs: strategy: matrix: py-ver-major: [3] - py-ver-minor: [8, 9, 10, 11, 12, 13] + py-ver-minor: [9, 10, 11, 12, 13] step: [lint, unit, bandit, mypy] env: diff --git a/Makefile b/Makefile index 1b08f4290..5b9dd214e 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ MODULE=cwltool # `SHELL=bash` doesn't work for some, so don't use BASH-isms like # `[[` conditional expressions. -PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py) setup.py +PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py tests/cwl-conformance/*.py) setup.py DEVPKGS=diff_cover pylint pep257 pydocstyle 'tox<4' tox-pyenv auto-walrus \ isort wheel autoflake pyupgrade bandit -rlint-requirements.txt\ -rtest-requirements.txt -rmypy-requirements.txt -rdocs/requirements.txt @@ -190,7 +190,7 @@ shellcheck: FORCE cwltool-in-docker.sh pyupgrade: $(PYSOURCES) - pyupgrade --exit-zero-even-if-changed --py38-plus $^ + pyupgrade --exit-zero-even-if-changed --py39-plus $^ auto-walrus $^ release-test: FORCE diff --git a/cwltool/argparser.py b/cwltool/argparser.py index efced5386..7b3125d94 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -3,19 +3,8 @@ import argparse import os import urllib -from typing import ( - Any, - Callable, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Sequence, - Type, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence, Sequence +from typing import Any, Callable, Optional, Union, cast from .loghandler import _logger from .process import Process, shortname @@ -718,7 +707,7 @@ def arg_parser() -> argparse.ArgumentParser: return parser -def get_default_args() -> Dict[str, Any]: +def get_default_args() -> dict[str, Any]: """Get default values of cwltool's command line options.""" ap = arg_parser() args = ap.parse_args([]) @@ -732,7 +721,7 @@ class FSAction(argparse.Action): def __init__( self, - option_strings: List[str], + option_strings: list[str], dest: str, nargs: Any = None, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, @@ -770,7 +759,7 @@ class FSAppendAction(argparse.Action): def __init__( self, - option_strings: List[str], + option_strings: list[str], dest: str, nargs: Any = None, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, @@ -827,7 +816,7 @@ class AppendAction(argparse.Action): def __init__( self, - option_strings: List[str], + option_strings: list[str], dest: str, nargs: Any = None, **kwargs: Any, @@ -859,7 +848,7 @@ def add_argument( toolparser: argparse.ArgumentParser, name: str, inptype: Any, - records: List[str], + records: list[str], description: str = "", default: Any = None, input_required: bool = True, @@ -888,9 +877,9 @@ def add_argument( return None ahelp = description.replace("%", "%%") - action: Optional[Union[Type[argparse.Action], str]] = None + action: Optional[Union[type[argparse.Action], str]] = None atype: Optional[Any] = None - typekw: Dict[str, Any] = {} + typekw: dict[str, Any] = {} if inptype == "File": action = FileAction @@ -962,8 +951,8 @@ def add_argument( def generate_parser( toolparser: argparse.ArgumentParser, tool: Process, - namemap: Dict[str, str], - records: List[str], + namemap: dict[str, str], + records: list[str], input_required: bool = True, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, base_uri: str = "", diff --git a/cwltool/builder.py b/cwltool/builder.py index 2ba1e6543..066a77f86 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -3,21 +3,9 @@ import copy import logging import math +from collections.abc import MutableMapping, MutableSequence from decimal import Decimal -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Type, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Any, Callable, Optional, Union, cast from cwl_utils import expression from cwl_utils.file_formats import check_format @@ -55,7 +43,7 @@ ) from .pathmapper import PathMapper -INPUT_OBJ_VOCAB: Dict[str, str] = { +INPUT_OBJ_VOCAB: dict[str, str] = { "Any": "https://w3id.org/cwl/salad#Any", "File": "https://w3id.org/cwl/cwl#File", "Directory": "https://w3id.org/cwl/cwl#Directory", @@ -107,16 +95,16 @@ class Builder(HasReqsHints): def __init__( self, job: CWLObjectType, - files: List[CWLObjectType], - bindings: List[CWLObjectType], + files: list[CWLObjectType], + bindings: list[CWLObjectType], schemaDefs: MutableMapping[str, CWLObjectType], names: Names, - requirements: List[CWLObjectType], - hints: List[CWLObjectType], - resources: Dict[str, Union[int, float]], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], + resources: dict[str, Union[int, float]], mutation_manager: Optional[MutationManager], formatgraph: Optional[Graph], - make_fs_access: Type[StdFsAccess], + make_fs_access: type[StdFsAccess], fs_access: StdFsAccess, job_script_provider: Optional[DependenciesConfiguration], timeout: float, @@ -172,7 +160,7 @@ def __init__( self.find_default_container: Optional[Callable[[], str]] = None self.container_engine = container_engine - def build_job_script(self, commands: List[str]) -> Optional[str]: + def build_job_script(self, commands: list[str]) -> Optional[str]: if self.job_script_provider is not None: return self.job_script_provider.build_job_script(self, commands) return None @@ -180,11 +168,11 @@ def build_job_script(self, commands: List[str]) -> Optional[str]: def bind_input( self, schema: CWLObjectType, - datum: Union[CWLObjectType, List[CWLObjectType]], + datum: Union[CWLObjectType, list[CWLObjectType]], discover_secondaryFiles: bool, - lead_pos: Optional[Union[int, List[int]]] = None, - tail_pos: Optional[Union[str, List[int]]] = None, - ) -> List[MutableMapping[str, Union[str, List[int]]]]: + lead_pos: Optional[Union[int, list[int]]] = None, + tail_pos: Optional[Union[str, list[int]]] = None, + ) -> list[MutableMapping[str, Union[str, list[int]]]]: """ Bind an input object to the command line. @@ -200,8 +188,8 @@ def bind_input( if lead_pos is None: lead_pos = [] - bindings: List[MutableMapping[str, Union[str, List[int]]]] = [] - binding: Union[MutableMapping[str, Union[str, List[int]]], CommentedMap] = {} + bindings: list[MutableMapping[str, Union[str, list[int]]]] = [] + binding: Union[MutableMapping[str, Union[str, list[int]]], CommentedMap] = {} value_from_expression = False if "inputBinding" in schema and isinstance(schema["inputBinding"], MutableMapping): binding = CommentedMap(schema["inputBinding"].items()) @@ -324,7 +312,7 @@ def bind_input( if schema["type"] == "record": datum = cast(CWLObjectType, datum) - for f in cast(List[CWLObjectType], schema["fields"]): + for f in cast(list[CWLObjectType], schema["fields"]): name = cast(str, f["name"]) if name in datum and datum[name] is not None: bindings.extend( @@ -372,7 +360,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: self.files.append(datum) loadContents_sourceline: Union[ - None, MutableMapping[str, Union[str, List[int]]], CWLObjectType + None, MutableMapping[str, Union[str, list[int]]], CWLObjectType ] = None if binding and binding.get("loadContents"): loadContents_sourceline = binding @@ -513,7 +501,7 @@ def addsf( if "format" in schema: eval_format: Any = self.do_eval(schema["format"]) if isinstance(eval_format, str): - evaluated_format: Union[str, List[str]] = eval_format + evaluated_format: Union[str, list[str]] = eval_format elif isinstance(eval_format, MutableSequence): for index, entry in enumerate(eval_format): message = None @@ -541,7 +529,7 @@ def addsf( raise SourceLine( schema["format"], index, WorkflowException, debug ).makeError(message) - evaluated_format = cast(List[str], eval_format) + evaluated_format = cast(list[str], eval_format) else: raise SourceLine(schema, "format", WorkflowException, debug).makeError( "An expression in the 'format' field must " @@ -586,8 +574,8 @@ def addsf( # Position to front of the sort key if binding: for bi in bindings: - bi["position"] = cast(List[int], binding["position"]) + cast( - List[int], bi["position"] + bi["position"] = cast(list[int], binding["position"]) + cast( + list[int], bi["position"] ) bindings.append(binding) @@ -618,7 +606,7 @@ def tostr(self, value: Union[MutableMapping[str, str], Any]) -> str: else: return str(value) - def generate_arg(self, binding: CWLObjectType) -> List[str]: + def generate_arg(self, binding: CWLObjectType) -> list[str]: value = binding.get("datum") debug = _logger.isEnabledFor(logging.DEBUG) if "valueFrom" in binding: @@ -648,7 +636,7 @@ def generate_arg(self, binding: CWLObjectType) -> List[str]: argl = [itemSeparator.join([self.tostr(v) for v in value])] elif binding.get("valueFrom"): value = [self.tostr(v) for v in value] - return cast(List[str], ([prefix] if prefix else [])) + cast(List[str], value) + return cast(list[str], ([prefix] if prefix else [])) + cast(list[str], value) elif prefix and value: return [prefix] else: diff --git a/cwltool/checker.py b/cwltool/checker.py index 676245698..7742adf5c 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -1,19 +1,8 @@ """Static checking of CWL workflow connectivity.""" from collections import namedtuple -from typing import ( - Any, - Dict, - Iterator, - List, - Literal, - MutableMapping, - MutableSequence, - Optional, - Sized, - Union, - cast, -) +from collections.abc import Iterator, MutableMapping, MutableSequence, Sized +from typing import Any, Literal, Optional, Union, cast from schema_salad.exceptions import ValidationException from schema_salad.sourceline import SourceLine, bullets, strip_dup_lineno @@ -25,8 +14,7 @@ from .utils import CWLObjectType, CWLOutputType, SinkType, aslist -def _get_type(tp): - # type: (Any) -> Any +def _get_type(tp: Any) -> Any: if isinstance(tp, MutableMapping): if tp.get("type") not in ("array", "record", "enum"): return tp["type"] @@ -98,10 +86,10 @@ def can_assign_src_to_sink(src: SinkType, sink: Optional[SinkType], strict: bool if src["type"] == "record" and sink["type"] == "record": return _compare_records(src, sink, strict) if src["type"] == "File" and sink["type"] == "File": - for sinksf in cast(List[CWLObjectType], sink.get("secondaryFiles", [])): + for sinksf in cast(list[CWLObjectType], sink.get("secondaryFiles", [])): if not [ 1 - for srcsf in cast(List[CWLObjectType], src.get("secondaryFiles", [])) + for srcsf in cast(list[CWLObjectType], src.get("secondaryFiles", [])) if sinksf == srcsf ]: if strict: @@ -167,7 +155,7 @@ def _rec_fields(rec: MutableMapping[str, Any]) -> MutableMapping[str, Any]: return True -def missing_subset(fullset: List[Any], subset: List[Any]) -> List[Any]: +def missing_subset(fullset: list[Any], subset: list[Any]) -> list[Any]: missing = [] for i in subset: if i not in fullset: @@ -176,11 +164,11 @@ def missing_subset(fullset: List[Any], subset: List[Any]) -> List[Any]: def static_checker( - workflow_inputs: List[CWLObjectType], + workflow_inputs: list[CWLObjectType], workflow_outputs: MutableSequence[CWLObjectType], step_inputs: MutableSequence[CWLObjectType], - step_outputs: List[CWLObjectType], - param_to_step: Dict[str, CWLObjectType], + step_outputs: list[CWLObjectType], + param_to_step: dict[str, CWLObjectType], ) -> None: """ Check if all source and sink types of a workflow are compatible before run time. @@ -191,7 +179,7 @@ def static_checker( # sink parameters: step_inputs and workflow_outputs # make a dictionary of source parameters, indexed by the "id" field - src_dict: Dict[str, CWLObjectType] = {} + src_dict: dict[str, CWLObjectType] = {} for param in workflow_inputs + step_outputs: src_dict[cast(str, param["id"])] = param @@ -245,7 +233,7 @@ def static_checker( ", ".join( shortname(cast(str, s["id"])) for s in cast( - List[Dict[str, Union[str, bool]]], + list[dict[str, Union[str, bool]]], param_to_step[sink["id"]]["inputs"], ) if not s.get("not_connected") @@ -328,11 +316,11 @@ def static_checker( def check_all_types( - src_dict: Dict[str, CWLObjectType], + src_dict: dict[str, CWLObjectType], sinks: MutableSequence[CWLObjectType], sourceField: Union[Literal["source"], Literal["outputSource"]], - param_to_step: Dict[str, CWLObjectType], -) -> Dict[str, List[SrcSink]]: + param_to_step: dict[str, CWLObjectType], +) -> dict[str, list[SrcSink]]: """ Given a list of sinks, check if their types match with the types of their sources. @@ -340,7 +328,7 @@ def check_all_types( (from :py:func:`check_types`) :raises ValidationException: if a sourceField is missing """ - validation = {"warning": [], "exception": []} # type: Dict[str, List[SrcSink]] + validation: dict[str, list[SrcSink]] = {"warning": [], "exception": []} for sink in sinks: if sourceField in sink: valueFrom = cast(Optional[str], sink.get("valueFrom")) @@ -351,18 +339,18 @@ def check_all_types( extra_message = "pickValue is: %s" % pickValue if isinstance(sink[sourceField], MutableSequence): - linkMerge = cast( + linkMerge: Optional[str] = cast( Optional[str], sink.get( "linkMerge", ("merge_nested" if len(cast(Sized, sink[sourceField])) > 1 else None), ), - ) # type: Optional[str] + ) if pickValue in ["first_non_null", "the_only_non_null"]: linkMerge = None - srcs_of_sink = [] # type: List[CWLObjectType] + srcs_of_sink: list[CWLObjectType] = [] for parm_id in cast(MutableSequence[str], sink[sourceField]): srcs_of_sink += [src_dict[parm_id]] if is_conditional_step(param_to_step, parm_id) and pickValue is None: @@ -404,10 +392,10 @@ def check_all_types( snk_typ = sink["type"] if "null" not in src_typ: - src_typ = ["null"] + cast(List[Any], src_typ) + src_typ = ["null"] + cast(list[Any], src_typ) if "null" not in cast( - Union[List[str], CWLObjectType], snk_typ + Union[list[str], CWLObjectType], snk_typ ): # Given our type names this works even if not a list validation["warning"].append( SrcSink( @@ -440,7 +428,7 @@ def check_all_types( return validation -def circular_dependency_checker(step_inputs: List[CWLObjectType]) -> None: +def circular_dependency_checker(step_inputs: list[CWLObjectType]) -> None: """ Check if a workflow has circular dependency. @@ -448,8 +436,8 @@ def circular_dependency_checker(step_inputs: List[CWLObjectType]) -> None: """ adjacency = get_dependency_tree(step_inputs) vertices = adjacency.keys() - processed: List[str] = [] - cycles: List[List[str]] = [] + processed: list[str] = [] + cycles: list[list[str]] = [] for vertex in vertices: if vertex not in processed: traversal_path = [vertex] @@ -461,7 +449,7 @@ def circular_dependency_checker(step_inputs: List[CWLObjectType]) -> None: raise ValidationException(exception_msg) -def get_dependency_tree(step_inputs: List[CWLObjectType]) -> Dict[str, List[str]]: +def get_dependency_tree(step_inputs: list[CWLObjectType]) -> dict[str, list[str]]: """Get the dependency tree in the form of adjacency list.""" adjacency = {} # adjacency list of the dependency tree for step_input in step_inputs: @@ -482,10 +470,10 @@ def get_dependency_tree(step_inputs: List[CWLObjectType]) -> Dict[str, List[str] def processDFS( - adjacency: Dict[str, List[str]], - traversal_path: List[str], - processed: List[str], - cycles: List[List[str]], + adjacency: dict[str, list[str]], + traversal_path: list[str], + processed: list[str], + cycles: list[list[str]], ) -> None: """Perform depth first search.""" tip = traversal_path[-1] @@ -509,14 +497,14 @@ def get_step_id(field_id: str) -> str: return step_id -def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool: +def is_conditional_step(param_to_step: dict[str, CWLObjectType], parm_id: str) -> bool: if (source_step := param_to_step.get(parm_id)) is not None: if source_step.get("when") is not None: return True return False -def is_all_output_method_loop_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool: +def is_all_output_method_loop_step(param_to_step: dict[str, CWLObjectType], parm_id: str) -> bool: """Check if a step contains a `loop` directive with `all_iterations` outputMethod.""" source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id) if source_step is not None: diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index eb0b1a4f5..de1878593 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -12,25 +12,11 @@ import threading import urllib import urllib.parse +from collections.abc import Generator, Mapping, MutableMapping, MutableSequence from enum import Enum from functools import cmp_to_key, partial -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Generator, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - Pattern, - Set, - TextIO, - Type, - Union, - cast, -) +from re import Pattern +from typing import TYPE_CHECKING, Any, Optional, TextIO, Union, cast from mypy_extensions import mypyc_attr from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -163,8 +149,8 @@ def __init__( builder: Builder, script: str, output_callback: Optional[OutputCallbackType], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], outdir: Optional[str] = None, tmpdir: Optional[str] = None, ) -> None: @@ -241,7 +227,7 @@ def job( raise WorkflowException("Abstract operation cannot be executed.") -def remove_path(f): # type: (CWLObjectType) -> None +def remove_path(f: CWLObjectType) -> None: if "path" in f: del f["path"] @@ -334,7 +320,7 @@ def __init__( self.output_callback = output_callback self.cachebuilder = cachebuilder self.outdir = jobcache - self.prov_obj = None # type: Optional[ProvenanceProfile] + self.prov_obj: Optional[ProvenanceProfile] = None def run( self, @@ -392,7 +378,7 @@ def check_valid_locations(fs_access: StdFsAccess, ob: CWLObjectType) -> None: raise ValidationException("Does not exist or is not a Directory: '%s'" % location) -OutputPortsType = Dict[str, Optional[CWLOutputType]] +OutputPortsType = dict[str, Optional[CWLOutputType]] class ParameterOutputWorkflowException(WorkflowException): @@ -411,13 +397,13 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext """Initialize this CommandLineTool.""" super().__init__(toolpath_object, loadingContext) self.prov_obj = loadingContext.prov_obj - self.path_check_mode = ( + self.path_check_mode: PathCheckingMode = ( PathCheckingMode.RELAXED if loadingContext.relax_path_checks else PathCheckingMode.STRICT - ) # type: PathCheckingMode + ) - def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: + def make_job_runner(self, runtimeContext: RuntimeContext) -> type[JobBase]: dockerReq, dockerRequired = self.get_requirement("DockerRequirement") mpiReq, mpiRequired = self.get_requirement(MPIRequirementName) @@ -477,7 +463,7 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: @staticmethod def make_path_mapper( - reffiles: List[CWLObjectType], + reffiles: list[CWLObjectType], stagedir: str, runtimeContext: RuntimeContext, separateDirs: bool, @@ -499,9 +485,9 @@ def updatePathmap(self, outdir: str, pathmap: PathMapper, fn: CWLObjectType) -> ("Writable" if fn.get("writable") else "") + cast(str, fn["class"]), False, ) - for sf in cast(List[CWLObjectType], fn.get("secondaryFiles", [])): + for sf in cast(list[CWLObjectType], fn.get("secondaryFiles", [])): self.updatePathmap(outdir, pathmap, sf) - for ls in cast(List[CWLObjectType], fn.get("listing", [])): + for ls in cast(list[CWLObjectType], fn.get("listing", [])): self.updatePathmap(os.path.join(outdir, cast(str, fn["basename"])), pathmap, ls) def _initialworkdir(self, j: JobBase, builder: Builder) -> None: @@ -517,7 +503,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: cwl_version ) < ORDERED_VERSIONS.index("v1.1.0-dev1") - ls = [] # type: List[CWLObjectType] + ls: list[CWLObjectType] = [] if isinstance(initialWorkdir["listing"], str): # "listing" is just a string (must be an expression) so # just evaluate it and use the result as if it was in @@ -587,7 +573,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: raise SourceLine(initialWorkdir, "listing", WorkflowException, debug).makeError( message ) - ls = cast(List[CWLObjectType], ls_evaluated) + ls = cast(list[CWLObjectType], ls_evaluated) else: # "listing" is an array of either expressions or Dirent so # evaluate each item @@ -634,10 +620,10 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: for e in entry: ec = cast(CWLObjectType, e) ec["writable"] = t.get("writable", False) - ls.extend(cast(List[CWLObjectType], entry)) + ls.extend(cast(list[CWLObjectType], entry)) continue - et = {} # type: CWLObjectType + et: CWLObjectType = {} if isinstance(entry, Mapping) and entry.get("class") in ( "File", "Directory", @@ -686,7 +672,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: if not initwd_item: continue if isinstance(initwd_item, MutableSequence): - ls.extend(cast(List[CWLObjectType], initwd_item)) + ls.extend(cast(list[CWLObjectType], initwd_item)) else: ls.append(cast(CWLObjectType, initwd_item)) @@ -850,9 +836,9 @@ def job( cmdline = ["docker", "run", dockerimg] + cmdline # not really run using docker, just for hashing purposes - keydict = { + keydict: dict[str, Union[MutableSequence[Union[str, int]], CWLObjectType]] = { "cmdline": cmdline - } # type: Dict[str, Union[MutableSequence[Union[str, int]], CWLObjectType]] + } for shortcut in ["stdin", "stdout", "stderr"]: if shortcut in self.tool: @@ -1071,8 +1057,8 @@ def update_status_output_callback( j.inplace_update = cast(bool, inplaceUpdateReq["inplaceUpdate"]) normalizeFilesDirs(j.generatefiles) - readers = {} # type: Dict[str, CWLObjectType] - muts = set() # type: Set[str] + readers: dict[str, CWLObjectType] = {} + muts: set[str] = set() if builder.mutation_manager is not None: @@ -1103,7 +1089,7 @@ def register_reader(f: CWLObjectType) -> None: timelimit, _ = self.get_requirement("ToolTimeLimit") if timelimit is not None: with SourceLine(timelimit, "timelimit", ValidationException, debug): - limit_field = cast(Dict[str, Union[str, int]], timelimit)["timelimit"] + limit_field = cast(dict[str, Union[str, int]], timelimit)["timelimit"] if isinstance(limit_field, str): timelimit_eval = builder.do_eval(limit_field) if timelimit_eval and not isinstance(timelimit_eval, int): @@ -1142,7 +1128,7 @@ def register_reader(f: CWLObjectType) -> None: required_env = {} evr, _ = self.get_requirement("EnvVarRequirement") if evr is not None: - for eindex, t3 in enumerate(cast(List[Dict[str, str]], evr["envDef"])): + for eindex, t3 in enumerate(cast(list[dict[str, str]], evr["envDef"])): env_value_field = t3["envValue"] if "${" in env_value_field or "$(" in env_value_field: env_value_eval = builder.do_eval(env_value_field) @@ -1160,7 +1146,7 @@ def register_reader(f: CWLObjectType) -> None: shellcmd, _ = self.get_requirement("ShellCommandRequirement") if shellcmd is not None: - cmd = [] # type: List[str] + cmd: list[str] = [] for b in builder.bindings: arg = builder.generate_arg(b) if b.get("shellQuote", True): @@ -1201,7 +1187,7 @@ def register_reader(f: CWLObjectType) -> None: def collect_output_ports( self, - ports: Union[CommentedSeq, Set[CWLObjectType]], + ports: Union[CommentedSeq, set[CWLObjectType]], builder: Builder, outdir: str, rcode: int, @@ -1209,7 +1195,7 @@ def collect_output_ports( jobname: str = "", readers: Optional[MutableMapping[str, CWLObjectType]] = None, ) -> OutputPortsType: - ret = {} # type: OutputPortsType + ret: OutputPortsType = {} debug = _logger.isEnabledFor(logging.DEBUG) cwl_version = self.metadata.get(ORIGINAL_CWLVERSION, None) if cwl_version != "v1.0": @@ -1284,16 +1270,16 @@ def collect_output( fs_access: StdFsAccess, compute_checksum: bool = True, ) -> Optional[CWLOutputType]: - r = [] # type: List[CWLOutputType] + r: list[CWLOutputType] = [] empty_and_optional = False debug = _logger.isEnabledFor(logging.DEBUG) result: Optional[CWLOutputType] = None if "outputBinding" in schema: binding = cast( - MutableMapping[str, Union[bool, str, List[str]]], + MutableMapping[str, Union[bool, str, list[str]]], schema["outputBinding"], ) - globpatterns = [] # type: List[str] + globpatterns: list[str] = [] revmap = partial(revmap_file, builder, outdir) @@ -1359,7 +1345,7 @@ def collect_output( _logger.error("Unexpected error from fs_access", exc_info=True) raise - for files in cast(List[Dict[str, Optional[CWLOutputType]]], r): + for files in cast(list[dict[str, Optional[CWLOutputType]]], r): rfile = files.copy() revmap(rfile) if files["class"] == "Directory": @@ -1515,7 +1501,7 @@ def collect_output( and schema["type"]["type"] == "record" ): out = {} - for field in cast(List[CWLObjectType], schema["type"]["fields"]): + for field in cast(list[CWLObjectType], schema["type"]["fields"]): out[shortname(cast(str, field["name"]))] = self.collect_output( field, builder, outdir, fs_access, compute_checksum=compute_checksum ) diff --git a/cwltool/context.py b/cwltool/context.py index 283936d61..237a90968 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -5,20 +5,8 @@ import shutil import tempfile import threading -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - List, - Literal, - Optional, - TextIO, - Tuple, - Union, -) +from collections.abc import Iterable +from typing import IO, TYPE_CHECKING, Any, Callable, Literal, Optional, TextIO, Union from ruamel.yaml.comments import CommentedMap from schema_salad.avro.schema import Names @@ -46,7 +34,7 @@ class ContextBase: """Shared kwargs based initializer for :py:class:`RuntimeContext` and :py:class:`LoadingContext`.""" - def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: """Initialize.""" if kwargs: for k, v in kwargs.items(): @@ -87,13 +75,13 @@ def set_log_dir(outdir: str, log_dir: str, subdir_name: str) -> str: class LoadingContext(ContextBase): - def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: """Initialize the LoadingContext from the kwargs.""" self.debug: bool = False self.metadata: CWLObjectType = {} - self.requirements: Optional[List[CWLObjectType]] = None - self.hints: Optional[List[CWLObjectType]] = None - self.overrides_list: List[CWLObjectType] = [] + self.requirements: Optional[list[CWLObjectType]] = None + self.hints: Optional[list[CWLObjectType]] = None + self.overrides_list: list[CWLObjectType] = [] self.loader: Optional[Loader] = None self.avsc_names: Optional[Names] = None self.disable_js_validation: bool = False @@ -117,7 +105,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.singularity: bool = False self.podman: bool = False self.eval_timeout: float = 60 - self.codegen_idx: Dict[str, Tuple[Any, "LoadingOptions"]] = {} + self.codegen_idx: dict[str, tuple[Any, "LoadingOptions"]] = {} self.fast_parser = False self.skip_resolve_all = False self.skip_schemas = False @@ -136,11 +124,11 @@ class RuntimeContext(ContextBase): tmp_outdir_prefix: str = "" stagedir: str = "" - def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: """Initialize the RuntimeContext from the kwargs.""" select_resources_callable = Callable[ - [Dict[str, Union[int, float]], RuntimeContext], - Dict[str, Union[int, float]], + [dict[str, Union[int, float]], RuntimeContext], + dict[str, Union[int, float]], ] self.user_space_docker_cmd: Optional[str] = None self.secret_store: Optional["SecretStore"] = None diff --git a/cwltool/cuda.py b/cwltool/cuda.py index 719bfd867..1394ec239 100644 --- a/cwltool/cuda.py +++ b/cwltool/cuda.py @@ -2,13 +2,12 @@ import subprocess # nosec import xml.dom.minidom # nosec -from typing import Tuple from .loghandler import _logger from .utils import CWLObjectType -def cuda_version_and_device_count() -> Tuple[str, int]: +def cuda_version_and_device_count() -> tuple[str, int]: """Determine the CUDA version and number of attached CUDA GPUs.""" try: out = subprocess.check_output(["nvidia-smi", "-q", "-x"]) # nosec diff --git a/cwltool/cwlprov/__init__.py b/cwltool/cwlprov/__init__.py index b8ff8d14d..a09a57c34 100644 --- a/cwltool/cwlprov/__init__.py +++ b/cwltool/cwlprov/__init__.py @@ -6,10 +6,10 @@ import re import uuid from getpass import getuser -from typing import IO, Any, Callable, Dict, List, Optional, Tuple, TypedDict, Union +from typing import IO, Any, Callable, Optional, TypedDict, Union -def _whoami() -> Tuple[str, str]: +def _whoami() -> tuple[str, str]: """Return the current operating system account as (username, fullname).""" username = getuser() try: @@ -106,8 +106,8 @@ def _valid_orcid(orcid: Optional[str]) -> str: { "uri": str, "about": str, - "content": Optional[Union[str, List[str]]], - "oa:motivatedBy": Dict[str, str], + "content": Optional[Union[str, list[str]]], + "oa:motivatedBy": dict[str, str], }, ) @@ -116,11 +116,11 @@ class Aggregate(TypedDict, total=False): """RO Aggregate class.""" uri: Optional[str] - bundledAs: Optional[Dict[str, Any]] + bundledAs: Optional[dict[str, Any]] mediatype: Optional[str] - conformsTo: Optional[Union[str, List[str]]] + conformsTo: Optional[Union[str, list[str]]] createdOn: Optional[str] - createdBy: Optional[Dict[str, str]] + createdBy: Optional[dict[str, str]] # Aggregate.bundledAs is actually type Aggregate, but cyclic definitions are not supported diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index ce8d63ad4..59d835fff 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -3,22 +3,11 @@ import logging import urllib import uuid +from collections.abc import MutableMapping, MutableSequence, Sequence from io import BytesIO from pathlib import PurePath, PurePosixPath from socket import getfqdn -from typing import ( - TYPE_CHECKING, - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Sequence, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from prov.identifier import Identifier, QualifiedName from prov.model import PROV, PROV_LABEL, PROV_TYPE, PROV_VALUE, ProvDocument, ProvEntity @@ -117,7 +106,7 @@ def __str__(self) -> str: """Represent this Provenvance profile as a string.""" return f"ProvenanceProfile <{self.workflow_run_uri}> in <{self.research_object}>" - def generate_prov_doc(self) -> Tuple[str, ProvDocument]: + def generate_prov_doc(self) -> tuple[str, ProvDocument]: """Add basic namespaces.""" def host_provenance(document: ProvDocument) -> None: @@ -177,7 +166,7 @@ def host_provenance(document: ProvDocument) -> None: # by a user account, as cwltool is a command line tool account = self.document.agent(ACCOUNT_UUID) if self.orcid or self.full_name: - person: Dict[Union[str, Identifier], Any] = { + person: dict[Union[str, Identifier], Any] = { PROV_TYPE: PROV["Person"], "prov:type": SCHEMA["Person"], } @@ -291,7 +280,7 @@ def record_process_end( self.generate_output_prov(outputs, process_run_id, process_name) self.document.wasEndedBy(process_run_id, None, self.workflow_run_uri, when) - def declare_file(self, value: CWLObjectType) -> Tuple[ProvEntity, ProvEntity, str]: + def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, str]: if value["class"] != "File": raise ValueError("Must have class:File: %s" % value) # Need to determine file hash aka RO filename @@ -399,10 +388,10 @@ def declare_directory(self, value: CWLObjectType) -> ProvEntity: # dir_bundle.identifier, {PROV["type"]: ORE["ResourceMap"], # ORE["describes"]: coll_b.identifier}) - coll_attribs: List[Tuple[Union[str, Identifier], Any]] = [ + coll_attribs: list[tuple[Union[str, Identifier], Any]] = [ (ORE["isDescribedBy"], dir_bundle.identifier) ] - coll_b_attribs: List[Tuple[Union[str, Identifier], Any]] = [] + coll_b_attribs: list[tuple[Union[str, Identifier], Any]] = [] # FIXME: .listing might not be populated yet - hopefully # a later call to this method will sort that @@ -469,7 +458,7 @@ def declare_directory(self, value: CWLObjectType) -> ProvEntity: self.research_object.add_uri(coll.identifier.uri) return coll - def declare_string(self, value: str) -> Tuple[ProvEntity, str]: + def declare_string(self, value: str) -> tuple[ProvEntity, str]: """Save as string in UTF-8.""" byte_s = BytesIO(str(value).encode(ENCODING)) data_file = self.research_object.add_data_file(byte_s, content_type=TEXT_PLAIN) @@ -518,7 +507,7 @@ def declare_artefact(self, value: Any) -> ProvEntity: # Already processed this value, but it might not be in this PROV entities = self.document.get_record(value["@id"]) if entities: - return cast(List[ProvEntity], entities)[0] + return cast(list[ProvEntity], entities)[0] # else, unknown in PROV, re-add below as if it's fresh # Base case - we found a File we need to update @@ -549,7 +538,7 @@ def declare_artefact(self, value: Any) -> ProvEntity: coll.add_asserted_type(CWLPROV[value["class"]]) # Let's iterate and recurse - coll_attribs: List[Tuple[Union[str, Identifier], Any]] = [] + coll_attribs: list[tuple[Union[str, Identifier], Any]] = [] for key, val in value.items(): v_ent = self.declare_artefact(val) self.document.membership(coll, v_ent) @@ -601,7 +590,7 @@ def declare_artefact(self, value: Any) -> ProvEntity: def used_artefacts( self, - job_order: Union[CWLObjectType, List[CWLObjectType]], + job_order: Union[CWLObjectType, list[CWLObjectType]], process_run_id: str, name: Optional[str] = None, ) -> None: @@ -704,7 +693,7 @@ def activity_has_provenance(self, activity: str, prov_ids: Sequence[Identifier]) """Add http://www.w3.org/TR/prov-aq/ relations to nested PROV files.""" # NOTE: The below will only work if the corresponding metadata/provenance arcp URI # is a pre-registered namespace in the PROV Document - attribs: List[Tuple[Union[str, Identifier], Any]] = [ + attribs: list[tuple[Union[str, Identifier], Any]] = [ (PROV["has_provenance"], prov_id) for prov_id in prov_ids ] self.document.activity(activity, other_attributes=attribs) @@ -713,7 +702,7 @@ def activity_has_provenance(self, activity: str, prov_ids: Sequence[Identifier]) uris = [i.uri for i in prov_ids] self.research_object.add_annotation(activity, uris, PROV["has_provenance"].uri) - def finalize_prov_profile(self, name: Optional[str]) -> List[QualifiedName]: + def finalize_prov_profile(self, name: Optional[str]) -> list[QualifiedName]: """Transfer the provenance related files to the RO.""" # NOTE: Relative posix path if name is None: diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index 7c6eaf5d6..ac60afc92 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -7,20 +7,9 @@ import tempfile import urllib import uuid +from collections.abc import MutableMapping, MutableSequence from pathlib import Path, PurePosixPath -from typing import ( - IO, - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Set, - Tuple, - Union, - cast, -) +from typing import IO, Any, Optional, Union, cast import prov.model as provM from prov.model import PROV, ProvDocument @@ -75,12 +64,12 @@ def __init__( self.folder = create_tmp_dir(temp_prefix_ro) self.closed = False # map of filename "data/de/alsdklkas": 12398123 bytes - self.bagged_size: Dict[str, int] = {} - self.tagfiles: Set[str] = set() - self._file_provenance: Dict[str, Aggregate] = {} - self._external_aggregates: List[Aggregate] = [] - self.annotations: List[Annotation] = [] - self._content_types: Dict[str, str] = {} + self.bagged_size: dict[str, int] = {} + self.tagfiles: set[str] = set() + self._file_provenance: dict[str, Aggregate] = {} + self._external_aggregates: list[Aggregate] = [] + self.annotations: list[Annotation] = [] + self._content_types: dict[str, str] = {} self.fsaccess = fsaccess # These should be replaced by generate_prov_doc when workflow/run IDs are known: self.engine_uuid = f"urn:uuid:{uuid.uuid4()}" @@ -202,14 +191,14 @@ def add_tagfile(self, path: str, timestamp: Optional[datetime.datetime] = None) "conformsTo": None, } - def _ro_aggregates(self) -> List[Aggregate]: + def _ro_aggregates(self) -> list[Aggregate]: """Gather dictionary of files to be added to the manifest.""" def guess_mediatype( rel_path: str, - ) -> Tuple[Optional[str], Optional[Union[str, List[str]]]]: + ) -> tuple[Optional[str], Optional[Union[str, list[str]]]]: """Return the mediatypes.""" - media_types: Dict[Union[str, None], str] = { + media_types: dict[Union[str, None], str] = { # Adapted from # https://w3id.org/bundle/2014-11-05/#media-types "txt": TEXT_PLAIN, @@ -223,12 +212,12 @@ def guess_mediatype( "provn": 'text/provenance-notation; charset="UTF-8"', "nt": "application/n-triples", } - conforms_to: Dict[Union[str, None], str] = { + conforms_to: dict[Union[str, None], str] = { "provn": "http://www.w3.org/TR/2013/REC-prov-n-20130430/", "cwl": "https://w3id.org/cwl/", } - prov_conforms_to: Dict[str, str] = { + prov_conforms_to: dict[str, str] = { "provn": "http://www.w3.org/TR/2013/REC-prov-n-20130430/", "rdf": "http://www.w3.org/TR/2013/REC-prov-o-20130430/", "ttl": "http://www.w3.org/TR/2013/REC-prov-o-20130430/", @@ -244,7 +233,7 @@ def guess_mediatype( extension = None mediatype: Optional[str] = media_types.get(extension, None) - conformsTo: Optional[Union[str, List[str]]] = conforms_to.get(extension, None) + conformsTo: Optional[Union[str, list[str]]] = conforms_to.get(extension, None) # TODO: Open CWL file to read its declared "cwlVersion", e.g. # cwlVersion = "v1.0" @@ -261,7 +250,7 @@ def guess_mediatype( conformsTo = prov_conforms_to[extension] return (mediatype, conformsTo) - aggregates: List[Aggregate] = [] + aggregates: list[Aggregate] = [] for path in self.bagged_size.keys(): temp_path = PurePosixPath(path) folder = temp_path.parent @@ -291,7 +280,7 @@ def guess_mediatype( bundledAs.update(self._file_provenance[path]) else: aggregate_dict["bundledAs"] = cast( - Optional[Dict[str, Any]], self._file_provenance[path] + Optional[dict[str, Any]], self._file_provenance[path] ) else: # Probably made outside wf run, part of job object? @@ -343,7 +332,7 @@ def add_uri(self, uri: str, timestamp: Optional[datetime.datetime] = None) -> Ag return aggr def add_annotation( - self, about: str, content: List[str], motivated_by: str = "oa:describing" + self, about: str, content: list[str], motivated_by: str = "oa:describing" ) -> str: """Cheap URI relativize for current directory and /.""" self.self_check() @@ -359,9 +348,9 @@ def add_annotation( self.annotations.append(ann) return uri - def _ro_annotations(self) -> List[Annotation]: + def _ro_annotations(self) -> list[Annotation]: """Append base RO and provenance annotations to the list of annotations.""" - annotations: List[Annotation] = [] + annotations: list[Annotation] = [] annotations.append( { "uri": uuid.uuid4().urn, @@ -511,7 +500,7 @@ def add_data_file( def _self_made( self, timestamp: Optional[datetime.datetime] = None - ) -> Tuple[str, Dict[str, str]]: # createdOn, createdBy + ) -> tuple[str, dict[str, str]]: # createdOn, createdBy if timestamp is None: timestamp = datetime.datetime.now() return ( @@ -519,7 +508,7 @@ def _self_made( {"uri": self.engine_uuid, "name": self.cwltool_version}, ) - def add_to_manifest(self, rel_path: str, checksums: Dict[str, str]) -> None: + def add_to_manifest(self, rel_path: str, checksums: dict[str, str]) -> None: """Add files to the research object manifest.""" self.self_check() if PurePosixPath(rel_path).is_absolute(): diff --git a/cwltool/cwlprov/writablebagfile.py b/cwltool/cwlprov/writablebagfile.py index d5ff3c731..06d7d0bf7 100644 --- a/cwltool/cwlprov/writablebagfile.py +++ b/cwltool/cwlprov/writablebagfile.py @@ -8,10 +8,11 @@ import uuid from array import array from collections import OrderedDict +from collections.abc import MutableMapping from io import FileIO, TextIOWrapper from mmap import mmap from pathlib import Path, PurePosixPath -from typing import Any, BinaryIO, Dict, MutableMapping, Optional, Union, cast +from typing import Any, BinaryIO, Optional, Union, cast from schema_salad.utils import json_dumps @@ -246,7 +247,7 @@ def create_job( relativised_input_objecttemp: CWLObjectType = {} research_object._relativise_files(copied) - def jdefault(o: Any) -> Dict[Any, Any]: + def jdefault(o: Any) -> dict[Any, Any]: return dict(o) if is_output: diff --git a/cwltool/cwlrdf.py b/cwltool/cwlrdf.py index dbe9e2f97..126f0c780 100644 --- a/cwltool/cwlrdf.py +++ b/cwltool/cwlrdf.py @@ -1,6 +1,7 @@ import urllib from codecs import StreamWriter -from typing import IO, Any, Dict, Iterator, Optional, TextIO, Union, cast +from collections.abc import Iterator +from typing import IO, Any, Optional, TextIO, Union, cast from rdflib import Graph from rdflib.query import ResultRow @@ -117,7 +118,7 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: - dotname: Dict[str, str] = {} + dotname: dict[str, str] = {} clusternode = {} stdout.write("compound=true\n") diff --git a/cwltool/cwlviewer.py b/cwltool/cwlviewer.py index e544a568e..769343964 100644 --- a/cwltool/cwlviewer.py +++ b/cwltool/cwlviewer.py @@ -1,7 +1,8 @@ """Visualize a CWL workflow.""" +from collections.abc import Iterator from pathlib import Path -from typing import Iterator, List, cast +from typing import cast from urllib.parse import urlparse import pydot @@ -154,7 +155,7 @@ def _get_root_graph_uri(self) -> rdflib.term.Identifier: with open(_get_root_query_path) as f: get_root_query = f.read() root = cast( - List[rdflib.query.ResultRow], + list[rdflib.query.ResultRow], list( self._rdf_graph.query( get_root_query, diff --git a/cwltool/docker.py b/cwltool/docker.py index d0f628b15..b03ae635c 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -9,8 +9,9 @@ import subprocess # nosec import sys import threading +from collections.abc import MutableMapping from io import StringIO # pylint: disable=redefined-builtin -from typing import Callable, Dict, List, MutableMapping, Optional, Set, Tuple, cast +from typing import Callable, Optional, cast import requests @@ -23,13 +24,13 @@ from .pathmapper import MapperEnt, PathMapper from .utils import CWLObjectType, create_tmp_dir, ensure_writable -_IMAGES: Set[str] = set() +_IMAGES: set[str] = set() _IMAGES_LOCK = threading.Lock() -__docker_machine_mounts: Optional[List[str]] = None +__docker_machine_mounts: Optional[list[str]] = None __docker_machine_mounts_lock = threading.Lock() -def _get_docker_machine_mounts() -> List[str]: +def _get_docker_machine_mounts() -> list[str]: global __docker_machine_mounts if __docker_machine_mounts is None: with __docker_machine_mounts_lock: @@ -83,9 +84,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Initialize a command line builder using the Docker software container engine.""" @@ -94,7 +95,7 @@ def __init__( def get_image( self, - docker_requirement: Dict[str, str], + docker_requirement: dict[str, str], pull_image: bool, force_pull: bool, tmp_outdir_prefix: str, @@ -127,7 +128,7 @@ def get_image( except (OSError, subprocess.CalledProcessError, UnicodeError): pass - cmd: List[str] = [] + cmd: list[str] = [] if "dockerFile" in docker_requirement: dockerfile_dir = create_tmp_dir(tmp_outdir_prefix) with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as dfile: @@ -204,13 +205,13 @@ def get_from_requirements( if not shutil.which(self.docker_exec): raise WorkflowException(f"{self.docker_exec} executable is not available") - if self.get_image(cast(Dict[str, str], r), pull_image, force_pull, tmp_outdir_prefix): + if self.get_image(cast(dict[str, str], r), pull_image, force_pull, tmp_outdir_prefix): return cast(Optional[str], r["dockerImageId"]) raise WorkflowException("Docker image %s not found" % r["dockerImageId"]) @staticmethod def append_volume( - runtime: List[str], + runtime: list[str], source: str, target: str, writable: bool = False, @@ -233,7 +234,7 @@ def append_volume( os.makedirs(source) def add_file_or_directory_volume( - self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str] ) -> None: """Append volume a file/dir mapping to the runtime option list.""" if not volume.resolved.startswith("_:"): @@ -242,7 +243,7 @@ def add_file_or_directory_volume( def add_writable_file_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -266,7 +267,7 @@ def add_writable_file_volume( def add_writable_directory_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -295,7 +296,7 @@ def add_writable_directory_volume( shutil.copytree(volume.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt or new_dir) - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: # spec currently says "HOME must be set to the designated output # directory." but spec might change to designated temp directory. # runtime.append("--env=HOME=/tmp") @@ -306,7 +307,7 @@ def _required_env(self) -> Dict[str, str]: def create_runtime( self, env: MutableMapping[str, str], runtimeContext: RuntimeContext - ) -> Tuple[List[str], Optional[str]]: + ) -> tuple[list[str], Optional[str]]: any_path_okay = self.builder.get_requirement("DockerRequirement")[1] or False user_space_docker_cmd = runtimeContext.user_space_docker_cmd if user_space_docker_cmd: @@ -445,9 +446,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Initialize a command line builder using the Podman software container engine.""" diff --git a/cwltool/docker_id.py b/cwltool/docker_id.py index bb436b2cb..90484b686 100644 --- a/cwltool/docker_id.py +++ b/cwltool/docker_id.py @@ -1,10 +1,10 @@ """Helper functions for docker.""" import subprocess # nosec -from typing import List, Optional, Tuple +from typing import Optional -def docker_vm_id() -> Tuple[Optional[int], Optional[int]]: +def docker_vm_id() -> tuple[Optional[int], Optional[int]]: """ Return the User ID and Group ID of the default docker user inside the VM. @@ -21,7 +21,7 @@ def docker_vm_id() -> Tuple[Optional[int], Optional[int]]: return (None, None) -def check_output_and_strip(cmd: List[str]) -> Optional[str]: +def check_output_and_strip(cmd: list[str]) -> Optional[str]: """ Pass a command list to :py:func:`subprocess.check_output`. @@ -48,7 +48,7 @@ def docker_machine_name() -> Optional[str]: return check_output_and_strip(["docker-machine", "active"]) -def cmd_output_matches(check_cmd: List[str], expected_status: str) -> bool: +def cmd_output_matches(check_cmd: list[str], expected_status: str) -> bool: """ Run a command and compares output to expected. @@ -80,7 +80,7 @@ def docker_machine_running() -> bool: return cmd_output_matches(["docker-machine", "status", machine_name], "Running") -def cmd_output_to_int(cmd: List[str]) -> Optional[int]: +def cmd_output_to_int(cmd: list[str]) -> Optional[int]: """ Run the provided command and returns the integer value of the result. @@ -97,7 +97,7 @@ def cmd_output_to_int(cmd: List[str]) -> Optional[int]: return None -def boot2docker_id() -> Tuple[Optional[int], Optional[int]]: +def boot2docker_id() -> tuple[Optional[int], Optional[int]]: """ Get the UID and GID of the docker user inside a running boot2docker vm. @@ -108,7 +108,7 @@ def boot2docker_id() -> Tuple[Optional[int], Optional[int]]: return (uid, gid) -def docker_machine_id() -> Tuple[Optional[int], Optional[int]]: +def docker_machine_id() -> tuple[Optional[int], Optional[int]]: """ Ask docker-machine for active machine and gets the UID of the docker user. diff --git a/cwltool/env_to_stdout.py b/cwltool/env_to_stdout.py index 33b832479..0309fe08f 100644 --- a/cwltool/env_to_stdout.py +++ b/cwltool/env_to_stdout.py @@ -11,10 +11,9 @@ """ import os -from typing import Dict -def deserialize_env(data: str) -> Dict[str, str]: +def deserialize_env(data: str) -> dict[str, str]: """Deserialize the output of `env -0` to dictionary.""" result = {} for item in data.strip("\0").split("\0"): diff --git a/cwltool/executors.py b/cwltool/executors.py index bfc87f9c7..6070462ab 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -7,18 +7,9 @@ import os import threading from abc import ABCMeta, abstractmethod +from collections.abc import Iterable, MutableSequence from threading import Lock -from typing import ( - Dict, - Iterable, - List, - MutableSequence, - Optional, - Set, - Tuple, - Union, - cast, -) +from typing import Optional, Union, cast import psutil from mypy_extensions import mypyc_attr @@ -50,8 +41,8 @@ class JobExecutor(metaclass=ABCMeta): def __init__(self) -> None: """Initialize.""" self.final_output: MutableSequence[Optional[CWLObjectType]] = [] - self.final_status: List[str] = [] - self.output_dirs: Set[str] = set() + self.final_status: list[str] = [] + self.output_dirs: set[str] = set() def __call__( self, @@ -59,7 +50,7 @@ def __call__( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: logging.Logger = _logger, - ) -> Tuple[Optional[CWLObjectType], str]: + ) -> tuple[Optional[CWLObjectType], str]: return self.execute(process, job_order_object, runtime_context, logger) def output_callback(self, out: Optional[CWLObjectType], process_status: str) -> None: @@ -83,7 +74,7 @@ def execute( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: logging.Logger = _logger, - ) -> Tuple[Union[Optional[CWLObjectType]], str]: + ) -> tuple[Union[Optional[CWLObjectType]], str]: """Execute the process.""" self.final_output = [] @@ -112,7 +103,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: runtime_context.toplevel = True runtime_context.workflow_eval_lock = threading.Condition(threading.RLock()) - job_reqs: Optional[List[CWLObjectType]] = None + job_reqs: Optional[list[CWLObjectType]] = None if "https://w3id.org/cwl/cwl#requirements" in job_order_object: if process.metadata.get(ORIGINAL_CWLVERSION) == "v1.0": raise WorkflowException( @@ -121,7 +112,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: "can set the cwlVersion to v1.1" ) job_reqs = cast( - List[CWLObjectType], + list[CWLObjectType], job_order_object["https://w3id.org/cwl/cwl#requirements"], ) elif "cwl:defaults" in process.metadata and "https://w3id.org/cwl/cwl#requirements" in cast( @@ -134,7 +125,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: "can set the cwlVersion to v1.1" ) job_reqs = cast( - Optional[List[CWLObjectType]], + Optional[list[CWLObjectType]], cast(CWLObjectType, process.metadata["cwl:defaults"])[ "https://w3id.org/cwl/cwl#requirements" ], @@ -277,8 +268,8 @@ class MultithreadedJobExecutor(JobExecutor): def __init__(self) -> None: """Initialize.""" super().__init__() - self.exceptions: List[WorkflowException] = [] - self.pending_jobs: List[JobsType] = [] + self.exceptions: list[WorkflowException] = [] + self.pending_jobs: list[JobsType] = [] self.pending_jobs_lock = threading.Lock() self.max_ram = int(psutil.virtual_memory().available / 2**20) @@ -289,10 +280,10 @@ def __init__(self) -> None: self.allocated_cuda: int = 0 def select_resources( - self, request: Dict[str, Union[int, float]], runtime_context: RuntimeContext - ) -> Dict[str, Union[int, float]]: # pylint: disable=unused-argument + self, request: dict[str, Union[int, float]], runtime_context: RuntimeContext + ) -> dict[str, Union[int, float]]: # pylint: disable=unused-argument """Naïve check for available cpu cores and memory.""" - result: Dict[str, Union[int, float]] = {} + result: dict[str, Union[int, float]] = {} maxrsc = {"cores": self.max_cores, "ram": self.max_ram} resources_types = {"cores", "ram"} if "cudaDeviceCountMin" in request or "cudaDeviceCountMax" in request: @@ -491,5 +482,5 @@ def execute( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: Optional[logging.Logger] = None, - ) -> Tuple[Optional[CWLObjectType], str]: + ) -> tuple[Optional[CWLObjectType], str]: return {}, "success" diff --git a/cwltool/factory.py b/cwltool/factory.py index 85d7344e6..eaf98e3cf 100644 --- a/cwltool/factory.py +++ b/cwltool/factory.py @@ -1,5 +1,5 @@ import os -from typing import Any, Dict, Optional, Union +from typing import Any, Optional, Union from . import load_tool from .context import LoadingContext, RuntimeContext @@ -62,7 +62,7 @@ def __init__( else: self.loading_context = loading_context - def make(self, cwl: Union[str, Dict[str, Any]]) -> Callable: + def make(self, cwl: Union[str, dict[str, Any]]) -> Callable: """Instantiate a CWL object from a CWl document.""" load = load_tool.load_tool(cwl, self.loading_context) if isinstance(load, int): diff --git a/cwltool/flatten.py b/cwltool/flatten.py index 420d90d04..5c9738cbf 100644 --- a/cwltool/flatten.py +++ b/cwltool/flatten.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, List, cast +from typing import Any, Callable, cast # http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html @@ -22,4 +22,4 @@ def flatten(thing, ltypes=(list, tuple)): else: thing_list[i : i + 1] = thing_list[i] i += 1 - return cast(Callable[[Any], List[Any]], ltype)(thing_list) + return cast(Callable[[Any], list[Any]], ltype)(thing_list) diff --git a/cwltool/job.py b/cwltool/job.py index 1731a5350..b360be25f 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -16,24 +16,10 @@ import time import uuid from abc import ABCMeta, abstractmethod +from collections.abc import Iterable, Mapping, MutableMapping, MutableSequence +from re import Match from threading import Timer -from typing import ( - IO, - TYPE_CHECKING, - Callable, - Dict, - Iterable, - List, - Mapping, - Match, - MutableMapping, - MutableSequence, - Optional, - TextIO, - Tuple, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Callable, Optional, TextIO, Union, cast import psutil from prov.model import PROV @@ -122,9 +108,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Initialize the job object.""" @@ -140,7 +126,7 @@ def __init__( self.requirements = requirements self.hints = hints self.name = name - self.command_line: List[str] = [] + self.command_line: list[str] = [] self.pathmapper = PathMapper([], "", "") self.make_path_mapper = make_path_mapper self.generatemapper: Optional[PathMapper] = None @@ -228,7 +214,7 @@ def is_streamable(file: str) -> bool: def _execute( self, - runtime: List[str], + runtime: list[str], env: MutableMapping[str, str], runtimeContext: RuntimeContext, monitor_function: Optional[Callable[["subprocess.Popen[str]"], None]] = None, @@ -321,7 +307,7 @@ def stderr_stdout_log_path( commands = [str(x) for x in runtime + self.command_line] if runtimeContext.secret_store is not None: commands = cast( - List[str], + list[str], runtimeContext.secret_store.retrieve(cast(CWLOutputType, commands)), ) env = cast( @@ -456,7 +442,7 @@ def stderr_stdout_log_path( shutil.rmtree(self.tmpdir, True) @abstractmethod - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: """Variables required by the CWL spec (HOME, TMPDIR, etc). Note that with containers, the paths will (likely) be those from @@ -481,7 +467,7 @@ def prepare_environment( applied (in that order). """ # Start empty - env: Dict[str, str] = {} + env: dict[str, str] = {} # Preserve any env vars if runtimeContext.preserve_entire_environment: @@ -589,7 +575,7 @@ def run( self._execute([], self.environment, runtimeContext, monitor_function) - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: env = {} env["HOME"] = self.outdir env["TMPDIR"] = self.tmpdir @@ -623,24 +609,24 @@ def create_runtime( self, env: MutableMapping[str, str], runtime_context: RuntimeContext, - ) -> Tuple[List[str], Optional[str]]: + ) -> tuple[list[str], Optional[str]]: """Return the list of commands to run the selected container engine.""" @staticmethod @abstractmethod - def append_volume(runtime: List[str], source: str, target: str, writable: bool = False) -> None: + def append_volume(runtime: list[str], source: str, target: str, writable: bool = False) -> None: """Add binding arguments to the runtime list.""" @abstractmethod def add_file_or_directory_volume( - self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str] ) -> None: """Append volume a file/dir mapping to the runtime option list.""" @abstractmethod def add_writable_file_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -650,7 +636,7 @@ def add_writable_file_volume( @abstractmethod def add_writable_directory_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -674,7 +660,7 @@ def _preserve_environment_on_containers_warning( def create_file_and_add_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], secret_store: Optional[SecretStore], @@ -706,7 +692,7 @@ def create_file_and_add_volume( def add_volumes( self, pathmapper: PathMapper, - runtime: List[str], + runtime: list[str], tmpdir_prefix: str, secret_store: Optional[SecretStore] = None, any_path_okay: bool = False, @@ -918,7 +904,7 @@ def docker_monitor( def _job_popen( - commands: List[str], + commands: list[str], stdin_path: Optional[str], stdout_path: Optional[str], stderr_path: Optional[str], diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index d6352f918..7a58a8330 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -7,18 +7,9 @@ import re import urllib import uuid +from collections.abc import MutableMapping, MutableSequence from functools import partial -from typing import ( - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Tuple, - Union, - cast, -) +from typing import Any, Optional, Union, cast from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -93,7 +84,7 @@ def resolve_tool_uri( resolver: Optional[ResolverType] = None, fetcher_constructor: Optional[FetcherCallableType] = None, document_loader: Optional[Loader] = None, -) -> Tuple[str, str]: +) -> tuple[str, str]: uri = None # type: Optional[str] split = urllib.parse.urlsplit(argsworkflow) # In case of Windows path, urlsplit misjudge Drive letters as scheme, here we are skipping that @@ -117,7 +108,7 @@ def resolve_tool_uri( def fetch_document( argsworkflow: Union[str, CWLObjectType], loadingContext: Optional[LoadingContext] = None, -) -> Tuple[LoadingContext, CommentedMap, str]: +) -> tuple[LoadingContext, CommentedMap, str]: """Retrieve a CWL document.""" if loadingContext is None: loadingContext = LoadingContext() @@ -144,7 +135,7 @@ def fetch_document( return loadingContext, workflowobj, uri if isinstance(argsworkflow, MutableMapping): uri = cast(str, argsworkflow["id"]) if argsworkflow.get("id") else "_:" + str(uuid.uuid4()) - workflowobj = cast(CommentedMap, cmap(cast(Dict[str, Any], argsworkflow), fn=uri)) + workflowobj = cast(CommentedMap, cmap(cast(dict[str, Any], argsworkflow), fn=uri)) loadingContext.loader.idx[uri] = workflowobj return loadingContext, workflowobj, uri raise ValidationException("Must be URI or object: '%s'" % argsworkflow) @@ -306,7 +297,7 @@ def fast_parser( uri: str, loadingContext: LoadingContext, fetcher: Fetcher, -) -> Tuple[Union[CommentedMap, CommentedSeq], CommentedMap]: +) -> tuple[Union[CommentedMap, CommentedSeq], CommentedMap]: lopt = cwl_v1_2.LoadingOptions(idx=loadingContext.codegen_idx, fileuri=fileuri, fetcher=fetcher) if uri not in loadingContext.codegen_idx: @@ -326,7 +317,7 @@ def fast_parser( processobj = cwl_v1_2.save(objects, relative_uris=False) - metadata: Dict[str, Any] = {} + metadata: dict[str, Any] = {} metadata["id"] = loadopt.fileuri if loadopt.namespaces: @@ -353,7 +344,7 @@ def fast_parser( objects, loadopt = loadingContext.codegen_idx[nofrag] fileobj = cmap( cast( - Union[int, float, str, Dict[str, Any], List[Any], None], + Union[int, float, str, dict[str, Any], list[Any], None], cwl_v1_2.save(objects, relative_uris=False), ) ) @@ -370,7 +361,7 @@ def fast_parser( return cast( Union[CommentedMap, CommentedSeq], - cmap(cast(Union[Dict[str, Any], List[Any]], processobj)), + cmap(cast(Union[dict[str, Any], list[Any]], processobj)), ), cast(CommentedMap, cmap(metadata)) @@ -379,7 +370,7 @@ def resolve_and_validate_document( workflowobj: Union[CommentedMap, CommentedSeq], uri: str, preprocess_only: bool = False, -) -> Tuple[LoadingContext, str]: +) -> tuple[LoadingContext, str]: """Validate a CWL document.""" if not loadingContext.loader: raise ValueError("loadingContext must have a loader.") @@ -394,7 +385,7 @@ def resolve_and_validate_document( if "cwl:tool" in workflowobj: jobobj, _ = loader.resolve_all(workflowobj, uri) uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) - del cast(Dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"] + del cast(dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri, loadingContext)[1] @@ -624,17 +615,17 @@ def resolve_overrides( ov: IdxResultType, ov_uri: str, baseurl: str, -) -> List[CWLObjectType]: +) -> list[CWLObjectType]: ovloader = Loader(overrides_ctx) ret, _ = ovloader.resolve_all(ov, baseurl) if not isinstance(ret, CommentedMap): raise Exception("Expected CommentedMap, got %s" % type(ret)) cwl_docloader = get_schema("v1.0")[0] cwl_docloader.resolve_all(ret, ov_uri) - return cast(List[CWLObjectType], ret["http://commonwl.org/cwltool#overrides"]) + return cast(list[CWLObjectType], ret["http://commonwl.org/cwltool#overrides"]) -def load_overrides(ov: str, base_url: str) -> List[CWLObjectType]: +def load_overrides(ov: str, base_url: str) -> list[CWLObjectType]: ovloader = Loader(overrides_ctx) return resolve_overrides(ovloader.fetch(ov), ov, base_url) @@ -644,7 +635,7 @@ def recursive_resolve_and_validate_document( workflowobj: Union[CommentedMap, CommentedSeq], uri: str, preprocess_only: bool = False, -) -> Tuple[LoadingContext, str, Process]: +) -> tuple[LoadingContext, str, Process]: """Validate a CWL document, checking that a tool object can be built.""" loadingContext, uri = resolve_and_validate_document( loadingContext, diff --git a/cwltool/main.py b/cwltool/main.py index 30f299f09..9477cb1a2 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -15,21 +15,8 @@ import urllib import warnings from codecs import getwriter -from typing import ( - IO, - Any, - Callable, - Dict, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - Sized, - Tuple, - Union, - cast, -) +from collections.abc import Mapping, MutableMapping, MutableSequence, Sized +from typing import IO, Any, Callable, Optional, Union, cast import argcomplete import coloredlogs @@ -185,11 +172,11 @@ def append_word_to_default_user_agent(word: str) -> None: def generate_example_input( inptype: Optional[CWLOutputType], default: Optional[CWLOutputType], -) -> Tuple[Any, str]: +) -> tuple[Any, str]: """Convert a single input schema into an example.""" example = None comment = "" - defaults = { + defaults: CWLObjectType = { "null": "null", "Any": "null", "boolean": False, @@ -202,7 +189,7 @@ def generate_example_input( "Directory": ruamel.yaml.comments.CommentedMap( [("class", "Directory"), ("path", "a/directory/path")] ), - } # type: CWLObjectType + } if isinstance(inptype, MutableSequence): optional = False if "null" in inptype: @@ -244,7 +231,7 @@ def generate_example_input( if default is not None: example = default elif inptype["type"] == "enum": - symbols = cast(List[str], inptype["symbols"]) + symbols = cast(list[str], inptype["symbols"]) if default is not None: example = default elif "default" in inptype: @@ -260,7 +247,7 @@ def generate_example_input( comment = '"{}" record type.'.format(inptype["name"]) else: comment = "Anonymous record type." - for field in cast(List[CWLObjectType], inptype["fields"]): + for field in cast(list[CWLObjectType], inptype["fields"]): value, f_comment = generate_example_input(field["type"], None) example.insert(0, shortname(cast(str, field["name"])), value, f_comment) elif "default" in inptype: @@ -343,7 +330,7 @@ def generate_input_template(tool: Process) -> CWLObjectType: """Generate an example input object for the given CWL process.""" template = ruamel.yaml.comments.CommentedMap() for inp in cast( - List[MutableMapping[str, str]], + list[MutableMapping[str, str]], realize_input_schema(tool.tool["inputs"], tool.schemaDefs), ): name = shortname(inp["id"]) @@ -356,9 +343,9 @@ def load_job_order( args: argparse.Namespace, stdin: IO[Any], fetcher_constructor: Optional[FetcherCallableType], - overrides_list: List[CWLObjectType], + overrides_list: list[CWLObjectType], tool_file_uri: str, -) -> Tuple[Optional[CWLObjectType], str, Loader]: +) -> tuple[Optional[CWLObjectType], str, Loader]: job_order_object = None job_order_file = None @@ -423,8 +410,8 @@ def init_job_order( ) -> CWLObjectType: secrets_req, _ = process.get_requirement("http://commonwl.org/cwltool#Secrets") if job_order_object is None: - namemap = {} # type: Dict[str, str] - records = [] # type: List[str] + namemap: dict[str, str] = {} + records: list[str] = [] toolparser = generate_parser( argparse.ArgumentParser(prog=args.workflow), process, @@ -463,7 +450,7 @@ def init_job_order( if secret_store and secrets_req: secret_store.store( - [shortname(sc) for sc in cast(List[str], secrets_req["secrets"])], + [shortname(sc) for sc in cast(list[str], secrets_req["secrets"])], job_order_object, ) @@ -486,7 +473,7 @@ def path_to_loc(p: CWLObjectType) -> None: p["location"] = p["path"] del p["path"] - ns = {} # type: ContextType + ns: ContextType = {} ns.update(cast(ContextType, job_order_object.get("$namespaces", {}))) ns.update(cast(ContextType, process.metadata.get("$namespaces", {}))) ld = Loader(ns) @@ -532,7 +519,7 @@ def expand_formats(p: CWLObjectType) -> None: if secret_store and secrets_req: secret_store.store( - [shortname(sc) for sc in cast(List[str], secrets_req["secrets"])], + [shortname(sc) for sc in cast(list[str], secrets_req["secrets"])], job_order_object, ) @@ -583,7 +570,7 @@ def prov_deps( def remove_non_cwl(deps: CWLObjectType) -> None: if "secondaryFiles" in deps: - sec_files = cast(List[CWLObjectType], deps["secondaryFiles"]) + sec_files = cast(list[CWLObjectType], deps["secondaryFiles"]) for index, entry in enumerate(sec_files): if not ("format" in entry and entry["format"] == CWL_IANA): del sec_files[index] @@ -602,11 +589,11 @@ def find_deps( nestdirs: bool = True, ) -> CWLObjectType: """Find the dependencies of the CWL document.""" - deps = { + deps: CWLObjectType = { "class": "File", "location": uri, "format": CWL_IANA, - } # type: CWLObjectType + } def loadref(base: str, uri: str) -> Union[CommentedMap, CommentedSeq, str, None]: return document_loader.fetch(document_loader.fetcher.urljoin(base, uri)) @@ -638,7 +625,7 @@ def print_pack( return json_dumps(target, indent=4, default=str) -def supported_cwl_versions(enable_dev: bool) -> List[str]: +def supported_cwl_versions(enable_dev: bool) -> list[str]: # ALLUPDATES and UPDATES are dicts if enable_dev: versions = list(ALLUPDATES) @@ -692,8 +679,8 @@ def formatTime(self, record: logging.LogRecord, datefmt: Optional[str] = None) - def setup_provenance( args: argparse.Namespace, runtimeContext: RuntimeContext, - argsl: Optional[List[str]] = None, -) -> Tuple[ProvOut, "logging.StreamHandler[ProvOut]"]: + argsl: Optional[list[str]] = None, +) -> tuple[ProvOut, "logging.StreamHandler[ProvOut]"]: if not args.compute_checksum: _logger.error("--provenance incompatible with --no-compute-checksum") raise ArgumentException() @@ -940,7 +927,7 @@ def print_targets( _logger.info("%s steps targets:", prefix[:-1]) for t in tool.tool["steps"]: print(f" {prefix}{shortname(t['id'])}", file=stdout) - run: Union[str, Process, Dict[str, Any]] = t["run"] + run: Union[str, Process, dict[str, Any]] = t["run"] if isinstance(run, str): process = make_tool(run, loading_context) elif isinstance(run, dict): @@ -951,7 +938,7 @@ def print_targets( def main( - argsl: Optional[List[str]] = None, + argsl: Optional[list[str]] = None, args: Optional[argparse.Namespace] = None, job_order_object: Optional[CWLObjectType] = None, stdin: IO[Any] = sys.stdin, @@ -998,7 +985,7 @@ def main( if args is None: if argsl is None: argsl = sys.argv[1:] - addl = [] # type: List[str] + addl: list[str] = [] if "CWLTOOL_OPTIONS" in os.environ: c_opts = os.environ["CWLTOOL_OPTIONS"].split(" ") addl = [x for x in c_opts if x != ""] @@ -1250,7 +1237,7 @@ def main( if args.parallel: temp_executor = MultithreadedJobExecutor() runtimeContext.select_resources = temp_executor.select_resources - real_executor = temp_executor # type: JobExecutor + real_executor: JobExecutor = temp_executor else: real_executor = SingleJobExecutor() else: @@ -1260,7 +1247,7 @@ def main( runtimeContext.basedir = input_basedir if isinstance(tool, ProcessGenerator): - tfjob_order = {} # type: CWLObjectType + tfjob_order: CWLObjectType = {} if loadingContext.jobdefaults: tfjob_order.update(loadingContext.jobdefaults) if job_order_object: diff --git a/cwltool/mpi.py b/cwltool/mpi.py index 2cc1122c6..a7bdcbe03 100644 --- a/cwltool/mpi.py +++ b/cwltool/mpi.py @@ -3,7 +3,8 @@ import inspect import os import re -from typing import List, Mapping, MutableMapping, Optional, Type, TypeVar, Union +from collections.abc import Mapping, MutableMapping +from typing import Optional, TypeVar, Union from schema_salad.utils import yaml_no_ts @@ -18,9 +19,9 @@ def __init__( runner: str = "mpirun", nproc_flag: str = "-n", default_nproc: Union[int, str] = 1, - extra_flags: Optional[List[str]] = None, - env_pass: Optional[List[str]] = None, - env_pass_regex: Optional[List[str]] = None, + extra_flags: Optional[list[str]] = None, + env_pass: Optional[list[str]] = None, + env_pass_regex: Optional[list[str]] = None, env_set: Optional[Mapping[str, str]] = None, ) -> None: """ @@ -46,7 +47,7 @@ def __init__( self.env_set = env_set or {} @classmethod - def load(cls: Type[MpiConfigT], config_file_name: str) -> MpiConfigT: + def load(cls: type[MpiConfigT], config_file_name: str) -> MpiConfigT: """Create the MpiConfig object from the contents of a YAML file. The file must contain exactly one object, whose attributes must diff --git a/cwltool/mutation.py b/cwltool/mutation.py index 077b92cb7..9f58a86cf 100644 --- a/cwltool/mutation.py +++ b/cwltool/mutation.py @@ -1,5 +1,5 @@ from collections import namedtuple -from typing import Dict, cast +from typing import cast from .errors import WorkflowException from .utils import CWLObjectType @@ -20,7 +20,7 @@ class MutationManager: def __init__(self) -> None: """Initialize.""" - self.generations: Dict[str, MutationState] = {} + self.generations: dict[str, MutationState] = {} def register_reader(self, stepname: str, obj: CWLObjectType) -> None: loc = cast(str, obj["location"]) diff --git a/cwltool/pack.py b/cwltool/pack.py index c9fbc4e04..99684e003 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -2,17 +2,8 @@ import copy import urllib -from typing import ( - Any, - Callable, - Dict, - MutableMapping, - MutableSequence, - Optional, - Set, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence +from typing import Any, Callable, Optional, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.ref_resolver import Loader, SubLoader @@ -30,7 +21,7 @@ def find_run( d: Union[CWLObjectType, ResolveType], loadref: LoadRefType, - runs: Set[str], + runs: set[str], ) -> None: if isinstance(d, MutableSequence): for s in d: @@ -46,7 +37,7 @@ def find_run( def find_ids( d: Union[CWLObjectType, CWLOutputType, MutableSequence[CWLObjectType], None], - ids: Set[str], + ids: set[str], ) -> None: if isinstance(d, MutableSequence): for s in d: @@ -59,7 +50,7 @@ def find_ids( find_ids(cast(CWLOutputType, s2), ids) -def replace_refs(d: Any, rewrite: Dict[str, str], stem: str, newstem: str) -> None: +def replace_refs(d: Any, rewrite: dict[str, str], stem: str, newstem: str) -> None: if isinstance(d, MutableSequence): for s, v in enumerate(d): if isinstance(v, str): @@ -88,7 +79,7 @@ def replace_refs(d: Any, rewrite: Dict[str, str], stem: str, newstem: str) -> No def import_embed( d: Union[MutableSequence[CWLObjectType], CWLObjectType, CWLOutputType], - seen: Set[str], + seen: set[str], ) -> None: if isinstance(d, MutableSequence): for v in d: @@ -114,7 +105,7 @@ def import_embed( def pack( loadingContext: LoadingContext, uri: str, - rewrite_out: Optional[Dict[str, str]] = None, + rewrite_out: Optional[dict[str, str]] = None, loader: Optional[Loader] = None, ) -> CWLObjectType: # The workflow document we have in memory right now may have been @@ -153,7 +144,7 @@ def pack( document_loader.idx[po["id"]] = CommentedMap(po.items()) document_loader.idx[metadata["id"]] = CommentedMap(metadata.items()) - found_versions = {cast(str, loadingContext.metadata["cwlVersion"])} # type: Set[str] + found_versions: set[str] = {cast(str, loadingContext.metadata["cwlVersion"])} def loadref(base: Optional[str], lr_uri: str) -> ResolveType: lr_loadingContext = loadingContext.copy() @@ -167,15 +158,15 @@ def loadref(base: Optional[str], lr_uri: str) -> ResolveType: raise Exception("loader should not be None") return lr_loadingContext.loader.resolve_ref(lr_uri, base_url=base)[0] - input_ids: Set[str] = set() - output_ids: Set[str] = set() + input_ids: set[str] = set() + output_ids: set[str] = set() if isinstance(processobj, MutableSequence): mainobj = processobj[0] else: mainobj = processobj - find_ids(cast(Dict[str, Any], mainobj)["inputs"], input_ids) - find_ids(cast(Dict[str, Any], mainobj)["outputs"], output_ids) + find_ids(cast(dict[str, Any], mainobj)["inputs"], input_ids) + find_ids(cast(dict[str, Any], mainobj)["outputs"], output_ids) runs = {uri} find_run(processobj, loadref, runs) @@ -190,15 +181,15 @@ def loadref(base: Optional[str], lr_uri: str) -> ResolveType: for f in runs: find_ids(document_loader.resolve_ref(f)[0], input_ids) - input_names: Set[str] = set() - output_names: Set[str] = set() + input_names: set[str] = set() + output_names: set[str] = set() - rewrite_inputs: Dict[str, str] = {} - rewrite_outputs: Dict[str, str] = {} + rewrite_inputs: dict[str, str] = {} + rewrite_outputs: dict[str, str] = {} mainpath, _ = urllib.parse.urldefrag(uri) - def rewrite_id(r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str]) -> None: + def rewrite_id(r: str, mainuri: str, rewrite: dict[str, str], names: set[str]) -> None: if r == mainuri: rewrite[r] = "#main" elif r.startswith(mainuri) and r[len(mainuri)] in ("#", "/"): @@ -225,7 +216,7 @@ def rewrite_id(r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str]) - packed = CommentedMap((("$graph", CommentedSeq()), ("cwlVersion", update_to_version))) namespaces = metadata.get("$namespaces", None) - schemas: Set[str] = set() + schemas: set[str] = set() if "$schemas" in metadata: for each_schema in metadata["$schemas"]: schemas.add(each_schema) @@ -261,7 +252,7 @@ def rewrite_id(r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str]) - "Operation", ): continue - dc = cast(Dict[str, Any], copy.deepcopy(dcr)) + dc = cast(dict[str, Any], copy.deepcopy(dcr)) v = rewrite_inputs[r] dc["id"] = v for n in ("name", "cwlVersion", "$namespaces", "$schemas"): diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 0a06eb47b..86fd9ae82 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -4,17 +4,8 @@ import stat import urllib import uuid -from typing import ( - Dict, - ItemsView, - Iterable, - Iterator, - KeysView, - List, - Optional, - Tuple, - cast, -) +from collections.abc import ItemsView, Iterable, Iterator, KeysView +from typing import Optional, cast from mypy_extensions import mypyc_attr from schema_salad.exceptions import ValidationException @@ -92,20 +83,20 @@ class PathMapper: def __init__( self, - referenced_files: List[CWLObjectType], + referenced_files: list[CWLObjectType], basedir: str, stagedir: str, separateDirs: bool = True, ) -> None: """Initialize the PathMapper.""" - self._pathmap: Dict[str, MapperEnt] = {} + self._pathmap: dict[str, MapperEnt] = {} self.stagedir = stagedir self.separateDirs = separateDirs self.setup(dedup(referenced_files), basedir) def visitlisting( self, - listing: List[CWLObjectType], + listing: list[CWLObjectType], stagedir: str, basedir: str, copy: bool = False, @@ -147,7 +138,7 @@ def visit( if location.startswith("file://"): staged = False self.visitlisting( - cast(List[CWLObjectType], obj.get("listing", [])), + cast(list[CWLObjectType], obj.get("listing", [])), tgt, basedir, copy=copy, @@ -189,14 +180,14 @@ def visit( deref, tgt, "WritableFile" if copy else "File", staged ) self.visitlisting( - cast(List[CWLObjectType], obj.get("secondaryFiles", [])), + cast(list[CWLObjectType], obj.get("secondaryFiles", [])), stagedir, basedir, copy=copy, staged=staged, ) - def setup(self, referenced_files: List[CWLObjectType], basedir: str) -> None: + def setup(self, referenced_files: list[CWLObjectType], basedir: str) -> None: # Go through each file and set the target to its own directory along # with any secondary files. stagedir = self.stagedir @@ -246,7 +237,7 @@ def parents(path: str) -> Iterable[str]: def reversemap( self, target: str, - ) -> Optional[Tuple[str, str]]: + ) -> Optional[tuple[str, str]]: """Find the (source, resolved_path) for the given target, if any.""" for k, v in self._pathmap.items(): if v[1] == target: diff --git a/cwltool/process.py b/cwltool/process.py index bde035118..ff96985c5 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -13,25 +13,9 @@ import textwrap import urllib.parse import uuid +from collections.abc import Iterable, Iterator, MutableMapping, MutableSequence, Sized from os import scandir -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - Iterator, - List, - MutableMapping, - MutableSequence, - Optional, - Set, - Sized, - Tuple, - Type, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast from cwl_utils import expression from mypy_extensions import mypyc_attr @@ -161,14 +145,14 @@ def filter(self, record: logging.LogRecord) -> bool: "vocab_res_proc.yml", ) -SCHEMA_CACHE: Dict[ - str, Tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader] +SCHEMA_CACHE: dict[ + str, tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader] ] = {} SCHEMA_FILE: Optional[CWLObjectType] = None SCHEMA_DIR: Optional[CWLObjectType] = None SCHEMA_ANY: Optional[CWLObjectType] = None -custom_schemas: Dict[str, Tuple[str, str]] = {} +custom_schemas: dict[str, tuple[str, str]] = {} def use_standard_schema(version: str) -> None: @@ -186,11 +170,11 @@ def use_custom_schema(version: str, name: str, text: str) -> None: def get_schema( version: str, -) -> Tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader]: +) -> tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader]: if version in SCHEMA_CACHE: return SCHEMA_CACHE[version] - cache: Dict[str, Union[str, Graph, bool]] = {} + cache: dict[str, Union[str, Graph, bool]] = {} version = version.split("#")[-1] if ".dev" in version: version = ".".join(version.split(".")[:-1]) @@ -244,7 +228,7 @@ def stage_files( :raises WorkflowException: if there is a file staging conflict """ items = pathmapper.items() if not symlink else pathmapper.items_exclude_children() - targets: Dict[str, MapperEnt] = {} + targets: dict[str, MapperEnt] = {} for key, entry in list(items): if "File" not in entry.type: continue @@ -309,11 +293,11 @@ def stage_files( def relocateOutputs( outputObj: CWLObjectType, destination_path: str, - source_directories: Set[str], + source_directories: set[str], action: str, fs_access: StdFsAccess, compute_checksum: bool = True, - path_mapper: Type[PathMapper] = PathMapper, + path_mapper: type[PathMapper] = PathMapper, ) -> CWLObjectType: adjustDirObjs(outputObj, functools.partial(get_listing, fs_access, recursive=True)) @@ -414,7 +398,7 @@ def add_sizes(fsaccess: StdFsAccess, obj: CWLObjectType) -> None: def fill_in_defaults( - inputs: List[CWLObjectType], + inputs: list[CWLObjectType], job: CWLObjectType, fsaccess: StdFsAccess, ) -> None: @@ -578,7 +562,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext self.tool["id"] = "_:" + str(uuid.uuid4()) self.requirements.extend( cast( - List[CWLObjectType], + list[CWLObjectType], get_overrides(getdefault(loadingContext.overrides_list, []), self.tool["id"]).get( "requirements", [] ), @@ -617,7 +601,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext avroize_type(cast(MutableSequence[CWLOutputType], sdtypes)) av = make_valid_avro( sdtypes, - {cast(str, t["name"]): cast(Dict[str, Any], t) for t in sdtypes}, + {cast(str, t["name"]): cast(dict[str, Any], t) for t in sdtypes}, set(), vocab=INPUT_OBJ_VOCAB, ) @@ -655,9 +639,9 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext c["type"] = avroize_type(c["type"], c["name"]) if key == "inputs": - cast(List[CWLObjectType], self.inputs_record_schema["fields"]).append(c) + cast(list[CWLObjectType], self.inputs_record_schema["fields"]).append(c) elif key == "outputs": - cast(List[CWLObjectType], self.outputs_record_schema["fields"]).append(c) + cast(list[CWLObjectType], self.outputs_record_schema["fields"]).append(c) with SourceLine(toolpath_object, "inputs", ValidationException, debug): self.inputs_record_schema = cast( @@ -681,7 +665,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext if toolpath_object.get("class") is not None and not getdefault( loadingContext.disable_js_validation, False ): - validate_js_options: Optional[Dict[str, Union[List[str], str, int]]] = None + validate_js_options: Optional[dict[str, Union[list[str], str, int]]] = None if loadingContext.js_hint_options_file is not None: try: with open(loadingContext.js_hint_options_file) as options_file: @@ -784,7 +768,7 @@ def _init_job(self, joborder: CWLObjectType, runtime_context: RuntimeContext) -> v = job[k] dircount = [0] - def inc(d: List[int]) -> None: + def inc(d: list[int]) -> None: d[0] += 1 visit_class(v, ("Directory",), lambda x: inc(dircount)) # noqa: B023 @@ -820,7 +804,7 @@ def inc(d: List[int]) -> None: except (ValidationException, WorkflowException) as err: raise WorkflowException("Invalid job input record:\n" + str(err)) from err - files: List[CWLObjectType] = [] + files: list[CWLObjectType] = [] bindings = CommentedSeq() outdir = "" tmpdir = "" @@ -947,7 +931,7 @@ def inc(d: List[int]) -> None: def evalResources( self, builder: Builder, runtimeContext: RuntimeContext - ) -> Dict[str, Union[int, float]]: + ) -> dict[str, Union[int, float]]: resourceReq, _ = self.get_requirement("ResourceRequirement") if resourceReq is None: resourceReq = {} @@ -957,7 +941,7 @@ def evalResources( ram = 1024 else: ram = 256 - request: Dict[str, Union[int, float, str]] = { + request: dict[str, Union[int, float, str]] = { "coresMin": 1, "coresMax": 1, "ramMin": ram, @@ -1005,7 +989,7 @@ def evalResources( request[a + "Min"] = mn request[a + "Max"] = cast(Union[int, float], mx) - request_evaluated = cast(Dict[str, Union[int, float]], request) + request_evaluated = cast(dict[str, Union[int, float]], request) if runtimeContext.select_resources is not None: # Call select resources hook return runtimeContext.select_resources(request_evaluated, runtimeContext) @@ -1038,7 +1022,7 @@ def checkRequirements( f"Unsupported requirement {entry['class']}." ) - def validate_hints(self, avsc_names: Names, hints: List[CWLObjectType], strict: bool) -> None: + def validate_hints(self, avsc_names: Names, hints: list[CWLObjectType], strict: bool) -> None: """Process the hints field.""" if self.doc_loader is None: return @@ -1085,10 +1069,10 @@ def __str__(self) -> str: return f"{type(self).__name__}: {self.tool['id']}" -_names: Set[str] = set() +_names: set[str] = set() -def uniquename(stem: str, names: Optional[Set[str]] = None) -> str: +def uniquename(stem: str, names: Optional[set[str]] = None) -> str: global _names if names is None: names = _names @@ -1123,8 +1107,8 @@ def nestdir(base: str, deps: CWLObjectType) -> CWLObjectType: def mergedirs( listing: MutableSequence[CWLObjectType], ) -> MutableSequence[CWLObjectType]: - r: List[CWLObjectType] = [] - ents: Dict[str, CWLObjectType] = {} + r: list[CWLObjectType] = [] + ents: dict[str, CWLObjectType] = {} for e in listing: basename = cast(str, e["basename"]) if basename not in ents: @@ -1138,14 +1122,14 @@ def mergedirs( if e.get("listing"): # name already in entries # merge it into the existing listing - cast(List[CWLObjectType], ents[basename].setdefault("listing", [])).extend( - cast(List[CWLObjectType], e["listing"]) + cast(list[CWLObjectType], ents[basename].setdefault("listing", [])).extend( + cast(list[CWLObjectType], e["listing"]) ) for e in ents.values(): if e["class"] == "Directory" and "listing" in e: e["listing"] = cast( MutableSequence[CWLOutputType], - mergedirs(cast(List[CWLObjectType], e["listing"])), + mergedirs(cast(list[CWLObjectType], e["listing"])), ) r.extend(ents.values()) return r @@ -1157,8 +1141,8 @@ def mergedirs( def scandeps( base: str, doc: Union[CWLObjectType, MutableSequence[CWLObjectType]], - reffields: Set[str], - urlfields: Set[str], + reffields: set[str], + urlfields: set[str], loadref: Callable[[str, str], Union[CommentedMap, CommentedSeq, str, None]], urljoin: Callable[[str, str], str] = urllib.parse.urljoin, nestdirs: bool = True, diff --git a/cwltool/procgenerator.py b/cwltool/procgenerator.py index 34c1e650f..9839ce5d4 100644 --- a/cwltool/procgenerator.py +++ b/cwltool/procgenerator.py @@ -1,5 +1,5 @@ import copy -from typing import Dict, Optional, Tuple, cast +from typing import Optional, cast from ruamel.yaml.comments import CommentedMap from schema_salad.exceptions import ValidationException @@ -99,12 +99,12 @@ def result( job_order: CWLObjectType, jobout: CWLObjectType, runtimeContext: RuntimeContext, - ) -> Tuple[Process, CWLObjectType]: + ) -> tuple[Process, CWLObjectType]: try: loadingContext = self.loadingContext.copy() loadingContext.metadata = {} embedded_tool = load_tool( - cast(Dict[str, str], jobout["runProcess"])["location"], loadingContext + cast(dict[str, str], jobout["runProcess"])["location"], loadingContext ) except ValidationException as vexc: if runtimeContext.debug: diff --git a/cwltool/run_job.py b/cwltool/run_job.py index 307872f7a..5a81ce20c 100644 --- a/cwltool/run_job.py +++ b/cwltool/run_job.py @@ -4,10 +4,10 @@ import os import subprocess # nosec import sys -from typing import BinaryIO, Dict, List, Optional, TextIO, Union +from typing import BinaryIO, Optional, TextIO, Union -def handle_software_environment(cwl_env: Dict[str, str], script: str) -> Dict[str, str]: +def handle_software_environment(cwl_env: dict[str, str], script: str) -> dict[str, str]: """Update the provided environment dict by running the script.""" exec_env = cwl_env.copy() exec_env["_CWLTOOL"] = "1" @@ -29,7 +29,7 @@ def handle_software_environment(cwl_env: Dict[str, str], script: str) -> Dict[st return env -def main(argv: List[str]) -> int: +def main(argv: list[str]) -> int: """ Read in the configuration JSON and execute the commands. diff --git a/cwltool/secrets.py b/cwltool/secrets.py index f35f24c37..c73e0108c 100644 --- a/cwltool/secrets.py +++ b/cwltool/secrets.py @@ -1,7 +1,8 @@ """Minimal in memory storage of secrets.""" import uuid -from typing import Dict, List, MutableMapping, MutableSequence, Optional, cast +from collections.abc import MutableMapping, MutableSequence +from typing import Optional, cast from .utils import CWLObjectType, CWLOutputType @@ -11,7 +12,7 @@ class SecretStore: def __init__(self) -> None: """Initialize the secret store.""" - self.secrets: Dict[str, str] = {} + self.secrets: dict[str, str] = {} def add(self, value: Optional[CWLOutputType]) -> Optional[CWLOutputType]: """ @@ -28,7 +29,7 @@ def add(self, value: Optional[CWLOutputType]) -> Optional[CWLOutputType]: return placeholder return value - def store(self, secrets: List[str], job: CWLObjectType) -> None: + def store(self, secrets: list[str], job: CWLObjectType) -> None: """Sanitize the job object of any of the given secrets.""" for j in job: if j in secrets: diff --git a/cwltool/singularity.py b/cwltool/singularity.py index c43183ac7..0029d3950 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -6,8 +6,9 @@ import re import shutil import sys +from collections.abc import MutableMapping from subprocess import check_call, check_output # nosec -from typing import Callable, Dict, List, MutableMapping, Optional, Tuple, cast +from typing import Callable, Optional, cast from schema_salad.sourceline import SourceLine from spython.main import Client @@ -28,13 +29,13 @@ # This is a list containing major and minor versions as integer. # (The number of minor version digits can vary among different distributions, # therefore we need a list here.) -_SINGULARITY_VERSION: Optional[List[int]] = None +_SINGULARITY_VERSION: Optional[list[int]] = None # Cached flavor / distribution of singularity # Can be singularity, singularity-ce or apptainer _SINGULARITY_FLAVOR: str = "" -def get_version() -> Tuple[List[int], str]: +def get_version() -> tuple[list[int], str]: """ Parse the output of 'singularity --version' to determine the flavor and version. @@ -131,9 +132,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Builder for invoking the Singularty software container engine.""" @@ -141,7 +142,7 @@ def __init__( @staticmethod def get_image( - dockerRequirement: Dict[str, str], + dockerRequirement: dict[str, str], pull_image: bool, tmp_outdir_prefix: str, force_pull: bool = False, @@ -247,7 +248,7 @@ def get_image( dockerRequirement["dockerImageId"] = path found = True if (force_pull or not found) and pull_image: - cmd = [] # type: List[str] + cmd: list[str] = [] if "dockerPull" in dockerRequirement: if cache_folder: env = os.environ.copy() @@ -338,7 +339,7 @@ def get_from_requirements( if not bool(shutil.which("singularity")): raise WorkflowException("singularity executable is not available") - if not self.get_image(cast(Dict[str, str], r), pull_image, tmp_outdir_prefix, force_pull): + if not self.get_image(cast(dict[str, str], r), pull_image, tmp_outdir_prefix, force_pull): raise WorkflowException("Container image {} not found".format(r["dockerImageId"])) if "CWL_SINGULARITY_CACHE" in os.environ: @@ -350,7 +351,7 @@ def get_from_requirements( return os.path.abspath(img_path) @staticmethod - def append_volume(runtime: List[str], source: str, target: str, writable: bool = False) -> None: + def append_volume(runtime: list[str], source: str, target: str, writable: bool = False) -> None: """Add binding arguments to the runtime list.""" if is_version_3_9_or_newer(): DockerCommandLineJob.append_volume(runtime, source, target, writable, skip_mkdirs=True) @@ -364,7 +365,7 @@ def append_volume(runtime: List[str], source: str, target: str, writable: bool = runtime.append(vol) def add_file_or_directory_volume( - self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str] ) -> None: if not volume.resolved.startswith("_:"): if host_outdir_tgt is not None and not is_version_3_4_or_newer(): @@ -380,7 +381,7 @@ def add_file_or_directory_volume( def add_writable_file_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -417,7 +418,7 @@ def add_writable_file_volume( def add_writable_directory_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -452,7 +453,7 @@ def add_writable_directory_volume( shutil.copytree(volume.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt or new_dir) - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: return { "TMPDIR": self.CONTAINER_TMPDIR, "HOME": self.builder.outdir, @@ -460,7 +461,7 @@ def _required_env(self) -> Dict[str, str]: def create_runtime( self, env: MutableMapping[str, str], runtime_context: RuntimeContext - ) -> Tuple[List[str], Optional[str]]: + ) -> tuple[list[str], Optional[str]]: """Return the Singularity runtime list of commands and options.""" any_path_okay = self.builder.get_requirement("DockerRequirement")[1] or False runtime = [ diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index ec99bda05..de34f8f6d 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -10,17 +10,8 @@ import argparse import os import string -from typing import ( - TYPE_CHECKING, - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence +from typing import TYPE_CHECKING, Any, Optional, Union, cast from .utils import HasReqsHints @@ -79,7 +70,7 @@ def __init__(self, args: argparse.Namespace) -> None: if self.tool_dependency_dir and not os.path.exists(self.tool_dependency_dir): os.makedirs(self.tool_dependency_dir) - def build_job_script(self, builder: "Builder", command: List[str]) -> str: + def build_job_script(self, builder: "Builder", command: list[str]) -> str: ensure_galaxy_lib_available() resolution_config_dict = { "use": self.use_tool_dependencies, @@ -103,14 +94,14 @@ def build_job_script(self, builder: "Builder", command: List[str]) -> str: ) ) - template_kwds: Dict[str, str] = dict(handle_dependencies=handle_dependencies) + template_kwds: dict[str, str] = dict(handle_dependencies=handle_dependencies) job_script = COMMAND_WITH_DEPENDENCIES_TEMPLATE.substitute(template_kwds) return job_script def get_dependencies(builder: HasReqsHints) -> ToolRequirements: (software_requirement, _) = builder.get_requirement("SoftwareRequirement") - dependencies: List[Union["ToolRequirement", Dict[str, Any]]] = [] + dependencies: list[Union["ToolRequirement", dict[str, Any]]] = [] if software_requirement and software_requirement.get("packages"): packages = cast( MutableSequence[MutableMapping[str, Union[str, MutableSequence[str]]]], diff --git a/cwltool/stdfsaccess.py b/cwltool/stdfsaccess.py index 069289111..056b4b912 100644 --- a/cwltool/stdfsaccess.py +++ b/cwltool/stdfsaccess.py @@ -3,7 +3,7 @@ import glob import os import urllib -from typing import IO, Any, List +from typing import IO, Any from schema_salad.ref_resolver import file_uri, uri_file_path @@ -31,7 +31,7 @@ def __init__(self, basedir: str) -> None: def _abs(self, p: str) -> str: return abspath(p, self.basedir) - def glob(self, pattern: str) -> List[str]: + def glob(self, pattern: str) -> list[str]: return [file_uri(str(self._abs(line))) for line in glob.glob(self._abs(pattern))] def open(self, fn: str, mode: str) -> IO[Any]: @@ -49,7 +49,7 @@ def isfile(self, fn: str) -> bool: def isdir(self, fn: str) -> bool: return os.path.isdir(self._abs(fn)) - def listdir(self, fn: str) -> List[str]: + def listdir(self, fn: str) -> list[str]: return [abspath(urllib.parse.quote(entry), fn) for entry in os.listdir(self._abs(fn))] def join(self, path, *paths): # type: (str, *str) -> str diff --git a/cwltool/subgraph.py b/cwltool/subgraph.py index f6df7e69f..550dc7838 100644 --- a/cwltool/subgraph.py +++ b/cwltool/subgraph.py @@ -1,18 +1,7 @@ import urllib from collections import namedtuple -from typing import ( - Any, - Dict, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - Set, - Tuple, - Union, - cast, -) +from collections.abc import Mapping, MutableMapping, MutableSequence +from typing import Any, Optional, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -33,7 +22,7 @@ def subgraph_visit( current: str, nodes: MutableMapping[str, Node], - visited: Set[str], + visited: set[str], direction: str, ) -> None: if current in visited: @@ -48,7 +37,7 @@ def subgraph_visit( subgraph_visit(c, nodes, visited, direction) -def declare_node(nodes: Dict[str, Node], nodeid: str, tp: Optional[str]) -> Node: +def declare_node(nodes: dict[str, Node], nodeid: str, tp: Optional[str]) -> Node: if nodeid in nodes: n = nodes[nodeid] if n.type is None: @@ -59,8 +48,8 @@ def declare_node(nodes: Dict[str, Node], nodeid: str, tp: Optional[str]) -> Node def find_step( - steps: List[WorkflowStep], stepid: str, loading_context: LoadingContext -) -> Tuple[Optional[CWLObjectType], Optional[WorkflowStep]]: + steps: list[WorkflowStep], stepid: str, loading_context: LoadingContext +) -> tuple[Optional[CWLObjectType], Optional[WorkflowStep]]: """Find the step (raw dictionary and WorkflowStep) for a given step id.""" for st in steps: st_tool_id = st.tool["id"] @@ -114,7 +103,7 @@ def get_subgraph( if tool.tool["class"] != "Workflow": raise Exception("Can only extract subgraph from workflow") - nodes: Dict[str, Node] = {} + nodes: dict[str, Node] = {} for inp in tool.tool["inputs"]: declare_node(nodes, inp["id"], INPUT) @@ -149,7 +138,7 @@ def get_subgraph( nodes[out].up.append(st["id"]) # Find all the downstream nodes from the starting points - visited_down: Set[str] = set() + visited_down: set[str] = set() for r in roots: if nodes[r].type == OUTPUT: subgraph_visit(r, nodes, visited_down, UP) @@ -157,8 +146,8 @@ def get_subgraph( subgraph_visit(r, nodes, visited_down, DOWN) # Now make sure all the nodes are connected to upstream inputs - visited: Set[str] = set() - rewire: Dict[str, Tuple[str, CWLObjectType]] = {} + visited: set[str] = set() + rewire: dict[str, tuple[str, CWLObjectType]] = {} for v in visited_down: visited.add(v) if nodes[v].type in (STEP, OUTPUT): @@ -221,7 +210,7 @@ def get_step(tool: Workflow, step_id: str, loading_context: LoadingContext) -> C extracted["inputs"] = CommentedSeq() extracted["outputs"] = CommentedSeq() - for in_port in cast(List[CWLObjectType], step["in"]): + for in_port in cast(list[CWLObjectType], step["in"]): name = "#" + cast(str, in_port["id"]).split("#")[-1].split("/")[-1] inp: CWLObjectType = {"id": name, "type": "Any"} if "default" in in_port: @@ -231,7 +220,7 @@ def get_step(tool: Workflow, step_id: str, loading_context: LoadingContext) -> C if "linkMerge" in in_port: del in_port["linkMerge"] - for outport in cast(List[Union[str, Mapping[str, Any]]], step["out"]): + for outport in cast(list[Union[str, Mapping[str, Any]]], step["out"]): if isinstance(outport, Mapping): outport_id = cast(str, outport["id"]) else: @@ -256,7 +245,7 @@ def get_step(tool: Workflow, step_id: str, loading_context: LoadingContext) -> C def get_process( tool: Workflow, step_id: str, loading_context: LoadingContext -) -> Tuple[Any, WorkflowStep]: +) -> tuple[Any, WorkflowStep]: """Find the underlying Process for a given Workflow step id.""" if loading_context.loader is None: raise Exception("loading_context.loader cannot be None") diff --git a/cwltool/udocker.py b/cwltool/udocker.py index 6598d6a7c..ea3fc78ca 100644 --- a/cwltool/udocker.py +++ b/cwltool/udocker.py @@ -1,7 +1,5 @@ """Enables Docker software containers via the udocker runtime.""" -from typing import List - from .docker import DockerCommandLineJob @@ -10,7 +8,7 @@ class UDockerCommandLineJob(DockerCommandLineJob): @staticmethod def append_volume( - runtime: List[str], + runtime: list[str], source: str, target: str, writable: bool = False, diff --git a/cwltool/update.py b/cwltool/update.py index 4fd66b37a..67e1f4257 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -1,15 +1,7 @@ import copy +from collections.abc import MutableMapping, MutableSequence from functools import partial -from typing import ( - Callable, - Dict, - MutableMapping, - MutableSequence, - Optional, - Tuple, - Union, - cast, -) +from typing import Callable, Optional, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException @@ -20,7 +12,7 @@ from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field -def v1_2to1_3dev1(doc: CommentedMap, loader: Loader, baseuri: str) -> Tuple[CommentedMap, str]: +def v1_2to1_3dev1(doc: CommentedMap, loader: Loader, baseuri: str) -> tuple[CommentedMap, str]: """Public updater for v1.2 to v1.3.0-dev1.""" doc = copy.deepcopy(doc) @@ -78,7 +70,7 @@ def rewrite_loop_requirements(t: CWLObjectType) -> None: def v1_1to1_2( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.1 to v1.2.""" doc = copy.deepcopy(doc) @@ -94,7 +86,7 @@ def v1_1to1_2( def v1_0to1_1( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.0 to v1.1.""" doc = copy.deepcopy(doc) @@ -195,21 +187,21 @@ def fix_inputBinding(t: CWLObjectType) -> None: def v1_1_0dev1to1_1( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.1.0-dev1 to v1.1.""" return (doc, "v1.1") def v1_2_0dev1todev2( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev1 to v1.2.0-dev2.""" return (doc, "v1.2.0-dev2") def v1_2_0dev2todev3( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev2 to v1.2.0-dev3.""" doc = copy.deepcopy(doc) @@ -232,21 +224,21 @@ def update_pickvalue(t: CWLObjectType) -> None: def v1_2_0dev3todev4( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev3 to v1.2.0-dev4.""" return (doc, "v1.2.0-dev4") def v1_2_0dev4todev5( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev4 to v1.2.0-dev5.""" return (doc, "v1.2.0-dev5") def v1_2_0dev5to1_2( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev5 to v1.2.""" return (doc, "v1.2") @@ -264,13 +256,13 @@ def v1_2_0dev5to1_2( "v1.3.0-dev1", ] -UPDATES: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]] = { +UPDATES: dict[str, Optional[Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]]]] = { "v1.0": v1_0to1_1, "v1.1": v1_1to1_2, "v1.2": v1_2to1_3dev1, } -DEVUPDATES: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]] = { +DEVUPDATES: dict[str, Optional[Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]]]] = { "v1.1.0-dev1": v1_1_0dev1to1_1, "v1.2.0-dev1": v1_2_0dev1todev2, "v1.2.0-dev2": v1_2_0dev2todev3, @@ -291,7 +283,7 @@ def v1_2_0dev5to1_2( def identity( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Do-nothing, CWL document upgrade function.""" return (doc, cast(str, doc["cwlVersion"])) @@ -300,7 +292,7 @@ def checkversion( doc: Union[CommentedSeq, CommentedMap], metadata: CommentedMap, enable_dev: bool, -) -> Tuple[CommentedMap, str]: +) -> tuple[CommentedMap, str]: """Check the validity of the version of the give CWL document. Returns the document and the validated version string. @@ -365,7 +357,7 @@ def update( (cdoc, version) = checkversion(doc, metadata, enable_dev) originalversion = copy.copy(version) - nextupdate: Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]] = identity + nextupdate: Optional[Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]]] = identity while version != update_to and nextupdate: (cdoc, version) = nextupdate(cdoc, loader, baseuri) diff --git a/cwltool/utils.py b/cwltool/utils.py index c8620994a..e460842a9 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -19,9 +19,11 @@ import tempfile import urllib import uuid +from collections.abc import Generator, Iterable, MutableMapping, MutableSequence from datetime import datetime from email.utils import parsedate_to_datetime from functools import partial +from importlib.resources import as_file, files from itertools import zip_longest from pathlib import Path, PurePosixPath from tempfile import NamedTemporaryFile @@ -31,17 +33,9 @@ Any, Callable, Deque, - Dict, - Generator, - Iterable, - List, Literal, - MutableMapping, - MutableSequence, NamedTuple, Optional, - Set, - Tuple, TypedDict, Union, cast, @@ -54,11 +48,6 @@ from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader -if sys.version_info >= (3, 9): - from importlib.resources import as_file, files -else: - from importlib_resources import as_file, files - if TYPE_CHECKING: from .command_line_tool import CallbackJob, ExpressionJob from .job import CommandLineJob, JobBase @@ -92,13 +81,13 @@ OutputCallbackType = Callable[[Optional[CWLObjectType], str], None] ResolverType = Callable[["Loader", str], Optional[str]] DestinationsType = MutableMapping[str, Optional[CWLOutputType]] -ScatterDestinationsType = MutableMapping[str, List[Optional[CWLOutputType]]] +ScatterDestinationsType = MutableMapping[str, list[Optional[CWLOutputType]]] ScatterOutputCallbackType = Callable[[Optional[ScatterDestinationsType], str], None] SinkType = Union[CWLOutputType, CWLObjectType] DirectoryType = TypedDict( - "DirectoryType", {"class": str, "listing": List[CWLObjectType], "basename": str} + "DirectoryType", {"class": str, "listing": list[CWLObjectType], "basename": str} ) -JSONType = Union[Dict[str, "JSONType"], List["JSONType"], str, int, float, bool, None] +JSONType = Union[dict[str, "JSONType"], list["JSONType"], str, int, float, bool, None] class WorkflowStateItem(NamedTuple): @@ -109,7 +98,7 @@ class WorkflowStateItem(NamedTuple): success: str -ParametersType = List[CWLObjectType] +ParametersType = list[CWLObjectType] StepType = CWLObjectType # WorkflowStep LoadListingType = Union[Literal["no_listing"], Literal["shallow_listing"], Literal["deep_listing"]] @@ -143,7 +132,7 @@ def copytree_with_merge(src: str, dst: str) -> None: shutil.copy2(spath, dpath) -def cmp_like_py2(dict1: Dict[str, Any], dict2: Dict[str, Any]) -> int: +def cmp_like_py2(dict1: dict[str, Any], dict2: dict[str, Any]) -> int: """ Compare in the same manner as Python2. @@ -259,20 +248,20 @@ def adjustDirObjs(rec: Any, op: Union[Callable[..., Any], "partial[Any]"]) -> No visit_class(rec, ("Directory",), op) -def dedup(listing: List[CWLObjectType]) -> List[CWLObjectType]: +def dedup(listing: list[CWLObjectType]) -> list[CWLObjectType]: marksub = set() - def mark(d: Dict[str, str]) -> None: + def mark(d: dict[str, str]) -> None: marksub.add(d["location"]) for entry in listing: if entry["class"] == "Directory": - for e in cast(List[CWLObjectType], entry.get("listing", [])): + for e in cast(list[CWLObjectType], entry.get("listing", [])): adjustFileObjs(e, mark) adjustDirObjs(e, mark) dd = [] - markdup: Set[str] = set() + markdup: set[str] = set() for r in listing: if r["location"] not in marksub and r["location"] not in markdup: dd.append(r) @@ -284,14 +273,14 @@ def mark(d: Dict[str, str]) -> None: def get_listing(fs_access: "StdFsAccess", rec: CWLObjectType, recursive: bool = True) -> None: """Expand, recursively, any 'listing' fields in a Directory.""" if rec.get("class") != "Directory": - finddirs: List[CWLObjectType] = [] + finddirs: list[CWLObjectType] = [] visit_class(rec, ("Directory",), finddirs.append) for f in finddirs: get_listing(fs_access, f, recursive=recursive) return if "listing" in rec: return - listing: List[CWLOutputType] = [] + listing: list[CWLOutputType] = [] loc = cast(str, rec["location"]) for ld in fs_access.listdir(loc): parse = urllib.parse.urlparse(ld) @@ -310,7 +299,7 @@ def get_listing(fs_access: "StdFsAccess", rec: CWLObjectType, recursive: bool = rec["listing"] = listing -def trim_listing(obj: Dict[str, Any]) -> None: +def trim_listing(obj: dict[str, Any]) -> None: """ Remove 'listing' field from Directory objects that are file references. @@ -322,7 +311,7 @@ def trim_listing(obj: Dict[str, Any]) -> None: del obj["listing"] -def downloadHttpFile(httpurl: str) -> Tuple[str, Optional[datetime]]: +def downloadHttpFile(httpurl: str) -> tuple[str, Optional[datetime]]: """ Download a remote file, possibly using a locally cached copy. @@ -414,7 +403,7 @@ def normalizeFilesDirs( ] ] ) -> None: - def addLocation(d: Dict[str, Any]) -> None: + def addLocation(d: dict[str, Any]) -> None: if "location" not in d: if d["class"] == "File" and ("contents" not in d): raise ValidationException( @@ -484,10 +473,10 @@ class HasReqsHints: def __init__(self) -> None: """Initialize this reqs decorator.""" - self.requirements: List[CWLObjectType] = [] - self.hints: List[CWLObjectType] = [] + self.requirements: list[CWLObjectType] = [] + self.hints: list[CWLObjectType] = [] - def get_requirement(self, feature: str) -> Tuple[Optional[CWLObjectType], Optional[bool]]: + def get_requirement(self, feature: str) -> tuple[Optional[CWLObjectType], Optional[bool]]: """Retrieve the named feature from the requirements field, or the hints field.""" for item in reversed(self.requirements): if item["class"] == feature: diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index de4adaa14..b43b7ef0d 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -3,17 +3,8 @@ import json import logging from collections import namedtuple -from typing import ( - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Tuple, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence +from typing import Any, Optional, Union, cast from cwl_utils.errors import SubstitutionError from cwl_utils.expression import scanner as scan_expression @@ -63,7 +54,7 @@ def get_expressions( tool: Union[CommentedMap, str, CommentedSeq], schema: Optional[Union[Schema, ArraySchema]], source_line: Optional[SourceLine] = None, -) -> List[Tuple[str, Optional[SourceLine]]]: +) -> list[tuple[str, Optional[SourceLine]]]: debug = _logger.isEnabledFor(logging.DEBUG) if is_expression(tool, schema): return [(cast(str, tool), source_line)] @@ -124,8 +115,8 @@ def get_expressions( def jshint_js( js_text: str, - globals: Optional[List[str]] = None, - options: Optional[Dict[str, Union[List[str], str, int]]] = None, + globals: Optional[list[str]] = None, + options: Optional[dict[str, Union[list[str], str, int]]] = None, container_engine: str = "docker", eval_timeout: float = 60, ) -> JSHintJSReturn: @@ -177,7 +168,7 @@ def dump_jshint_error() -> None: except ValueError: dump_jshint_error() - jshint_errors: List[str] = [] + jshint_errors: list[str] = [] js_text_lines = js_text.split("\n") @@ -193,7 +184,7 @@ def dump_jshint_error() -> None: return JSHintJSReturn(jshint_errors, jshint_json.get("globals", [])) -def print_js_hint_messages(js_hint_messages: List[str], source_line: Optional[SourceLine]) -> None: +def print_js_hint_messages(js_hint_messages: list[str], source_line: Optional[SourceLine]) -> None: """Log the message from JSHint, using the line number.""" if source_line is not None: for js_hint_message in js_hint_messages: @@ -203,7 +194,7 @@ def print_js_hint_messages(js_hint_messages: List[str], source_line: Optional[So def validate_js_expressions( tool: CommentedMap, schema: Schema, - jshint_options: Optional[Dict[str, Union[List[str], str, int]]] = None, + jshint_options: Optional[dict[str, Union[list[str], str, int]]] = None, container_engine: str = "docker", eval_timeout: float = 60, ) -> None: diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 982ec7e70..3bf32251f 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -3,16 +3,8 @@ import functools import logging import random -from typing import ( - Callable, - Dict, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - cast, -) +from collections.abc import Mapping, MutableMapping, MutableSequence +from typing import Callable, Optional, cast from uuid import UUID from mypy_extensions import mypyc_attr @@ -98,7 +90,7 @@ def __init__( loadingContext.requirements = self.requirements loadingContext.hints = self.hints - self.steps: List[WorkflowStep] = [] + self.steps: list[WorkflowStep] = [] validation_errors = [] for index, step in enumerate(self.tool.get("steps", [])): try: @@ -119,9 +111,9 @@ def __init__( workflow_inputs = self.tool["inputs"] workflow_outputs = self.tool["outputs"] - step_inputs: List[CWLObjectType] = [] - step_outputs: List[CWLObjectType] = [] - param_to_step: Dict[str, CWLObjectType] = {} + step_inputs: list[CWLObjectType] = [] + step_outputs: list[CWLObjectType] = [] + param_to_step: dict[str, CWLObjectType] = {} for step in self.steps: step_inputs.extend(step.tool["inputs"]) step_outputs.extend(step.tool["outputs"]) @@ -220,7 +212,7 @@ def __init__( loadingContext.requirements.append(parent_req) loadingContext.requirements.extend( cast( - List[CWLObjectType], + list[CWLObjectType], get_overrides(getdefault(loadingContext.overrides_list, []), self.id).get( "requirements", [] ), diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index d144128e6..6cd0b2e7c 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -3,19 +3,8 @@ import functools import logging import threading -from typing import ( - TYPE_CHECKING, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Set, - Sized, - Tuple, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence, Sized +from typing import TYPE_CHECKING, Optional, Union, cast from cwl_utils import expression from schema_salad.sourceline import SourceLine @@ -89,11 +78,11 @@ def __init__( ) -> None: """Initialize.""" self.dest = dest - self._completed: Set[int] = set() + self._completed: set[int] = set() self.processStatus = "success" self.total = total self.output_callback = output_callback - self.steps: List[Optional[JobsGeneratorType]] = [] + self.steps: list[Optional[JobsGeneratorType]] = [] @property def completed(self) -> int: @@ -123,7 +112,7 @@ def receive_scatter_output(self, index: int, jobout: CWLObjectType, processStatu def setTotal( self, total: int, - steps: List[Optional[JobsGeneratorType]], + steps: list[Optional[JobsGeneratorType]], ) -> None: """ Set the total number of expected outputs along with the steps. @@ -137,7 +126,7 @@ def setTotal( def parallel_steps( - steps: List[Optional[JobsGeneratorType]], + steps: list[Optional[JobsGeneratorType]], rc: ReceiveScatterOutput, runtimeContext: RuntimeContext, ) -> JobsGeneratorType: @@ -187,7 +176,7 @@ def nested_crossproduct_scatter( rc = ReceiveScatterOutput(output_callback, output, jobl) - steps: List[Optional[JobsGeneratorType]] = [] + steps: list[Optional[JobsGeneratorType]] = [] for index in range(0, jobl): sjob: Optional[CWLObjectType] = copy.copy(joborder) assert sjob is not None # nosec @@ -254,11 +243,11 @@ def _flat_crossproduct_scatter( callback: ReceiveScatterOutput, startindex: int, runtimeContext: RuntimeContext, -) -> Tuple[List[Optional[JobsGeneratorType]], int]: +) -> tuple[list[Optional[JobsGeneratorType]], int]: """Inner loop.""" scatter_key = scatter_keys[0] jobl = len(cast(Sized, joborder[scatter_key])) - steps: List[Optional[JobsGeneratorType]] = [] + steps: list[Optional[JobsGeneratorType]] = [] put = startindex for index in range(0, jobl): sjob: Optional[CWLObjectType] = copy.copy(joborder) @@ -309,7 +298,7 @@ def dotproduct_scatter( rc = ReceiveScatterOutput(output_callback, output, jobl) - steps: List[Optional[JobsGeneratorType]] = [] + steps: list[Optional[JobsGeneratorType]] = [] for index in range(0, jobl): sjobo: Optional[CWLObjectType] = copy.copy(joborder) assert sjobo is not None # nosec @@ -357,7 +346,7 @@ def match_types( elif linkMerge: if iid not in inputobj: inputobj[iid] = [] - sourceTypes = cast(List[Optional[CWLOutputType]], inputobj[iid]) + sourceTypes = cast(list[Optional[CWLOutputType]], inputobj[iid]) if linkMerge == "merge_nested": sourceTypes.append(src.value) elif linkMerge == "merge_flattened": @@ -380,7 +369,7 @@ def match_types( def object_from_state( - state: Dict[str, Optional[WorkflowStateItem]], + state: dict[str, Optional[WorkflowStateItem]], params: ParametersType, frag_only: bool, supportsMultipleInput: bool, @@ -487,7 +476,7 @@ def __init__(self, workflow: "Workflow", runtimeContext: RuntimeContext) -> None self.prov_obj = workflow.provenance_object self.parent_wf = workflow.parent_wf self.steps = [WorkflowJobStep(s) for s in workflow.steps] - self.state: Dict[str, Optional[WorkflowStateItem]] = {} + self.state: dict[str, Optional[WorkflowStateItem]] = {} self.processStatus = "" self.did_callback = False self.made_progress: Optional[bool] = None @@ -554,7 +543,7 @@ def do_output_callback(self, final_output_callback: OutputCallbackType) -> None: def receive_output( self, step: WorkflowJobStep, - outputparms: List[CWLObjectType], + outputparms: list[CWLObjectType], final_output_callback: OutputCallbackType, jobout: CWLObjectType, processStatus: str, @@ -701,7 +690,7 @@ def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType] return psio if "scatter" in step.tool: - scatter = cast(List[str], aslist(step.tool["scatter"])) + scatter = cast(list[str], aslist(step.tool["scatter"])) method = step.tool.get("scatterMethod") if method is None and len(scatter) != 1: raise WorkflowException( @@ -961,7 +950,7 @@ def loop_callback( try: loop = cast(MutableSequence[CWLObjectType], self.step.tool.get("loop", [])) outputMethod = self.step.tool.get("outputMethod", "last_iteration") - state: Dict[str, Optional[WorkflowStateItem]] = {} + state: dict[str, Optional[WorkflowStateItem]] = {} for i in self.step.tool["outputs"]: if "id" in i: iid = cast(str, i["id"]) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index a7d0dacb8..ccd7737f4 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,6 +1,7 @@ mypy==1.11.2 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 +cwltest types-requests types-setuptools types-psutil diff --git a/pyproject.toml b/pyproject.toml index 05a2b82f7..4f3f91c31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,4 +19,4 @@ write_to = "cwltool/_version.py" [tool.black] line-length = 100 -target-version = [ "py38" ] +target-version = [ "py39" ] diff --git a/setup.py b/setup.py index fa39a378b..40c3fd8d4 100644 --- a/setup.py +++ b/setup.py @@ -129,7 +129,6 @@ "prov == 1.5.1", "mypy-extensions", "psutil >= 5.6.6", - "importlib_resources>=1.4;python_version<'3.9'", "coloredlogs", "pydot >= 1.4.1, <3", "argcomplete >= 1.12.0", @@ -143,7 +142,7 @@ "galaxy-util <24.2", ], }, - python_requires=">=3.8, <3.14", + python_requires=">=3.9, <3.14", use_scm_version=True, setup_requires=PYTEST_RUNNER + ["setuptools_scm>=8.0.4,<9"], test_suite="tests", @@ -169,7 +168,6 @@ "Operating System :: POSIX", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/tests/cwl-conformance/cwltool-conftest.py b/tests/cwl-conformance/cwltool-conftest.py index 3e2b83990..c87cf0ef7 100644 --- a/tests/cwl-conformance/cwltool-conftest.py +++ b/tests/cwl-conformance/cwltool-conftest.py @@ -6,20 +6,20 @@ import json from io import StringIO -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Optional from cwltest import utils def pytest_cwl_execute_test( config: utils.CWLTestConfig, processfile: str, jobfile: Optional[str] -) -> Tuple[int, Optional[Dict[str, Any]]]: +) -> tuple[int, Optional[dict[str, Any]]]: """Use the CWL reference runner (cwltool) to execute tests.""" from cwltool import main from cwltool.errors import WorkflowException stdout = StringIO() - argsl: List[str] = [f"--outdir={config.outdir}"] + argsl: list[str] = [f"--outdir={config.outdir}"] if config.runner_quiet: argsl.append("--quiet") elif config.verbose: diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index ae18a41ae..b903c04d6 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -6,7 +6,7 @@ from pathlib import Path from shutil import which from types import ModuleType -from typing import Optional, Tuple +from typing import Optional import pytest @@ -56,7 +56,7 @@ def test_biocontainers_resolution(tmp_path: Path) -> None: @pytest.fixture(scope="session") -def bioconda_setup(request: pytest.FixtureRequest) -> Tuple[Optional[int], str]: +def bioconda_setup(request: pytest.FixtureRequest) -> tuple[Optional[int], str]: """ Caches the conda environment created for seqtk_seq.cwl. @@ -108,7 +108,7 @@ def bioconda_setup(request: pytest.FixtureRequest) -> Tuple[Optional[int], str]: @pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed") -def test_bioconda(bioconda_setup: Tuple[Optional[int], str]) -> None: +def test_bioconda(bioconda_setup: tuple[Optional[int], str]) -> None: error_code, stderr = bioconda_setup assert error_code == 0, stderr diff --git a/tests/test_environment.py b/tests/test_environment.py index ba87041b3..a4bfd1ac3 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -2,8 +2,9 @@ import os from abc import ABC, abstractmethod +from collections.abc import Mapping from pathlib import Path -from typing import Any, Callable, Dict, List, Mapping, Union +from typing import Any, Callable, Union import pytest @@ -17,7 +18,7 @@ # TODO: maybe add regex? Env = Mapping[str, str] CheckerTypes = Union[None, str, Callable[[str, Env], bool]] -EnvChecks = Dict[str, CheckerTypes] +EnvChecks = dict[str, CheckerTypes] def assert_envvar_matches(check: CheckerTypes, k: str, env: Mapping[str, str]) -> None: @@ -66,7 +67,7 @@ def checks(tmp_prefix: str) -> EnvChecks: """Return a mapping from environment variable names to how to check for correctness.""" # Any flags to pass to cwltool to force use of the correct container - flags: List[str] + flags: list[str] # Does the env tool (maybe in our container) accept a `-0` flag? env_accepts_null: bool diff --git a/tests/test_examples.py b/tests/test_examples.py index 4d479e313..23d17dcb2 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -9,7 +9,7 @@ import urllib.parse from io import StringIO from pathlib import Path -from typing import Any, Dict, List, Union, cast +from typing import Any, Union, cast import cwl_utils.expression as expr import pydot @@ -69,7 +69,7 @@ def test_expression_match(expression: str, expected: bool) -> None: assert (match is not None) == expected -interpolate_input = { +interpolate_input: dict[str, Any] = { "foo": { "bar": {"baz": "zab1"}, "b ar": {"baz": 2}, @@ -77,7 +77,7 @@ def test_expression_match(expression: str, expected: bool) -> None: 'b"ar': {"baz": None}, }, "lst": ["A", "B"], -} # type: Dict[str, Any] +} interpolate_parameters = [ ("$(foo)", interpolate_input["foo"]), @@ -410,7 +410,7 @@ def loadref( raise Exception("test case can't load things") scanned_deps = cast( - List[Dict[str, Any]], + list[dict[str, Any]], cwltool.process.scandeps( cast(str, obj["id"]), obj, @@ -473,7 +473,7 @@ def loadref( assert scanned_deps == expected_deps scanned_deps2 = cast( - List[Dict[str, Any]], + list[dict[str, Any]], cwltool.process.scandeps( cast(str, obj["id"]), obj, @@ -515,7 +515,7 @@ def loadref( raise Exception("test case can't load things") scanned_deps = cast( - List[Dict[str, Any]], + list[dict[str, Any]], cwltool.process.scandeps( "", obj, @@ -576,7 +576,7 @@ def test_scandeps_defaults_with_secondaryfiles() -> None: def test_dedupe() -> None: - not_deduped = [ + not_deduped: list[CWLObjectType] = [ {"class": "File", "location": "file:///example/a"}, {"class": "File", "location": "file:///example/a"}, {"class": "File", "location": "file:///example/d"}, @@ -585,7 +585,7 @@ def test_dedupe() -> None: "location": "file:///example/c", "listing": [{"class": "File", "location": "file:///example/d"}], }, - ] # type: List[CWLObjectType] + ] expected = [ {"class": "File", "location": "file:///example/a"}, @@ -649,7 +649,7 @@ def test_dedupe() -> None: @pytest.mark.parametrize("name, source, sink, expected", source_to_sink) def test_compare_types( - name: str, source: Dict[str, Any], sink: Dict[str, Any], expected: bool + name: str, source: dict[str, Any], sink: dict[str, Any], expected: bool ) -> None: assert can_assign_src_to_sink(source, sink) == expected, name @@ -675,7 +675,7 @@ def test_compare_types( @pytest.mark.parametrize("name, source, sink, expected", source_to_sink_strict) def test_compare_types_strict( - name: str, source: Dict[str, Any], sink: Dict[str, Any], expected: bool + name: str, source: dict[str, Any], sink: dict[str, Any], expected: bool ) -> None: assert can_assign_src_to_sink(source, sink, strict=True) == expected, name @@ -1682,7 +1682,7 @@ def test_arguments_self() -> None: else: factory.runtime_context.use_container = False check = factory.make(get_data("tests/wf/paramref_arguments_self.cwl")) - outputs = cast(Dict[str, Any], check()) + outputs = cast(dict[str, Any], check()) assert "self_review" in outputs assert len(outputs) == 1 assert outputs["self_review"]["checksum"] == "sha1$724ba28f4a9a1b472057ff99511ed393a45552e1" diff --git a/tests/test_fetch.py b/tests/test_fetch.py index e55491d90..962b7d7e5 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import Any, List, Optional +from typing import Any, Optional from urllib.parse import urljoin, urlsplit import pytest @@ -25,7 +25,7 @@ def __init__( ) -> None: """Create a Fetcher that provides a fixed result for testing purposes.""" - def fetch_text(self, url: str, content_types: Optional[List[str]] = None) -> str: + def fetch_text(self, url: str, content_types: Optional[list[str]] = None) -> str: if url == "baz:bar/foo.cwl": return """ cwlVersion: v1.0 diff --git a/tests/test_http_input.py b/tests/test_http_input.py index 6b4d9b479..e80260ff9 100644 --- a/tests/test_http_input.py +++ b/tests/test_http_input.py @@ -1,10 +1,7 @@ import os -import sys from datetime import datetime from pathlib import Path -from typing import List -import pytest from pytest_httpserver import HTTPServer from cwltool.pathmapper import PathMapper @@ -15,7 +12,7 @@ def test_http_path_mapping(tmp_path: Path) -> None: input_file_path = ( "https://raw.githubusercontent.com/common-workflow-language/cwltool/main/tests/2.fasta" ) - base_file: List[CWLObjectType] = [ + base_file: list[CWLObjectType] = [ { "class": "File", "location": "https://raw.githubusercontent.com/common-workflow-language/" @@ -34,7 +31,6 @@ def test_http_path_mapping(tmp_path: Path) -> None: assert ">Sequence 561 BP; 135 A; 106 C; 98 G; 222 T; 0 other;" in contents -@pytest.mark.skipif(sys.version_info < (3, 7), reason="timesout on CI") def test_modification_date(tmp_path: Path) -> None: """Local copies of remote files should preserve last modification date.""" # Initialize the server @@ -58,7 +54,7 @@ def test_modification_date(tmp_path: Path) -> None: ) location = httpserver.url_for(f"/{remote_file_name}") - base_file: List[CWLObjectType] = [ + base_file: list[CWLObjectType] = [ { "class": "File", "location": location, diff --git a/tests/test_js_sandbox.py b/tests/test_js_sandbox.py index f4839e8a0..2c5df6339 100644 --- a/tests/test_js_sandbox.py +++ b/tests/test_js_sandbox.py @@ -5,7 +5,7 @@ import shutil import threading from pathlib import Path -from typing import Any, List +from typing import Any import pytest from cwl_utils import sandboxjs @@ -48,8 +48,8 @@ def test_value_from_two_concatenated_expressions() -> None: def hide_nodejs(temp_dir: Path) -> str: """Generate a new PATH that hides node{js,}.""" - paths: List[str] = os.environ.get("PATH", "").split(":") - names: List[str] = [] + paths: list[str] = os.environ.get("PATH", "").split(":") + names: list[str] = [] for name in ("nodejs", "node"): path = shutil.which(name) if path: diff --git a/tests/test_loop.py b/tests/test_loop.py index bf908196d..e8a043611 100644 --- a/tests/test_loop.py +++ b/tests/test_loop.py @@ -1,8 +1,8 @@ """Test the 1.3 loop feature.""" import json +from collections.abc import MutableMapping, MutableSequence from io import StringIO -from typing import MutableMapping, MutableSequence from cwltool.main import main diff --git a/tests/test_loop_ext.py b/tests/test_loop_ext.py index 499dd17b4..1769d64ad 100644 --- a/tests/test_loop_ext.py +++ b/tests/test_loop_ext.py @@ -1,8 +1,8 @@ """Test the prototype cwltool:Loop extension.""" import json +from collections.abc import MutableMapping, MutableSequence from io import StringIO -from typing import MutableMapping, MutableSequence from cwltool.main import main diff --git a/tests/test_mpi.py b/tests/test_mpi.py index 643907a39..92f0e353c 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -3,9 +3,10 @@ import json import os.path import sys +from collections.abc import Generator, MutableMapping from io import StringIO from pathlib import Path -from typing import Any, Generator, List, MutableMapping, Optional, Tuple +from typing import Any, Optional import pytest from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -75,12 +76,12 @@ def __init__(self): else: self.indata = sys.stdin.read().encode(sys.stdin.encoding) - def run_once(self, args: List[str]): + def run_once(self, args: list[str]): subprocess.run( args, input=self.indata, stdout=sys.stdout, stderr=sys.stderr ).check_returncode() - def run_many(self, n: int, args: List[str]): + def run_many(self, n: int, args: list[str]): for i in range(n): self.run_once(args) @@ -122,7 +123,7 @@ def make_processes_input(np: int, tmp_path: Path) -> Path: return input_file -def cwltool_args(fake_mpi_conf: str) -> List[str]: +def cwltool_args(fake_mpi_conf: str) -> list[str]: return ["--enable-ext", "--enable-dev", "--mpi-config-file", fake_mpi_conf] @@ -296,10 +297,10 @@ def schema_ext11() -> Generator[Names, None, None]: def mk_tool( schema: Names, - opts: List[str], - reqs: Optional[List[CommentedMap]] = None, - hints: Optional[List[CommentedMap]] = None, -) -> Tuple[LoadingContext, RuntimeContext, CommentedMap]: + opts: list[str], + reqs: Optional[list[CommentedMap]] = None, + hints: Optional[list[CommentedMap]] = None, +) -> tuple[LoadingContext, RuntimeContext, CommentedMap]: tool = basetool.copy() if reqs is not None: diff --git a/tests/test_override.py b/tests/test_override.py index 980c853bb..93c836c84 100644 --- a/tests/test_override.py +++ b/tests/test_override.py @@ -1,6 +1,5 @@ import json from io import StringIO -from typing import Dict, List import pytest @@ -76,7 +75,7 @@ @needs_docker @pytest.mark.parametrize("parameters,result", override_parameters) -def test_overrides(parameters: List[str], result: Dict[str, str]) -> None: +def test_overrides(parameters: list[str], result: dict[str, str]) -> None: sio = StringIO() assert main(parameters, stdout=sio) == 0 @@ -119,7 +118,7 @@ def test_overrides(parameters: List[str], result: Dict[str, str]) -> None: @needs_docker @pytest.mark.parametrize("parameters,expected_error", failing_override_parameters) -def test_overrides_fails(parameters: List[str], expected_error: str) -> None: +def test_overrides_fails(parameters: list[str], expected_error: str) -> None: sio = StringIO() assert main(parameters, stderr=sio) == 1 diff --git a/tests/test_pack.py b/tests/test_pack.py index 1d38e35e8..a65996f8f 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -5,7 +5,6 @@ from functools import partial from io import StringIO from pathlib import Path -from typing import Dict import pytest from schema_salad.utils import yaml_no_ts @@ -95,7 +94,7 @@ def test_pack_fragment() -> None: def test_pack_rewrites() -> None: - rewrites: Dict[str, str] = {} + rewrites: dict[str, str] = {} loadingContext, workflowobj, uri = fetch_document(get_data("tests/wf/default-wf5.cwl")) loadingContext.do_update = False diff --git a/tests/test_path_checks.py b/tests/test_path_checks.py index 01ab7fe17..096de9942 100644 --- a/tests/test_path_checks.py +++ b/tests/test_path_checks.py @@ -1,7 +1,7 @@ import urllib.parse from io import BytesIO from pathlib import Path -from typing import IO, Any, List, cast +from typing import IO, Any, cast import pytest from ruamel.yaml.comments import CommentedMap @@ -112,7 +112,7 @@ def test_unicode_in_output_files(tmp_path: Path, filename: str) -> None: class StubFsAccess(StdFsAccess): """Stub fs access object that doesn't rely on the filesystem.""" - def glob(self, pattern: str) -> List[str]: + def glob(self, pattern: str) -> list[str]: """glob.""" return [pattern] diff --git a/tests/test_pathmapper.py b/tests/test_pathmapper.py index b7cf2f6a1..4ffac24bd 100644 --- a/tests/test_pathmapper.py +++ b/tests/test_pathmapper.py @@ -1,5 +1,3 @@ -from typing import List, Tuple - import pytest from cwltool.pathmapper import PathMapper @@ -10,7 +8,7 @@ def test_subclass() -> None: class SubPathMapper(PathMapper): def __init__( self, - referenced_files: List[CWLObjectType], + referenced_files: list[CWLObjectType], basedir: str, stagedir: str, new: str, @@ -81,7 +79,7 @@ def test_normalizeFilesDirs(name: str, file_dir: CWLObjectType, expected: CWLObj @pytest.mark.parametrize("filename,expected", basename_generation_parameters) -def test_basename_field_generation(filename: str, expected: Tuple[str, str]) -> None: +def test_basename_field_generation(filename: str, expected: tuple[str, str]) -> None: nameroot, nameext = expected expected2 = { "class": "File", diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 83eb61c22..e8d8416be 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -3,8 +3,9 @@ import pickle import sys import urllib +from collections.abc import Generator from pathlib import Path -from typing import IO, Any, Generator, cast +from typing import IO, Any, cast import arcp import bagit diff --git a/tests/test_relocate.py b/tests/test_relocate.py index 81877c776..692e995fa 100644 --- a/tests/test_relocate.py +++ b/tests/test_relocate.py @@ -1,18 +1,13 @@ import json import os import shutil -import sys +from io import StringIO from pathlib import Path from cwltool.main import main from .util import get_data, needs_docker -if sys.version_info[0] < 3: - from StringIO import StringIO -else: - from io import StringIO - @needs_docker def test_for_910(tmp_path: Path) -> None: diff --git a/tests/test_secrets.py b/tests/test_secrets.py index bd90bee78..a8c0b67af 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -1,7 +1,7 @@ import shutil import tempfile from io import StringIO -from typing import Callable, Dict, List, Tuple, Union +from typing import Callable, Union import pytest @@ -13,7 +13,7 @@ @pytest.fixture -def secrets() -> Tuple[SecretStore, CWLObjectType]: +def secrets() -> tuple[SecretStore, CWLObjectType]: """Fixture to return a secret store.""" sec_store = SecretStore() job: CWLObjectType = {"foo": "bar", "baz": "quux"} @@ -22,7 +22,7 @@ def secrets() -> Tuple[SecretStore, CWLObjectType]: return sec_store, job -def test_obscuring(secrets: Tuple[SecretStore, CWLObjectType]) -> None: +def test_obscuring(secrets: tuple[SecretStore, CWLObjectType]) -> None: """Basic test of secret store.""" storage, obscured = secrets assert obscured["foo"] != "bar" @@ -41,8 +41,8 @@ def test_obscuring(secrets: Tuple[SecretStore, CWLObjectType]) -> None: @pytest.mark.parametrize("factory,expected", obscured_factories_expected) def test_secrets( factory: Callable[[str], CWLObjectType], - expected: Union[str, List[str], Dict[str, str]], - secrets: Tuple[SecretStore, CWLObjectType], + expected: Union[str, list[str], dict[str, str]], + secrets: tuple[SecretStore, CWLObjectType], ) -> None: storage, obscured = secrets obs = obscured["foo"] diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index 73fe240d0..18a588cf8 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -6,7 +6,7 @@ import subprocess import sys from pathlib import Path -from typing import List, cast +from typing import cast import pytest from ruamel.yaml.comments import CommentedMap @@ -318,7 +318,7 @@ def test_docker_tmpdir_prefix(tmp_path: Path) -> None: "docker", ) job = DockerCommandLineJob(builder, {}, CommandLineTool.make_path_mapper, [], [], "") - runtime: List[str] = [] + runtime: list[str] = [] volume_writable_file = MapperEnt( resolved=get_data("tests/2.fastq"), target="foo", type=None, staged=None diff --git a/tests/test_toolargparse.py b/tests/test_toolargparse.py index 11ce5e3db..2e50fe722 100644 --- a/tests/test_toolargparse.py +++ b/tests/test_toolargparse.py @@ -1,7 +1,7 @@ import argparse from io import StringIO from pathlib import Path -from typing import Callable, List +from typing import Callable import pytest @@ -296,7 +296,7 @@ def test_argparser_without_doc() -> None: ), ], ) -def test_argparse_append_with_default(job_order: List[str], expected_values: List[str]) -> None: +def test_argparse_append_with_default(job_order: list[str], expected_values: list[str]) -> None: """ Confirm that the appended arguments must not include the default. diff --git a/tests/util.py b/tests/util.py index 0547cfa9a..44d2f108c 100644 --- a/tests/util.py +++ b/tests/util.py @@ -8,9 +8,10 @@ import shutil import subprocess import sys +from collections.abc import Generator, Mapping from contextlib import ExitStack from pathlib import Path -from typing import Dict, Generator, List, Mapping, Optional, Tuple, Union +from typing import Optional, Union import pytest @@ -83,11 +84,11 @@ def env_accepts_null() -> bool: def get_main_output( - args: List[str], + args: list[str], replacement_env: Optional[Mapping[str, str]] = None, extra_env: Optional[Mapping[str, str]] = None, monkeypatch: Optional[pytest.MonkeyPatch] = None, -) -> Tuple[Optional[int], str, str]: +) -> tuple[Optional[int], str, str]: """Run cwltool main. args: the command line args to call it with @@ -127,13 +128,13 @@ def get_main_output( def get_tool_env( tmp_path: Path, - flag_args: List[str], + flag_args: list[str], inputs_file: Optional[str] = None, replacement_env: Optional[Mapping[str, str]] = None, extra_env: Optional[Mapping[str, str]] = None, monkeypatch: Optional[pytest.MonkeyPatch] = None, runtime_env_accepts_null: Optional[bool] = None, -) -> Dict[str, str]: +) -> dict[str, str]: """Get the env vars for a tool's invocation.""" # GNU env accepts the -0 option to end each variable's # printing with "\0". No such luck on BSD-ish. diff --git a/tox.ini b/tox.ini index c75d0cc47..2a5a431b9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] envlist = - py3{8,9,10,11,12,13}-lint - py3{8,9,10,11,12,13}-unit - py3{8,9,10,11,12,13}-bandit - py3{8,9,10,11,12,13}-mypy + py3{9,10,11,12,13}-lint + py3{9,10,11,12,13}-unit + py3{9,10,11,12,13}-bandit + py3{9,10,11,12,13}-mypy py312-lintreadme py312-shellcheck py312-pydocstyle @@ -16,7 +16,6 @@ testpaths = tests [gh-actions] python = - 3.8: py38 3.9: py39 3.10: py310 3.11: py311 @@ -25,13 +24,13 @@ python = [testenv] skipsdist = - py3{8,9,10,11,12,13}-!{unit,mypy,lintreadme} = True + py3{9,10,11,12,13}-!{unit,mypy,lintreadme} = True description = - py3{8,9,10,11,12,13}-unit: Run the unit tests - py3{8,9,10,11,12,13}-lint: Lint the Python code - py3{8,9,10,11,12,13}-bandit: Search for common security issues - py3{8,9,10,11,12,13}-mypy: Check for type safety + py3{9,10,11,12,13}-unit: Run the unit tests + py3{9,10,11,12,13}-lint: Lint the Python code + py3{9,10,11,12,13}-bandit: Search for common security issues + py3{9,10,11,12,13}-mypy: Check for type safety py312-pydocstyle: docstring style checker py312-shellcheck: syntax check for shell scripts py312-lintreadme: Lint the README.rst→.md conversion @@ -44,14 +43,14 @@ passenv = SINGULARITY_FAKEROOT extras = - py3{8,9,10,11,12,13}-unit: deps + py3{9,10,11,12,13}-unit: deps deps = - py3{8,9,10,11,12,13}-{unit,lint,bandit,mypy}: -rrequirements.txt - py3{8,9,10,11,12,13}-{unit,mypy}: -rtest-requirements.txt - py3{8,9,10,11,12,13}-lint: -rlint-requirements.txt - py3{8,9,10,11,12,13}-bandit: bandit - py3{8,9,10,11,12,13}-mypy: -rmypy-requirements.txt + py3{9,10,11,12,13}-{unit,lint,bandit,mypy}: -rrequirements.txt + py3{9,10,11,12,13}-{unit,mypy}: -rtest-requirements.txt + py3{9,10,11,12,13}-lint: -rlint-requirements.txt + py3{9,10,11,12,13}-bandit: bandit + py3{9,10,11,12,13}-mypy: -rmypy-requirements.txt py312-pydocstyle: pydocstyle py312-pydocstyle: diff-cover py312-lintreadme: twine @@ -63,20 +62,20 @@ setenv = HOME = {envtmpdir} commands_pre = - py3{8,9,10,11,12,13}-unit: python -m pip install -U pip setuptools wheel + py3{9,10,11,12,13}-unit: python -m pip install -U pip setuptools wheel py312-lintreadme: python -m build --outdir {distdir} commands = - py3{8,9,10,11,12,13}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs} - py3{8,9,10,11,12,13}-bandit: bandit -r cwltool - py3{8,9,10,11,12,13}-lint: make flake8 format-check codespell-check - py3{8,9,10,11,12,13}-mypy: make mypy PYTEST_EXTRA={posargs} - py3{8,9,10,11,12}-mypy: make mypyc PYTEST_EXTRA={posargs} + py3{9,10,11,12,13}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs} + py3{9,10,11,12,13}-bandit: bandit -r cwltool + py3{9,10,11,12,13}-lint: make flake8 format-check codespell-check + py3{9,10,11,12,13}-mypy: make mypy PYTEST_EXTRA={posargs} + py3{9,10,11,12}-mypy: make mypyc PYTEST_EXTRA={posargs} py312-shellcheck: make shellcheck py312-pydocstyle: make diff_pydocstyle_report py312-lintreadme: twine check {distdir}/* skip_install = - py3{8,9,10,11,12,13}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true + py3{9,10,11,12,13}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true allowlist_externals = make From a9567667af28688042a5369bf35213d7932c3d9c Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 4 Oct 2024 14:56:29 +0200 Subject: [PATCH 13/78] add more docs --- cwltool/builder.py | 2 ++ cwltool/checker.py | 2 ++ cwltool/command_line_tool.py | 2 ++ cwltool/cwlprov/provenance_profile.py | 1 + cwltool/flatten.py | 13 +++++++++---- cwltool/load_tool.py | 1 + cwltool/main.py | 1 + cwltool/pack.py | 1 + cwltool/pathmapper.py | 7 +++++-- cwltool/process.py | 1 + cwltool/software_requirements.py | 1 + cwltool/stdfsaccess.py | 2 ++ cwltool/subgraph.py | 6 ++++++ 13 files changed, 34 insertions(+), 6 deletions(-) diff --git a/cwltool/builder.py b/cwltool/builder.py index 066a77f86..e1de5b857 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -161,6 +161,7 @@ def __init__( self.container_engine = container_engine def build_job_script(self, commands: list[str]) -> Optional[str]: + """Use the job_script_provider to turn the commands into a job script.""" if self.job_script_provider is not None: return self.job_script_provider.build_job_script(self, commands) return None @@ -607,6 +608,7 @@ def tostr(self, value: Union[MutableMapping[str, str], Any]) -> str: return str(value) def generate_arg(self, binding: CWLObjectType) -> list[str]: + """Convert an input binding to a list of command line arguments.""" value = binding.get("datum") debug = _logger.isEnabledFor(logging.DEBUG) if "valueFrom" in binding: diff --git a/cwltool/checker.py b/cwltool/checker.py index 7742adf5c..17cba77ba 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -156,6 +156,7 @@ def _rec_fields(rec: MutableMapping[str, Any]) -> MutableMapping[str, Any]: def missing_subset(fullset: list[Any], subset: list[Any]) -> list[Any]: + """Calculate the items missing from the fullset given the subset.""" missing = [] for i in subset: if i not in fullset: @@ -498,6 +499,7 @@ def get_step_id(field_id: str) -> str: def is_conditional_step(param_to_step: dict[str, CWLObjectType], parm_id: str) -> bool: + """Return True if the step given by the parm_id is a conditional step.""" if (source_step := param_to_step.get(parm_id)) is not None: if source_step.get("when") is not None: return True diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index de1878593..e201fb12b 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -228,6 +228,7 @@ def job( def remove_path(f: CWLObjectType) -> None: + """Remove any 'path' property, if present.""" if "path" in f: del f["path"] @@ -404,6 +405,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext ) def make_job_runner(self, runtimeContext: RuntimeContext) -> type[JobBase]: + """Return the correct CommandLineJob class given the container settings.""" dockerReq, dockerRequired = self.get_requirement("DockerRequirement") mpiReq, mpiRequired = self.get_requirement(MPIRequirementName) diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index 59d835fff..d4dfd6cb4 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -281,6 +281,7 @@ def record_process_end( self.document.wasEndedBy(process_run_id, None, self.workflow_run_uri, when) def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, str]: + """Construct a FileEntity for the given CWL File object.""" if value["class"] != "File": raise ValueError("Must have class:File: %s" % value) # Need to determine file hash aka RO filename diff --git a/cwltool/flatten.py b/cwltool/flatten.py index 5c9738cbf..3c057ebbe 100644 --- a/cwltool/flatten.py +++ b/cwltool/flatten.py @@ -1,12 +1,17 @@ -from typing import Any, Callable, cast +""" +Our version of the popular flatten() method. + +http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html +""" -# http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html +from typing import Any, Callable, cast -def flatten(thing, ltypes=(list, tuple)): - # type: (Any, Any) -> List[Any] +def flatten(thing: Any) -> list[Any]: + """Flatten a list without recursion problems.""" if thing is None: return [] + ltypes = (list, tuple) if not isinstance(thing, ltypes): return [thing] diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 7a58a8330..4d7f3a930 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -626,6 +626,7 @@ def resolve_overrides( def load_overrides(ov: str, base_url: str) -> list[CWLObjectType]: + """Load and resolve any overrides.""" ovloader = Loader(overrides_ctx) return resolve_overrides(ovloader.fetch(ov), ov, base_url) diff --git a/cwltool/main.py b/cwltool/main.py index 9477cb1a2..99928d0bd 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -626,6 +626,7 @@ def print_pack( def supported_cwl_versions(enable_dev: bool) -> list[str]: + """Return a list of currently supported CWL versions.""" # ALLUPDATES and UPDATES are dicts if enable_dev: versions = list(ALLUPDATES) diff --git a/cwltool/pack.py b/cwltool/pack.py index 99684e003..d3705d5e4 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -51,6 +51,7 @@ def find_ids( def replace_refs(d: Any, rewrite: dict[str, str], stem: str, newstem: str) -> None: + """Replace references with the actual value.""" if isinstance(d, MutableSequence): for s, v in enumerate(d): if isinstance(v, str): diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 86fd9ae82..10cb7a733 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -188,8 +188,11 @@ def visit( ) def setup(self, referenced_files: list[CWLObjectType], basedir: str) -> None: - # Go through each file and set the target to its own directory along - # with any secondary files. + """ + For each file, set the target to its own directory. + + Also processes secondary files into that same directory. + """ stagedir = self.stagedir for fob in referenced_files: if self.separateDirs: diff --git a/cwltool/process.py b/cwltool/process.py index ff96985c5..fe5f84764 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -1073,6 +1073,7 @@ def __str__(self) -> str: def uniquename(stem: str, names: Optional[set[str]] = None) -> str: + """Construct a thread-unique name using the given stem as a prefix.""" global _names if names is None: names = _names diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index de34f8f6d..6ad84da4b 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -71,6 +71,7 @@ def __init__(self, args: argparse.Namespace) -> None: os.makedirs(self.tool_dependency_dir) def build_job_script(self, builder: "Builder", command: list[str]) -> str: + """Use the galaxy-tool-util library to construct a build script.""" ensure_galaxy_lib_available() resolution_config_dict = { "use": self.use_tool_dependencies, diff --git a/cwltool/stdfsaccess.py b/cwltool/stdfsaccess.py index 056b4b912..c58257f63 100644 --- a/cwltool/stdfsaccess.py +++ b/cwltool/stdfsaccess.py @@ -32,6 +32,7 @@ def _abs(self, p: str) -> str: return abspath(p, self.basedir) def glob(self, pattern: str) -> list[str]: + """Return a possibly empty list of absolute URI paths that match pathname.""" return [file_uri(str(self._abs(line))) for line in glob.glob(self._abs(pattern))] def open(self, fn: str, mode: str) -> IO[Any]: @@ -50,6 +51,7 @@ def isdir(self, fn: str) -> bool: return os.path.isdir(self._abs(fn)) def listdir(self, fn: str) -> list[str]: + """Return a list containing the absolute path URLs of the entries in the directory given by path.""" return [abspath(urllib.parse.quote(entry), fn) for entry in os.listdir(self._abs(fn))] def join(self, path, *paths): # type: (str, *str) -> str diff --git a/cwltool/subgraph.py b/cwltool/subgraph.py index 550dc7838..204e987a8 100644 --- a/cwltool/subgraph.py +++ b/cwltool/subgraph.py @@ -38,6 +38,12 @@ def subgraph_visit( def declare_node(nodes: dict[str, Node], nodeid: str, tp: Optional[str]) -> Node: + """ + Record the given nodeid in the graph. + + If the nodeid is already present, but its type is unset, set it. + :returns: The Node tuple (even if already present in the graph). + """ if nodeid in nodes: n = nodes[nodeid] if n.type is None: From 606ec26c754aadcfbec4f2898a05ca98e1539e4c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:26:53 +0000 Subject: [PATCH 14/78] Bump mypy from 1.11.2 to 1.12.0 Bumps [mypy](https://github.com/python/mypy) from 1.11.2 to 1.12.0. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.11.2...v1.12.0) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- mypy-requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index ccd7737f4..4aa7f0a2d 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.11.2 # also update pyproject.toml +mypy==1.12.0 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 cwltest diff --git a/pyproject.toml b/pyproject.toml index 4f3f91c31..4d93750ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.11.2", # also update mypy-requirements.txt + "mypy==1.12.0", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", From 74a08ca0e90a223f98fdec73c88f36d783a4dde0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Oct 2024 07:26:21 +0000 Subject: [PATCH 15/78] Update rdflib requirement from <7.1,>=4.2.2 to >=4.2.2,<7.2 Updates the requirements on [rdflib](https://github.com/RDFLib/rdflib) to permit the latest version. - [Release notes](https://github.com/RDFLib/rdflib/releases) - [Changelog](https://github.com/RDFLib/rdflib/blob/main/CHANGELOG.md) - [Commits](https://github.com/RDFLib/rdflib/compare/4.2.2...7.1.0) --- updated-dependencies: - dependency-name: rdflib dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b1fc2207d..3ac631838 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ requests>=2.6.1 ruamel.yaml>=0.16.0,<0.19 -rdflib>=4.2.2,<7.1 +rdflib>=4.2.2,<7.2 schema-salad>=8.7,<9 prov==1.5.1 mypy-extensions diff --git a/setup.py b/setup.py index 40c3fd8d4..9bbee10f1 100644 --- a/setup.py +++ b/setup.py @@ -124,7 +124,7 @@ "requests >= 2.6.1", # >= 2.6.1 to workaround # https://github.com/ionrock/cachecontrol/issues/137 "ruamel.yaml >= 0.16, < 0.19", - "rdflib >= 4.2.2, < 7.1.0", + "rdflib >= 4.2.2, < 7.2.0", "schema-salad >= 8.7, < 9", "prov == 1.5.1", "mypy-extensions", From e3e6bf9d51946579081ee3b41c28f1da194aa895 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:55:55 +0000 Subject: [PATCH 16/78] Bump mypy from 1.12.0 to 1.12.1 (#2057) * Bump mypy from 1.12.0 to 1.12.1 Bumps [mypy](https://github.com/python/mypy) from 1.12.0 to 1.12.1. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.12.0...v1.12.1) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update pyproject.toml --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- mypy-requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 4aa7f0a2d..d090f1943 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.12.0 # also update pyproject.toml +mypy==1.12.1 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 cwltest diff --git a/pyproject.toml b/pyproject.toml index 4d93750ff..c2546d266 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.12.0", # also update mypy-requirements.txt + "mypy==1.12.1", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", From 8dee8e9b4227b30a8a0c1f57e69d158e0c56f4ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 08:52:24 +0000 Subject: [PATCH 17/78] Bump mypy from 1.12.1 to 1.13.0 (#2058) * Bump mypy from 1.12.1 to 1.13.0 Bumps [mypy](https://github.com/python/mypy) from 1.12.1 to 1.13.0. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.12.1...v1.13.0) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update pyproject.toml --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- mypy-requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index d090f1943..5f18fa03a 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.12.1 # also update pyproject.toml +mypy==1.13.0 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 cwltest diff --git a/pyproject.toml b/pyproject.toml index c2546d266..cec213f52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.12.1", # also update mypy-requirements.txt + "mypy==1.13.0", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", From e3f6cf77f2e81bd97db96234b79c968227429143 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 23 Oct 2024 12:01:04 +0200 Subject: [PATCH 18/78] build binary wheels But don't publish them yet --- .circleci/config.yml | 104 +++++++++++++++++++++++++ .github/workflows/ci-tests.yml | 16 ++-- .github/workflows/wheels.yml | 130 +++++++++++++++++++++++++++++++ MANIFEST.in | 1 + cibw-requirements.txt | 1 + cwltool/software_requirements.py | 2 + pyproject.toml | 13 ++++ test-requirements.txt | 2 +- tests/test_dependencies.py | 10 +-- 9 files changed, 265 insertions(+), 14 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 .github/workflows/wheels.yml create mode 100644 cibw-requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..2127e18be --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,104 @@ +version: 2.1 + +parameters: + REF: + type: string + default: '' + description: Optional tag to build + +jobs: + arm-wheels: + parameters: + build: + type: string + image: + type: string + + machine: + image: ubuntu-2204:current + resource_class: arm.medium # two vCPUs + + environment: + CIBW_ARCHS: "aarch64" + CIBW_MANYLINUX_AARCH64_IMAGE: "<< parameters.image >>" + CIBW_MUSLLINUX_AARCH64_IMAGE: "<< parameters.image >>" + CIBW_BUILD: "<< parameters.build >>" + + steps: + - checkout + - when: + condition: << pipeline.parameters.REF >> + steps: + - run: + name: Checkout branch/tag << pipeline.parameters.REF >> + command: | + echo "Switching to branch/tag << pipeline.parameters.REF >> if it exists" + git checkout << pipeline.parameters.REF >> || true + git pull origin << pipeline.parameters.REF >> || true + - run: + name: install cibuildwheel and other build reqs + command: | + python3 -m pip install --upgrade pip setuptools setuptools_scm[toml] + python3 -m pip install -rcibw-requirements.txt + + - run: + name: pip freeze + command: | + python3 -m pip freeze + + - run: + name: list wheels + command: | + python3 -m cibuildwheel . --print-build-identifiers + + - run: + name: cibuildwheel + command: | + python3 -m cibuildwheel . + + - store_test_results: + path: test-results/ + + - store_artifacts: + path: wheelhouse/ + + # - when: + # condition: + # or: + # - matches: + # pattern: ".+" + # value: "<< pipeline.git.tag >>" + # - << pipeline.parameters.REF >> + # steps: + # - run: + # environment: + # TWINE_NONINTERACTIVE: "1" + # command: | + # python3 -m pip install twine + # python3 -m twine upload --verbose --skip-existing wheelhouse/* + +workflows: + wheels: # This is the name of the workflow, feel free to change it to better match your workflow. + # Inside the workflow, you define the jobs you want to run. + jobs: + - arm-wheels: + name: arm-wheels-manylinux_2_28 + filters: + tags: + only: /.*/ + build: "*manylinux*" + image: quay.io/pypa/manylinux_2_28_aarch64 + - arm-wheels: + name: arm-wheels-musllinux_1_1 + filters: + tags: + only: /.*/ + build: "*musllinux*" + image: quay.io/pypa/musllinux_1_1_aarch64 + - arm-wheels: + name: arm-wheels-musllinux_1_2 + filters: + tags: + only: /.*/ + build: "*musllinux*" + image: quay.io/pypa/musllinux_1_2_aarch64 diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 2095b53b6..e5d9a7e83 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -44,11 +44,11 @@ jobs: with: fetch-depth: 0 - - name: Set up Singularity + - name: Set up Singularity and environment-modules if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} run: | wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} @@ -132,10 +132,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Singularity + - name: Set up Singularity and environment-modules run: | wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. run: sudo usermod -c 'CI Runner' "$(whoami)" @@ -180,11 +180,11 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Singularity + - name: Set up Singularity and environment-modules if: ${{ matrix.container == 'singularity' }} run: | wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb + sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules - name: Singularity cache if: ${{ matrix.container == 'singularity' }} @@ -229,10 +229,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Singularity + - name: Set up Singularity and environment-modules run: | wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb + sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 000000000..9c14eb4e7 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,130 @@ +name: Python package build and publish + +on: + release: + types: [published] + workflow_dispatch: {} + repository_dispatch: {} + pull_request: + push: + branches: + - main + +concurrency: + group: wheels-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build_wheels: + name: ${{ matrix.image }} wheels + runs-on: ubuntu-24.04 + strategy: + matrix: + include: + - image: manylinux_2_28_x86_64 + build: "*manylinux*" + - image: musllinux_1_1_x86_64 + build: "*musllinux*" + - image: musllinux_1_2_x86_64 + build: "*musllinux*" + + steps: + - uses: actions/checkout@v4 + if: ${{ github.event_name != 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + - uses: actions/checkout@v4 + if: ${{ github.event_name == 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + ref: ${{ github.event.client_payload.ref }} + + # - name: Set up QEMU + # if: runner.os == 'Linux' + # uses: docker/setup-qemu-action@v2 + # with: + # platforms: all + + - name: Build wheels + uses: pypa/cibuildwheel@v2.21.3 + env: + CIBW_BUILD: ${{ matrix.build }} + CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} + CIBW_MUSLLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} + # configure cibuildwheel to build native 64-bit archs ('auto64'), and some + # emulated ones + # Linux arm64 wheels are built on circleci + CIBW_ARCHS_LINUX: auto64 # ppc64le s390x + + - uses: actions/upload-artifact@v4 + with: + name: artifact-${{ matrix.image }} + path: ./wheelhouse/*.whl + + build_sdist: + name: Build source distribution + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + if: ${{ github.event_name != 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + - uses: actions/checkout@v4 + if: ${{ github.event_name == 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + ref: ${{ github.event.client_payload.ref }} + + - name: Build sdist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v4 + with: + name: artifact-source + path: dist/*.tar.gz + + build_wheels_macos: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + # macos-13 is an intel runner, macos-14 is apple silicon + os: [macos-13, macos-14] + steps: + - uses: actions/checkout@v4 + if: ${{ github.event_name != 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + - uses: actions/checkout@v4 + if: ${{ github.event_name == 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + ref: ${{ github.event.client_payload.ref }} + + - name: Build wheels + uses: pypa/cibuildwheel@v2.21.3 + + - uses: actions/upload-artifact@v4 + with: + name: artifact-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl + + # upload_pypi: + # needs: [build_wheels, build_sdist] + # runs-on: ubuntu-24.04 + # environment: deploy + # permissions: + # id-token: write + # if: (github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'repository_dispatch' && github.event.client_payload.publish_wheel == true) + # steps: + # - uses: actions/download-artifact@v4 + # with: + # # unpacks default artifact into dist/ + # pattern: artifact-* + # merge-multiple: true + # path: dist + + # - name: Publish package distributions to PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # skip-existing: true diff --git a/MANIFEST.in b/MANIFEST.in index 187d19bea..7ee34f35e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -19,6 +19,7 @@ include tests/reloc/dir2/* include tests/checker_wf/* include tests/subgraph/* include tests/input_deps/* +recursive-include tests/test_deps_env include tests/trs/* include tests/wf/generator/* include cwltool/py.typed diff --git a/cibw-requirements.txt b/cibw-requirements.txt new file mode 100644 index 000000000..c4511439c --- /dev/null +++ b/cibw-requirements.txt @@ -0,0 +1 @@ +cibuildwheel==2.21.3 diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index 6ad84da4b..3d4d48f6b 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -50,6 +50,8 @@ class DependenciesConfiguration: def __init__(self, args: argparse.Namespace) -> None: """Initialize.""" + self.tool_dependency_dir: Optional[str] = None + self.dependency_resolvers_config_file: Optional[str] = None conf_file = getattr(args, "beta_dependency_resolvers_configuration", None) tool_dependency_dir = getattr(args, "beta_dependencies_directory", None) conda_dependencies = getattr(args, "beta_conda_dependencies", None) diff --git a/pyproject.toml b/pyproject.toml index cec213f52..a69720739 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,19 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] write_to = "cwltool/_version.py" +[tool.cibuildwheel] +test-command = "python -m pytest -n 2 --junitxml={project}/test-results/junit_$(python -V | awk '{print $2}')_${AUDITWHEEL_PLAT}.xml -k 'not (test_bioconda or test_env_filtering or test_udocker)' --pyargs cwltool" +test-requires = "-r test-requirements.txt" +test-extras = "deps" +skip = "pp*" +# ^ skip building wheels on PyPy (any version) +build-verbosity = 1 +environment = { CWLTOOL_USE_MYPYC="1", MYPYPATH="$(pwd)/mypy-stubs" } + +# Install system library +[tool.cibuildwheel.linux] +before-all = "apk add libxml2-dev libxslt-dev nodejs || yum install -y libxml2-devel libxslt-devel nodejs environment-modules || apt-get install -y --no-install-recommends libxml2-dev libxslt-dev nodejs environment-modules" + [tool.black] line-length = 100 target-version = [ "py39" ] diff --git a/test-requirements.txt b/test-requirements.txt index e545ee65a..8b0908f2e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,7 +3,7 @@ pytest>= 6.2,< 8.4 pytest-xdist>=3.2.0 # for the worksteal scheduler psutil # enhances pytest-xdist to allow "-n logical" pytest-httpserver -pytest-retry;python_version>'3.9' +pytest-retry;python_version>='3.9' mock>=2.0.0 pytest-mock>=1.10.0 pytest-cov diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index b903c04d6..f5ac0274b 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -119,15 +119,15 @@ def test_modules(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Do a basic smoke test using environment modules to satisfy a SoftwareRequirement.""" wflow = get_data("tests/random_lines.cwl") job = get_data("tests/random_lines_job.json") - monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles")) + monkeypatch.setenv("MODULEPATH", get_data("tests/test_deps_env/modulefiles")) error_code, _, stderr = get_main_output( [ "--outdir", str(tmp_path / "out"), - "--beta-dependency-resolvers-configuration", "--beta-dependencies-directory", str(tmp_path / "deps"), - "tests/test_deps_env_modules_resolvers_conf.yml", + "--beta-dependency-resolvers-configuration", + get_data("tests/test_deps_env_modules_resolvers_conf.yml"), "--debug", wflow, job, @@ -145,7 +145,7 @@ def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Do so by by running `env` as the tool and parsing its output. """ - monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles")) + monkeypatch.setenv("MODULEPATH", get_data("tests/test_deps_env/modulefiles")) tool_env = get_tool_env( tmp_path, [ @@ -155,6 +155,6 @@ def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> get_data("tests/env_with_software_req.yml"), ) - assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space " + assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space ", tool_env tool_path = tool_env["PATH"].split(":") assert get_data("tests/test_deps_env/random-lines/1.0/scripts") in tool_path From 6cfef62c21330672538fd5e9b45ec888569c0a6f Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 24 Oct 2024 15:35:55 +0200 Subject: [PATCH 19/78] binary wheels: only build on main and other optimizations - cibuildwheel: adjust pytest parallel execution - cicircle: upgrade to ubuntu 24.04 - Stop building for musllinux_1_1 https://github.com/pypa/manylinux/issues/1629 > musl libc 1.1 is EOL and Alpine Linux 3.12 also (support ended 2 years ago, May 1st, 2022). --- .circleci/config.yml | 15 ++++++--------- .github/workflows/wheels.yml | 3 --- pyproject.toml | 2 +- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2127e18be..fce789feb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,8 +15,8 @@ jobs: type: string machine: - image: ubuntu-2204:current - resource_class: arm.medium # two vCPUs + image: ubuntu-2404:current + resource_class: arm.medium # 2 vCPUs environment: CIBW_ARCHS: "aarch64" @@ -84,20 +84,17 @@ workflows: - arm-wheels: name: arm-wheels-manylinux_2_28 filters: + branches: + only: main tags: only: /.*/ build: "*manylinux*" image: quay.io/pypa/manylinux_2_28_aarch64 - - arm-wheels: - name: arm-wheels-musllinux_1_1 - filters: - tags: - only: /.*/ - build: "*musllinux*" - image: quay.io/pypa/musllinux_1_1_aarch64 - arm-wheels: name: arm-wheels-musllinux_1_2 filters: + branches: + only: main tags: only: /.*/ build: "*musllinux*" diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 9c14eb4e7..fb54729d4 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -5,7 +5,6 @@ on: types: [published] workflow_dispatch: {} repository_dispatch: {} - pull_request: push: branches: - main @@ -23,8 +22,6 @@ jobs: include: - image: manylinux_2_28_x86_64 build: "*manylinux*" - - image: musllinux_1_1_x86_64 - build: "*musllinux*" - image: musllinux_1_2_x86_64 build: "*musllinux*" diff --git a/pyproject.toml b/pyproject.toml index a69720739..cec96b76b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ build-backend = "setuptools.build_meta" write_to = "cwltool/_version.py" [tool.cibuildwheel] -test-command = "python -m pytest -n 2 --junitxml={project}/test-results/junit_$(python -V | awk '{print $2}')_${AUDITWHEEL_PLAT}.xml -k 'not (test_bioconda or test_env_filtering or test_udocker)' --pyargs cwltool" +test-command = "python -m pytest --ignore cwltool/schemas -n logical --dist worksteal --junitxml={project}/test-results/junit_$(python -V | awk '{print $2}')_${AUDITWHEEL_PLAT}.xml -k 'not (test_bioconda or test_env_filtering or test_udocker)' --pyargs cwltool" test-requires = "-r test-requirements.txt" test-extras = "deps" skip = "pp*" From d810958c3cb262db8f099c08a0d6e50989edbefd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 07:08:45 +0000 Subject: [PATCH 20/78] Update flake8-bugbear requirement from <24.9 to <24.11 Updates the requirements on [flake8-bugbear](https://github.com/PyCQA/flake8-bugbear) to permit the latest version. - [Release notes](https://github.com/PyCQA/flake8-bugbear/releases) - [Commits](https://github.com/PyCQA/flake8-bugbear/compare/16.4.1...24.10.31) --- updated-dependencies: - dependency-name: flake8-bugbear dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index 5af76cd93..ec1e6e1fe 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,3 +1,3 @@ -flake8-bugbear<24.9 +flake8-bugbear<24.11 black==24.* codespell From 9cda157cb4380e9d30dec29f0452c56d0c10d064 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Sat, 2 Nov 2024 10:30:37 +0100 Subject: [PATCH 21/78] cpu_count can be None, so fallback to 1 --- cwltool/executors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cwltool/executors.py b/cwltool/executors.py index 6070462ab..03fa948e7 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -273,7 +273,7 @@ def __init__(self) -> None: self.pending_jobs_lock = threading.Lock() self.max_ram = int(psutil.virtual_memory().available / 2**20) - self.max_cores = float(psutil.cpu_count()) + self.max_cores = float(psutil.cpu_count() or 1) self.max_cuda = cuda_version_and_device_count()[1] self.allocated_ram = float(0) self.allocated_cores = float(0) @@ -429,7 +429,7 @@ def run_jobs( logger: logging.Logger, runtime_context: RuntimeContext, ) -> None: - self.taskqueue: TaskQueue = TaskQueue(threading.Lock(), psutil.cpu_count()) + self.taskqueue: TaskQueue = TaskQueue(threading.Lock(), psutil.cpu_count() or 1) try: jobiter = process.job(job_order_object, self.output_callback, runtime_context) From ee30368e4f9175c76e194c9f5fa734a5ca4b767c Mon Sep 17 00:00:00 2001 From: Dominik Brilhaus Date: Fri, 8 Nov 2024 09:46:13 +0100 Subject: [PATCH 22/78] add note on docker platform issue (#2064) Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- README.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.rst b/README.rst index db40d0420..dce55fc95 100644 --- a/README.rst +++ b/README.rst @@ -189,6 +189,22 @@ and ``--tmp-outdir-prefix`` to somewhere under ``/Users``:: $ cwl-runner --tmp-outdir-prefix=/Users/username/project --tmpdir-prefix=/Users/username/project wc-tool.cwl wc-job.json + +Docker default platform on macOS with Apple Silicon +=================================================== + +If macOS users want to run CWL tools/workflows using ``cwltool`` with Docker and their software containers only have support for amd64 (64-bit x86) CPUs, but they have an Apple Silicon (aarch64/arm64) CPU, +they run into the error: + + WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested. + +To fix this, export the following environment variable before executing `cwltool`: + +``export DOCKER_DEFAULT_PLATFORM=linux/amd64`` + +To automatically have this variable set in the future, add the same command to ones respective shell profile (e.g. ``~/.zshrc``, ``~/.bash_profile``). + + Using uDocker ============= From 0b649350fdb7f9dc4acff3bb33fcfd062334df9f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 09:52:36 +0000 Subject: [PATCH 23/78] Update ruamel-yaml requirement from <0.18,>=0.16.0 to >=0.16.0,<0.19 (#2066) Updates the requirements on [ruamel-yaml]() to permit the latest version. --- updated-dependencies: - dependency-name: ruamel-yaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cec96b76b..7ddf547f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ requires = [ "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", - "ruamel.yaml>=0.16.0,<0.18", + "ruamel.yaml>=0.16.0,<0.19", "schema-salad>=8.7,<9", "cwl-utils>=0.32", "toml", From 048eb55aefd8d71d161fbc89ec0e888b8bfa0aa1 Mon Sep 17 00:00:00 2001 From: Kostas Mavrommatis Date: Mon, 11 Nov 2024 13:38:32 +0100 Subject: [PATCH 24/78] use max_cores in taskQueue instead of system cores (#2038) The class TascQueue accepts an argument `thread_count` that is used for the max size of a queue. In the `MultithreadedJobExecutor `class there is a variable defined (`max_cores`) that is getting its value from the available cores of the machine. Further down in the same class when TaskQueue is called instead of using the `max_cores` it is using `psutil.cpu_count()`. I suggest to use the self.max_cores in the call of TaskQueue in the file executor.py instead of psutil.cpu_count() Use case: when a job executor is setup as MultithreadedJobExecutor one can override the max_cores after the initialization of the object and limit the use to the specified cores. Additional enhancement would be to include an argument to allow the use to provide the number of cores available for use Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- cwltool/executors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwltool/executors.py b/cwltool/executors.py index 03fa948e7..e25426c9d 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -429,7 +429,7 @@ def run_jobs( logger: logging.Logger, runtime_context: RuntimeContext, ) -> None: - self.taskqueue: TaskQueue = TaskQueue(threading.Lock(), psutil.cpu_count() or 1) + self.taskqueue: TaskQueue = TaskQueue(threading.Lock(), int(math.ceil(self.max_cores))) try: jobiter = process.job(job_order_object, self.output_callback, runtime_context) From 1557c8ded38dc1af4a296e8fdd3116d7dfc5282e Mon Sep 17 00:00:00 2001 From: Sameeul Samee Date: Sun, 10 Nov 2024 06:23:56 -0500 Subject: [PATCH 25/78] Use "run" with singularity/apptainer instead of "exec", when possible Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- cwltool/singularity.py | 20 +++++++++++++++++++- tests/test_environment.py | 3 +++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 0029d3950..40d1bc9b5 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -75,6 +75,14 @@ def is_apptainer_1_or_newer() -> bool: return v[0][0] >= 1 +def is_apptainer_1_1_or_newer() -> bool: + """Check if apptainer singularity distribution is version 1.1 or higher.""" + v = get_version() + if v[1] != "apptainer": + return False + return v[0][0] >= 2 or (v[0][0] >= 1 and v[0][1] >= 1) + + def is_version_2_6() -> bool: """ Check if this singularity version is exactly version 2.6. @@ -119,6 +127,12 @@ def is_version_3_9_or_newer() -> bool: return v[0][0] >= 4 or (v[0][0] == 3 and v[0][1] >= 9) +def is_version_3_10_or_newer() -> bool: + """Detect if Singularity v3.10+ is available.""" + v = get_version() + return v[0][0] >= 4 or (v[0][0] == 3 and v[0][1] >= 10) + + def _normalize_image_id(string: str) -> str: return string.replace("/", "_") + ".img" @@ -464,14 +478,18 @@ def create_runtime( ) -> tuple[list[str], Optional[str]]: """Return the Singularity runtime list of commands and options.""" any_path_okay = self.builder.get_requirement("DockerRequirement")[1] or False + runtime = [ "singularity", "--quiet", - "exec", + "run" if is_apptainer_1_1_or_newer() or is_version_3_10_or_newer() else "exec", "--contain", "--ipc", "--cleanenv", ] + if is_apptainer_1_1_or_newer() or is_version_3_10_or_newer(): + runtime.append("--no-eval") + if singularity_supports_userns(): runtime.append("--userns") else: diff --git a/tests/test_environment.py b/tests/test_environment.py index a4bfd1ac3..488477aa7 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -159,6 +159,9 @@ def BIND(v: str, env: Env) -> bool: return v.startswith(tmp_prefix) and v.endswith(":/tmp") sing_vars["SINGULARITY_BIND"] = BIND + if vminor >= 10: + sing_vars["SINGULARITY_COMMAND"] = "run" + sing_vars["SINGULARITY_NO_EVAL"] = None result.update(sing_vars) From c3c92ebb7c5d485ace93c6a31139b05f2e9d82d8 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Tue, 12 Nov 2024 14:12:02 +0100 Subject: [PATCH 26/78] conformances testing: no longer skip tests for singularity/apptainer --- conformance-test.sh | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/conformance-test.sh b/conformance-test.sh index 36ea23b17..6df14a63c 100755 --- a/conformance-test.sh +++ b/conformance-test.sh @@ -97,19 +97,7 @@ if [[ "$VERSION" = *dev* ]] then CWLTOOL_OPTIONS+=" --enable-dev" fi -if [[ "$CONTAINER" = "singularity" ]]; then - CWLTOOL_OPTIONS+=" --singularity" - # This test fails because Singularity and Docker have - # different views on how to deal with this. - exclusions+=(docker_entrypoint) - if [[ "${VERSION}" = "v1.1" ]]; then - # This fails because of a difference (in Singularity vs Docker) in - # the way filehandles are passed to processes in the container and - # wc can tell somehow. - # See issue #1440 - exclusions+=(stdin_shorcut) - fi -elif [[ "$CONTAINER" = "podman" ]]; then +if [[ "$CONTAINER" = "podman" ]]; then CWLTOOL_OPTIONS+=" --podman" fi From e4d42b85d24cd5088e14cc31d67c2dee0c6fc40a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Nov 2024 07:39:02 +0000 Subject: [PATCH 27/78] Bump codecov/codecov-action from 4 to 5 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 4 to 5. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v4...v5) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index e5d9a7e83..6608e9a22 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -81,7 +81,7 @@ jobs: - name: Upload coverage to Codecov if: ${{ matrix.step == 'unit' }} - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: fail_ci_if_error: true env: @@ -217,7 +217,7 @@ jobs: path: | **/cwltool_conf*.xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: fail_ci_if_error: true env: @@ -302,7 +302,7 @@ jobs: - name: Test with tox run: tox - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: fail_ci_if_error: true env: From 6c86caa0571fd186d90a6600e0bb405596d4a5e0 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 27 Nov 2024 16:37:33 +0100 Subject: [PATCH 28/78] make-template: fix type error --- cwltool/main.py | 4 +- tests/CometAdapter.cwl | 228 +++++++++++++++++++++++++++++++++++++++++ tests/test_examples.py | 12 +++ 3 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 tests/CometAdapter.cwl diff --git a/cwltool/main.py b/cwltool/main.py index 99928d0bd..7aedce6b1 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -330,10 +330,10 @@ def generate_input_template(tool: Process) -> CWLObjectType: """Generate an example input object for the given CWL process.""" template = ruamel.yaml.comments.CommentedMap() for inp in cast( - list[MutableMapping[str, str]], + list[CWLObjectType], realize_input_schema(tool.tool["inputs"], tool.schemaDefs), ): - name = shortname(inp["id"]) + name = shortname(cast(str, inp["id"])) value, comment = generate_example_input(inp["type"], inp.get("default", None)) template.insert(0, name, value, comment) return template diff --git a/tests/CometAdapter.cwl b/tests/CometAdapter.cwl new file mode 100644 index 000000000..ed1b24bb6 --- /dev/null +++ b/tests/CometAdapter.cwl @@ -0,0 +1,228 @@ +# Copyright (c) 2002-present, The OpenMS Team -- EKU Tuebingen, ETH Zurich, and FU Berlin +# SPDX-License-Identifier: Apache-2.0 +label: CometAdapter +doc: Annotates MS/MS spectra using Comet. +inputs: + in: + doc: Input file + type: File + out: + doc: Output file + type: string + database: + doc: FASTA file + type: File + comet_executable: + doc: The Comet executable. Provide a full or relative path, or make sure it can be found in your PATH environment. + type: File + pin_out: + doc: Output file - for Percolator input + type: string? + default_params_file: + doc: Default Comet params file. All parameters of this take precedence. A template file can be generated using 'comet.exe -p' + type: File? + precursor_mass_tolerance: + doc: "Precursor monoisotopic mass tolerance (Comet parameter: peptide_mass_tolerance). See also precursor_error_units to set the unit." + type: double? + precursor_error_units: + doc: "Unit of precursor monoisotopic mass tolerance for parameter precursor_mass_tolerance (Comet parameter: peptide_mass_units)" + type: string? + isotope_error: + doc: This parameter controls whether the peptide_mass_tolerance takes into account possible isotope errors in the precursor mass measurement. Use -8/-4/0/4/8 only for SILAC. + type: string? + fragment_mass_tolerance: + doc: "This is half the bin size, which is used to segment the MS/MS spectrum. Thus, the value should be a bit higher than for other search engines, since the bin might not be centered around the peak apex (see 'fragment_bin_offset').CAUTION: Low tolerances have heavy impact on RAM usage (since Comet uses a lot of bins in this case). Consider using use_sparse_matrix and/or spectrum_batch_size." + type: double? + fragment_error_units: + doc: Fragment monoisotopic mass error units + type: string? + fragment_bin_offset: + doc: "Offset of fragment bins. Recommended by Comet: low-res: 0.4, high-res: 0.0" + type: double? + instrument: + doc: "Comets theoretical_fragment_ions parameter: theoretical fragment ion peak representation, high-res: sum of intensities plus flanking bins, ion trap (low-res) ms/ms: sum of intensities of central M bin only" + type: string? + use_A_ions: + doc: use A ions for PSM + type: boolean? + use_B_ions: + doc: use B ions for PSM + type: boolean? + use_C_ions: + doc: use C ions for PSM + type: boolean? + use_X_ions: + doc: use X ions for PSM + type: boolean? + use_Y_ions: + doc: use Y ions for PSM + type: boolean? + use_Z_ions: + doc: use Z ions for PSM + type: boolean? + use_NL_ions: + doc: use neutral loss (NH3, H2O) ions from b/y for PSM + type: boolean? + enzyme: + doc: The enzyme used for peptide digestion. + type: string? + second_enzyme: + doc: Additional enzyme used for peptide digestion. + type: string? + num_enzyme_termini: + doc: Specify the termini where the cleavage rule has to match + type: string? + missed_cleavages: + doc: Number of possible cleavage sites missed by the enzyme. It has no effect if enzyme is unspecific cleavage. + type: long? + min_peptide_length: + doc: Minimum peptide length to consider. + type: long? + max_peptide_length: + doc: Maximum peptide length to consider. + type: long? + num_hits: + doc: Number of peptide hits (PSMs) per spectrum in output file + type: long? + precursor_charge: + doc: "Precursor charge range to search (if spectrum is not annotated with a charge or if override_charge!=keep any known): 0:[num] == search all charges, 2:6 == from +2 to +6, 3:3 == +3" + type: string? + override_charge: + doc: "_keep any known_: keep any precursor charge state (from input), _ignore known_: ignore known precursor charge state and use precursor_charge parameter, _ignore outside range_: ignore precursor charges outside precursor_charge range, _keep known search unknown_: keep any known precursor charge state. For unknown charge states, search as singly charged if there is no signal above the precursor m/z or use the precursor_charge range" + type: string? + ms_level: + doc: MS level to analyze, valid are levels 2 (default) or 3 + type: long? + activation_method: + doc: If not ALL, only searches spectra of the given method + type: string? + digest_mass_range: + doc: MH+ peptide mass range to analyze + type: string? + max_fragment_charge: + doc: Set maximum fragment charge state to analyze as long as still lower than precursor charge - 1. (Allowed max 5) + type: long? + max_precursor_charge: + doc: set maximum precursor charge state to analyze (allowed max 9) + type: long? + clip_nterm_methionine: + doc: If set to true, also considers the peptide sequence w/o N-term methionine separately and applies appropriate N-term mods to it + type: boolean? + spectrum_batch_size: + doc: max. number of spectra to search at a time; use 0 to search the entire scan range in one batch + type: long? + mass_offsets: + doc: One or more mass offsets to search (values subtracted from deconvoluted precursor mass). Has to include 0.0 if you want the default mass to be searched. + type: double[]? + minimum_peaks: + doc: Required minimum number of peaks in spectrum to search (default 10) + type: long? + minimum_intensity: + doc: Minimum intensity value to read in + type: double? + remove_precursor_peak: + doc: no = no removal, yes = remove all peaks around precursor m/z, charge_reduced = remove all charge reduced precursor peaks (for ETD/ECD). phosphate_loss = remove the HPO3 (-80) and H3PO4 (-98) precursor phosphate neutral loss peaks. See also remove_precursor_tolerance + type: string? + remove_precursor_tolerance: + doc: one-sided tolerance for precursor removal in Thompson + type: double? + clear_mz_range: + doc: for iTRAQ/TMT type data; will clear out all peaks in the specified m/z range, if not 0:0 + type: string? + fixed_modifications: + doc: Fixed modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + type: string[]? + variable_modifications: + doc: Variable modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + type: string[]? + binary_modifications: + doc: "List of modification group indices. Indices correspond to the binary modification index used by comet to group individually searched lists of variable modifications.\nNote: if set, both variable_modifications and binary_modifications need to have the same number of entries as the N-th entry corresponds to the N-th variable_modification.\n if left empty (default), all entries are internally set to 0 generating all permutations of modified and unmodified residues.\n For a detailed explanation please see the parameter description in the Comet help." + type: long[]? + max_variable_mods_in_peptide: + doc: Set a maximum number of variable modifications per peptide + type: long? + require_variable_mod: + doc: If true, requires at least one variable modification per peptide + type: boolean? + reindex: + doc: Recalculate peptide to protein association using OpenMS. Annotates target-decoy information. + type: string? + log: + doc: Name of log file (created only when specified) + type: string? + debug: + doc: Sets the debug level + type: long? + threads: + doc: Sets the number of threads allowed to be used by the TOPP tool + type: long? + no_progress: + doc: Disables progress logging to command line + type: boolean? + force: + doc: Overrides tool-specific checks + type: boolean? + test: + doc: Enables the test mode (needed for internal use only) + type: boolean? + PeptideIndexing__decoy_string: + doc: String that was appended (or prefixed - see 'decoy_string_position' flag below) to the accessions in the protein database to indicate decoy proteins. If empty (default), it's determined automatically (checking for common terms, both as prefix and suffix). + type: string? + PeptideIndexing__decoy_string_position: + doc: Is the 'decoy_string' prepended (prefix) or appended (suffix) to the protein accession? (ignored if decoy_string is empty) + type: string? + PeptideIndexing__missing_decoy_action: + doc: "Action to take if NO peptide was assigned to a decoy protein (which indicates wrong database or decoy string): 'error' (exit with error, no output), 'warn' (exit with success, warning message), 'silent' (no action is taken, not even a warning)" + type: string? + PeptideIndexing__write_protein_sequence: + doc: If set, the protein sequences are stored as well. + type: boolean? + PeptideIndexing__write_protein_description: + doc: If set, the protein description is stored as well. + type: boolean? + PeptideIndexing__keep_unreferenced_proteins: + doc: If set, protein hits which are not referenced by any peptide are kept. + type: boolean? + PeptideIndexing__unmatched_action: + doc: "If peptide sequences cannot be matched to any protein: 1) raise an error; 2) warn (unmatched PepHits will miss target/decoy annotation with downstream problems); 3) remove the hit." + type: string? + PeptideIndexing__aaa_max: + doc: Maximal number of ambiguous amino acids (AAAs) allowed when matching to a protein database with AAAs. AAAs are 'B', 'J', 'Z' and 'X'. + type: long? + PeptideIndexing__mismatches_max: + doc: Maximal number of mismatched (mm) amino acids allowed when matching to a protein database. The required runtime is exponential in the number of mm's; apply with care. MM's are allowed in addition to AAA's. + type: long? + PeptideIndexing__IL_equivalent: + doc: Treat the isobaric amino acids isoleucine ('I') and leucine ('L') as equivalent (indistinguishable). Also occurrences of 'J' will be treated as 'I' thus avoiding ambiguous matching. + type: boolean? + PeptideIndexing__allow_nterm_protein_cleavage: + doc: Allow the protein N-terminus amino acid to clip. + type: string? + PeptideIndexing__enzyme__name: + doc: "Enzyme which determines valid cleavage sites - e.g. trypsin cleaves after lysine (K) or arginine (R), but not before proline (P). Default: deduce from input" + type: string? + PeptideIndexing__enzyme__specificity: + doc: "Specificity of the enzyme. Default: deduce from input.\n 'full': both internal cleavage sites must match.\n 'semi': one of two internal cleavage sites must match.\n 'none': allow all peptide hits no matter their context (enzyme is irrelevant)." + type: string? +outputs: + out: + type: File + outputBinding: + glob: $(inputs.out) + pin_out: + type: File? + outputBinding: + glob: $(inputs.pin_out) +cwlVersion: v1.2 +class: CommandLineTool +baseCommand: + - CometAdapter +requirements: + InlineJavascriptRequirement: {} + InitialWorkDirRequirement: + listing: + - entryname: cwl_inputs.json + entry: $(JSON.stringify(inputs)) +arguments: + - -ini + - cwl_inputs.json diff --git a/tests/test_examples.py b/tests/test_examples.py index 23d17dcb2..c6ec6d06a 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1894,3 +1894,15 @@ def test_input_named_id() -> None: ] ) assert exit_code == 0, stderr + + +def test_make_template() -> None: + """End-to-end test of --make-template, especially for mypyc mode.""" + exit_code, stdout, stderr = get_main_output( + [ + "--make-template", + "--debug", + get_data("tests/CometAdapter.cwl"), + ] + ) + assert exit_code == 0, stderr From 7fa57ddbf627b0a90efcde0151b92968620c9f9f Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Mon, 9 Dec 2024 12:12:20 +0100 Subject: [PATCH 29/78] --no-warnings includes schema-salad; --validate should complain to stdout --- cwltool/loghandler.py | 22 +++++++----- cwltool/main.py | 18 +++++----- tests/test_examples.py | 13 ++++--- tests/test_validate.py | 80 +++++++++++++++++++++++++++++++++++++++--- tests/util.py | 5 +-- 5 files changed, 107 insertions(+), 31 deletions(-) diff --git a/cwltool/loghandler.py b/cwltool/loghandler.py index 76daa8be9..c76830816 100644 --- a/cwltool/loghandler.py +++ b/cwltool/loghandler.py @@ -11,7 +11,7 @@ def configure_logging( - stderr_handler: logging.Handler, + err_handler: logging.Handler, no_warnings: bool, quiet: bool, debug: bool, @@ -21,25 +21,29 @@ def configure_logging( ) -> None: """Configure logging.""" rdflib_logger = logging.getLogger("rdflib.term") - rdflib_logger.addHandler(stderr_handler) + rdflib_logger.addHandler(err_handler) rdflib_logger.setLevel(logging.ERROR) deps_logger = logging.getLogger("galaxy.tool_util.deps") - deps_logger.addHandler(stderr_handler) + deps_logger.addHandler(err_handler) ss_logger = logging.getLogger("salad") - ss_logger.addHandler(stderr_handler) if no_warnings: - stderr_handler.setLevel(logging.ERROR) - if quiet: + err_handler.setLevel(logging.ERROR) + ss_logger.setLevel(logging.ERROR) + elif quiet: # Silence STDERR, not an eventual provenance log file - stderr_handler.setLevel(logging.WARN) + err_handler.setLevel(logging.WARN) + ss_logger.setLevel(logging.WARN) + else: + err_handler.setLevel(logging.INFO) + ss_logger.setLevel(logging.INFO) if debug: # Increase to debug for both stderr and provenance log file base_logger.setLevel(logging.DEBUG) - stderr_handler.setLevel(logging.DEBUG) + err_handler.setLevel(logging.DEBUG) rdflib_logger.setLevel(logging.DEBUG) deps_logger.setLevel(logging.DEBUG) fmtclass = coloredlogs.ColoredFormatter if enable_color else logging.Formatter formatter = fmtclass("%(levelname)s %(message)s") if timestamps: formatter = fmtclass("[%(asctime)s] %(levelname)s %(message)s", "%Y-%m-%d %H:%M:%S") - stderr_handler.setFormatter(formatter) + err_handler.setFormatter(formatter) diff --git a/cwltool/main.py b/cwltool/main.py index 7aedce6b1..17ccb11ce 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -967,12 +967,6 @@ def main( stdout = cast(IO[str], stdout) _logger.removeHandler(defaultStreamHandler) - stderr_handler = logger_handler - if stderr_handler is not None: - _logger.addHandler(stderr_handler) - else: - coloredlogs.install(logger=_logger, stream=stderr) - stderr_handler = _logger.handlers[-1] workflowobj = None prov_log_handler: Optional[logging.StreamHandler[ProvOut]] = None global docker_exe @@ -997,6 +991,13 @@ def main( if not args.cidfile_dir: args.cidfile_dir = os.getcwd() del args.record_container_id + if logger_handler is not None: + err_handler = logger_handler + _logger.addHandler(err_handler) + else: + coloredlogs.install(logger=_logger, stream=stdout if args.validate else stderr) + err_handler = _logger.handlers[-1] + logging.getLogger("salad").handlers = _logger.handlers if runtimeContext is None: runtimeContext = RuntimeContext(vars(args)) @@ -1015,7 +1016,7 @@ def main( setattr(args, key, val) configure_logging( - stderr_handler, + err_handler, args.no_warnings, args.quiet, runtimeContext.debug, @@ -1413,8 +1414,7 @@ def loc_to_path(obj: CWLObjectType) -> None: # public API for logging.StreamHandler prov_log_handler.close() close_ro(research_obj, args.provenance) - - _logger.removeHandler(stderr_handler) + _logger.removeHandler(err_handler) _logger.addHandler(defaultStreamHandler) diff --git a/tests/test_examples.py b/tests/test_examples.py index c6ec6d06a..f413976fd 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1820,9 +1820,9 @@ def test_validate_optional_src_with_mandatory_sink() -> None: ["--validate", get_data("tests/wf/optional_src_mandatory_sink.cwl")] ) assert exit_code == 0 - stderr = re.sub(r"\s\s+", " ", stderr) - assert 'Source \'opt_file\' of type ["null", "File"] may be incompatible' in stderr - assert "with sink 'r' of type \"File\"" in stderr + stdout = re.sub(r"\s\s+", " ", stdout) + assert 'Source \'opt_file\' of type ["null", "File"] may be incompatible' in stdout + assert "with sink 'r' of type \"File\"" in stdout def test_res_req_expr_float_1_0() -> None: @@ -1875,12 +1875,11 @@ def test_invalid_nested_array() -> None: ] ) assert exit_code == 1, stderr - stderr = re.sub(r"\n\s+", " ", stderr) - stderr = re.sub(r"\s\s+", " ", stderr) - assert "Tool definition failed validation:" in stderr + stdout = re.sub(r"\s\s+", " ", stdout) + assert "Tool definition failed validation:" in stdout assert ( "tests/nested-array.cwl:6:5: Field 'type' references unknown identifier 'string[][]'" - ) in stderr + ) in stdout def test_input_named_id() -> None: diff --git a/tests/test_validate.py b/tests/test_validate.py index 171a6b6c1..f2d89e473 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -1,5 +1,7 @@ """Tests --validation.""" +import io +import logging import re from .util import get_data, get_main_output @@ -43,13 +45,83 @@ def test_validate_with_invalid_input_object() -> None: ] ) assert exit_code == 1 - stderr = re.sub(r"\s\s+", " ", stderr) - assert "Invalid job input record" in stderr + stdout = re.sub(r"\s\s+", " ", stdout) + assert "Invalid job input record" in stdout assert ( "tests/wf/1st-workflow_bad_inputs.yml:2:1: * the 'ex' field is not " - "valid because the value is not string" in stderr + "valid because the value is not string" in stdout ) assert ( "tests/wf/1st-workflow_bad_inputs.yml:1:1: * the 'inp' field is not " - "valid because is not a dict. Expected a File object." in stderr + "valid because is not a dict. Expected a File object." in stdout + ) + + +def test_validate_quiet() -> None: + """Ensure that --validate --quiet prints the correct amount of information.""" + exit_code, stdout, stderr = get_main_output( + [ + "--validate", + "--quiet", + get_data("tests/CometAdapter.cwl"), + ] + ) + assert exit_code == 0 + stdout = re.sub(r"\s\s+", " ", stdout) + assert "INFO" not in stdout + assert "INFO" not in stderr + assert "tests/CometAdapter.cwl:9:3: object id" in stdout + assert "tests/CometAdapter.cwl#out' previously defined" in stdout + + +def test_validate_no_warnings() -> None: + """Ensure that --validate --no-warnings doesn't print any warnings.""" + exit_code, stdout, stderr = get_main_output( + [ + "--validate", + "--no-warnings", + get_data("tests/CometAdapter.cwl"), + ] ) + assert exit_code == 0 + stdout = re.sub(r"\s\s+", " ", stdout) + stderr = re.sub(r"\s\s+", " ", stderr) + assert "INFO" not in stdout + assert "INFO" not in stderr + assert "WARNING" not in stdout + assert "WARNING" not in stderr + assert "tests/CometAdapter.cwl:9:3: object id" not in stdout + assert "tests/CometAdapter.cwl:9:3: object id" not in stderr + assert "tests/CometAdapter.cwl#out' previously defined" not in stdout + assert "tests/CometAdapter.cwl#out' previously defined" not in stderr + + +def test_validate_custom_logger() -> None: + """Custom log handling test.""" + custom_log = io.StringIO() + handler = logging.StreamHandler(custom_log) + handler.setLevel(logging.DEBUG) + exit_code, stdout, stderr = get_main_output( + [ + "--validate", + get_data("tests/CometAdapter.cwl"), + ], + logger_handler=handler, + ) + custom_log_text = custom_log.getvalue() + assert exit_code == 0 + custom_log_text = re.sub(r"\s\s+", " ", custom_log_text) + stdout = re.sub(r"\s\s+", " ", stdout) + stderr = re.sub(r"\s\s+", " ", stderr) + assert "INFO" not in stdout + assert "INFO" not in stderr + assert "INFO" in custom_log_text + assert "WARNING" not in stdout + assert "WARNING" not in stderr + assert "WARNING" in custom_log_text + assert "tests/CometAdapter.cwl:9:3: object id" not in stdout + assert "tests/CometAdapter.cwl:9:3: object id" not in stderr + assert "tests/CometAdapter.cwl:9:3: object id" in custom_log_text + assert "tests/CometAdapter.cwl#out' previously defined" not in stdout + assert "tests/CometAdapter.cwl#out' previously defined" not in stderr + assert "tests/CometAdapter.cwl#out' previously defined" in custom_log_text diff --git a/tests/util.py b/tests/util.py index 44d2f108c..8dd0bf74e 100644 --- a/tests/util.py +++ b/tests/util.py @@ -11,7 +11,7 @@ from collections.abc import Generator, Mapping from contextlib import ExitStack from pathlib import Path -from typing import Optional, Union +from typing import Any, Optional, Union import pytest @@ -88,6 +88,7 @@ def get_main_output( replacement_env: Optional[Mapping[str, str]] = None, extra_env: Optional[Mapping[str, str]] = None, monkeypatch: Optional[pytest.MonkeyPatch] = None, + **extra_kwargs: Any, ) -> tuple[Optional[int], str, str]: """Run cwltool main. @@ -113,7 +114,7 @@ def get_main_output( monkeypatch.setenv(k, v) try: - rc = main(argsl=args, stdout=stdout, stderr=stderr) + rc = main(argsl=args, stdout=stdout, stderr=stderr, **extra_kwargs) except SystemExit as e: if isinstance(e.code, int): rc = e.code From 604fd1242f4923c64d4cb396474ce9be0c134f8a Mon Sep 17 00:00:00 2001 From: Sameeul Bashir Samee Date: Wed, 11 Dec 2024 09:58:33 -0500 Subject: [PATCH 30/78] Append "_latest" to image id if no tag is present (#2085) --- cwltool/singularity.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 40d1bc9b5..d0e46fb27 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -134,10 +134,14 @@ def is_version_3_10_or_newer() -> bool: def _normalize_image_id(string: str) -> str: + if ":" not in string: + string += "_latest" return string.replace("/", "_") + ".img" def _normalize_sif_id(string: str) -> str: + if ":" not in string: + string += "_latest" return string.replace("/", "_") + ".sif" From d3c7bd5d6c409e857b98f9034a55952ca95afdb3 Mon Sep 17 00:00:00 2001 From: Iacopo Colonnelli Date: Thu, 12 Dec 2024 13:19:58 +0100 Subject: [PATCH 31/78] Fix `cwltool:Loop` extension definition (#2081) This commit fixes one small error in the `cwltool:Loop` definition that were breaking the Schema SALAD codegen procedure. --- cwltool/extensions-v1.2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwltool/extensions-v1.2.yml b/cwltool/extensions-v1.2.yml index c39b15d07..ae371c671 100644 --- a/cwltool/extensions-v1.2.yml +++ b/cwltool/extensions-v1.2.yml @@ -236,7 +236,7 @@ $graph: name: LoopOutputModes symbols: [ last, all ] default: last - doc: + doc: | - Specify the desired method of dealing with loop outputs - Default. Propagates only the last computed element to the subsequent steps when the loop terminates. - Propagates a single array with all output values to the subsequent steps when the loop terminates. From 37d85390827b17c13bc90b8e4e707cad359bfa03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:41:09 +0100 Subject: [PATCH 32/78] Update flake8-bugbear requirement from <24.11 to <24.13 (#2086) Updates the requirements on [flake8-bugbear](https://github.com/PyCQA/flake8-bugbear) to permit the latest version. - [Release notes](https://github.com/PyCQA/flake8-bugbear/releases) - [Commits](https://github.com/PyCQA/flake8-bugbear/compare/16.4.1...24.12.12) --- updated-dependencies: - dependency-name: flake8-bugbear dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index ec1e6e1fe..abb223c85 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,3 +1,3 @@ -flake8-bugbear<24.11 +flake8-bugbear<24.13 black==24.* codespell From 56e159f1fff80c4f89b0279fffb5a38f223008df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 10:45:18 +0100 Subject: [PATCH 33/78] Bump cibuildwheel from 2.21.3 to 2.22.0 (#2077) Bumps [cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.21.3 to 2.22.0. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.21.3...v2.22) --- updated-dependencies: - dependency-name: cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- cibw-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibw-requirements.txt b/cibw-requirements.txt index c4511439c..833aca23d 100644 --- a/cibw-requirements.txt +++ b/cibw-requirements.txt @@ -1 +1 @@ -cibuildwheel==2.21.3 +cibuildwheel==2.22.0 From cc6772e01d523f7103a96ae99cc980f32d0ffcb9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 10:45:53 +0100 Subject: [PATCH 34/78] Bump sphinx-rtd-theme from 3.0.1 to 3.0.2 (#2069) Bumps [sphinx-rtd-theme](https://github.com/readthedocs/sphinx_rtd_theme) from 3.0.1 to 3.0.2. - [Changelog](https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/changelog.rst) - [Commits](https://github.com/readthedocs/sphinx_rtd_theme/compare/3.0.1...3.0.2) --- updated-dependencies: - dependency-name: sphinx-rtd-theme dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index d614584fc..fd3033b91 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ sphinx >= 2.2 -sphinx-rtd-theme==3.0.1 +sphinx-rtd-theme==3.0.2 sphinx-autoapi sphinx-autodoc-typehints sphinxcontrib-autoprogram From 6b8f06a9f6f6a570142c7aedc767fea2efa2a0cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 10:47:09 +0100 Subject: [PATCH 35/78] Bump pypa/cibuildwheel from 2.21.3 to 2.22.0 (#2076) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.21.3 to 2.22.0. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.21.3...v2.22.0) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index fb54729d4..ed10cd17d 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -43,7 +43,7 @@ jobs: # platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.21.3 + uses: pypa/cibuildwheel@v2.22.0 env: CIBW_BUILD: ${{ matrix.build }} CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} @@ -99,7 +99,7 @@ jobs: ref: ${{ github.event.client_payload.ref }} - name: Build wheels - uses: pypa/cibuildwheel@v2.21.3 + uses: pypa/cibuildwheel@v2.22.0 - uses: actions/upload-artifact@v4 with: From f1d192dd2b28902fd0098e133c2ef241557d27a8 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Sat, 30 Nov 2024 13:07:55 +0100 Subject: [PATCH 36/78] argpase: colorize the --help output with rich-argparse --- .github/workflows/ci-tests.yml | 2 +- cwltool/argparser.py | 352 ++++++++++++++++++--------------- cwltool/main.py | 7 +- pyproject.toml | 1 + requirements.txt | 1 + setup.py | 1 + tests/test_environment.py | 12 +- tests/test_misc_cli.py | 10 +- tests/test_singularity.py | 3 +- 9 files changed, 215 insertions(+), 174 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 6608e9a22..1f01160c8 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -157,7 +157,7 @@ jobs: chmod a-w . - name: run tests - run: APPTAINER_TMPDIR=${RUNNER_TEMP} make test + run: APPTAINER_TMPDIR=${RUNNER_TEMP} make test PYTEST_EXTRA=-vvv conformance_tests: diff --git a/cwltool/argparser.py b/cwltool/argparser.py index 7b3125d94..3622b627f 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -6,6 +6,9 @@ from collections.abc import MutableMapping, MutableSequence, Sequence from typing import Any, Callable, Optional, Union, cast +import rich.markup +from rich_argparse import HelpPreviewAction, RichHelpFormatter + from .loghandler import _logger from .process import Process, shortname from .resolver import ga4gh_tool_registries @@ -14,9 +17,11 @@ def arg_parser() -> argparse.ArgumentParser: + RichHelpFormatter.group_name_formatter = str parser = argparse.ArgumentParser( + formatter_class=RichHelpFormatter, description="Reference executor for Common Workflow Language standards. " - "Not for production use." + "Not for production use.", ) parser.add_argument("--basedir", type=str) parser.add_argument( @@ -26,23 +31,15 @@ def arg_parser() -> argparse.ArgumentParser: help="Output directory. The default is the current directory.", ) - parser.add_argument( - "--log-dir", - type=str, - default="", - help="Log your tools stdout/stderr to this location outside of container " - "This will only log stdout/stderr if you specify stdout/stderr in their " - "respective fields or capture it as an output", - ) - parser.add_argument( "--parallel", action="store_true", default=False, help="Run jobs in parallel. ", ) - envgroup = parser.add_mutually_exclusive_group() - envgroup.add_argument( + envgroup = parser.add_argument_group(title="Control environment variables") + env_exclusive = envgroup.add_mutually_exclusive_group() + env_exclusive.add_argument( "--preserve-environment", type=str, action="append", @@ -53,7 +50,7 @@ def arg_parser() -> argparse.ArgumentParser: default=[], dest="preserve_environment", ) - envgroup.add_argument( + env_exclusive.add_argument( "--preserve-entire-environment", action="store_true", help="Preserve all environment variables when running CommandLineTools " @@ -62,54 +59,10 @@ def arg_parser() -> argparse.ArgumentParser: dest="preserve_entire_environment", ) - containergroup = parser.add_mutually_exclusive_group() - containergroup.add_argument( - "--rm-container", - action="store_true", - default=True, - help="Delete Docker container used by jobs after they exit (default)", - dest="rm_container", - ) - - containergroup.add_argument( - "--leave-container", - action="store_false", - default=True, - help="Do not delete Docker container used by jobs after they exit", - dest="rm_container", - ) - - cidgroup = parser.add_argument_group( - "Options for recording the Docker container identifier into a file." - ) - cidgroup.add_argument( - # Disabled as containerid is now saved by default - "--record-container-id", - action="store_true", - default=False, - help=argparse.SUPPRESS, - dest="record_container_id", - ) - - cidgroup.add_argument( - "--cidfile-dir", - type=str, - help="Store the Docker container ID into a file in the specified directory.", - default=None, - dest="cidfile_dir", - ) - - cidgroup.add_argument( - "--cidfile-prefix", - type=str, - help="Specify a prefix to the container ID filename. " - "Final file name will be followed by a timestamp. " - "The default is no prefix.", - default=None, - dest="cidfile_prefix", + files_group = parser.add_argument_group( + title="Manage intermediate, temporary, or final output files" ) - - parser.add_argument( + files_group.add_argument( "--tmpdir-prefix", type=str, help="Path prefix for temporary directories. If --tmpdir-prefix is not " @@ -119,7 +72,7 @@ def arg_parser() -> argparse.ArgumentParser: default=DEFAULT_TMP_PREFIX, ) - intgroup = parser.add_mutually_exclusive_group() + intgroup = files_group.add_mutually_exclusive_group() intgroup.add_argument( "--tmp-outdir-prefix", type=str, @@ -137,7 +90,7 @@ def arg_parser() -> argparse.ArgumentParser: "troubleshooting of CWL documents.", ) - tmpgroup = parser.add_mutually_exclusive_group() + tmpgroup = files_group.add_mutually_exclusive_group() tmpgroup.add_argument( "--rm-tmpdir", action="store_true", @@ -154,7 +107,7 @@ def arg_parser() -> argparse.ArgumentParser: dest="rm_tmpdir", ) - outgroup = parser.add_mutually_exclusive_group() + outgroup = files_group.add_mutually_exclusive_group() outgroup.add_argument( "--move-outputs", action="store_const", @@ -184,30 +137,6 @@ def arg_parser() -> argparse.ArgumentParser: dest="move_outputs", ) - pullgroup = parser.add_mutually_exclusive_group() - pullgroup.add_argument( - "--enable-pull", - default=True, - action="store_true", - help="Try to pull Docker images", - dest="pull_image", - ) - - pullgroup.add_argument( - "--disable-pull", - default=True, - action="store_false", - help="Do not try to pull Docker images", - dest="pull_image", - ) - - parser.add_argument( - "--rdf-serializer", - help="Output RDF serialization format used by --print-rdf (one of " - "turtle (default), n3, nt, xml)", - default="turtle", - ) - parser.add_argument( "--eval-timeout", help="Time to wait for a Javascript expression to evaluate before giving " @@ -216,9 +145,7 @@ def arg_parser() -> argparse.ArgumentParser: default=60, ) - provgroup = parser.add_argument_group( - "Options for recording provenance information of the execution" - ) + provgroup = parser.add_argument_group("Recording provenance information of the execution") provgroup.add_argument( "--provenance", help="Save provenance to specified folder as a " @@ -276,7 +203,8 @@ def arg_parser() -> argparse.ArgumentParser: type=str, ) - printgroup = parser.add_mutually_exclusive_group() + non_exec_group = parser.add_argument_group(title="Non-execution options") + printgroup = non_exec_group.add_mutually_exclusive_group() printgroup.add_argument( "--print-rdf", action="store_true", @@ -324,6 +252,15 @@ def arg_parser() -> argparse.ArgumentParser: printgroup.add_argument( "--make-template", action="store_true", help="Generate a template input object" ) + non_exec_group.add_argument( + "--rdf-serializer", + help="Output RDF serialization format used by --print-rdf (one of " + "turtle (default), n3, nt, xml)", + default="turtle", + ) + non_exec_group.add_argument( + "--tool-help", action="store_true", help="Print command line help for tool" + ) strictgroup = parser.add_mutually_exclusive_group() strictgroup.add_argument( @@ -365,11 +302,27 @@ def arg_parser() -> argparse.ArgumentParser: dest="doc_cache", ) - volumegroup = parser.add_mutually_exclusive_group() - volumegroup.add_argument("--verbose", action="store_true", help="Default logging") - volumegroup.add_argument("--no-warnings", action="store_true", help="Only print errors.") - volumegroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.") - volumegroup.add_argument("--debug", action="store_true", help="Print even more logging") + volumegroup = parser.add_argument_group(title="Configure logging") + volume_exclusive = volumegroup.add_mutually_exclusive_group() + volume_exclusive.add_argument("--verbose", action="store_true", help="Default logging") + volume_exclusive.add_argument("--no-warnings", action="store_true", help="Only print errors.") + volume_exclusive.add_argument( + "--quiet", action="store_true", help="Only print warnings and errors." + ) + volume_exclusive.add_argument("--debug", action="store_true", help="Print even more logging") + volumegroup.add_argument( + "--log-dir", + type=str, + default="", + help="Log your tools stdout/stderr to this location outside of container " + "This will only log stdout/stderr if you specify stdout/stderr in their " + "respective fields or capture it as an output", + ) + volumegroup.add_argument( + "--timestamps", + action="store_true", + help="Add timestamps to the errors, warnings, and notifications.", + ) parser.add_argument( "--write-summary", @@ -380,30 +333,6 @@ def arg_parser() -> argparse.ArgumentParser: dest="write_summary", ) - parser.add_argument( - "--strict-memory-limit", - action="store_true", - help="When running with " - "software containers and the Docker engine, pass either the " - "calculated memory allocation from ResourceRequirements or the " - "default of 1 gigabyte to Docker's --memory option.", - ) - - parser.add_argument( - "--strict-cpu-limit", - action="store_true", - help="When running with " - "software containers and the Docker engine, pass either the " - "calculated cpu allocation from ResourceRequirements or the " - "default of 1 core to Docker's --cpu option. " - "Requires docker version >= v1.13.", - ) - - parser.add_argument( - "--timestamps", - action="store_true", - help="Add timestamps to the errors, warnings, and notifications.", - ) parser.add_argument( "--js-console", action="store_true", help="Enable javascript console output" ) @@ -418,7 +347,105 @@ def arg_parser() -> argparse.ArgumentParser: help="File of options to pass to jshint. " 'This includes the added option "includewarnings". ', ) - dockergroup = parser.add_mutually_exclusive_group() + container_group = parser.add_argument_group( + title="Software container engine selection and configuration" + ) + pullgroup = container_group.add_mutually_exclusive_group() + pullgroup.add_argument( + "--enable-pull", + default=True, + action="store_true", + help="Try to pull Docker images", + dest="pull_image", + ) + + pullgroup.add_argument( + "--disable-pull", + default=True, + action="store_false", + help="Do not try to pull Docker images", + dest="pull_image", + ) + container_group.add_argument( + "--force-docker-pull", + action="store_true", + default=False, + help="Pull latest software container image even if it is locally present", + dest="force_docker_pull", + ) + container_group.add_argument( + "--no-read-only", + action="store_true", + default=False, + help="Do not set root directory in the container as read-only", + dest="no_read_only", + ) + + container_group.add_argument( + "--default-container", + help="Specify a default software container to use for any " + "CommandLineTool without a DockerRequirement.", + ) + container_group.add_argument( + "--no-match-user", + action="store_true", + help="Disable passing the current uid to `docker run --user`", + ) + container_group.add_argument( + "--custom-net", + type=str, + help="Passed to `docker run` as the `--net` parameter when " + "NetworkAccess is true, which is its default setting.", + ) + + container_cleanup = container_group.add_mutually_exclusive_group() + container_cleanup.add_argument( + "--rm-container", + action="store_true", + default=True, + help="Delete Docker container used by jobs after they exit (default)", + dest="rm_container", + ) + + container_cleanup.add_argument( + "--leave-container", + action="store_false", + default=True, + help="Do not delete Docker container used by jobs after they exit", + dest="rm_container", + ) + + cidgroup = container_group.add_argument_group( + "Recording the Docker container identifier into a file" + ) + cidgroup.add_argument( + # Disabled as containerid is now saved by default + "--record-container-id", + action="store_true", + default=False, + help=argparse.SUPPRESS, + dest="record_container_id", + ) + + cidgroup.add_argument( + "--cidfile-dir", + type=str, + help="Store the Docker container ID into a file in the specified directory.", + default=None, + dest="cidfile_dir", + ) + + cidgroup.add_argument( + "--cidfile-prefix", + type=str, + help="Specify a prefix to the container ID filename. " + "Final file name will be followed by a timestamp. " + "The default is no prefix.", + default=None, + dest="cidfile_prefix", + ) + + dockergroup = container_group.add_mutually_exclusive_group() dockergroup.add_argument( "--user-space-docker-cmd", metavar="CMD", @@ -458,6 +485,24 @@ def arg_parser() -> argparse.ArgumentParser: "is specified under `hints`.", dest="use_container", ) + container_group.add_argument( + "--strict-memory-limit", + action="store_true", + help="When running with " + "software containers and the Docker engine, pass either the " + "calculated memory allocation from ResourceRequirements or the " + "default of 1 gigabyte to Docker's --memory option.", + ) + + container_group.add_argument( + "--strict-cpu-limit", + action="store_true", + help="When running with " + "software containers and the Docker engine, pass either the " + "calculated cpu allocation from ResourceRequirements or the " + "default of 1 core to Docker's --cpu option. " + "Requires docker version >= v1.13.", + ) dependency_resolvers_configuration_help = argparse.SUPPRESS dependencies_directory_help = argparse.SUPPRESS @@ -467,7 +512,7 @@ def arg_parser() -> argparse.ArgumentParser: if SOFTWARE_REQUIREMENTS_ENABLED: dependency_resolvers_configuration_help = ( "Dependency resolver " - "configuration file describing how to adapt 'SoftwareRequirement' " + "configuration file describing how to adapt `SoftwareRequirement` " "packages to current system." ) dependencies_directory_help = ( @@ -476,7 +521,7 @@ def arg_parser() -> argparse.ArgumentParser: use_biocontainers_help = ( "Use biocontainers for tools without an " "explicitly annotated Docker container." ) - conda_dependencies = "Short cut to use Conda to resolve 'SoftwareRequirement' packages." + conda_dependencies = "Short cut to use Conda to resolve `SoftwareRequirement` packages." parser.add_argument( "--beta-dependency-resolvers-configuration", @@ -499,8 +544,6 @@ def arg_parser() -> argparse.ArgumentParser: action="store_true", ) - parser.add_argument("--tool-help", action="store_true", help="Print command line help for tool") - parser.add_argument( "--relative-deps", choices=["primary", "cwd"], @@ -519,7 +562,7 @@ def arg_parser() -> argparse.ArgumentParser: parser.add_argument( "--enable-ext", action="store_true", - help="Enable loading and running 'cwltool:' extensions to the CWL standards.", + help="Enable loading and running `cwltool:` extensions to the CWL standards.", default=False, ) @@ -537,22 +580,6 @@ def arg_parser() -> argparse.ArgumentParser: help="Disable colored logging (default false)", ) - parser.add_argument( - "--default-container", - help="Specify a default software container to use for any " - "CommandLineTool without a DockerRequirement.", - ) - parser.add_argument( - "--no-match-user", - action="store_true", - help="Disable passing the current uid to `docker run --user`", - ) - parser.add_argument( - "--custom-net", - type=str, - help="Passed to `docker run` as the '--net' parameter when " - "NetworkAccess is true, which is its default setting.", - ) parser.add_argument( "--disable-validate", dest="do_validate", @@ -595,9 +622,9 @@ def arg_parser() -> argparse.ArgumentParser: parser.add_argument( "--on-error", - help="Desired workflow behavior when a step fails. One of 'stop' (do " - "not submit any more steps) or 'continue' (may submit other steps that " - "are not downstream from the error). Default is 'stop'.", + help="Desired workflow behavior when a step fails. One of `stop` (do " + "not submit any more steps) or `continue` (may submit other steps that " + "are not downstream from the error). Default is `stop`.", default="stop", choices=("stop", "continue"), ) @@ -625,21 +652,6 @@ def arg_parser() -> argparse.ArgumentParser: dest="relax_path_checks", ) - parser.add_argument( - "--force-docker-pull", - action="store_true", - default=False, - help="Pull latest software container image even if it is locally present", - dest="force_docker_pull", - ) - parser.add_argument( - "--no-read-only", - action="store_true", - default=False, - help="Do not set root directory in the container as read-only", - dest="no_read_only", - ) - parser.add_argument( "--overrides", type=str, @@ -647,7 +659,8 @@ def arg_parser() -> argparse.ArgumentParser: help="Read process requirement overrides from file.", ) - subgroup = parser.add_mutually_exclusive_group() + target_group = parser.add_argument_group(title="Target selection (optional)") + subgroup = target_group.add_mutually_exclusive_group() subgroup.add_argument( "--target", "-t", @@ -668,8 +681,8 @@ def arg_parser() -> argparse.ArgumentParser: default=None, help="Only executes the underlying Process (CommandLineTool, " "ExpressionTool, or sub-Workflow) for the given step in a workflow. " - "This will not include any step-level processing: 'scatter', 'when'; " - "and there will be no processing of step-level 'default', or 'valueFrom' " + "This will not include any step-level processing: `scatter`, `when`; " + "and there will be no processing of step-level `default`, or `valueFrom` " "input modifiers. However, requirements/hints from the step or parent " "workflow(s) will be inherited as usual." "The input object must match that Process's inputs.", @@ -703,7 +716,11 @@ def arg_parser() -> argparse.ArgumentParser: "formatted description of the required input values for the given " "`cwl_document`.", ) - + parser.add_argument( + "--generate-help-preview", + action=HelpPreviewAction, + path="help-preview.svg", # (optional) or "help-preview.html" or "help-preview.txt" + ) return parser @@ -855,6 +872,7 @@ def add_argument( urljoin: Callable[[str, str], str] = urllib.parse.urljoin, base_uri: str = "", ) -> None: + description = rich.markup.escape(description) if len(name) == 1: flag = "-" else: @@ -980,4 +998,10 @@ def generate_parser( base_uri, ) + toolparser.add_argument( + "--generate-help-preview", + action=HelpPreviewAction, + path="help-preview.svg", # (optional) or "help-preview.html" or "help-preview.txt" + ) + return toolparser diff --git a/cwltool/main.py b/cwltool/main.py index 17ccb11ce..b7ba40d40 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -22,6 +22,7 @@ import coloredlogs import requests import ruamel.yaml +from rich_argparse import RichHelpFormatter from ruamel.yaml.comments import CommentedMap, CommentedSeq from ruamel.yaml.main import YAML from schema_salad.exceptions import ValidationException @@ -413,7 +414,10 @@ def init_job_order( namemap: dict[str, str] = {} records: list[str] = [] toolparser = generate_parser( - argparse.ArgumentParser(prog=args.workflow), + argparse.ArgumentParser( + prog=args.workflow, + formatter_class=RichHelpFormatter, + ), process, namemap, records, @@ -976,6 +980,7 @@ def main( user_agent += f" {progname}" # append the real program name as well append_word_to_default_user_agent(user_agent) + err_handler: logging.Handler = defaultStreamHandler try: if args is None: if argsl is None: diff --git a/pyproject.toml b/pyproject.toml index 7ddf547f2..248c0e69d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ requires = [ "cwl-utils>=0.32", "toml", "argcomplete>=1.12.0", + "rich-argparse" ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index 3ac631838..3cbcf0027 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ argcomplete>=1.12.0 pyparsing!=3.0.2 # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 cwl-utils>=0.32 spython>=0.3.0 +rich-argparse diff --git a/setup.py b/setup.py index 9bbee10f1..d3fef7b26 100644 --- a/setup.py +++ b/setup.py @@ -135,6 +135,7 @@ "pyparsing != 3.0.2", # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 "cwl-utils >= 0.32", "spython >= 0.3.0", + "rich-argparse", ], extras_require={ "deps": [ diff --git a/tests/test_environment.py b/tests/test_environment.py index 488477aa7..4e9c602f1 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from collections.abc import Mapping from pathlib import Path -from typing import Any, Callable, Union +from typing import Callable, Union import pytest @@ -198,7 +198,7 @@ def BIND(v: str, env: Env) -> bool: @CRT_PARAMS -def test_basic(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> None: +def test_basic(crt_params: CheckHolder, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Test that basic env vars (only) show up.""" tmp_prefix = str(tmp_path / "canary") extra_env = { @@ -218,7 +218,9 @@ def test_basic(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> Non @CRT_PARAMS -def test_preserve_single(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> None: +def test_preserve_single( + crt_params: CheckHolder, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: """Test that preserving a single env var works.""" tmp_prefix = str(tmp_path / "canary") extra_env = { @@ -242,7 +244,9 @@ def test_preserve_single(crt_params: CheckHolder, tmp_path: Path, monkeypatch: A @CRT_PARAMS -def test_preserve_all(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> None: +def test_preserve_all( + crt_params: CheckHolder, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: """Test that preserving all works.""" tmp_prefix = str(tmp_path / "canary") extra_env = { diff --git a/tests/test_misc_cli.py b/tests/test_misc_cli.py index 307153e16..be314cad4 100644 --- a/tests/test_misc_cli.py +++ b/tests/test_misc_cli.py @@ -1,5 +1,7 @@ """Tests for various command line options.""" +import pytest + from cwltool.utils import versionstring from .util import get_data, get_main_output, needs_docker @@ -26,9 +28,13 @@ def test_empty_cmdling() -> None: assert "CWL document required, no input file was provided" in stderr -def test_tool_help() -> None: +def test_tool_help(monkeypatch: pytest.MonkeyPatch) -> None: """Test --tool-help.""" - return_code, stdout, stderr = get_main_output(["--tool-help", get_data("tests/echo.cwl")]) + return_code, stdout, stderr = get_main_output( + ["--tool-help", get_data("tests/echo.cwl")], + extra_env={"NO_COLOR": "1"}, + monkeypatch=monkeypatch, + ) assert return_code == 0 assert "job_order Job input json file" in stdout diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 0512f2e28..1139dfbc7 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -2,7 +2,6 @@ import shutil from pathlib import Path -from typing import Any import pytest @@ -19,7 +18,7 @@ @needs_singularity_2_6 -def test_singularity_pullfolder(tmp_path: Path, monkeypatch: Any) -> None: +def test_singularity_pullfolder(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Test singularity respects SINGULARITY_PULLFOLDER.""" workdir = tmp_path / "working_dir_new" workdir.mkdir() From 527884b7ae7a815798903887f8f58b89bd0bc2a9 Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Mon, 16 Dec 2024 17:34:50 -0500 Subject: [PATCH 37/78] Proposal: Improved `ProvenanceProfile` definition (#2082) Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- .gitignore | 1 + cwltool/context.py | 2 + cwltool/cwlprov/provenance_profile.py | 45 +------- cwltool/cwlprov/ro.py | 97 ++++++++++++++++-- cwltool/executors.py | 11 +- cwltool/main.py | 5 + cwltool/workflow.py | 3 +- mypy-stubs/rdflib/graph.pyi | 4 +- tests/test_provenance.py | 142 +++++++++++++++++++++----- 9 files changed, 225 insertions(+), 85 deletions(-) diff --git a/.gitignore b/.gitignore index fbe4b24fc..b4cab0e66 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ eggs/ *.egg .tox/ .pytest_cache +*.so # Editor Temps .*.sw? diff --git a/cwltool/context.py b/cwltool/context.py index 237a90968..bb281fd88 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -183,6 +183,8 @@ def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: self.orcid: str = "" self.cwl_full_name: str = "" self.process_run_id: Optional[str] = None + self.prov_host: bool = False + self.prov_user: bool = False self.prov_obj: Optional[ProvenanceProfile] = None self.mpi_config: MpiConfig = MpiConfig() self.default_stdout: Optional[Union[IO[bytes], TextIO]] = None diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index d4dfd6cb4..e8538e51b 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -6,7 +6,6 @@ from collections.abc import MutableMapping, MutableSequence, Sequence from io import BytesIO from pathlib import PurePath, PurePosixPath -from socket import getfqdn from typing import TYPE_CHECKING, Any, Optional, Union, cast from prov.identifier import Identifier, QualifiedName @@ -24,12 +23,10 @@ ACCOUNT_UUID, CWLPROV, ENCODING, - FOAF, METADATA, ORE, PROVENANCE, RO, - SCHEMA, SHA1, SHA256, TEXT_PLAIN, @@ -108,25 +105,6 @@ def __str__(self) -> str: def generate_prov_doc(self) -> tuple[str, ProvDocument]: """Add basic namespaces.""" - - def host_provenance(document: ProvDocument) -> None: - """Record host provenance.""" - document.add_namespace(CWLPROV) - document.add_namespace(UUID) - document.add_namespace(FOAF) - - hostname = getfqdn() - # won't have a foaf:accountServiceHomepage for unix hosts, but - # we can at least provide hostname - document.agent( - ACCOUNT_UUID, - { - PROV_TYPE: FOAF["OnlineAccount"], - "prov:location": hostname, - CWLPROV["hostname"]: hostname, - }, - ) - self.cwltool_version = f"cwltool {versionstring().split()[-1]}" self.document.add_namespace("wfprov", "http://purl.org/wf4ever/wfprov#") # document.add_namespace('prov', 'http://www.w3.org/ns/prov#') @@ -165,25 +143,10 @@ def host_provenance(document: ProvDocument) -> None: # .. but we always know cwltool was launched (directly or indirectly) # by a user account, as cwltool is a command line tool account = self.document.agent(ACCOUNT_UUID) - if self.orcid or self.full_name: - person: dict[Union[str, Identifier], Any] = { - PROV_TYPE: PROV["Person"], - "prov:type": SCHEMA["Person"], - } - if self.full_name: - person["prov:label"] = self.full_name - person["foaf:name"] = self.full_name - person["schema:name"] = self.full_name - else: - # TODO: Look up name from ORCID API? - pass - agent = self.document.agent(self.orcid or uuid.uuid4().urn, person) - self.document.actedOnBehalfOf(account, agent) - else: - if self.host_provenance: - host_provenance(self.document) - if self.user_provenance: - self.research_object.user_provenance(self.document) + if self.host_provenance: + self.research_object.host_provenance(self.document) + if self.user_provenance or self.orcid or self.full_name: + self.research_object.user_provenance(self.document) # The execution of cwltool wfengine = self.document.agent( self.engine_uuid, diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index ac60afc92..f58919a6b 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -9,10 +9,11 @@ import uuid from collections.abc import MutableMapping, MutableSequence from pathlib import Path, PurePosixPath -from typing import IO, Any, Optional, Union, cast +from socket import getfqdn +from typing import IO, TYPE_CHECKING, Any, Optional, Union, cast import prov.model as provM -from prov.model import PROV, ProvDocument +from prov.model import ProvDocument from ..loghandler import _logger from ..stdfsaccess import StdFsAccess @@ -27,6 +28,7 @@ from . import Aggregate, Annotation, AuthoredBy, _valid_orcid, _whoami, checksum_copy from .provenance_constants import ( ACCOUNT_UUID, + CWLPROV, CWLPROV_VERSION, DATA, ENCODING, @@ -35,6 +37,7 @@ METADATA, ORCID, PROVENANCE, + SCHEMA, SHA1, SHA256, SHA512, @@ -46,6 +49,9 @@ Hasher, ) +if TYPE_CHECKING: + from .provenance_profile import ProvenanceProfile # pylint: disable=unused-import + class ResearchObject: """CWLProv Research Object.""" @@ -82,6 +88,34 @@ def __init__( self._initialize() _logger.debug("[provenance] Temporary research object: %s", self.folder) + def initialize_provenance( + self, + full_name: str, + host_provenance: bool, + user_provenance: bool, + orcid: str, + fsaccess: StdFsAccess, + run_uuid: Optional[uuid.UUID] = None, + ) -> "ProvenanceProfile": + """ + Provide a provenance profile initialization hook function. + + Allows overriding the default strategy to define the + provenance profile concepts and associations to extend + details as needed. + """ + from .provenance_profile import ProvenanceProfile + + return ProvenanceProfile( + research_object=self, + full_name=full_name, + host_provenance=host_provenance, + user_provenance=user_provenance, + orcid=orcid, + fsaccess=fsaccess, + run_uuid=run_uuid, + ) + def self_check(self) -> None: """Raise ValueError if this RO is closed.""" if self.closed: @@ -117,10 +151,22 @@ def _initialize_bagit(self) -> None: bag_it_file.write("BagIt-Version: 0.97\n") bag_it_file.write(f"Tag-File-Character-Encoding: {ENCODING}\n") + def resolve_user(self) -> tuple[str, str]: + """ + Provide a user provenance hook function. + + Allows overriding the default strategy to retrieve user provenance + in case the calling code can provide a better resolution. + The function must return a tuple of the (username, fullname) + that identifies the user. This user will be applied on top + to any provided ORCID or fullname by agent association. + """ + return _whoami() + def user_provenance(self, document: ProvDocument) -> None: """Add the user provenance.""" self.self_check() - (username, fullname) = _whoami() + (username, fullname) = self.resolve_user() if not self.full_name: self.full_name = fullname @@ -132,19 +178,21 @@ def user_provenance(self, document: ProvDocument) -> None: ACCOUNT_UUID, { provM.PROV_TYPE: FOAF["OnlineAccount"], - "prov:label": username, + provM.PROV_LABEL: username, FOAF["accountName"]: username, }, ) user = document.agent( self.orcid or USER_UUID, - { - provM.PROV_TYPE: PROV["Person"], - "prov:label": self.full_name, - FOAF["name"]: self.full_name, - FOAF["account"]: account, - }, + [ + (provM.PROV_TYPE, SCHEMA["Person"]), + (provM.PROV_TYPE, provM.PROV["Person"]), + (provM.PROV_LABEL, self.full_name), + (FOAF["name"], self.full_name), + (FOAF["account"], account), + (SCHEMA["name"], self.full_name), + ], ) # cwltool may be started on the shell (directly by user), # by shell script (indirectly by user) @@ -156,6 +204,35 @@ def user_provenance(self, document: ProvDocument) -> None: # get their name wrong!) document.actedOnBehalfOf(account, user) + def resolve_host(self) -> tuple[str, str]: + """ + Provide a host provenance hook function. + + Allows overriding the default strategy to retrieve host provenance + in case the calling code can provide a better resolution. + The function must return a tuple of the (fqdn, uri) that identifies the host. + """ + fqdn = getfqdn() + return fqdn, fqdn # allow for (fqdn, uri) to be distinct, but the same by default + + def host_provenance(self, document: ProvDocument) -> None: + """Record host provenance.""" + document.add_namespace(CWLPROV) + document.add_namespace(UUID) + document.add_namespace(FOAF) + + hostname, uri = self.resolve_host() + # won't have a foaf:accountServiceHomepage for unix hosts, but + # we can at least provide hostname + document.agent( + ACCOUNT_UUID, + { + provM.PROV_TYPE: FOAF["OnlineAccount"], + provM.PROV_LOCATION: uri, + CWLPROV["hostname"]: hostname, + }, + ) + def add_tagfile(self, path: str, timestamp: Optional[datetime.datetime] = None) -> None: """Add tag files to our research object.""" self.self_check() diff --git a/cwltool/executors.py b/cwltool/executors.py index e25426c9d..33198d854 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -19,7 +19,6 @@ from .command_line_tool import CallbackJob, ExpressionJob from .context import RuntimeContext, getdefault from .cuda import cuda_version_and_device_count -from .cwlprov.provenance_profile import ProvenanceProfile from .errors import WorkflowException from .job import JobBase from .loghandler import _logger @@ -194,11 +193,13 @@ def run_jobs( # define provenance profile for single commandline tool if not isinstance(process, Workflow) and runtime_context.research_obj is not None: - process.provenance_object = ProvenanceProfile( - runtime_context.research_obj, + process.provenance_object = runtime_context.research_obj.initialize_provenance( full_name=runtime_context.cwl_full_name, - host_provenance=False, - user_provenance=False, + # following are only set from main when directly command line tool + # when nested in a workflow, they should be disabled since they would + # already have been provided/initialized by the parent workflow prov-obj + host_provenance=runtime_context.prov_host, + user_provenance=runtime_context.prov_user, orcid=runtime_context.orcid, # single tool execution, so RO UUID = wf UUID = tool UUID run_uuid=runtime_context.research_obj.ro_uuid, diff --git a/cwltool/main.py b/cwltool/main.py index b7ba40d40..a137d8a4f 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -1065,6 +1065,11 @@ def main( loadingContext = setup_loadingContext(loadingContext, runtimeContext, args) + if loadingContext.research_obj: + # early forward parameters required for a single command line tool + runtimeContext.prov_host = loadingContext.host_provenance + runtimeContext.prov_user = loadingContext.user_provenance + uri, tool_file_uri = resolve_tool_uri( args.workflow, resolver=loadingContext.resolver, diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 3bf32251f..899ac4643 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -72,8 +72,7 @@ def __init__( if is_main: run_uuid = loadingContext.research_obj.ro_uuid - self.provenance_object = ProvenanceProfile( - loadingContext.research_obj, + self.provenance_object = loadingContext.research_obj.initialize_provenance( full_name=loadingContext.cwl_full_name, host_provenance=loadingContext.host_provenance, user_provenance=loadingContext.user_provenance, diff --git a/mypy-stubs/rdflib/graph.pyi b/mypy-stubs/rdflib/graph.pyi index d3e6f2f54..9764972b2 100644 --- a/mypy-stubs/rdflib/graph.pyi +++ b/mypy-stubs/rdflib/graph.pyi @@ -16,7 +16,7 @@ from rdflib import query from rdflib.collection import Collection from rdflib.paths import Path from rdflib.resource import Resource -from rdflib.term import BNode, Identifier, Node +from rdflib.term import BNode, Identifier, Literal, Node class Graph(Node): base: Any = ... @@ -66,7 +66,7 @@ class Graph(Node): ) -> Iterable[Node]: ... def objects( self, subject: Optional[Any] = ..., predicate: Optional[Any] = ... - ) -> Iterable[Identifier]: ... + ) -> Iterable[Union[Identifier, Literal]]: ... def subject_predicates(self, object: Optional[Any] = ...) -> None: ... def subject_objects(self, predicate: Optional[Any] = ...) -> None: ... def predicate_objects(self, subject: Optional[Any] = ...) -> None: ... diff --git a/tests/test_provenance.py b/tests/test_provenance.py index e8d8416be..d7a2a698b 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -32,12 +32,23 @@ SCHEMA = Namespace("http://schema.org/") CWLPROV = Namespace("https://w3id.org/cwl/prov#") OA = Namespace("http://www.w3.org/ns/oa#") +FOAF = Namespace("http://xmlns.com/foaf/0.1/") -def cwltool(tmp_path: Path, *args: Any) -> Path: +TEST_ORCID = "https://orcid.org/0000-0003-4862-3349" + + +def cwltool(tmp_path: Path, *args: Any, with_orcid: bool = False) -> Path: prov_folder = tmp_path / "provenance" prov_folder.mkdir() - new_args = ["--provenance", str(prov_folder)] + new_args = [ + "--enable-user-provenance", + "--enable-host-provenance", + "--provenance", + str(prov_folder), + ] + if with_orcid: + new_args.extend(["--orcid", TEST_ORCID]) new_args.extend(args) # Run within a temporary directory to not pollute git checkout tmp_dir = tmp_path / "cwltool-run" @@ -49,61 +60,81 @@ def cwltool(tmp_path: Path, *args: Any) -> Path: @needs_docker -def test_hello_workflow(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_hello_workflow(tmp_path: Path, with_orcid: bool) -> None: check_provenance( cwltool( tmp_path, get_data("tests/wf/hello-workflow.cwl"), "--usermessage", "Hello workflow", - ) + with_orcid=with_orcid, + ), + with_orcid=with_orcid, ) @needs_docker -def test_hello_single_tool(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_hello_single_tool(tmp_path: Path, with_orcid: bool) -> None: check_provenance( cwltool( tmp_path, get_data("tests/wf/hello_single_tool.cwl"), "--message", "Hello tool", + with_orcid=with_orcid, ), single_tool=True, + with_orcid=with_orcid, ) @needs_docker -def test_revsort_workflow(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_revsort_workflow(tmp_path: Path, with_orcid: bool) -> None: folder = cwltool( tmp_path, get_data("tests/wf/revsort.cwl"), get_data("tests/wf/revsort-job.json"), + with_orcid=with_orcid, ) check_output_object(folder) - check_provenance(folder) + check_provenance(folder, with_orcid=with_orcid) @needs_docker -def test_revsort_workflow_shortcut(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_revsort_workflow_shortcut(tmp_path: Path, with_orcid: bool) -> None: """Confirm that using 'cwl:tool' shortcut still snapshots the CWL files.""" folder = cwltool( tmp_path, get_data("tests/wf/revsort-job-shortcut.json"), + with_orcid=with_orcid, ) check_output_object(folder) - check_provenance(folder) + check_provenance(folder, with_orcid=with_orcid) assert not (folder / "snapshot" / "revsort-job-shortcut.json").exists() assert len(list((folder / "snapshot").iterdir())) == 4 @needs_docker -def test_nested_workflow(tmp_path: Path) -> None: - check_provenance(cwltool(tmp_path, get_data("tests/wf/nested.cwl")), nested=True) +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_nested_workflow(tmp_path: Path, with_orcid: bool) -> None: + check_provenance( + cwltool( + tmp_path, + get_data("tests/wf/nested.cwl"), + with_orcid=with_orcid, + ), + nested=True, + with_orcid=with_orcid, + ) @needs_docker -def test_secondary_files_implicit(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_secondary_files_implicit(tmp_path: Path, with_orcid: bool) -> None: file1 = tmp_path / "foo1.txt" file1idx = tmp_path / "foo1.txt.idx" @@ -113,13 +144,20 @@ def test_secondary_files_implicit(tmp_path: Path) -> None: f.write("bar") # secondary will be picked up by .idx - folder = cwltool(tmp_path, get_data("tests/wf/sec-wf.cwl"), "--file1", str(file1)) - check_provenance(folder, secondary_files=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/sec-wf.cwl"), + "--file1", + str(file1), + with_orcid=with_orcid, + ) + check_provenance(folder, secondary_files=True, with_orcid=with_orcid) check_secondary_files(folder) @needs_docker -def test_secondary_files_explicit(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_secondary_files_explicit(tmp_path: Path, with_orcid: bool) -> None: # Deliberately do NOT have common basename or extension file1dir = tmp_path / "foo" file1dir.mkdir() @@ -154,22 +192,33 @@ def test_secondary_files_explicit(tmp_path: Path) -> None: j = json.dumps(job, ensure_ascii=True) fp.write(j.encode("ascii")) - folder = cwltool(tmp_path, get_data("tests/wf/sec-wf.cwl"), str(jobJson)) - check_provenance(folder, secondary_files=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/sec-wf.cwl"), + str(jobJson), + with_orcid=with_orcid, + ) + check_provenance(folder, secondary_files=True, with_orcid=with_orcid) check_secondary_files(folder) @needs_docker -def test_secondary_files_output(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_secondary_files_output(tmp_path: Path, with_orcid: bool) -> None: # secondary will be picked up by .idx - folder = cwltool(tmp_path, get_data("tests/wf/sec-wf-out.cwl")) - check_provenance(folder, secondary_files=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/sec-wf-out.cwl"), + with_orcid=with_orcid, + ) + check_provenance(folder, secondary_files=True, with_orcid=with_orcid) # Skipped, not the same secondary files as above # self.check_secondary_files() @needs_docker -def test_directory_workflow(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_directory_workflow(tmp_path: Path, with_orcid: bool) -> None: dir2 = tmp_path / "dir2" dir2.mkdir() sha1 = { @@ -185,8 +234,14 @@ def test_directory_workflow(tmp_path: Path) -> None: with open(dir2 / x, "w", encoding="ascii") as f: f.write(x) - folder = cwltool(tmp_path, get_data("tests/wf/directory.cwl"), "--dir", str(dir2)) - check_provenance(folder, directory=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/directory.cwl"), + "--dir", + str(dir2), + with_orcid=with_orcid, + ) + check_provenance(folder, directory=True, with_orcid=with_orcid) # Output should include ls stdout of filenames a b c on each line file_list = ( @@ -209,10 +264,12 @@ def test_directory_workflow(tmp_path: Path) -> None: @needs_docker -def test_no_data_files(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_no_data_files(tmp_path: Path, with_orcid: bool) -> None: folder = cwltool( tmp_path, get_data("tests/wf/conditional_step_no_inputs.cwl"), + with_orcid=with_orcid, ) check_bagit(folder) @@ -263,6 +320,7 @@ def check_provenance( single_tool: bool = False, directory: bool = False, secondary_files: bool = False, + with_orcid: bool = False, ) -> None: check_folders(base_path) check_bagit(base_path) @@ -273,6 +331,7 @@ def check_provenance( single_tool=single_tool, directory=directory, secondary_files=secondary_files, + with_orcid=with_orcid, ) @@ -463,6 +522,7 @@ def check_prov( single_tool: bool = False, directory: bool = False, secondary_files: bool = False, + with_orcid: bool = False, ) -> None: prov_file = base_path / "metadata" / "provenance" / "primary.cwlprov.nt" assert prov_file.is_file(), f"Can't find {prov_file}" @@ -485,7 +545,6 @@ def check_prov( # the has_provenance annotations in manifest.json instead # run should have been started by a wf engine - engines = set(g.subjects(RDF.type, WFPROV.WorkflowEngine)) assert engines, "Could not find WorkflowEngine" assert len(engines) == 1, "Found too many WorkflowEngines: %s" % engines @@ -502,6 +561,39 @@ def check_prov( PROV.SoftwareAgent, ) in g, "Engine not declared as SoftwareAgent" + # run should be associated to the user + accounts = set(g.subjects(RDF.type, FOAF.OnlineAccount)) + assert len(accounts) == 1 + account = accounts.pop() + people = set(g.subjects(RDF.type, SCHEMA.Person)) + assert len(people) == 1, "Can't find associated person in workflow run" + person = people.pop() + if with_orcid: + assert person == URIRef(TEST_ORCID) + else: + account_names = set(g.objects(account, FOAF.accountName)) + assert len(account_names) == 1 + account_name = cast(Literal, account_names.pop()) + machine_user = provenance._whoami()[0] + assert account_name.value == machine_user + + # find the random UUID assigned to cwltool + tool_agents = set(g.subjects(RDF.type, PROV.SoftwareAgent)) + n_all_agents = 2 + len(tool_agents) + agents = set(g.subjects(RDF.type, PROV.Agent)) + assert ( + len(agents) == n_all_agents + ), "There should be 1 agent per tool (engine), 1 user agent, and 1 cwltool agent" + agents.remove(person) + agents.remove(engine) # the main tool + remain_agents = agents - tool_agents + assert len(remain_agents) == 1 + assert ( + account, + PROV.actedOnBehalfOf, + person, + ) in g, "Association of cwltool agent acting for user is missing" + if single_tool: activities = set(g.subjects(RDF.type, PROV.Activity)) assert len(activities) == 1, "Too many activities: %s" % activities From c6782ff11d345dcc86a79e1d582fbbaa8a61e8c1 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 17 Dec 2024 17:14:26 +0100 Subject: [PATCH 38/78] singularity: improve testing on version 4.x+ --- .github/workflows/ci-tests.yml | 16 ++++++++-------- tests/test_environment.py | 33 ++++++++++++++++----------------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 1f01160c8..2ebb14c5f 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -47,8 +47,8 @@ jobs: - name: Set up Singularity and environment-modules if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} @@ -134,8 +134,8 @@ jobs: - name: Set up Singularity and environment-modules run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. run: sudo usermod -c 'CI Runner' "$(whoami)" @@ -183,8 +183,8 @@ jobs: - name: Set up Singularity and environment-modules if: ${{ matrix.container == 'singularity' }} run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Singularity cache if: ${{ matrix.container == 'singularity' }} @@ -231,8 +231,8 @@ jobs: - name: Set up Singularity and environment-modules run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Set up Python uses: actions/setup-python@v5 diff --git a/tests/test_environment.py b/tests/test_environment.py index 4e9c602f1..fa29fb924 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -7,6 +7,7 @@ from typing import Callable, Union import pytest +from packaging.version import Version from cwltool.singularity import get_version @@ -133,33 +134,31 @@ def PWD(v: str, env: Env) -> bool: } # Singularity variables appear to be in flux somewhat. - version = get_version()[0] - vmajor = version[0] - assert vmajor == 3, "Tests only work for Singularity 3" - vminor = version[1] + version = Version(".".join(map(str, get_version()[0]))) + assert version >= Version("3"), "Tests only work for Singularity 3+" sing_vars: EnvChecks = { "SINGULARITY_CONTAINER": None, "SINGULARITY_NAME": None, } - if vminor < 5: + if version < Version("3.5"): sing_vars["SINGULARITY_APPNAME"] = None - if vminor >= 5: + if (version >= Version("3.5")) and (version < Version("3.6")): + sing_vars["SINGULARITY_INIT"] = "1" + if version >= Version("3.5"): sing_vars["PROMPT_COMMAND"] = None sing_vars["SINGULARITY_ENVIRONMENT"] = None - if vminor == 5: - sing_vars["SINGULARITY_INIT"] = "1" - elif vminor > 5: + if version >= Version("3.6"): sing_vars["SINGULARITY_COMMAND"] = "exec" - if vminor >= 7: - if vminor > 9: - sing_vars["SINGULARITY_BIND"] = "" - else: + if version >= Version("3.7"): + if version > Version("3.9"): + sing_vars["SINGULARITY_BIND"] = "" + else: - def BIND(v: str, env: Env) -> bool: - return v.startswith(tmp_prefix) and v.endswith(":/tmp") + def BIND(v: str, env: Env) -> bool: + return v.startswith(tmp_prefix) and v.endswith(":/tmp") - sing_vars["SINGULARITY_BIND"] = BIND - if vminor >= 10: + sing_vars["SINGULARITY_BIND"] = BIND + if version >= Version("3.10"): sing_vars["SINGULARITY_COMMAND"] = "run" sing_vars["SINGULARITY_NO_EVAL"] = None From ae5fae45dcf3af00a3303f63a237bbc88cf0f67e Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Fri, 20 Dec 2024 08:48:37 -0800 Subject: [PATCH 39/78] README: Python supported versions list was outdated --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index dce55fc95..7c6ffe22d 100644 --- a/README.rst +++ b/README.rst @@ -52,7 +52,7 @@ and provide comprehensive validation of CWL files as well as provide other tools related to working with CWL. ``cwltool`` is written and tested for -`Python `_ ``3.x {x = 6, 8, 9, 10, 11}`` +`Python `_ ``3.x {x = 9, 10, 11, 12, 13}`` The reference implementation consists of two packages. The ``cwltool`` package is the primary Python module containing the reference implementation in the From 6a61bed8973ecc62dc95a5544dda428d6e9e963b Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Fri, 20 Dec 2024 09:20:13 -0800 Subject: [PATCH 40/78] docs: render the CLI groups too --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index d569f5586..1c6021bf4 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -3,4 +3,4 @@ cwltool Command Line Options .. autoprogram:: cwltool.argparser:arg_parser() :prog: cwltool - + :groups: From f453cdce5956fe6581f5ccdcb8aacb8c4f29f6d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 07:13:14 +0000 Subject: [PATCH 41/78] Bump mypy from 1.13.0 to 1.14.0 Bumps [mypy](https://github.com/python/mypy) from 1.13.0 to 1.14.0. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.13.0...v1.14.0) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- mypy-requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 5f18fa03a..f5944d2e8 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.13.0 # also update pyproject.toml +mypy==1.14.0 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 cwltest diff --git a/pyproject.toml b/pyproject.toml index 248c0e69d..deb1adc27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.13.0", # also update mypy-requirements.txt + "mypy==1.14.0", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", From baa668bc96ade54607465d21bc6cfa15c9bff13c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Dec 2024 07:48:50 +0000 Subject: [PATCH 42/78] Bump mypy from 1.14.0 to 1.14.1 Bumps [mypy](https://github.com/python/mypy) from 1.14.0 to 1.14.1. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.14.0...v1.14.1) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- mypy-requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index f5944d2e8..760428998 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.14.0 # also update pyproject.toml +mypy==1.14.1 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 cwltest diff --git a/pyproject.toml b/pyproject.toml index deb1adc27..b243171fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.14.0", # also update mypy-requirements.txt + "mypy==1.14.1", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", From a67c898958f6affc8cb9de05fe87c9228a4fc63e Mon Sep 17 00:00:00 2001 From: stxue1 Date: Thu, 9 Jan 2025 10:37:05 -0800 Subject: [PATCH 43/78] Change caching pathmapper to respect the pathmapper factory --- cwltool/command_line_tool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index e201fb12b..775360016 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -807,10 +807,10 @@ def job( cachecontext.tmpdir = "/tmp" # nosec cachecontext.stagedir = "/stage" cachebuilder = self._init_job(job_order, cachecontext) - cachebuilder.pathmapper = PathMapper( + cachebuilder.pathmapper = self.make_path_mapper( cachebuilder.files, - runtimeContext.basedir, cachebuilder.stagedir, + runtimeContext, separateDirs=False, ) _check_adjust = partial(check_adjust, self.path_check_mode.value, cachebuilder) From 459958beece07569898879c9df7a12fba71cf781 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 22 Jan 2025 14:09:57 +0100 Subject: [PATCH 44/78] Various cleanups to make the docs better. --- cwltool/checker.py | 74 +++++++++++++++++++++--------------- cwltool/command_line_tool.py | 2 +- cwltool/errors.py | 7 ++++ cwltool/main.py | 2 +- cwltool/mutation.py | 24 +++++++----- cwltool/pathmapper.py | 50 +++++++++++------------- cwltool/process.py | 4 +- cwltool/subgraph.py | 41 +++++++++++--------- cwltool/utils.py | 3 -- cwltool/validate_js.py | 11 ++++-- docs/conf.py | 3 +- tests/test_mpi.py | 2 +- tests/util.py | 2 +- 13 files changed, 125 insertions(+), 100 deletions(-) diff --git a/cwltool/checker.py b/cwltool/checker.py index 17cba77ba..a3b8ba5df 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -1,8 +1,7 @@ """Static checking of CWL workflow connectivity.""" -from collections import namedtuple from collections.abc import Iterator, MutableMapping, MutableSequence, Sized -from typing import Any, Literal, Optional, Union, cast +from typing import Any, Literal, NamedTuple, Optional, Union, cast from schema_salad.exceptions import ValidationException from schema_salad.sourceline import SourceLine, bullets, strip_dup_lineno @@ -184,8 +183,8 @@ def static_checker( for param in workflow_inputs + step_outputs: src_dict[cast(str, param["id"])] = param - step_inputs_val = check_all_types(src_dict, step_inputs, "source", param_to_step) - workflow_outputs_val = check_all_types( + step_inputs_val = _check_all_types(src_dict, step_inputs, "source", param_to_step) + workflow_outputs_val = _check_all_types( src_dict, workflow_outputs, "outputSource", param_to_step ) @@ -199,27 +198,34 @@ def static_checker( sink = warning.sink linkMerge = warning.linkMerge sinksf = sorted( - p["pattern"] for p in sink.get("secondaryFiles", []) if p.get("required", True) + cast(str, p["pattern"]) + for p in cast(MutableSequence[CWLObjectType], sink.get("secondaryFiles", [])) + if p.get("required", True) + ) + srcsf = sorted( + cast(str, p["pattern"]) + for p in cast(MutableSequence[CWLObjectType], src.get("secondaryFiles", [])) ) - srcsf = sorted(p["pattern"] for p in src.get("secondaryFiles", [])) # Every secondaryFile required by the sink, should be declared # by the source missing = missing_subset(srcsf, sinksf) + src_name = shortname(cast(str, src["id"])) + sink_id = cast(str, sink["id"]) + sink_name = shortname(sink_id) if missing: msg1 = "Parameter '{}' requires secondaryFiles {} but".format( - shortname(sink["id"]), + sink_name, missing, ) msg3 = SourceLine(src, "id").makeError( - "source '%s' does not provide those secondaryFiles." % (shortname(src["id"])) + "source '%s' does not provide those secondaryFiles." % (src_name) ) msg4 = SourceLine(src.get("_tool_entry", src), "secondaryFiles").makeError( "To resolve, add missing secondaryFiles patterns to definition of '%s' or" - % (shortname(src["id"])) + % (src_name) ) msg5 = SourceLine(sink.get("_tool_entry", sink), "secondaryFiles").makeError( - "mark missing secondaryFiles in definition of '%s' as optional." - % shortname(sink["id"]) + "mark missing secondaryFiles in definition of '%s' as optional." % (sink_name) ) msg = SourceLine(sink).makeError( "{}\n{}".format(msg1, bullets([msg3, msg4, msg5], " ")) @@ -229,13 +235,13 @@ def static_checker( msg = SourceLine(sink, "type").makeError( "'%s' is not an input parameter of %s, expected %s" % ( - shortname(sink["id"]), - param_to_step[sink["id"]]["run"], + sink_name, + param_to_step[sink_id]["run"], ", ".join( shortname(cast(str, s["id"])) for s in cast( list[dict[str, Union[str, bool]]], - param_to_step[sink["id"]]["inputs"], + param_to_step[sink_id]["inputs"], ) if not s.get("not_connected") ), @@ -247,12 +253,11 @@ def static_checker( msg = ( SourceLine(src, "type").makeError( "Source '%s' of type %s may be incompatible" - % (shortname(src["id"]), json_dumps(src["type"])) + % (src_name, json_dumps(src["type"])) ) + "\n" + SourceLine(sink, "type").makeError( - " with sink '%s' of type %s" - % (shortname(sink["id"]), json_dumps(sink["type"])) + " with sink '%s' of type %s" % (sink_name, json_dumps(sink["type"])) ) ) if linkMerge is not None: @@ -274,12 +279,12 @@ def static_checker( msg = ( SourceLine(src, "type").makeError( "Source '%s' of type %s is incompatible" - % (shortname(src["id"]), json_dumps(src["type"])) + % (shortname(cast(str, src["id"])), json_dumps(src["type"])) ) + "\n" + SourceLine(sink, "type").makeError( " with sink '{}' of type {}".format( - shortname(sink["id"]), json_dumps(sink["type"]) + shortname(cast(str, sink["id"])), json_dumps(sink["type"]) ) ) ) @@ -291,16 +296,17 @@ def static_checker( exception_msgs.append(msg) for sink in step_inputs: + sink_type = cast(Union[str, list[str], list[CWLObjectType], CWLObjectType], sink["type"]) if ( - "null" != sink["type"] - and "null" not in sink["type"] + "null" != sink_type + and "null" not in sink_type and "source" not in sink and "default" not in sink and "valueFrom" not in sink ): msg = SourceLine(sink).makeError( "Required parameter '%s' does not have source, default, or valueFrom expression" - % shortname(sink["id"]) + % shortname(cast(str, sink["id"])) ) exception_msgs.append(msg) @@ -313,15 +319,21 @@ def static_checker( raise ValidationException(all_exception_msg) -SrcSink = namedtuple("SrcSink", ["src", "sink", "linkMerge", "message"]) +class _SrcSink(NamedTuple): + """An error or warning message about a connection between two points of the workflow graph.""" + + src: CWLObjectType + sink: CWLObjectType + linkMerge: Optional[str] + message: Optional[str] -def check_all_types( +def _check_all_types( src_dict: dict[str, CWLObjectType], sinks: MutableSequence[CWLObjectType], sourceField: Union[Literal["source"], Literal["outputSource"]], param_to_step: dict[str, CWLObjectType], -) -> dict[str, list[SrcSink]]: +) -> dict[str, list[_SrcSink]]: """ Given a list of sinks, check if their types match with the types of their sources. @@ -329,7 +341,7 @@ def check_all_types( (from :py:func:`check_types`) :raises ValidationException: if a sourceField is missing """ - validation: dict[str, list[SrcSink]] = {"warning": [], "exception": []} + validation: dict[str, list[_SrcSink]] = {"warning": [], "exception": []} for sink in sinks: if sourceField in sink: valueFrom = cast(Optional[str], sink.get("valueFrom")) @@ -356,7 +368,7 @@ def check_all_types( srcs_of_sink += [src_dict[parm_id]] if is_conditional_step(param_to_step, parm_id) and pickValue is None: validation["warning"].append( - SrcSink( + _SrcSink( src_dict[parm_id], sink, linkMerge, @@ -380,7 +392,7 @@ def check_all_types( if pickValue is not None: validation["warning"].append( - SrcSink( + _SrcSink( src_dict[parm_id], sink, linkMerge, @@ -399,7 +411,7 @@ def check_all_types( Union[list[str], CWLObjectType], snk_typ ): # Given our type names this works even if not a list validation["warning"].append( - SrcSink( + _SrcSink( src_dict[parm_id], sink, linkMerge, @@ -419,11 +431,11 @@ def check_all_types( check_result = check_types(src, sink, linkMerge, valueFrom) if check_result == "warning": validation["warning"].append( - SrcSink(src, sink, linkMerge, message=extra_message) + _SrcSink(src, sink, linkMerge, message=extra_message) ) elif check_result == "exception": validation["exception"].append( - SrcSink(src, sink, linkMerge, message=extra_message) + _SrcSink(src, sink, linkMerge, message=extra_message) ) return validation diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 775360016..1fe1a7044 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -276,7 +276,7 @@ def revmap_file(builder: Builder, outdir: str, f: CWLObjectType) -> Optional[CWL ) revmap_f = builder.pathmapper.reversemap(path) - if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith("Writable"): + if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith("Writable"): # type: ignore[union-attr] f["location"] = revmap_f[1] elif ( uripath == outdir diff --git a/cwltool/errors.py b/cwltool/errors.py index 045b9b383..2b7e50aed 100644 --- a/cwltool/errors.py +++ b/cwltool/errors.py @@ -11,6 +11,13 @@ from cwl_utils.errors import GraphTargetMissingException as GraphTargetMissingException from cwl_utils.errors import WorkflowException as WorkflowException +__all__ = ( + "GraphTargetMissingException", + "WorkflowException", + "UnsupportedRequirement", + "ArgumentException", +) + class UnsupportedRequirement(WorkflowException): pass diff --git a/cwltool/main.py b/cwltool/main.py index a137d8a4f..c658c3685 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -16,6 +16,7 @@ import warnings from codecs import getwriter from collections.abc import Mapping, MutableMapping, MutableSequence, Sized +from importlib.resources import files from typing import IO, Any, Callable, Optional, Union, cast import argcomplete @@ -96,7 +97,6 @@ CWLOutputType, HasReqsHints, adjustDirObjs, - files, normalizeFilesDirs, processes_to_kill, trim_listing, diff --git a/cwltool/mutation.py b/cwltool/mutation.py index 9f58a86cf..622807ec6 100644 --- a/cwltool/mutation.py +++ b/cwltool/mutation.py @@ -1,10 +1,16 @@ -from collections import namedtuple -from typing import cast +"""Support for InplaceUpdateRequirement.""" + +from typing import NamedTuple, cast from .errors import WorkflowException from .utils import CWLObjectType -MutationState = namedtuple("MutationState", ["generation", "readers", "stepname"]) + +class _MutationState(NamedTuple): + generation: int + readers: list[str] + stepname: str + _generation = "http://commonwl.org/cwltool#generation" @@ -20,11 +26,11 @@ class MutationManager: def __init__(self) -> None: """Initialize.""" - self.generations: dict[str, MutationState] = {} + self.generations: dict[str, _MutationState] = {} def register_reader(self, stepname: str, obj: CWLObjectType) -> None: loc = cast(str, obj["location"]) - current = self.generations.get(loc, MutationState(0, [], "")) + current = self.generations.get(loc, _MutationState(0, [], "")) obj_generation = obj.get(_generation, 0) if obj_generation != current.generation: @@ -40,7 +46,7 @@ def register_reader(self, stepname: str, obj: CWLObjectType) -> None: def release_reader(self, stepname: str, obj: CWLObjectType) -> None: loc = cast(str, obj["location"]) - current = self.generations.get(loc, MutationState(0, [], "")) + current = self.generations.get(loc, _MutationState(0, [], "")) obj_generation = obj.get(_generation, 0) if obj_generation != current.generation: @@ -55,7 +61,7 @@ def release_reader(self, stepname: str, obj: CWLObjectType) -> None: def register_mutation(self, stepname: str, obj: CWLObjectType) -> None: loc = cast(str, obj["location"]) - current = self.generations.get(loc, MutationState(0, [], "")) + current = self.generations.get(loc, _MutationState(0, [], "")) obj_generation = obj.get(_generation, 0) if len(current.readers) > 0: @@ -73,11 +79,11 @@ def register_mutation(self, stepname: str, obj: CWLObjectType) -> None: ) ) - self.generations[loc] = MutationState(current.generation + 1, current.readers, stepname) + self.generations[loc] = _MutationState(current.generation + 1, current.readers, stepname) def set_generation(self, obj: CWLObjectType) -> None: loc = cast(str, obj["location"]) - current = self.generations.get(loc, MutationState(0, [], "")) + current = self.generations.get(loc, _MutationState(0, [], "")) obj[_generation] = current.generation def unset_generation(self, obj: CWLObjectType) -> None: diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 10cb7a733..774414f0a 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -1,11 +1,10 @@ -import collections import logging import os import stat import urllib import uuid from collections.abc import ItemsView, Iterable, Iterator, KeysView -from typing import Optional, cast +from typing import NamedTuple, Optional, cast from mypy_extensions import mypyc_attr from schema_salad.exceptions import ValidationException @@ -16,31 +15,24 @@ from .stdfsaccess import abspath from .utils import CWLObjectType, dedup, downloadHttpFile -MapperEnt = collections.namedtuple("MapperEnt", ["resolved", "target", "type", "staged"]) -""" Mapper entries. -.. py:attribute:: resolved - :type: str +class MapperEnt(NamedTuple): + """Mapper entries.""" - The "real" path on the local file system (after resolving relative paths - and traversing symlinks - -.. py:attribute:: target - :type: str - - The path on the target file system (under stagedir) - -.. py:attribute:: type - :type: str - - The object type. One of "File", "Directory", "CreateFile", "WritableFile", - or "CreateWritableFile". - -.. py:attribute:: staged - :type: bool - - If the File has been staged yet -""" + resolved: str + """ + The "real" path on the local file system (after resolving relative paths + and traversing symlinks + """ + target: str + """The path on the target file system (under stagedir)""" + type: Optional[str] + """ + The object type. One of "File", "Directory", "CreateFile", "WritableFile", + or "CreateWritableFile". + """ + staged: Optional[bool] + """If the File has been staged yet.""" @mypyc_attr(allow_interpreted_subclasses=True) @@ -149,7 +141,7 @@ def visit( ab = abspath(path, basedir) if "contents" in obj and path.startswith("_:"): self._pathmap[path] = MapperEnt( - obj["contents"], + cast(str, obj["contents"]), tgt, "CreateWritableFile" if copy else "CreateFile", staged, @@ -247,8 +239,10 @@ def reversemap( return (k, v[0]) return None - def update(self, key: str, resolved: str, target: str, ctype: str, stage: bool) -> MapperEnt: - """Update an existine entry.""" + def update( + self, key: str, resolved: str, target: str, ctype: Optional[str], stage: Optional[bool] + ) -> MapperEnt: + """Update an existing entry.""" m = MapperEnt(resolved, target, ctype, stage) self._pathmap[key] = m return m diff --git a/cwltool/process.py b/cwltool/process.py index fe5f84764..9f985a5d5 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -14,6 +14,7 @@ import urllib.parse import uuid from collections.abc import Iterable, Iterator, MutableMapping, MutableSequence, Sized +from importlib.resources import files from os import scandir from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast @@ -54,7 +55,6 @@ aslist, cmp_like_py2, ensure_writable, - files, get_listing, normalizeFilesDirs, random_outdir, @@ -230,7 +230,7 @@ def stage_files( items = pathmapper.items() if not symlink else pathmapper.items_exclude_children() targets: dict[str, MapperEnt] = {} for key, entry in list(items): - if "File" not in entry.type: + if entry.type is None or "File" not in entry.type: continue if entry.target not in targets: targets[entry.target] = entry diff --git a/cwltool/subgraph.py b/cwltool/subgraph.py index 204e987a8..0a0289cc3 100644 --- a/cwltool/subgraph.py +++ b/cwltool/subgraph.py @@ -1,7 +1,6 @@ import urllib -from collections import namedtuple from collections.abc import Mapping, MutableMapping, MutableSequence -from typing import Any, Optional, Union, cast +from typing import Any, NamedTuple, Optional, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -11,7 +10,13 @@ from .utils import CWLObjectType, aslist from .workflow import Workflow, WorkflowStep -Node = namedtuple("Node", ("up", "down", "type")) + +class _Node(NamedTuple): + up: list[str] + down: list[str] + type: Optional[str] + + UP = "up" DOWN = "down" INPUT = "input" @@ -19,9 +24,9 @@ STEP = "step" -def subgraph_visit( +def _subgraph_visit( current: str, - nodes: MutableMapping[str, Node], + nodes: MutableMapping[str, _Node], visited: set[str], direction: str, ) -> None: @@ -34,10 +39,10 @@ def subgraph_visit( if direction == UP: d = nodes[current].up for c in d: - subgraph_visit(c, nodes, visited, direction) + _subgraph_visit(c, nodes, visited, direction) -def declare_node(nodes: dict[str, Node], nodeid: str, tp: Optional[str]) -> Node: +def _declare_node(nodes: dict[str, _Node], nodeid: str, tp: Optional[str]) -> _Node: """ Record the given nodeid in the graph. @@ -47,9 +52,9 @@ def declare_node(nodes: dict[str, Node], nodeid: str, tp: Optional[str]) -> Node if nodeid in nodes: n = nodes[nodeid] if n.type is None: - nodes[nodeid] = Node(n.up, n.down, tp) + nodes[nodeid] = _Node(n.up, n.down, tp) else: - nodes[nodeid] = Node([], [], tp) + nodes[nodeid] = _Node([], [], tp) return nodes[nodeid] @@ -109,22 +114,22 @@ def get_subgraph( if tool.tool["class"] != "Workflow": raise Exception("Can only extract subgraph from workflow") - nodes: dict[str, Node] = {} + nodes: dict[str, _Node] = {} for inp in tool.tool["inputs"]: - declare_node(nodes, inp["id"], INPUT) + _declare_node(nodes, inp["id"], INPUT) for out in tool.tool["outputs"]: - declare_node(nodes, out["id"], OUTPUT) + _declare_node(nodes, out["id"], OUTPUT) for i in aslist(out.get("outputSource", CommentedSeq)): # source is upstream from output (dependency) nodes[out["id"]].up.append(i) # output is downstream from source - declare_node(nodes, i, None) + _declare_node(nodes, i, None) nodes[i].down.append(out["id"]) for st in tool.tool["steps"]: - step = declare_node(nodes, st["id"], STEP) + step = _declare_node(nodes, st["id"], STEP) for i in st["in"]: if "source" not in i: continue @@ -132,7 +137,7 @@ def get_subgraph( # source is upstream from step (dependency) step.up.append(src) # step is downstream from source - declare_node(nodes, src, None) + _declare_node(nodes, src, None) nodes[src].down.append(st["id"]) for out in st["out"]: if isinstance(out, Mapping) and "id" in out: @@ -140,16 +145,16 @@ def get_subgraph( # output is downstream from step step.down.append(out) # step is upstream from output - declare_node(nodes, out, None) + _declare_node(nodes, out, None) nodes[out].up.append(st["id"]) # Find all the downstream nodes from the starting points visited_down: set[str] = set() for r in roots: if nodes[r].type == OUTPUT: - subgraph_visit(r, nodes, visited_down, UP) + _subgraph_visit(r, nodes, visited_down, UP) else: - subgraph_visit(r, nodes, visited_down, DOWN) + _subgraph_visit(r, nodes, visited_down, DOWN) # Now make sure all the nodes are connected to upstream inputs visited: set[str] = set() diff --git a/cwltool/utils.py b/cwltool/utils.py index e460842a9..2c7f6ba12 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -23,7 +23,6 @@ from datetime import datetime from email.utils import parsedate_to_datetime from functools import partial -from importlib.resources import as_file, files from itertools import zip_longest from pathlib import Path, PurePosixPath from tempfile import NamedTemporaryFile @@ -54,8 +53,6 @@ from .stdfsaccess import StdFsAccess from .workflow_job import WorkflowJob -__all__ = ["files", "as_file"] - __random_outdir: Optional[str] = None CONTENT_LIMIT = 64 * 1024 diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index b43b7ef0d..3a490b68d 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -2,9 +2,9 @@ import itertools import json import logging -from collections import namedtuple from collections.abc import MutableMapping, MutableSequence -from typing import Any, Optional, Union, cast +from importlib.resources import files +from typing import Any, NamedTuple, Optional, Union, cast from cwl_utils.errors import SubstitutionError from cwl_utils.expression import scanner as scan_expression @@ -23,7 +23,6 @@ from .errors import WorkflowException from .loghandler import _logger -from .utils import files def is_expression(tool: Any, schema: Optional[Schema]) -> bool: @@ -110,7 +109,11 @@ def get_expressions( return [] -JSHintJSReturn = namedtuple("JSHintJSReturn", ["errors", "globals"]) +class JSHintJSReturn(NamedTuple): + """List of errors and the final values of the globals from running javascript.""" + + errors: list[str] + globals: list[str] def jshint_js( diff --git a/docs/conf.py b/docs/conf.py index 6e04b5d64..e476f4191 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,7 +51,8 @@ intersphinx_mapping = { "python": ("https://docs.python.org/3", None), "schema_salad": ("https://schema-salad.readthedocs.io/en/stable/", None), - "rdflib": ("https://rdflib.readthedocs.io/en/6.2.0/", None), + "rdflib": ("https://rdflib.readthedocs.io/en/stable/", None), + "cwl_utils": ("https://cwl-utils.readthedocs.io/en/stable/", None), # "ruamel.yaml": ("https://yaml.readthedocs.io/en/stable/", None), } diff --git a/tests/test_mpi.py b/tests/test_mpi.py index 92f0e353c..9da32fd05 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -4,6 +4,7 @@ import os.path import sys from collections.abc import Generator, MutableMapping +from importlib.resources import files from io import StringIO from pathlib import Path from typing import Any, Optional @@ -20,7 +21,6 @@ from cwltool.context import LoadingContext, RuntimeContext from cwltool.main import main from cwltool.mpi import MpiConfig, MPIRequirementName -from cwltool.utils import files from .util import get_data, working_directory diff --git a/tests/util.py b/tests/util.py index 8dd0bf74e..d7624bc5e 100644 --- a/tests/util.py +++ b/tests/util.py @@ -10,6 +10,7 @@ import sys from collections.abc import Generator, Mapping from contextlib import ExitStack +from importlib.resources import as_file, files from pathlib import Path from typing import Any, Optional, Union @@ -18,7 +19,6 @@ from cwltool.env_to_stdout import deserialize_env from cwltool.main import main from cwltool.singularity import is_version_2_6, is_version_3_or_newer -from cwltool.utils import as_file, files def force_default_container(default_container_id: str, _: str) -> str: From f6aeeae01ca1d821f2be1966f48f5257100f90e5 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 22 Jan 2025 15:16:06 +0100 Subject: [PATCH 45/78] typo --- cwltool/cwlprov/ro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index f58919a6b..be10d3d64 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -668,7 +668,7 @@ def _relativise_files( del structure["path"] if structure.get("class") == "Directory": - # TODO: Generate anonymoys Directory with a "listing" + # TODO: Generate anonymous Directory with a "listing" # pointing to the hashed files del structure["location"] From dfa1dd0ec1618b5546df3ddbf41230111308574e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jan 2025 07:02:47 +0000 Subject: [PATCH 46/78] Update black requirement from ==24.* to ==25.* Updates the requirements on [black](https://github.com/psf/black) to permit the latest version. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/24.1a1...25.1.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index abb223c85..a2c6768cf 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,3 +1,3 @@ flake8-bugbear<24.13 -black==24.* +black==25.* codespell From 85976ec0000cffb510984ff1d3da2c13d313be03 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 29 Jan 2025 14:34:21 +0100 Subject: [PATCH 47/78] black 2025 reformat --- cwltool/load_tool.py | 6 +++--- cwltool/process.py | 2 +- cwltool/utils.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 4d7f3a930..f7f4936cd 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -142,7 +142,7 @@ def fetch_document( def _convert_stdstreams_to_files( - workflowobj: Union[CWLObjectType, MutableSequence[Union[CWLObjectType, str, int]], str] + workflowobj: Union[CWLObjectType, MutableSequence[Union[CWLObjectType, str, int]], str], ) -> None: if isinstance(workflowobj, MutableMapping): if workflowobj.get("class") == "CommandLineTool": @@ -219,7 +219,7 @@ def _convert_stdstreams_to_files( def _add_blank_ids( - workflowobj: Union[CWLObjectType, MutableSequence[Union[CWLObjectType, str]]] + workflowobj: Union[CWLObjectType, MutableSequence[Union[CWLObjectType, str]]], ) -> None: if isinstance(workflowobj, MutableMapping): if ( @@ -247,7 +247,7 @@ def _add_blank_ids( def _fast_parser_convert_stdstreams_to_files( - processobj: Union[cwl_v1_2.Process, MutableSequence[cwl_v1_2.Process]] + processobj: Union[cwl_v1_2.Process, MutableSequence[cwl_v1_2.Process]], ) -> None: if isinstance(processobj, cwl_v1_2.CommandLineTool): cwl_v1_2_utils.convert_stdstreams_to_files(processobj) diff --git a/cwltool/process.py b/cwltool/process.py index 9f985a5d5..5271e4643 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -305,7 +305,7 @@ def relocateOutputs( return outputObj def _collectDirEntries( - obj: Union[CWLObjectType, MutableSequence[CWLObjectType], None] + obj: Union[CWLObjectType, MutableSequence[CWLObjectType], None], ) -> Iterator[CWLObjectType]: if isinstance(obj, dict): if obj.get("class") in ("File", "Directory"): diff --git a/cwltool/utils.py b/cwltool/utils.py index 2c7f6ba12..c37dbe8e7 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -398,7 +398,7 @@ def normalizeFilesDirs( MutableMapping[str, Any], DirectoryType, ] - ] + ], ) -> None: def addLocation(d: dict[str, Any]) -> None: if "location" not in d: From 51516cfa746ab7124c9a512109e53406ea42abcd Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 30 Jan 2025 11:15:09 +0100 Subject: [PATCH 48/78] Add missing cwl-runner refs And mark executable --- cwltool/schemas/v1.1.0-dev1/cwl-runner.cwl | 0 cwltool/schemas/v1.1/cwl-runner.cwl | 0 cwltool/schemas/v1.2.0-dev1/cwl-runner.cwl | 0 cwltool/schemas/v1.2.0-dev2/cwl-runner.cwl | 0 cwltool/schemas/v1.2.0-dev3/cwl-runner.cwl | 0 cwltool/schemas/v1.2.0-dev4/cwl-runner.cwl | 0 cwltool/schemas/v1.2.0-dev5/cwl-runner.cwl | 0 cwltool/schemas/v1.3.0-dev1/cwl-runner.cwl | 0 tests/CometAdapter.cwl | 1 + tests/arg-empty-prefix-separate-false.cwl | 0 tests/checker_wf/optional_array_mismatch.cwl | 1 + tests/default_values_list.cwl | 0 tests/echo-badposition-expr.cwl | 0 tests/echo-position-expr.cwl | 0 tests/echo-stdout-log-dir.cwl | 0 tests/env.cwl | 0 tests/env2.cwl | 0 tests/env3.cwl | 0 tests/env4.cwl | 0 tests/input_deps/docker-array-secondaryfiles.cwl | 0 tests/iwdr_bad_expr.cwl | 0 tests/iwdr_dir_literal_real_file.cwl | 0 tests/load_contents-array.cwl | 1 + tests/loop-ext/all-output-loop-no-iteration.cwl | 0 tests/loop-ext/all-output-loop.cwl | 0 tests/loop-ext/default-value-loop.cwl | 0 tests/loop-ext/invalid-loop-command-line-tool.cwl | 0 tests/loop-ext/invalid-loop-expression-tool.cwl | 0 tests/loop-ext/invalid-loop-hint.cwl | 0 tests/loop-ext/invalid-loop-scatter.cwl | 0 tests/loop-ext/invalid-loop-when-exception.cwl | 0 tests/loop-ext/invalid-loop-when.cwl | 0 tests/loop-ext/invalid-loop-workflow.cwl | 0 .../loop-ext/invalid-multi-source-loop-no-requirement.cwl | 0 tests/loop-ext/invalid-no-loopWhen.cwl | 0 tests/loop-ext/invalid-non-boolean-loopWhen.cwl | 0 tests/loop-ext/invalid-non-boolean-loopWhen2.cwl | 0 tests/loop-ext/invalid-value-from-loop-no-requirement.cwl | 0 tests/loop-ext/loop-inside-loop-all.cwl | 0 tests/loop-ext/loop-inside-loop.cwl | 0 tests/loop-ext/loop-inside-scatter.cwl | 0 tests/loop-ext/multi-source-loop.cwl | 0 tests/loop-ext/opt-var-loop.cwl | 0 tests/loop-ext/scatter-inside-loop.cwl | 0 tests/loop-ext/single-var-loop-no-iteration.cwl | 0 tests/loop-ext/single-var-loop.cwl | 0 tests/loop-ext/two-vars-loop-2.cwl | 0 tests/loop-ext/two-vars-loop.cwl | 0 tests/loop-ext/value-from-loop.cwl | 0 tests/loop/all-output-loop-no-iteration.cwl | 0 tests/loop/all-output-loop.cwl | 0 tests/loop/default-value-loop.cwl | 0 tests/loop/invalid-loop-scatter.cwl | 0 tests/loop/invalid-loop-when-exception.cwl | 0 tests/loop/invalid-loop-when-exception2.cwl | 0 tests/loop/invalid-multi-source-loop-no-requirement.cwl | 0 tests/loop/invalid-no-loopWhen.cwl | 0 tests/loop/invalid-non-boolean-loopWhen.cwl | 0 tests/loop/invalid-non-boolean-loopWhen2.cwl | 0 tests/loop/invalid-value-from-loop-no-requirement.cwl | 0 tests/loop/loop-inside-loop-all.cwl | 0 tests/loop/loop-inside-loop.cwl | 0 tests/loop/loop-inside-scatter.cwl | 0 tests/loop/multi-source-loop.cwl | 0 tests/loop/opt-var-loop.cwl | 0 tests/loop/scatter-inside-loop.cwl | 0 tests/loop/single-var-loop-no-iteration.cwl | 0 tests/loop/single-var-loop.cwl | 0 tests/loop/two-vars-loop-2.cwl | 0 tests/loop/two-vars-loop.cwl | 0 tests/loop/value-from-loop.cwl | 0 tests/nested-array.cwl | 1 + tests/non_portable.cwl | 0 tests/non_portable2.cwl | 0 tests/override/env-tool.cwl | 0 tests/override/env-tool_v1.1.0-dev1.cwl | 0 tests/override/env-tool_v1.1.cwl | 0 tests/portable.cwl | 0 tests/reloc/test.cwl | 1 + tests/scatter_numbers.cwl | 0 tests/secondary-files-bad.cwl | 0 tests/secondary-files-required-container.cwl | 0 tests/secondary-files-required-missing-container.cwl | 0 tests/secondary-files-string-v1.cwl | 0 tests/secondary-files.cwl | 0 tests/sing_pullfolder_test.cwl | 0 tests/subgraph/1432.cwl | 1 + tests/subgraph/env-tool2.cwl | 1 + tests/subgraph/env-tool2_no_env.cwl | 1 + tests/subgraph/env-tool2_req.cwl | 1 + tests/subgraph/env-wf2.cwl | 0 tests/subgraph/env-wf2_hint_collision.cwl | 0 tests/subgraph/env-wf2_hint_req_collision.cwl | 0 tests/subgraph/env-wf2_long.cwl | 0 tests/subgraph/env-wf2_only_hint.cwl | 0 tests/subgraph/env-wf2_req_collision.cwl | 0 tests/subgraph/env-wf2_subwf-packed.cwl | 3 ++- tests/subgraph/env-wf2_subwf.cwl | 0 tests/subgraph/env-wf2_subwf_b.cwl | 0 tests/subgraph/steplevel-resreq.cwl | 0 tests/subgraph/timelimit2-wf.cwl | 0 tests/test_examples.py | 2 +- tests/test_validate.py | 8 ++++---- tests/trs/Dockstore.cwl | 0 tests/trs/md5sum-tool.cwl | 0 tests/trs/md5sum-workflow.cwl | 0 tests/utf_doc_example.cwl | 0 tests/wc-tool-bad-hints.cwl | 0 tests/wc-tool-bad-reqs.cwl | 0 tests/wf/1496.cwl | 1 + tests/wf/1590.cwl | 1 + tests/wf/1st-workflow.cwl | 0 tests/wf/811-12.cwl | 1 + tests/wf/811.cwl | 1 + tests/wf/816_tool.cwl | 1 + tests/wf/816_wf.cwl | 1 + tests/wf/910.cwl | 0 tests/wf/arguments.cwl | 0 tests/wf/bad-stderr-expr.cwl | 0 tests/wf/bad-stdin-expr.cwl | 0 tests/wf/bad-stdout-expr.cwl | 0 tests/wf/bad_networkaccess.cwl | 0 tests/wf/bad_timelimit.cwl | 0 tests/wf/cache_test_workflow.cwl | 0 tests/wf/cat-tool.cwl | 0 tests/wf/conditional_step_no_inputs.cwl | 0 tests/wf/conflict.cwl | 0 tests/wf/cores_float.cwl | 0 tests/wf/default-dir5.cwl | 0 tests/wf/default-wf5.cwl | 0 tests/wf/double-nested.cwl | 0 tests/wf/expect_trick_packed.cwl | 0 tests/wf/extract_region_specs.cwl | 1 + tests/wf/floats_small_and_large.cwl | 1 + tests/wf/generator/pytoolgen.cwl | 0 tests/wf/input_named_id.cwl | 1 + tests/wf/iwd-passthrough1.cwl | 0 tests/wf/iwdr-empty.cwl | 0 tests/wf/iwdr-entry.cwl | 0 tests/wf/iwdr-passthrough-successive.cwl | 0 tests/wf/literalfile.cwl | 0 tests/wf/malformed_outputs.cwl | 0 tests/wf/missing-tool.cwl | 0 tests/wf/mpi_env.cwl | 0 tests/wf/mpi_expr.cwl | 0 tests/wf/mpi_line_count.cwl | 0 tests/wf/mpi_simple.cwl | 0 tests/wf/mpi_simple_wf.cwl | 0 tests/wf/nested.cwl | 0 tests/wf/networkaccess-fail.cwl | 0 tests/wf/networkaccess.cwl | 0 tests/wf/no-parameters-echo.cwl | 0 tests/wf/nvidia-smi-cc.cwl | 1 + tests/wf/nvidia-smi-container.cwl | 1 + tests/wf/nvidia-smi-max.cwl | 1 + tests/wf/nvidia-smi-range.cwl | 1 + tests/wf/nvidia-smi.cwl | 1 + tests/wf/operation/abstract-cosifer.cwl | 1 + tests/wf/operation/operation-single.cwl | 1 + tests/wf/optional-numerical-output-0.cwl | 0 tests/wf/optional_src_mandatory_sink.cwl | 0 tests/wf/packed-with-loadlisting.cwl | 0 tests/wf/packed_no_main.cwl | 1 + tests/wf/paramref_arguments_roundtrip.cwl | 0 tests/wf/paramref_arguments_self.cwl | 0 tests/wf/record_outputeval.cwl | 1 + tests/wf/resreq_expr_float_v1_0.cwl | 1 + tests/wf/resreq_expr_float_v1_2.cwl | 1 + tests/wf/scatter2.cwl | 0 tests/wf/scatter2_subwf.cwl | 0 tests/wf/schemadef-bug-1473.cwl | 1 + tests/wf/schemadef-tool-12.cwl | 0 tests/wf/schemadef-tool.cwl | 0 tests/wf/sec-tool.cwl | 0 tests/wf/sec-wf-out.cwl | 0 tests/wf/sec-wf.cwl | 0 tests/wf/secret_job.cwl | 0 tests/wf/secret_wf.cwl | 0 tests/wf/separate_without_prefix.cwl | 0 tests/wf/shm_size.cwl | 0 tests/wf/storage_float.cwl | 0 tests/wf/tar-param.cwl | 0 tests/wf/three_step_color.cwl | 0 tests/wf/timelimit-fail.cwl | 0 tests/wf/timelimit.cwl | 0 tests/wf/touch_tool.cwl | 0 tests/wf/trick_defaults.cwl | 0 tests/wf/trick_defaults2.cwl | 0 tests/wf/vf-concat.cwl | 0 tests/wf/workreuse-fail.cwl | 0 tests/wf/workreuse.cwl | 0 tests/with_doc.cwl | 0 tests/without_doc.cwl | 0 193 files changed, 37 insertions(+), 6 deletions(-) mode change 100644 => 100755 cwltool/schemas/v1.1.0-dev1/cwl-runner.cwl mode change 100644 => 100755 cwltool/schemas/v1.1/cwl-runner.cwl mode change 100644 => 100755 cwltool/schemas/v1.2.0-dev1/cwl-runner.cwl mode change 100644 => 100755 cwltool/schemas/v1.2.0-dev2/cwl-runner.cwl mode change 100644 => 100755 cwltool/schemas/v1.2.0-dev3/cwl-runner.cwl mode change 100644 => 100755 cwltool/schemas/v1.2.0-dev4/cwl-runner.cwl mode change 100644 => 100755 cwltool/schemas/v1.2.0-dev5/cwl-runner.cwl mode change 100644 => 100755 cwltool/schemas/v1.3.0-dev1/cwl-runner.cwl mode change 100644 => 100755 tests/CometAdapter.cwl mode change 100644 => 100755 tests/arg-empty-prefix-separate-false.cwl mode change 100644 => 100755 tests/checker_wf/optional_array_mismatch.cwl mode change 100644 => 100755 tests/default_values_list.cwl mode change 100644 => 100755 tests/echo-badposition-expr.cwl mode change 100644 => 100755 tests/echo-position-expr.cwl mode change 100644 => 100755 tests/echo-stdout-log-dir.cwl mode change 100644 => 100755 tests/env.cwl mode change 100644 => 100755 tests/env2.cwl mode change 100644 => 100755 tests/env3.cwl mode change 100644 => 100755 tests/env4.cwl mode change 100644 => 100755 tests/input_deps/docker-array-secondaryfiles.cwl mode change 100644 => 100755 tests/iwdr_bad_expr.cwl mode change 100644 => 100755 tests/iwdr_dir_literal_real_file.cwl mode change 100644 => 100755 tests/load_contents-array.cwl mode change 100644 => 100755 tests/loop-ext/all-output-loop-no-iteration.cwl mode change 100644 => 100755 tests/loop-ext/all-output-loop.cwl mode change 100644 => 100755 tests/loop-ext/default-value-loop.cwl mode change 100644 => 100755 tests/loop-ext/invalid-loop-command-line-tool.cwl mode change 100644 => 100755 tests/loop-ext/invalid-loop-expression-tool.cwl mode change 100644 => 100755 tests/loop-ext/invalid-loop-hint.cwl mode change 100644 => 100755 tests/loop-ext/invalid-loop-scatter.cwl mode change 100644 => 100755 tests/loop-ext/invalid-loop-when-exception.cwl mode change 100644 => 100755 tests/loop-ext/invalid-loop-when.cwl mode change 100644 => 100755 tests/loop-ext/invalid-loop-workflow.cwl mode change 100644 => 100755 tests/loop-ext/invalid-multi-source-loop-no-requirement.cwl mode change 100644 => 100755 tests/loop-ext/invalid-no-loopWhen.cwl mode change 100644 => 100755 tests/loop-ext/invalid-non-boolean-loopWhen.cwl mode change 100644 => 100755 tests/loop-ext/invalid-non-boolean-loopWhen2.cwl mode change 100644 => 100755 tests/loop-ext/invalid-value-from-loop-no-requirement.cwl mode change 100644 => 100755 tests/loop-ext/loop-inside-loop-all.cwl mode change 100644 => 100755 tests/loop-ext/loop-inside-loop.cwl mode change 100644 => 100755 tests/loop-ext/loop-inside-scatter.cwl mode change 100644 => 100755 tests/loop-ext/multi-source-loop.cwl mode change 100644 => 100755 tests/loop-ext/opt-var-loop.cwl mode change 100644 => 100755 tests/loop-ext/scatter-inside-loop.cwl mode change 100644 => 100755 tests/loop-ext/single-var-loop-no-iteration.cwl mode change 100644 => 100755 tests/loop-ext/single-var-loop.cwl mode change 100644 => 100755 tests/loop-ext/two-vars-loop-2.cwl mode change 100644 => 100755 tests/loop-ext/two-vars-loop.cwl mode change 100644 => 100755 tests/loop-ext/value-from-loop.cwl mode change 100644 => 100755 tests/loop/all-output-loop-no-iteration.cwl mode change 100644 => 100755 tests/loop/all-output-loop.cwl mode change 100644 => 100755 tests/loop/default-value-loop.cwl mode change 100644 => 100755 tests/loop/invalid-loop-scatter.cwl mode change 100644 => 100755 tests/loop/invalid-loop-when-exception.cwl mode change 100644 => 100755 tests/loop/invalid-loop-when-exception2.cwl mode change 100644 => 100755 tests/loop/invalid-multi-source-loop-no-requirement.cwl mode change 100644 => 100755 tests/loop/invalid-no-loopWhen.cwl mode change 100644 => 100755 tests/loop/invalid-non-boolean-loopWhen.cwl mode change 100644 => 100755 tests/loop/invalid-non-boolean-loopWhen2.cwl mode change 100644 => 100755 tests/loop/invalid-value-from-loop-no-requirement.cwl mode change 100644 => 100755 tests/loop/loop-inside-loop-all.cwl mode change 100644 => 100755 tests/loop/loop-inside-loop.cwl mode change 100644 => 100755 tests/loop/loop-inside-scatter.cwl mode change 100644 => 100755 tests/loop/multi-source-loop.cwl mode change 100644 => 100755 tests/loop/opt-var-loop.cwl mode change 100644 => 100755 tests/loop/scatter-inside-loop.cwl mode change 100644 => 100755 tests/loop/single-var-loop-no-iteration.cwl mode change 100644 => 100755 tests/loop/single-var-loop.cwl mode change 100644 => 100755 tests/loop/two-vars-loop-2.cwl mode change 100644 => 100755 tests/loop/two-vars-loop.cwl mode change 100644 => 100755 tests/loop/value-from-loop.cwl mode change 100644 => 100755 tests/nested-array.cwl mode change 100644 => 100755 tests/non_portable.cwl mode change 100644 => 100755 tests/non_portable2.cwl mode change 100644 => 100755 tests/override/env-tool.cwl mode change 100644 => 100755 tests/override/env-tool_v1.1.0-dev1.cwl mode change 100644 => 100755 tests/override/env-tool_v1.1.cwl mode change 100644 => 100755 tests/portable.cwl mode change 100644 => 100755 tests/reloc/test.cwl mode change 100644 => 100755 tests/scatter_numbers.cwl mode change 100644 => 100755 tests/secondary-files-bad.cwl mode change 100644 => 100755 tests/secondary-files-required-container.cwl mode change 100644 => 100755 tests/secondary-files-required-missing-container.cwl mode change 100644 => 100755 tests/secondary-files-string-v1.cwl mode change 100644 => 100755 tests/secondary-files.cwl mode change 100644 => 100755 tests/sing_pullfolder_test.cwl mode change 100644 => 100755 tests/subgraph/1432.cwl mode change 100644 => 100755 tests/subgraph/env-tool2.cwl mode change 100644 => 100755 tests/subgraph/env-tool2_no_env.cwl mode change 100644 => 100755 tests/subgraph/env-tool2_req.cwl mode change 100644 => 100755 tests/subgraph/env-wf2.cwl mode change 100644 => 100755 tests/subgraph/env-wf2_hint_collision.cwl mode change 100644 => 100755 tests/subgraph/env-wf2_hint_req_collision.cwl mode change 100644 => 100755 tests/subgraph/env-wf2_long.cwl mode change 100644 => 100755 tests/subgraph/env-wf2_only_hint.cwl mode change 100644 => 100755 tests/subgraph/env-wf2_req_collision.cwl mode change 100644 => 100755 tests/subgraph/env-wf2_subwf-packed.cwl mode change 100644 => 100755 tests/subgraph/env-wf2_subwf.cwl mode change 100644 => 100755 tests/subgraph/env-wf2_subwf_b.cwl mode change 100644 => 100755 tests/subgraph/steplevel-resreq.cwl mode change 100644 => 100755 tests/subgraph/timelimit2-wf.cwl mode change 100644 => 100755 tests/trs/Dockstore.cwl mode change 100644 => 100755 tests/trs/md5sum-tool.cwl mode change 100644 => 100755 tests/trs/md5sum-workflow.cwl mode change 100644 => 100755 tests/utf_doc_example.cwl mode change 100644 => 100755 tests/wc-tool-bad-hints.cwl mode change 100644 => 100755 tests/wc-tool-bad-reqs.cwl mode change 100644 => 100755 tests/wf/1496.cwl mode change 100644 => 100755 tests/wf/1590.cwl mode change 100644 => 100755 tests/wf/1st-workflow.cwl mode change 100644 => 100755 tests/wf/811-12.cwl mode change 100644 => 100755 tests/wf/811.cwl mode change 100644 => 100755 tests/wf/816_tool.cwl mode change 100644 => 100755 tests/wf/816_wf.cwl mode change 100644 => 100755 tests/wf/910.cwl mode change 100644 => 100755 tests/wf/arguments.cwl mode change 100644 => 100755 tests/wf/bad-stderr-expr.cwl mode change 100644 => 100755 tests/wf/bad-stdin-expr.cwl mode change 100644 => 100755 tests/wf/bad-stdout-expr.cwl mode change 100644 => 100755 tests/wf/bad_networkaccess.cwl mode change 100644 => 100755 tests/wf/bad_timelimit.cwl mode change 100644 => 100755 tests/wf/cache_test_workflow.cwl mode change 100644 => 100755 tests/wf/cat-tool.cwl mode change 100644 => 100755 tests/wf/conditional_step_no_inputs.cwl mode change 100644 => 100755 tests/wf/conflict.cwl mode change 100644 => 100755 tests/wf/cores_float.cwl mode change 100644 => 100755 tests/wf/default-dir5.cwl mode change 100644 => 100755 tests/wf/default-wf5.cwl mode change 100644 => 100755 tests/wf/double-nested.cwl mode change 100644 => 100755 tests/wf/expect_trick_packed.cwl mode change 100644 => 100755 tests/wf/extract_region_specs.cwl mode change 100644 => 100755 tests/wf/floats_small_and_large.cwl mode change 100644 => 100755 tests/wf/generator/pytoolgen.cwl mode change 100644 => 100755 tests/wf/input_named_id.cwl mode change 100644 => 100755 tests/wf/iwd-passthrough1.cwl mode change 100644 => 100755 tests/wf/iwdr-empty.cwl mode change 100644 => 100755 tests/wf/iwdr-entry.cwl mode change 100644 => 100755 tests/wf/iwdr-passthrough-successive.cwl mode change 100644 => 100755 tests/wf/literalfile.cwl mode change 100644 => 100755 tests/wf/malformed_outputs.cwl mode change 100644 => 100755 tests/wf/missing-tool.cwl mode change 100644 => 100755 tests/wf/mpi_env.cwl mode change 100644 => 100755 tests/wf/mpi_expr.cwl mode change 100644 => 100755 tests/wf/mpi_line_count.cwl mode change 100644 => 100755 tests/wf/mpi_simple.cwl mode change 100644 => 100755 tests/wf/mpi_simple_wf.cwl mode change 100644 => 100755 tests/wf/nested.cwl mode change 100644 => 100755 tests/wf/networkaccess-fail.cwl mode change 100644 => 100755 tests/wf/networkaccess.cwl mode change 100644 => 100755 tests/wf/no-parameters-echo.cwl mode change 100644 => 100755 tests/wf/nvidia-smi-cc.cwl mode change 100644 => 100755 tests/wf/nvidia-smi-container.cwl mode change 100644 => 100755 tests/wf/nvidia-smi-max.cwl mode change 100644 => 100755 tests/wf/nvidia-smi-range.cwl mode change 100644 => 100755 tests/wf/nvidia-smi.cwl mode change 100644 => 100755 tests/wf/operation/abstract-cosifer.cwl mode change 100644 => 100755 tests/wf/operation/operation-single.cwl mode change 100644 => 100755 tests/wf/optional-numerical-output-0.cwl mode change 100644 => 100755 tests/wf/optional_src_mandatory_sink.cwl mode change 100644 => 100755 tests/wf/packed-with-loadlisting.cwl mode change 100644 => 100755 tests/wf/packed_no_main.cwl mode change 100644 => 100755 tests/wf/paramref_arguments_roundtrip.cwl mode change 100644 => 100755 tests/wf/paramref_arguments_self.cwl mode change 100644 => 100755 tests/wf/record_outputeval.cwl mode change 100644 => 100755 tests/wf/resreq_expr_float_v1_0.cwl mode change 100644 => 100755 tests/wf/resreq_expr_float_v1_2.cwl mode change 100644 => 100755 tests/wf/scatter2.cwl mode change 100644 => 100755 tests/wf/scatter2_subwf.cwl mode change 100644 => 100755 tests/wf/schemadef-bug-1473.cwl mode change 100644 => 100755 tests/wf/schemadef-tool-12.cwl mode change 100644 => 100755 tests/wf/schemadef-tool.cwl mode change 100644 => 100755 tests/wf/sec-tool.cwl mode change 100644 => 100755 tests/wf/sec-wf-out.cwl mode change 100644 => 100755 tests/wf/sec-wf.cwl mode change 100644 => 100755 tests/wf/secret_job.cwl mode change 100644 => 100755 tests/wf/secret_wf.cwl mode change 100644 => 100755 tests/wf/separate_without_prefix.cwl mode change 100644 => 100755 tests/wf/shm_size.cwl mode change 100644 => 100755 tests/wf/storage_float.cwl mode change 100644 => 100755 tests/wf/tar-param.cwl mode change 100644 => 100755 tests/wf/three_step_color.cwl mode change 100644 => 100755 tests/wf/timelimit-fail.cwl mode change 100644 => 100755 tests/wf/timelimit.cwl mode change 100644 => 100755 tests/wf/touch_tool.cwl mode change 100644 => 100755 tests/wf/trick_defaults.cwl mode change 100644 => 100755 tests/wf/trick_defaults2.cwl mode change 100644 => 100755 tests/wf/vf-concat.cwl mode change 100644 => 100755 tests/wf/workreuse-fail.cwl mode change 100644 => 100755 tests/wf/workreuse.cwl mode change 100644 => 100755 tests/with_doc.cwl mode change 100644 => 100755 tests/without_doc.cwl diff --git a/cwltool/schemas/v1.1.0-dev1/cwl-runner.cwl b/cwltool/schemas/v1.1.0-dev1/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.1/cwl-runner.cwl b/cwltool/schemas/v1.1/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.2.0-dev1/cwl-runner.cwl b/cwltool/schemas/v1.2.0-dev1/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.2.0-dev2/cwl-runner.cwl b/cwltool/schemas/v1.2.0-dev2/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.2.0-dev3/cwl-runner.cwl b/cwltool/schemas/v1.2.0-dev3/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.2.0-dev4/cwl-runner.cwl b/cwltool/schemas/v1.2.0-dev4/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.2.0-dev5/cwl-runner.cwl b/cwltool/schemas/v1.2.0-dev5/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.3.0-dev1/cwl-runner.cwl b/cwltool/schemas/v1.3.0-dev1/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/tests/CometAdapter.cwl b/tests/CometAdapter.cwl old mode 100644 new mode 100755 index ed1b24bb6..3d7083560 --- a/tests/CometAdapter.cwl +++ b/tests/CometAdapter.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner # Copyright (c) 2002-present, The OpenMS Team -- EKU Tuebingen, ETH Zurich, and FU Berlin # SPDX-License-Identifier: Apache-2.0 label: CometAdapter diff --git a/tests/arg-empty-prefix-separate-false.cwl b/tests/arg-empty-prefix-separate-false.cwl old mode 100644 new mode 100755 diff --git a/tests/checker_wf/optional_array_mismatch.cwl b/tests/checker_wf/optional_array_mismatch.cwl old mode 100644 new mode 100755 index d6f052757..a557927f2 --- a/tests/checker_wf/optional_array_mismatch.cwl +++ b/tests/checker_wf/optional_array_mismatch.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: Workflow inputs: diff --git a/tests/default_values_list.cwl b/tests/default_values_list.cwl old mode 100644 new mode 100755 diff --git a/tests/echo-badposition-expr.cwl b/tests/echo-badposition-expr.cwl old mode 100644 new mode 100755 diff --git a/tests/echo-position-expr.cwl b/tests/echo-position-expr.cwl old mode 100644 new mode 100755 diff --git a/tests/echo-stdout-log-dir.cwl b/tests/echo-stdout-log-dir.cwl old mode 100644 new mode 100755 diff --git a/tests/env.cwl b/tests/env.cwl old mode 100644 new mode 100755 diff --git a/tests/env2.cwl b/tests/env2.cwl old mode 100644 new mode 100755 diff --git a/tests/env3.cwl b/tests/env3.cwl old mode 100644 new mode 100755 diff --git a/tests/env4.cwl b/tests/env4.cwl old mode 100644 new mode 100755 diff --git a/tests/input_deps/docker-array-secondaryfiles.cwl b/tests/input_deps/docker-array-secondaryfiles.cwl old mode 100644 new mode 100755 diff --git a/tests/iwdr_bad_expr.cwl b/tests/iwdr_bad_expr.cwl old mode 100644 new mode 100755 diff --git a/tests/iwdr_dir_literal_real_file.cwl b/tests/iwdr_dir_literal_real_file.cwl old mode 100644 new mode 100755 diff --git a/tests/load_contents-array.cwl b/tests/load_contents-array.cwl old mode 100644 new mode 100755 index f6b786ec6..eaad62436 --- a/tests/load_contents-array.cwl +++ b/tests/load_contents-array.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: "v1.2" class: CommandLineTool baseCommand: echo diff --git a/tests/loop-ext/all-output-loop-no-iteration.cwl b/tests/loop-ext/all-output-loop-no-iteration.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/all-output-loop.cwl b/tests/loop-ext/all-output-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/default-value-loop.cwl b/tests/loop-ext/default-value-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-command-line-tool.cwl b/tests/loop-ext/invalid-loop-command-line-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-expression-tool.cwl b/tests/loop-ext/invalid-loop-expression-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-hint.cwl b/tests/loop-ext/invalid-loop-hint.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-scatter.cwl b/tests/loop-ext/invalid-loop-scatter.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-when-exception.cwl b/tests/loop-ext/invalid-loop-when-exception.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-when.cwl b/tests/loop-ext/invalid-loop-when.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-workflow.cwl b/tests/loop-ext/invalid-loop-workflow.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-multi-source-loop-no-requirement.cwl b/tests/loop-ext/invalid-multi-source-loop-no-requirement.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-no-loopWhen.cwl b/tests/loop-ext/invalid-no-loopWhen.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-non-boolean-loopWhen.cwl b/tests/loop-ext/invalid-non-boolean-loopWhen.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-non-boolean-loopWhen2.cwl b/tests/loop-ext/invalid-non-boolean-loopWhen2.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-value-from-loop-no-requirement.cwl b/tests/loop-ext/invalid-value-from-loop-no-requirement.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/loop-inside-loop-all.cwl b/tests/loop-ext/loop-inside-loop-all.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/loop-inside-loop.cwl b/tests/loop-ext/loop-inside-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/loop-inside-scatter.cwl b/tests/loop-ext/loop-inside-scatter.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/multi-source-loop.cwl b/tests/loop-ext/multi-source-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/opt-var-loop.cwl b/tests/loop-ext/opt-var-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/scatter-inside-loop.cwl b/tests/loop-ext/scatter-inside-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/single-var-loop-no-iteration.cwl b/tests/loop-ext/single-var-loop-no-iteration.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/single-var-loop.cwl b/tests/loop-ext/single-var-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/two-vars-loop-2.cwl b/tests/loop-ext/two-vars-loop-2.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/two-vars-loop.cwl b/tests/loop-ext/two-vars-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/value-from-loop.cwl b/tests/loop-ext/value-from-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/all-output-loop-no-iteration.cwl b/tests/loop/all-output-loop-no-iteration.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/all-output-loop.cwl b/tests/loop/all-output-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/default-value-loop.cwl b/tests/loop/default-value-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-loop-scatter.cwl b/tests/loop/invalid-loop-scatter.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-loop-when-exception.cwl b/tests/loop/invalid-loop-when-exception.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-loop-when-exception2.cwl b/tests/loop/invalid-loop-when-exception2.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-multi-source-loop-no-requirement.cwl b/tests/loop/invalid-multi-source-loop-no-requirement.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-no-loopWhen.cwl b/tests/loop/invalid-no-loopWhen.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-non-boolean-loopWhen.cwl b/tests/loop/invalid-non-boolean-loopWhen.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-non-boolean-loopWhen2.cwl b/tests/loop/invalid-non-boolean-loopWhen2.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-value-from-loop-no-requirement.cwl b/tests/loop/invalid-value-from-loop-no-requirement.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/loop-inside-loop-all.cwl b/tests/loop/loop-inside-loop-all.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/loop-inside-loop.cwl b/tests/loop/loop-inside-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/loop-inside-scatter.cwl b/tests/loop/loop-inside-scatter.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/multi-source-loop.cwl b/tests/loop/multi-source-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/opt-var-loop.cwl b/tests/loop/opt-var-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/scatter-inside-loop.cwl b/tests/loop/scatter-inside-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/single-var-loop-no-iteration.cwl b/tests/loop/single-var-loop-no-iteration.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/single-var-loop.cwl b/tests/loop/single-var-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/two-vars-loop-2.cwl b/tests/loop/two-vars-loop-2.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/two-vars-loop.cwl b/tests/loop/two-vars-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/value-from-loop.cwl b/tests/loop/value-from-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/nested-array.cwl b/tests/nested-array.cwl old mode 100644 new mode 100755 index 8272614fc..6aa3650b8 --- a/tests/nested-array.cwl +++ b/tests/nested-array.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool baseCommand: echo diff --git a/tests/non_portable.cwl b/tests/non_portable.cwl old mode 100644 new mode 100755 diff --git a/tests/non_portable2.cwl b/tests/non_portable2.cwl old mode 100644 new mode 100755 diff --git a/tests/override/env-tool.cwl b/tests/override/env-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/override/env-tool_v1.1.0-dev1.cwl b/tests/override/env-tool_v1.1.0-dev1.cwl old mode 100644 new mode 100755 diff --git a/tests/override/env-tool_v1.1.cwl b/tests/override/env-tool_v1.1.cwl old mode 100644 new mode 100755 diff --git a/tests/portable.cwl b/tests/portable.cwl old mode 100644 new mode 100755 diff --git a/tests/reloc/test.cwl b/tests/reloc/test.cwl old mode 100644 new mode 100755 index 41d37474e..c88cf48da --- a/tests/reloc/test.cwl +++ b/tests/reloc/test.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool inputs: diff --git a/tests/scatter_numbers.cwl b/tests/scatter_numbers.cwl old mode 100644 new mode 100755 diff --git a/tests/secondary-files-bad.cwl b/tests/secondary-files-bad.cwl old mode 100644 new mode 100755 diff --git a/tests/secondary-files-required-container.cwl b/tests/secondary-files-required-container.cwl old mode 100644 new mode 100755 diff --git a/tests/secondary-files-required-missing-container.cwl b/tests/secondary-files-required-missing-container.cwl old mode 100644 new mode 100755 diff --git a/tests/secondary-files-string-v1.cwl b/tests/secondary-files-string-v1.cwl old mode 100644 new mode 100755 diff --git a/tests/secondary-files.cwl b/tests/secondary-files.cwl old mode 100644 new mode 100755 diff --git a/tests/sing_pullfolder_test.cwl b/tests/sing_pullfolder_test.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/1432.cwl b/tests/subgraph/1432.cwl old mode 100644 new mode 100755 index 54d553875..41283abfe --- a/tests/subgraph/1432.cwl +++ b/tests/subgraph/1432.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: Workflow cwlVersion: v1.2 inputs: diff --git a/tests/subgraph/env-tool2.cwl b/tests/subgraph/env-tool2.cwl old mode 100644 new mode 100755 index f568e5450..14c8a6826 --- a/tests/subgraph/env-tool2.cwl +++ b/tests/subgraph/env-tool2.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: CommandLineTool cwlVersion: v1.2 inputs: diff --git a/tests/subgraph/env-tool2_no_env.cwl b/tests/subgraph/env-tool2_no_env.cwl old mode 100644 new mode 100755 index c4a6f6327..2b3711b77 --- a/tests/subgraph/env-tool2_no_env.cwl +++ b/tests/subgraph/env-tool2_no_env.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: CommandLineTool cwlVersion: v1.2 inputs: diff --git a/tests/subgraph/env-tool2_req.cwl b/tests/subgraph/env-tool2_req.cwl old mode 100644 new mode 100755 index 96b0bc3d1..943108056 --- a/tests/subgraph/env-tool2_req.cwl +++ b/tests/subgraph/env-tool2_req.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: CommandLineTool cwlVersion: v1.2 inputs: diff --git a/tests/subgraph/env-wf2.cwl b/tests/subgraph/env-wf2.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_hint_collision.cwl b/tests/subgraph/env-wf2_hint_collision.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_hint_req_collision.cwl b/tests/subgraph/env-wf2_hint_req_collision.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_long.cwl b/tests/subgraph/env-wf2_long.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_only_hint.cwl b/tests/subgraph/env-wf2_only_hint.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_req_collision.cwl b/tests/subgraph/env-wf2_req_collision.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_subwf-packed.cwl b/tests/subgraph/env-wf2_subwf-packed.cwl old mode 100644 new mode 100755 index ed119f0f2..e1c09649d --- a/tests/subgraph/env-wf2_subwf-packed.cwl +++ b/tests/subgraph/env-wf2_subwf-packed.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner { "$graph": [ { @@ -127,4 +128,4 @@ } ], "cwlVersion": "v1.2" -} \ No newline at end of file +} diff --git a/tests/subgraph/env-wf2_subwf.cwl b/tests/subgraph/env-wf2_subwf.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_subwf_b.cwl b/tests/subgraph/env-wf2_subwf_b.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/steplevel-resreq.cwl b/tests/subgraph/steplevel-resreq.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/timelimit2-wf.cwl b/tests/subgraph/timelimit2-wf.cwl old mode 100644 new mode 100755 diff --git a/tests/test_examples.py b/tests/test_examples.py index f413976fd..b78965bb4 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1878,7 +1878,7 @@ def test_invalid_nested_array() -> None: stdout = re.sub(r"\s\s+", " ", stdout) assert "Tool definition failed validation:" in stdout assert ( - "tests/nested-array.cwl:6:5: Field 'type' references unknown identifier 'string[][]'" + "tests/nested-array.cwl:7:5: Field 'type' references unknown identifier 'string[][]'" ) in stdout diff --git a/tests/test_validate.py b/tests/test_validate.py index f2d89e473..8e664d9aa 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -70,7 +70,7 @@ def test_validate_quiet() -> None: stdout = re.sub(r"\s\s+", " ", stdout) assert "INFO" not in stdout assert "INFO" not in stderr - assert "tests/CometAdapter.cwl:9:3: object id" in stdout + assert "tests/CometAdapter.cwl:10:3: object id" in stdout assert "tests/CometAdapter.cwl#out' previously defined" in stdout @@ -119,9 +119,9 @@ def test_validate_custom_logger() -> None: assert "WARNING" not in stdout assert "WARNING" not in stderr assert "WARNING" in custom_log_text - assert "tests/CometAdapter.cwl:9:3: object id" not in stdout - assert "tests/CometAdapter.cwl:9:3: object id" not in stderr - assert "tests/CometAdapter.cwl:9:3: object id" in custom_log_text + assert "tests/CometAdapter.cwl:10:3: object id" not in stdout + assert "tests/CometAdapter.cwl:10:3: object id" not in stderr + assert "tests/CometAdapter.cwl:10:3: object id" in custom_log_text assert "tests/CometAdapter.cwl#out' previously defined" not in stdout assert "tests/CometAdapter.cwl#out' previously defined" not in stderr assert "tests/CometAdapter.cwl#out' previously defined" in custom_log_text diff --git a/tests/trs/Dockstore.cwl b/tests/trs/Dockstore.cwl old mode 100644 new mode 100755 diff --git a/tests/trs/md5sum-tool.cwl b/tests/trs/md5sum-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/trs/md5sum-workflow.cwl b/tests/trs/md5sum-workflow.cwl old mode 100644 new mode 100755 diff --git a/tests/utf_doc_example.cwl b/tests/utf_doc_example.cwl old mode 100644 new mode 100755 diff --git a/tests/wc-tool-bad-hints.cwl b/tests/wc-tool-bad-hints.cwl old mode 100644 new mode 100755 diff --git a/tests/wc-tool-bad-reqs.cwl b/tests/wc-tool-bad-reqs.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/1496.cwl b/tests/wf/1496.cwl old mode 100644 new mode 100755 index 74f6ef49e..d5da025dd --- a/tests/wf/1496.cwl +++ b/tests/wf/1496.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool diff --git a/tests/wf/1590.cwl b/tests/wf/1590.cwl old mode 100644 new mode 100755 index 19c8ad331..de602a500 --- a/tests/wf/1590.cwl +++ b/tests/wf/1590.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner { "baseCommand": [ "cat" diff --git a/tests/wf/1st-workflow.cwl b/tests/wf/1st-workflow.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/811-12.cwl b/tests/wf/811-12.cwl old mode 100644 new mode 100755 index f4403f45b..ab0b2bd92 --- a/tests/wf/811-12.cwl +++ b/tests/wf/811-12.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: Workflow diff --git a/tests/wf/811.cwl b/tests/wf/811.cwl old mode 100644 new mode 100755 index 16a4c828d..8ef826fdc --- a/tests/wf/811.cwl +++ b/tests/wf/811.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: Workflow diff --git a/tests/wf/816_tool.cwl b/tests/wf/816_tool.cwl old mode 100644 new mode 100755 index 00395abe8..c48d3e373 --- a/tests/wf/816_tool.cwl +++ b/tests/wf/816_tool.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: CommandLineTool requirements: diff --git a/tests/wf/816_wf.cwl b/tests/wf/816_wf.cwl old mode 100644 new mode 100755 index a64b130b6..9db143916 --- a/tests/wf/816_wf.cwl +++ b/tests/wf/816_wf.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: Workflow cwlVersion: v1.0 inputs: diff --git a/tests/wf/910.cwl b/tests/wf/910.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/arguments.cwl b/tests/wf/arguments.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/bad-stderr-expr.cwl b/tests/wf/bad-stderr-expr.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/bad-stdin-expr.cwl b/tests/wf/bad-stdin-expr.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/bad-stdout-expr.cwl b/tests/wf/bad-stdout-expr.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/bad_networkaccess.cwl b/tests/wf/bad_networkaccess.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/bad_timelimit.cwl b/tests/wf/bad_timelimit.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/cache_test_workflow.cwl b/tests/wf/cache_test_workflow.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/cat-tool.cwl b/tests/wf/cat-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/conditional_step_no_inputs.cwl b/tests/wf/conditional_step_no_inputs.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/conflict.cwl b/tests/wf/conflict.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/cores_float.cwl b/tests/wf/cores_float.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/default-dir5.cwl b/tests/wf/default-dir5.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/default-wf5.cwl b/tests/wf/default-wf5.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/double-nested.cwl b/tests/wf/double-nested.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/expect_trick_packed.cwl b/tests/wf/expect_trick_packed.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/extract_region_specs.cwl b/tests/wf/extract_region_specs.cwl old mode 100644 new mode 100755 index 279fa4400..437222a13 --- a/tests/wf/extract_region_specs.cwl +++ b/tests/wf/extract_region_specs.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner { "cwlVersion": "v1.0", "class": "CommandLineTool", diff --git a/tests/wf/floats_small_and_large.cwl b/tests/wf/floats_small_and_large.cwl old mode 100644 new mode 100755 index 434327361..1637bd9af --- a/tests/wf/floats_small_and_large.cwl +++ b/tests/wf/floats_small_and_large.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: CommandLineTool baseCommand: echo diff --git a/tests/wf/generator/pytoolgen.cwl b/tests/wf/generator/pytoolgen.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/input_named_id.cwl b/tests/wf/input_named_id.cwl old mode 100644 new mode 100755 index e559f967b..82a0353aa --- a/tests/wf/input_named_id.cwl +++ b/tests/wf/input_named_id.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner label: FeatureFinderIdentification doc: "" inputs: diff --git a/tests/wf/iwd-passthrough1.cwl b/tests/wf/iwd-passthrough1.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/iwdr-empty.cwl b/tests/wf/iwdr-empty.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/iwdr-entry.cwl b/tests/wf/iwdr-entry.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/iwdr-passthrough-successive.cwl b/tests/wf/iwdr-passthrough-successive.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/literalfile.cwl b/tests/wf/literalfile.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/malformed_outputs.cwl b/tests/wf/malformed_outputs.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/missing-tool.cwl b/tests/wf/missing-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/mpi_env.cwl b/tests/wf/mpi_env.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/mpi_expr.cwl b/tests/wf/mpi_expr.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/mpi_line_count.cwl b/tests/wf/mpi_line_count.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/mpi_simple.cwl b/tests/wf/mpi_simple.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/mpi_simple_wf.cwl b/tests/wf/mpi_simple_wf.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/nested.cwl b/tests/wf/nested.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/networkaccess-fail.cwl b/tests/wf/networkaccess-fail.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/networkaccess.cwl b/tests/wf/networkaccess.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/no-parameters-echo.cwl b/tests/wf/no-parameters-echo.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/nvidia-smi-cc.cwl b/tests/wf/nvidia-smi-cc.cwl old mode 100644 new mode 100755 index a4f315b0e..92e781d26 --- a/tests/wf/nvidia-smi-cc.cwl +++ b/tests/wf/nvidia-smi-cc.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool $namespaces: diff --git a/tests/wf/nvidia-smi-container.cwl b/tests/wf/nvidia-smi-container.cwl old mode 100644 new mode 100755 index 84fd72d83..22092fae4 --- a/tests/wf/nvidia-smi-container.cwl +++ b/tests/wf/nvidia-smi-container.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool $namespaces: diff --git a/tests/wf/nvidia-smi-max.cwl b/tests/wf/nvidia-smi-max.cwl old mode 100644 new mode 100755 index d3d4d5e9c..1f8687d4d --- a/tests/wf/nvidia-smi-max.cwl +++ b/tests/wf/nvidia-smi-max.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool $namespaces: diff --git a/tests/wf/nvidia-smi-range.cwl b/tests/wf/nvidia-smi-range.cwl old mode 100644 new mode 100755 index 19d3ea43c..2e131a373 --- a/tests/wf/nvidia-smi-range.cwl +++ b/tests/wf/nvidia-smi-range.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool $namespaces: diff --git a/tests/wf/nvidia-smi.cwl b/tests/wf/nvidia-smi.cwl old mode 100644 new mode 100755 index 8e227d0c5..8a17f8e17 --- a/tests/wf/nvidia-smi.cwl +++ b/tests/wf/nvidia-smi.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool $namespaces: diff --git a/tests/wf/operation/abstract-cosifer.cwl b/tests/wf/operation/abstract-cosifer.cwl old mode 100644 new mode 100755 index 057620844..2a50cded7 --- a/tests/wf/operation/abstract-cosifer.cwl +++ b/tests/wf/operation/abstract-cosifer.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: Operation cwlVersion: v1.2 diff --git a/tests/wf/operation/operation-single.cwl b/tests/wf/operation/operation-single.cwl old mode 100644 new mode 100755 index 2119bf02e..63b6a6bf0 --- a/tests/wf/operation/operation-single.cwl +++ b/tests/wf/operation/operation-single.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: Workflow cwlVersion: v1.2 id: abstract_cosifer_workflow diff --git a/tests/wf/optional-numerical-output-0.cwl b/tests/wf/optional-numerical-output-0.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/optional_src_mandatory_sink.cwl b/tests/wf/optional_src_mandatory_sink.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/packed-with-loadlisting.cwl b/tests/wf/packed-with-loadlisting.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/packed_no_main.cwl b/tests/wf/packed_no_main.cwl old mode 100644 new mode 100755 index 8307a083e..aafd0de58 --- a/tests/wf/packed_no_main.cwl +++ b/tests/wf/packed_no_main.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 $graph: - id: echo diff --git a/tests/wf/paramref_arguments_roundtrip.cwl b/tests/wf/paramref_arguments_roundtrip.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/paramref_arguments_self.cwl b/tests/wf/paramref_arguments_self.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/record_outputeval.cwl b/tests/wf/record_outputeval.cwl old mode 100644 new mode 100755 index 45daf9b4d..e7b59cb6c --- a/tests/wf/record_outputeval.cwl +++ b/tests/wf/record_outputeval.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool requirements: diff --git a/tests/wf/resreq_expr_float_v1_0.cwl b/tests/wf/resreq_expr_float_v1_0.cwl old mode 100644 new mode 100755 index d5ea08cde..a36877c0f --- a/tests/wf/resreq_expr_float_v1_0.cwl +++ b/tests/wf/resreq_expr_float_v1_0.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: CommandLineTool requirements: diff --git a/tests/wf/resreq_expr_float_v1_2.cwl b/tests/wf/resreq_expr_float_v1_2.cwl old mode 100644 new mode 100755 index cfb1b2a13..ca72364b7 --- a/tests/wf/resreq_expr_float_v1_2.cwl +++ b/tests/wf/resreq_expr_float_v1_2.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool requirements: diff --git a/tests/wf/scatter2.cwl b/tests/wf/scatter2.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/scatter2_subwf.cwl b/tests/wf/scatter2_subwf.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/schemadef-bug-1473.cwl b/tests/wf/schemadef-bug-1473.cwl old mode 100644 new mode 100755 index ad87ae08e..beab3211d --- a/tests/wf/schemadef-bug-1473.cwl +++ b/tests/wf/schemadef-bug-1473.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner { "$graph": [ { diff --git a/tests/wf/schemadef-tool-12.cwl b/tests/wf/schemadef-tool-12.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/schemadef-tool.cwl b/tests/wf/schemadef-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/sec-tool.cwl b/tests/wf/sec-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/sec-wf-out.cwl b/tests/wf/sec-wf-out.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/sec-wf.cwl b/tests/wf/sec-wf.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/secret_job.cwl b/tests/wf/secret_job.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/secret_wf.cwl b/tests/wf/secret_wf.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/separate_without_prefix.cwl b/tests/wf/separate_without_prefix.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/shm_size.cwl b/tests/wf/shm_size.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/storage_float.cwl b/tests/wf/storage_float.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/tar-param.cwl b/tests/wf/tar-param.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/three_step_color.cwl b/tests/wf/three_step_color.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/timelimit-fail.cwl b/tests/wf/timelimit-fail.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/timelimit.cwl b/tests/wf/timelimit.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/touch_tool.cwl b/tests/wf/touch_tool.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/trick_defaults.cwl b/tests/wf/trick_defaults.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/trick_defaults2.cwl b/tests/wf/trick_defaults2.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/vf-concat.cwl b/tests/wf/vf-concat.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/workreuse-fail.cwl b/tests/wf/workreuse-fail.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/workreuse.cwl b/tests/wf/workreuse.cwl old mode 100644 new mode 100755 diff --git a/tests/with_doc.cwl b/tests/with_doc.cwl old mode 100644 new mode 100755 diff --git a/tests/without_doc.cwl b/tests/without_doc.cwl old mode 100644 new mode 100755 From 289b10887406115c0fe189b0e4532a8ca5bd5013 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Feb 2025 07:51:11 +0000 Subject: [PATCH 49/78] Bump mypy from 1.14.1 to 1.15.0 Bumps [mypy](https://github.com/python/mypy) from 1.14.1 to 1.15.0. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.14.1...v1.15.0) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- mypy-requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 760428998..a29e35bb2 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.14.1 # also update pyproject.toml +mypy==1.15.0 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 cwltest diff --git a/pyproject.toml b/pyproject.toml index b243171fa..cb7d837a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.14.1", # also update mypy-requirements.txt + "mypy==1.15.0", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", From f741c0d4dfd789e364ca7b4745a733a9e2c91164 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 5 Feb 2025 09:33:17 +0100 Subject: [PATCH 50/78] setup.py: modernize use of mypycify; switch to skiplist --- setup.py | 111 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 68 insertions(+), 43 deletions(-) diff --git a/setup.py b/setup.py index d3fef7b26..181154a0e 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,15 @@ #!/usr/bin/env python3 """Setup for the reference implementation of the CWL standards.""" +import glob import os import sys import warnings +from typing import TYPE_CHECKING, Any -from setuptools import setup +from setuptools import Extension, setup + +if TYPE_CHECKING: + from typing_extensions import TypeGuard if os.name == "nt": warnings.warn( @@ -20,6 +25,31 @@ stacklevel=1, ) + +def _is_list_of_setuptools_extension(items: list[Any]) -> "TypeGuard[list[Extension]]": + return all(isinstance(item, Extension) for item in items) + + +def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> list[str]: + """ + Find all interesting data files, for setup(package_data=). + + Arguments: + root: The directory to search in. + globs: A list of glob patterns to accept files. + """ + rv_dirs = [root for root, dirs, files in os.walk(base)] + rv = [] + for rv_dir in rv_dirs: + files = [] + for pat in globs: + files += glob.glob(os.path.join(rv_dir, pat)) + if not files: + continue + rv.extend([os.path.relpath(f, root) for f in files]) + return rv + + SETUP_DIR = os.path.dirname(__file__) README = os.path.join(SETUP_DIR, "README.rst") @@ -34,55 +64,50 @@ USE_MYPYC = True if USE_MYPYC: - mypyc_targets = [ - "cwltool/argparser.py", - "cwltool/builder.py", - "cwltool/checker.py", - "cwltool/command_line_tool.py", - # "cwltool/context.py", # monkeypatching - "cwltool/cwlrdf.py", - "cwltool/docker_id.py", - "cwltool/docker.py", - "cwltool/udocker.py", - "cwltool/errors.py", - "cwltool/executors.py", - "cwltool/factory.py", - "cwltool/flatten.py", - # "cwltool/__init__.py", - "cwltool/job.py", - "cwltool/load_tool.py", - # "cwltool/loghandler.py", # so we can monkeypatch the logger from tests - # "cwltool/__main__.py", - "cwltool/main.py", - "cwltool/mutation.py", - "cwltool/pack.py", - "cwltool/pathmapper.py", - "cwltool/process.py", - "cwltool/procgenerator.py", - # "cwltool/cwlprov/__init__.py", - "cwltool/cwlprov/provenance_constants.py", - "cwltool/cwlprov/provenance_profile.py", - "cwltool/cwlprov/ro.py", - # "cwltool/cwlprov/writablebagfile.py", # WritableBag is having issues - "cwltool/resolver.py", - "cwltool/secrets.py", - "cwltool/singularity.py", - "cwltool/software_requirements.py", - # "cwltool/stdfsaccess.py", # StdFsAccess needs to be subclassable - "cwltool/subgraph.py", - "cwltool/update.py", - "cwltool/utils.py", - "cwltool/validate_js.py", - "cwltool/workflow.py", + mypyc_skiplist = tuple( + os.path.join("cwltool", x) + for x in ( + "context.py", # monkeypatching + "__init__.py", + "loghandler.py", # so we can monkeypatch the logger from tests + "__main__.py", + "cwlprov/__init__.py", + "cuda.py", # for monkeypatch + "run_job.py", + "cwlprov/writablebagfile.py", # WritableBag is having issues + "stdfsaccess.py", # StdFsAccess needs to be subclassable + ) + ) + + everything = [os.path.join("cwltool", x) for x in _find_package_data("cwltool", ["*.py"])] + # Start with all the .py files + all_real_pys = [ + x for x in everything if not x.startswith(os.path.join("mypy", "typeshed") + os.sep) ] + # Strip out anything in our skiplist + mypyc_targets = [x for x in all_real_pys if x not in mypyc_skiplist] + + # Strip out any test code + mypyc_targets = [x for x in mypyc_targets if not x.startswith(("tests" + os.sep))] - from mypyc.build import mypycify # type: ignore[import-untyped] + mypyc_targets.sort() + + from mypyc.build import mypycify opt_level = os.getenv("MYPYC_OPT_LEVEL", "3") - ext_modules = mypycify(mypyc_targets, opt_level=opt_level) + debug_level = os.getenv("MYPYC_DEBUG_LEVEL", "1") + force_multifile = os.getenv("MYPYC_MULTI_FILE", "") == "1" + ext_modules = mypycify( + mypyc_targets, + opt_level=opt_level, + debug_level=debug_level, + multi_file=force_multifile, + ) else: ext_modules = [] +assert _is_list_of_setuptools_extension(ext_modules), "Expected mypycify to use setuptools" + setup( name="cwltool", description="Common workflow language reference implementation", From 393b6982d31838cb8f4c938a009a8fa24021dab7 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 5 Feb 2025 10:18:40 +0100 Subject: [PATCH 51/78] cwlviewer: use importlib instead of __file__. --- cwltool/cwlviewer.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/cwltool/cwlviewer.py b/cwltool/cwlviewer.py index 769343964..36166c485 100644 --- a/cwltool/cwlviewer.py +++ b/cwltool/cwlviewer.py @@ -1,18 +1,28 @@ """Visualize a CWL workflow.""" from collections.abc import Iterator -from pathlib import Path +from importlib.resources import files from typing import cast from urllib.parse import urlparse import pydot import rdflib -_queries_dir = (Path(__file__).parent / "rdfqueries").resolve() -_get_inner_edges_query_path = _queries_dir / "get_inner_edges.sparql" -_get_input_edges_query_path = _queries_dir / "get_input_edges.sparql" -_get_output_edges_query_path = _queries_dir / "get_output_edges.sparql" -_get_root_query_path = _queries_dir / "get_root.sparql" + +def _get_inner_edges_query() -> str: + return files("cwltool").joinpath("rdfqueries/get_inner_edges.sparql").read_text() + + +def _get_input_edges_query() -> str: + return files("cwltool").joinpath("rdfqueries/get_input_edges.sparql").read_text() + + +def _get_output_edges_query() -> str: + return files("cwltool").joinpath("rdfqueries/get_output_edges.sparql").read_text() + + +def _get_root_query() -> str: + return files("cwltool").joinpath("rdfqueries/get_root.sparql").read_text() class CWLViewer: @@ -33,8 +43,7 @@ def _load_cwl_graph(self, rdf_description: str) -> rdflib.graph.Graph: return rdf_graph def _set_inner_edges(self) -> None: - with open(_get_inner_edges_query_path) as f: - get_inner_edges_query = f.read() + get_inner_edges_query = _get_inner_edges_query() inner_edges = cast( Iterator[rdflib.query.ResultRow], self._rdf_graph.query( @@ -96,8 +105,7 @@ def _set_inner_edges(self) -> None: ) def _set_input_edges(self) -> None: - with open(_get_input_edges_query_path) as f: - get_input_edges_query = f.read() + get_input_edges_query = _get_input_edges_query() inputs_subgraph = pydot.Subgraph(graph_name="cluster_inputs") self._dot_graph.add_subgraph(inputs_subgraph) inputs_subgraph.set("rank", "same") @@ -124,8 +132,7 @@ def _set_input_edges(self) -> None: self._dot_graph.add_edge(pydot.Edge(str(input_row["input"]), str(input_row["step"]))) def _set_output_edges(self) -> None: - with open(_get_output_edges_query_path) as f: - get_output_edges = f.read() + get_output_edges = _get_output_edges_query() outputs_graph = pydot.Subgraph(graph_name="cluster_outputs") self._dot_graph.add_subgraph(outputs_graph) outputs_graph.set("rank", "same") @@ -152,8 +159,7 @@ def _set_output_edges(self) -> None: self._dot_graph.add_edge(pydot.Edge(output_edge_row["step"], output_edge_row["output"])) def _get_root_graph_uri(self) -> rdflib.term.Identifier: - with open(_get_root_query_path) as f: - get_root_query = f.read() + get_root_query = _get_root_query() root = cast( list[rdflib.query.ResultRow], list( From dc7fd2e6d768f6ddf1be38edcd4c910b63e85e2f Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 5 Feb 2025 10:48:16 +0100 Subject: [PATCH 52/78] fix type errors discovered by mypyc --- cwltool/cuda.py | 7 ++++--- cwltool/singularity_utils.py | 14 +++++++------- cwltool/workflow_job.py | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/cwltool/cuda.py b/cwltool/cuda.py index 1394ec239..d607b83da 100644 --- a/cwltool/cuda.py +++ b/cwltool/cuda.py @@ -2,6 +2,7 @@ import subprocess # nosec import xml.dom.minidom # nosec +from typing import Union from .loghandler import _logger from .utils import CWLObjectType @@ -10,9 +11,9 @@ def cuda_version_and_device_count() -> tuple[str, int]: """Determine the CUDA version and number of attached CUDA GPUs.""" try: - out = subprocess.check_output(["nvidia-smi", "-q", "-x"]) # nosec + out: Union[str, bytes] = subprocess.check_output(["nvidia-smi", "-q", "-x"]) # nosec except Exception as e: - _logger.warning("Error checking CUDA version with nvidia-smi: %s", e) + _logger.warning("Error checking CUDA version with nvidia-smi: %s", e, exc_info=e) return ("", 0) dm = xml.dom.minidom.parseString(out) # nosec @@ -62,5 +63,5 @@ def cuda_check(cuda_req: CWLObjectType, requestCount: int) -> int: return 0 return requestCount except Exception as e: - _logger.warning("Error checking CUDA requirements: %s", e) + _logger.warning("Error checking CUDA requirements: %s", e, exc_info=e) return 0 diff --git a/cwltool/singularity_utils.py b/cwltool/singularity_utils.py index e4cc88918..13f7ed3f6 100644 --- a/cwltool/singularity_utils.py +++ b/cwltool/singularity_utils.py @@ -2,7 +2,7 @@ import os import os.path -from subprocess import DEVNULL, PIPE, Popen, TimeoutExpired # nosec +import subprocess # nosec from typing import Optional _USERNS: Optional[bool] = None @@ -14,17 +14,17 @@ def singularity_supports_userns() -> bool: if _USERNS is None: try: hello_image = os.path.join(os.path.dirname(__file__), "hello.simg") - result = Popen( # nosec + result = subprocess.run( # nosec ["singularity", "exec", "--userns", hello_image, "true"], - stderr=PIPE, - stdout=DEVNULL, - universal_newlines=True, - ).communicate(timeout=60)[1] + capture_output=True, + timeout=60, + text=True, + ).stderr _USERNS = ( "No valid /bin/sh" in result or "/bin/sh doesn't exist in container" in result or "executable file not found in" in result ) - except TimeoutExpired: + except subprocess.TimeoutExpired: _USERNS = False return _USERNS diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 6cd0b2e7c..b552641e1 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -406,7 +406,7 @@ def object_from_state( ("merge_nested" if len(connections) > 1 else None), ), ), - valueFrom=cast(str, inp.get("valueFrom")), + valueFrom=cast(Optional[str], inp.get("valueFrom")), ): raise WorkflowException( "Type mismatch between source '%s' (%s) and " From a1de5fad7ce211440b1ad4972e3e42b59ecd8ac3 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 5 Feb 2025 19:25:30 +0100 Subject: [PATCH 53/78] more exception tracebacks, when in debug mode --- cwltool/command_line_tool.py | 2 +- cwltool/cwlprov/provenance_profile.py | 2 +- cwltool/executors.py | 5 ++++- cwltool/job.py | 27 ++++++++++++++++++++------- cwltool/main.py | 2 +- cwltool/procgenerator.py | 6 +++--- cwltool/resolver.py | 4 ++-- cwltool/workflow.py | 4 +++- 8 files changed, 35 insertions(+), 17 deletions(-) diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 1fe1a7044..2319e6211 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -1342,7 +1342,7 @@ def collect_output( ] ) except OSError as e: - _logger.warning(str(e)) + _logger.warning(str(e), exc_info=builder.debug) except Exception: _logger.error("Unexpected error from fs_access", exc_info=True) raise diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index e8538e51b..e2208378f 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -546,7 +546,7 @@ def declare_artefact(self, value: Any) -> ProvEntity: # FIXME: list value does not support adding "@id" return coll except TypeError: - _logger.warning("Unrecognized type %s of %r", type(value), value) + _logger.warning("Unrecognized type %s of %r", type(value), value, exc_info=True) # Let's just fall back to Python repr() entity = self.document.entity(uuid.uuid4().urn, {PROV_LABEL: repr(value)}) self.research_object.add_uri(entity.identifier.uri) diff --git a/cwltool/executors.py b/cwltool/executors.py index 33198d854..9d0559726 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -326,7 +326,10 @@ def _runner( self.exceptions.append(err) except Exception as err: # pylint: disable=broad-except _logger.exception(f"Got workflow error: {err}") - self.exceptions.append(WorkflowException(str(err))) + wf_exc = WorkflowException(str(err)) + wf_exc.__cause__ = err + wf_exc.__suppress_context__ = True + self.exceptions.append(wf_exc) finally: if runtime_context.workflow_eval_lock: with runtime_context.workflow_eval_lock: diff --git a/cwltool/job.py b/cwltool/job.py index b360be25f..2c6bb9f77 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -376,17 +376,30 @@ def stderr_stdout_log_path( except OSError as e: if e.errno == 2: if runtime: - _logger.error("'%s' not found: %s", runtime[0], str(e)) + _logger.error( + "'%s' not found: %s", runtime[0], str(e), exc_info=runtimeContext.debug + ) else: - _logger.error("'%s' not found: %s", self.command_line[0], str(e)) + _logger.error( + "'%s' not found: %s", + self.command_line[0], + str(e), + exc_info=runtimeContext.debug, + ) else: - _logger.exception("Exception while running job") + _logger.exception( + "Exception while running job: %s", str(e), exc_info=runtimeContext.debug + ) processStatus = "permanentFail" except WorkflowException as err: - _logger.error("[job %s] Job error:\n%s", self.name, str(err)) + _logger.error( + "[job %s] Job error:\n%s", self.name, str(err), exc_info=runtimeContext.debug + ) processStatus = "permanentFail" - except Exception: - _logger.exception("Exception while running job") + except Exception as err: + _logger.exception( + "Exception while running job: %s.", str(err), exc_info=runtimeContext.debug + ) processStatus = "permanentFail" if ( runtimeContext.research_obj is not None @@ -795,7 +808,7 @@ def run( ) except Exception as err: container = "Singularity" if runtimeContext.singularity else "Docker" - _logger.debug("%s error", container, exc_info=True) + _logger.debug("%s error", container, exc_info=runtimeContext.debug) if docker_is_req: raise UnsupportedRequirement( f"{container} is required to run this tool: {str(err)}" diff --git a/cwltool/main.py b/cwltool/main.py index c658c3685..90cb2e2c8 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -1289,7 +1289,7 @@ def main( if isinstance(err.code, int): return err.code else: - _logger.debug("Non-integer SystemExit: %s", err.code) + _logger.debug("Non-integer SystemExit: %s", err.code, exc_info=args.debug) return 1 del args.workflow diff --git a/cwltool/procgenerator.py b/cwltool/procgenerator.py index 9839ce5d4..07123f906 100644 --- a/cwltool/procgenerator.py +++ b/cwltool/procgenerator.py @@ -57,7 +57,7 @@ def job( except WorkflowException: raise except Exception as exc: - _logger.exception("Unexpected exception") + _logger.exception("Unexpected exception", exc_info=runtimeContext.debug) raise WorkflowException(str(exc)) from exc @@ -80,7 +80,7 @@ def __init__( self.embedded_tool = load_tool(toolpath_object["run"], loadingContext) except ValidationException as vexc: if loadingContext.debug: - _logger.exception("Validation exception") + _logger.exception("Validation exception", exc_info=loadingContext.debug) raise WorkflowException( "Tool definition %s failed validation:\n%s" % (toolpath_object["run"], indent(str(vexc))) @@ -108,7 +108,7 @@ def result( ) except ValidationException as vexc: if runtimeContext.debug: - _logger.exception("Validation exception") + _logger.exception("Validation exception", exc_info=runtimeContext.debug) raise WorkflowException( "Tool definition %s failed validation:\n%s" % (jobout["runProcess"], indent(str(vexc))) diff --git a/cwltool/resolver.py b/cwltool/resolver.py index e48957f26..918a9b24e 100644 --- a/cwltool/resolver.py +++ b/cwltool/resolver.py @@ -15,8 +15,8 @@ def resolve_local(document_loader: Optional[Loader], uri: str) -> Optional[str]: try: pathobj = Path(pathpart).resolve() - except OSError: - _logger.debug("local resolver could not resolve %s", uri) + except OSError as exc: + _logger.debug("local resolver could not resolve %s due to %s", uri, str(exc)) return None if pathobj.is_file(): diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 899ac4643..a6e2ba189 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -442,7 +442,9 @@ def job( runtimeContext, ) except WorkflowException: - _logger.error("Exception on step '%s'", runtimeContext.name) + _logger.error( + "Exception on step '%s'", runtimeContext.name, exc_info=runtimeContext.debug + ) raise except Exception as exc: _logger.exception("Unexpected exception") From a4d67ac16615e336f74aa90cb7efb512e9c12b8f Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 11 Feb 2025 18:22:17 +0100 Subject: [PATCH 54/78] gh-actions: remove one last usage of the now-deprecated Ubuntu 20.04 --- .github/workflows/ci-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 2ebb14c5f..1295eebf1 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -89,7 +89,7 @@ jobs: tox-style: name: Linters - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: matrix: From 38555fc8d0c678fa39af66d22fc78310b6868489 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Feb 2025 07:43:50 +0000 Subject: [PATCH 55/78] Update galaxy-util requirement from <24.2 to <24.3 Updates the requirements on [galaxy-util](https://github.com/galaxyproject/galaxy) to permit the latest version. - [Release notes](https://github.com/galaxyproject/galaxy/releases) - [Commits](https://github.com/galaxyproject/galaxy/compare/galaxy-util-19.9.0...v24.2.0) --- updated-dependencies: - dependency-name: galaxy-util dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- mypy-requirements.txt | 2 +- setup.py | 2 +- test-requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index a29e35bb2..7ebcd84e5 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -7,4 +7,4 @@ types-setuptools types-psutil types-mock galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.2 -galaxy-util<24.2 +galaxy-util<24.3 diff --git a/setup.py b/setup.py index 181154a0e..bb207b4fe 100644 --- a/setup.py +++ b/setup.py @@ -165,7 +165,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li extras_require={ "deps": [ "galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.2", - "galaxy-util <24.2", + "galaxy-util <24.3", ], }, python_requires=">=3.9, <3.14", diff --git a/test-requirements.txt b/test-requirements.txt index 8b0908f2e..e4aa6fd30 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,4 +10,4 @@ pytest-cov arcp>=0.2.0 -r requirements.txt galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.2 -galaxy-util<24.2 +galaxy-util<24.3 From 1b5633876aabd4cb57ef3f1fe91c853f3ee82e46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Feb 2025 08:06:12 +0000 Subject: [PATCH 56/78] Update galaxy-tool-util requirement Updates the requirements on [galaxy-tool-util](https://github.com/galaxyproject/galaxy) to permit the latest version. - [Release notes](https://github.com/galaxyproject/galaxy/releases) - [Commits](https://github.com/galaxyproject/galaxy/compare/galaxy-tool-util-22.1.3...v24.2.0) --- updated-dependencies: - dependency-name: galaxy-tool-util dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- mypy-requirements.txt | 2 +- setup.py | 2 +- test-requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 7ebcd84e5..65e517172 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -6,5 +6,5 @@ types-requests types-setuptools types-psutil types-mock -galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.2 +galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.3 galaxy-util<24.3 diff --git a/setup.py b/setup.py index bb207b4fe..4aed7320b 100644 --- a/setup.py +++ b/setup.py @@ -164,7 +164,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li ], extras_require={ "deps": [ - "galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.2", + "galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.3", "galaxy-util <24.3", ], }, diff --git a/test-requirements.txt b/test-requirements.txt index e4aa6fd30..c5ca6ef73 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -9,5 +9,5 @@ pytest-mock>=1.10.0 pytest-cov arcp>=0.2.0 -r requirements.txt -galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.2 +galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.3 galaxy-util<24.3 From d439cd342173b5919e07d298304d0185ade9382f Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 20 Feb 2025 15:10:36 +0100 Subject: [PATCH 57/78] tox mypyc: run on Python 3.13 --- tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 2a5a431b9..c5740a845 100644 --- a/tox.ini +++ b/tox.ini @@ -69,8 +69,7 @@ commands = py3{9,10,11,12,13}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs} py3{9,10,11,12,13}-bandit: bandit -r cwltool py3{9,10,11,12,13}-lint: make flake8 format-check codespell-check - py3{9,10,11,12,13}-mypy: make mypy PYTEST_EXTRA={posargs} - py3{9,10,11,12}-mypy: make mypyc PYTEST_EXTRA={posargs} + py3{9,10,11,12,13}-mypy: make mypy mypyc PYTEST_EXTRA={posargs} py312-shellcheck: make shellcheck py312-pydocstyle: make diff_pydocstyle_report py312-lintreadme: twine check {distdir}/* From 0f3c0407829c5d1afc336a3b21d0267151b89f39 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 20 Feb 2025 11:19:36 +0100 Subject: [PATCH 58/78] check the initial working directory before the cache attempt --- MANIFEST.in | 1 + cwltool/command_line_tool.py | 13 +- tests/test_caching.py | 193 + tests/test_examples.py | 135 - tests/wf/inp-filelist.txt | 9999 +++++++++++++++++++++++++ tests/wf/iwd-container-entryname1.cwl | 23 + tests/wf/iwd-container-entryname3.cwl | 24 + tests/wf/loadContents-input.yml | 3 + 8 files changed, 10255 insertions(+), 136 deletions(-) create mode 100644 tests/test_caching.py create mode 100644 tests/wf/inp-filelist.txt create mode 100644 tests/wf/iwd-container-entryname1.cwl create mode 100644 tests/wf/iwd-container-entryname3.cwl create mode 100644 tests/wf/loadContents-input.yml diff --git a/MANIFEST.in b/MANIFEST.in index 7ee34f35e..83b8c3fa5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -11,6 +11,7 @@ include tests/loop-ext/* include tests/tmp1/tmp2/tmp3/.gitkeep include tests/tmp4/alpha/* include tests/wf/* +include tests/wf/inp-filelist.txt include tests/wf/operation/* include tests/override/* include tests/reloc/*.cwl diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 2319e6211..d4bdbe3fb 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -492,7 +492,14 @@ def updatePathmap(self, outdir: str, pathmap: PathMapper, fn: CWLObjectType) -> for ls in cast(list[CWLObjectType], fn.get("listing", [])): self.updatePathmap(os.path.join(outdir, cast(str, fn["basename"])), pathmap, ls) - def _initialworkdir(self, j: JobBase, builder: Builder) -> None: + def _initialworkdir(self, j: Optional[JobBase], builder: Builder) -> None: + """ + Test and initialize the working directory. + + :param j: A :py:class:`~cwltool.job.CommandLineJob` or a + specialized container-based job. + If 'None', then only tests will be performed, no setup. + """ initialWorkdir, _ = self.get_requirement("InitialWorkDirRequirement") if initialWorkdir is None: return @@ -760,6 +767,9 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: "is in 'requirements'." ) + if j is None: + return # Only testing + with SourceLine(initialWorkdir, "listing", WorkflowException, debug): j.generatefiles["listing"] = ls for entry in ls: @@ -824,6 +834,7 @@ def job( _check_adjust, ) visit_class([cachebuilder.files, cachebuilder.bindings], ("File"), _checksum) + self._initialworkdir(None, cachebuilder) # test the initial working directory cmdline = flatten(list(map(cachebuilder.generate_arg, cachebuilder.bindings))) docker_req, _ = self.get_requirement("DockerRequirement") diff --git a/tests/test_caching.py b/tests/test_caching.py new file mode 100644 index 000000000..92691dbfd --- /dev/null +++ b/tests/test_caching.py @@ -0,0 +1,193 @@ +import re +from pathlib import Path + +import pytest + +from .util import get_data, get_main_output, needs_docker + +test_factors = [(""), ("--parallel"), ("--debug"), ("--parallel --debug")] + + +@needs_docker +@pytest.mark.parametrize("factor", test_factors) +def test_cid_file_non_existing_dir(tmp_path: Path, factor: str) -> None: + """Test that --cachedir with a bad path should produce a specific error.""" + test_file = "cache_test_workflow.cwl" + bad_cidfile_dir = tmp_path / "cidfile-dir-badpath" + commands = factor.split() + commands.extend( + [ + "--record-container-id", + "--cidfile-dir", + str(bad_cidfile_dir), + get_data("tests/wf/" + test_file), + ] + ) + error_code, _, stderr = get_main_output(commands) + stderr = re.sub(r"\s\s+", " ", stderr) + assert "directory doesn't exist, please create it first" in stderr, stderr + assert error_code == 2 or error_code == 1, stderr + + +@needs_docker +@pytest.mark.parametrize("factor", test_factors) +def test_wf_without_container(tmp_path: Path, factor: str) -> None: + """Confirm that we can run a workflow without a container.""" + test_file = "hello-workflow.cwl" + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend( + [ + "--cachedir", + cache_dir, + "--outdir", + str(tmp_path / "outdir"), + get_data("tests/wf/" + test_file), + "--usermessage", + "hello", + ] + ) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "completed success" in stderr + assert error_code == 0 + + +@needs_docker +@pytest.mark.parametrize("factor", test_factors) +def test_issue_740_fixed(tmp_path: Path, factor: str) -> None: + """Confirm that re-running a particular workflow with caching succeeds.""" + test_file = "cache_test_workflow.cwl" + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "completed success" in stderr + assert error_code == 0 + + commands = factor.split() + commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "Output of job will be cached in" not in stderr + assert error_code == 0, stderr + + +@needs_docker +@pytest.mark.parametrize("factor", test_factors) +def test_cache_relative_paths(tmp_path: Path, factor: str) -> None: + """Confirm that re-running a particular workflow with caching succeeds.""" + test_file = "secondary-files.cwl" + test_job_file = "secondary-files-job.yml" + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend( + [ + "--out", + str(tmp_path / "out"), + "--cachedir", + cache_dir, + get_data(f"tests/{test_file}"), + get_data(f"tests/{test_job_file}"), + ] + ) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "completed success" in stderr + assert error_code == 0 + + commands = factor.split() + commands.extend( + [ + "--out", + str(tmp_path / "out2"), + "--cachedir", + cache_dir, + get_data(f"tests/{test_file}"), + get_data(f"tests/{test_job_file}"), + ] + ) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "Output of job will be cached in" not in stderr + assert error_code == 0, stderr + + assert (tmp_path / "cwltool_cache" / "27903451fc1ee10c148a0bdeb845b2cf").exists() + + +@pytest.mark.parametrize("factor", test_factors) +def test_cache_default_literal_file(tmp_path: Path, factor: str) -> None: + """Confirm that running a CLT with a default literal file with caching succeeds.""" + test_file = "tests/wf/extract_region_specs.cwl" + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend( + [ + "--out", + str(tmp_path / "out"), + "--cachedir", + cache_dir, + get_data(test_file), + ] + ) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "completed success" in stderr + assert error_code == 0 + + +@needs_docker +@pytest.mark.parametrize("factor", test_factors) +def test_cache_dockerreq_hint_instead_of_req(tmp_path: Path, factor: str) -> None: + """The cache must not be checked when there is an invalid use of an absolute path in iwdr.listing.""" + cache_dir = str(tmp_path / "cwltool_cache") + test_job_file = "tests/wf/loadContents-input.yml" + # First, run the iwd-container-entryname1 conformance tests with caching turned on + test1_file = "tests/wf/iwd-container-entryname1.cwl" + commands1 = factor.split() + commands1.extend( + [ + "--out", + str(tmp_path / "out1"), + "--cachedir", + cache_dir, + get_data(test1_file), + get_data(test_job_file), + ] + ) + error_code1, _, stderr1 = get_main_output(commands1) + + stderr1 = re.sub(r"\s\s+", " ", stderr1) + assert "completed success" in stderr1 + assert error_code1 == 0 + # Second, run the iwd-container-entryname3 test, which should fail + # even though it would be a cache hit, except that its DockerRequirement is + # in `hints` instead of `requirements` and one of the initial working directory + # items has an absolute path starting with `/`. + test2_file = "tests/wf/iwd-container-entryname3.cwl" + commands2 = factor.split() + commands2.extend( + [ + "--out", + str(tmp_path / "out2"), + "--cachedir", + cache_dir, + get_data(test2_file), + get_data(test_job_file), + ] + ) + error_code2, _, stderr2 = get_main_output(commands2) + + stderr2 = re.sub(r"\s\s+", " ", stderr2) + assert ( + "at index 0 of listing is invalid, name can only start with '/' " + "when DockerRequirement is in 'requirements" in stderr2 + ) + assert error_code2 == 1 diff --git a/tests/test_examples.py b/tests/test_examples.py index b78965bb4..a6696d77b 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1115,27 +1115,6 @@ def test_cid_file_dir_arg_is_file_instead_of_dir(tmp_path: Path, factor: str) -> assert error_code == 2 or error_code == 1, stderr -@needs_docker -@pytest.mark.parametrize("factor", test_factors) -def test_cid_file_non_existing_dir(tmp_path: Path, factor: str) -> None: - """Test that --cachedir with a bad path should produce a specific error.""" - test_file = "cache_test_workflow.cwl" - bad_cidfile_dir = tmp_path / "cidfile-dir-badpath" - commands = factor.split() - commands.extend( - [ - "--record-container-id", - "--cidfile-dir", - str(bad_cidfile_dir), - get_data("tests/wf/" + test_file), - ] - ) - error_code, _, stderr = get_main_output(commands) - stderr = re.sub(r"\s\s+", " ", stderr) - assert "directory doesn't exist, please create it first" in stderr, stderr - assert error_code == 2 or error_code == 1, stderr - - @needs_docker @pytest.mark.parametrize("factor", test_factors) def test_cid_file_w_prefix(tmp_path: Path, factor: str) -> None: @@ -1233,120 +1212,6 @@ def test_secondary_files_v1_0(tmp_path: Path, factor: str) -> None: assert error_code == 0 -@needs_docker -@pytest.mark.parametrize("factor", test_factors) -def test_wf_without_container(tmp_path: Path, factor: str) -> None: - """Confirm that we can run a workflow without a container.""" - test_file = "hello-workflow.cwl" - cache_dir = str(tmp_path / "cwltool_cache") - commands = factor.split() - commands.extend( - [ - "--cachedir", - cache_dir, - "--outdir", - str(tmp_path / "outdir"), - get_data("tests/wf/" + test_file), - "--usermessage", - "hello", - ] - ) - error_code, _, stderr = get_main_output(commands) - - stderr = re.sub(r"\s\s+", " ", stderr) - assert "completed success" in stderr - assert error_code == 0 - - -@needs_docker -@pytest.mark.parametrize("factor", test_factors) -def test_issue_740_fixed(tmp_path: Path, factor: str) -> None: - """Confirm that re-running a particular workflow with caching succeeds.""" - test_file = "cache_test_workflow.cwl" - cache_dir = str(tmp_path / "cwltool_cache") - commands = factor.split() - commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) - error_code, _, stderr = get_main_output(commands) - - stderr = re.sub(r"\s\s+", " ", stderr) - assert "completed success" in stderr - assert error_code == 0 - - commands = factor.split() - commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) - error_code, _, stderr = get_main_output(commands) - - stderr = re.sub(r"\s\s+", " ", stderr) - assert "Output of job will be cached in" not in stderr - assert error_code == 0, stderr - - -@needs_docker -@pytest.mark.parametrize("factor", test_factors) -def test_cache_relative_paths(tmp_path: Path, factor: str) -> None: - """Confirm that re-running a particular workflow with caching succeeds.""" - test_file = "secondary-files.cwl" - test_job_file = "secondary-files-job.yml" - cache_dir = str(tmp_path / "cwltool_cache") - commands = factor.split() - commands.extend( - [ - "--out", - str(tmp_path / "out"), - "--cachedir", - cache_dir, - get_data(f"tests/{test_file}"), - get_data(f"tests/{test_job_file}"), - ] - ) - error_code, _, stderr = get_main_output(commands) - - stderr = re.sub(r"\s\s+", " ", stderr) - assert "completed success" in stderr - assert error_code == 0 - - commands = factor.split() - commands.extend( - [ - "--out", - str(tmp_path / "out2"), - "--cachedir", - cache_dir, - get_data(f"tests/{test_file}"), - get_data(f"tests/{test_job_file}"), - ] - ) - error_code, _, stderr = get_main_output(commands) - - stderr = re.sub(r"\s\s+", " ", stderr) - assert "Output of job will be cached in" not in stderr - assert error_code == 0, stderr - - assert (tmp_path / "cwltool_cache" / "27903451fc1ee10c148a0bdeb845b2cf").exists() - - -@pytest.mark.parametrize("factor", test_factors) -def test_cache_default_literal_file(tmp_path: Path, factor: str) -> None: - """Confirm that running a CLT with a default literal file with caching succeeds.""" - test_file = "tests/wf/extract_region_specs.cwl" - cache_dir = str(tmp_path / "cwltool_cache") - commands = factor.split() - commands.extend( - [ - "--out", - str(tmp_path / "out"), - "--cachedir", - cache_dir, - get_data(test_file), - ] - ) - error_code, _, stderr = get_main_output(commands) - - stderr = re.sub(r"\s\s+", " ", stderr) - assert "completed success" in stderr - assert error_code == 0 - - def test_write_summary(tmp_path: Path) -> None: """Test --write-summary.""" commands = [ diff --git a/tests/wf/inp-filelist.txt b/tests/wf/inp-filelist.txt new file mode 100644 index 000000000..232ddf670 --- /dev/null +++ b/tests/wf/inp-filelist.txt @@ -0,0 +1,9999 @@ +example_input_file1.txt +example_input_file2.txt +example_input_file3.txt +example_input_file4.txt +example_input_file5.txt +example_input_file6.txt +example_input_file7.txt +example_input_file8.txt +example_input_file9.txt +example_input_file10.txt +example_input_file11.txt +example_input_file12.txt +example_input_file13.txt +example_input_file14.txt +example_input_file15.txt +example_input_file16.txt +example_input_file17.txt +example_input_file18.txt +example_input_file19.txt +example_input_file20.txt +example_input_file21.txt +example_input_file22.txt +example_input_file23.txt +example_input_file24.txt +example_input_file25.txt +example_input_file26.txt +example_input_file27.txt +example_input_file28.txt +example_input_file29.txt +example_input_file30.txt +example_input_file31.txt +example_input_file32.txt +example_input_file33.txt +example_input_file34.txt +example_input_file35.txt +example_input_file36.txt +example_input_file37.txt +example_input_file38.txt +example_input_file39.txt +example_input_file40.txt +example_input_file41.txt +example_input_file42.txt +example_input_file43.txt +example_input_file44.txt +example_input_file45.txt +example_input_file46.txt +example_input_file47.txt +example_input_file48.txt +example_input_file49.txt +example_input_file50.txt +example_input_file51.txt +example_input_file52.txt +example_input_file53.txt +example_input_file54.txt +example_input_file55.txt +example_input_file56.txt +example_input_file57.txt +example_input_file58.txt +example_input_file59.txt +example_input_file60.txt +example_input_file61.txt +example_input_file62.txt +example_input_file63.txt +example_input_file64.txt +example_input_file65.txt +example_input_file66.txt +example_input_file67.txt +example_input_file68.txt +example_input_file69.txt +example_input_file70.txt +example_input_file71.txt +example_input_file72.txt +example_input_file73.txt +example_input_file74.txt +example_input_file75.txt +example_input_file76.txt +example_input_file77.txt +example_input_file78.txt +example_input_file79.txt +example_input_file80.txt +example_input_file81.txt +example_input_file82.txt +example_input_file83.txt +example_input_file84.txt +example_input_file85.txt +example_input_file86.txt +example_input_file87.txt +example_input_file88.txt +example_input_file89.txt +example_input_file90.txt +example_input_file91.txt +example_input_file92.txt +example_input_file93.txt +example_input_file94.txt +example_input_file95.txt +example_input_file96.txt +example_input_file97.txt +example_input_file98.txt +example_input_file99.txt +example_input_file100.txt +example_input_file101.txt +example_input_file102.txt +example_input_file103.txt +example_input_file104.txt +example_input_file105.txt +example_input_file106.txt +example_input_file107.txt +example_input_file108.txt +example_input_file109.txt +example_input_file110.txt +example_input_file111.txt +example_input_file112.txt +example_input_file113.txt +example_input_file114.txt +example_input_file115.txt +example_input_file116.txt +example_input_file117.txt +example_input_file118.txt +example_input_file119.txt +example_input_file120.txt +example_input_file121.txt +example_input_file122.txt +example_input_file123.txt +example_input_file124.txt +example_input_file125.txt +example_input_file126.txt +example_input_file127.txt +example_input_file128.txt +example_input_file129.txt +example_input_file130.txt +example_input_file131.txt +example_input_file132.txt +example_input_file133.txt +example_input_file134.txt +example_input_file135.txt +example_input_file136.txt +example_input_file137.txt +example_input_file138.txt +example_input_file139.txt +example_input_file140.txt +example_input_file141.txt +example_input_file142.txt +example_input_file143.txt +example_input_file144.txt +example_input_file145.txt +example_input_file146.txt +example_input_file147.txt +example_input_file148.txt +example_input_file149.txt +example_input_file150.txt +example_input_file151.txt +example_input_file152.txt +example_input_file153.txt +example_input_file154.txt +example_input_file155.txt +example_input_file156.txt +example_input_file157.txt +example_input_file158.txt +example_input_file159.txt +example_input_file160.txt +example_input_file161.txt +example_input_file162.txt +example_input_file163.txt +example_input_file164.txt +example_input_file165.txt +example_input_file166.txt +example_input_file167.txt +example_input_file168.txt +example_input_file169.txt +example_input_file170.txt +example_input_file171.txt +example_input_file172.txt +example_input_file173.txt +example_input_file174.txt +example_input_file175.txt +example_input_file176.txt +example_input_file177.txt +example_input_file178.txt +example_input_file179.txt +example_input_file180.txt +example_input_file181.txt +example_input_file182.txt +example_input_file183.txt +example_input_file184.txt +example_input_file185.txt +example_input_file186.txt +example_input_file187.txt +example_input_file188.txt +example_input_file189.txt +example_input_file190.txt +example_input_file191.txt +example_input_file192.txt +example_input_file193.txt +example_input_file194.txt +example_input_file195.txt +example_input_file196.txt +example_input_file197.txt +example_input_file198.txt +example_input_file199.txt +example_input_file200.txt +example_input_file201.txt +example_input_file202.txt +example_input_file203.txt +example_input_file204.txt +example_input_file205.txt +example_input_file206.txt +example_input_file207.txt +example_input_file208.txt +example_input_file209.txt +example_input_file210.txt +example_input_file211.txt +example_input_file212.txt +example_input_file213.txt +example_input_file214.txt +example_input_file215.txt +example_input_file216.txt +example_input_file217.txt +example_input_file218.txt +example_input_file219.txt +example_input_file220.txt +example_input_file221.txt +example_input_file222.txt +example_input_file223.txt +example_input_file224.txt +example_input_file225.txt +example_input_file226.txt +example_input_file227.txt +example_input_file228.txt +example_input_file229.txt +example_input_file230.txt +example_input_file231.txt +example_input_file232.txt +example_input_file233.txt +example_input_file234.txt +example_input_file235.txt +example_input_file236.txt +example_input_file237.txt +example_input_file238.txt +example_input_file239.txt +example_input_file240.txt +example_input_file241.txt +example_input_file242.txt +example_input_file243.txt +example_input_file244.txt +example_input_file245.txt +example_input_file246.txt +example_input_file247.txt +example_input_file248.txt +example_input_file249.txt +example_input_file250.txt +example_input_file251.txt +example_input_file252.txt +example_input_file253.txt +example_input_file254.txt +example_input_file255.txt +example_input_file256.txt +example_input_file257.txt +example_input_file258.txt +example_input_file259.txt +example_input_file260.txt +example_input_file261.txt +example_input_file262.txt +example_input_file263.txt +example_input_file264.txt +example_input_file265.txt +example_input_file266.txt +example_input_file267.txt +example_input_file268.txt +example_input_file269.txt +example_input_file270.txt +example_input_file271.txt +example_input_file272.txt +example_input_file273.txt +example_input_file274.txt +example_input_file275.txt +example_input_file276.txt +example_input_file277.txt +example_input_file278.txt +example_input_file279.txt +example_input_file280.txt +example_input_file281.txt +example_input_file282.txt +example_input_file283.txt +example_input_file284.txt +example_input_file285.txt +example_input_file286.txt +example_input_file287.txt +example_input_file288.txt +example_input_file289.txt +example_input_file290.txt +example_input_file291.txt +example_input_file292.txt +example_input_file293.txt +example_input_file294.txt +example_input_file295.txt +example_input_file296.txt +example_input_file297.txt +example_input_file298.txt +example_input_file299.txt +example_input_file300.txt +example_input_file301.txt +example_input_file302.txt +example_input_file303.txt +example_input_file304.txt +example_input_file305.txt +example_input_file306.txt +example_input_file307.txt +example_input_file308.txt +example_input_file309.txt +example_input_file310.txt +example_input_file311.txt +example_input_file312.txt +example_input_file313.txt +example_input_file314.txt +example_input_file315.txt +example_input_file316.txt +example_input_file317.txt +example_input_file318.txt +example_input_file319.txt +example_input_file320.txt +example_input_file321.txt +example_input_file322.txt +example_input_file323.txt +example_input_file324.txt +example_input_file325.txt +example_input_file326.txt +example_input_file327.txt +example_input_file328.txt +example_input_file329.txt +example_input_file330.txt +example_input_file331.txt +example_input_file332.txt +example_input_file333.txt +example_input_file334.txt +example_input_file335.txt +example_input_file336.txt +example_input_file337.txt +example_input_file338.txt +example_input_file339.txt +example_input_file340.txt +example_input_file341.txt +example_input_file342.txt +example_input_file343.txt +example_input_file344.txt +example_input_file345.txt +example_input_file346.txt +example_input_file347.txt +example_input_file348.txt +example_input_file349.txt +example_input_file350.txt +example_input_file351.txt +example_input_file352.txt +example_input_file353.txt +example_input_file354.txt +example_input_file355.txt +example_input_file356.txt +example_input_file357.txt +example_input_file358.txt +example_input_file359.txt +example_input_file360.txt +example_input_file361.txt +example_input_file362.txt +example_input_file363.txt +example_input_file364.txt +example_input_file365.txt +example_input_file366.txt +example_input_file367.txt +example_input_file368.txt +example_input_file369.txt +example_input_file370.txt +example_input_file371.txt +example_input_file372.txt +example_input_file373.txt +example_input_file374.txt +example_input_file375.txt +example_input_file376.txt +example_input_file377.txt +example_input_file378.txt +example_input_file379.txt +example_input_file380.txt +example_input_file381.txt +example_input_file382.txt +example_input_file383.txt +example_input_file384.txt +example_input_file385.txt +example_input_file386.txt +example_input_file387.txt +example_input_file388.txt +example_input_file389.txt +example_input_file390.txt +example_input_file391.txt +example_input_file392.txt +example_input_file393.txt +example_input_file394.txt +example_input_file395.txt +example_input_file396.txt +example_input_file397.txt +example_input_file398.txt +example_input_file399.txt +example_input_file400.txt +example_input_file401.txt +example_input_file402.txt +example_input_file403.txt +example_input_file404.txt +example_input_file405.txt +example_input_file406.txt +example_input_file407.txt +example_input_file408.txt +example_input_file409.txt +example_input_file410.txt +example_input_file411.txt +example_input_file412.txt +example_input_file413.txt +example_input_file414.txt +example_input_file415.txt +example_input_file416.txt +example_input_file417.txt +example_input_file418.txt +example_input_file419.txt +example_input_file420.txt +example_input_file421.txt +example_input_file422.txt +example_input_file423.txt +example_input_file424.txt +example_input_file425.txt +example_input_file426.txt +example_input_file427.txt +example_input_file428.txt +example_input_file429.txt +example_input_file430.txt +example_input_file431.txt +example_input_file432.txt +example_input_file433.txt +example_input_file434.txt +example_input_file435.txt +example_input_file436.txt +example_input_file437.txt +example_input_file438.txt +example_input_file439.txt +example_input_file440.txt +example_input_file441.txt +example_input_file442.txt +example_input_file443.txt +example_input_file444.txt +example_input_file445.txt +example_input_file446.txt +example_input_file447.txt +example_input_file448.txt +example_input_file449.txt +example_input_file450.txt +example_input_file451.txt +example_input_file452.txt +example_input_file453.txt +example_input_file454.txt +example_input_file455.txt +example_input_file456.txt +example_input_file457.txt +example_input_file458.txt +example_input_file459.txt +example_input_file460.txt +example_input_file461.txt +example_input_file462.txt +example_input_file463.txt +example_input_file464.txt +example_input_file465.txt +example_input_file466.txt +example_input_file467.txt +example_input_file468.txt +example_input_file469.txt +example_input_file470.txt +example_input_file471.txt +example_input_file472.txt +example_input_file473.txt +example_input_file474.txt +example_input_file475.txt +example_input_file476.txt +example_input_file477.txt +example_input_file478.txt +example_input_file479.txt +example_input_file480.txt +example_input_file481.txt +example_input_file482.txt +example_input_file483.txt +example_input_file484.txt +example_input_file485.txt +example_input_file486.txt +example_input_file487.txt +example_input_file488.txt +example_input_file489.txt +example_input_file490.txt +example_input_file491.txt +example_input_file492.txt +example_input_file493.txt +example_input_file494.txt +example_input_file495.txt +example_input_file496.txt +example_input_file497.txt +example_input_file498.txt +example_input_file499.txt +example_input_file500.txt +example_input_file501.txt +example_input_file502.txt +example_input_file503.txt +example_input_file504.txt +example_input_file505.txt +example_input_file506.txt +example_input_file507.txt +example_input_file508.txt +example_input_file509.txt +example_input_file510.txt +example_input_file511.txt +example_input_file512.txt +example_input_file513.txt +example_input_file514.txt +example_input_file515.txt +example_input_file516.txt +example_input_file517.txt +example_input_file518.txt +example_input_file519.txt +example_input_file520.txt +example_input_file521.txt +example_input_file522.txt +example_input_file523.txt +example_input_file524.txt +example_input_file525.txt +example_input_file526.txt +example_input_file527.txt +example_input_file528.txt +example_input_file529.txt +example_input_file530.txt +example_input_file531.txt +example_input_file532.txt +example_input_file533.txt +example_input_file534.txt +example_input_file535.txt +example_input_file536.txt +example_input_file537.txt +example_input_file538.txt +example_input_file539.txt +example_input_file540.txt +example_input_file541.txt +example_input_file542.txt +example_input_file543.txt +example_input_file544.txt +example_input_file545.txt +example_input_file546.txt +example_input_file547.txt +example_input_file548.txt +example_input_file549.txt +example_input_file550.txt +example_input_file551.txt +example_input_file552.txt +example_input_file553.txt +example_input_file554.txt +example_input_file555.txt +example_input_file556.txt +example_input_file557.txt +example_input_file558.txt +example_input_file559.txt +example_input_file560.txt +example_input_file561.txt +example_input_file562.txt +example_input_file563.txt +example_input_file564.txt +example_input_file565.txt +example_input_file566.txt +example_input_file567.txt +example_input_file568.txt +example_input_file569.txt +example_input_file570.txt +example_input_file571.txt +example_input_file572.txt +example_input_file573.txt +example_input_file574.txt +example_input_file575.txt +example_input_file576.txt +example_input_file577.txt +example_input_file578.txt +example_input_file579.txt +example_input_file580.txt +example_input_file581.txt +example_input_file582.txt +example_input_file583.txt +example_input_file584.txt +example_input_file585.txt +example_input_file586.txt +example_input_file587.txt +example_input_file588.txt +example_input_file589.txt +example_input_file590.txt +example_input_file591.txt +example_input_file592.txt +example_input_file593.txt +example_input_file594.txt +example_input_file595.txt +example_input_file596.txt +example_input_file597.txt +example_input_file598.txt +example_input_file599.txt +example_input_file600.txt +example_input_file601.txt +example_input_file602.txt +example_input_file603.txt +example_input_file604.txt +example_input_file605.txt +example_input_file606.txt +example_input_file607.txt +example_input_file608.txt +example_input_file609.txt +example_input_file610.txt +example_input_file611.txt +example_input_file612.txt +example_input_file613.txt +example_input_file614.txt +example_input_file615.txt +example_input_file616.txt +example_input_file617.txt +example_input_file618.txt +example_input_file619.txt +example_input_file620.txt +example_input_file621.txt +example_input_file622.txt +example_input_file623.txt +example_input_file624.txt +example_input_file625.txt +example_input_file626.txt +example_input_file627.txt +example_input_file628.txt +example_input_file629.txt +example_input_file630.txt +example_input_file631.txt +example_input_file632.txt +example_input_file633.txt +example_input_file634.txt +example_input_file635.txt +example_input_file636.txt +example_input_file637.txt +example_input_file638.txt +example_input_file639.txt +example_input_file640.txt +example_input_file641.txt +example_input_file642.txt +example_input_file643.txt +example_input_file644.txt +example_input_file645.txt +example_input_file646.txt +example_input_file647.txt +example_input_file648.txt +example_input_file649.txt +example_input_file650.txt +example_input_file651.txt +example_input_file652.txt +example_input_file653.txt +example_input_file654.txt +example_input_file655.txt +example_input_file656.txt +example_input_file657.txt +example_input_file658.txt +example_input_file659.txt +example_input_file660.txt +example_input_file661.txt +example_input_file662.txt +example_input_file663.txt +example_input_file664.txt +example_input_file665.txt +example_input_file666.txt +example_input_file667.txt +example_input_file668.txt +example_input_file669.txt +example_input_file670.txt +example_input_file671.txt +example_input_file672.txt +example_input_file673.txt +example_input_file674.txt +example_input_file675.txt +example_input_file676.txt +example_input_file677.txt +example_input_file678.txt +example_input_file679.txt +example_input_file680.txt +example_input_file681.txt +example_input_file682.txt +example_input_file683.txt +example_input_file684.txt +example_input_file685.txt +example_input_file686.txt +example_input_file687.txt +example_input_file688.txt +example_input_file689.txt +example_input_file690.txt +example_input_file691.txt +example_input_file692.txt +example_input_file693.txt +example_input_file694.txt +example_input_file695.txt +example_input_file696.txt +example_input_file697.txt +example_input_file698.txt +example_input_file699.txt +example_input_file700.txt +example_input_file701.txt +example_input_file702.txt +example_input_file703.txt +example_input_file704.txt +example_input_file705.txt +example_input_file706.txt +example_input_file707.txt +example_input_file708.txt +example_input_file709.txt +example_input_file710.txt +example_input_file711.txt +example_input_file712.txt +example_input_file713.txt +example_input_file714.txt +example_input_file715.txt +example_input_file716.txt +example_input_file717.txt +example_input_file718.txt +example_input_file719.txt +example_input_file720.txt +example_input_file721.txt +example_input_file722.txt +example_input_file723.txt +example_input_file724.txt +example_input_file725.txt +example_input_file726.txt +example_input_file727.txt +example_input_file728.txt +example_input_file729.txt +example_input_file730.txt +example_input_file731.txt +example_input_file732.txt +example_input_file733.txt +example_input_file734.txt +example_input_file735.txt +example_input_file736.txt +example_input_file737.txt +example_input_file738.txt +example_input_file739.txt +example_input_file740.txt +example_input_file741.txt +example_input_file742.txt +example_input_file743.txt +example_input_file744.txt +example_input_file745.txt +example_input_file746.txt +example_input_file747.txt +example_input_file748.txt +example_input_file749.txt +example_input_file750.txt +example_input_file751.txt +example_input_file752.txt +example_input_file753.txt +example_input_file754.txt +example_input_file755.txt +example_input_file756.txt +example_input_file757.txt +example_input_file758.txt +example_input_file759.txt +example_input_file760.txt +example_input_file761.txt +example_input_file762.txt +example_input_file763.txt +example_input_file764.txt +example_input_file765.txt +example_input_file766.txt +example_input_file767.txt +example_input_file768.txt +example_input_file769.txt +example_input_file770.txt +example_input_file771.txt +example_input_file772.txt +example_input_file773.txt +example_input_file774.txt +example_input_file775.txt +example_input_file776.txt +example_input_file777.txt +example_input_file778.txt +example_input_file779.txt +example_input_file780.txt +example_input_file781.txt +example_input_file782.txt +example_input_file783.txt +example_input_file784.txt +example_input_file785.txt +example_input_file786.txt +example_input_file787.txt +example_input_file788.txt +example_input_file789.txt +example_input_file790.txt +example_input_file791.txt +example_input_file792.txt +example_input_file793.txt +example_input_file794.txt +example_input_file795.txt +example_input_file796.txt +example_input_file797.txt +example_input_file798.txt +example_input_file799.txt +example_input_file800.txt +example_input_file801.txt +example_input_file802.txt +example_input_file803.txt +example_input_file804.txt +example_input_file805.txt +example_input_file806.txt +example_input_file807.txt +example_input_file808.txt +example_input_file809.txt +example_input_file810.txt +example_input_file811.txt +example_input_file812.txt +example_input_file813.txt +example_input_file814.txt +example_input_file815.txt +example_input_file816.txt +example_input_file817.txt +example_input_file818.txt +example_input_file819.txt +example_input_file820.txt +example_input_file821.txt +example_input_file822.txt +example_input_file823.txt +example_input_file824.txt +example_input_file825.txt +example_input_file826.txt +example_input_file827.txt +example_input_file828.txt +example_input_file829.txt +example_input_file830.txt +example_input_file831.txt +example_input_file832.txt +example_input_file833.txt +example_input_file834.txt +example_input_file835.txt +example_input_file836.txt +example_input_file837.txt +example_input_file838.txt +example_input_file839.txt +example_input_file840.txt +example_input_file841.txt +example_input_file842.txt +example_input_file843.txt +example_input_file844.txt +example_input_file845.txt +example_input_file846.txt +example_input_file847.txt +example_input_file848.txt +example_input_file849.txt +example_input_file850.txt +example_input_file851.txt +example_input_file852.txt +example_input_file853.txt +example_input_file854.txt +example_input_file855.txt +example_input_file856.txt +example_input_file857.txt +example_input_file858.txt +example_input_file859.txt +example_input_file860.txt +example_input_file861.txt +example_input_file862.txt +example_input_file863.txt +example_input_file864.txt +example_input_file865.txt +example_input_file866.txt +example_input_file867.txt +example_input_file868.txt +example_input_file869.txt +example_input_file870.txt +example_input_file871.txt +example_input_file872.txt +example_input_file873.txt +example_input_file874.txt +example_input_file875.txt +example_input_file876.txt +example_input_file877.txt +example_input_file878.txt +example_input_file879.txt +example_input_file880.txt +example_input_file881.txt +example_input_file882.txt +example_input_file883.txt +example_input_file884.txt +example_input_file885.txt +example_input_file886.txt +example_input_file887.txt +example_input_file888.txt +example_input_file889.txt +example_input_file890.txt +example_input_file891.txt +example_input_file892.txt +example_input_file893.txt +example_input_file894.txt +example_input_file895.txt +example_input_file896.txt +example_input_file897.txt +example_input_file898.txt +example_input_file899.txt +example_input_file900.txt +example_input_file901.txt +example_input_file902.txt +example_input_file903.txt +example_input_file904.txt +example_input_file905.txt +example_input_file906.txt +example_input_file907.txt +example_input_file908.txt +example_input_file909.txt +example_input_file910.txt +example_input_file911.txt +example_input_file912.txt +example_input_file913.txt +example_input_file914.txt +example_input_file915.txt +example_input_file916.txt +example_input_file917.txt +example_input_file918.txt +example_input_file919.txt +example_input_file920.txt +example_input_file921.txt +example_input_file922.txt +example_input_file923.txt +example_input_file924.txt +example_input_file925.txt +example_input_file926.txt +example_input_file927.txt +example_input_file928.txt +example_input_file929.txt +example_input_file930.txt +example_input_file931.txt +example_input_file932.txt +example_input_file933.txt +example_input_file934.txt +example_input_file935.txt +example_input_file936.txt +example_input_file937.txt +example_input_file938.txt +example_input_file939.txt +example_input_file940.txt +example_input_file941.txt +example_input_file942.txt +example_input_file943.txt +example_input_file944.txt +example_input_file945.txt +example_input_file946.txt +example_input_file947.txt +example_input_file948.txt +example_input_file949.txt +example_input_file950.txt +example_input_file951.txt +example_input_file952.txt +example_input_file953.txt +example_input_file954.txt +example_input_file955.txt +example_input_file956.txt +example_input_file957.txt +example_input_file958.txt +example_input_file959.txt +example_input_file960.txt +example_input_file961.txt +example_input_file962.txt +example_input_file963.txt +example_input_file964.txt +example_input_file965.txt +example_input_file966.txt +example_input_file967.txt +example_input_file968.txt +example_input_file969.txt +example_input_file970.txt +example_input_file971.txt +example_input_file972.txt +example_input_file973.txt +example_input_file974.txt +example_input_file975.txt +example_input_file976.txt +example_input_file977.txt +example_input_file978.txt +example_input_file979.txt +example_input_file980.txt +example_input_file981.txt +example_input_file982.txt +example_input_file983.txt +example_input_file984.txt +example_input_file985.txt +example_input_file986.txt +example_input_file987.txt +example_input_file988.txt +example_input_file989.txt +example_input_file990.txt +example_input_file991.txt +example_input_file992.txt +example_input_file993.txt +example_input_file994.txt +example_input_file995.txt +example_input_file996.txt +example_input_file997.txt +example_input_file998.txt +example_input_file999.txt +example_input_file1000.txt +example_input_file1001.txt +example_input_file1002.txt +example_input_file1003.txt +example_input_file1004.txt +example_input_file1005.txt +example_input_file1006.txt +example_input_file1007.txt +example_input_file1008.txt +example_input_file1009.txt +example_input_file1010.txt +example_input_file1011.txt +example_input_file1012.txt +example_input_file1013.txt +example_input_file1014.txt +example_input_file1015.txt +example_input_file1016.txt +example_input_file1017.txt +example_input_file1018.txt +example_input_file1019.txt +example_input_file1020.txt +example_input_file1021.txt +example_input_file1022.txt +example_input_file1023.txt +example_input_file1024.txt +example_input_file1025.txt +example_input_file1026.txt +example_input_file1027.txt +example_input_file1028.txt +example_input_file1029.txt +example_input_file1030.txt +example_input_file1031.txt +example_input_file1032.txt +example_input_file1033.txt +example_input_file1034.txt +example_input_file1035.txt +example_input_file1036.txt +example_input_file1037.txt +example_input_file1038.txt +example_input_file1039.txt +example_input_file1040.txt +example_input_file1041.txt +example_input_file1042.txt +example_input_file1043.txt +example_input_file1044.txt +example_input_file1045.txt +example_input_file1046.txt +example_input_file1047.txt +example_input_file1048.txt +example_input_file1049.txt +example_input_file1050.txt +example_input_file1051.txt +example_input_file1052.txt +example_input_file1053.txt +example_input_file1054.txt +example_input_file1055.txt +example_input_file1056.txt +example_input_file1057.txt +example_input_file1058.txt +example_input_file1059.txt +example_input_file1060.txt +example_input_file1061.txt +example_input_file1062.txt +example_input_file1063.txt +example_input_file1064.txt +example_input_file1065.txt +example_input_file1066.txt +example_input_file1067.txt +example_input_file1068.txt +example_input_file1069.txt +example_input_file1070.txt +example_input_file1071.txt +example_input_file1072.txt +example_input_file1073.txt +example_input_file1074.txt +example_input_file1075.txt +example_input_file1076.txt +example_input_file1077.txt +example_input_file1078.txt +example_input_file1079.txt +example_input_file1080.txt +example_input_file1081.txt +example_input_file1082.txt +example_input_file1083.txt +example_input_file1084.txt +example_input_file1085.txt +example_input_file1086.txt +example_input_file1087.txt +example_input_file1088.txt +example_input_file1089.txt +example_input_file1090.txt +example_input_file1091.txt +example_input_file1092.txt +example_input_file1093.txt +example_input_file1094.txt +example_input_file1095.txt +example_input_file1096.txt +example_input_file1097.txt +example_input_file1098.txt +example_input_file1099.txt +example_input_file1100.txt +example_input_file1101.txt +example_input_file1102.txt +example_input_file1103.txt +example_input_file1104.txt +example_input_file1105.txt +example_input_file1106.txt +example_input_file1107.txt +example_input_file1108.txt +example_input_file1109.txt +example_input_file1110.txt +example_input_file1111.txt +example_input_file1112.txt +example_input_file1113.txt +example_input_file1114.txt +example_input_file1115.txt +example_input_file1116.txt +example_input_file1117.txt +example_input_file1118.txt +example_input_file1119.txt +example_input_file1120.txt +example_input_file1121.txt +example_input_file1122.txt +example_input_file1123.txt +example_input_file1124.txt +example_input_file1125.txt +example_input_file1126.txt +example_input_file1127.txt +example_input_file1128.txt +example_input_file1129.txt +example_input_file1130.txt +example_input_file1131.txt +example_input_file1132.txt +example_input_file1133.txt +example_input_file1134.txt +example_input_file1135.txt +example_input_file1136.txt +example_input_file1137.txt +example_input_file1138.txt +example_input_file1139.txt +example_input_file1140.txt +example_input_file1141.txt +example_input_file1142.txt +example_input_file1143.txt +example_input_file1144.txt +example_input_file1145.txt +example_input_file1146.txt +example_input_file1147.txt +example_input_file1148.txt +example_input_file1149.txt +example_input_file1150.txt +example_input_file1151.txt +example_input_file1152.txt +example_input_file1153.txt +example_input_file1154.txt +example_input_file1155.txt +example_input_file1156.txt +example_input_file1157.txt +example_input_file1158.txt +example_input_file1159.txt +example_input_file1160.txt +example_input_file1161.txt +example_input_file1162.txt +example_input_file1163.txt +example_input_file1164.txt +example_input_file1165.txt +example_input_file1166.txt +example_input_file1167.txt +example_input_file1168.txt +example_input_file1169.txt +example_input_file1170.txt +example_input_file1171.txt +example_input_file1172.txt +example_input_file1173.txt +example_input_file1174.txt +example_input_file1175.txt +example_input_file1176.txt +example_input_file1177.txt +example_input_file1178.txt +example_input_file1179.txt +example_input_file1180.txt +example_input_file1181.txt +example_input_file1182.txt +example_input_file1183.txt +example_input_file1184.txt +example_input_file1185.txt +example_input_file1186.txt +example_input_file1187.txt +example_input_file1188.txt +example_input_file1189.txt +example_input_file1190.txt +example_input_file1191.txt +example_input_file1192.txt +example_input_file1193.txt +example_input_file1194.txt +example_input_file1195.txt +example_input_file1196.txt +example_input_file1197.txt +example_input_file1198.txt +example_input_file1199.txt +example_input_file1200.txt +example_input_file1201.txt +example_input_file1202.txt +example_input_file1203.txt +example_input_file1204.txt +example_input_file1205.txt +example_input_file1206.txt +example_input_file1207.txt +example_input_file1208.txt +example_input_file1209.txt +example_input_file1210.txt +example_input_file1211.txt +example_input_file1212.txt +example_input_file1213.txt +example_input_file1214.txt +example_input_file1215.txt +example_input_file1216.txt +example_input_file1217.txt +example_input_file1218.txt +example_input_file1219.txt +example_input_file1220.txt +example_input_file1221.txt +example_input_file1222.txt +example_input_file1223.txt +example_input_file1224.txt +example_input_file1225.txt +example_input_file1226.txt +example_input_file1227.txt +example_input_file1228.txt +example_input_file1229.txt +example_input_file1230.txt +example_input_file1231.txt +example_input_file1232.txt +example_input_file1233.txt +example_input_file1234.txt +example_input_file1235.txt +example_input_file1236.txt +example_input_file1237.txt +example_input_file1238.txt +example_input_file1239.txt +example_input_file1240.txt +example_input_file1241.txt +example_input_file1242.txt +example_input_file1243.txt +example_input_file1244.txt +example_input_file1245.txt +example_input_file1246.txt +example_input_file1247.txt +example_input_file1248.txt +example_input_file1249.txt +example_input_file1250.txt +example_input_file1251.txt +example_input_file1252.txt +example_input_file1253.txt +example_input_file1254.txt +example_input_file1255.txt +example_input_file1256.txt +example_input_file1257.txt +example_input_file1258.txt +example_input_file1259.txt +example_input_file1260.txt +example_input_file1261.txt +example_input_file1262.txt +example_input_file1263.txt +example_input_file1264.txt +example_input_file1265.txt +example_input_file1266.txt +example_input_file1267.txt +example_input_file1268.txt +example_input_file1269.txt +example_input_file1270.txt +example_input_file1271.txt +example_input_file1272.txt +example_input_file1273.txt +example_input_file1274.txt +example_input_file1275.txt +example_input_file1276.txt +example_input_file1277.txt +example_input_file1278.txt +example_input_file1279.txt +example_input_file1280.txt +example_input_file1281.txt +example_input_file1282.txt +example_input_file1283.txt +example_input_file1284.txt +example_input_file1285.txt +example_input_file1286.txt +example_input_file1287.txt +example_input_file1288.txt +example_input_file1289.txt +example_input_file1290.txt +example_input_file1291.txt +example_input_file1292.txt +example_input_file1293.txt +example_input_file1294.txt +example_input_file1295.txt +example_input_file1296.txt +example_input_file1297.txt +example_input_file1298.txt +example_input_file1299.txt +example_input_file1300.txt +example_input_file1301.txt +example_input_file1302.txt +example_input_file1303.txt +example_input_file1304.txt +example_input_file1305.txt +example_input_file1306.txt +example_input_file1307.txt +example_input_file1308.txt +example_input_file1309.txt +example_input_file1310.txt +example_input_file1311.txt +example_input_file1312.txt +example_input_file1313.txt +example_input_file1314.txt +example_input_file1315.txt +example_input_file1316.txt +example_input_file1317.txt +example_input_file1318.txt +example_input_file1319.txt +example_input_file1320.txt +example_input_file1321.txt +example_input_file1322.txt +example_input_file1323.txt +example_input_file1324.txt +example_input_file1325.txt +example_input_file1326.txt +example_input_file1327.txt +example_input_file1328.txt +example_input_file1329.txt +example_input_file1330.txt +example_input_file1331.txt +example_input_file1332.txt +example_input_file1333.txt +example_input_file1334.txt +example_input_file1335.txt +example_input_file1336.txt +example_input_file1337.txt +example_input_file1338.txt +example_input_file1339.txt +example_input_file1340.txt +example_input_file1341.txt +example_input_file1342.txt +example_input_file1343.txt +example_input_file1344.txt +example_input_file1345.txt +example_input_file1346.txt +example_input_file1347.txt +example_input_file1348.txt +example_input_file1349.txt +example_input_file1350.txt +example_input_file1351.txt +example_input_file1352.txt +example_input_file1353.txt +example_input_file1354.txt +example_input_file1355.txt +example_input_file1356.txt +example_input_file1357.txt +example_input_file1358.txt +example_input_file1359.txt +example_input_file1360.txt +example_input_file1361.txt +example_input_file1362.txt +example_input_file1363.txt +example_input_file1364.txt +example_input_file1365.txt +example_input_file1366.txt +example_input_file1367.txt +example_input_file1368.txt +example_input_file1369.txt +example_input_file1370.txt +example_input_file1371.txt +example_input_file1372.txt +example_input_file1373.txt +example_input_file1374.txt +example_input_file1375.txt +example_input_file1376.txt +example_input_file1377.txt +example_input_file1378.txt +example_input_file1379.txt +example_input_file1380.txt +example_input_file1381.txt +example_input_file1382.txt +example_input_file1383.txt +example_input_file1384.txt +example_input_file1385.txt +example_input_file1386.txt +example_input_file1387.txt +example_input_file1388.txt +example_input_file1389.txt +example_input_file1390.txt +example_input_file1391.txt +example_input_file1392.txt +example_input_file1393.txt +example_input_file1394.txt +example_input_file1395.txt +example_input_file1396.txt +example_input_file1397.txt +example_input_file1398.txt +example_input_file1399.txt +example_input_file1400.txt +example_input_file1401.txt +example_input_file1402.txt +example_input_file1403.txt +example_input_file1404.txt +example_input_file1405.txt +example_input_file1406.txt +example_input_file1407.txt +example_input_file1408.txt +example_input_file1409.txt +example_input_file1410.txt +example_input_file1411.txt +example_input_file1412.txt +example_input_file1413.txt +example_input_file1414.txt +example_input_file1415.txt +example_input_file1416.txt +example_input_file1417.txt +example_input_file1418.txt +example_input_file1419.txt +example_input_file1420.txt +example_input_file1421.txt +example_input_file1422.txt +example_input_file1423.txt +example_input_file1424.txt +example_input_file1425.txt +example_input_file1426.txt +example_input_file1427.txt +example_input_file1428.txt +example_input_file1429.txt +example_input_file1430.txt +example_input_file1431.txt +example_input_file1432.txt +example_input_file1433.txt +example_input_file1434.txt +example_input_file1435.txt +example_input_file1436.txt +example_input_file1437.txt +example_input_file1438.txt +example_input_file1439.txt +example_input_file1440.txt +example_input_file1441.txt +example_input_file1442.txt +example_input_file1443.txt +example_input_file1444.txt +example_input_file1445.txt +example_input_file1446.txt +example_input_file1447.txt +example_input_file1448.txt +example_input_file1449.txt +example_input_file1450.txt +example_input_file1451.txt +example_input_file1452.txt +example_input_file1453.txt +example_input_file1454.txt +example_input_file1455.txt +example_input_file1456.txt +example_input_file1457.txt +example_input_file1458.txt +example_input_file1459.txt +example_input_file1460.txt +example_input_file1461.txt +example_input_file1462.txt +example_input_file1463.txt +example_input_file1464.txt +example_input_file1465.txt +example_input_file1466.txt +example_input_file1467.txt +example_input_file1468.txt +example_input_file1469.txt +example_input_file1470.txt +example_input_file1471.txt +example_input_file1472.txt +example_input_file1473.txt +example_input_file1474.txt +example_input_file1475.txt +example_input_file1476.txt +example_input_file1477.txt +example_input_file1478.txt +example_input_file1479.txt +example_input_file1480.txt +example_input_file1481.txt +example_input_file1482.txt +example_input_file1483.txt +example_input_file1484.txt +example_input_file1485.txt +example_input_file1486.txt +example_input_file1487.txt +example_input_file1488.txt +example_input_file1489.txt +example_input_file1490.txt +example_input_file1491.txt +example_input_file1492.txt +example_input_file1493.txt +example_input_file1494.txt +example_input_file1495.txt +example_input_file1496.txt +example_input_file1497.txt +example_input_file1498.txt +example_input_file1499.txt +example_input_file1500.txt +example_input_file1501.txt +example_input_file1502.txt +example_input_file1503.txt +example_input_file1504.txt +example_input_file1505.txt +example_input_file1506.txt +example_input_file1507.txt +example_input_file1508.txt +example_input_file1509.txt +example_input_file1510.txt +example_input_file1511.txt +example_input_file1512.txt +example_input_file1513.txt +example_input_file1514.txt +example_input_file1515.txt +example_input_file1516.txt +example_input_file1517.txt +example_input_file1518.txt +example_input_file1519.txt +example_input_file1520.txt +example_input_file1521.txt +example_input_file1522.txt +example_input_file1523.txt +example_input_file1524.txt +example_input_file1525.txt +example_input_file1526.txt +example_input_file1527.txt +example_input_file1528.txt +example_input_file1529.txt +example_input_file1530.txt +example_input_file1531.txt +example_input_file1532.txt +example_input_file1533.txt +example_input_file1534.txt +example_input_file1535.txt +example_input_file1536.txt +example_input_file1537.txt +example_input_file1538.txt +example_input_file1539.txt +example_input_file1540.txt +example_input_file1541.txt +example_input_file1542.txt +example_input_file1543.txt +example_input_file1544.txt +example_input_file1545.txt +example_input_file1546.txt +example_input_file1547.txt +example_input_file1548.txt +example_input_file1549.txt +example_input_file1550.txt +example_input_file1551.txt +example_input_file1552.txt +example_input_file1553.txt +example_input_file1554.txt +example_input_file1555.txt +example_input_file1556.txt +example_input_file1557.txt +example_input_file1558.txt +example_input_file1559.txt +example_input_file1560.txt +example_input_file1561.txt +example_input_file1562.txt +example_input_file1563.txt +example_input_file1564.txt +example_input_file1565.txt +example_input_file1566.txt +example_input_file1567.txt +example_input_file1568.txt +example_input_file1569.txt +example_input_file1570.txt +example_input_file1571.txt +example_input_file1572.txt +example_input_file1573.txt +example_input_file1574.txt +example_input_file1575.txt +example_input_file1576.txt +example_input_file1577.txt +example_input_file1578.txt +example_input_file1579.txt +example_input_file1580.txt +example_input_file1581.txt +example_input_file1582.txt +example_input_file1583.txt +example_input_file1584.txt +example_input_file1585.txt +example_input_file1586.txt +example_input_file1587.txt +example_input_file1588.txt +example_input_file1589.txt +example_input_file1590.txt +example_input_file1591.txt +example_input_file1592.txt +example_input_file1593.txt +example_input_file1594.txt +example_input_file1595.txt +example_input_file1596.txt +example_input_file1597.txt +example_input_file1598.txt +example_input_file1599.txt +example_input_file1600.txt +example_input_file1601.txt +example_input_file1602.txt +example_input_file1603.txt +example_input_file1604.txt +example_input_file1605.txt +example_input_file1606.txt +example_input_file1607.txt +example_input_file1608.txt +example_input_file1609.txt +example_input_file1610.txt +example_input_file1611.txt +example_input_file1612.txt +example_input_file1613.txt +example_input_file1614.txt +example_input_file1615.txt +example_input_file1616.txt +example_input_file1617.txt +example_input_file1618.txt +example_input_file1619.txt +example_input_file1620.txt +example_input_file1621.txt +example_input_file1622.txt +example_input_file1623.txt +example_input_file1624.txt +example_input_file1625.txt +example_input_file1626.txt +example_input_file1627.txt +example_input_file1628.txt +example_input_file1629.txt +example_input_file1630.txt +example_input_file1631.txt +example_input_file1632.txt +example_input_file1633.txt +example_input_file1634.txt +example_input_file1635.txt +example_input_file1636.txt +example_input_file1637.txt +example_input_file1638.txt +example_input_file1639.txt +example_input_file1640.txt +example_input_file1641.txt +example_input_file1642.txt +example_input_file1643.txt +example_input_file1644.txt +example_input_file1645.txt +example_input_file1646.txt +example_input_file1647.txt +example_input_file1648.txt +example_input_file1649.txt +example_input_file1650.txt +example_input_file1651.txt +example_input_file1652.txt +example_input_file1653.txt +example_input_file1654.txt +example_input_file1655.txt +example_input_file1656.txt +example_input_file1657.txt +example_input_file1658.txt +example_input_file1659.txt +example_input_file1660.txt +example_input_file1661.txt +example_input_file1662.txt +example_input_file1663.txt +example_input_file1664.txt +example_input_file1665.txt +example_input_file1666.txt +example_input_file1667.txt +example_input_file1668.txt +example_input_file1669.txt +example_input_file1670.txt +example_input_file1671.txt +example_input_file1672.txt +example_input_file1673.txt +example_input_file1674.txt +example_input_file1675.txt +example_input_file1676.txt +example_input_file1677.txt +example_input_file1678.txt +example_input_file1679.txt +example_input_file1680.txt +example_input_file1681.txt +example_input_file1682.txt +example_input_file1683.txt +example_input_file1684.txt +example_input_file1685.txt +example_input_file1686.txt +example_input_file1687.txt +example_input_file1688.txt +example_input_file1689.txt +example_input_file1690.txt +example_input_file1691.txt +example_input_file1692.txt +example_input_file1693.txt +example_input_file1694.txt +example_input_file1695.txt +example_input_file1696.txt +example_input_file1697.txt +example_input_file1698.txt +example_input_file1699.txt +example_input_file1700.txt +example_input_file1701.txt +example_input_file1702.txt +example_input_file1703.txt +example_input_file1704.txt +example_input_file1705.txt +example_input_file1706.txt +example_input_file1707.txt +example_input_file1708.txt +example_input_file1709.txt +example_input_file1710.txt +example_input_file1711.txt +example_input_file1712.txt +example_input_file1713.txt +example_input_file1714.txt +example_input_file1715.txt +example_input_file1716.txt +example_input_file1717.txt +example_input_file1718.txt +example_input_file1719.txt +example_input_file1720.txt +example_input_file1721.txt +example_input_file1722.txt +example_input_file1723.txt +example_input_file1724.txt +example_input_file1725.txt +example_input_file1726.txt +example_input_file1727.txt +example_input_file1728.txt +example_input_file1729.txt +example_input_file1730.txt +example_input_file1731.txt +example_input_file1732.txt +example_input_file1733.txt +example_input_file1734.txt +example_input_file1735.txt +example_input_file1736.txt +example_input_file1737.txt +example_input_file1738.txt +example_input_file1739.txt +example_input_file1740.txt +example_input_file1741.txt +example_input_file1742.txt +example_input_file1743.txt +example_input_file1744.txt +example_input_file1745.txt +example_input_file1746.txt +example_input_file1747.txt +example_input_file1748.txt +example_input_file1749.txt +example_input_file1750.txt +example_input_file1751.txt +example_input_file1752.txt +example_input_file1753.txt +example_input_file1754.txt +example_input_file1755.txt +example_input_file1756.txt +example_input_file1757.txt +example_input_file1758.txt +example_input_file1759.txt +example_input_file1760.txt +example_input_file1761.txt +example_input_file1762.txt +example_input_file1763.txt +example_input_file1764.txt +example_input_file1765.txt +example_input_file1766.txt +example_input_file1767.txt +example_input_file1768.txt +example_input_file1769.txt +example_input_file1770.txt +example_input_file1771.txt +example_input_file1772.txt +example_input_file1773.txt +example_input_file1774.txt +example_input_file1775.txt +example_input_file1776.txt +example_input_file1777.txt +example_input_file1778.txt +example_input_file1779.txt +example_input_file1780.txt +example_input_file1781.txt +example_input_file1782.txt +example_input_file1783.txt +example_input_file1784.txt +example_input_file1785.txt +example_input_file1786.txt +example_input_file1787.txt +example_input_file1788.txt +example_input_file1789.txt +example_input_file1790.txt +example_input_file1791.txt +example_input_file1792.txt +example_input_file1793.txt +example_input_file1794.txt +example_input_file1795.txt +example_input_file1796.txt +example_input_file1797.txt +example_input_file1798.txt +example_input_file1799.txt +example_input_file1800.txt +example_input_file1801.txt +example_input_file1802.txt +example_input_file1803.txt +example_input_file1804.txt +example_input_file1805.txt +example_input_file1806.txt +example_input_file1807.txt +example_input_file1808.txt +example_input_file1809.txt +example_input_file1810.txt +example_input_file1811.txt +example_input_file1812.txt +example_input_file1813.txt +example_input_file1814.txt +example_input_file1815.txt +example_input_file1816.txt +example_input_file1817.txt +example_input_file1818.txt +example_input_file1819.txt +example_input_file1820.txt +example_input_file1821.txt +example_input_file1822.txt +example_input_file1823.txt +example_input_file1824.txt +example_input_file1825.txt +example_input_file1826.txt +example_input_file1827.txt +example_input_file1828.txt +example_input_file1829.txt +example_input_file1830.txt +example_input_file1831.txt +example_input_file1832.txt +example_input_file1833.txt +example_input_file1834.txt +example_input_file1835.txt +example_input_file1836.txt +example_input_file1837.txt +example_input_file1838.txt +example_input_file1839.txt +example_input_file1840.txt +example_input_file1841.txt +example_input_file1842.txt +example_input_file1843.txt +example_input_file1844.txt +example_input_file1845.txt +example_input_file1846.txt +example_input_file1847.txt +example_input_file1848.txt +example_input_file1849.txt +example_input_file1850.txt +example_input_file1851.txt +example_input_file1852.txt +example_input_file1853.txt +example_input_file1854.txt +example_input_file1855.txt +example_input_file1856.txt +example_input_file1857.txt +example_input_file1858.txt +example_input_file1859.txt +example_input_file1860.txt +example_input_file1861.txt +example_input_file1862.txt +example_input_file1863.txt +example_input_file1864.txt +example_input_file1865.txt +example_input_file1866.txt +example_input_file1867.txt +example_input_file1868.txt +example_input_file1869.txt +example_input_file1870.txt +example_input_file1871.txt +example_input_file1872.txt +example_input_file1873.txt +example_input_file1874.txt +example_input_file1875.txt +example_input_file1876.txt +example_input_file1877.txt +example_input_file1878.txt +example_input_file1879.txt +example_input_file1880.txt +example_input_file1881.txt +example_input_file1882.txt +example_input_file1883.txt +example_input_file1884.txt +example_input_file1885.txt +example_input_file1886.txt +example_input_file1887.txt +example_input_file1888.txt +example_input_file1889.txt +example_input_file1890.txt +example_input_file1891.txt +example_input_file1892.txt +example_input_file1893.txt +example_input_file1894.txt +example_input_file1895.txt +example_input_file1896.txt +example_input_file1897.txt +example_input_file1898.txt +example_input_file1899.txt +example_input_file1900.txt +example_input_file1901.txt +example_input_file1902.txt +example_input_file1903.txt +example_input_file1904.txt +example_input_file1905.txt +example_input_file1906.txt +example_input_file1907.txt +example_input_file1908.txt +example_input_file1909.txt +example_input_file1910.txt +example_input_file1911.txt +example_input_file1912.txt +example_input_file1913.txt +example_input_file1914.txt +example_input_file1915.txt +example_input_file1916.txt +example_input_file1917.txt +example_input_file1918.txt +example_input_file1919.txt +example_input_file1920.txt +example_input_file1921.txt +example_input_file1922.txt +example_input_file1923.txt +example_input_file1924.txt +example_input_file1925.txt +example_input_file1926.txt +example_input_file1927.txt +example_input_file1928.txt +example_input_file1929.txt +example_input_file1930.txt +example_input_file1931.txt +example_input_file1932.txt +example_input_file1933.txt +example_input_file1934.txt +example_input_file1935.txt +example_input_file1936.txt +example_input_file1937.txt +example_input_file1938.txt +example_input_file1939.txt +example_input_file1940.txt +example_input_file1941.txt +example_input_file1942.txt +example_input_file1943.txt +example_input_file1944.txt +example_input_file1945.txt +example_input_file1946.txt +example_input_file1947.txt +example_input_file1948.txt +example_input_file1949.txt +example_input_file1950.txt +example_input_file1951.txt +example_input_file1952.txt +example_input_file1953.txt +example_input_file1954.txt +example_input_file1955.txt +example_input_file1956.txt +example_input_file1957.txt +example_input_file1958.txt +example_input_file1959.txt +example_input_file1960.txt +example_input_file1961.txt +example_input_file1962.txt +example_input_file1963.txt +example_input_file1964.txt +example_input_file1965.txt +example_input_file1966.txt +example_input_file1967.txt +example_input_file1968.txt +example_input_file1969.txt +example_input_file1970.txt +example_input_file1971.txt +example_input_file1972.txt +example_input_file1973.txt +example_input_file1974.txt +example_input_file1975.txt +example_input_file1976.txt +example_input_file1977.txt +example_input_file1978.txt +example_input_file1979.txt +example_input_file1980.txt +example_input_file1981.txt +example_input_file1982.txt +example_input_file1983.txt +example_input_file1984.txt +example_input_file1985.txt +example_input_file1986.txt +example_input_file1987.txt +example_input_file1988.txt +example_input_file1989.txt +example_input_file1990.txt +example_input_file1991.txt +example_input_file1992.txt +example_input_file1993.txt +example_input_file1994.txt +example_input_file1995.txt +example_input_file1996.txt +example_input_file1997.txt +example_input_file1998.txt +example_input_file1999.txt +example_input_file2000.txt +example_input_file2001.txt +example_input_file2002.txt +example_input_file2003.txt +example_input_file2004.txt +example_input_file2005.txt +example_input_file2006.txt +example_input_file2007.txt +example_input_file2008.txt +example_input_file2009.txt +example_input_file2010.txt +example_input_file2011.txt +example_input_file2012.txt +example_input_file2013.txt +example_input_file2014.txt +example_input_file2015.txt +example_input_file2016.txt +example_input_file2017.txt +example_input_file2018.txt +example_input_file2019.txt +example_input_file2020.txt +example_input_file2021.txt +example_input_file2022.txt +example_input_file2023.txt +example_input_file2024.txt +example_input_file2025.txt +example_input_file2026.txt +example_input_file2027.txt +example_input_file2028.txt +example_input_file2029.txt +example_input_file2030.txt +example_input_file2031.txt +example_input_file2032.txt +example_input_file2033.txt +example_input_file2034.txt +example_input_file2035.txt +example_input_file2036.txt +example_input_file2037.txt +example_input_file2038.txt +example_input_file2039.txt +example_input_file2040.txt +example_input_file2041.txt +example_input_file2042.txt +example_input_file2043.txt +example_input_file2044.txt +example_input_file2045.txt +example_input_file2046.txt +example_input_file2047.txt +example_input_file2048.txt +example_input_file2049.txt +example_input_file2050.txt +example_input_file2051.txt +example_input_file2052.txt +example_input_file2053.txt +example_input_file2054.txt +example_input_file2055.txt +example_input_file2056.txt +example_input_file2057.txt +example_input_file2058.txt +example_input_file2059.txt +example_input_file2060.txt +example_input_file2061.txt +example_input_file2062.txt +example_input_file2063.txt +example_input_file2064.txt +example_input_file2065.txt +example_input_file2066.txt +example_input_file2067.txt +example_input_file2068.txt +example_input_file2069.txt +example_input_file2070.txt +example_input_file2071.txt +example_input_file2072.txt +example_input_file2073.txt +example_input_file2074.txt +example_input_file2075.txt +example_input_file2076.txt +example_input_file2077.txt +example_input_file2078.txt +example_input_file2079.txt +example_input_file2080.txt +example_input_file2081.txt +example_input_file2082.txt +example_input_file2083.txt +example_input_file2084.txt +example_input_file2085.txt +example_input_file2086.txt +example_input_file2087.txt +example_input_file2088.txt +example_input_file2089.txt +example_input_file2090.txt +example_input_file2091.txt +example_input_file2092.txt +example_input_file2093.txt +example_input_file2094.txt +example_input_file2095.txt +example_input_file2096.txt +example_input_file2097.txt +example_input_file2098.txt +example_input_file2099.txt +example_input_file2100.txt +example_input_file2101.txt +example_input_file2102.txt +example_input_file2103.txt +example_input_file2104.txt +example_input_file2105.txt +example_input_file2106.txt +example_input_file2107.txt +example_input_file2108.txt +example_input_file2109.txt +example_input_file2110.txt +example_input_file2111.txt +example_input_file2112.txt +example_input_file2113.txt +example_input_file2114.txt +example_input_file2115.txt +example_input_file2116.txt +example_input_file2117.txt +example_input_file2118.txt +example_input_file2119.txt +example_input_file2120.txt +example_input_file2121.txt +example_input_file2122.txt +example_input_file2123.txt +example_input_file2124.txt +example_input_file2125.txt +example_input_file2126.txt +example_input_file2127.txt +example_input_file2128.txt +example_input_file2129.txt +example_input_file2130.txt +example_input_file2131.txt +example_input_file2132.txt +example_input_file2133.txt +example_input_file2134.txt +example_input_file2135.txt +example_input_file2136.txt +example_input_file2137.txt +example_input_file2138.txt +example_input_file2139.txt +example_input_file2140.txt +example_input_file2141.txt +example_input_file2142.txt +example_input_file2143.txt +example_input_file2144.txt +example_input_file2145.txt +example_input_file2146.txt +example_input_file2147.txt +example_input_file2148.txt +example_input_file2149.txt +example_input_file2150.txt +example_input_file2151.txt +example_input_file2152.txt +example_input_file2153.txt +example_input_file2154.txt +example_input_file2155.txt +example_input_file2156.txt +example_input_file2157.txt +example_input_file2158.txt +example_input_file2159.txt +example_input_file2160.txt +example_input_file2161.txt +example_input_file2162.txt +example_input_file2163.txt +example_input_file2164.txt +example_input_file2165.txt +example_input_file2166.txt +example_input_file2167.txt +example_input_file2168.txt +example_input_file2169.txt +example_input_file2170.txt +example_input_file2171.txt +example_input_file2172.txt +example_input_file2173.txt +example_input_file2174.txt +example_input_file2175.txt +example_input_file2176.txt +example_input_file2177.txt +example_input_file2178.txt +example_input_file2179.txt +example_input_file2180.txt +example_input_file2181.txt +example_input_file2182.txt +example_input_file2183.txt +example_input_file2184.txt +example_input_file2185.txt +example_input_file2186.txt +example_input_file2187.txt +example_input_file2188.txt +example_input_file2189.txt +example_input_file2190.txt +example_input_file2191.txt +example_input_file2192.txt +example_input_file2193.txt +example_input_file2194.txt +example_input_file2195.txt +example_input_file2196.txt +example_input_file2197.txt +example_input_file2198.txt +example_input_file2199.txt +example_input_file2200.txt +example_input_file2201.txt +example_input_file2202.txt +example_input_file2203.txt +example_input_file2204.txt +example_input_file2205.txt +example_input_file2206.txt +example_input_file2207.txt +example_input_file2208.txt +example_input_file2209.txt +example_input_file2210.txt +example_input_file2211.txt +example_input_file2212.txt +example_input_file2213.txt +example_input_file2214.txt +example_input_file2215.txt +example_input_file2216.txt +example_input_file2217.txt +example_input_file2218.txt +example_input_file2219.txt +example_input_file2220.txt +example_input_file2221.txt +example_input_file2222.txt +example_input_file2223.txt +example_input_file2224.txt +example_input_file2225.txt +example_input_file2226.txt +example_input_file2227.txt +example_input_file2228.txt +example_input_file2229.txt +example_input_file2230.txt +example_input_file2231.txt +example_input_file2232.txt +example_input_file2233.txt +example_input_file2234.txt +example_input_file2235.txt +example_input_file2236.txt +example_input_file2237.txt +example_input_file2238.txt +example_input_file2239.txt +example_input_file2240.txt +example_input_file2241.txt +example_input_file2242.txt +example_input_file2243.txt +example_input_file2244.txt +example_input_file2245.txt +example_input_file2246.txt +example_input_file2247.txt +example_input_file2248.txt +example_input_file2249.txt +example_input_file2250.txt +example_input_file2251.txt +example_input_file2252.txt +example_input_file2253.txt +example_input_file2254.txt +example_input_file2255.txt +example_input_file2256.txt +example_input_file2257.txt +example_input_file2258.txt +example_input_file2259.txt +example_input_file2260.txt +example_input_file2261.txt +example_input_file2262.txt +example_input_file2263.txt +example_input_file2264.txt +example_input_file2265.txt +example_input_file2266.txt +example_input_file2267.txt +example_input_file2268.txt +example_input_file2269.txt +example_input_file2270.txt +example_input_file2271.txt +example_input_file2272.txt +example_input_file2273.txt +example_input_file2274.txt +example_input_file2275.txt +example_input_file2276.txt +example_input_file2277.txt +example_input_file2278.txt +example_input_file2279.txt +example_input_file2280.txt +example_input_file2281.txt +example_input_file2282.txt +example_input_file2283.txt +example_input_file2284.txt +example_input_file2285.txt +example_input_file2286.txt +example_input_file2287.txt +example_input_file2288.txt +example_input_file2289.txt +example_input_file2290.txt +example_input_file2291.txt +example_input_file2292.txt +example_input_file2293.txt +example_input_file2294.txt +example_input_file2295.txt +example_input_file2296.txt +example_input_file2297.txt +example_input_file2298.txt +example_input_file2299.txt +example_input_file2300.txt +example_input_file2301.txt +example_input_file2302.txt +example_input_file2303.txt +example_input_file2304.txt +example_input_file2305.txt +example_input_file2306.txt +example_input_file2307.txt +example_input_file2308.txt +example_input_file2309.txt +example_input_file2310.txt +example_input_file2311.txt +example_input_file2312.txt +example_input_file2313.txt +example_input_file2314.txt +example_input_file2315.txt +example_input_file2316.txt +example_input_file2317.txt +example_input_file2318.txt +example_input_file2319.txt +example_input_file2320.txt +example_input_file2321.txt +example_input_file2322.txt +example_input_file2323.txt +example_input_file2324.txt +example_input_file2325.txt +example_input_file2326.txt +example_input_file2327.txt +example_input_file2328.txt +example_input_file2329.txt +example_input_file2330.txt +example_input_file2331.txt +example_input_file2332.txt +example_input_file2333.txt +example_input_file2334.txt +example_input_file2335.txt +example_input_file2336.txt +example_input_file2337.txt +example_input_file2338.txt +example_input_file2339.txt +example_input_file2340.txt +example_input_file2341.txt +example_input_file2342.txt +example_input_file2343.txt +example_input_file2344.txt +example_input_file2345.txt +example_input_file2346.txt +example_input_file2347.txt +example_input_file2348.txt +example_input_file2349.txt +example_input_file2350.txt +example_input_file2351.txt +example_input_file2352.txt +example_input_file2353.txt +example_input_file2354.txt +example_input_file2355.txt +example_input_file2356.txt +example_input_file2357.txt +example_input_file2358.txt +example_input_file2359.txt +example_input_file2360.txt +example_input_file2361.txt +example_input_file2362.txt +example_input_file2363.txt +example_input_file2364.txt +example_input_file2365.txt +example_input_file2366.txt +example_input_file2367.txt +example_input_file2368.txt +example_input_file2369.txt +example_input_file2370.txt +example_input_file2371.txt +example_input_file2372.txt +example_input_file2373.txt +example_input_file2374.txt +example_input_file2375.txt +example_input_file2376.txt +example_input_file2377.txt +example_input_file2378.txt +example_input_file2379.txt +example_input_file2380.txt +example_input_file2381.txt +example_input_file2382.txt +example_input_file2383.txt +example_input_file2384.txt +example_input_file2385.txt +example_input_file2386.txt +example_input_file2387.txt +example_input_file2388.txt +example_input_file2389.txt +example_input_file2390.txt +example_input_file2391.txt +example_input_file2392.txt +example_input_file2393.txt +example_input_file2394.txt +example_input_file2395.txt +example_input_file2396.txt +example_input_file2397.txt +example_input_file2398.txt +example_input_file2399.txt +example_input_file2400.txt +example_input_file2401.txt +example_input_file2402.txt +example_input_file2403.txt +example_input_file2404.txt +example_input_file2405.txt +example_input_file2406.txt +example_input_file2407.txt +example_input_file2408.txt +example_input_file2409.txt +example_input_file2410.txt +example_input_file2411.txt +example_input_file2412.txt +example_input_file2413.txt +example_input_file2414.txt +example_input_file2415.txt +example_input_file2416.txt +example_input_file2417.txt +example_input_file2418.txt +example_input_file2419.txt +example_input_file2420.txt +example_input_file2421.txt +example_input_file2422.txt +example_input_file2423.txt +example_input_file2424.txt +example_input_file2425.txt +example_input_file2426.txt +example_input_file2427.txt +example_input_file2428.txt +example_input_file2429.txt +example_input_file2430.txt +example_input_file2431.txt +example_input_file2432.txt +example_input_file2433.txt +example_input_file2434.txt +example_input_file2435.txt +example_input_file2436.txt +example_input_file2437.txt +example_input_file2438.txt +example_input_file2439.txt +example_input_file2440.txt +example_input_file2441.txt +example_input_file2442.txt +example_input_file2443.txt +example_input_file2444.txt +example_input_file2445.txt +example_input_file2446.txt +example_input_file2447.txt +example_input_file2448.txt +example_input_file2449.txt +example_input_file2450.txt +example_input_file2451.txt +example_input_file2452.txt +example_input_file2453.txt +example_input_file2454.txt +example_input_file2455.txt +example_input_file2456.txt +example_input_file2457.txt +example_input_file2458.txt +example_input_file2459.txt +example_input_file2460.txt +example_input_file2461.txt +example_input_file2462.txt +example_input_file2463.txt +example_input_file2464.txt +example_input_file2465.txt +example_input_file2466.txt +example_input_file2467.txt +example_input_file2468.txt +example_input_file2469.txt +example_input_file2470.txt +example_input_file2471.txt +example_input_file2472.txt +example_input_file2473.txt +example_input_file2474.txt +example_input_file2475.txt +example_input_file2476.txt +example_input_file2477.txt +example_input_file2478.txt +example_input_file2479.txt +example_input_file2480.txt +example_input_file2481.txt +example_input_file2482.txt +example_input_file2483.txt +example_input_file2484.txt +example_input_file2485.txt +example_input_file2486.txt +example_input_file2487.txt +example_input_file2488.txt +example_input_file2489.txt +example_input_file2490.txt +example_input_file2491.txt +example_input_file2492.txt +example_input_file2493.txt +example_input_file2494.txt +example_input_file2495.txt +example_input_file2496.txt +example_input_file2497.txt +example_input_file2498.txt +example_input_file2499.txt +example_input_file2500.txt +example_input_file2501.txt +example_input_file2502.txt +example_input_file2503.txt +example_input_file2504.txt +example_input_file2505.txt +example_input_file2506.txt +example_input_file2507.txt +example_input_file2508.txt +example_input_file2509.txt +example_input_file2510.txt +example_input_file2511.txt +example_input_file2512.txt +example_input_file2513.txt +example_input_file2514.txt +example_input_file2515.txt +example_input_file2516.txt +example_input_file2517.txt +example_input_file2518.txt +example_input_file2519.txt +example_input_file2520.txt +example_input_file2521.txt +example_input_file2522.txt +example_input_file2523.txt +example_input_file2524.txt +example_input_file2525.txt +example_input_file2526.txt +example_input_file2527.txt +example_input_file2528.txt +example_input_file2529.txt +example_input_file2530.txt +example_input_file2531.txt +example_input_file2532.txt +example_input_file2533.txt +example_input_file2534.txt +example_input_file2535.txt +example_input_file2536.txt +example_input_file2537.txt +example_input_file2538.txt +example_input_file2539.txt +example_input_file2540.txt +example_input_file2541.txt +example_input_file2542.txt +example_input_file2543.txt +example_input_file2544.txt +example_input_file2545.txt +example_input_file2546.txt +example_input_file2547.txt +example_input_file2548.txt +example_input_file2549.txt +example_input_file2550.txt +example_input_file2551.txt +example_input_file2552.txt +example_input_file2553.txt +example_input_file2554.txt +example_input_file2555.txt +example_input_file2556.txt +example_input_file2557.txt +example_input_file2558.txt +example_input_file2559.txt +example_input_file2560.txt +example_input_file2561.txt +example_input_file2562.txt +example_input_file2563.txt +example_input_file2564.txt +example_input_file2565.txt +example_input_file2566.txt +example_input_file2567.txt +example_input_file2568.txt +example_input_file2569.txt +example_input_file2570.txt +example_input_file2571.txt +example_input_file2572.txt +example_input_file2573.txt +example_input_file2574.txt +example_input_file2575.txt +example_input_file2576.txt +example_input_file2577.txt +example_input_file2578.txt +example_input_file2579.txt +example_input_file2580.txt +example_input_file2581.txt +example_input_file2582.txt +example_input_file2583.txt +example_input_file2584.txt +example_input_file2585.txt +example_input_file2586.txt +example_input_file2587.txt +example_input_file2588.txt +example_input_file2589.txt +example_input_file2590.txt +example_input_file2591.txt +example_input_file2592.txt +example_input_file2593.txt +example_input_file2594.txt +example_input_file2595.txt +example_input_file2596.txt +example_input_file2597.txt +example_input_file2598.txt +example_input_file2599.txt +example_input_file2600.txt +example_input_file2601.txt +example_input_file2602.txt +example_input_file2603.txt +example_input_file2604.txt +example_input_file2605.txt +example_input_file2606.txt +example_input_file2607.txt +example_input_file2608.txt +example_input_file2609.txt +example_input_file2610.txt +example_input_file2611.txt +example_input_file2612.txt +example_input_file2613.txt +example_input_file2614.txt +example_input_file2615.txt +example_input_file2616.txt +example_input_file2617.txt +example_input_file2618.txt +example_input_file2619.txt +example_input_file2620.txt +example_input_file2621.txt +example_input_file2622.txt +example_input_file2623.txt +example_input_file2624.txt +example_input_file2625.txt +example_input_file2626.txt +example_input_file2627.txt +example_input_file2628.txt +example_input_file2629.txt +example_input_file2630.txt +example_input_file2631.txt +example_input_file2632.txt +example_input_file2633.txt +example_input_file2634.txt +example_input_file2635.txt +example_input_file2636.txt +example_input_file2637.txt +example_input_file2638.txt +example_input_file2639.txt +example_input_file2640.txt +example_input_file2641.txt +example_input_file2642.txt +example_input_file2643.txt +example_input_file2644.txt +example_input_file2645.txt +example_input_file2646.txt +example_input_file2647.txt +example_input_file2648.txt +example_input_file2649.txt +example_input_file2650.txt +example_input_file2651.txt +example_input_file2652.txt +example_input_file2653.txt +example_input_file2654.txt +example_input_file2655.txt +example_input_file2656.txt +example_input_file2657.txt +example_input_file2658.txt +example_input_file2659.txt +example_input_file2660.txt +example_input_file2661.txt +example_input_file2662.txt +example_input_file2663.txt +example_input_file2664.txt +example_input_file2665.txt +example_input_file2666.txt +example_input_file2667.txt +example_input_file2668.txt +example_input_file2669.txt +example_input_file2670.txt +example_input_file2671.txt +example_input_file2672.txt +example_input_file2673.txt +example_input_file2674.txt +example_input_file2675.txt +example_input_file2676.txt +example_input_file2677.txt +example_input_file2678.txt +example_input_file2679.txt +example_input_file2680.txt +example_input_file2681.txt +example_input_file2682.txt +example_input_file2683.txt +example_input_file2684.txt +example_input_file2685.txt +example_input_file2686.txt +example_input_file2687.txt +example_input_file2688.txt +example_input_file2689.txt +example_input_file2690.txt +example_input_file2691.txt +example_input_file2692.txt +example_input_file2693.txt +example_input_file2694.txt +example_input_file2695.txt +example_input_file2696.txt +example_input_file2697.txt +example_input_file2698.txt +example_input_file2699.txt +example_input_file2700.txt +example_input_file2701.txt +example_input_file2702.txt +example_input_file2703.txt +example_input_file2704.txt +example_input_file2705.txt +example_input_file2706.txt +example_input_file2707.txt +example_input_file2708.txt +example_input_file2709.txt +example_input_file2710.txt +example_input_file2711.txt +example_input_file2712.txt +example_input_file2713.txt +example_input_file2714.txt +example_input_file2715.txt +example_input_file2716.txt +example_input_file2717.txt +example_input_file2718.txt +example_input_file2719.txt +example_input_file2720.txt +example_input_file2721.txt +example_input_file2722.txt +example_input_file2723.txt +example_input_file2724.txt +example_input_file2725.txt +example_input_file2726.txt +example_input_file2727.txt +example_input_file2728.txt +example_input_file2729.txt +example_input_file2730.txt +example_input_file2731.txt +example_input_file2732.txt +example_input_file2733.txt +example_input_file2734.txt +example_input_file2735.txt +example_input_file2736.txt +example_input_file2737.txt +example_input_file2738.txt +example_input_file2739.txt +example_input_file2740.txt +example_input_file2741.txt +example_input_file2742.txt +example_input_file2743.txt +example_input_file2744.txt +example_input_file2745.txt +example_input_file2746.txt +example_input_file2747.txt +example_input_file2748.txt +example_input_file2749.txt +example_input_file2750.txt +example_input_file2751.txt +example_input_file2752.txt +example_input_file2753.txt +example_input_file2754.txt +example_input_file2755.txt +example_input_file2756.txt +example_input_file2757.txt +example_input_file2758.txt +example_input_file2759.txt +example_input_file2760.txt +example_input_file2761.txt +example_input_file2762.txt +example_input_file2763.txt +example_input_file2764.txt +example_input_file2765.txt +example_input_file2766.txt +example_input_file2767.txt +example_input_file2768.txt +example_input_file2769.txt +example_input_file2770.txt +example_input_file2771.txt +example_input_file2772.txt +example_input_file2773.txt +example_input_file2774.txt +example_input_file2775.txt +example_input_file2776.txt +example_input_file2777.txt +example_input_file2778.txt +example_input_file2779.txt +example_input_file2780.txt +example_input_file2781.txt +example_input_file2782.txt +example_input_file2783.txt +example_input_file2784.txt +example_input_file2785.txt +example_input_file2786.txt +example_input_file2787.txt +example_input_file2788.txt +example_input_file2789.txt +example_input_file2790.txt +example_input_file2791.txt +example_input_file2792.txt +example_input_file2793.txt +example_input_file2794.txt +example_input_file2795.txt +example_input_file2796.txt +example_input_file2797.txt +example_input_file2798.txt +example_input_file2799.txt +example_input_file2800.txt +example_input_file2801.txt +example_input_file2802.txt +example_input_file2803.txt +example_input_file2804.txt +example_input_file2805.txt +example_input_file2806.txt +example_input_file2807.txt +example_input_file2808.txt +example_input_file2809.txt +example_input_file2810.txt +example_input_file2811.txt +example_input_file2812.txt +example_input_file2813.txt +example_input_file2814.txt +example_input_file2815.txt +example_input_file2816.txt +example_input_file2817.txt +example_input_file2818.txt +example_input_file2819.txt +example_input_file2820.txt +example_input_file2821.txt +example_input_file2822.txt +example_input_file2823.txt +example_input_file2824.txt +example_input_file2825.txt +example_input_file2826.txt +example_input_file2827.txt +example_input_file2828.txt +example_input_file2829.txt +example_input_file2830.txt +example_input_file2831.txt +example_input_file2832.txt +example_input_file2833.txt +example_input_file2834.txt +example_input_file2835.txt +example_input_file2836.txt +example_input_file2837.txt +example_input_file2838.txt +example_input_file2839.txt +example_input_file2840.txt +example_input_file2841.txt +example_input_file2842.txt +example_input_file2843.txt +example_input_file2844.txt +example_input_file2845.txt +example_input_file2846.txt +example_input_file2847.txt +example_input_file2848.txt +example_input_file2849.txt +example_input_file2850.txt +example_input_file2851.txt +example_input_file2852.txt +example_input_file2853.txt +example_input_file2854.txt +example_input_file2855.txt +example_input_file2856.txt +example_input_file2857.txt +example_input_file2858.txt +example_input_file2859.txt +example_input_file2860.txt +example_input_file2861.txt +example_input_file2862.txt +example_input_file2863.txt +example_input_file2864.txt +example_input_file2865.txt +example_input_file2866.txt +example_input_file2867.txt +example_input_file2868.txt +example_input_file2869.txt +example_input_file2870.txt +example_input_file2871.txt +example_input_file2872.txt +example_input_file2873.txt +example_input_file2874.txt +example_input_file2875.txt +example_input_file2876.txt +example_input_file2877.txt +example_input_file2878.txt +example_input_file2879.txt +example_input_file2880.txt +example_input_file2881.txt +example_input_file2882.txt +example_input_file2883.txt +example_input_file2884.txt +example_input_file2885.txt +example_input_file2886.txt +example_input_file2887.txt +example_input_file2888.txt +example_input_file2889.txt +example_input_file2890.txt +example_input_file2891.txt +example_input_file2892.txt +example_input_file2893.txt +example_input_file2894.txt +example_input_file2895.txt +example_input_file2896.txt +example_input_file2897.txt +example_input_file2898.txt +example_input_file2899.txt +example_input_file2900.txt +example_input_file2901.txt +example_input_file2902.txt +example_input_file2903.txt +example_input_file2904.txt +example_input_file2905.txt +example_input_file2906.txt +example_input_file2907.txt +example_input_file2908.txt +example_input_file2909.txt +example_input_file2910.txt +example_input_file2911.txt +example_input_file2912.txt +example_input_file2913.txt +example_input_file2914.txt +example_input_file2915.txt +example_input_file2916.txt +example_input_file2917.txt +example_input_file2918.txt +example_input_file2919.txt +example_input_file2920.txt +example_input_file2921.txt +example_input_file2922.txt +example_input_file2923.txt +example_input_file2924.txt +example_input_file2925.txt +example_input_file2926.txt +example_input_file2927.txt +example_input_file2928.txt +example_input_file2929.txt +example_input_file2930.txt +example_input_file2931.txt +example_input_file2932.txt +example_input_file2933.txt +example_input_file2934.txt +example_input_file2935.txt +example_input_file2936.txt +example_input_file2937.txt +example_input_file2938.txt +example_input_file2939.txt +example_input_file2940.txt +example_input_file2941.txt +example_input_file2942.txt +example_input_file2943.txt +example_input_file2944.txt +example_input_file2945.txt +example_input_file2946.txt +example_input_file2947.txt +example_input_file2948.txt +example_input_file2949.txt +example_input_file2950.txt +example_input_file2951.txt +example_input_file2952.txt +example_input_file2953.txt +example_input_file2954.txt +example_input_file2955.txt +example_input_file2956.txt +example_input_file2957.txt +example_input_file2958.txt +example_input_file2959.txt +example_input_file2960.txt +example_input_file2961.txt +example_input_file2962.txt +example_input_file2963.txt +example_input_file2964.txt +example_input_file2965.txt +example_input_file2966.txt +example_input_file2967.txt +example_input_file2968.txt +example_input_file2969.txt +example_input_file2970.txt +example_input_file2971.txt +example_input_file2972.txt +example_input_file2973.txt +example_input_file2974.txt +example_input_file2975.txt +example_input_file2976.txt +example_input_file2977.txt +example_input_file2978.txt +example_input_file2979.txt +example_input_file2980.txt +example_input_file2981.txt +example_input_file2982.txt +example_input_file2983.txt +example_input_file2984.txt +example_input_file2985.txt +example_input_file2986.txt +example_input_file2987.txt +example_input_file2988.txt +example_input_file2989.txt +example_input_file2990.txt +example_input_file2991.txt +example_input_file2992.txt +example_input_file2993.txt +example_input_file2994.txt +example_input_file2995.txt +example_input_file2996.txt +example_input_file2997.txt +example_input_file2998.txt +example_input_file2999.txt +example_input_file3000.txt +example_input_file3001.txt +example_input_file3002.txt +example_input_file3003.txt +example_input_file3004.txt +example_input_file3005.txt +example_input_file3006.txt +example_input_file3007.txt +example_input_file3008.txt +example_input_file3009.txt +example_input_file3010.txt +example_input_file3011.txt +example_input_file3012.txt +example_input_file3013.txt +example_input_file3014.txt +example_input_file3015.txt +example_input_file3016.txt +example_input_file3017.txt +example_input_file3018.txt +example_input_file3019.txt +example_input_file3020.txt +example_input_file3021.txt +example_input_file3022.txt +example_input_file3023.txt +example_input_file3024.txt +example_input_file3025.txt +example_input_file3026.txt +example_input_file3027.txt +example_input_file3028.txt +example_input_file3029.txt +example_input_file3030.txt +example_input_file3031.txt +example_input_file3032.txt +example_input_file3033.txt +example_input_file3034.txt +example_input_file3035.txt +example_input_file3036.txt +example_input_file3037.txt +example_input_file3038.txt +example_input_file3039.txt +example_input_file3040.txt +example_input_file3041.txt +example_input_file3042.txt +example_input_file3043.txt +example_input_file3044.txt +example_input_file3045.txt +example_input_file3046.txt +example_input_file3047.txt +example_input_file3048.txt +example_input_file3049.txt +example_input_file3050.txt +example_input_file3051.txt +example_input_file3052.txt +example_input_file3053.txt +example_input_file3054.txt +example_input_file3055.txt +example_input_file3056.txt +example_input_file3057.txt +example_input_file3058.txt +example_input_file3059.txt +example_input_file3060.txt +example_input_file3061.txt +example_input_file3062.txt +example_input_file3063.txt +example_input_file3064.txt +example_input_file3065.txt +example_input_file3066.txt +example_input_file3067.txt +example_input_file3068.txt +example_input_file3069.txt +example_input_file3070.txt +example_input_file3071.txt +example_input_file3072.txt +example_input_file3073.txt +example_input_file3074.txt +example_input_file3075.txt +example_input_file3076.txt +example_input_file3077.txt +example_input_file3078.txt +example_input_file3079.txt +example_input_file3080.txt +example_input_file3081.txt +example_input_file3082.txt +example_input_file3083.txt +example_input_file3084.txt +example_input_file3085.txt +example_input_file3086.txt +example_input_file3087.txt +example_input_file3088.txt +example_input_file3089.txt +example_input_file3090.txt +example_input_file3091.txt +example_input_file3092.txt +example_input_file3093.txt +example_input_file3094.txt +example_input_file3095.txt +example_input_file3096.txt +example_input_file3097.txt +example_input_file3098.txt +example_input_file3099.txt +example_input_file3100.txt +example_input_file3101.txt +example_input_file3102.txt +example_input_file3103.txt +example_input_file3104.txt +example_input_file3105.txt +example_input_file3106.txt +example_input_file3107.txt +example_input_file3108.txt +example_input_file3109.txt +example_input_file3110.txt +example_input_file3111.txt +example_input_file3112.txt +example_input_file3113.txt +example_input_file3114.txt +example_input_file3115.txt +example_input_file3116.txt +example_input_file3117.txt +example_input_file3118.txt +example_input_file3119.txt +example_input_file3120.txt +example_input_file3121.txt +example_input_file3122.txt +example_input_file3123.txt +example_input_file3124.txt +example_input_file3125.txt +example_input_file3126.txt +example_input_file3127.txt +example_input_file3128.txt +example_input_file3129.txt +example_input_file3130.txt +example_input_file3131.txt +example_input_file3132.txt +example_input_file3133.txt +example_input_file3134.txt +example_input_file3135.txt +example_input_file3136.txt +example_input_file3137.txt +example_input_file3138.txt +example_input_file3139.txt +example_input_file3140.txt +example_input_file3141.txt +example_input_file3142.txt +example_input_file3143.txt +example_input_file3144.txt +example_input_file3145.txt +example_input_file3146.txt +example_input_file3147.txt +example_input_file3148.txt +example_input_file3149.txt +example_input_file3150.txt +example_input_file3151.txt +example_input_file3152.txt +example_input_file3153.txt +example_input_file3154.txt +example_input_file3155.txt +example_input_file3156.txt +example_input_file3157.txt +example_input_file3158.txt +example_input_file3159.txt +example_input_file3160.txt +example_input_file3161.txt +example_input_file3162.txt +example_input_file3163.txt +example_input_file3164.txt +example_input_file3165.txt +example_input_file3166.txt +example_input_file3167.txt +example_input_file3168.txt +example_input_file3169.txt +example_input_file3170.txt +example_input_file3171.txt +example_input_file3172.txt +example_input_file3173.txt +example_input_file3174.txt +example_input_file3175.txt +example_input_file3176.txt +example_input_file3177.txt +example_input_file3178.txt +example_input_file3179.txt +example_input_file3180.txt +example_input_file3181.txt +example_input_file3182.txt +example_input_file3183.txt +example_input_file3184.txt +example_input_file3185.txt +example_input_file3186.txt +example_input_file3187.txt +example_input_file3188.txt +example_input_file3189.txt +example_input_file3190.txt +example_input_file3191.txt +example_input_file3192.txt +example_input_file3193.txt +example_input_file3194.txt +example_input_file3195.txt +example_input_file3196.txt +example_input_file3197.txt +example_input_file3198.txt +example_input_file3199.txt +example_input_file3200.txt +example_input_file3201.txt +example_input_file3202.txt +example_input_file3203.txt +example_input_file3204.txt +example_input_file3205.txt +example_input_file3206.txt +example_input_file3207.txt +example_input_file3208.txt +example_input_file3209.txt +example_input_file3210.txt +example_input_file3211.txt +example_input_file3212.txt +example_input_file3213.txt +example_input_file3214.txt +example_input_file3215.txt +example_input_file3216.txt +example_input_file3217.txt +example_input_file3218.txt +example_input_file3219.txt +example_input_file3220.txt +example_input_file3221.txt +example_input_file3222.txt +example_input_file3223.txt +example_input_file3224.txt +example_input_file3225.txt +example_input_file3226.txt +example_input_file3227.txt +example_input_file3228.txt +example_input_file3229.txt +example_input_file3230.txt +example_input_file3231.txt +example_input_file3232.txt +example_input_file3233.txt +example_input_file3234.txt +example_input_file3235.txt +example_input_file3236.txt +example_input_file3237.txt +example_input_file3238.txt +example_input_file3239.txt +example_input_file3240.txt +example_input_file3241.txt +example_input_file3242.txt +example_input_file3243.txt +example_input_file3244.txt +example_input_file3245.txt +example_input_file3246.txt +example_input_file3247.txt +example_input_file3248.txt +example_input_file3249.txt +example_input_file3250.txt +example_input_file3251.txt +example_input_file3252.txt +example_input_file3253.txt +example_input_file3254.txt +example_input_file3255.txt +example_input_file3256.txt +example_input_file3257.txt +example_input_file3258.txt +example_input_file3259.txt +example_input_file3260.txt +example_input_file3261.txt +example_input_file3262.txt +example_input_file3263.txt +example_input_file3264.txt +example_input_file3265.txt +example_input_file3266.txt +example_input_file3267.txt +example_input_file3268.txt +example_input_file3269.txt +example_input_file3270.txt +example_input_file3271.txt +example_input_file3272.txt +example_input_file3273.txt +example_input_file3274.txt +example_input_file3275.txt +example_input_file3276.txt +example_input_file3277.txt +example_input_file3278.txt +example_input_file3279.txt +example_input_file3280.txt +example_input_file3281.txt +example_input_file3282.txt +example_input_file3283.txt +example_input_file3284.txt +example_input_file3285.txt +example_input_file3286.txt +example_input_file3287.txt +example_input_file3288.txt +example_input_file3289.txt +example_input_file3290.txt +example_input_file3291.txt +example_input_file3292.txt +example_input_file3293.txt +example_input_file3294.txt +example_input_file3295.txt +example_input_file3296.txt +example_input_file3297.txt +example_input_file3298.txt +example_input_file3299.txt +example_input_file3300.txt +example_input_file3301.txt +example_input_file3302.txt +example_input_file3303.txt +example_input_file3304.txt +example_input_file3305.txt +example_input_file3306.txt +example_input_file3307.txt +example_input_file3308.txt +example_input_file3309.txt +example_input_file3310.txt +example_input_file3311.txt +example_input_file3312.txt +example_input_file3313.txt +example_input_file3314.txt +example_input_file3315.txt +example_input_file3316.txt +example_input_file3317.txt +example_input_file3318.txt +example_input_file3319.txt +example_input_file3320.txt +example_input_file3321.txt +example_input_file3322.txt +example_input_file3323.txt +example_input_file3324.txt +example_input_file3325.txt +example_input_file3326.txt +example_input_file3327.txt +example_input_file3328.txt +example_input_file3329.txt +example_input_file3330.txt +example_input_file3331.txt +example_input_file3332.txt +example_input_file3333.txt +example_input_file3334.txt +example_input_file3335.txt +example_input_file3336.txt +example_input_file3337.txt +example_input_file3338.txt +example_input_file3339.txt +example_input_file3340.txt +example_input_file3341.txt +example_input_file3342.txt +example_input_file3343.txt +example_input_file3344.txt +example_input_file3345.txt +example_input_file3346.txt +example_input_file3347.txt +example_input_file3348.txt +example_input_file3349.txt +example_input_file3350.txt +example_input_file3351.txt +example_input_file3352.txt +example_input_file3353.txt +example_input_file3354.txt +example_input_file3355.txt +example_input_file3356.txt +example_input_file3357.txt +example_input_file3358.txt +example_input_file3359.txt +example_input_file3360.txt +example_input_file3361.txt +example_input_file3362.txt +example_input_file3363.txt +example_input_file3364.txt +example_input_file3365.txt +example_input_file3366.txt +example_input_file3367.txt +example_input_file3368.txt +example_input_file3369.txt +example_input_file3370.txt +example_input_file3371.txt +example_input_file3372.txt +example_input_file3373.txt +example_input_file3374.txt +example_input_file3375.txt +example_input_file3376.txt +example_input_file3377.txt +example_input_file3378.txt +example_input_file3379.txt +example_input_file3380.txt +example_input_file3381.txt +example_input_file3382.txt +example_input_file3383.txt +example_input_file3384.txt +example_input_file3385.txt +example_input_file3386.txt +example_input_file3387.txt +example_input_file3388.txt +example_input_file3389.txt +example_input_file3390.txt +example_input_file3391.txt +example_input_file3392.txt +example_input_file3393.txt +example_input_file3394.txt +example_input_file3395.txt +example_input_file3396.txt +example_input_file3397.txt +example_input_file3398.txt +example_input_file3399.txt +example_input_file3400.txt +example_input_file3401.txt +example_input_file3402.txt +example_input_file3403.txt +example_input_file3404.txt +example_input_file3405.txt +example_input_file3406.txt +example_input_file3407.txt +example_input_file3408.txt +example_input_file3409.txt +example_input_file3410.txt +example_input_file3411.txt +example_input_file3412.txt +example_input_file3413.txt +example_input_file3414.txt +example_input_file3415.txt +example_input_file3416.txt +example_input_file3417.txt +example_input_file3418.txt +example_input_file3419.txt +example_input_file3420.txt +example_input_file3421.txt +example_input_file3422.txt +example_input_file3423.txt +example_input_file3424.txt +example_input_file3425.txt +example_input_file3426.txt +example_input_file3427.txt +example_input_file3428.txt +example_input_file3429.txt +example_input_file3430.txt +example_input_file3431.txt +example_input_file3432.txt +example_input_file3433.txt +example_input_file3434.txt +example_input_file3435.txt +example_input_file3436.txt +example_input_file3437.txt +example_input_file3438.txt +example_input_file3439.txt +example_input_file3440.txt +example_input_file3441.txt +example_input_file3442.txt +example_input_file3443.txt +example_input_file3444.txt +example_input_file3445.txt +example_input_file3446.txt +example_input_file3447.txt +example_input_file3448.txt +example_input_file3449.txt +example_input_file3450.txt +example_input_file3451.txt +example_input_file3452.txt +example_input_file3453.txt +example_input_file3454.txt +example_input_file3455.txt +example_input_file3456.txt +example_input_file3457.txt +example_input_file3458.txt +example_input_file3459.txt +example_input_file3460.txt +example_input_file3461.txt +example_input_file3462.txt +example_input_file3463.txt +example_input_file3464.txt +example_input_file3465.txt +example_input_file3466.txt +example_input_file3467.txt +example_input_file3468.txt +example_input_file3469.txt +example_input_file3470.txt +example_input_file3471.txt +example_input_file3472.txt +example_input_file3473.txt +example_input_file3474.txt +example_input_file3475.txt +example_input_file3476.txt +example_input_file3477.txt +example_input_file3478.txt +example_input_file3479.txt +example_input_file3480.txt +example_input_file3481.txt +example_input_file3482.txt +example_input_file3483.txt +example_input_file3484.txt +example_input_file3485.txt +example_input_file3486.txt +example_input_file3487.txt +example_input_file3488.txt +example_input_file3489.txt +example_input_file3490.txt +example_input_file3491.txt +example_input_file3492.txt +example_input_file3493.txt +example_input_file3494.txt +example_input_file3495.txt +example_input_file3496.txt +example_input_file3497.txt +example_input_file3498.txt +example_input_file3499.txt +example_input_file3500.txt +example_input_file3501.txt +example_input_file3502.txt +example_input_file3503.txt +example_input_file3504.txt +example_input_file3505.txt +example_input_file3506.txt +example_input_file3507.txt +example_input_file3508.txt +example_input_file3509.txt +example_input_file3510.txt +example_input_file3511.txt +example_input_file3512.txt +example_input_file3513.txt +example_input_file3514.txt +example_input_file3515.txt +example_input_file3516.txt +example_input_file3517.txt +example_input_file3518.txt +example_input_file3519.txt +example_input_file3520.txt +example_input_file3521.txt +example_input_file3522.txt +example_input_file3523.txt +example_input_file3524.txt +example_input_file3525.txt +example_input_file3526.txt +example_input_file3527.txt +example_input_file3528.txt +example_input_file3529.txt +example_input_file3530.txt +example_input_file3531.txt +example_input_file3532.txt +example_input_file3533.txt +example_input_file3534.txt +example_input_file3535.txt +example_input_file3536.txt +example_input_file3537.txt +example_input_file3538.txt +example_input_file3539.txt +example_input_file3540.txt +example_input_file3541.txt +example_input_file3542.txt +example_input_file3543.txt +example_input_file3544.txt +example_input_file3545.txt +example_input_file3546.txt +example_input_file3547.txt +example_input_file3548.txt +example_input_file3549.txt +example_input_file3550.txt +example_input_file3551.txt +example_input_file3552.txt +example_input_file3553.txt +example_input_file3554.txt +example_input_file3555.txt +example_input_file3556.txt +example_input_file3557.txt +example_input_file3558.txt +example_input_file3559.txt +example_input_file3560.txt +example_input_file3561.txt +example_input_file3562.txt +example_input_file3563.txt +example_input_file3564.txt +example_input_file3565.txt +example_input_file3566.txt +example_input_file3567.txt +example_input_file3568.txt +example_input_file3569.txt +example_input_file3570.txt +example_input_file3571.txt +example_input_file3572.txt +example_input_file3573.txt +example_input_file3574.txt +example_input_file3575.txt +example_input_file3576.txt +example_input_file3577.txt +example_input_file3578.txt +example_input_file3579.txt +example_input_file3580.txt +example_input_file3581.txt +example_input_file3582.txt +example_input_file3583.txt +example_input_file3584.txt +example_input_file3585.txt +example_input_file3586.txt +example_input_file3587.txt +example_input_file3588.txt +example_input_file3589.txt +example_input_file3590.txt +example_input_file3591.txt +example_input_file3592.txt +example_input_file3593.txt +example_input_file3594.txt +example_input_file3595.txt +example_input_file3596.txt +example_input_file3597.txt +example_input_file3598.txt +example_input_file3599.txt +example_input_file3600.txt +example_input_file3601.txt +example_input_file3602.txt +example_input_file3603.txt +example_input_file3604.txt +example_input_file3605.txt +example_input_file3606.txt +example_input_file3607.txt +example_input_file3608.txt +example_input_file3609.txt +example_input_file3610.txt +example_input_file3611.txt +example_input_file3612.txt +example_input_file3613.txt +example_input_file3614.txt +example_input_file3615.txt +example_input_file3616.txt +example_input_file3617.txt +example_input_file3618.txt +example_input_file3619.txt +example_input_file3620.txt +example_input_file3621.txt +example_input_file3622.txt +example_input_file3623.txt +example_input_file3624.txt +example_input_file3625.txt +example_input_file3626.txt +example_input_file3627.txt +example_input_file3628.txt +example_input_file3629.txt +example_input_file3630.txt +example_input_file3631.txt +example_input_file3632.txt +example_input_file3633.txt +example_input_file3634.txt +example_input_file3635.txt +example_input_file3636.txt +example_input_file3637.txt +example_input_file3638.txt +example_input_file3639.txt +example_input_file3640.txt +example_input_file3641.txt +example_input_file3642.txt +example_input_file3643.txt +example_input_file3644.txt +example_input_file3645.txt +example_input_file3646.txt +example_input_file3647.txt +example_input_file3648.txt +example_input_file3649.txt +example_input_file3650.txt +example_input_file3651.txt +example_input_file3652.txt +example_input_file3653.txt +example_input_file3654.txt +example_input_file3655.txt +example_input_file3656.txt +example_input_file3657.txt +example_input_file3658.txt +example_input_file3659.txt +example_input_file3660.txt +example_input_file3661.txt +example_input_file3662.txt +example_input_file3663.txt +example_input_file3664.txt +example_input_file3665.txt +example_input_file3666.txt +example_input_file3667.txt +example_input_file3668.txt +example_input_file3669.txt +example_input_file3670.txt +example_input_file3671.txt +example_input_file3672.txt +example_input_file3673.txt +example_input_file3674.txt +example_input_file3675.txt +example_input_file3676.txt +example_input_file3677.txt +example_input_file3678.txt +example_input_file3679.txt +example_input_file3680.txt +example_input_file3681.txt +example_input_file3682.txt +example_input_file3683.txt +example_input_file3684.txt +example_input_file3685.txt +example_input_file3686.txt +example_input_file3687.txt +example_input_file3688.txt +example_input_file3689.txt +example_input_file3690.txt +example_input_file3691.txt +example_input_file3692.txt +example_input_file3693.txt +example_input_file3694.txt +example_input_file3695.txt +example_input_file3696.txt +example_input_file3697.txt +example_input_file3698.txt +example_input_file3699.txt +example_input_file3700.txt +example_input_file3701.txt +example_input_file3702.txt +example_input_file3703.txt +example_input_file3704.txt +example_input_file3705.txt +example_input_file3706.txt +example_input_file3707.txt +example_input_file3708.txt +example_input_file3709.txt +example_input_file3710.txt +example_input_file3711.txt +example_input_file3712.txt +example_input_file3713.txt +example_input_file3714.txt +example_input_file3715.txt +example_input_file3716.txt +example_input_file3717.txt +example_input_file3718.txt +example_input_file3719.txt +example_input_file3720.txt +example_input_file3721.txt +example_input_file3722.txt +example_input_file3723.txt +example_input_file3724.txt +example_input_file3725.txt +example_input_file3726.txt +example_input_file3727.txt +example_input_file3728.txt +example_input_file3729.txt +example_input_file3730.txt +example_input_file3731.txt +example_input_file3732.txt +example_input_file3733.txt +example_input_file3734.txt +example_input_file3735.txt +example_input_file3736.txt +example_input_file3737.txt +example_input_file3738.txt +example_input_file3739.txt +example_input_file3740.txt +example_input_file3741.txt +example_input_file3742.txt +example_input_file3743.txt +example_input_file3744.txt +example_input_file3745.txt +example_input_file3746.txt +example_input_file3747.txt +example_input_file3748.txt +example_input_file3749.txt +example_input_file3750.txt +example_input_file3751.txt +example_input_file3752.txt +example_input_file3753.txt +example_input_file3754.txt +example_input_file3755.txt +example_input_file3756.txt +example_input_file3757.txt +example_input_file3758.txt +example_input_file3759.txt +example_input_file3760.txt +example_input_file3761.txt +example_input_file3762.txt +example_input_file3763.txt +example_input_file3764.txt +example_input_file3765.txt +example_input_file3766.txt +example_input_file3767.txt +example_input_file3768.txt +example_input_file3769.txt +example_input_file3770.txt +example_input_file3771.txt +example_input_file3772.txt +example_input_file3773.txt +example_input_file3774.txt +example_input_file3775.txt +example_input_file3776.txt +example_input_file3777.txt +example_input_file3778.txt +example_input_file3779.txt +example_input_file3780.txt +example_input_file3781.txt +example_input_file3782.txt +example_input_file3783.txt +example_input_file3784.txt +example_input_file3785.txt +example_input_file3786.txt +example_input_file3787.txt +example_input_file3788.txt +example_input_file3789.txt +example_input_file3790.txt +example_input_file3791.txt +example_input_file3792.txt +example_input_file3793.txt +example_input_file3794.txt +example_input_file3795.txt +example_input_file3796.txt +example_input_file3797.txt +example_input_file3798.txt +example_input_file3799.txt +example_input_file3800.txt +example_input_file3801.txt +example_input_file3802.txt +example_input_file3803.txt +example_input_file3804.txt +example_input_file3805.txt +example_input_file3806.txt +example_input_file3807.txt +example_input_file3808.txt +example_input_file3809.txt +example_input_file3810.txt +example_input_file3811.txt +example_input_file3812.txt +example_input_file3813.txt +example_input_file3814.txt +example_input_file3815.txt +example_input_file3816.txt +example_input_file3817.txt +example_input_file3818.txt +example_input_file3819.txt +example_input_file3820.txt +example_input_file3821.txt +example_input_file3822.txt +example_input_file3823.txt +example_input_file3824.txt +example_input_file3825.txt +example_input_file3826.txt +example_input_file3827.txt +example_input_file3828.txt +example_input_file3829.txt +example_input_file3830.txt +example_input_file3831.txt +example_input_file3832.txt +example_input_file3833.txt +example_input_file3834.txt +example_input_file3835.txt +example_input_file3836.txt +example_input_file3837.txt +example_input_file3838.txt +example_input_file3839.txt +example_input_file3840.txt +example_input_file3841.txt +example_input_file3842.txt +example_input_file3843.txt +example_input_file3844.txt +example_input_file3845.txt +example_input_file3846.txt +example_input_file3847.txt +example_input_file3848.txt +example_input_file3849.txt +example_input_file3850.txt +example_input_file3851.txt +example_input_file3852.txt +example_input_file3853.txt +example_input_file3854.txt +example_input_file3855.txt +example_input_file3856.txt +example_input_file3857.txt +example_input_file3858.txt +example_input_file3859.txt +example_input_file3860.txt +example_input_file3861.txt +example_input_file3862.txt +example_input_file3863.txt +example_input_file3864.txt +example_input_file3865.txt +example_input_file3866.txt +example_input_file3867.txt +example_input_file3868.txt +example_input_file3869.txt +example_input_file3870.txt +example_input_file3871.txt +example_input_file3872.txt +example_input_file3873.txt +example_input_file3874.txt +example_input_file3875.txt +example_input_file3876.txt +example_input_file3877.txt +example_input_file3878.txt +example_input_file3879.txt +example_input_file3880.txt +example_input_file3881.txt +example_input_file3882.txt +example_input_file3883.txt +example_input_file3884.txt +example_input_file3885.txt +example_input_file3886.txt +example_input_file3887.txt +example_input_file3888.txt +example_input_file3889.txt +example_input_file3890.txt +example_input_file3891.txt +example_input_file3892.txt +example_input_file3893.txt +example_input_file3894.txt +example_input_file3895.txt +example_input_file3896.txt +example_input_file3897.txt +example_input_file3898.txt +example_input_file3899.txt +example_input_file3900.txt +example_input_file3901.txt +example_input_file3902.txt +example_input_file3903.txt +example_input_file3904.txt +example_input_file3905.txt +example_input_file3906.txt +example_input_file3907.txt +example_input_file3908.txt +example_input_file3909.txt +example_input_file3910.txt +example_input_file3911.txt +example_input_file3912.txt +example_input_file3913.txt +example_input_file3914.txt +example_input_file3915.txt +example_input_file3916.txt +example_input_file3917.txt +example_input_file3918.txt +example_input_file3919.txt +example_input_file3920.txt +example_input_file3921.txt +example_input_file3922.txt +example_input_file3923.txt +example_input_file3924.txt +example_input_file3925.txt +example_input_file3926.txt +example_input_file3927.txt +example_input_file3928.txt +example_input_file3929.txt +example_input_file3930.txt +example_input_file3931.txt +example_input_file3932.txt +example_input_file3933.txt +example_input_file3934.txt +example_input_file3935.txt +example_input_file3936.txt +example_input_file3937.txt +example_input_file3938.txt +example_input_file3939.txt +example_input_file3940.txt +example_input_file3941.txt +example_input_file3942.txt +example_input_file3943.txt +example_input_file3944.txt +example_input_file3945.txt +example_input_file3946.txt +example_input_file3947.txt +example_input_file3948.txt +example_input_file3949.txt +example_input_file3950.txt +example_input_file3951.txt +example_input_file3952.txt +example_input_file3953.txt +example_input_file3954.txt +example_input_file3955.txt +example_input_file3956.txt +example_input_file3957.txt +example_input_file3958.txt +example_input_file3959.txt +example_input_file3960.txt +example_input_file3961.txt +example_input_file3962.txt +example_input_file3963.txt +example_input_file3964.txt +example_input_file3965.txt +example_input_file3966.txt +example_input_file3967.txt +example_input_file3968.txt +example_input_file3969.txt +example_input_file3970.txt +example_input_file3971.txt +example_input_file3972.txt +example_input_file3973.txt +example_input_file3974.txt +example_input_file3975.txt +example_input_file3976.txt +example_input_file3977.txt +example_input_file3978.txt +example_input_file3979.txt +example_input_file3980.txt +example_input_file3981.txt +example_input_file3982.txt +example_input_file3983.txt +example_input_file3984.txt +example_input_file3985.txt +example_input_file3986.txt +example_input_file3987.txt +example_input_file3988.txt +example_input_file3989.txt +example_input_file3990.txt +example_input_file3991.txt +example_input_file3992.txt +example_input_file3993.txt +example_input_file3994.txt +example_input_file3995.txt +example_input_file3996.txt +example_input_file3997.txt +example_input_file3998.txt +example_input_file3999.txt +example_input_file4000.txt +example_input_file4001.txt +example_input_file4002.txt +example_input_file4003.txt +example_input_file4004.txt +example_input_file4005.txt +example_input_file4006.txt +example_input_file4007.txt +example_input_file4008.txt +example_input_file4009.txt +example_input_file4010.txt +example_input_file4011.txt +example_input_file4012.txt +example_input_file4013.txt +example_input_file4014.txt +example_input_file4015.txt +example_input_file4016.txt +example_input_file4017.txt +example_input_file4018.txt +example_input_file4019.txt +example_input_file4020.txt +example_input_file4021.txt +example_input_file4022.txt +example_input_file4023.txt +example_input_file4024.txt +example_input_file4025.txt +example_input_file4026.txt +example_input_file4027.txt +example_input_file4028.txt +example_input_file4029.txt +example_input_file4030.txt +example_input_file4031.txt +example_input_file4032.txt +example_input_file4033.txt +example_input_file4034.txt +example_input_file4035.txt +example_input_file4036.txt +example_input_file4037.txt +example_input_file4038.txt +example_input_file4039.txt +example_input_file4040.txt +example_input_file4041.txt +example_input_file4042.txt +example_input_file4043.txt +example_input_file4044.txt +example_input_file4045.txt +example_input_file4046.txt +example_input_file4047.txt +example_input_file4048.txt +example_input_file4049.txt +example_input_file4050.txt +example_input_file4051.txt +example_input_file4052.txt +example_input_file4053.txt +example_input_file4054.txt +example_input_file4055.txt +example_input_file4056.txt +example_input_file4057.txt +example_input_file4058.txt +example_input_file4059.txt +example_input_file4060.txt +example_input_file4061.txt +example_input_file4062.txt +example_input_file4063.txt +example_input_file4064.txt +example_input_file4065.txt +example_input_file4066.txt +example_input_file4067.txt +example_input_file4068.txt +example_input_file4069.txt +example_input_file4070.txt +example_input_file4071.txt +example_input_file4072.txt +example_input_file4073.txt +example_input_file4074.txt +example_input_file4075.txt +example_input_file4076.txt +example_input_file4077.txt +example_input_file4078.txt +example_input_file4079.txt +example_input_file4080.txt +example_input_file4081.txt +example_input_file4082.txt +example_input_file4083.txt +example_input_file4084.txt +example_input_file4085.txt +example_input_file4086.txt +example_input_file4087.txt +example_input_file4088.txt +example_input_file4089.txt +example_input_file4090.txt +example_input_file4091.txt +example_input_file4092.txt +example_input_file4093.txt +example_input_file4094.txt +example_input_file4095.txt +example_input_file4096.txt +example_input_file4097.txt +example_input_file4098.txt +example_input_file4099.txt +example_input_file4100.txt +example_input_file4101.txt +example_input_file4102.txt +example_input_file4103.txt +example_input_file4104.txt +example_input_file4105.txt +example_input_file4106.txt +example_input_file4107.txt +example_input_file4108.txt +example_input_file4109.txt +example_input_file4110.txt +example_input_file4111.txt +example_input_file4112.txt +example_input_file4113.txt +example_input_file4114.txt +example_input_file4115.txt +example_input_file4116.txt +example_input_file4117.txt +example_input_file4118.txt +example_input_file4119.txt +example_input_file4120.txt +example_input_file4121.txt +example_input_file4122.txt +example_input_file4123.txt +example_input_file4124.txt +example_input_file4125.txt +example_input_file4126.txt +example_input_file4127.txt +example_input_file4128.txt +example_input_file4129.txt +example_input_file4130.txt +example_input_file4131.txt +example_input_file4132.txt +example_input_file4133.txt +example_input_file4134.txt +example_input_file4135.txt +example_input_file4136.txt +example_input_file4137.txt +example_input_file4138.txt +example_input_file4139.txt +example_input_file4140.txt +example_input_file4141.txt +example_input_file4142.txt +example_input_file4143.txt +example_input_file4144.txt +example_input_file4145.txt +example_input_file4146.txt +example_input_file4147.txt +example_input_file4148.txt +example_input_file4149.txt +example_input_file4150.txt +example_input_file4151.txt +example_input_file4152.txt +example_input_file4153.txt +example_input_file4154.txt +example_input_file4155.txt +example_input_file4156.txt +example_input_file4157.txt +example_input_file4158.txt +example_input_file4159.txt +example_input_file4160.txt +example_input_file4161.txt +example_input_file4162.txt +example_input_file4163.txt +example_input_file4164.txt +example_input_file4165.txt +example_input_file4166.txt +example_input_file4167.txt +example_input_file4168.txt +example_input_file4169.txt +example_input_file4170.txt +example_input_file4171.txt +example_input_file4172.txt +example_input_file4173.txt +example_input_file4174.txt +example_input_file4175.txt +example_input_file4176.txt +example_input_file4177.txt +example_input_file4178.txt +example_input_file4179.txt +example_input_file4180.txt +example_input_file4181.txt +example_input_file4182.txt +example_input_file4183.txt +example_input_file4184.txt +example_input_file4185.txt +example_input_file4186.txt +example_input_file4187.txt +example_input_file4188.txt +example_input_file4189.txt +example_input_file4190.txt +example_input_file4191.txt +example_input_file4192.txt +example_input_file4193.txt +example_input_file4194.txt +example_input_file4195.txt +example_input_file4196.txt +example_input_file4197.txt +example_input_file4198.txt +example_input_file4199.txt +example_input_file4200.txt +example_input_file4201.txt +example_input_file4202.txt +example_input_file4203.txt +example_input_file4204.txt +example_input_file4205.txt +example_input_file4206.txt +example_input_file4207.txt +example_input_file4208.txt +example_input_file4209.txt +example_input_file4210.txt +example_input_file4211.txt +example_input_file4212.txt +example_input_file4213.txt +example_input_file4214.txt +example_input_file4215.txt +example_input_file4216.txt +example_input_file4217.txt +example_input_file4218.txt +example_input_file4219.txt +example_input_file4220.txt +example_input_file4221.txt +example_input_file4222.txt +example_input_file4223.txt +example_input_file4224.txt +example_input_file4225.txt +example_input_file4226.txt +example_input_file4227.txt +example_input_file4228.txt +example_input_file4229.txt +example_input_file4230.txt +example_input_file4231.txt +example_input_file4232.txt +example_input_file4233.txt +example_input_file4234.txt +example_input_file4235.txt +example_input_file4236.txt +example_input_file4237.txt +example_input_file4238.txt +example_input_file4239.txt +example_input_file4240.txt +example_input_file4241.txt +example_input_file4242.txt +example_input_file4243.txt +example_input_file4244.txt +example_input_file4245.txt +example_input_file4246.txt +example_input_file4247.txt +example_input_file4248.txt +example_input_file4249.txt +example_input_file4250.txt +example_input_file4251.txt +example_input_file4252.txt +example_input_file4253.txt +example_input_file4254.txt +example_input_file4255.txt +example_input_file4256.txt +example_input_file4257.txt +example_input_file4258.txt +example_input_file4259.txt +example_input_file4260.txt +example_input_file4261.txt +example_input_file4262.txt +example_input_file4263.txt +example_input_file4264.txt +example_input_file4265.txt +example_input_file4266.txt +example_input_file4267.txt +example_input_file4268.txt +example_input_file4269.txt +example_input_file4270.txt +example_input_file4271.txt +example_input_file4272.txt +example_input_file4273.txt +example_input_file4274.txt +example_input_file4275.txt +example_input_file4276.txt +example_input_file4277.txt +example_input_file4278.txt +example_input_file4279.txt +example_input_file4280.txt +example_input_file4281.txt +example_input_file4282.txt +example_input_file4283.txt +example_input_file4284.txt +example_input_file4285.txt +example_input_file4286.txt +example_input_file4287.txt +example_input_file4288.txt +example_input_file4289.txt +example_input_file4290.txt +example_input_file4291.txt +example_input_file4292.txt +example_input_file4293.txt +example_input_file4294.txt +example_input_file4295.txt +example_input_file4296.txt +example_input_file4297.txt +example_input_file4298.txt +example_input_file4299.txt +example_input_file4300.txt +example_input_file4301.txt +example_input_file4302.txt +example_input_file4303.txt +example_input_file4304.txt +example_input_file4305.txt +example_input_file4306.txt +example_input_file4307.txt +example_input_file4308.txt +example_input_file4309.txt +example_input_file4310.txt +example_input_file4311.txt +example_input_file4312.txt +example_input_file4313.txt +example_input_file4314.txt +example_input_file4315.txt +example_input_file4316.txt +example_input_file4317.txt +example_input_file4318.txt +example_input_file4319.txt +example_input_file4320.txt +example_input_file4321.txt +example_input_file4322.txt +example_input_file4323.txt +example_input_file4324.txt +example_input_file4325.txt +example_input_file4326.txt +example_input_file4327.txt +example_input_file4328.txt +example_input_file4329.txt +example_input_file4330.txt +example_input_file4331.txt +example_input_file4332.txt +example_input_file4333.txt +example_input_file4334.txt +example_input_file4335.txt +example_input_file4336.txt +example_input_file4337.txt +example_input_file4338.txt +example_input_file4339.txt +example_input_file4340.txt +example_input_file4341.txt +example_input_file4342.txt +example_input_file4343.txt +example_input_file4344.txt +example_input_file4345.txt +example_input_file4346.txt +example_input_file4347.txt +example_input_file4348.txt +example_input_file4349.txt +example_input_file4350.txt +example_input_file4351.txt +example_input_file4352.txt +example_input_file4353.txt +example_input_file4354.txt +example_input_file4355.txt +example_input_file4356.txt +example_input_file4357.txt +example_input_file4358.txt +example_input_file4359.txt +example_input_file4360.txt +example_input_file4361.txt +example_input_file4362.txt +example_input_file4363.txt +example_input_file4364.txt +example_input_file4365.txt +example_input_file4366.txt +example_input_file4367.txt +example_input_file4368.txt +example_input_file4369.txt +example_input_file4370.txt +example_input_file4371.txt +example_input_file4372.txt +example_input_file4373.txt +example_input_file4374.txt +example_input_file4375.txt +example_input_file4376.txt +example_input_file4377.txt +example_input_file4378.txt +example_input_file4379.txt +example_input_file4380.txt +example_input_file4381.txt +example_input_file4382.txt +example_input_file4383.txt +example_input_file4384.txt +example_input_file4385.txt +example_input_file4386.txt +example_input_file4387.txt +example_input_file4388.txt +example_input_file4389.txt +example_input_file4390.txt +example_input_file4391.txt +example_input_file4392.txt +example_input_file4393.txt +example_input_file4394.txt +example_input_file4395.txt +example_input_file4396.txt +example_input_file4397.txt +example_input_file4398.txt +example_input_file4399.txt +example_input_file4400.txt +example_input_file4401.txt +example_input_file4402.txt +example_input_file4403.txt +example_input_file4404.txt +example_input_file4405.txt +example_input_file4406.txt +example_input_file4407.txt +example_input_file4408.txt +example_input_file4409.txt +example_input_file4410.txt +example_input_file4411.txt +example_input_file4412.txt +example_input_file4413.txt +example_input_file4414.txt +example_input_file4415.txt +example_input_file4416.txt +example_input_file4417.txt +example_input_file4418.txt +example_input_file4419.txt +example_input_file4420.txt +example_input_file4421.txt +example_input_file4422.txt +example_input_file4423.txt +example_input_file4424.txt +example_input_file4425.txt +example_input_file4426.txt +example_input_file4427.txt +example_input_file4428.txt +example_input_file4429.txt +example_input_file4430.txt +example_input_file4431.txt +example_input_file4432.txt +example_input_file4433.txt +example_input_file4434.txt +example_input_file4435.txt +example_input_file4436.txt +example_input_file4437.txt +example_input_file4438.txt +example_input_file4439.txt +example_input_file4440.txt +example_input_file4441.txt +example_input_file4442.txt +example_input_file4443.txt +example_input_file4444.txt +example_input_file4445.txt +example_input_file4446.txt +example_input_file4447.txt +example_input_file4448.txt +example_input_file4449.txt +example_input_file4450.txt +example_input_file4451.txt +example_input_file4452.txt +example_input_file4453.txt +example_input_file4454.txt +example_input_file4455.txt +example_input_file4456.txt +example_input_file4457.txt +example_input_file4458.txt +example_input_file4459.txt +example_input_file4460.txt +example_input_file4461.txt +example_input_file4462.txt +example_input_file4463.txt +example_input_file4464.txt +example_input_file4465.txt +example_input_file4466.txt +example_input_file4467.txt +example_input_file4468.txt +example_input_file4469.txt +example_input_file4470.txt +example_input_file4471.txt +example_input_file4472.txt +example_input_file4473.txt +example_input_file4474.txt +example_input_file4475.txt +example_input_file4476.txt +example_input_file4477.txt +example_input_file4478.txt +example_input_file4479.txt +example_input_file4480.txt +example_input_file4481.txt +example_input_file4482.txt +example_input_file4483.txt +example_input_file4484.txt +example_input_file4485.txt +example_input_file4486.txt +example_input_file4487.txt +example_input_file4488.txt +example_input_file4489.txt +example_input_file4490.txt +example_input_file4491.txt +example_input_file4492.txt +example_input_file4493.txt +example_input_file4494.txt +example_input_file4495.txt +example_input_file4496.txt +example_input_file4497.txt +example_input_file4498.txt +example_input_file4499.txt +example_input_file4500.txt +example_input_file4501.txt +example_input_file4502.txt +example_input_file4503.txt +example_input_file4504.txt +example_input_file4505.txt +example_input_file4506.txt +example_input_file4507.txt +example_input_file4508.txt +example_input_file4509.txt +example_input_file4510.txt +example_input_file4511.txt +example_input_file4512.txt +example_input_file4513.txt +example_input_file4514.txt +example_input_file4515.txt +example_input_file4516.txt +example_input_file4517.txt +example_input_file4518.txt +example_input_file4519.txt +example_input_file4520.txt +example_input_file4521.txt +example_input_file4522.txt +example_input_file4523.txt +example_input_file4524.txt +example_input_file4525.txt +example_input_file4526.txt +example_input_file4527.txt +example_input_file4528.txt +example_input_file4529.txt +example_input_file4530.txt +example_input_file4531.txt +example_input_file4532.txt +example_input_file4533.txt +example_input_file4534.txt +example_input_file4535.txt +example_input_file4536.txt +example_input_file4537.txt +example_input_file4538.txt +example_input_file4539.txt +example_input_file4540.txt +example_input_file4541.txt +example_input_file4542.txt +example_input_file4543.txt +example_input_file4544.txt +example_input_file4545.txt +example_input_file4546.txt +example_input_file4547.txt +example_input_file4548.txt +example_input_file4549.txt +example_input_file4550.txt +example_input_file4551.txt +example_input_file4552.txt +example_input_file4553.txt +example_input_file4554.txt +example_input_file4555.txt +example_input_file4556.txt +example_input_file4557.txt +example_input_file4558.txt +example_input_file4559.txt +example_input_file4560.txt +example_input_file4561.txt +example_input_file4562.txt +example_input_file4563.txt +example_input_file4564.txt +example_input_file4565.txt +example_input_file4566.txt +example_input_file4567.txt +example_input_file4568.txt +example_input_file4569.txt +example_input_file4570.txt +example_input_file4571.txt +example_input_file4572.txt +example_input_file4573.txt +example_input_file4574.txt +example_input_file4575.txt +example_input_file4576.txt +example_input_file4577.txt +example_input_file4578.txt +example_input_file4579.txt +example_input_file4580.txt +example_input_file4581.txt +example_input_file4582.txt +example_input_file4583.txt +example_input_file4584.txt +example_input_file4585.txt +example_input_file4586.txt +example_input_file4587.txt +example_input_file4588.txt +example_input_file4589.txt +example_input_file4590.txt +example_input_file4591.txt +example_input_file4592.txt +example_input_file4593.txt +example_input_file4594.txt +example_input_file4595.txt +example_input_file4596.txt +example_input_file4597.txt +example_input_file4598.txt +example_input_file4599.txt +example_input_file4600.txt +example_input_file4601.txt +example_input_file4602.txt +example_input_file4603.txt +example_input_file4604.txt +example_input_file4605.txt +example_input_file4606.txt +example_input_file4607.txt +example_input_file4608.txt +example_input_file4609.txt +example_input_file4610.txt +example_input_file4611.txt +example_input_file4612.txt +example_input_file4613.txt +example_input_file4614.txt +example_input_file4615.txt +example_input_file4616.txt +example_input_file4617.txt +example_input_file4618.txt +example_input_file4619.txt +example_input_file4620.txt +example_input_file4621.txt +example_input_file4622.txt +example_input_file4623.txt +example_input_file4624.txt +example_input_file4625.txt +example_input_file4626.txt +example_input_file4627.txt +example_input_file4628.txt +example_input_file4629.txt +example_input_file4630.txt +example_input_file4631.txt +example_input_file4632.txt +example_input_file4633.txt +example_input_file4634.txt +example_input_file4635.txt +example_input_file4636.txt +example_input_file4637.txt +example_input_file4638.txt +example_input_file4639.txt +example_input_file4640.txt +example_input_file4641.txt +example_input_file4642.txt +example_input_file4643.txt +example_input_file4644.txt +example_input_file4645.txt +example_input_file4646.txt +example_input_file4647.txt +example_input_file4648.txt +example_input_file4649.txt +example_input_file4650.txt +example_input_file4651.txt +example_input_file4652.txt +example_input_file4653.txt +example_input_file4654.txt +example_input_file4655.txt +example_input_file4656.txt +example_input_file4657.txt +example_input_file4658.txt +example_input_file4659.txt +example_input_file4660.txt +example_input_file4661.txt +example_input_file4662.txt +example_input_file4663.txt +example_input_file4664.txt +example_input_file4665.txt +example_input_file4666.txt +example_input_file4667.txt +example_input_file4668.txt +example_input_file4669.txt +example_input_file4670.txt +example_input_file4671.txt +example_input_file4672.txt +example_input_file4673.txt +example_input_file4674.txt +example_input_file4675.txt +example_input_file4676.txt +example_input_file4677.txt +example_input_file4678.txt +example_input_file4679.txt +example_input_file4680.txt +example_input_file4681.txt +example_input_file4682.txt +example_input_file4683.txt +example_input_file4684.txt +example_input_file4685.txt +example_input_file4686.txt +example_input_file4687.txt +example_input_file4688.txt +example_input_file4689.txt +example_input_file4690.txt +example_input_file4691.txt +example_input_file4692.txt +example_input_file4693.txt +example_input_file4694.txt +example_input_file4695.txt +example_input_file4696.txt +example_input_file4697.txt +example_input_file4698.txt +example_input_file4699.txt +example_input_file4700.txt +example_input_file4701.txt +example_input_file4702.txt +example_input_file4703.txt +example_input_file4704.txt +example_input_file4705.txt +example_input_file4706.txt +example_input_file4707.txt +example_input_file4708.txt +example_input_file4709.txt +example_input_file4710.txt +example_input_file4711.txt +example_input_file4712.txt +example_input_file4713.txt +example_input_file4714.txt +example_input_file4715.txt +example_input_file4716.txt +example_input_file4717.txt +example_input_file4718.txt +example_input_file4719.txt +example_input_file4720.txt +example_input_file4721.txt +example_input_file4722.txt +example_input_file4723.txt +example_input_file4724.txt +example_input_file4725.txt +example_input_file4726.txt +example_input_file4727.txt +example_input_file4728.txt +example_input_file4729.txt +example_input_file4730.txt +example_input_file4731.txt +example_input_file4732.txt +example_input_file4733.txt +example_input_file4734.txt +example_input_file4735.txt +example_input_file4736.txt +example_input_file4737.txt +example_input_file4738.txt +example_input_file4739.txt +example_input_file4740.txt +example_input_file4741.txt +example_input_file4742.txt +example_input_file4743.txt +example_input_file4744.txt +example_input_file4745.txt +example_input_file4746.txt +example_input_file4747.txt +example_input_file4748.txt +example_input_file4749.txt +example_input_file4750.txt +example_input_file4751.txt +example_input_file4752.txt +example_input_file4753.txt +example_input_file4754.txt +example_input_file4755.txt +example_input_file4756.txt +example_input_file4757.txt +example_input_file4758.txt +example_input_file4759.txt +example_input_file4760.txt +example_input_file4761.txt +example_input_file4762.txt +example_input_file4763.txt +example_input_file4764.txt +example_input_file4765.txt +example_input_file4766.txt +example_input_file4767.txt +example_input_file4768.txt +example_input_file4769.txt +example_input_file4770.txt +example_input_file4771.txt +example_input_file4772.txt +example_input_file4773.txt +example_input_file4774.txt +example_input_file4775.txt +example_input_file4776.txt +example_input_file4777.txt +example_input_file4778.txt +example_input_file4779.txt +example_input_file4780.txt +example_input_file4781.txt +example_input_file4782.txt +example_input_file4783.txt +example_input_file4784.txt +example_input_file4785.txt +example_input_file4786.txt +example_input_file4787.txt +example_input_file4788.txt +example_input_file4789.txt +example_input_file4790.txt +example_input_file4791.txt +example_input_file4792.txt +example_input_file4793.txt +example_input_file4794.txt +example_input_file4795.txt +example_input_file4796.txt +example_input_file4797.txt +example_input_file4798.txt +example_input_file4799.txt +example_input_file4800.txt +example_input_file4801.txt +example_input_file4802.txt +example_input_file4803.txt +example_input_file4804.txt +example_input_file4805.txt +example_input_file4806.txt +example_input_file4807.txt +example_input_file4808.txt +example_input_file4809.txt +example_input_file4810.txt +example_input_file4811.txt +example_input_file4812.txt +example_input_file4813.txt +example_input_file4814.txt +example_input_file4815.txt +example_input_file4816.txt +example_input_file4817.txt +example_input_file4818.txt +example_input_file4819.txt +example_input_file4820.txt +example_input_file4821.txt +example_input_file4822.txt +example_input_file4823.txt +example_input_file4824.txt +example_input_file4825.txt +example_input_file4826.txt +example_input_file4827.txt +example_input_file4828.txt +example_input_file4829.txt +example_input_file4830.txt +example_input_file4831.txt +example_input_file4832.txt +example_input_file4833.txt +example_input_file4834.txt +example_input_file4835.txt +example_input_file4836.txt +example_input_file4837.txt +example_input_file4838.txt +example_input_file4839.txt +example_input_file4840.txt +example_input_file4841.txt +example_input_file4842.txt +example_input_file4843.txt +example_input_file4844.txt +example_input_file4845.txt +example_input_file4846.txt +example_input_file4847.txt +example_input_file4848.txt +example_input_file4849.txt +example_input_file4850.txt +example_input_file4851.txt +example_input_file4852.txt +example_input_file4853.txt +example_input_file4854.txt +example_input_file4855.txt +example_input_file4856.txt +example_input_file4857.txt +example_input_file4858.txt +example_input_file4859.txt +example_input_file4860.txt +example_input_file4861.txt +example_input_file4862.txt +example_input_file4863.txt +example_input_file4864.txt +example_input_file4865.txt +example_input_file4866.txt +example_input_file4867.txt +example_input_file4868.txt +example_input_file4869.txt +example_input_file4870.txt +example_input_file4871.txt +example_input_file4872.txt +example_input_file4873.txt +example_input_file4874.txt +example_input_file4875.txt +example_input_file4876.txt +example_input_file4877.txt +example_input_file4878.txt +example_input_file4879.txt +example_input_file4880.txt +example_input_file4881.txt +example_input_file4882.txt +example_input_file4883.txt +example_input_file4884.txt +example_input_file4885.txt +example_input_file4886.txt +example_input_file4887.txt +example_input_file4888.txt +example_input_file4889.txt +example_input_file4890.txt +example_input_file4891.txt +example_input_file4892.txt +example_input_file4893.txt +example_input_file4894.txt +example_input_file4895.txt +example_input_file4896.txt +example_input_file4897.txt +example_input_file4898.txt +example_input_file4899.txt +example_input_file4900.txt +example_input_file4901.txt +example_input_file4902.txt +example_input_file4903.txt +example_input_file4904.txt +example_input_file4905.txt +example_input_file4906.txt +example_input_file4907.txt +example_input_file4908.txt +example_input_file4909.txt +example_input_file4910.txt +example_input_file4911.txt +example_input_file4912.txt +example_input_file4913.txt +example_input_file4914.txt +example_input_file4915.txt +example_input_file4916.txt +example_input_file4917.txt +example_input_file4918.txt +example_input_file4919.txt +example_input_file4920.txt +example_input_file4921.txt +example_input_file4922.txt +example_input_file4923.txt +example_input_file4924.txt +example_input_file4925.txt +example_input_file4926.txt +example_input_file4927.txt +example_input_file4928.txt +example_input_file4929.txt +example_input_file4930.txt +example_input_file4931.txt +example_input_file4932.txt +example_input_file4933.txt +example_input_file4934.txt +example_input_file4935.txt +example_input_file4936.txt +example_input_file4937.txt +example_input_file4938.txt +example_input_file4939.txt +example_input_file4940.txt +example_input_file4941.txt +example_input_file4942.txt +example_input_file4943.txt +example_input_file4944.txt +example_input_file4945.txt +example_input_file4946.txt +example_input_file4947.txt +example_input_file4948.txt +example_input_file4949.txt +example_input_file4950.txt +example_input_file4951.txt +example_input_file4952.txt +example_input_file4953.txt +example_input_file4954.txt +example_input_file4955.txt +example_input_file4956.txt +example_input_file4957.txt +example_input_file4958.txt +example_input_file4959.txt +example_input_file4960.txt +example_input_file4961.txt +example_input_file4962.txt +example_input_file4963.txt +example_input_file4964.txt +example_input_file4965.txt +example_input_file4966.txt +example_input_file4967.txt +example_input_file4968.txt +example_input_file4969.txt +example_input_file4970.txt +example_input_file4971.txt +example_input_file4972.txt +example_input_file4973.txt +example_input_file4974.txt +example_input_file4975.txt +example_input_file4976.txt +example_input_file4977.txt +example_input_file4978.txt +example_input_file4979.txt +example_input_file4980.txt +example_input_file4981.txt +example_input_file4982.txt +example_input_file4983.txt +example_input_file4984.txt +example_input_file4985.txt +example_input_file4986.txt +example_input_file4987.txt +example_input_file4988.txt +example_input_file4989.txt +example_input_file4990.txt +example_input_file4991.txt +example_input_file4992.txt +example_input_file4993.txt +example_input_file4994.txt +example_input_file4995.txt +example_input_file4996.txt +example_input_file4997.txt +example_input_file4998.txt +example_input_file4999.txt +example_input_file5000.txt +example_input_file5001.txt +example_input_file5002.txt +example_input_file5003.txt +example_input_file5004.txt +example_input_file5005.txt +example_input_file5006.txt +example_input_file5007.txt +example_input_file5008.txt +example_input_file5009.txt +example_input_file5010.txt +example_input_file5011.txt +example_input_file5012.txt +example_input_file5013.txt +example_input_file5014.txt +example_input_file5015.txt +example_input_file5016.txt +example_input_file5017.txt +example_input_file5018.txt +example_input_file5019.txt +example_input_file5020.txt +example_input_file5021.txt +example_input_file5022.txt +example_input_file5023.txt +example_input_file5024.txt +example_input_file5025.txt +example_input_file5026.txt +example_input_file5027.txt +example_input_file5028.txt +example_input_file5029.txt +example_input_file5030.txt +example_input_file5031.txt +example_input_file5032.txt +example_input_file5033.txt +example_input_file5034.txt +example_input_file5035.txt +example_input_file5036.txt +example_input_file5037.txt +example_input_file5038.txt +example_input_file5039.txt +example_input_file5040.txt +example_input_file5041.txt +example_input_file5042.txt +example_input_file5043.txt +example_input_file5044.txt +example_input_file5045.txt +example_input_file5046.txt +example_input_file5047.txt +example_input_file5048.txt +example_input_file5049.txt +example_input_file5050.txt +example_input_file5051.txt +example_input_file5052.txt +example_input_file5053.txt +example_input_file5054.txt +example_input_file5055.txt +example_input_file5056.txt +example_input_file5057.txt +example_input_file5058.txt +example_input_file5059.txt +example_input_file5060.txt +example_input_file5061.txt +example_input_file5062.txt +example_input_file5063.txt +example_input_file5064.txt +example_input_file5065.txt +example_input_file5066.txt +example_input_file5067.txt +example_input_file5068.txt +example_input_file5069.txt +example_input_file5070.txt +example_input_file5071.txt +example_input_file5072.txt +example_input_file5073.txt +example_input_file5074.txt +example_input_file5075.txt +example_input_file5076.txt +example_input_file5077.txt +example_input_file5078.txt +example_input_file5079.txt +example_input_file5080.txt +example_input_file5081.txt +example_input_file5082.txt +example_input_file5083.txt +example_input_file5084.txt +example_input_file5085.txt +example_input_file5086.txt +example_input_file5087.txt +example_input_file5088.txt +example_input_file5089.txt +example_input_file5090.txt +example_input_file5091.txt +example_input_file5092.txt +example_input_file5093.txt +example_input_file5094.txt +example_input_file5095.txt +example_input_file5096.txt +example_input_file5097.txt +example_input_file5098.txt +example_input_file5099.txt +example_input_file5100.txt +example_input_file5101.txt +example_input_file5102.txt +example_input_file5103.txt +example_input_file5104.txt +example_input_file5105.txt +example_input_file5106.txt +example_input_file5107.txt +example_input_file5108.txt +example_input_file5109.txt +example_input_file5110.txt +example_input_file5111.txt +example_input_file5112.txt +example_input_file5113.txt +example_input_file5114.txt +example_input_file5115.txt +example_input_file5116.txt +example_input_file5117.txt +example_input_file5118.txt +example_input_file5119.txt +example_input_file5120.txt +example_input_file5121.txt +example_input_file5122.txt +example_input_file5123.txt +example_input_file5124.txt +example_input_file5125.txt +example_input_file5126.txt +example_input_file5127.txt +example_input_file5128.txt +example_input_file5129.txt +example_input_file5130.txt +example_input_file5131.txt +example_input_file5132.txt +example_input_file5133.txt +example_input_file5134.txt +example_input_file5135.txt +example_input_file5136.txt +example_input_file5137.txt +example_input_file5138.txt +example_input_file5139.txt +example_input_file5140.txt +example_input_file5141.txt +example_input_file5142.txt +example_input_file5143.txt +example_input_file5144.txt +example_input_file5145.txt +example_input_file5146.txt +example_input_file5147.txt +example_input_file5148.txt +example_input_file5149.txt +example_input_file5150.txt +example_input_file5151.txt +example_input_file5152.txt +example_input_file5153.txt +example_input_file5154.txt +example_input_file5155.txt +example_input_file5156.txt +example_input_file5157.txt +example_input_file5158.txt +example_input_file5159.txt +example_input_file5160.txt +example_input_file5161.txt +example_input_file5162.txt +example_input_file5163.txt +example_input_file5164.txt +example_input_file5165.txt +example_input_file5166.txt +example_input_file5167.txt +example_input_file5168.txt +example_input_file5169.txt +example_input_file5170.txt +example_input_file5171.txt +example_input_file5172.txt +example_input_file5173.txt +example_input_file5174.txt +example_input_file5175.txt +example_input_file5176.txt +example_input_file5177.txt +example_input_file5178.txt +example_input_file5179.txt +example_input_file5180.txt +example_input_file5181.txt +example_input_file5182.txt +example_input_file5183.txt +example_input_file5184.txt +example_input_file5185.txt +example_input_file5186.txt +example_input_file5187.txt +example_input_file5188.txt +example_input_file5189.txt +example_input_file5190.txt +example_input_file5191.txt +example_input_file5192.txt +example_input_file5193.txt +example_input_file5194.txt +example_input_file5195.txt +example_input_file5196.txt +example_input_file5197.txt +example_input_file5198.txt +example_input_file5199.txt +example_input_file5200.txt +example_input_file5201.txt +example_input_file5202.txt +example_input_file5203.txt +example_input_file5204.txt +example_input_file5205.txt +example_input_file5206.txt +example_input_file5207.txt +example_input_file5208.txt +example_input_file5209.txt +example_input_file5210.txt +example_input_file5211.txt +example_input_file5212.txt +example_input_file5213.txt +example_input_file5214.txt +example_input_file5215.txt +example_input_file5216.txt +example_input_file5217.txt +example_input_file5218.txt +example_input_file5219.txt +example_input_file5220.txt +example_input_file5221.txt +example_input_file5222.txt +example_input_file5223.txt +example_input_file5224.txt +example_input_file5225.txt +example_input_file5226.txt +example_input_file5227.txt +example_input_file5228.txt +example_input_file5229.txt +example_input_file5230.txt +example_input_file5231.txt +example_input_file5232.txt +example_input_file5233.txt +example_input_file5234.txt +example_input_file5235.txt +example_input_file5236.txt +example_input_file5237.txt +example_input_file5238.txt +example_input_file5239.txt +example_input_file5240.txt +example_input_file5241.txt +example_input_file5242.txt +example_input_file5243.txt +example_input_file5244.txt +example_input_file5245.txt +example_input_file5246.txt +example_input_file5247.txt +example_input_file5248.txt +example_input_file5249.txt +example_input_file5250.txt +example_input_file5251.txt +example_input_file5252.txt +example_input_file5253.txt +example_input_file5254.txt +example_input_file5255.txt +example_input_file5256.txt +example_input_file5257.txt +example_input_file5258.txt +example_input_file5259.txt +example_input_file5260.txt +example_input_file5261.txt +example_input_file5262.txt +example_input_file5263.txt +example_input_file5264.txt +example_input_file5265.txt +example_input_file5266.txt +example_input_file5267.txt +example_input_file5268.txt +example_input_file5269.txt +example_input_file5270.txt +example_input_file5271.txt +example_input_file5272.txt +example_input_file5273.txt +example_input_file5274.txt +example_input_file5275.txt +example_input_file5276.txt +example_input_file5277.txt +example_input_file5278.txt +example_input_file5279.txt +example_input_file5280.txt +example_input_file5281.txt +example_input_file5282.txt +example_input_file5283.txt +example_input_file5284.txt +example_input_file5285.txt +example_input_file5286.txt +example_input_file5287.txt +example_input_file5288.txt +example_input_file5289.txt +example_input_file5290.txt +example_input_file5291.txt +example_input_file5292.txt +example_input_file5293.txt +example_input_file5294.txt +example_input_file5295.txt +example_input_file5296.txt +example_input_file5297.txt +example_input_file5298.txt +example_input_file5299.txt +example_input_file5300.txt +example_input_file5301.txt +example_input_file5302.txt +example_input_file5303.txt +example_input_file5304.txt +example_input_file5305.txt +example_input_file5306.txt +example_input_file5307.txt +example_input_file5308.txt +example_input_file5309.txt +example_input_file5310.txt +example_input_file5311.txt +example_input_file5312.txt +example_input_file5313.txt +example_input_file5314.txt +example_input_file5315.txt +example_input_file5316.txt +example_input_file5317.txt +example_input_file5318.txt +example_input_file5319.txt +example_input_file5320.txt +example_input_file5321.txt +example_input_file5322.txt +example_input_file5323.txt +example_input_file5324.txt +example_input_file5325.txt +example_input_file5326.txt +example_input_file5327.txt +example_input_file5328.txt +example_input_file5329.txt +example_input_file5330.txt +example_input_file5331.txt +example_input_file5332.txt +example_input_file5333.txt +example_input_file5334.txt +example_input_file5335.txt +example_input_file5336.txt +example_input_file5337.txt +example_input_file5338.txt +example_input_file5339.txt +example_input_file5340.txt +example_input_file5341.txt +example_input_file5342.txt +example_input_file5343.txt +example_input_file5344.txt +example_input_file5345.txt +example_input_file5346.txt +example_input_file5347.txt +example_input_file5348.txt +example_input_file5349.txt +example_input_file5350.txt +example_input_file5351.txt +example_input_file5352.txt +example_input_file5353.txt +example_input_file5354.txt +example_input_file5355.txt +example_input_file5356.txt +example_input_file5357.txt +example_input_file5358.txt +example_input_file5359.txt +example_input_file5360.txt +example_input_file5361.txt +example_input_file5362.txt +example_input_file5363.txt +example_input_file5364.txt +example_input_file5365.txt +example_input_file5366.txt +example_input_file5367.txt +example_input_file5368.txt +example_input_file5369.txt +example_input_file5370.txt +example_input_file5371.txt +example_input_file5372.txt +example_input_file5373.txt +example_input_file5374.txt +example_input_file5375.txt +example_input_file5376.txt +example_input_file5377.txt +example_input_file5378.txt +example_input_file5379.txt +example_input_file5380.txt +example_input_file5381.txt +example_input_file5382.txt +example_input_file5383.txt +example_input_file5384.txt +example_input_file5385.txt +example_input_file5386.txt +example_input_file5387.txt +example_input_file5388.txt +example_input_file5389.txt +example_input_file5390.txt +example_input_file5391.txt +example_input_file5392.txt +example_input_file5393.txt +example_input_file5394.txt +example_input_file5395.txt +example_input_file5396.txt +example_input_file5397.txt +example_input_file5398.txt +example_input_file5399.txt +example_input_file5400.txt +example_input_file5401.txt +example_input_file5402.txt +example_input_file5403.txt +example_input_file5404.txt +example_input_file5405.txt +example_input_file5406.txt +example_input_file5407.txt +example_input_file5408.txt +example_input_file5409.txt +example_input_file5410.txt +example_input_file5411.txt +example_input_file5412.txt +example_input_file5413.txt +example_input_file5414.txt +example_input_file5415.txt +example_input_file5416.txt +example_input_file5417.txt +example_input_file5418.txt +example_input_file5419.txt +example_input_file5420.txt +example_input_file5421.txt +example_input_file5422.txt +example_input_file5423.txt +example_input_file5424.txt +example_input_file5425.txt +example_input_file5426.txt +example_input_file5427.txt +example_input_file5428.txt +example_input_file5429.txt +example_input_file5430.txt +example_input_file5431.txt +example_input_file5432.txt +example_input_file5433.txt +example_input_file5434.txt +example_input_file5435.txt +example_input_file5436.txt +example_input_file5437.txt +example_input_file5438.txt +example_input_file5439.txt +example_input_file5440.txt +example_input_file5441.txt +example_input_file5442.txt +example_input_file5443.txt +example_input_file5444.txt +example_input_file5445.txt +example_input_file5446.txt +example_input_file5447.txt +example_input_file5448.txt +example_input_file5449.txt +example_input_file5450.txt +example_input_file5451.txt +example_input_file5452.txt +example_input_file5453.txt +example_input_file5454.txt +example_input_file5455.txt +example_input_file5456.txt +example_input_file5457.txt +example_input_file5458.txt +example_input_file5459.txt +example_input_file5460.txt +example_input_file5461.txt +example_input_file5462.txt +example_input_file5463.txt +example_input_file5464.txt +example_input_file5465.txt +example_input_file5466.txt +example_input_file5467.txt +example_input_file5468.txt +example_input_file5469.txt +example_input_file5470.txt +example_input_file5471.txt +example_input_file5472.txt +example_input_file5473.txt +example_input_file5474.txt +example_input_file5475.txt +example_input_file5476.txt +example_input_file5477.txt +example_input_file5478.txt +example_input_file5479.txt +example_input_file5480.txt +example_input_file5481.txt +example_input_file5482.txt +example_input_file5483.txt +example_input_file5484.txt +example_input_file5485.txt +example_input_file5486.txt +example_input_file5487.txt +example_input_file5488.txt +example_input_file5489.txt +example_input_file5490.txt +example_input_file5491.txt +example_input_file5492.txt +example_input_file5493.txt +example_input_file5494.txt +example_input_file5495.txt +example_input_file5496.txt +example_input_file5497.txt +example_input_file5498.txt +example_input_file5499.txt +example_input_file5500.txt +example_input_file5501.txt +example_input_file5502.txt +example_input_file5503.txt +example_input_file5504.txt +example_input_file5505.txt +example_input_file5506.txt +example_input_file5507.txt +example_input_file5508.txt +example_input_file5509.txt +example_input_file5510.txt +example_input_file5511.txt +example_input_file5512.txt +example_input_file5513.txt +example_input_file5514.txt +example_input_file5515.txt +example_input_file5516.txt +example_input_file5517.txt +example_input_file5518.txt +example_input_file5519.txt +example_input_file5520.txt +example_input_file5521.txt +example_input_file5522.txt +example_input_file5523.txt +example_input_file5524.txt +example_input_file5525.txt +example_input_file5526.txt +example_input_file5527.txt +example_input_file5528.txt +example_input_file5529.txt +example_input_file5530.txt +example_input_file5531.txt +example_input_file5532.txt +example_input_file5533.txt +example_input_file5534.txt +example_input_file5535.txt +example_input_file5536.txt +example_input_file5537.txt +example_input_file5538.txt +example_input_file5539.txt +example_input_file5540.txt +example_input_file5541.txt +example_input_file5542.txt +example_input_file5543.txt +example_input_file5544.txt +example_input_file5545.txt +example_input_file5546.txt +example_input_file5547.txt +example_input_file5548.txt +example_input_file5549.txt +example_input_file5550.txt +example_input_file5551.txt +example_input_file5552.txt +example_input_file5553.txt +example_input_file5554.txt +example_input_file5555.txt +example_input_file5556.txt +example_input_file5557.txt +example_input_file5558.txt +example_input_file5559.txt +example_input_file5560.txt +example_input_file5561.txt +example_input_file5562.txt +example_input_file5563.txt +example_input_file5564.txt +example_input_file5565.txt +example_input_file5566.txt +example_input_file5567.txt +example_input_file5568.txt +example_input_file5569.txt +example_input_file5570.txt +example_input_file5571.txt +example_input_file5572.txt +example_input_file5573.txt +example_input_file5574.txt +example_input_file5575.txt +example_input_file5576.txt +example_input_file5577.txt +example_input_file5578.txt +example_input_file5579.txt +example_input_file5580.txt +example_input_file5581.txt +example_input_file5582.txt +example_input_file5583.txt +example_input_file5584.txt +example_input_file5585.txt +example_input_file5586.txt +example_input_file5587.txt +example_input_file5588.txt +example_input_file5589.txt +example_input_file5590.txt +example_input_file5591.txt +example_input_file5592.txt +example_input_file5593.txt +example_input_file5594.txt +example_input_file5595.txt +example_input_file5596.txt +example_input_file5597.txt +example_input_file5598.txt +example_input_file5599.txt +example_input_file5600.txt +example_input_file5601.txt +example_input_file5602.txt +example_input_file5603.txt +example_input_file5604.txt +example_input_file5605.txt +example_input_file5606.txt +example_input_file5607.txt +example_input_file5608.txt +example_input_file5609.txt +example_input_file5610.txt +example_input_file5611.txt +example_input_file5612.txt +example_input_file5613.txt +example_input_file5614.txt +example_input_file5615.txt +example_input_file5616.txt +example_input_file5617.txt +example_input_file5618.txt +example_input_file5619.txt +example_input_file5620.txt +example_input_file5621.txt +example_input_file5622.txt +example_input_file5623.txt +example_input_file5624.txt +example_input_file5625.txt +example_input_file5626.txt +example_input_file5627.txt +example_input_file5628.txt +example_input_file5629.txt +example_input_file5630.txt +example_input_file5631.txt +example_input_file5632.txt +example_input_file5633.txt +example_input_file5634.txt +example_input_file5635.txt +example_input_file5636.txt +example_input_file5637.txt +example_input_file5638.txt +example_input_file5639.txt +example_input_file5640.txt +example_input_file5641.txt +example_input_file5642.txt +example_input_file5643.txt +example_input_file5644.txt +example_input_file5645.txt +example_input_file5646.txt +example_input_file5647.txt +example_input_file5648.txt +example_input_file5649.txt +example_input_file5650.txt +example_input_file5651.txt +example_input_file5652.txt +example_input_file5653.txt +example_input_file5654.txt +example_input_file5655.txt +example_input_file5656.txt +example_input_file5657.txt +example_input_file5658.txt +example_input_file5659.txt +example_input_file5660.txt +example_input_file5661.txt +example_input_file5662.txt +example_input_file5663.txt +example_input_file5664.txt +example_input_file5665.txt +example_input_file5666.txt +example_input_file5667.txt +example_input_file5668.txt +example_input_file5669.txt +example_input_file5670.txt +example_input_file5671.txt +example_input_file5672.txt +example_input_file5673.txt +example_input_file5674.txt +example_input_file5675.txt +example_input_file5676.txt +example_input_file5677.txt +example_input_file5678.txt +example_input_file5679.txt +example_input_file5680.txt +example_input_file5681.txt +example_input_file5682.txt +example_input_file5683.txt +example_input_file5684.txt +example_input_file5685.txt +example_input_file5686.txt +example_input_file5687.txt +example_input_file5688.txt +example_input_file5689.txt +example_input_file5690.txt +example_input_file5691.txt +example_input_file5692.txt +example_input_file5693.txt +example_input_file5694.txt +example_input_file5695.txt +example_input_file5696.txt +example_input_file5697.txt +example_input_file5698.txt +example_input_file5699.txt +example_input_file5700.txt +example_input_file5701.txt +example_input_file5702.txt +example_input_file5703.txt +example_input_file5704.txt +example_input_file5705.txt +example_input_file5706.txt +example_input_file5707.txt +example_input_file5708.txt +example_input_file5709.txt +example_input_file5710.txt +example_input_file5711.txt +example_input_file5712.txt +example_input_file5713.txt +example_input_file5714.txt +example_input_file5715.txt +example_input_file5716.txt +example_input_file5717.txt +example_input_file5718.txt +example_input_file5719.txt +example_input_file5720.txt +example_input_file5721.txt +example_input_file5722.txt +example_input_file5723.txt +example_input_file5724.txt +example_input_file5725.txt +example_input_file5726.txt +example_input_file5727.txt +example_input_file5728.txt +example_input_file5729.txt +example_input_file5730.txt +example_input_file5731.txt +example_input_file5732.txt +example_input_file5733.txt +example_input_file5734.txt +example_input_file5735.txt +example_input_file5736.txt +example_input_file5737.txt +example_input_file5738.txt +example_input_file5739.txt +example_input_file5740.txt +example_input_file5741.txt +example_input_file5742.txt +example_input_file5743.txt +example_input_file5744.txt +example_input_file5745.txt +example_input_file5746.txt +example_input_file5747.txt +example_input_file5748.txt +example_input_file5749.txt +example_input_file5750.txt +example_input_file5751.txt +example_input_file5752.txt +example_input_file5753.txt +example_input_file5754.txt +example_input_file5755.txt +example_input_file5756.txt +example_input_file5757.txt +example_input_file5758.txt +example_input_file5759.txt +example_input_file5760.txt +example_input_file5761.txt +example_input_file5762.txt +example_input_file5763.txt +example_input_file5764.txt +example_input_file5765.txt +example_input_file5766.txt +example_input_file5767.txt +example_input_file5768.txt +example_input_file5769.txt +example_input_file5770.txt +example_input_file5771.txt +example_input_file5772.txt +example_input_file5773.txt +example_input_file5774.txt +example_input_file5775.txt +example_input_file5776.txt +example_input_file5777.txt +example_input_file5778.txt +example_input_file5779.txt +example_input_file5780.txt +example_input_file5781.txt +example_input_file5782.txt +example_input_file5783.txt +example_input_file5784.txt +example_input_file5785.txt +example_input_file5786.txt +example_input_file5787.txt +example_input_file5788.txt +example_input_file5789.txt +example_input_file5790.txt +example_input_file5791.txt +example_input_file5792.txt +example_input_file5793.txt +example_input_file5794.txt +example_input_file5795.txt +example_input_file5796.txt +example_input_file5797.txt +example_input_file5798.txt +example_input_file5799.txt +example_input_file5800.txt +example_input_file5801.txt +example_input_file5802.txt +example_input_file5803.txt +example_input_file5804.txt +example_input_file5805.txt +example_input_file5806.txt +example_input_file5807.txt +example_input_file5808.txt +example_input_file5809.txt +example_input_file5810.txt +example_input_file5811.txt +example_input_file5812.txt +example_input_file5813.txt +example_input_file5814.txt +example_input_file5815.txt +example_input_file5816.txt +example_input_file5817.txt +example_input_file5818.txt +example_input_file5819.txt +example_input_file5820.txt +example_input_file5821.txt +example_input_file5822.txt +example_input_file5823.txt +example_input_file5824.txt +example_input_file5825.txt +example_input_file5826.txt +example_input_file5827.txt +example_input_file5828.txt +example_input_file5829.txt +example_input_file5830.txt +example_input_file5831.txt +example_input_file5832.txt +example_input_file5833.txt +example_input_file5834.txt +example_input_file5835.txt +example_input_file5836.txt +example_input_file5837.txt +example_input_file5838.txt +example_input_file5839.txt +example_input_file5840.txt +example_input_file5841.txt +example_input_file5842.txt +example_input_file5843.txt +example_input_file5844.txt +example_input_file5845.txt +example_input_file5846.txt +example_input_file5847.txt +example_input_file5848.txt +example_input_file5849.txt +example_input_file5850.txt +example_input_file5851.txt +example_input_file5852.txt +example_input_file5853.txt +example_input_file5854.txt +example_input_file5855.txt +example_input_file5856.txt +example_input_file5857.txt +example_input_file5858.txt +example_input_file5859.txt +example_input_file5860.txt +example_input_file5861.txt +example_input_file5862.txt +example_input_file5863.txt +example_input_file5864.txt +example_input_file5865.txt +example_input_file5866.txt +example_input_file5867.txt +example_input_file5868.txt +example_input_file5869.txt +example_input_file5870.txt +example_input_file5871.txt +example_input_file5872.txt +example_input_file5873.txt +example_input_file5874.txt +example_input_file5875.txt +example_input_file5876.txt +example_input_file5877.txt +example_input_file5878.txt +example_input_file5879.txt +example_input_file5880.txt +example_input_file5881.txt +example_input_file5882.txt +example_input_file5883.txt +example_input_file5884.txt +example_input_file5885.txt +example_input_file5886.txt +example_input_file5887.txt +example_input_file5888.txt +example_input_file5889.txt +example_input_file5890.txt +example_input_file5891.txt +example_input_file5892.txt +example_input_file5893.txt +example_input_file5894.txt +example_input_file5895.txt +example_input_file5896.txt +example_input_file5897.txt +example_input_file5898.txt +example_input_file5899.txt +example_input_file5900.txt +example_input_file5901.txt +example_input_file5902.txt +example_input_file5903.txt +example_input_file5904.txt +example_input_file5905.txt +example_input_file5906.txt +example_input_file5907.txt +example_input_file5908.txt +example_input_file5909.txt +example_input_file5910.txt +example_input_file5911.txt +example_input_file5912.txt +example_input_file5913.txt +example_input_file5914.txt +example_input_file5915.txt +example_input_file5916.txt +example_input_file5917.txt +example_input_file5918.txt +example_input_file5919.txt +example_input_file5920.txt +example_input_file5921.txt +example_input_file5922.txt +example_input_file5923.txt +example_input_file5924.txt +example_input_file5925.txt +example_input_file5926.txt +example_input_file5927.txt +example_input_file5928.txt +example_input_file5929.txt +example_input_file5930.txt +example_input_file5931.txt +example_input_file5932.txt +example_input_file5933.txt +example_input_file5934.txt +example_input_file5935.txt +example_input_file5936.txt +example_input_file5937.txt +example_input_file5938.txt +example_input_file5939.txt +example_input_file5940.txt +example_input_file5941.txt +example_input_file5942.txt +example_input_file5943.txt +example_input_file5944.txt +example_input_file5945.txt +example_input_file5946.txt +example_input_file5947.txt +example_input_file5948.txt +example_input_file5949.txt +example_input_file5950.txt +example_input_file5951.txt +example_input_file5952.txt +example_input_file5953.txt +example_input_file5954.txt +example_input_file5955.txt +example_input_file5956.txt +example_input_file5957.txt +example_input_file5958.txt +example_input_file5959.txt +example_input_file5960.txt +example_input_file5961.txt +example_input_file5962.txt +example_input_file5963.txt +example_input_file5964.txt +example_input_file5965.txt +example_input_file5966.txt +example_input_file5967.txt +example_input_file5968.txt +example_input_file5969.txt +example_input_file5970.txt +example_input_file5971.txt +example_input_file5972.txt +example_input_file5973.txt +example_input_file5974.txt +example_input_file5975.txt +example_input_file5976.txt +example_input_file5977.txt +example_input_file5978.txt +example_input_file5979.txt +example_input_file5980.txt +example_input_file5981.txt +example_input_file5982.txt +example_input_file5983.txt +example_input_file5984.txt +example_input_file5985.txt +example_input_file5986.txt +example_input_file5987.txt +example_input_file5988.txt +example_input_file5989.txt +example_input_file5990.txt +example_input_file5991.txt +example_input_file5992.txt +example_input_file5993.txt +example_input_file5994.txt +example_input_file5995.txt +example_input_file5996.txt +example_input_file5997.txt +example_input_file5998.txt +example_input_file5999.txt +example_input_file6000.txt +example_input_file6001.txt +example_input_file6002.txt +example_input_file6003.txt +example_input_file6004.txt +example_input_file6005.txt +example_input_file6006.txt +example_input_file6007.txt +example_input_file6008.txt +example_input_file6009.txt +example_input_file6010.txt +example_input_file6011.txt +example_input_file6012.txt +example_input_file6013.txt +example_input_file6014.txt +example_input_file6015.txt +example_input_file6016.txt +example_input_file6017.txt +example_input_file6018.txt +example_input_file6019.txt +example_input_file6020.txt +example_input_file6021.txt +example_input_file6022.txt +example_input_file6023.txt +example_input_file6024.txt +example_input_file6025.txt +example_input_file6026.txt +example_input_file6027.txt +example_input_file6028.txt +example_input_file6029.txt +example_input_file6030.txt +example_input_file6031.txt +example_input_file6032.txt +example_input_file6033.txt +example_input_file6034.txt +example_input_file6035.txt +example_input_file6036.txt +example_input_file6037.txt +example_input_file6038.txt +example_input_file6039.txt +example_input_file6040.txt +example_input_file6041.txt +example_input_file6042.txt +example_input_file6043.txt +example_input_file6044.txt +example_input_file6045.txt +example_input_file6046.txt +example_input_file6047.txt +example_input_file6048.txt +example_input_file6049.txt +example_input_file6050.txt +example_input_file6051.txt +example_input_file6052.txt +example_input_file6053.txt +example_input_file6054.txt +example_input_file6055.txt +example_input_file6056.txt +example_input_file6057.txt +example_input_file6058.txt +example_input_file6059.txt +example_input_file6060.txt +example_input_file6061.txt +example_input_file6062.txt +example_input_file6063.txt +example_input_file6064.txt +example_input_file6065.txt +example_input_file6066.txt +example_input_file6067.txt +example_input_file6068.txt +example_input_file6069.txt +example_input_file6070.txt +example_input_file6071.txt +example_input_file6072.txt +example_input_file6073.txt +example_input_file6074.txt +example_input_file6075.txt +example_input_file6076.txt +example_input_file6077.txt +example_input_file6078.txt +example_input_file6079.txt +example_input_file6080.txt +example_input_file6081.txt +example_input_file6082.txt +example_input_file6083.txt +example_input_file6084.txt +example_input_file6085.txt +example_input_file6086.txt +example_input_file6087.txt +example_input_file6088.txt +example_input_file6089.txt +example_input_file6090.txt +example_input_file6091.txt +example_input_file6092.txt +example_input_file6093.txt +example_input_file6094.txt +example_input_file6095.txt +example_input_file6096.txt +example_input_file6097.txt +example_input_file6098.txt +example_input_file6099.txt +example_input_file6100.txt +example_input_file6101.txt +example_input_file6102.txt +example_input_file6103.txt +example_input_file6104.txt +example_input_file6105.txt +example_input_file6106.txt +example_input_file6107.txt +example_input_file6108.txt +example_input_file6109.txt +example_input_file6110.txt +example_input_file6111.txt +example_input_file6112.txt +example_input_file6113.txt +example_input_file6114.txt +example_input_file6115.txt +example_input_file6116.txt +example_input_file6117.txt +example_input_file6118.txt +example_input_file6119.txt +example_input_file6120.txt +example_input_file6121.txt +example_input_file6122.txt +example_input_file6123.txt +example_input_file6124.txt +example_input_file6125.txt +example_input_file6126.txt +example_input_file6127.txt +example_input_file6128.txt +example_input_file6129.txt +example_input_file6130.txt +example_input_file6131.txt +example_input_file6132.txt +example_input_file6133.txt +example_input_file6134.txt +example_input_file6135.txt +example_input_file6136.txt +example_input_file6137.txt +example_input_file6138.txt +example_input_file6139.txt +example_input_file6140.txt +example_input_file6141.txt +example_input_file6142.txt +example_input_file6143.txt +example_input_file6144.txt +example_input_file6145.txt +example_input_file6146.txt +example_input_file6147.txt +example_input_file6148.txt +example_input_file6149.txt +example_input_file6150.txt +example_input_file6151.txt +example_input_file6152.txt +example_input_file6153.txt +example_input_file6154.txt +example_input_file6155.txt +example_input_file6156.txt +example_input_file6157.txt +example_input_file6158.txt +example_input_file6159.txt +example_input_file6160.txt +example_input_file6161.txt +example_input_file6162.txt +example_input_file6163.txt +example_input_file6164.txt +example_input_file6165.txt +example_input_file6166.txt +example_input_file6167.txt +example_input_file6168.txt +example_input_file6169.txt +example_input_file6170.txt +example_input_file6171.txt +example_input_file6172.txt +example_input_file6173.txt +example_input_file6174.txt +example_input_file6175.txt +example_input_file6176.txt +example_input_file6177.txt +example_input_file6178.txt +example_input_file6179.txt +example_input_file6180.txt +example_input_file6181.txt +example_input_file6182.txt +example_input_file6183.txt +example_input_file6184.txt +example_input_file6185.txt +example_input_file6186.txt +example_input_file6187.txt +example_input_file6188.txt +example_input_file6189.txt +example_input_file6190.txt +example_input_file6191.txt +example_input_file6192.txt +example_input_file6193.txt +example_input_file6194.txt +example_input_file6195.txt +example_input_file6196.txt +example_input_file6197.txt +example_input_file6198.txt +example_input_file6199.txt +example_input_file6200.txt +example_input_file6201.txt +example_input_file6202.txt +example_input_file6203.txt +example_input_file6204.txt +example_input_file6205.txt +example_input_file6206.txt +example_input_file6207.txt +example_input_file6208.txt +example_input_file6209.txt +example_input_file6210.txt +example_input_file6211.txt +example_input_file6212.txt +example_input_file6213.txt +example_input_file6214.txt +example_input_file6215.txt +example_input_file6216.txt +example_input_file6217.txt +example_input_file6218.txt +example_input_file6219.txt +example_input_file6220.txt +example_input_file6221.txt +example_input_file6222.txt +example_input_file6223.txt +example_input_file6224.txt +example_input_file6225.txt +example_input_file6226.txt +example_input_file6227.txt +example_input_file6228.txt +example_input_file6229.txt +example_input_file6230.txt +example_input_file6231.txt +example_input_file6232.txt +example_input_file6233.txt +example_input_file6234.txt +example_input_file6235.txt +example_input_file6236.txt +example_input_file6237.txt +example_input_file6238.txt +example_input_file6239.txt +example_input_file6240.txt +example_input_file6241.txt +example_input_file6242.txt +example_input_file6243.txt +example_input_file6244.txt +example_input_file6245.txt +example_input_file6246.txt +example_input_file6247.txt +example_input_file6248.txt +example_input_file6249.txt +example_input_file6250.txt +example_input_file6251.txt +example_input_file6252.txt +example_input_file6253.txt +example_input_file6254.txt +example_input_file6255.txt +example_input_file6256.txt +example_input_file6257.txt +example_input_file6258.txt +example_input_file6259.txt +example_input_file6260.txt +example_input_file6261.txt +example_input_file6262.txt +example_input_file6263.txt +example_input_file6264.txt +example_input_file6265.txt +example_input_file6266.txt +example_input_file6267.txt +example_input_file6268.txt +example_input_file6269.txt +example_input_file6270.txt +example_input_file6271.txt +example_input_file6272.txt +example_input_file6273.txt +example_input_file6274.txt +example_input_file6275.txt +example_input_file6276.txt +example_input_file6277.txt +example_input_file6278.txt +example_input_file6279.txt +example_input_file6280.txt +example_input_file6281.txt +example_input_file6282.txt +example_input_file6283.txt +example_input_file6284.txt +example_input_file6285.txt +example_input_file6286.txt +example_input_file6287.txt +example_input_file6288.txt +example_input_file6289.txt +example_input_file6290.txt +example_input_file6291.txt +example_input_file6292.txt +example_input_file6293.txt +example_input_file6294.txt +example_input_file6295.txt +example_input_file6296.txt +example_input_file6297.txt +example_input_file6298.txt +example_input_file6299.txt +example_input_file6300.txt +example_input_file6301.txt +example_input_file6302.txt +example_input_file6303.txt +example_input_file6304.txt +example_input_file6305.txt +example_input_file6306.txt +example_input_file6307.txt +example_input_file6308.txt +example_input_file6309.txt +example_input_file6310.txt +example_input_file6311.txt +example_input_file6312.txt +example_input_file6313.txt +example_input_file6314.txt +example_input_file6315.txt +example_input_file6316.txt +example_input_file6317.txt +example_input_file6318.txt +example_input_file6319.txt +example_input_file6320.txt +example_input_file6321.txt +example_input_file6322.txt +example_input_file6323.txt +example_input_file6324.txt +example_input_file6325.txt +example_input_file6326.txt +example_input_file6327.txt +example_input_file6328.txt +example_input_file6329.txt +example_input_file6330.txt +example_input_file6331.txt +example_input_file6332.txt +example_input_file6333.txt +example_input_file6334.txt +example_input_file6335.txt +example_input_file6336.txt +example_input_file6337.txt +example_input_file6338.txt +example_input_file6339.txt +example_input_file6340.txt +example_input_file6341.txt +example_input_file6342.txt +example_input_file6343.txt +example_input_file6344.txt +example_input_file6345.txt +example_input_file6346.txt +example_input_file6347.txt +example_input_file6348.txt +example_input_file6349.txt +example_input_file6350.txt +example_input_file6351.txt +example_input_file6352.txt +example_input_file6353.txt +example_input_file6354.txt +example_input_file6355.txt +example_input_file6356.txt +example_input_file6357.txt +example_input_file6358.txt +example_input_file6359.txt +example_input_file6360.txt +example_input_file6361.txt +example_input_file6362.txt +example_input_file6363.txt +example_input_file6364.txt +example_input_file6365.txt +example_input_file6366.txt +example_input_file6367.txt +example_input_file6368.txt +example_input_file6369.txt +example_input_file6370.txt +example_input_file6371.txt +example_input_file6372.txt +example_input_file6373.txt +example_input_file6374.txt +example_input_file6375.txt +example_input_file6376.txt +example_input_file6377.txt +example_input_file6378.txt +example_input_file6379.txt +example_input_file6380.txt +example_input_file6381.txt +example_input_file6382.txt +example_input_file6383.txt +example_input_file6384.txt +example_input_file6385.txt +example_input_file6386.txt +example_input_file6387.txt +example_input_file6388.txt +example_input_file6389.txt +example_input_file6390.txt +example_input_file6391.txt +example_input_file6392.txt +example_input_file6393.txt +example_input_file6394.txt +example_input_file6395.txt +example_input_file6396.txt +example_input_file6397.txt +example_input_file6398.txt +example_input_file6399.txt +example_input_file6400.txt +example_input_file6401.txt +example_input_file6402.txt +example_input_file6403.txt +example_input_file6404.txt +example_input_file6405.txt +example_input_file6406.txt +example_input_file6407.txt +example_input_file6408.txt +example_input_file6409.txt +example_input_file6410.txt +example_input_file6411.txt +example_input_file6412.txt +example_input_file6413.txt +example_input_file6414.txt +example_input_file6415.txt +example_input_file6416.txt +example_input_file6417.txt +example_input_file6418.txt +example_input_file6419.txt +example_input_file6420.txt +example_input_file6421.txt +example_input_file6422.txt +example_input_file6423.txt +example_input_file6424.txt +example_input_file6425.txt +example_input_file6426.txt +example_input_file6427.txt +example_input_file6428.txt +example_input_file6429.txt +example_input_file6430.txt +example_input_file6431.txt +example_input_file6432.txt +example_input_file6433.txt +example_input_file6434.txt +example_input_file6435.txt +example_input_file6436.txt +example_input_file6437.txt +example_input_file6438.txt +example_input_file6439.txt +example_input_file6440.txt +example_input_file6441.txt +example_input_file6442.txt +example_input_file6443.txt +example_input_file6444.txt +example_input_file6445.txt +example_input_file6446.txt +example_input_file6447.txt +example_input_file6448.txt +example_input_file6449.txt +example_input_file6450.txt +example_input_file6451.txt +example_input_file6452.txt +example_input_file6453.txt +example_input_file6454.txt +example_input_file6455.txt +example_input_file6456.txt +example_input_file6457.txt +example_input_file6458.txt +example_input_file6459.txt +example_input_file6460.txt +example_input_file6461.txt +example_input_file6462.txt +example_input_file6463.txt +example_input_file6464.txt +example_input_file6465.txt +example_input_file6466.txt +example_input_file6467.txt +example_input_file6468.txt +example_input_file6469.txt +example_input_file6470.txt +example_input_file6471.txt +example_input_file6472.txt +example_input_file6473.txt +example_input_file6474.txt +example_input_file6475.txt +example_input_file6476.txt +example_input_file6477.txt +example_input_file6478.txt +example_input_file6479.txt +example_input_file6480.txt +example_input_file6481.txt +example_input_file6482.txt +example_input_file6483.txt +example_input_file6484.txt +example_input_file6485.txt +example_input_file6486.txt +example_input_file6487.txt +example_input_file6488.txt +example_input_file6489.txt +example_input_file6490.txt +example_input_file6491.txt +example_input_file6492.txt +example_input_file6493.txt +example_input_file6494.txt +example_input_file6495.txt +example_input_file6496.txt +example_input_file6497.txt +example_input_file6498.txt +example_input_file6499.txt +example_input_file6500.txt +example_input_file6501.txt +example_input_file6502.txt +example_input_file6503.txt +example_input_file6504.txt +example_input_file6505.txt +example_input_file6506.txt +example_input_file6507.txt +example_input_file6508.txt +example_input_file6509.txt +example_input_file6510.txt +example_input_file6511.txt +example_input_file6512.txt +example_input_file6513.txt +example_input_file6514.txt +example_input_file6515.txt +example_input_file6516.txt +example_input_file6517.txt +example_input_file6518.txt +example_input_file6519.txt +example_input_file6520.txt +example_input_file6521.txt +example_input_file6522.txt +example_input_file6523.txt +example_input_file6524.txt +example_input_file6525.txt +example_input_file6526.txt +example_input_file6527.txt +example_input_file6528.txt +example_input_file6529.txt +example_input_file6530.txt +example_input_file6531.txt +example_input_file6532.txt +example_input_file6533.txt +example_input_file6534.txt +example_input_file6535.txt +example_input_file6536.txt +example_input_file6537.txt +example_input_file6538.txt +example_input_file6539.txt +example_input_file6540.txt +example_input_file6541.txt +example_input_file6542.txt +example_input_file6543.txt +example_input_file6544.txt +example_input_file6545.txt +example_input_file6546.txt +example_input_file6547.txt +example_input_file6548.txt +example_input_file6549.txt +example_input_file6550.txt +example_input_file6551.txt +example_input_file6552.txt +example_input_file6553.txt +example_input_file6554.txt +example_input_file6555.txt +example_input_file6556.txt +example_input_file6557.txt +example_input_file6558.txt +example_input_file6559.txt +example_input_file6560.txt +example_input_file6561.txt +example_input_file6562.txt +example_input_file6563.txt +example_input_file6564.txt +example_input_file6565.txt +example_input_file6566.txt +example_input_file6567.txt +example_input_file6568.txt +example_input_file6569.txt +example_input_file6570.txt +example_input_file6571.txt +example_input_file6572.txt +example_input_file6573.txt +example_input_file6574.txt +example_input_file6575.txt +example_input_file6576.txt +example_input_file6577.txt +example_input_file6578.txt +example_input_file6579.txt +example_input_file6580.txt +example_input_file6581.txt +example_input_file6582.txt +example_input_file6583.txt +example_input_file6584.txt +example_input_file6585.txt +example_input_file6586.txt +example_input_file6587.txt +example_input_file6588.txt +example_input_file6589.txt +example_input_file6590.txt +example_input_file6591.txt +example_input_file6592.txt +example_input_file6593.txt +example_input_file6594.txt +example_input_file6595.txt +example_input_file6596.txt +example_input_file6597.txt +example_input_file6598.txt +example_input_file6599.txt +example_input_file6600.txt +example_input_file6601.txt +example_input_file6602.txt +example_input_file6603.txt +example_input_file6604.txt +example_input_file6605.txt +example_input_file6606.txt +example_input_file6607.txt +example_input_file6608.txt +example_input_file6609.txt +example_input_file6610.txt +example_input_file6611.txt +example_input_file6612.txt +example_input_file6613.txt +example_input_file6614.txt +example_input_file6615.txt +example_input_file6616.txt +example_input_file6617.txt +example_input_file6618.txt +example_input_file6619.txt +example_input_file6620.txt +example_input_file6621.txt +example_input_file6622.txt +example_input_file6623.txt +example_input_file6624.txt +example_input_file6625.txt +example_input_file6626.txt +example_input_file6627.txt +example_input_file6628.txt +example_input_file6629.txt +example_input_file6630.txt +example_input_file6631.txt +example_input_file6632.txt +example_input_file6633.txt +example_input_file6634.txt +example_input_file6635.txt +example_input_file6636.txt +example_input_file6637.txt +example_input_file6638.txt +example_input_file6639.txt +example_input_file6640.txt +example_input_file6641.txt +example_input_file6642.txt +example_input_file6643.txt +example_input_file6644.txt +example_input_file6645.txt +example_input_file6646.txt +example_input_file6647.txt +example_input_file6648.txt +example_input_file6649.txt +example_input_file6650.txt +example_input_file6651.txt +example_input_file6652.txt +example_input_file6653.txt +example_input_file6654.txt +example_input_file6655.txt +example_input_file6656.txt +example_input_file6657.txt +example_input_file6658.txt +example_input_file6659.txt +example_input_file6660.txt +example_input_file6661.txt +example_input_file6662.txt +example_input_file6663.txt +example_input_file6664.txt +example_input_file6665.txt +example_input_file6666.txt +example_input_file6667.txt +example_input_file6668.txt +example_input_file6669.txt +example_input_file6670.txt +example_input_file6671.txt +example_input_file6672.txt +example_input_file6673.txt +example_input_file6674.txt +example_input_file6675.txt +example_input_file6676.txt +example_input_file6677.txt +example_input_file6678.txt +example_input_file6679.txt +example_input_file6680.txt +example_input_file6681.txt +example_input_file6682.txt +example_input_file6683.txt +example_input_file6684.txt +example_input_file6685.txt +example_input_file6686.txt +example_input_file6687.txt +example_input_file6688.txt +example_input_file6689.txt +example_input_file6690.txt +example_input_file6691.txt +example_input_file6692.txt +example_input_file6693.txt +example_input_file6694.txt +example_input_file6695.txt +example_input_file6696.txt +example_input_file6697.txt +example_input_file6698.txt +example_input_file6699.txt +example_input_file6700.txt +example_input_file6701.txt +example_input_file6702.txt +example_input_file6703.txt +example_input_file6704.txt +example_input_file6705.txt +example_input_file6706.txt +example_input_file6707.txt +example_input_file6708.txt +example_input_file6709.txt +example_input_file6710.txt +example_input_file6711.txt +example_input_file6712.txt +example_input_file6713.txt +example_input_file6714.txt +example_input_file6715.txt +example_input_file6716.txt +example_input_file6717.txt +example_input_file6718.txt +example_input_file6719.txt +example_input_file6720.txt +example_input_file6721.txt +example_input_file6722.txt +example_input_file6723.txt +example_input_file6724.txt +example_input_file6725.txt +example_input_file6726.txt +example_input_file6727.txt +example_input_file6728.txt +example_input_file6729.txt +example_input_file6730.txt +example_input_file6731.txt +example_input_file6732.txt +example_input_file6733.txt +example_input_file6734.txt +example_input_file6735.txt +example_input_file6736.txt +example_input_file6737.txt +example_input_file6738.txt +example_input_file6739.txt +example_input_file6740.txt +example_input_file6741.txt +example_input_file6742.txt +example_input_file6743.txt +example_input_file6744.txt +example_input_file6745.txt +example_input_file6746.txt +example_input_file6747.txt +example_input_file6748.txt +example_input_file6749.txt +example_input_file6750.txt +example_input_file6751.txt +example_input_file6752.txt +example_input_file6753.txt +example_input_file6754.txt +example_input_file6755.txt +example_input_file6756.txt +example_input_file6757.txt +example_input_file6758.txt +example_input_file6759.txt +example_input_file6760.txt +example_input_file6761.txt +example_input_file6762.txt +example_input_file6763.txt +example_input_file6764.txt +example_input_file6765.txt +example_input_file6766.txt +example_input_file6767.txt +example_input_file6768.txt +example_input_file6769.txt +example_input_file6770.txt +example_input_file6771.txt +example_input_file6772.txt +example_input_file6773.txt +example_input_file6774.txt +example_input_file6775.txt +example_input_file6776.txt +example_input_file6777.txt +example_input_file6778.txt +example_input_file6779.txt +example_input_file6780.txt +example_input_file6781.txt +example_input_file6782.txt +example_input_file6783.txt +example_input_file6784.txt +example_input_file6785.txt +example_input_file6786.txt +example_input_file6787.txt +example_input_file6788.txt +example_input_file6789.txt +example_input_file6790.txt +example_input_file6791.txt +example_input_file6792.txt +example_input_file6793.txt +example_input_file6794.txt +example_input_file6795.txt +example_input_file6796.txt +example_input_file6797.txt +example_input_file6798.txt +example_input_file6799.txt +example_input_file6800.txt +example_input_file6801.txt +example_input_file6802.txt +example_input_file6803.txt +example_input_file6804.txt +example_input_file6805.txt +example_input_file6806.txt +example_input_file6807.txt +example_input_file6808.txt +example_input_file6809.txt +example_input_file6810.txt +example_input_file6811.txt +example_input_file6812.txt +example_input_file6813.txt +example_input_file6814.txt +example_input_file6815.txt +example_input_file6816.txt +example_input_file6817.txt +example_input_file6818.txt +example_input_file6819.txt +example_input_file6820.txt +example_input_file6821.txt +example_input_file6822.txt +example_input_file6823.txt +example_input_file6824.txt +example_input_file6825.txt +example_input_file6826.txt +example_input_file6827.txt +example_input_file6828.txt +example_input_file6829.txt +example_input_file6830.txt +example_input_file6831.txt +example_input_file6832.txt +example_input_file6833.txt +example_input_file6834.txt +example_input_file6835.txt +example_input_file6836.txt +example_input_file6837.txt +example_input_file6838.txt +example_input_file6839.txt +example_input_file6840.txt +example_input_file6841.txt +example_input_file6842.txt +example_input_file6843.txt +example_input_file6844.txt +example_input_file6845.txt +example_input_file6846.txt +example_input_file6847.txt +example_input_file6848.txt +example_input_file6849.txt +example_input_file6850.txt +example_input_file6851.txt +example_input_file6852.txt +example_input_file6853.txt +example_input_file6854.txt +example_input_file6855.txt +example_input_file6856.txt +example_input_file6857.txt +example_input_file6858.txt +example_input_file6859.txt +example_input_file6860.txt +example_input_file6861.txt +example_input_file6862.txt +example_input_file6863.txt +example_input_file6864.txt +example_input_file6865.txt +example_input_file6866.txt +example_input_file6867.txt +example_input_file6868.txt +example_input_file6869.txt +example_input_file6870.txt +example_input_file6871.txt +example_input_file6872.txt +example_input_file6873.txt +example_input_file6874.txt +example_input_file6875.txt +example_input_file6876.txt +example_input_file6877.txt +example_input_file6878.txt +example_input_file6879.txt +example_input_file6880.txt +example_input_file6881.txt +example_input_file6882.txt +example_input_file6883.txt +example_input_file6884.txt +example_input_file6885.txt +example_input_file6886.txt +example_input_file6887.txt +example_input_file6888.txt +example_input_file6889.txt +example_input_file6890.txt +example_input_file6891.txt +example_input_file6892.txt +example_input_file6893.txt +example_input_file6894.txt +example_input_file6895.txt +example_input_file6896.txt +example_input_file6897.txt +example_input_file6898.txt +example_input_file6899.txt +example_input_file6900.txt +example_input_file6901.txt +example_input_file6902.txt +example_input_file6903.txt +example_input_file6904.txt +example_input_file6905.txt +example_input_file6906.txt +example_input_file6907.txt +example_input_file6908.txt +example_input_file6909.txt +example_input_file6910.txt +example_input_file6911.txt +example_input_file6912.txt +example_input_file6913.txt +example_input_file6914.txt +example_input_file6915.txt +example_input_file6916.txt +example_input_file6917.txt +example_input_file6918.txt +example_input_file6919.txt +example_input_file6920.txt +example_input_file6921.txt +example_input_file6922.txt +example_input_file6923.txt +example_input_file6924.txt +example_input_file6925.txt +example_input_file6926.txt +example_input_file6927.txt +example_input_file6928.txt +example_input_file6929.txt +example_input_file6930.txt +example_input_file6931.txt +example_input_file6932.txt +example_input_file6933.txt +example_input_file6934.txt +example_input_file6935.txt +example_input_file6936.txt +example_input_file6937.txt +example_input_file6938.txt +example_input_file6939.txt +example_input_file6940.txt +example_input_file6941.txt +example_input_file6942.txt +example_input_file6943.txt +example_input_file6944.txt +example_input_file6945.txt +example_input_file6946.txt +example_input_file6947.txt +example_input_file6948.txt +example_input_file6949.txt +example_input_file6950.txt +example_input_file6951.txt +example_input_file6952.txt +example_input_file6953.txt +example_input_file6954.txt +example_input_file6955.txt +example_input_file6956.txt +example_input_file6957.txt +example_input_file6958.txt +example_input_file6959.txt +example_input_file6960.txt +example_input_file6961.txt +example_input_file6962.txt +example_input_file6963.txt +example_input_file6964.txt +example_input_file6965.txt +example_input_file6966.txt +example_input_file6967.txt +example_input_file6968.txt +example_input_file6969.txt +example_input_file6970.txt +example_input_file6971.txt +example_input_file6972.txt +example_input_file6973.txt +example_input_file6974.txt +example_input_file6975.txt +example_input_file6976.txt +example_input_file6977.txt +example_input_file6978.txt +example_input_file6979.txt +example_input_file6980.txt +example_input_file6981.txt +example_input_file6982.txt +example_input_file6983.txt +example_input_file6984.txt +example_input_file6985.txt +example_input_file6986.txt +example_input_file6987.txt +example_input_file6988.txt +example_input_file6989.txt +example_input_file6990.txt +example_input_file6991.txt +example_input_file6992.txt +example_input_file6993.txt +example_input_file6994.txt +example_input_file6995.txt +example_input_file6996.txt +example_input_file6997.txt +example_input_file6998.txt +example_input_file6999.txt +example_input_file7000.txt +example_input_file7001.txt +example_input_file7002.txt +example_input_file7003.txt +example_input_file7004.txt +example_input_file7005.txt +example_input_file7006.txt +example_input_file7007.txt +example_input_file7008.txt +example_input_file7009.txt +example_input_file7010.txt +example_input_file7011.txt +example_input_file7012.txt +example_input_file7013.txt +example_input_file7014.txt +example_input_file7015.txt +example_input_file7016.txt +example_input_file7017.txt +example_input_file7018.txt +example_input_file7019.txt +example_input_file7020.txt +example_input_file7021.txt +example_input_file7022.txt +example_input_file7023.txt +example_input_file7024.txt +example_input_file7025.txt +example_input_file7026.txt +example_input_file7027.txt +example_input_file7028.txt +example_input_file7029.txt +example_input_file7030.txt +example_input_file7031.txt +example_input_file7032.txt +example_input_file7033.txt +example_input_file7034.txt +example_input_file7035.txt +example_input_file7036.txt +example_input_file7037.txt +example_input_file7038.txt +example_input_file7039.txt +example_input_file7040.txt +example_input_file7041.txt +example_input_file7042.txt +example_input_file7043.txt +example_input_file7044.txt +example_input_file7045.txt +example_input_file7046.txt +example_input_file7047.txt +example_input_file7048.txt +example_input_file7049.txt +example_input_file7050.txt +example_input_file7051.txt +example_input_file7052.txt +example_input_file7053.txt +example_input_file7054.txt +example_input_file7055.txt +example_input_file7056.txt +example_input_file7057.txt +example_input_file7058.txt +example_input_file7059.txt +example_input_file7060.txt +example_input_file7061.txt +example_input_file7062.txt +example_input_file7063.txt +example_input_file7064.txt +example_input_file7065.txt +example_input_file7066.txt +example_input_file7067.txt +example_input_file7068.txt +example_input_file7069.txt +example_input_file7070.txt +example_input_file7071.txt +example_input_file7072.txt +example_input_file7073.txt +example_input_file7074.txt +example_input_file7075.txt +example_input_file7076.txt +example_input_file7077.txt +example_input_file7078.txt +example_input_file7079.txt +example_input_file7080.txt +example_input_file7081.txt +example_input_file7082.txt +example_input_file7083.txt +example_input_file7084.txt +example_input_file7085.txt +example_input_file7086.txt +example_input_file7087.txt +example_input_file7088.txt +example_input_file7089.txt +example_input_file7090.txt +example_input_file7091.txt +example_input_file7092.txt +example_input_file7093.txt +example_input_file7094.txt +example_input_file7095.txt +example_input_file7096.txt +example_input_file7097.txt +example_input_file7098.txt +example_input_file7099.txt +example_input_file7100.txt +example_input_file7101.txt +example_input_file7102.txt +example_input_file7103.txt +example_input_file7104.txt +example_input_file7105.txt +example_input_file7106.txt +example_input_file7107.txt +example_input_file7108.txt +example_input_file7109.txt +example_input_file7110.txt +example_input_file7111.txt +example_input_file7112.txt +example_input_file7113.txt +example_input_file7114.txt +example_input_file7115.txt +example_input_file7116.txt +example_input_file7117.txt +example_input_file7118.txt +example_input_file7119.txt +example_input_file7120.txt +example_input_file7121.txt +example_input_file7122.txt +example_input_file7123.txt +example_input_file7124.txt +example_input_file7125.txt +example_input_file7126.txt +example_input_file7127.txt +example_input_file7128.txt +example_input_file7129.txt +example_input_file7130.txt +example_input_file7131.txt +example_input_file7132.txt +example_input_file7133.txt +example_input_file7134.txt +example_input_file7135.txt +example_input_file7136.txt +example_input_file7137.txt +example_input_file7138.txt +example_input_file7139.txt +example_input_file7140.txt +example_input_file7141.txt +example_input_file7142.txt +example_input_file7143.txt +example_input_file7144.txt +example_input_file7145.txt +example_input_file7146.txt +example_input_file7147.txt +example_input_file7148.txt +example_input_file7149.txt +example_input_file7150.txt +example_input_file7151.txt +example_input_file7152.txt +example_input_file7153.txt +example_input_file7154.txt +example_input_file7155.txt +example_input_file7156.txt +example_input_file7157.txt +example_input_file7158.txt +example_input_file7159.txt +example_input_file7160.txt +example_input_file7161.txt +example_input_file7162.txt +example_input_file7163.txt +example_input_file7164.txt +example_input_file7165.txt +example_input_file7166.txt +example_input_file7167.txt +example_input_file7168.txt +example_input_file7169.txt +example_input_file7170.txt +example_input_file7171.txt +example_input_file7172.txt +example_input_file7173.txt +example_input_file7174.txt +example_input_file7175.txt +example_input_file7176.txt +example_input_file7177.txt +example_input_file7178.txt +example_input_file7179.txt +example_input_file7180.txt +example_input_file7181.txt +example_input_file7182.txt +example_input_file7183.txt +example_input_file7184.txt +example_input_file7185.txt +example_input_file7186.txt +example_input_file7187.txt +example_input_file7188.txt +example_input_file7189.txt +example_input_file7190.txt +example_input_file7191.txt +example_input_file7192.txt +example_input_file7193.txt +example_input_file7194.txt +example_input_file7195.txt +example_input_file7196.txt +example_input_file7197.txt +example_input_file7198.txt +example_input_file7199.txt +example_input_file7200.txt +example_input_file7201.txt +example_input_file7202.txt +example_input_file7203.txt +example_input_file7204.txt +example_input_file7205.txt +example_input_file7206.txt +example_input_file7207.txt +example_input_file7208.txt +example_input_file7209.txt +example_input_file7210.txt +example_input_file7211.txt +example_input_file7212.txt +example_input_file7213.txt +example_input_file7214.txt +example_input_file7215.txt +example_input_file7216.txt +example_input_file7217.txt +example_input_file7218.txt +example_input_file7219.txt +example_input_file7220.txt +example_input_file7221.txt +example_input_file7222.txt +example_input_file7223.txt +example_input_file7224.txt +example_input_file7225.txt +example_input_file7226.txt +example_input_file7227.txt +example_input_file7228.txt +example_input_file7229.txt +example_input_file7230.txt +example_input_file7231.txt +example_input_file7232.txt +example_input_file7233.txt +example_input_file7234.txt +example_input_file7235.txt +example_input_file7236.txt +example_input_file7237.txt +example_input_file7238.txt +example_input_file7239.txt +example_input_file7240.txt +example_input_file7241.txt +example_input_file7242.txt +example_input_file7243.txt +example_input_file7244.txt +example_input_file7245.txt +example_input_file7246.txt +example_input_file7247.txt +example_input_file7248.txt +example_input_file7249.txt +example_input_file7250.txt +example_input_file7251.txt +example_input_file7252.txt +example_input_file7253.txt +example_input_file7254.txt +example_input_file7255.txt +example_input_file7256.txt +example_input_file7257.txt +example_input_file7258.txt +example_input_file7259.txt +example_input_file7260.txt +example_input_file7261.txt +example_input_file7262.txt +example_input_file7263.txt +example_input_file7264.txt +example_input_file7265.txt +example_input_file7266.txt +example_input_file7267.txt +example_input_file7268.txt +example_input_file7269.txt +example_input_file7270.txt +example_input_file7271.txt +example_input_file7272.txt +example_input_file7273.txt +example_input_file7274.txt +example_input_file7275.txt +example_input_file7276.txt +example_input_file7277.txt +example_input_file7278.txt +example_input_file7279.txt +example_input_file7280.txt +example_input_file7281.txt +example_input_file7282.txt +example_input_file7283.txt +example_input_file7284.txt +example_input_file7285.txt +example_input_file7286.txt +example_input_file7287.txt +example_input_file7288.txt +example_input_file7289.txt +example_input_file7290.txt +example_input_file7291.txt +example_input_file7292.txt +example_input_file7293.txt +example_input_file7294.txt +example_input_file7295.txt +example_input_file7296.txt +example_input_file7297.txt +example_input_file7298.txt +example_input_file7299.txt +example_input_file7300.txt +example_input_file7301.txt +example_input_file7302.txt +example_input_file7303.txt +example_input_file7304.txt +example_input_file7305.txt +example_input_file7306.txt +example_input_file7307.txt +example_input_file7308.txt +example_input_file7309.txt +example_input_file7310.txt +example_input_file7311.txt +example_input_file7312.txt +example_input_file7313.txt +example_input_file7314.txt +example_input_file7315.txt +example_input_file7316.txt +example_input_file7317.txt +example_input_file7318.txt +example_input_file7319.txt +example_input_file7320.txt +example_input_file7321.txt +example_input_file7322.txt +example_input_file7323.txt +example_input_file7324.txt +example_input_file7325.txt +example_input_file7326.txt +example_input_file7327.txt +example_input_file7328.txt +example_input_file7329.txt +example_input_file7330.txt +example_input_file7331.txt +example_input_file7332.txt +example_input_file7333.txt +example_input_file7334.txt +example_input_file7335.txt +example_input_file7336.txt +example_input_file7337.txt +example_input_file7338.txt +example_input_file7339.txt +example_input_file7340.txt +example_input_file7341.txt +example_input_file7342.txt +example_input_file7343.txt +example_input_file7344.txt +example_input_file7345.txt +example_input_file7346.txt +example_input_file7347.txt +example_input_file7348.txt +example_input_file7349.txt +example_input_file7350.txt +example_input_file7351.txt +example_input_file7352.txt +example_input_file7353.txt +example_input_file7354.txt +example_input_file7355.txt +example_input_file7356.txt +example_input_file7357.txt +example_input_file7358.txt +example_input_file7359.txt +example_input_file7360.txt +example_input_file7361.txt +example_input_file7362.txt +example_input_file7363.txt +example_input_file7364.txt +example_input_file7365.txt +example_input_file7366.txt +example_input_file7367.txt +example_input_file7368.txt +example_input_file7369.txt +example_input_file7370.txt +example_input_file7371.txt +example_input_file7372.txt +example_input_file7373.txt +example_input_file7374.txt +example_input_file7375.txt +example_input_file7376.txt +example_input_file7377.txt +example_input_file7378.txt +example_input_file7379.txt +example_input_file7380.txt +example_input_file7381.txt +example_input_file7382.txt +example_input_file7383.txt +example_input_file7384.txt +example_input_file7385.txt +example_input_file7386.txt +example_input_file7387.txt +example_input_file7388.txt +example_input_file7389.txt +example_input_file7390.txt +example_input_file7391.txt +example_input_file7392.txt +example_input_file7393.txt +example_input_file7394.txt +example_input_file7395.txt +example_input_file7396.txt +example_input_file7397.txt +example_input_file7398.txt +example_input_file7399.txt +example_input_file7400.txt +example_input_file7401.txt +example_input_file7402.txt +example_input_file7403.txt +example_input_file7404.txt +example_input_file7405.txt +example_input_file7406.txt +example_input_file7407.txt +example_input_file7408.txt +example_input_file7409.txt +example_input_file7410.txt +example_input_file7411.txt +example_input_file7412.txt +example_input_file7413.txt +example_input_file7414.txt +example_input_file7415.txt +example_input_file7416.txt +example_input_file7417.txt +example_input_file7418.txt +example_input_file7419.txt +example_input_file7420.txt +example_input_file7421.txt +example_input_file7422.txt +example_input_file7423.txt +example_input_file7424.txt +example_input_file7425.txt +example_input_file7426.txt +example_input_file7427.txt +example_input_file7428.txt +example_input_file7429.txt +example_input_file7430.txt +example_input_file7431.txt +example_input_file7432.txt +example_input_file7433.txt +example_input_file7434.txt +example_input_file7435.txt +example_input_file7436.txt +example_input_file7437.txt +example_input_file7438.txt +example_input_file7439.txt +example_input_file7440.txt +example_input_file7441.txt +example_input_file7442.txt +example_input_file7443.txt +example_input_file7444.txt +example_input_file7445.txt +example_input_file7446.txt +example_input_file7447.txt +example_input_file7448.txt +example_input_file7449.txt +example_input_file7450.txt +example_input_file7451.txt +example_input_file7452.txt +example_input_file7453.txt +example_input_file7454.txt +example_input_file7455.txt +example_input_file7456.txt +example_input_file7457.txt +example_input_file7458.txt +example_input_file7459.txt +example_input_file7460.txt +example_input_file7461.txt +example_input_file7462.txt +example_input_file7463.txt +example_input_file7464.txt +example_input_file7465.txt +example_input_file7466.txt +example_input_file7467.txt +example_input_file7468.txt +example_input_file7469.txt +example_input_file7470.txt +example_input_file7471.txt +example_input_file7472.txt +example_input_file7473.txt +example_input_file7474.txt +example_input_file7475.txt +example_input_file7476.txt +example_input_file7477.txt +example_input_file7478.txt +example_input_file7479.txt +example_input_file7480.txt +example_input_file7481.txt +example_input_file7482.txt +example_input_file7483.txt +example_input_file7484.txt +example_input_file7485.txt +example_input_file7486.txt +example_input_file7487.txt +example_input_file7488.txt +example_input_file7489.txt +example_input_file7490.txt +example_input_file7491.txt +example_input_file7492.txt +example_input_file7493.txt +example_input_file7494.txt +example_input_file7495.txt +example_input_file7496.txt +example_input_file7497.txt +example_input_file7498.txt +example_input_file7499.txt +example_input_file7500.txt +example_input_file7501.txt +example_input_file7502.txt +example_input_file7503.txt +example_input_file7504.txt +example_input_file7505.txt +example_input_file7506.txt +example_input_file7507.txt +example_input_file7508.txt +example_input_file7509.txt +example_input_file7510.txt +example_input_file7511.txt +example_input_file7512.txt +example_input_file7513.txt +example_input_file7514.txt +example_input_file7515.txt +example_input_file7516.txt +example_input_file7517.txt +example_input_file7518.txt +example_input_file7519.txt +example_input_file7520.txt +example_input_file7521.txt +example_input_file7522.txt +example_input_file7523.txt +example_input_file7524.txt +example_input_file7525.txt +example_input_file7526.txt +example_input_file7527.txt +example_input_file7528.txt +example_input_file7529.txt +example_input_file7530.txt +example_input_file7531.txt +example_input_file7532.txt +example_input_file7533.txt +example_input_file7534.txt +example_input_file7535.txt +example_input_file7536.txt +example_input_file7537.txt +example_input_file7538.txt +example_input_file7539.txt +example_input_file7540.txt +example_input_file7541.txt +example_input_file7542.txt +example_input_file7543.txt +example_input_file7544.txt +example_input_file7545.txt +example_input_file7546.txt +example_input_file7547.txt +example_input_file7548.txt +example_input_file7549.txt +example_input_file7550.txt +example_input_file7551.txt +example_input_file7552.txt +example_input_file7553.txt +example_input_file7554.txt +example_input_file7555.txt +example_input_file7556.txt +example_input_file7557.txt +example_input_file7558.txt +example_input_file7559.txt +example_input_file7560.txt +example_input_file7561.txt +example_input_file7562.txt +example_input_file7563.txt +example_input_file7564.txt +example_input_file7565.txt +example_input_file7566.txt +example_input_file7567.txt +example_input_file7568.txt +example_input_file7569.txt +example_input_file7570.txt +example_input_file7571.txt +example_input_file7572.txt +example_input_file7573.txt +example_input_file7574.txt +example_input_file7575.txt +example_input_file7576.txt +example_input_file7577.txt +example_input_file7578.txt +example_input_file7579.txt +example_input_file7580.txt +example_input_file7581.txt +example_input_file7582.txt +example_input_file7583.txt +example_input_file7584.txt +example_input_file7585.txt +example_input_file7586.txt +example_input_file7587.txt +example_input_file7588.txt +example_input_file7589.txt +example_input_file7590.txt +example_input_file7591.txt +example_input_file7592.txt +example_input_file7593.txt +example_input_file7594.txt +example_input_file7595.txt +example_input_file7596.txt +example_input_file7597.txt +example_input_file7598.txt +example_input_file7599.txt +example_input_file7600.txt +example_input_file7601.txt +example_input_file7602.txt +example_input_file7603.txt +example_input_file7604.txt +example_input_file7605.txt +example_input_file7606.txt +example_input_file7607.txt +example_input_file7608.txt +example_input_file7609.txt +example_input_file7610.txt +example_input_file7611.txt +example_input_file7612.txt +example_input_file7613.txt +example_input_file7614.txt +example_input_file7615.txt +example_input_file7616.txt +example_input_file7617.txt +example_input_file7618.txt +example_input_file7619.txt +example_input_file7620.txt +example_input_file7621.txt +example_input_file7622.txt +example_input_file7623.txt +example_input_file7624.txt +example_input_file7625.txt +example_input_file7626.txt +example_input_file7627.txt +example_input_file7628.txt +example_input_file7629.txt +example_input_file7630.txt +example_input_file7631.txt +example_input_file7632.txt +example_input_file7633.txt +example_input_file7634.txt +example_input_file7635.txt +example_input_file7636.txt +example_input_file7637.txt +example_input_file7638.txt +example_input_file7639.txt +example_input_file7640.txt +example_input_file7641.txt +example_input_file7642.txt +example_input_file7643.txt +example_input_file7644.txt +example_input_file7645.txt +example_input_file7646.txt +example_input_file7647.txt +example_input_file7648.txt +example_input_file7649.txt +example_input_file7650.txt +example_input_file7651.txt +example_input_file7652.txt +example_input_file7653.txt +example_input_file7654.txt +example_input_file7655.txt +example_input_file7656.txt +example_input_file7657.txt +example_input_file7658.txt +example_input_file7659.txt +example_input_file7660.txt +example_input_file7661.txt +example_input_file7662.txt +example_input_file7663.txt +example_input_file7664.txt +example_input_file7665.txt +example_input_file7666.txt +example_input_file7667.txt +example_input_file7668.txt +example_input_file7669.txt +example_input_file7670.txt +example_input_file7671.txt +example_input_file7672.txt +example_input_file7673.txt +example_input_file7674.txt +example_input_file7675.txt +example_input_file7676.txt +example_input_file7677.txt +example_input_file7678.txt +example_input_file7679.txt +example_input_file7680.txt +example_input_file7681.txt +example_input_file7682.txt +example_input_file7683.txt +example_input_file7684.txt +example_input_file7685.txt +example_input_file7686.txt +example_input_file7687.txt +example_input_file7688.txt +example_input_file7689.txt +example_input_file7690.txt +example_input_file7691.txt +example_input_file7692.txt +example_input_file7693.txt +example_input_file7694.txt +example_input_file7695.txt +example_input_file7696.txt +example_input_file7697.txt +example_input_file7698.txt +example_input_file7699.txt +example_input_file7700.txt +example_input_file7701.txt +example_input_file7702.txt +example_input_file7703.txt +example_input_file7704.txt +example_input_file7705.txt +example_input_file7706.txt +example_input_file7707.txt +example_input_file7708.txt +example_input_file7709.txt +example_input_file7710.txt +example_input_file7711.txt +example_input_file7712.txt +example_input_file7713.txt +example_input_file7714.txt +example_input_file7715.txt +example_input_file7716.txt +example_input_file7717.txt +example_input_file7718.txt +example_input_file7719.txt +example_input_file7720.txt +example_input_file7721.txt +example_input_file7722.txt +example_input_file7723.txt +example_input_file7724.txt +example_input_file7725.txt +example_input_file7726.txt +example_input_file7727.txt +example_input_file7728.txt +example_input_file7729.txt +example_input_file7730.txt +example_input_file7731.txt +example_input_file7732.txt +example_input_file7733.txt +example_input_file7734.txt +example_input_file7735.txt +example_input_file7736.txt +example_input_file7737.txt +example_input_file7738.txt +example_input_file7739.txt +example_input_file7740.txt +example_input_file7741.txt +example_input_file7742.txt +example_input_file7743.txt +example_input_file7744.txt +example_input_file7745.txt +example_input_file7746.txt +example_input_file7747.txt +example_input_file7748.txt +example_input_file7749.txt +example_input_file7750.txt +example_input_file7751.txt +example_input_file7752.txt +example_input_file7753.txt +example_input_file7754.txt +example_input_file7755.txt +example_input_file7756.txt +example_input_file7757.txt +example_input_file7758.txt +example_input_file7759.txt +example_input_file7760.txt +example_input_file7761.txt +example_input_file7762.txt +example_input_file7763.txt +example_input_file7764.txt +example_input_file7765.txt +example_input_file7766.txt +example_input_file7767.txt +example_input_file7768.txt +example_input_file7769.txt +example_input_file7770.txt +example_input_file7771.txt +example_input_file7772.txt +example_input_file7773.txt +example_input_file7774.txt +example_input_file7775.txt +example_input_file7776.txt +example_input_file7777.txt +example_input_file7778.txt +example_input_file7779.txt +example_input_file7780.txt +example_input_file7781.txt +example_input_file7782.txt +example_input_file7783.txt +example_input_file7784.txt +example_input_file7785.txt +example_input_file7786.txt +example_input_file7787.txt +example_input_file7788.txt +example_input_file7789.txt +example_input_file7790.txt +example_input_file7791.txt +example_input_file7792.txt +example_input_file7793.txt +example_input_file7794.txt +example_input_file7795.txt +example_input_file7796.txt +example_input_file7797.txt +example_input_file7798.txt +example_input_file7799.txt +example_input_file7800.txt +example_input_file7801.txt +example_input_file7802.txt +example_input_file7803.txt +example_input_file7804.txt +example_input_file7805.txt +example_input_file7806.txt +example_input_file7807.txt +example_input_file7808.txt +example_input_file7809.txt +example_input_file7810.txt +example_input_file7811.txt +example_input_file7812.txt +example_input_file7813.txt +example_input_file7814.txt +example_input_file7815.txt +example_input_file7816.txt +example_input_file7817.txt +example_input_file7818.txt +example_input_file7819.txt +example_input_file7820.txt +example_input_file7821.txt +example_input_file7822.txt +example_input_file7823.txt +example_input_file7824.txt +example_input_file7825.txt +example_input_file7826.txt +example_input_file7827.txt +example_input_file7828.txt +example_input_file7829.txt +example_input_file7830.txt +example_input_file7831.txt +example_input_file7832.txt +example_input_file7833.txt +example_input_file7834.txt +example_input_file7835.txt +example_input_file7836.txt +example_input_file7837.txt +example_input_file7838.txt +example_input_file7839.txt +example_input_file7840.txt +example_input_file7841.txt +example_input_file7842.txt +example_input_file7843.txt +example_input_file7844.txt +example_input_file7845.txt +example_input_file7846.txt +example_input_file7847.txt +example_input_file7848.txt +example_input_file7849.txt +example_input_file7850.txt +example_input_file7851.txt +example_input_file7852.txt +example_input_file7853.txt +example_input_file7854.txt +example_input_file7855.txt +example_input_file7856.txt +example_input_file7857.txt +example_input_file7858.txt +example_input_file7859.txt +example_input_file7860.txt +example_input_file7861.txt +example_input_file7862.txt +example_input_file7863.txt +example_input_file7864.txt +example_input_file7865.txt +example_input_file7866.txt +example_input_file7867.txt +example_input_file7868.txt +example_input_file7869.txt +example_input_file7870.txt +example_input_file7871.txt +example_input_file7872.txt +example_input_file7873.txt +example_input_file7874.txt +example_input_file7875.txt +example_input_file7876.txt +example_input_file7877.txt +example_input_file7878.txt +example_input_file7879.txt +example_input_file7880.txt +example_input_file7881.txt +example_input_file7882.txt +example_input_file7883.txt +example_input_file7884.txt +example_input_file7885.txt +example_input_file7886.txt +example_input_file7887.txt +example_input_file7888.txt +example_input_file7889.txt +example_input_file7890.txt +example_input_file7891.txt +example_input_file7892.txt +example_input_file7893.txt +example_input_file7894.txt +example_input_file7895.txt +example_input_file7896.txt +example_input_file7897.txt +example_input_file7898.txt +example_input_file7899.txt +example_input_file7900.txt +example_input_file7901.txt +example_input_file7902.txt +example_input_file7903.txt +example_input_file7904.txt +example_input_file7905.txt +example_input_file7906.txt +example_input_file7907.txt +example_input_file7908.txt +example_input_file7909.txt +example_input_file7910.txt +example_input_file7911.txt +example_input_file7912.txt +example_input_file7913.txt +example_input_file7914.txt +example_input_file7915.txt +example_input_file7916.txt +example_input_file7917.txt +example_input_file7918.txt +example_input_file7919.txt +example_input_file7920.txt +example_input_file7921.txt +example_input_file7922.txt +example_input_file7923.txt +example_input_file7924.txt +example_input_file7925.txt +example_input_file7926.txt +example_input_file7927.txt +example_input_file7928.txt +example_input_file7929.txt +example_input_file7930.txt +example_input_file7931.txt +example_input_file7932.txt +example_input_file7933.txt +example_input_file7934.txt +example_input_file7935.txt +example_input_file7936.txt +example_input_file7937.txt +example_input_file7938.txt +example_input_file7939.txt +example_input_file7940.txt +example_input_file7941.txt +example_input_file7942.txt +example_input_file7943.txt +example_input_file7944.txt +example_input_file7945.txt +example_input_file7946.txt +example_input_file7947.txt +example_input_file7948.txt +example_input_file7949.txt +example_input_file7950.txt +example_input_file7951.txt +example_input_file7952.txt +example_input_file7953.txt +example_input_file7954.txt +example_input_file7955.txt +example_input_file7956.txt +example_input_file7957.txt +example_input_file7958.txt +example_input_file7959.txt +example_input_file7960.txt +example_input_file7961.txt +example_input_file7962.txt +example_input_file7963.txt +example_input_file7964.txt +example_input_file7965.txt +example_input_file7966.txt +example_input_file7967.txt +example_input_file7968.txt +example_input_file7969.txt +example_input_file7970.txt +example_input_file7971.txt +example_input_file7972.txt +example_input_file7973.txt +example_input_file7974.txt +example_input_file7975.txt +example_input_file7976.txt +example_input_file7977.txt +example_input_file7978.txt +example_input_file7979.txt +example_input_file7980.txt +example_input_file7981.txt +example_input_file7982.txt +example_input_file7983.txt +example_input_file7984.txt +example_input_file7985.txt +example_input_file7986.txt +example_input_file7987.txt +example_input_file7988.txt +example_input_file7989.txt +example_input_file7990.txt +example_input_file7991.txt +example_input_file7992.txt +example_input_file7993.txt +example_input_file7994.txt +example_input_file7995.txt +example_input_file7996.txt +example_input_file7997.txt +example_input_file7998.txt +example_input_file7999.txt +example_input_file8000.txt +example_input_file8001.txt +example_input_file8002.txt +example_input_file8003.txt +example_input_file8004.txt +example_input_file8005.txt +example_input_file8006.txt +example_input_file8007.txt +example_input_file8008.txt +example_input_file8009.txt +example_input_file8010.txt +example_input_file8011.txt +example_input_file8012.txt +example_input_file8013.txt +example_input_file8014.txt +example_input_file8015.txt +example_input_file8016.txt +example_input_file8017.txt +example_input_file8018.txt +example_input_file8019.txt +example_input_file8020.txt +example_input_file8021.txt +example_input_file8022.txt +example_input_file8023.txt +example_input_file8024.txt +example_input_file8025.txt +example_input_file8026.txt +example_input_file8027.txt +example_input_file8028.txt +example_input_file8029.txt +example_input_file8030.txt +example_input_file8031.txt +example_input_file8032.txt +example_input_file8033.txt +example_input_file8034.txt +example_input_file8035.txt +example_input_file8036.txt +example_input_file8037.txt +example_input_file8038.txt +example_input_file8039.txt +example_input_file8040.txt +example_input_file8041.txt +example_input_file8042.txt +example_input_file8043.txt +example_input_file8044.txt +example_input_file8045.txt +example_input_file8046.txt +example_input_file8047.txt +example_input_file8048.txt +example_input_file8049.txt +example_input_file8050.txt +example_input_file8051.txt +example_input_file8052.txt +example_input_file8053.txt +example_input_file8054.txt +example_input_file8055.txt +example_input_file8056.txt +example_input_file8057.txt +example_input_file8058.txt +example_input_file8059.txt +example_input_file8060.txt +example_input_file8061.txt +example_input_file8062.txt +example_input_file8063.txt +example_input_file8064.txt +example_input_file8065.txt +example_input_file8066.txt +example_input_file8067.txt +example_input_file8068.txt +example_input_file8069.txt +example_input_file8070.txt +example_input_file8071.txt +example_input_file8072.txt +example_input_file8073.txt +example_input_file8074.txt +example_input_file8075.txt +example_input_file8076.txt +example_input_file8077.txt +example_input_file8078.txt +example_input_file8079.txt +example_input_file8080.txt +example_input_file8081.txt +example_input_file8082.txt +example_input_file8083.txt +example_input_file8084.txt +example_input_file8085.txt +example_input_file8086.txt +example_input_file8087.txt +example_input_file8088.txt +example_input_file8089.txt +example_input_file8090.txt +example_input_file8091.txt +example_input_file8092.txt +example_input_file8093.txt +example_input_file8094.txt +example_input_file8095.txt +example_input_file8096.txt +example_input_file8097.txt +example_input_file8098.txt +example_input_file8099.txt +example_input_file8100.txt +example_input_file8101.txt +example_input_file8102.txt +example_input_file8103.txt +example_input_file8104.txt +example_input_file8105.txt +example_input_file8106.txt +example_input_file8107.txt +example_input_file8108.txt +example_input_file8109.txt +example_input_file8110.txt +example_input_file8111.txt +example_input_file8112.txt +example_input_file8113.txt +example_input_file8114.txt +example_input_file8115.txt +example_input_file8116.txt +example_input_file8117.txt +example_input_file8118.txt +example_input_file8119.txt +example_input_file8120.txt +example_input_file8121.txt +example_input_file8122.txt +example_input_file8123.txt +example_input_file8124.txt +example_input_file8125.txt +example_input_file8126.txt +example_input_file8127.txt +example_input_file8128.txt +example_input_file8129.txt +example_input_file8130.txt +example_input_file8131.txt +example_input_file8132.txt +example_input_file8133.txt +example_input_file8134.txt +example_input_file8135.txt +example_input_file8136.txt +example_input_file8137.txt +example_input_file8138.txt +example_input_file8139.txt +example_input_file8140.txt +example_input_file8141.txt +example_input_file8142.txt +example_input_file8143.txt +example_input_file8144.txt +example_input_file8145.txt +example_input_file8146.txt +example_input_file8147.txt +example_input_file8148.txt +example_input_file8149.txt +example_input_file8150.txt +example_input_file8151.txt +example_input_file8152.txt +example_input_file8153.txt +example_input_file8154.txt +example_input_file8155.txt +example_input_file8156.txt +example_input_file8157.txt +example_input_file8158.txt +example_input_file8159.txt +example_input_file8160.txt +example_input_file8161.txt +example_input_file8162.txt +example_input_file8163.txt +example_input_file8164.txt +example_input_file8165.txt +example_input_file8166.txt +example_input_file8167.txt +example_input_file8168.txt +example_input_file8169.txt +example_input_file8170.txt +example_input_file8171.txt +example_input_file8172.txt +example_input_file8173.txt +example_input_file8174.txt +example_input_file8175.txt +example_input_file8176.txt +example_input_file8177.txt +example_input_file8178.txt +example_input_file8179.txt +example_input_file8180.txt +example_input_file8181.txt +example_input_file8182.txt +example_input_file8183.txt +example_input_file8184.txt +example_input_file8185.txt +example_input_file8186.txt +example_input_file8187.txt +example_input_file8188.txt +example_input_file8189.txt +example_input_file8190.txt +example_input_file8191.txt +example_input_file8192.txt +example_input_file8193.txt +example_input_file8194.txt +example_input_file8195.txt +example_input_file8196.txt +example_input_file8197.txt +example_input_file8198.txt +example_input_file8199.txt +example_input_file8200.txt +example_input_file8201.txt +example_input_file8202.txt +example_input_file8203.txt +example_input_file8204.txt +example_input_file8205.txt +example_input_file8206.txt +example_input_file8207.txt +example_input_file8208.txt +example_input_file8209.txt +example_input_file8210.txt +example_input_file8211.txt +example_input_file8212.txt +example_input_file8213.txt +example_input_file8214.txt +example_input_file8215.txt +example_input_file8216.txt +example_input_file8217.txt +example_input_file8218.txt +example_input_file8219.txt +example_input_file8220.txt +example_input_file8221.txt +example_input_file8222.txt +example_input_file8223.txt +example_input_file8224.txt +example_input_file8225.txt +example_input_file8226.txt +example_input_file8227.txt +example_input_file8228.txt +example_input_file8229.txt +example_input_file8230.txt +example_input_file8231.txt +example_input_file8232.txt +example_input_file8233.txt +example_input_file8234.txt +example_input_file8235.txt +example_input_file8236.txt +example_input_file8237.txt +example_input_file8238.txt +example_input_file8239.txt +example_input_file8240.txt +example_input_file8241.txt +example_input_file8242.txt +example_input_file8243.txt +example_input_file8244.txt +example_input_file8245.txt +example_input_file8246.txt +example_input_file8247.txt +example_input_file8248.txt +example_input_file8249.txt +example_input_file8250.txt +example_input_file8251.txt +example_input_file8252.txt +example_input_file8253.txt +example_input_file8254.txt +example_input_file8255.txt +example_input_file8256.txt +example_input_file8257.txt +example_input_file8258.txt +example_input_file8259.txt +example_input_file8260.txt +example_input_file8261.txt +example_input_file8262.txt +example_input_file8263.txt +example_input_file8264.txt +example_input_file8265.txt +example_input_file8266.txt +example_input_file8267.txt +example_input_file8268.txt +example_input_file8269.txt +example_input_file8270.txt +example_input_file8271.txt +example_input_file8272.txt +example_input_file8273.txt +example_input_file8274.txt +example_input_file8275.txt +example_input_file8276.txt +example_input_file8277.txt +example_input_file8278.txt +example_input_file8279.txt +example_input_file8280.txt +example_input_file8281.txt +example_input_file8282.txt +example_input_file8283.txt +example_input_file8284.txt +example_input_file8285.txt +example_input_file8286.txt +example_input_file8287.txt +example_input_file8288.txt +example_input_file8289.txt +example_input_file8290.txt +example_input_file8291.txt +example_input_file8292.txt +example_input_file8293.txt +example_input_file8294.txt +example_input_file8295.txt +example_input_file8296.txt +example_input_file8297.txt +example_input_file8298.txt +example_input_file8299.txt +example_input_file8300.txt +example_input_file8301.txt +example_input_file8302.txt +example_input_file8303.txt +example_input_file8304.txt +example_input_file8305.txt +example_input_file8306.txt +example_input_file8307.txt +example_input_file8308.txt +example_input_file8309.txt +example_input_file8310.txt +example_input_file8311.txt +example_input_file8312.txt +example_input_file8313.txt +example_input_file8314.txt +example_input_file8315.txt +example_input_file8316.txt +example_input_file8317.txt +example_input_file8318.txt +example_input_file8319.txt +example_input_file8320.txt +example_input_file8321.txt +example_input_file8322.txt +example_input_file8323.txt +example_input_file8324.txt +example_input_file8325.txt +example_input_file8326.txt +example_input_file8327.txt +example_input_file8328.txt +example_input_file8329.txt +example_input_file8330.txt +example_input_file8331.txt +example_input_file8332.txt +example_input_file8333.txt +example_input_file8334.txt +example_input_file8335.txt +example_input_file8336.txt +example_input_file8337.txt +example_input_file8338.txt +example_input_file8339.txt +example_input_file8340.txt +example_input_file8341.txt +example_input_file8342.txt +example_input_file8343.txt +example_input_file8344.txt +example_input_file8345.txt +example_input_file8346.txt +example_input_file8347.txt +example_input_file8348.txt +example_input_file8349.txt +example_input_file8350.txt +example_input_file8351.txt +example_input_file8352.txt +example_input_file8353.txt +example_input_file8354.txt +example_input_file8355.txt +example_input_file8356.txt +example_input_file8357.txt +example_input_file8358.txt +example_input_file8359.txt +example_input_file8360.txt +example_input_file8361.txt +example_input_file8362.txt +example_input_file8363.txt +example_input_file8364.txt +example_input_file8365.txt +example_input_file8366.txt +example_input_file8367.txt +example_input_file8368.txt +example_input_file8369.txt +example_input_file8370.txt +example_input_file8371.txt +example_input_file8372.txt +example_input_file8373.txt +example_input_file8374.txt +example_input_file8375.txt +example_input_file8376.txt +example_input_file8377.txt +example_input_file8378.txt +example_input_file8379.txt +example_input_file8380.txt +example_input_file8381.txt +example_input_file8382.txt +example_input_file8383.txt +example_input_file8384.txt +example_input_file8385.txt +example_input_file8386.txt +example_input_file8387.txt +example_input_file8388.txt +example_input_file8389.txt +example_input_file8390.txt +example_input_file8391.txt +example_input_file8392.txt +example_input_file8393.txt +example_input_file8394.txt +example_input_file8395.txt +example_input_file8396.txt +example_input_file8397.txt +example_input_file8398.txt +example_input_file8399.txt +example_input_file8400.txt +example_input_file8401.txt +example_input_file8402.txt +example_input_file8403.txt +example_input_file8404.txt +example_input_file8405.txt +example_input_file8406.txt +example_input_file8407.txt +example_input_file8408.txt +example_input_file8409.txt +example_input_file8410.txt +example_input_file8411.txt +example_input_file8412.txt +example_input_file8413.txt +example_input_file8414.txt +example_input_file8415.txt +example_input_file8416.txt +example_input_file8417.txt +example_input_file8418.txt +example_input_file8419.txt +example_input_file8420.txt +example_input_file8421.txt +example_input_file8422.txt +example_input_file8423.txt +example_input_file8424.txt +example_input_file8425.txt +example_input_file8426.txt +example_input_file8427.txt +example_input_file8428.txt +example_input_file8429.txt +example_input_file8430.txt +example_input_file8431.txt +example_input_file8432.txt +example_input_file8433.txt +example_input_file8434.txt +example_input_file8435.txt +example_input_file8436.txt +example_input_file8437.txt +example_input_file8438.txt +example_input_file8439.txt +example_input_file8440.txt +example_input_file8441.txt +example_input_file8442.txt +example_input_file8443.txt +example_input_file8444.txt +example_input_file8445.txt +example_input_file8446.txt +example_input_file8447.txt +example_input_file8448.txt +example_input_file8449.txt +example_input_file8450.txt +example_input_file8451.txt +example_input_file8452.txt +example_input_file8453.txt +example_input_file8454.txt +example_input_file8455.txt +example_input_file8456.txt +example_input_file8457.txt +example_input_file8458.txt +example_input_file8459.txt +example_input_file8460.txt +example_input_file8461.txt +example_input_file8462.txt +example_input_file8463.txt +example_input_file8464.txt +example_input_file8465.txt +example_input_file8466.txt +example_input_file8467.txt +example_input_file8468.txt +example_input_file8469.txt +example_input_file8470.txt +example_input_file8471.txt +example_input_file8472.txt +example_input_file8473.txt +example_input_file8474.txt +example_input_file8475.txt +example_input_file8476.txt +example_input_file8477.txt +example_input_file8478.txt +example_input_file8479.txt +example_input_file8480.txt +example_input_file8481.txt +example_input_file8482.txt +example_input_file8483.txt +example_input_file8484.txt +example_input_file8485.txt +example_input_file8486.txt +example_input_file8487.txt +example_input_file8488.txt +example_input_file8489.txt +example_input_file8490.txt +example_input_file8491.txt +example_input_file8492.txt +example_input_file8493.txt +example_input_file8494.txt +example_input_file8495.txt +example_input_file8496.txt +example_input_file8497.txt +example_input_file8498.txt +example_input_file8499.txt +example_input_file8500.txt +example_input_file8501.txt +example_input_file8502.txt +example_input_file8503.txt +example_input_file8504.txt +example_input_file8505.txt +example_input_file8506.txt +example_input_file8507.txt +example_input_file8508.txt +example_input_file8509.txt +example_input_file8510.txt +example_input_file8511.txt +example_input_file8512.txt +example_input_file8513.txt +example_input_file8514.txt +example_input_file8515.txt +example_input_file8516.txt +example_input_file8517.txt +example_input_file8518.txt +example_input_file8519.txt +example_input_file8520.txt +example_input_file8521.txt +example_input_file8522.txt +example_input_file8523.txt +example_input_file8524.txt +example_input_file8525.txt +example_input_file8526.txt +example_input_file8527.txt +example_input_file8528.txt +example_input_file8529.txt +example_input_file8530.txt +example_input_file8531.txt +example_input_file8532.txt +example_input_file8533.txt +example_input_file8534.txt +example_input_file8535.txt +example_input_file8536.txt +example_input_file8537.txt +example_input_file8538.txt +example_input_file8539.txt +example_input_file8540.txt +example_input_file8541.txt +example_input_file8542.txt +example_input_file8543.txt +example_input_file8544.txt +example_input_file8545.txt +example_input_file8546.txt +example_input_file8547.txt +example_input_file8548.txt +example_input_file8549.txt +example_input_file8550.txt +example_input_file8551.txt +example_input_file8552.txt +example_input_file8553.txt +example_input_file8554.txt +example_input_file8555.txt +example_input_file8556.txt +example_input_file8557.txt +example_input_file8558.txt +example_input_file8559.txt +example_input_file8560.txt +example_input_file8561.txt +example_input_file8562.txt +example_input_file8563.txt +example_input_file8564.txt +example_input_file8565.txt +example_input_file8566.txt +example_input_file8567.txt +example_input_file8568.txt +example_input_file8569.txt +example_input_file8570.txt +example_input_file8571.txt +example_input_file8572.txt +example_input_file8573.txt +example_input_file8574.txt +example_input_file8575.txt +example_input_file8576.txt +example_input_file8577.txt +example_input_file8578.txt +example_input_file8579.txt +example_input_file8580.txt +example_input_file8581.txt +example_input_file8582.txt +example_input_file8583.txt +example_input_file8584.txt +example_input_file8585.txt +example_input_file8586.txt +example_input_file8587.txt +example_input_file8588.txt +example_input_file8589.txt +example_input_file8590.txt +example_input_file8591.txt +example_input_file8592.txt +example_input_file8593.txt +example_input_file8594.txt +example_input_file8595.txt +example_input_file8596.txt +example_input_file8597.txt +example_input_file8598.txt +example_input_file8599.txt +example_input_file8600.txt +example_input_file8601.txt +example_input_file8602.txt +example_input_file8603.txt +example_input_file8604.txt +example_input_file8605.txt +example_input_file8606.txt +example_input_file8607.txt +example_input_file8608.txt +example_input_file8609.txt +example_input_file8610.txt +example_input_file8611.txt +example_input_file8612.txt +example_input_file8613.txt +example_input_file8614.txt +example_input_file8615.txt +example_input_file8616.txt +example_input_file8617.txt +example_input_file8618.txt +example_input_file8619.txt +example_input_file8620.txt +example_input_file8621.txt +example_input_file8622.txt +example_input_file8623.txt +example_input_file8624.txt +example_input_file8625.txt +example_input_file8626.txt +example_input_file8627.txt +example_input_file8628.txt +example_input_file8629.txt +example_input_file8630.txt +example_input_file8631.txt +example_input_file8632.txt +example_input_file8633.txt +example_input_file8634.txt +example_input_file8635.txt +example_input_file8636.txt +example_input_file8637.txt +example_input_file8638.txt +example_input_file8639.txt +example_input_file8640.txt +example_input_file8641.txt +example_input_file8642.txt +example_input_file8643.txt +example_input_file8644.txt +example_input_file8645.txt +example_input_file8646.txt +example_input_file8647.txt +example_input_file8648.txt +example_input_file8649.txt +example_input_file8650.txt +example_input_file8651.txt +example_input_file8652.txt +example_input_file8653.txt +example_input_file8654.txt +example_input_file8655.txt +example_input_file8656.txt +example_input_file8657.txt +example_input_file8658.txt +example_input_file8659.txt +example_input_file8660.txt +example_input_file8661.txt +example_input_file8662.txt +example_input_file8663.txt +example_input_file8664.txt +example_input_file8665.txt +example_input_file8666.txt +example_input_file8667.txt +example_input_file8668.txt +example_input_file8669.txt +example_input_file8670.txt +example_input_file8671.txt +example_input_file8672.txt +example_input_file8673.txt +example_input_file8674.txt +example_input_file8675.txt +example_input_file8676.txt +example_input_file8677.txt +example_input_file8678.txt +example_input_file8679.txt +example_input_file8680.txt +example_input_file8681.txt +example_input_file8682.txt +example_input_file8683.txt +example_input_file8684.txt +example_input_file8685.txt +example_input_file8686.txt +example_input_file8687.txt +example_input_file8688.txt +example_input_file8689.txt +example_input_file8690.txt +example_input_file8691.txt +example_input_file8692.txt +example_input_file8693.txt +example_input_file8694.txt +example_input_file8695.txt +example_input_file8696.txt +example_input_file8697.txt +example_input_file8698.txt +example_input_file8699.txt +example_input_file8700.txt +example_input_file8701.txt +example_input_file8702.txt +example_input_file8703.txt +example_input_file8704.txt +example_input_file8705.txt +example_input_file8706.txt +example_input_file8707.txt +example_input_file8708.txt +example_input_file8709.txt +example_input_file8710.txt +example_input_file8711.txt +example_input_file8712.txt +example_input_file8713.txt +example_input_file8714.txt +example_input_file8715.txt +example_input_file8716.txt +example_input_file8717.txt +example_input_file8718.txt +example_input_file8719.txt +example_input_file8720.txt +example_input_file8721.txt +example_input_file8722.txt +example_input_file8723.txt +example_input_file8724.txt +example_input_file8725.txt +example_input_file8726.txt +example_input_file8727.txt +example_input_file8728.txt +example_input_file8729.txt +example_input_file8730.txt +example_input_file8731.txt +example_input_file8732.txt +example_input_file8733.txt +example_input_file8734.txt +example_input_file8735.txt +example_input_file8736.txt +example_input_file8737.txt +example_input_file8738.txt +example_input_file8739.txt +example_input_file8740.txt +example_input_file8741.txt +example_input_file8742.txt +example_input_file8743.txt +example_input_file8744.txt +example_input_file8745.txt +example_input_file8746.txt +example_input_file8747.txt +example_input_file8748.txt +example_input_file8749.txt +example_input_file8750.txt +example_input_file8751.txt +example_input_file8752.txt +example_input_file8753.txt +example_input_file8754.txt +example_input_file8755.txt +example_input_file8756.txt +example_input_file8757.txt +example_input_file8758.txt +example_input_file8759.txt +example_input_file8760.txt +example_input_file8761.txt +example_input_file8762.txt +example_input_file8763.txt +example_input_file8764.txt +example_input_file8765.txt +example_input_file8766.txt +example_input_file8767.txt +example_input_file8768.txt +example_input_file8769.txt +example_input_file8770.txt +example_input_file8771.txt +example_input_file8772.txt +example_input_file8773.txt +example_input_file8774.txt +example_input_file8775.txt +example_input_file8776.txt +example_input_file8777.txt +example_input_file8778.txt +example_input_file8779.txt +example_input_file8780.txt +example_input_file8781.txt +example_input_file8782.txt +example_input_file8783.txt +example_input_file8784.txt +example_input_file8785.txt +example_input_file8786.txt +example_input_file8787.txt +example_input_file8788.txt +example_input_file8789.txt +example_input_file8790.txt +example_input_file8791.txt +example_input_file8792.txt +example_input_file8793.txt +example_input_file8794.txt +example_input_file8795.txt +example_input_file8796.txt +example_input_file8797.txt +example_input_file8798.txt +example_input_file8799.txt +example_input_file8800.txt +example_input_file8801.txt +example_input_file8802.txt +example_input_file8803.txt +example_input_file8804.txt +example_input_file8805.txt +example_input_file8806.txt +example_input_file8807.txt +example_input_file8808.txt +example_input_file8809.txt +example_input_file8810.txt +example_input_file8811.txt +example_input_file8812.txt +example_input_file8813.txt +example_input_file8814.txt +example_input_file8815.txt +example_input_file8816.txt +example_input_file8817.txt +example_input_file8818.txt +example_input_file8819.txt +example_input_file8820.txt +example_input_file8821.txt +example_input_file8822.txt +example_input_file8823.txt +example_input_file8824.txt +example_input_file8825.txt +example_input_file8826.txt +example_input_file8827.txt +example_input_file8828.txt +example_input_file8829.txt +example_input_file8830.txt +example_input_file8831.txt +example_input_file8832.txt +example_input_file8833.txt +example_input_file8834.txt +example_input_file8835.txt +example_input_file8836.txt +example_input_file8837.txt +example_input_file8838.txt +example_input_file8839.txt +example_input_file8840.txt +example_input_file8841.txt +example_input_file8842.txt +example_input_file8843.txt +example_input_file8844.txt +example_input_file8845.txt +example_input_file8846.txt +example_input_file8847.txt +example_input_file8848.txt +example_input_file8849.txt +example_input_file8850.txt +example_input_file8851.txt +example_input_file8852.txt +example_input_file8853.txt +example_input_file8854.txt +example_input_file8855.txt +example_input_file8856.txt +example_input_file8857.txt +example_input_file8858.txt +example_input_file8859.txt +example_input_file8860.txt +example_input_file8861.txt +example_input_file8862.txt +example_input_file8863.txt +example_input_file8864.txt +example_input_file8865.txt +example_input_file8866.txt +example_input_file8867.txt +example_input_file8868.txt +example_input_file8869.txt +example_input_file8870.txt +example_input_file8871.txt +example_input_file8872.txt +example_input_file8873.txt +example_input_file8874.txt +example_input_file8875.txt +example_input_file8876.txt +example_input_file8877.txt +example_input_file8878.txt +example_input_file8879.txt +example_input_file8880.txt +example_input_file8881.txt +example_input_file8882.txt +example_input_file8883.txt +example_input_file8884.txt +example_input_file8885.txt +example_input_file8886.txt +example_input_file8887.txt +example_input_file8888.txt +example_input_file8889.txt +example_input_file8890.txt +example_input_file8891.txt +example_input_file8892.txt +example_input_file8893.txt +example_input_file8894.txt +example_input_file8895.txt +example_input_file8896.txt +example_input_file8897.txt +example_input_file8898.txt +example_input_file8899.txt +example_input_file8900.txt +example_input_file8901.txt +example_input_file8902.txt +example_input_file8903.txt +example_input_file8904.txt +example_input_file8905.txt +example_input_file8906.txt +example_input_file8907.txt +example_input_file8908.txt +example_input_file8909.txt +example_input_file8910.txt +example_input_file8911.txt +example_input_file8912.txt +example_input_file8913.txt +example_input_file8914.txt +example_input_file8915.txt +example_input_file8916.txt +example_input_file8917.txt +example_input_file8918.txt +example_input_file8919.txt +example_input_file8920.txt +example_input_file8921.txt +example_input_file8922.txt +example_input_file8923.txt +example_input_file8924.txt +example_input_file8925.txt +example_input_file8926.txt +example_input_file8927.txt +example_input_file8928.txt +example_input_file8929.txt +example_input_file8930.txt +example_input_file8931.txt +example_input_file8932.txt +example_input_file8933.txt +example_input_file8934.txt +example_input_file8935.txt +example_input_file8936.txt +example_input_file8937.txt +example_input_file8938.txt +example_input_file8939.txt +example_input_file8940.txt +example_input_file8941.txt +example_input_file8942.txt +example_input_file8943.txt +example_input_file8944.txt +example_input_file8945.txt +example_input_file8946.txt +example_input_file8947.txt +example_input_file8948.txt +example_input_file8949.txt +example_input_file8950.txt +example_input_file8951.txt +example_input_file8952.txt +example_input_file8953.txt +example_input_file8954.txt +example_input_file8955.txt +example_input_file8956.txt +example_input_file8957.txt +example_input_file8958.txt +example_input_file8959.txt +example_input_file8960.txt +example_input_file8961.txt +example_input_file8962.txt +example_input_file8963.txt +example_input_file8964.txt +example_input_file8965.txt +example_input_file8966.txt +example_input_file8967.txt +example_input_file8968.txt +example_input_file8969.txt +example_input_file8970.txt +example_input_file8971.txt +example_input_file8972.txt +example_input_file8973.txt +example_input_file8974.txt +example_input_file8975.txt +example_input_file8976.txt +example_input_file8977.txt +example_input_file8978.txt +example_input_file8979.txt +example_input_file8980.txt +example_input_file8981.txt +example_input_file8982.txt +example_input_file8983.txt +example_input_file8984.txt +example_input_file8985.txt +example_input_file8986.txt +example_input_file8987.txt +example_input_file8988.txt +example_input_file8989.txt +example_input_file8990.txt +example_input_file8991.txt +example_input_file8992.txt +example_input_file8993.txt +example_input_file8994.txt +example_input_file8995.txt +example_input_file8996.txt +example_input_file8997.txt +example_input_file8998.txt +example_input_file8999.txt +example_input_file9000.txt +example_input_file9001.txt +example_input_file9002.txt +example_input_file9003.txt +example_input_file9004.txt +example_input_file9005.txt +example_input_file9006.txt +example_input_file9007.txt +example_input_file9008.txt +example_input_file9009.txt +example_input_file9010.txt +example_input_file9011.txt +example_input_file9012.txt +example_input_file9013.txt +example_input_file9014.txt +example_input_file9015.txt +example_input_file9016.txt +example_input_file9017.txt +example_input_file9018.txt +example_input_file9019.txt +example_input_file9020.txt +example_input_file9021.txt +example_input_file9022.txt +example_input_file9023.txt +example_input_file9024.txt +example_input_file9025.txt +example_input_file9026.txt +example_input_file9027.txt +example_input_file9028.txt +example_input_file9029.txt +example_input_file9030.txt +example_input_file9031.txt +example_input_file9032.txt +example_input_file9033.txt +example_input_file9034.txt +example_input_file9035.txt +example_input_file9036.txt +example_input_file9037.txt +example_input_file9038.txt +example_input_file9039.txt +example_input_file9040.txt +example_input_file9041.txt +example_input_file9042.txt +example_input_file9043.txt +example_input_file9044.txt +example_input_file9045.txt +example_input_file9046.txt +example_input_file9047.txt +example_input_file9048.txt +example_input_file9049.txt +example_input_file9050.txt +example_input_file9051.txt +example_input_file9052.txt +example_input_file9053.txt +example_input_file9054.txt +example_input_file9055.txt +example_input_file9056.txt +example_input_file9057.txt +example_input_file9058.txt +example_input_file9059.txt +example_input_file9060.txt +example_input_file9061.txt +example_input_file9062.txt +example_input_file9063.txt +example_input_file9064.txt +example_input_file9065.txt +example_input_file9066.txt +example_input_file9067.txt +example_input_file9068.txt +example_input_file9069.txt +example_input_file9070.txt +example_input_file9071.txt +example_input_file9072.txt +example_input_file9073.txt +example_input_file9074.txt +example_input_file9075.txt +example_input_file9076.txt +example_input_file9077.txt +example_input_file9078.txt +example_input_file9079.txt +example_input_file9080.txt +example_input_file9081.txt +example_input_file9082.txt +example_input_file9083.txt +example_input_file9084.txt +example_input_file9085.txt +example_input_file9086.txt +example_input_file9087.txt +example_input_file9088.txt +example_input_file9089.txt +example_input_file9090.txt +example_input_file9091.txt +example_input_file9092.txt +example_input_file9093.txt +example_input_file9094.txt +example_input_file9095.txt +example_input_file9096.txt +example_input_file9097.txt +example_input_file9098.txt +example_input_file9099.txt +example_input_file9100.txt +example_input_file9101.txt +example_input_file9102.txt +example_input_file9103.txt +example_input_file9104.txt +example_input_file9105.txt +example_input_file9106.txt +example_input_file9107.txt +example_input_file9108.txt +example_input_file9109.txt +example_input_file9110.txt +example_input_file9111.txt +example_input_file9112.txt +example_input_file9113.txt +example_input_file9114.txt +example_input_file9115.txt +example_input_file9116.txt +example_input_file9117.txt +example_input_file9118.txt +example_input_file9119.txt +example_input_file9120.txt +example_input_file9121.txt +example_input_file9122.txt +example_input_file9123.txt +example_input_file9124.txt +example_input_file9125.txt +example_input_file9126.txt +example_input_file9127.txt +example_input_file9128.txt +example_input_file9129.txt +example_input_file9130.txt +example_input_file9131.txt +example_input_file9132.txt +example_input_file9133.txt +example_input_file9134.txt +example_input_file9135.txt +example_input_file9136.txt +example_input_file9137.txt +example_input_file9138.txt +example_input_file9139.txt +example_input_file9140.txt +example_input_file9141.txt +example_input_file9142.txt +example_input_file9143.txt +example_input_file9144.txt +example_input_file9145.txt +example_input_file9146.txt +example_input_file9147.txt +example_input_file9148.txt +example_input_file9149.txt +example_input_file9150.txt +example_input_file9151.txt +example_input_file9152.txt +example_input_file9153.txt +example_input_file9154.txt +example_input_file9155.txt +example_input_file9156.txt +example_input_file9157.txt +example_input_file9158.txt +example_input_file9159.txt +example_input_file9160.txt +example_input_file9161.txt +example_input_file9162.txt +example_input_file9163.txt +example_input_file9164.txt +example_input_file9165.txt +example_input_file9166.txt +example_input_file9167.txt +example_input_file9168.txt +example_input_file9169.txt +example_input_file9170.txt +example_input_file9171.txt +example_input_file9172.txt +example_input_file9173.txt +example_input_file9174.txt +example_input_file9175.txt +example_input_file9176.txt +example_input_file9177.txt +example_input_file9178.txt +example_input_file9179.txt +example_input_file9180.txt +example_input_file9181.txt +example_input_file9182.txt +example_input_file9183.txt +example_input_file9184.txt +example_input_file9185.txt +example_input_file9186.txt +example_input_file9187.txt +example_input_file9188.txt +example_input_file9189.txt +example_input_file9190.txt +example_input_file9191.txt +example_input_file9192.txt +example_input_file9193.txt +example_input_file9194.txt +example_input_file9195.txt +example_input_file9196.txt +example_input_file9197.txt +example_input_file9198.txt +example_input_file9199.txt +example_input_file9200.txt +example_input_file9201.txt +example_input_file9202.txt +example_input_file9203.txt +example_input_file9204.txt +example_input_file9205.txt +example_input_file9206.txt +example_input_file9207.txt +example_input_file9208.txt +example_input_file9209.txt +example_input_file9210.txt +example_input_file9211.txt +example_input_file9212.txt +example_input_file9213.txt +example_input_file9214.txt +example_input_file9215.txt +example_input_file9216.txt +example_input_file9217.txt +example_input_file9218.txt +example_input_file9219.txt +example_input_file9220.txt +example_input_file9221.txt +example_input_file9222.txt +example_input_file9223.txt +example_input_file9224.txt +example_input_file9225.txt +example_input_file9226.txt +example_input_file9227.txt +example_input_file9228.txt +example_input_file9229.txt +example_input_file9230.txt +example_input_file9231.txt +example_input_file9232.txt +example_input_file9233.txt +example_input_file9234.txt +example_input_file9235.txt +example_input_file9236.txt +example_input_file9237.txt +example_input_file9238.txt +example_input_file9239.txt +example_input_file9240.txt +example_input_file9241.txt +example_input_file9242.txt +example_input_file9243.txt +example_input_file9244.txt +example_input_file9245.txt +example_input_file9246.txt +example_input_file9247.txt +example_input_file9248.txt +example_input_file9249.txt +example_input_file9250.txt +example_input_file9251.txt +example_input_file9252.txt +example_input_file9253.txt +example_input_file9254.txt +example_input_file9255.txt +example_input_file9256.txt +example_input_file9257.txt +example_input_file9258.txt +example_input_file9259.txt +example_input_file9260.txt +example_input_file9261.txt +example_input_file9262.txt +example_input_file9263.txt +example_input_file9264.txt +example_input_file9265.txt +example_input_file9266.txt +example_input_file9267.txt +example_input_file9268.txt +example_input_file9269.txt +example_input_file9270.txt +example_input_file9271.txt +example_input_file9272.txt +example_input_file9273.txt +example_input_file9274.txt +example_input_file9275.txt +example_input_file9276.txt +example_input_file9277.txt +example_input_file9278.txt +example_input_file9279.txt +example_input_file9280.txt +example_input_file9281.txt +example_input_file9282.txt +example_input_file9283.txt +example_input_file9284.txt +example_input_file9285.txt +example_input_file9286.txt +example_input_file9287.txt +example_input_file9288.txt +example_input_file9289.txt +example_input_file9290.txt +example_input_file9291.txt +example_input_file9292.txt +example_input_file9293.txt +example_input_file9294.txt +example_input_file9295.txt +example_input_file9296.txt +example_input_file9297.txt +example_input_file9298.txt +example_input_file9299.txt +example_input_file9300.txt +example_input_file9301.txt +example_input_file9302.txt +example_input_file9303.txt +example_input_file9304.txt +example_input_file9305.txt +example_input_file9306.txt +example_input_file9307.txt +example_input_file9308.txt +example_input_file9309.txt +example_input_file9310.txt +example_input_file9311.txt +example_input_file9312.txt +example_input_file9313.txt +example_input_file9314.txt +example_input_file9315.txt +example_input_file9316.txt +example_input_file9317.txt +example_input_file9318.txt +example_input_file9319.txt +example_input_file9320.txt +example_input_file9321.txt +example_input_file9322.txt +example_input_file9323.txt +example_input_file9324.txt +example_input_file9325.txt +example_input_file9326.txt +example_input_file9327.txt +example_input_file9328.txt +example_input_file9329.txt +example_input_file9330.txt +example_input_file9331.txt +example_input_file9332.txt +example_input_file9333.txt +example_input_file9334.txt +example_input_file9335.txt +example_input_file9336.txt +example_input_file9337.txt +example_input_file9338.txt +example_input_file9339.txt +example_input_file9340.txt +example_input_file9341.txt +example_input_file9342.txt +example_input_file9343.txt +example_input_file9344.txt +example_input_file9345.txt +example_input_file9346.txt +example_input_file9347.txt +example_input_file9348.txt +example_input_file9349.txt +example_input_file9350.txt +example_input_file9351.txt +example_input_file9352.txt +example_input_file9353.txt +example_input_file9354.txt +example_input_file9355.txt +example_input_file9356.txt +example_input_file9357.txt +example_input_file9358.txt +example_input_file9359.txt +example_input_file9360.txt +example_input_file9361.txt +example_input_file9362.txt +example_input_file9363.txt +example_input_file9364.txt +example_input_file9365.txt +example_input_file9366.txt +example_input_file9367.txt +example_input_file9368.txt +example_input_file9369.txt +example_input_file9370.txt +example_input_file9371.txt +example_input_file9372.txt +example_input_file9373.txt +example_input_file9374.txt +example_input_file9375.txt +example_input_file9376.txt +example_input_file9377.txt +example_input_file9378.txt +example_input_file9379.txt +example_input_file9380.txt +example_input_file9381.txt +example_input_file9382.txt +example_input_file9383.txt +example_input_file9384.txt +example_input_file9385.txt +example_input_file9386.txt +example_input_file9387.txt +example_input_file9388.txt +example_input_file9389.txt +example_input_file9390.txt +example_input_file9391.txt +example_input_file9392.txt +example_input_file9393.txt +example_input_file9394.txt +example_input_file9395.txt +example_input_file9396.txt +example_input_file9397.txt +example_input_file9398.txt +example_input_file9399.txt +example_input_file9400.txt +example_input_file9401.txt +example_input_file9402.txt +example_input_file9403.txt +example_input_file9404.txt +example_input_file9405.txt +example_input_file9406.txt +example_input_file9407.txt +example_input_file9408.txt +example_input_file9409.txt +example_input_file9410.txt +example_input_file9411.txt +example_input_file9412.txt +example_input_file9413.txt +example_input_file9414.txt +example_input_file9415.txt +example_input_file9416.txt +example_input_file9417.txt +example_input_file9418.txt +example_input_file9419.txt +example_input_file9420.txt +example_input_file9421.txt +example_input_file9422.txt +example_input_file9423.txt +example_input_file9424.txt +example_input_file9425.txt +example_input_file9426.txt +example_input_file9427.txt +example_input_file9428.txt +example_input_file9429.txt +example_input_file9430.txt +example_input_file9431.txt +example_input_file9432.txt +example_input_file9433.txt +example_input_file9434.txt +example_input_file9435.txt +example_input_file9436.txt +example_input_file9437.txt +example_input_file9438.txt +example_input_file9439.txt +example_input_file9440.txt +example_input_file9441.txt +example_input_file9442.txt +example_input_file9443.txt +example_input_file9444.txt +example_input_file9445.txt +example_input_file9446.txt +example_input_file9447.txt +example_input_file9448.txt +example_input_file9449.txt +example_input_file9450.txt +example_input_file9451.txt +example_input_file9452.txt +example_input_file9453.txt +example_input_file9454.txt +example_input_file9455.txt +example_input_file9456.txt +example_input_file9457.txt +example_input_file9458.txt +example_input_file9459.txt +example_input_file9460.txt +example_input_file9461.txt +example_input_file9462.txt +example_input_file9463.txt +example_input_file9464.txt +example_input_file9465.txt +example_input_file9466.txt +example_input_file9467.txt +example_input_file9468.txt +example_input_file9469.txt +example_input_file9470.txt +example_input_file9471.txt +example_input_file9472.txt +example_input_file9473.txt +example_input_file9474.txt +example_input_file9475.txt +example_input_file9476.txt +example_input_file9477.txt +example_input_file9478.txt +example_input_file9479.txt +example_input_file9480.txt +example_input_file9481.txt +example_input_file9482.txt +example_input_file9483.txt +example_input_file9484.txt +example_input_file9485.txt +example_input_file9486.txt +example_input_file9487.txt +example_input_file9488.txt +example_input_file9489.txt +example_input_file9490.txt +example_input_file9491.txt +example_input_file9492.txt +example_input_file9493.txt +example_input_file9494.txt +example_input_file9495.txt +example_input_file9496.txt +example_input_file9497.txt +example_input_file9498.txt +example_input_file9499.txt +example_input_file9500.txt +example_input_file9501.txt +example_input_file9502.txt +example_input_file9503.txt +example_input_file9504.txt +example_input_file9505.txt +example_input_file9506.txt +example_input_file9507.txt +example_input_file9508.txt +example_input_file9509.txt +example_input_file9510.txt +example_input_file9511.txt +example_input_file9512.txt +example_input_file9513.txt +example_input_file9514.txt +example_input_file9515.txt +example_input_file9516.txt +example_input_file9517.txt +example_input_file9518.txt +example_input_file9519.txt +example_input_file9520.txt +example_input_file9521.txt +example_input_file9522.txt +example_input_file9523.txt +example_input_file9524.txt +example_input_file9525.txt +example_input_file9526.txt +example_input_file9527.txt +example_input_file9528.txt +example_input_file9529.txt +example_input_file9530.txt +example_input_file9531.txt +example_input_file9532.txt +example_input_file9533.txt +example_input_file9534.txt +example_input_file9535.txt +example_input_file9536.txt +example_input_file9537.txt +example_input_file9538.txt +example_input_file9539.txt +example_input_file9540.txt +example_input_file9541.txt +example_input_file9542.txt +example_input_file9543.txt +example_input_file9544.txt +example_input_file9545.txt +example_input_file9546.txt +example_input_file9547.txt +example_input_file9548.txt +example_input_file9549.txt +example_input_file9550.txt +example_input_file9551.txt +example_input_file9552.txt +example_input_file9553.txt +example_input_file9554.txt +example_input_file9555.txt +example_input_file9556.txt +example_input_file9557.txt +example_input_file9558.txt +example_input_file9559.txt +example_input_file9560.txt +example_input_file9561.txt +example_input_file9562.txt +example_input_file9563.txt +example_input_file9564.txt +example_input_file9565.txt +example_input_file9566.txt +example_input_file9567.txt +example_input_file9568.txt +example_input_file9569.txt +example_input_file9570.txt +example_input_file9571.txt +example_input_file9572.txt +example_input_file9573.txt +example_input_file9574.txt +example_input_file9575.txt +example_input_file9576.txt +example_input_file9577.txt +example_input_file9578.txt +example_input_file9579.txt +example_input_file9580.txt +example_input_file9581.txt +example_input_file9582.txt +example_input_file9583.txt +example_input_file9584.txt +example_input_file9585.txt +example_input_file9586.txt +example_input_file9587.txt +example_input_file9588.txt +example_input_file9589.txt +example_input_file9590.txt +example_input_file9591.txt +example_input_file9592.txt +example_input_file9593.txt +example_input_file9594.txt +example_input_file9595.txt +example_input_file9596.txt +example_input_file9597.txt +example_input_file9598.txt +example_input_file9599.txt +example_input_file9600.txt +example_input_file9601.txt +example_input_file9602.txt +example_input_file9603.txt +example_input_file9604.txt +example_input_file9605.txt +example_input_file9606.txt +example_input_file9607.txt +example_input_file9608.txt +example_input_file9609.txt +example_input_file9610.txt +example_input_file9611.txt +example_input_file9612.txt +example_input_file9613.txt +example_input_file9614.txt +example_input_file9615.txt +example_input_file9616.txt +example_input_file9617.txt +example_input_file9618.txt +example_input_file9619.txt +example_input_file9620.txt +example_input_file9621.txt +example_input_file9622.txt +example_input_file9623.txt +example_input_file9624.txt +example_input_file9625.txt +example_input_file9626.txt +example_input_file9627.txt +example_input_file9628.txt +example_input_file9629.txt +example_input_file9630.txt +example_input_file9631.txt +example_input_file9632.txt +example_input_file9633.txt +example_input_file9634.txt +example_input_file9635.txt +example_input_file9636.txt +example_input_file9637.txt +example_input_file9638.txt +example_input_file9639.txt +example_input_file9640.txt +example_input_file9641.txt +example_input_file9642.txt +example_input_file9643.txt +example_input_file9644.txt +example_input_file9645.txt +example_input_file9646.txt +example_input_file9647.txt +example_input_file9648.txt +example_input_file9649.txt +example_input_file9650.txt +example_input_file9651.txt +example_input_file9652.txt +example_input_file9653.txt +example_input_file9654.txt +example_input_file9655.txt +example_input_file9656.txt +example_input_file9657.txt +example_input_file9658.txt +example_input_file9659.txt +example_input_file9660.txt +example_input_file9661.txt +example_input_file9662.txt +example_input_file9663.txt +example_input_file9664.txt +example_input_file9665.txt +example_input_file9666.txt +example_input_file9667.txt +example_input_file9668.txt +example_input_file9669.txt +example_input_file9670.txt +example_input_file9671.txt +example_input_file9672.txt +example_input_file9673.txt +example_input_file9674.txt +example_input_file9675.txt +example_input_file9676.txt +example_input_file9677.txt +example_input_file9678.txt +example_input_file9679.txt +example_input_file9680.txt +example_input_file9681.txt +example_input_file9682.txt +example_input_file9683.txt +example_input_file9684.txt +example_input_file9685.txt +example_input_file9686.txt +example_input_file9687.txt +example_input_file9688.txt +example_input_file9689.txt +example_input_file9690.txt +example_input_file9691.txt +example_input_file9692.txt +example_input_file9693.txt +example_input_file9694.txt +example_input_file9695.txt +example_input_file9696.txt +example_input_file9697.txt +example_input_file9698.txt +example_input_file9699.txt +example_input_file9700.txt +example_input_file9701.txt +example_input_file9702.txt +example_input_file9703.txt +example_input_file9704.txt +example_input_file9705.txt +example_input_file9706.txt +example_input_file9707.txt +example_input_file9708.txt +example_input_file9709.txt +example_input_file9710.txt +example_input_file9711.txt +example_input_file9712.txt +example_input_file9713.txt +example_input_file9714.txt +example_input_file9715.txt +example_input_file9716.txt +example_input_file9717.txt +example_input_file9718.txt +example_input_file9719.txt +example_input_file9720.txt +example_input_file9721.txt +example_input_file9722.txt +example_input_file9723.txt +example_input_file9724.txt +example_input_file9725.txt +example_input_file9726.txt +example_input_file9727.txt +example_input_file9728.txt +example_input_file9729.txt +example_input_file9730.txt +example_input_file9731.txt +example_input_file9732.txt +example_input_file9733.txt +example_input_file9734.txt +example_input_file9735.txt +example_input_file9736.txt +example_input_file9737.txt +example_input_file9738.txt +example_input_file9739.txt +example_input_file9740.txt +example_input_file9741.txt +example_input_file9742.txt +example_input_file9743.txt +example_input_file9744.txt +example_input_file9745.txt +example_input_file9746.txt +example_input_file9747.txt +example_input_file9748.txt +example_input_file9749.txt +example_input_file9750.txt +example_input_file9751.txt +example_input_file9752.txt +example_input_file9753.txt +example_input_file9754.txt +example_input_file9755.txt +example_input_file9756.txt +example_input_file9757.txt +example_input_file9758.txt +example_input_file9759.txt +example_input_file9760.txt +example_input_file9761.txt +example_input_file9762.txt +example_input_file9763.txt +example_input_file9764.txt +example_input_file9765.txt +example_input_file9766.txt +example_input_file9767.txt +example_input_file9768.txt +example_input_file9769.txt +example_input_file9770.txt +example_input_file9771.txt +example_input_file9772.txt +example_input_file9773.txt +example_input_file9774.txt +example_input_file9775.txt +example_input_file9776.txt +example_input_file9777.txt +example_input_file9778.txt +example_input_file9779.txt +example_input_file9780.txt +example_input_file9781.txt +example_input_file9782.txt +example_input_file9783.txt +example_input_file9784.txt +example_input_file9785.txt +example_input_file9786.txt +example_input_file9787.txt +example_input_file9788.txt +example_input_file9789.txt +example_input_file9790.txt +example_input_file9791.txt +example_input_file9792.txt +example_input_file9793.txt +example_input_file9794.txt +example_input_file9795.txt +example_input_file9796.txt +example_input_file9797.txt +example_input_file9798.txt +example_input_file9799.txt +example_input_file9800.txt +example_input_file9801.txt +example_input_file9802.txt +example_input_file9803.txt +example_input_file9804.txt +example_input_file9805.txt +example_input_file9806.txt +example_input_file9807.txt +example_input_file9808.txt +example_input_file9809.txt +example_input_file9810.txt +example_input_file9811.txt +example_input_file9812.txt +example_input_file9813.txt +example_input_file9814.txt +example_input_file9815.txt +example_input_file9816.txt +example_input_file9817.txt +example_input_file9818.txt +example_input_file9819.txt +example_input_file9820.txt +example_input_file9821.txt +example_input_file9822.txt +example_input_file9823.txt +example_input_file9824.txt +example_input_file9825.txt +example_input_file9826.txt +example_input_file9827.txt +example_input_file9828.txt +example_input_file9829.txt +example_input_file9830.txt +example_input_file9831.txt +example_input_file9832.txt +example_input_file9833.txt +example_input_file9834.txt +example_input_file9835.txt +example_input_file9836.txt +example_input_file9837.txt +example_input_file9838.txt +example_input_file9839.txt +example_input_file9840.txt +example_input_file9841.txt +example_input_file9842.txt +example_input_file9843.txt +example_input_file9844.txt +example_input_file9845.txt +example_input_file9846.txt +example_input_file9847.txt +example_input_file9848.txt +example_input_file9849.txt +example_input_file9850.txt +example_input_file9851.txt +example_input_file9852.txt +example_input_file9853.txt +example_input_file9854.txt +example_input_file9855.txt +example_input_file9856.txt +example_input_file9857.txt +example_input_file9858.txt +example_input_file9859.txt +example_input_file9860.txt +example_input_file9861.txt +example_input_file9862.txt +example_input_file9863.txt +example_input_file9864.txt +example_input_file9865.txt +example_input_file9866.txt +example_input_file9867.txt +example_input_file9868.txt +example_input_file9869.txt +example_input_file9870.txt +example_input_file9871.txt +example_input_file9872.txt +example_input_file9873.txt +example_input_file9874.txt +example_input_file9875.txt +example_input_file9876.txt +example_input_file9877.txt +example_input_file9878.txt +example_input_file9879.txt +example_input_file9880.txt +example_input_file9881.txt +example_input_file9882.txt +example_input_file9883.txt +example_input_file9884.txt +example_input_file9885.txt +example_input_file9886.txt +example_input_file9887.txt +example_input_file9888.txt +example_input_file9889.txt +example_input_file9890.txt +example_input_file9891.txt +example_input_file9892.txt +example_input_file9893.txt +example_input_file9894.txt +example_input_file9895.txt +example_input_file9896.txt +example_input_file9897.txt +example_input_file9898.txt +example_input_file9899.txt +example_input_file9900.txt +example_input_file9901.txt +example_input_file9902.txt +example_input_file9903.txt +example_input_file9904.txt +example_input_file9905.txt +example_input_file9906.txt +example_input_file9907.txt +example_input_file9908.txt +example_input_file9909.txt +example_input_file9910.txt +example_input_file9911.txt +example_input_file9912.txt +example_input_file9913.txt +example_input_file9914.txt +example_input_file9915.txt +example_input_file9916.txt +example_input_file9917.txt +example_input_file9918.txt +example_input_file9919.txt +example_input_file9920.txt +example_input_file9921.txt +example_input_file9922.txt +example_input_file9923.txt +example_input_file9924.txt +example_input_file9925.txt +example_input_file9926.txt +example_input_file9927.txt +example_input_file9928.txt +example_input_file9929.txt +example_input_file9930.txt +example_input_file9931.txt +example_input_file9932.txt +example_input_file9933.txt +example_input_file9934.txt +example_input_file9935.txt +example_input_file9936.txt +example_input_file9937.txt +example_input_file9938.txt +example_input_file9939.txt +example_input_file9940.txt +example_input_file9941.txt +example_input_file9942.txt +example_input_file9943.txt +example_input_file9944.txt +example_input_file9945.txt +example_input_file9946.txt +example_input_file9947.txt +example_input_file9948.txt +example_input_file9949.txt +example_input_file9950.txt +example_input_file9951.txt +example_input_file9952.txt +example_input_file9953.txt +example_input_file9954.txt +example_input_file9955.txt +example_input_file9956.txt +example_input_file9957.txt +example_input_file9958.txt +example_input_file9959.txt +example_input_file9960.txt +example_input_file9961.txt +example_input_file9962.txt +example_input_file9963.txt +example_input_file9964.txt +example_input_file9965.txt +example_input_file9966.txt +example_input_file9967.txt +example_input_file9968.txt +example_input_file9969.txt +example_input_file9970.txt +example_input_file9971.txt +example_input_file9972.txt +example_input_file9973.txt +example_input_file9974.txt +example_input_file9975.txt +example_input_file9976.txt +example_input_file9977.txt +example_input_file9978.txt +example_input_file9979.txt +example_input_file9980.txt +example_input_file9981.txt +example_input_file9982.txt +example_input_file9983.txt +example_input_file9984.txt +example_input_file9985.txt +example_input_file9986.txt +example_input_file9987.txt +example_input_file9988.txt +example_input_file9989.txt +example_input_file9990.txt +example_input_file9991.txt +example_input_file9992.txt +example_input_file9993.txt +example_input_file9994.txt +example_input_file9995.txt +example_input_file9996.txt +example_input_file9997.txt +example_input_file9998.txt +example_input_file9999.txt diff --git a/tests/wf/iwd-container-entryname1.cwl b/tests/wf/iwd-container-entryname1.cwl new file mode 100644 index 000000000..3fe16c9e8 --- /dev/null +++ b/tests/wf/iwd-container-entryname1.cwl @@ -0,0 +1,23 @@ +cwlVersion: v1.2 +class: CommandLineTool +doc: | + When executing in a container, entryname can have an absolute path + to a mount location inside the container. +inputs: + filelist: File +outputs: + head: + type: File + outputBinding: + glob: head.txt +requirements: + DockerRequirement: + dockerPull: docker.io/debian:stable-slim + dockerOutputDirectory: /output + InitialWorkDirRequirement: + listing: + - entryname: /tmp2j3y7rpb/input/stuff.txt # Give it a weird prefix to minimize chance of conflict with a real file + entry: $(inputs.filelist) + ShellCommandRequirement: {} +arguments: + - {shellQuote: false, valueFrom: "head -n10 /tmp2j3y7rpb/input/stuff.txt > /output/head.txt"} diff --git a/tests/wf/iwd-container-entryname3.cwl b/tests/wf/iwd-container-entryname3.cwl new file mode 100644 index 000000000..cfa14e631 --- /dev/null +++ b/tests/wf/iwd-container-entryname3.cwl @@ -0,0 +1,24 @@ +cwlVersion: v1.2 +class: CommandLineTool +doc: | + Must fail if entryname is an absolute path and DockerRequirement is + not in the 'requirements' section. +inputs: + filelist: File +outputs: + head: + type: File + outputBinding: + glob: head.txt +hints: + DockerRequirement: + dockerPull: docker.io/debian:stable-slim + dockerOutputDirectory: /output +requirements: + InitialWorkDirRequirement: + listing: + - entryname: /tmp2j3y7rpb/input/stuff.txt # Give it a weird prefix to minimize chance of conflict with a real file + entry: $(inputs.filelist) + ShellCommandRequirement: {} +arguments: + - {shellQuote: false, valueFrom: "head -n10 /tmp2j3y7rpb/input/stuff.txt > /output/head.txt"} diff --git a/tests/wf/loadContents-input.yml b/tests/wf/loadContents-input.yml new file mode 100644 index 000000000..a44fd27ca --- /dev/null +++ b/tests/wf/loadContents-input.yml @@ -0,0 +1,3 @@ +filelist: + class: File + location: inp-filelist.txt From cc9a59f73f15aa150d192a12da182586e87d31d2 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 20 Feb 2025 17:07:40 +0100 Subject: [PATCH 59/78] MPI: move hints/reqs + DockerRequirement checking into Process._init_job() Move cache tests to their own file and add a new test --- build-cwltool-docker.sh | 2 +- cwltool/command_line_tool.py | 22 ---------------- cwltool/process.py | 37 +++++++++++++++++++++++++- tests/test_caching.py | 21 --------------- tests/test_environment.py | 2 +- tests/test_examples.py | 21 +++++++++++++++ tests/test_mpi.py | 50 ++++++++++++++++++++++-------------- 7 files changed, 90 insertions(+), 65 deletions(-) diff --git a/build-cwltool-docker.sh b/build-cwltool-docker.sh index a70fdf4df..9f7163afb 100755 --- a/build-cwltool-docker.sh +++ b/build-cwltool-docker.sh @@ -8,4 +8,4 @@ ${engine} run -t -v /var/run/docker.sock:/var/run/docker.sock \ -v /tmp:/tmp \ -v "$PWD":/tmp/cwltool \ quay.io/commonwl/cwltool_module /bin/sh -c \ - "apk add gcc bash git && pip install -r/tmp/cwltool/test-requirements.txt ; pytest -k 'not (test_bioconda or test_double_overwrite or test_env_filtering or test_biocontainers or test_disable_file_overwrite_without_ext or test_disable_file_creation_in_outdir_with_ext or test_write_write_conflict or test_directory_literal_with_real_inputs_inside or test_revsort_workflow or test_stdin_with_id_preset or test_no_compute_chcksum or test_packed_workflow_execution[tests/wf/count-lines1-wf.cwl-tests/wf/wc-job.json-False] or test_sequential_workflow or test_single_process_subwf_subwf_inline_step)' --ignore-glob '*test_udocker.py' -n 0 -v -rs --pyargs cwltool" + "apk add gcc bash git && pip install -r/tmp/cwltool/test-requirements.txt ; pytest -k 'not (test_bioconda or test_double_overwrite or test_env_filtering or test_biocontainers or test_disable_file_overwrite_without_ext or test_disable_file_creation_in_outdir_with_ext or test_write_write_conflict or test_directory_literal_with_real_inputs_inside or test_revsort_workflow or test_stdin_with_id_preset or test_no_compute_chcksum or test_packed_workflow_execution[tests/wf/count-lines1-wf.cwl-tests/wf/wc-job.json-False] or test_sequential_workflow or test_single_process_subwf_subwf_inline_step or test_cache_dockerreq_hint_instead_of_req)' --ignore-glob '*test_udocker.py' -n 0 -v -rs --pyargs cwltool" diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index d4bdbe3fb..cbff45bd0 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -431,28 +431,6 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> type[JobBase]: return SingularityCommandLineJob elif runtimeContext.user_space_docker_cmd: return UDockerCommandLineJob - if mpiReq is not None: - if mpiRequired: - if dockerRequired: - raise UnsupportedRequirement( - "No support for Docker and MPIRequirement both being required" - ) - else: - _logger.warning( - "MPI has been required while Docker is hinted, discarding Docker hint(s)" - ) - self.hints = [h for h in self.hints if h["class"] != "DockerRequirement"] - return CommandLineJob - else: - if dockerRequired: - _logger.warning( - "Docker has been required while MPI is hinted, discarding MPI hint(s)" - ) - self.hints = [h for h in self.hints if h["class"] != MPIRequirementName] - else: - raise UnsupportedRequirement( - "Both Docker and MPI have been hinted - don't know what to do" - ) if runtimeContext.podman: return PodmanCommandLineJob return DockerCommandLineJob diff --git a/cwltool/process.py b/cwltool/process.py index 5271e4643..6d1fc7ee7 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -810,12 +810,47 @@ def inc(d: list[int]) -> None: tmpdir = "" stagedir = "" - docker_req, _ = self.get_requirement("DockerRequirement") + docker_req, docker_required = self.get_requirement("DockerRequirement") default_docker = None + mpi_req, mpi_required = self.get_requirement(MPIRequirementName) if docker_req is None and runtime_context.default_container: default_docker = runtime_context.default_container + if ( + docker_req is not None + and runtime_context.use_container + and not runtime_context.singularity + and not runtime_context.user_space_docker_cmd + and mpi_req is not None + ): + if mpi_required: + if docker_required: + raise UnsupportedRequirement( + "No support for DockerRequirement and MPIRequirement " + "both being required, unless Singularity or uDocker is being used." + ) + else: + _logger.warning( + "MPI has been required while DockerRequirement is hinted " + "and neither Singularity nor uDocker is being used, discarding Docker hint(s)." + ) + self.hints = [h for h in self.hints if h["class"] != "DockerRequirement"] + docker_req = None + docker_required = False + else: + if docker_required: + _logger.warning( + "Docker has been required (and neither Singularity nor " + "uDocker is being used) while MPI is hinted, discarding MPI hint(s)/" + ) + self.hints = [h for h in self.hints if h["class"] != MPIRequirementName] + else: + raise UnsupportedRequirement( + "Both Docker and MPI have been hinted and neither " + "Singularity nor uDocker are being used - don't know what to do." + ) + if (docker_req or default_docker) and runtime_context.use_container: if docker_req is not None: # Check if docker output directory is absolute diff --git a/tests/test_caching.py b/tests/test_caching.py index 92691dbfd..db3bf0246 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -8,27 +8,6 @@ test_factors = [(""), ("--parallel"), ("--debug"), ("--parallel --debug")] -@needs_docker -@pytest.mark.parametrize("factor", test_factors) -def test_cid_file_non_existing_dir(tmp_path: Path, factor: str) -> None: - """Test that --cachedir with a bad path should produce a specific error.""" - test_file = "cache_test_workflow.cwl" - bad_cidfile_dir = tmp_path / "cidfile-dir-badpath" - commands = factor.split() - commands.extend( - [ - "--record-container-id", - "--cidfile-dir", - str(bad_cidfile_dir), - get_data("tests/wf/" + test_file), - ] - ) - error_code, _, stderr = get_main_output(commands) - stderr = re.sub(r"\s\s+", " ", stderr) - assert "directory doesn't exist, please create it first" in stderr, stderr - assert error_code == 2 or error_code == 1, stderr - - @needs_docker @pytest.mark.parametrize("factor", test_factors) def test_wf_without_container(tmp_path: Path, factor: str) -> None: diff --git a/tests/test_environment.py b/tests/test_environment.py index fa29fb924..fd2a160ec 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -204,7 +204,7 @@ def test_basic(crt_params: CheckHolder, tmp_path: Path, monkeypatch: pytest.Monk "USEDVAR": "VARVAL", "UNUSEDVAR": "VARVAL", } - args = crt_params.flags + [f"--tmpdir-prefix={tmp_prefix}"] + args = crt_params.flags + [f"--tmpdir-prefix={tmp_prefix}", "--debug"] env = get_tool_env( tmp_path, args, diff --git a/tests/test_examples.py b/tests/test_examples.py index a6696d77b..0fe7f471c 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1115,6 +1115,27 @@ def test_cid_file_dir_arg_is_file_instead_of_dir(tmp_path: Path, factor: str) -> assert error_code == 2 or error_code == 1, stderr +@needs_docker +@pytest.mark.parametrize("factor", test_factors) +def test_cid_file_non_existing_dir(tmp_path: Path, factor: str) -> None: + """Test that --cidefile-dir with a bad path should produce a specific error.""" + test_file = "cache_test_workflow.cwl" + bad_cidfile_dir = tmp_path / "cidfile-dir-badpath" + commands = factor.split() + commands.extend( + [ + "--record-container-id", + "--cidfile-dir", + str(bad_cidfile_dir), + get_data("tests/wf/" + test_file), + ] + ) + error_code, _, stderr = get_main_output(commands) + stderr = re.sub(r"\s\s+", " ", stderr) + assert "directory doesn't exist, please create it first" in stderr, stderr + assert error_code == 2 or error_code == 1, stderr + + @needs_docker @pytest.mark.parametrize("factor", test_factors) def test_cid_file_w_prefix(tmp_path: Path, factor: str) -> None: diff --git a/tests/test_mpi.py b/tests/test_mpi.py index 9da32fd05..b36392a8a 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -7,18 +7,19 @@ from importlib.resources import files from io import StringIO from pathlib import Path -from typing import Any, Optional +from typing import Any, Optional, cast import pytest from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import Names +from schema_salad.ref_resolver import file_uri from schema_salad.utils import yaml_no_ts import cwltool.load_tool import cwltool.singularity import cwltool.udocker from cwltool.command_line_tool import CommandLineTool -from cwltool.context import LoadingContext, RuntimeContext +from cwltool.context import RuntimeContext from cwltool.main import main from cwltool.mpi import MpiConfig, MPIRequirementName @@ -292,7 +293,14 @@ def schema_ext11() -> Generator[Names, None, None]: mpiReq = CommentedMap({"class": MPIRequirementName, "processes": 1}) containerReq = CommentedMap({"class": "DockerRequirement"}) -basetool = CommentedMap({"cwlVersion": "v1.1", "inputs": CommentedSeq(), "outputs": CommentedSeq()}) +basetool = CommentedMap( + { + "cwlVersion": "v1.1", + "class": "CommandLineTool", + "inputs": CommentedSeq(), + "outputs": CommentedSeq(), + } +) def mk_tool( @@ -300,7 +308,7 @@ def mk_tool( opts: list[str], reqs: Optional[list[CommentedMap]] = None, hints: Optional[list[CommentedMap]] = None, -) -> tuple[LoadingContext, RuntimeContext, CommentedMap]: +) -> tuple[RuntimeContext, CommandLineTool]: tool = basetool.copy() if reqs is not None: @@ -310,53 +318,57 @@ def mk_tool( args = cwltool.argparser.arg_parser().parse_args(opts) args.enable_ext = True + args.basedir = os.path.dirname(os.path.abspath(".")) rc = RuntimeContext(vars(args)) lc = cwltool.main.setup_loadingContext(None, rc, args) lc.avsc_names = schema - return lc, rc, tool + tool["id"] = file_uri(os.path.abspath("./mktool.cwl")) + assert lc.loader is not None + lc.loader.idx[tool["id"]] = tool + return rc, cast(CommandLineTool, cwltool.load_tool.load_tool(tool, lc)) def test_singularity(schema_ext11: Names) -> None: - lc, rc, tool = mk_tool(schema_ext11, ["--singularity"], reqs=[mpiReq, containerReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, ["--singularity"], reqs=[mpiReq, containerReq]) + clt._init_job({}, rc) jr = clt.make_job_runner(rc) assert jr is cwltool.singularity.SingularityCommandLineJob def test_udocker(schema_ext11: Names) -> None: - lc, rc, tool = mk_tool(schema_ext11, ["--udocker"], reqs=[mpiReq, containerReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, ["--udocker"], reqs=[mpiReq, containerReq]) + clt._init_job({}, rc) jr = clt.make_job_runner(rc) assert jr is cwltool.udocker.UDockerCommandLineJob def test_docker_hint(schema_ext11: Names) -> None: # Docker hint, MPI required - lc, rc, tool = mk_tool(schema_ext11, [], hints=[containerReq], reqs=[mpiReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, [], hints=[containerReq], reqs=[mpiReq]) + clt._init_job({}, rc) jr = clt.make_job_runner(rc) assert jr is cwltool.job.CommandLineJob def test_docker_required(schema_ext11: Names) -> None: # Docker required, MPI hinted - lc, rc, tool = mk_tool(schema_ext11, [], reqs=[containerReq], hints=[mpiReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, [], reqs=[containerReq], hints=[mpiReq]) + clt._init_job({}, rc) jr = clt.make_job_runner(rc) assert jr is cwltool.docker.DockerCommandLineJob def test_docker_mpi_both_required(schema_ext11: Names) -> None: # Both required - error - lc, rc, tool = mk_tool(schema_ext11, [], reqs=[mpiReq, containerReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, [], reqs=[mpiReq, containerReq]) with pytest.raises(cwltool.errors.UnsupportedRequirement): - clt.make_job_runner(rc) + clt._init_job({}, rc) + clt.make_job_runner(rc) def test_docker_mpi_both_hinted(schema_ext11: Names) -> None: # Both hinted - error - lc, rc, tool = mk_tool(schema_ext11, [], hints=[mpiReq, containerReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, [], hints=[mpiReq, containerReq]) with pytest.raises(cwltool.errors.UnsupportedRequirement): - clt.make_job_runner(rc) + clt._init_job({}, rc) + clt.make_job_runner(rc) From 2dce710246e091f0189fab41b589ee062ee94500 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Sat, 1 Mar 2025 13:39:09 +0100 Subject: [PATCH 60/78] Fix DeprecationWarning: Nesting argument groups is deprecated. This was making the pytest output very noisy. Also update the help test as the `--cid-*` options work for both the Docker and Podman container engines. --- cwltool/argparser.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cwltool/argparser.py b/cwltool/argparser.py index 3622b627f..3f07cea9d 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -415,9 +415,7 @@ def arg_parser() -> argparse.ArgumentParser: dest="rm_container", ) - cidgroup = container_group.add_argument_group( - "Recording the Docker container identifier into a file" - ) + cidgroup = parser.add_argument_group("Recording the software container identifier into a file") cidgroup.add_argument( # Disabled as containerid is now saved by default "--record-container-id", @@ -430,7 +428,7 @@ def arg_parser() -> argparse.ArgumentParser: cidgroup.add_argument( "--cidfile-dir", type=str, - help="Store the Docker container ID into a file in the specified directory.", + help="Store the software container ID into a file in the specified directory.", default=None, dest="cidfile_dir", ) @@ -438,7 +436,7 @@ def arg_parser() -> argparse.ArgumentParser: cidgroup.add_argument( "--cidfile-prefix", type=str, - help="Specify a prefix to the container ID filename. " + help="Specify a prefix to the software container ID filename. " "Final file name will be followed by a timestamp. " "The default is no prefix.", default=None, From 2f8c9c78aab9e15a9341ada13cf583be039c39fd Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 8 Apr 2025 14:12:23 +0200 Subject: [PATCH 61/78] lint: remove unneeded global statements --- cwltool/main.py | 1 - cwltool/process.py | 1 - 2 files changed, 2 deletions(-) diff --git a/cwltool/main.py b/cwltool/main.py index 90cb2e2c8..b317ff27a 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -119,7 +119,6 @@ def _terminate_processes() -> None: continuing to execute while it kills the processes that they've spawned. This may occasionally lead to unexpected behaviour. """ - global docker_exe # It's possible that another thread will spawn a new task while # we're executing, so it's not safe to use a for loop here. while processes_to_kill: diff --git a/cwltool/process.py b/cwltool/process.py index 6d1fc7ee7..bc2aaf145 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -1109,7 +1109,6 @@ def __str__(self) -> str: def uniquename(stem: str, names: Optional[set[str]] = None) -> str: """Construct a thread-unique name using the given stem as a prefix.""" - global _names if names is None: names = _names c = 1 From f94719e862f86cc88600caf3628faba6c0d05042 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 8 Apr 2025 12:40:15 +0200 Subject: [PATCH 62/78] ProcessGenerator: allow for pickling --- pyproject.toml | 2 +- requirements.txt | 2 +- setup.py | 3 ++- tests/test_subclass_mypyc.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cb7d837a7..35db6cf42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ requires = [ "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", "ruamel.yaml>=0.16.0,<0.19", - "schema-salad>=8.7,<9", + "schema-salad>=8.9,<9", "cwl-utils>=0.32", "toml", "argcomplete>=1.12.0", diff --git a/requirements.txt b/requirements.txt index 3cbcf0027..621aa968a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ requests>=2.6.1 ruamel.yaml>=0.16.0,<0.19 rdflib>=4.2.2,<7.2 -schema-salad>=8.7,<9 +schema-salad>=8.9,<9 prov==1.5.1 mypy-extensions psutil>=5.6.6 diff --git a/setup.py b/setup.py index 4aed7320b..57eb06407 100644 --- a/setup.py +++ b/setup.py @@ -74,6 +74,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li "cwlprov/__init__.py", "cuda.py", # for monkeypatch "run_job.py", + "procgenerator.py", # for ProcessGenerator "cwlprov/writablebagfile.py", # WritableBag is having issues "stdfsaccess.py", # StdFsAccess needs to be subclassable ) @@ -150,7 +151,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li # https://github.com/ionrock/cachecontrol/issues/137 "ruamel.yaml >= 0.16, < 0.19", "rdflib >= 4.2.2, < 7.2.0", - "schema-salad >= 8.7, < 9", + "schema-salad >= 8.9, < 9", "prov == 1.5.1", "mypy-extensions", "psutil >= 5.6.6", diff --git a/tests/test_subclass_mypyc.py b/tests/test_subclass_mypyc.py index aa4964b34..14cdc9dc6 100644 --- a/tests/test_subclass_mypyc.py +++ b/tests/test_subclass_mypyc.py @@ -13,11 +13,13 @@ from cwltool.builder import Builder from cwltool.command_line_tool import CommandLineTool, ExpressionTool from cwltool.context import LoadingContext, RuntimeContext +from cwltool.procgenerator import ProcessGenerator from cwltool.stdfsaccess import StdFsAccess from cwltool.update import INTERNAL_VERSION from cwltool.workflow import Workflow from .test_anon_types import snippet +from .util import get_data @pytest.mark.parametrize("snippet", snippet) @@ -52,6 +54,35 @@ def test_pickle_unpickle_workflow(snippet: CommentedMap) -> None: assert pickle.loads(stream) +def test_pickle_processgenerator() -> None: + """We can pickle ProcessGenerator.""" + + pytoolgen = get_data("tests/wf/generator/pytoolgen.cwl") + a = ProcessGenerator( + CommentedMap( + { + "cwlVersion": "v1.0", + "id": f"file://{pytoolgen}", + "$namespaces": {"cwltool": "http://commonwl.org/cwltool#"}, + "class": "cwltool:ProcessGenerator", + "inputs": {}, + "outputs": {}, + "run": { + "id": f"file://{pytoolgen}#f9f617af-3088-4ade-bbf3-acd1d6f51355", + "cwlVersion": "v1.0", + "class": "CommandLineTool", + "inputs": {}, + "outputs": {}, + "baseCommand": "echo", + }, + }, + ), + LoadingContext(), + ) + + assert pickle.dumps(a) + + def test_serialize_builder() -> None: """We can pickle Builder.""" runtime_context = RuntimeContext() From 346f42118f820a82e4ec742673dc68bbc8838fd6 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 19 May 2025 14:12:41 -0400 Subject: [PATCH 63/78] Step name callback (#2109) --- build-cwltool-docker.sh | 2 +- cwltool/context.py | 5 ++++ cwltool/workflow_job.py | 7 ++++- tests/test_context.py | 64 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 75 insertions(+), 3 deletions(-) diff --git a/build-cwltool-docker.sh b/build-cwltool-docker.sh index 9f7163afb..3dfa7bb66 100755 --- a/build-cwltool-docker.sh +++ b/build-cwltool-docker.sh @@ -8,4 +8,4 @@ ${engine} run -t -v /var/run/docker.sock:/var/run/docker.sock \ -v /tmp:/tmp \ -v "$PWD":/tmp/cwltool \ quay.io/commonwl/cwltool_module /bin/sh -c \ - "apk add gcc bash git && pip install -r/tmp/cwltool/test-requirements.txt ; pytest -k 'not (test_bioconda or test_double_overwrite or test_env_filtering or test_biocontainers or test_disable_file_overwrite_without_ext or test_disable_file_creation_in_outdir_with_ext or test_write_write_conflict or test_directory_literal_with_real_inputs_inside or test_revsort_workflow or test_stdin_with_id_preset or test_no_compute_chcksum or test_packed_workflow_execution[tests/wf/count-lines1-wf.cwl-tests/wf/wc-job.json-False] or test_sequential_workflow or test_single_process_subwf_subwf_inline_step or test_cache_dockerreq_hint_instead_of_req)' --ignore-glob '*test_udocker.py' -n 0 -v -rs --pyargs cwltool" + "apk add gcc bash git && pip install -r/tmp/cwltool/test-requirements.txt ; pytest -k 'not (test_bioconda or test_double_overwrite or test_env_filtering or test_biocontainers or test_disable_file_overwrite_without_ext or test_disable_file_creation_in_outdir_with_ext or test_write_write_conflict or test_directory_literal_with_real_inputs_inside or test_revsort_workflow or test_stdin_with_id_preset or test_no_compute_chcksum or test_packed_workflow_execution[tests/wf/count-lines1-wf.cwl-tests/wf/wc-job.json-False] or test_sequential_workflow or test_single_process_subwf_subwf_inline_step or test_cache_dockerreq_hint_instead_of_req or test_workflow_job_step_name_callback)' --ignore-glob '*test_udocker.py' -n 0 -v -rs --pyargs cwltool" diff --git a/cwltool/context.py b/cwltool/context.py index bb281fd88..83dd5a190 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -29,6 +29,7 @@ from .process import Process from .secrets import SecretStore from .software_requirements import DependenciesConfiguration + from .workflow_job import WorkflowJobStep class ContextBase: @@ -191,6 +192,10 @@ def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: self.default_stderr: Optional[Union[IO[bytes], TextIO]] = None self.validate_only: bool = False self.validate_stdout: Optional["SupportsWrite[str]"] = None + self.workflow_job_step_name_callback: Optional[ + Callable[[WorkflowJobStep, CWLObjectType], str] + ] = None + super().__init__(kwargs) if self.tmp_outdir_prefix == "": self.tmp_outdir_prefix = self.tmpdir_prefix diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index b552641e1..368d1cc02 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -60,7 +60,12 @@ def job( ) -> JobsGeneratorType: runtimeContext = runtimeContext.copy() runtimeContext.part_of = self.name - runtimeContext.name = shortname(self.id) + + if runtimeContext.workflow_job_step_name_callback is not None: + vfinputs = {shortname(k): v for k, v in joborder.items()} + runtimeContext.name = runtimeContext.workflow_job_step_name_callback(self, vfinputs) + else: + runtimeContext.name = shortname(self.id) _logger.info("[%s] start", self.name) diff --git a/tests/test_context.py b/tests/test_context.py index acb7dfbca..be1d89575 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,10 +1,16 @@ import subprocess import sys +import logging +from io import StringIO +from typing import MutableMapping, cast from cwltool.context import RuntimeContext from cwltool.factory import Factory +from cwltool.utils import CWLObjectType +from cwltool.workflow_job import WorkflowJobStep +from pathlib import Path -from .util import get_data +from .util import get_data, needs_docker def test_replace_default_stdout_stderr() -> None: @@ -26,3 +32,59 @@ def test_replace_default_stdout_stderr() -> None: assert echo(inp="foo") == {"out": "foo\n"} sys.stdout = original_stdout sys.stderr = original_stderr + + +@needs_docker +def test_workflow_job_step_name_callback() -> None: + """Test ability to hook custom workflow step naming""" + + stream = StringIO() + streamhandler = logging.StreamHandler(stream) + _logger = logging.getLogger("cwltool") + _logger.addHandler(streamhandler) + + try: + runtime_context = RuntimeContext() + + def step_name_hook(step: WorkflowJobStep, job: CWLObjectType) -> str: + j1 = cast(MutableMapping[str, CWLObjectType], job) + inp = cast(MutableMapping[str, str], j1.get("revtool_input", j1.get("sorted_input"))) + return "%s on %s" % ( + step.name, + inp.get("basename"), + ) + + runtime_context.workflow_job_step_name_callback = step_name_hook + + factory = Factory(None, None, runtime_context) + revsort = factory.make(get_data("tests/wf/revsort.cwl")) + + result = revsort( + workflow_input={ + "class": "File", + "location": Path(get_data("tests/wf/whale.txt")).as_uri(), + "format": "https://www.iana.org/assignments/media-types/text/plain", + } + ) + + result = cast(CWLObjectType, result) + + sorted_out = cast(MutableMapping[str, str], result["sorted_output"]) + loc = sorted_out["location"] + + assert result == { + "sorted_output": { + "basename": "output.txt", + "checksum": "sha1$b9214658cc453331b62c2282b772a5c063dbd284", + "class": "File", + "http://commonwl.org/cwltool#generation": 0, + "nameext": ".txt", + "nameroot": "output", + "size": 1111, + "location": loc, + }, + } + + assert "rev on whale.txt" in stream.getvalue() + finally: + _logger.removeHandler(streamhandler) From bdcc818b610d6229eacd32e04d25814d38e049b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 30 May 2025 07:23:14 +0000 Subject: [PATCH 64/78] Bump mypy from 1.15.0 to 1.16.0 Bumps [mypy](https://github.com/python/mypy) from 1.15.0 to 1.16.0. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.15.0...v1.16.0) --- updated-dependencies: - dependency-name: mypy dependency-version: 1.16.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- mypy-requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 65e517172..3dc19c4e3 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.15.0 # also update pyproject.toml +mypy==1.16.0 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 cwltest diff --git a/pyproject.toml b/pyproject.toml index 35db6cf42..222d23159 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.15.0", # also update mypy-requirements.txt + "mypy==1.16.0", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", From a9773724db43b78b579b5ce8b7ab0b67c83e1a21 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 30 May 2025 14:53:54 +0200 Subject: [PATCH 65/78] mypy 1.16 type simplifications --- cwltool/checker.py | 14 ++++++-------- cwltool/command_line_tool.py | 4 ++-- cwltool/cwlprov/ro.py | 4 ++-- cwltool/main.py | 7 +++++-- cwltool/pack.py | 8 ++++---- cwltool/process.py | 6 +++--- cwltool/secrets.py | 10 +++++----- cwltool/update.py | 2 +- setup.py | 2 +- tests/test_context.py | 9 +++++---- 10 files changed, 34 insertions(+), 32 deletions(-) diff --git a/cwltool/checker.py b/cwltool/checker.py index a3b8ba5df..b3637db20 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -54,7 +54,7 @@ def check_types( def merge_flatten_type(src: SinkType) -> CWLOutputType: """Return the merge flattened type of the source type.""" if isinstance(src, MutableSequence): - return [merge_flatten_type(cast(SinkType, t)) for t in src] + return [merge_flatten_type(t) for t in src] if isinstance(src, MutableMapping) and src.get("type") == "array": return src return {"items": src, "type": "array"} @@ -94,22 +94,20 @@ def can_assign_src_to_sink(src: SinkType, sink: Optional[SinkType], strict: bool if strict: return False return True - return can_assign_src_to_sink( - cast(SinkType, src["type"]), cast(Optional[SinkType], sink["type"]), strict - ) + return can_assign_src_to_sink(src["type"], sink["type"], strict) if isinstance(src, MutableSequence): if strict: for this_src in src: - if not can_assign_src_to_sink(cast(SinkType, this_src), sink): + if not can_assign_src_to_sink(this_src, sink): return False return True for this_src in src: - if this_src != "null" and can_assign_src_to_sink(cast(SinkType, this_src), sink): + if this_src != "null" and can_assign_src_to_sink(this_src, sink): return True return False if isinstance(sink, MutableSequence): for this_sink in sink: - if can_assign_src_to_sink(src, cast(SinkType, this_sink)): + if can_assign_src_to_sink(src, this_sink): return True return False return bool(src == sink) @@ -257,7 +255,7 @@ def static_checker( ) + "\n" + SourceLine(sink, "type").makeError( - " with sink '%s' of type %s" % (sink_name, json_dumps(sink["type"])) + " with sink '{}' of type {}".format(sink_name, json_dumps(sink["type"])) ) ) if linkMerge is not None: diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index cbff45bd0..eb7bf9e23 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -501,7 +501,7 @@ def _initialworkdir(self, j: Optional[JobBase], builder: Builder) -> None: if not isinstance(ls_evaluated, MutableSequence): fail = ls_evaluated else: - ls_evaluated2 = cast(MutableSequence[Union[None, CWLOutputType]], ls_evaluated) + ls_evaluated2 = ls_evaluated for entry in ls_evaluated2: if entry == None: # noqa if classic_dirent: @@ -1389,7 +1389,7 @@ def collect_output( "Multiple matches for output item that is a single file." ) else: - result = cast(CWLOutputType, result[0]) + result = result[0] if "secondaryFiles" in schema: with SourceLine(schema, "secondaryFiles", WorkflowException, debug): diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index be10d3d64..19334d2b8 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -674,7 +674,7 @@ def _relativise_files( for val in structure.values(): try: - self._relativise_files(cast(CWLOutputType, val)) + self._relativise_files(val) except OSError: pass return @@ -682,4 +682,4 @@ def _relativise_files( if isinstance(structure, MutableSequence): for obj in structure: # Recurse and rewrite any nested File objects - self._relativise_files(cast(CWLOutputType, obj)) + self._relativise_files(obj) diff --git a/cwltool/main.py b/cwltool/main.py index b317ff27a..f68d05052 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -304,7 +304,7 @@ def realize_input_schema( if isinstance(entry["type"], Mapping): entry["type"] = cast( CWLOutputType, - realize_input_schema([cast(CWLObjectType, entry["type"])], schema_defs), + realize_input_schema([entry["type"]], schema_defs), ) if entry["type"] == "array": items = entry["items"] if not isinstance(entry["items"], str) else [entry["items"]] @@ -373,7 +373,10 @@ def load_job_order( content_types=CWL_CONTENT_TYPES, ) - if job_order_object is not None and "http://commonwl.org/cwltool#overrides" in job_order_object: + if ( + isinstance(job_order_object, CommentedMap) + and "http://commonwl.org/cwltool#overrides" in job_order_object + ): ov_uri = file_uri(job_order_file or input_basedir) overrides_list.extend(resolve_overrides(job_order_object, ov_uri, tool_file_uri)) del job_order_object["http://commonwl.org/cwltool#overrides"] diff --git a/cwltool/pack.py b/cwltool/pack.py index d3705d5e4..6d96c83a1 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -32,7 +32,7 @@ def find_run( runs.add(d["run"]) find_run(loadref(None, d["run"]), loadref, runs) for s in d.values(): - find_run(s, loadref, runs) + find_run(cast(Union[CWLObjectType, ResolveType], s), loadref, runs) def find_ids( @@ -47,7 +47,7 @@ def find_ids( if i in d and isinstance(d[i], str): ids.add(cast(str, d[i])) for s2 in d.values(): - find_ids(cast(CWLOutputType, s2), ids) + find_ids(s2, ids) def replace_refs(d: Any, rewrite: dict[str, str], stem: str, newstem: str) -> None: @@ -84,7 +84,7 @@ def import_embed( ) -> None: if isinstance(d, MutableSequence): for v in d: - import_embed(cast(CWLOutputType, v), seen) + import_embed(v, seen) elif isinstance(d, MutableMapping): for n in ("id", "name"): if n in d: @@ -100,7 +100,7 @@ def import_embed( break for k in sorted(d.keys()): - import_embed(cast(CWLOutputType, d[k]), seen) + import_embed(d[k], seen) def pack( diff --git a/cwltool/process.py b/cwltool/process.py index bc2aaf145..ce0f52b73 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -444,7 +444,7 @@ def avroize_type( cast(MutableSequence[CWLOutputType], field_type["items"]), name_prefix ) else: - field_type["type"] = avroize_type(cast(CWLOutputType, field_type["type"]), name_prefix) + field_type["type"] = avroize_type(field_type["type"], name_prefix) elif field_type == "File": return "org.w3id.cwl.cwl.File" elif field_type == "Directory": @@ -490,10 +490,10 @@ def var_spool_cwl_detector( r = True elif isinstance(obj, MutableMapping): for mkey, mvalue in obj.items(): - r = var_spool_cwl_detector(cast(CWLOutputType, mvalue), obj, mkey) or r + r = var_spool_cwl_detector(mvalue, obj, mkey) or r elif isinstance(obj, MutableSequence): for lkey, lvalue in enumerate(obj): - r = var_spool_cwl_detector(cast(CWLOutputType, lvalue), obj, lkey) or r + r = var_spool_cwl_detector(lvalue, obj, lkey) or r return r diff --git a/cwltool/secrets.py b/cwltool/secrets.py index c73e0108c..3d54774e1 100644 --- a/cwltool/secrets.py +++ b/cwltool/secrets.py @@ -2,7 +2,7 @@ import uuid from collections.abc import MutableMapping, MutableSequence -from typing import Optional, cast +from typing import Optional from .utils import CWLObjectType, CWLOutputType @@ -43,11 +43,11 @@ def has_secret(self, value: CWLOutputType) -> bool: return True elif isinstance(value, MutableMapping): for this_value in value.values(): - if self.has_secret(cast(CWLOutputType, this_value)): + if self.has_secret(this_value): return True elif isinstance(value, MutableSequence): for this_value in value: - if self.has_secret(cast(CWLOutputType, this_value)): + if self.has_secret(this_value): return True return False @@ -58,7 +58,7 @@ def retrieve(self, value: CWLOutputType) -> CWLOutputType: value = value.replace(key, this_value) return value elif isinstance(value, MutableMapping): - return {k: self.retrieve(cast(CWLOutputType, v)) for k, v in value.items()} + return {k: self.retrieve(v) for k, v in value.items()} elif isinstance(value, MutableSequence): - return [self.retrieve(cast(CWLOutputType, v)) for v in value] + return [self.retrieve(v) for v in value] return value diff --git a/cwltool/update.py b/cwltool/update.py index 67e1f4257..290498031 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -132,7 +132,7 @@ def update_secondaryFiles( new_seq[index] = update_secondaryFiles(entry) return new_seq elif isinstance(t, MutableSequence): - return CommentedSeq([update_secondaryFiles(cast(CWLOutputType, p)) for p in t]) + return CommentedSeq([update_secondaryFiles(p) for p in t]) elif isinstance(t, MutableMapping): return cast(MutableMapping[str, str], t) elif top: diff --git a/setup.py b/setup.py index 57eb06407..e1e7510b8 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li mypyc_targets = [x for x in all_real_pys if x not in mypyc_skiplist] # Strip out any test code - mypyc_targets = [x for x in mypyc_targets if not x.startswith(("tests" + os.sep))] + mypyc_targets = [x for x in mypyc_targets if not x.startswith("tests" + os.sep)] mypyc_targets.sort() diff --git a/tests/test_context.py b/tests/test_context.py index be1d89575..5f3666ce9 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,14 +1,15 @@ +import logging import subprocess import sys -import logging from io import StringIO -from typing import MutableMapping, cast +from pathlib import Path +from typing import cast +from collections.abc import MutableMapping from cwltool.context import RuntimeContext from cwltool.factory import Factory from cwltool.utils import CWLObjectType from cwltool.workflow_job import WorkflowJobStep -from pathlib import Path from .util import get_data, needs_docker @@ -49,7 +50,7 @@ def test_workflow_job_step_name_callback() -> None: def step_name_hook(step: WorkflowJobStep, job: CWLObjectType) -> str: j1 = cast(MutableMapping[str, CWLObjectType], job) inp = cast(MutableMapping[str, str], j1.get("revtool_input", j1.get("sorted_input"))) - return "%s on %s" % ( + return "{} on {}".format( step.name, inp.get("basename"), ) From 8f31b737249f094599592b4373a2eeb77f2b883c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Apr 2025 08:29:36 +0000 Subject: [PATCH 66/78] Bump pypa/cibuildwheel from 2.22.0 to 2.23.3 Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.22.0 to 2.23.3. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.22.0...v2.23.3) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-version: 2.23.3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index ed10cd17d..6f8547b71 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -43,7 +43,7 @@ jobs: # platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.22.0 + uses: pypa/cibuildwheel@v2.23.3 env: CIBW_BUILD: ${{ matrix.build }} CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} @@ -99,7 +99,7 @@ jobs: ref: ${{ github.event.client_payload.ref }} - name: Build wheels - uses: pypa/cibuildwheel@v2.22.0 + uses: pypa/cibuildwheel@v2.23.3 - uses: actions/upload-artifact@v4 with: From a2fbed450fb38d669d462ed66b13754743251ef6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Apr 2025 07:48:06 +0000 Subject: [PATCH 67/78] Bump cibuildwheel from 2.22.0 to 2.23.3 Bumps [cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.22.0 to 2.23.3. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.22...v2.23.3) --- updated-dependencies: - dependency-name: cibuildwheel dependency-version: 2.23.3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- cibw-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibw-requirements.txt b/cibw-requirements.txt index 833aca23d..0e314b8bf 100644 --- a/cibw-requirements.txt +++ b/cibw-requirements.txt @@ -1 +1 @@ -cibuildwheel==2.22.0 +cibuildwheel==2.23.3 From 0418dabeecf11358bbbadac2a2101235116f6b1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Jun 2025 08:04:52 +0000 Subject: [PATCH 68/78] Update pytest requirement from <8.4,>=6.2 to >=6.2,<8.5 Updates the requirements on [pytest](https://github.com/pytest-dev/pytest) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.2.0...8.4.0) --- updated-dependencies: - dependency-name: pytest dependency-version: 8.4.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e1e7510b8..960b26cc5 100644 --- a/setup.py +++ b/setup.py @@ -175,7 +175,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li test_suite="tests", tests_require=[ "bagit >= 1.6.4, < 1.9", - "pytest >= 6.2, < 8.4", + "pytest >= 6.2, < 8.5", "mock >= 2.0.0", "pytest-mock >= 1.10.0", "pytest-httpserver", diff --git a/test-requirements.txt b/test-requirements.txt index c5ca6ef73..e2d83d4fe 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,5 @@ bagit>=1.6.4,<1.9 -pytest>= 6.2,< 8.4 +pytest>= 6.2,< 8.5 pytest-xdist>=3.2.0 # for the worksteal scheduler psutil # enhances pytest-xdist to allow "-n logical" pytest-httpserver From 6976e99c6ef2d9ee9f4d60a07582f7a847e73768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz?= Date: Tue, 10 Jun 2025 11:16:48 +0200 Subject: [PATCH 69/78] Allow for newer pydot version (#2019) Co-authored-by: "FeRD (Frank Dana)" Co-authored-by: Michael R. Crusoe --- cwltool/cwlviewer.py | 20 ++++++++++++++------ mypy-stubs/pydot.pyi | 4 ++-- requirements.txt | 2 +- setup.py | 2 +- tests/test_context.py | 2 +- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/cwltool/cwlviewer.py b/cwltool/cwlviewer.py index 36166c485..92e494dde 100644 --- a/cwltool/cwlviewer.py +++ b/cwltool/cwlviewer.py @@ -99,8 +99,8 @@ def _set_inner_edges(self) -> None: self._dot_graph.add_node(n) self._dot_graph.add_edge( pydot.Edge( - str(inner_edge_row["source_step"]), - str(inner_edge_row["target_step"]), + pydot.quote_id_if_necessary(str(inner_edge_row["source_step"])), + pydot.quote_id_if_necessary(str(inner_edge_row["target_step"])), ) ) @@ -109,7 +109,6 @@ def _set_input_edges(self) -> None: inputs_subgraph = pydot.Subgraph(graph_name="cluster_inputs") self._dot_graph.add_subgraph(inputs_subgraph) inputs_subgraph.set("rank", "same") - inputs_subgraph.create_attribute_methods(["style"]) inputs_subgraph.set("style", "dashed") inputs_subgraph.set("label", "Workflow Inputs") @@ -129,14 +128,18 @@ def _set_input_edges(self) -> None: ) n.set_name(str(input_row["input"])) inputs_subgraph.add_node(n) - self._dot_graph.add_edge(pydot.Edge(str(input_row["input"]), str(input_row["step"]))) + self._dot_graph.add_edge( + pydot.Edge( + pydot.quote_id_if_necessary(str(input_row["input"])), + pydot.quote_id_if_necessary(str(input_row["step"])), + ) + ) def _set_output_edges(self) -> None: get_output_edges = _get_output_edges_query() outputs_graph = pydot.Subgraph(graph_name="cluster_outputs") self._dot_graph.add_subgraph(outputs_graph) outputs_graph.set("rank", "same") - outputs_graph.create_attribute_methods(["style"]) outputs_graph.set("style", "dashed") outputs_graph.set("label", "Workflow Outputs") outputs_graph.set("labelloc", "b") @@ -156,7 +159,12 @@ def _set_output_edges(self) -> None: ) n.set_name(str(output_edge_row["output"])) outputs_graph.add_node(n) - self._dot_graph.add_edge(pydot.Edge(output_edge_row["step"], output_edge_row["output"])) + self._dot_graph.add_edge( + pydot.Edge( + pydot.quote_id_if_necessary(output_edge_row["step"]), + pydot.quote_id_if_necessary(output_edge_row["output"]), + ) + ) def _get_root_graph_uri(self) -> rdflib.term.Identifier: get_root_query = _get_root_query() diff --git a/mypy-stubs/pydot.pyi b/mypy-stubs/pydot.pyi index bd0ab3147..ecf3a453b 100644 --- a/mypy-stubs/pydot.pyi +++ b/mypy-stubs/pydot.pyi @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Sequence, Union +from typing import Any, Dict, List, Optional, Sequence, Union PY3: Any str_type = str @@ -12,6 +12,7 @@ def is_windows() -> bool: ... def is_anaconda() -> bool: ... def get_executable_extension() -> str: ... def graph_from_dot_data(s: str) -> List["Dot"]: ... +def quote_id_if_necessary(s: str, unquoted_keywords: Optional[Sequence[str]] = None) -> str: ... class Common: def set_parent_graph(self, parent_graph: "Graph") -> None: ... @@ -21,7 +22,6 @@ class Common: def get_attributes(self) -> Dict[str, str]: ... def set_sequence(self, seq: str) -> None: ... def get_sequence(self) -> str: ... - def create_attribute_methods(self, obj_attributes: List[str]) -> None: ... class Error(Exception): value: Any diff --git a/requirements.txt b/requirements.txt index 621aa968a..8c36ccddb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ mypy-extensions psutil>=5.6.6 importlib_resources>=1.4;python_version<'3.9' coloredlogs -pydot>=1.4.1,<3 +pydot>=1.4.1 argcomplete>=1.12.0 pyparsing!=3.0.2 # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 cwl-utils>=0.32 diff --git a/setup.py b/setup.py index 960b26cc5..02a648095 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li "mypy-extensions", "psutil >= 5.6.6", "coloredlogs", - "pydot >= 1.4.1, <3", + "pydot >= 1.4.1", "argcomplete >= 1.12.0", "pyparsing != 3.0.2", # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 "cwl-utils >= 0.32", diff --git a/tests/test_context.py b/tests/test_context.py index 5f3666ce9..505dfd635 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,10 +1,10 @@ import logging import subprocess import sys +from collections.abc import MutableMapping from io import StringIO from pathlib import Path from typing import cast -from collections.abc import MutableMapping from cwltool.context import RuntimeContext from cwltool.factory import Factory From 734da5f557ccebbbe7c6c0f4068ab022e37d402a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Jun 2025 18:37:53 +0200 Subject: [PATCH 70/78] Bump mypy from 1.16.0 to 1.16.1 (#2125) Bumps [mypy](https://github.com/python/mypy) from 1.16.0 to 1.16.1. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.16.0...v1.16.1) --- updated-dependencies: - dependency-name: mypy dependency-version: 1.16.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mypy-requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 3dc19c4e3..b9352492c 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.16.0 # also update pyproject.toml +mypy==1.16.1 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 cwltest diff --git a/pyproject.toml b/pyproject.toml index 222d23159..047d56ca4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.16.0", # also update mypy-requirements.txt + "mypy==1.16.1", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", From c7fd7cc415379d7b723234f7327d753d11c5e4b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Jun 2025 13:02:20 +0200 Subject: [PATCH 71/78] Update bagit requirement from <1.9,>=1.6.4 to >=1.6.4,<1.10 (#2123) Updates the requirements on [bagit](https://github.com/LibraryOfCongress/bagit-python) to permit the latest version. - [Release notes](https://github.com/LibraryOfCongress/bagit-python/releases) - [Commits](https://github.com/LibraryOfCongress/bagit-python/compare/v1.6.4...v1.9.0) --- updated-dependencies: - dependency-name: bagit dependency-version: 1.9.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 02a648095..d37daaf6c 100644 --- a/setup.py +++ b/setup.py @@ -174,7 +174,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li setup_requires=PYTEST_RUNNER + ["setuptools_scm>=8.0.4,<9"], test_suite="tests", tests_require=[ - "bagit >= 1.6.4, < 1.9", + "bagit >= 1.6.4, < 1.10", "pytest >= 6.2, < 8.5", "mock >= 2.0.0", "pytest-mock >= 1.10.0", diff --git a/test-requirements.txt b/test-requirements.txt index e2d83d4fe..f9b61511b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,4 @@ -bagit>=1.6.4,<1.9 +bagit>=1.6.4,<1.10 pytest>= 6.2,< 8.5 pytest-xdist>=3.2.0 # for the worksteal scheduler psutil # enhances pytest-xdist to allow "-n logical" From be42e9ce0a78d5fb6930f4fb6d23ed27dbd6b9de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Jun 2025 13:03:11 +0200 Subject: [PATCH 72/78] Bump pypa/cibuildwheel from 2.23.3 to 3.0.0 (#2122) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.23.3 to 3.0.0. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.23.3...v3.0.0) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-version: 3.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 6f8547b71..8ba8bff93 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -43,7 +43,7 @@ jobs: # platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.23.3 + uses: pypa/cibuildwheel@v3.0.0 env: CIBW_BUILD: ${{ matrix.build }} CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} @@ -99,7 +99,7 @@ jobs: ref: ${{ github.event.client_payload.ref }} - name: Build wheels - uses: pypa/cibuildwheel@v2.23.3 + uses: pypa/cibuildwheel@v3.0.0 - uses: actions/upload-artifact@v4 with: From db503403a93dc6d4b540052864368f9cab3854cf Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 18 Jun 2025 13:27:14 +0200 Subject: [PATCH 73/78] cibuildwheel 3.x: pypy is now opt-in, no need to skip --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 047d56ca4..a76d027a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,8 +22,6 @@ write_to = "cwltool/_version.py" test-command = "python -m pytest --ignore cwltool/schemas -n logical --dist worksteal --junitxml={project}/test-results/junit_$(python -V | awk '{print $2}')_${AUDITWHEEL_PLAT}.xml -k 'not (test_bioconda or test_env_filtering or test_udocker)' --pyargs cwltool" test-requires = "-r test-requirements.txt" test-extras = "deps" -skip = "pp*" -# ^ skip building wheels on PyPy (any version) build-verbosity = 1 environment = { CWLTOOL_USE_MYPYC="1", MYPYPATH="$(pwd)/mypy-stubs" } From 5aae4c59ba9e1c160293dd1472d5ab98088aaa7f Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 18 Jun 2025 13:31:13 +0200 Subject: [PATCH 74/78] adjust dependencies given that Python 3.9 is the minimum version --- docs/requirements.txt | 1 - pyproject.toml | 1 - requirements.txt | 1 - test-requirements.txt | 2 +- 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index fd3033b91..e9d89216c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,4 +3,3 @@ sphinx-rtd-theme==3.0.2 sphinx-autoapi sphinx-autodoc-typehints sphinxcontrib-autoprogram -importlib_resources;python_version<'3.9' diff --git a/pyproject.toml b/pyproject.toml index a76d027a0..8667e8fbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,6 @@ requires = [ "mypy==1.16.1", # also update mypy-requirements.txt "types-requests", "types-psutil", - "importlib_resources>=1.4;python_version<'3.9'", "ruamel.yaml>=0.16.0,<0.19", "schema-salad>=8.9,<9", "cwl-utils>=0.32", diff --git a/requirements.txt b/requirements.txt index 8c36ccddb..47fafda8f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,6 @@ schema-salad>=8.9,<9 prov==1.5.1 mypy-extensions psutil>=5.6.6 -importlib_resources>=1.4;python_version<'3.9' coloredlogs pydot>=1.4.1 argcomplete>=1.12.0 diff --git a/test-requirements.txt b/test-requirements.txt index f9b61511b..7ca4d45be 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,7 +3,7 @@ pytest>= 6.2,< 8.5 pytest-xdist>=3.2.0 # for the worksteal scheduler psutil # enhances pytest-xdist to allow "-n logical" pytest-httpserver -pytest-retry;python_version>='3.9' +pytest-retry mock>=2.0.0 pytest-mock>=1.10.0 pytest-cov From 8e9d7fe42e675a14993f8ad13ef7d3d59e6cdd5f Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 18 Jun 2025 13:51:49 +0200 Subject: [PATCH 75/78] cibuildwheel musllinux: pin to older image https://github.com/pypa/manylinux/issues/1795 --- .circleci/config.yml | 2 +- .github/workflows/wheels.yml | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fce789feb..eb16ee3de 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,4 +98,4 @@ workflows: tags: only: /.*/ build: "*musllinux*" - image: quay.io/pypa/musllinux_1_2_aarch64 + image: quay.io/pypa/musllinux_1_2_aarch64:2025.02.02-1 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 8ba8bff93..534d9b2c8 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -22,7 +22,7 @@ jobs: include: - image: manylinux_2_28_x86_64 build: "*manylinux*" - - image: musllinux_1_2_x86_64 + - image: musllinux_1_2_x86_64:2025.02.02-1 build: "*musllinux*" steps: @@ -53,9 +53,13 @@ jobs: # Linux arm64 wheels are built on circleci CIBW_ARCHS_LINUX: auto64 # ppc64le s390x + + - id: image_name_fix + run: | + image=${{ matrix.image }}; echo "fixed_image_name=${image/:/_}" >>${GITHUB_OUTPUT} - uses: actions/upload-artifact@v4 with: - name: artifact-${{ matrix.image }} + name: artifact-${{steps.image_name_fix.outputs.fixed_image_name}} path: ./wheelhouse/*.whl build_sdist: From b201191a2758d4be24e7041c635f863def483bd9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Jun 2025 09:58:50 +0000 Subject: [PATCH 76/78] Update galaxy-tool-util requirement Updates the requirements on [galaxy-tool-util](https://github.com/galaxyproject/galaxy) to permit the latest version. - [Release notes](https://github.com/galaxyproject/galaxy/releases) - [Commits](https://github.com/galaxyproject/galaxy/compare/galaxy-tool-util-22.1.3...v25.0.0) --- updated-dependencies: - dependency-name: galaxy-tool-util dependency-version: 25.0.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- mypy-requirements.txt | 2 +- setup.py | 2 +- test-requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index b9352492c..b9035581f 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -6,5 +6,5 @@ types-requests types-setuptools types-psutil types-mock -galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.3 +galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<25.1 galaxy-util<24.3 diff --git a/setup.py b/setup.py index d37daaf6c..4376ef88b 100644 --- a/setup.py +++ b/setup.py @@ -165,7 +165,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li ], extras_require={ "deps": [ - "galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.3", + "galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<25.1", "galaxy-util <24.3", ], }, diff --git a/test-requirements.txt b/test-requirements.txt index 7ca4d45be..9c7d5cf93 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -9,5 +9,5 @@ pytest-mock>=1.10.0 pytest-cov arcp>=0.2.0 -r requirements.txt -galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.3 +galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<25.1 galaxy-util<24.3 From df4b71e96d80c40bcfca0ca908fadb838db407c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Jun 2025 09:54:24 +0000 Subject: [PATCH 77/78] Update galaxy-util requirement from <24.3 to <25.1 Updates the requirements on [galaxy-util](https://github.com/galaxyproject/galaxy) to permit the latest version. - [Release notes](https://github.com/galaxyproject/galaxy/releases) - [Commits](https://github.com/galaxyproject/galaxy/compare/galaxy-util-19.9.0...v25.0.0) --- updated-dependencies: - dependency-name: galaxy-util dependency-version: 25.0.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- mypy-requirements.txt | 2 +- setup.py | 2 +- test-requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index b9035581f..3523889d6 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -7,4 +7,4 @@ types-setuptools types-psutil types-mock galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<25.1 -galaxy-util<24.3 +galaxy-util<25.1 diff --git a/setup.py b/setup.py index 4376ef88b..4e9494193 100644 --- a/setup.py +++ b/setup.py @@ -166,7 +166,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li extras_require={ "deps": [ "galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<25.1", - "galaxy-util <24.3", + "galaxy-util <25.1", ], }, python_requires=">=3.9, <3.14", diff --git a/test-requirements.txt b/test-requirements.txt index 9c7d5cf93..42565f630 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,4 +10,4 @@ pytest-cov arcp>=0.2.0 -r requirements.txt galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<25.1 -galaxy-util<24.3 +galaxy-util<25.1 From dbcec2a6c856e219c5cb9c05494c4ad5b14ff202 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 19 Jun 2025 17:01:34 +0200 Subject: [PATCH 78/78] workaround https://github.com/galaxyproject/galaxy/pull/20525 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 4e9494193..8f8098f0a 100644 --- a/setup.py +++ b/setup.py @@ -167,6 +167,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li "deps": [ "galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<25.1", "galaxy-util <25.1", + "pillow", # workaround for https://github.com/galaxyproject/galaxy/pull/20525 ], }, python_requires=">=3.9, <3.14",