From 798bc322b3472077ca7d90f0b035cf06a0521653 Mon Sep 17 00:00:00 2001 From: nvuillam Date: Tue, 11 Aug 2026 23:59:07 +0200 Subject: [PATCH 1/2] Never write inside the analyzed sources, only in REPORT_OUTPUT_FOLDER Four linters generated a temporary ignore file at the root of the linted repository and deleted it after their run (secretlint, sqlfluff, cljstyle, coffeelint). A file appearing then disappearing there aborts the project-mode linters walking the tree at the same moment: trivy fails the whole run with "walk dir error: unknown error with .megalinter-secretlintignore: no such file or directory". Being a race, it turns random builds red. MegaLinter now writes only in REPORT_OUTPUT_FOLDER: - secretlint uses the generic ignore-file forwarding, whose generated file already lands in the report folder - sqlfluff receives excluded directories through the ignore_paths key of a generated configuration copied to the report folder, its user configuration being preserved - cljstyle uses its native repeatable --ignore argument, which adds patterns to the ignore set of the user .cljstyle instead of replacing it, so the custom CljstyleLinter class is not needed anymore - coffeelint is disabled: CoffeeScript tooling is discontinued and the tool has no exclusion option at all, reading .coffeelintignore from its working directory only The cli_lint_mode_project_exclude_workspace_file_name property and the write_workspace_generated_file helper are removed, including from the descriptor JSON schema so that a descriptor reintroducing a write in the sources fails build validation. REPORT_OUTPUT_FOLDER is also always excluded now, even when EXCLUDED_DIRECTORIES is overridden, and it is forwarded to project-mode linters even when it does not exist yet when their command line is built: reporters keep writing into it while linters run. --- .automation/generated/linters_matrix.json | 2 - .claude/agents/descriptor-expert.md | 1 - .claude/rules/descriptors.md | 7 +- .claude/skills/add-linter/SKILL.md | 2 +- CHANGELOG.md | 6 ++ Dockerfile | 4 - flavors/dotnetweb/Dockerfile | 4 - flavors/dotnetweb/flavor.json | 1 - flavors/javascript/Dockerfile | 4 - flavors/javascript/flavor.json | 1 - megalinter/Linter.py | 67 ++++------------ megalinter/descriptors/all_flavors.json | 2 - .../clojure.megalinter-descriptor.yml | 6 +- .../coffee.megalinter-descriptor.yml | 6 +- .../repository.megalinter-descriptor.yml | 1 - .../megalinter-descriptor.jsonschema.json | 10 --- .../descriptors/sql.megalinter-descriptor.yml | 1 - megalinter/linters/CljstyleLinter.py | 25 ------ megalinter/linters/SqlFluffLinter.py | 51 ++++++++++++ .../tests/test_megalinter/linter_test.py | 30 +++++++ .../test_megalinter/linter_timeout_test.py | 1 - .../test_megalinter/sqlfluff_linter_test.py | 79 +++++++++++++++++++ .../tests/test_megalinter/utils_test.py | 24 +++++- megalinter/utils.py | 10 ++- 24 files changed, 225 insertions(+), 120 deletions(-) delete mode 100644 megalinter/linters/CljstyleLinter.py create mode 100644 megalinter/tests/test_megalinter/sqlfluff_linter_test.py diff --git a/.automation/generated/linters_matrix.json b/.automation/generated/linters_matrix.json index 7f0f8c7ee3b..f9491750fb1 100644 --- a/.automation/generated/linters_matrix.json +++ b/.automation/generated/linters_matrix.json @@ -15,7 +15,6 @@ "clojure_clj_kondo", "clojure_cljstyle", "cloudformation_cfn_lint", - "coffee_coffeelint", "copypaste_jscpd", "cpp_cppcheck", "cpp_cpplint", @@ -145,7 +144,6 @@ "c_cpplint", "c_clang_format", "cloudformation_cfn_lint", - "coffee_coffeelint", "cpp_cppcheck", "cpp_cpplint", "cpp_clang_format", diff --git a/.claude/agents/descriptor-expert.md b/.claude/agents/descriptor-expert.md index 3a1cc68be91..797160823a3 100644 --- a/.claude/agents/descriptor-expert.md +++ b/.claude/agents/descriptor-expert.md @@ -97,7 +97,6 @@ Fill in as many as applicable when creating or reviewing a descriptor: | `cli_lint_mode_project_exclude_seed_values` | Defaults re-included when the flag REPLACES the tool's built-in defaults (bandit `-x`, devskim `-g`) | | `cli_lint_mode_project_exclude_config_key` | Dotted key of the resolved config list the flag REPLACES (checkov `skip-path`, trivy `scan.skip-dirs`) — entries re-emitted first | | `cli_lint_mode_project_exclude_ignore_file_arg_name` | Flag receiving a generated ignore file (prettier/markdownlint `--ignore-path`); with `_seed_files` (workspace files merged in, first existing wins), `_pass_existing` (files re-passed when the flag replaces their discovery) | -| `cli_lint_mode_project_exclude_workspace_file_name` | Generated ignore file written at workspace root only if absent, removed after run (sqlfluff, coffeelint, secretlint) — for tools that only discover ignore files inside the repo | | `ignore_file_name` | Ignore file (e.g., `.eslintignore`) | | `cli_lint_ignore_arg_name` | Ignore file argument | | `cli_version_arg_name` | Version arg if not `--version` | diff --git a/.claude/rules/descriptors.md b/.claude/rules/descriptors.md index 1c125a8415b..22d544dcbb8 100644 --- a/.claude/rules/descriptors.md +++ b/.claude/rules/descriptors.md @@ -59,8 +59,6 @@ cli_lint_mode_project_exclude_seed_values: ["**/.git/**"] # defaults to re-in cli_lint_mode_project_exclude_ignore_file_arg_name: "--ignore-path" # argument receiving the generated ignore file cli_lint_mode_project_exclude_ignore_file_seed_files: [".toolignore"] # workspace files merged into it (first existing wins) cli_lint_mode_project_exclude_ignore_file_pass_existing: [".gitignore"] # files re-passed via the same arg when it replaces their discovery -# When the tool only discovers ignore files inside the analyzed repository: -cli_lint_mode_project_exclude_workspace_file_name: ".toolignore" # written at workspace root only if absent, removed after the run ``` One more property preserves config-file lists that the CLI flag would replace: @@ -69,12 +67,13 @@ One more property preserves config-file lists that the CLI flag would replace: cli_lint_mode_project_exclude_config_key: "scan.skip-dirs" # dotted key path of the resolved config list the flag REPLACES: its entries are re-emitted first ``` +**Never write inside the analyzed sources.** Every file MegaLinter generates goes to `REPORT_OUTPUT_FOLDER` (`write_report_generated_file`). A file created then deleted at the workspace root during the run crashes the project-mode linters walking the tree at the same moment (trivy aborts with `walk dir error … no such file or directory`), and the failure is timing-dependent, so it surfaces as a random red build. A tool that can only read an ignore file it discovers itself inside the repository gets **no** exclusion forwarding — say so in `disabled_reason` or in `linter_text`, do not write the file. + **Choosing the mechanism** (exactly ONE per linter — the base class applies all declared mechanisms, so declaring two forwards twice): 1. Native CLI exclusion flag → `..._exclude_arg_name` (+ value template / separator / seed values / config key). 2. Flag taking an ignore FILE → `..._exclude_ignore_file_*` (generated in the report folder, merged with seeds). -3. Tool only discovers ignore files inside the repo → `..._exclude_workspace_file_name` (temp file at workspace root, only if absent, auto-removed). -4. Anything needing a generated/merged CONFIG (yamllint extends, rubocop inherit_from, phpstan includes, TOML/PHP configs…) → override `manage_excluded_directories_config(cmd)` in the linter class. It is called only in project mode when forwarding is active (single gate: `is_project_exclude_forwarding_active`, overridable via `FORWARD_EXCLUDED_DIRECTORIES` / `_FORWARD_EXCLUDED_DIRECTORIES`). Use the base helpers `find_cli_argument_value_index`, `replace_or_append_cli_argument`, `write_report_generated_file`, `write_workspace_generated_file`, `read_workspace_file_lines`, and call `log_project_exclude_forwarding` so the action shows in the console log. +3. Anything needing a generated/merged CONFIG (yamllint extends, rubocop inherit_from, phpstan includes, sqlfluff ignore_paths, TOML/PHP configs…) → override `manage_excluded_directories_config(cmd)` in the linter class. It is called only in project mode when forwarding is active (single gate: `is_project_exclude_forwarding_active`, overridable via `FORWARD_EXCLUDED_DIRECTORIES` / `_FORWARD_EXCLUDED_DIRECTORIES`). Use the base helpers `find_cli_argument_value_index`, `replace_or_append_cli_argument`, `write_report_generated_file`, `read_workspace_file_lines`, and call `log_project_exclude_forwarding` so the action shows in the console log. Rules and known traps (each was hit for real — verify against official docs/source before filling anything): diff --git a/.claude/skills/add-linter/SKILL.md b/.claude/skills/add-linter/SKILL.md index 7efb187f5dc..36c51287359 100644 --- a/.claude/skills/add-linter/SKILL.md +++ b/.claude/skills/add-linter/SKILL.md @@ -68,8 +68,8 @@ Add the linter entry with **as many properties as possible**. Even though the JS - **Excluded directories forwarding** — when `project` is a supported lint mode, MegaLinter must forward `EXCLUDED_DIRECTORIES` to the tool or it will scan `node_modules`/build caches raw. Pick exactly ONE mechanism (see `.claude/rules/descriptors.md` → "Project Lint Mode: Forwarding Excluded Directories" for full semantics and known traps): - native CLI flag → `cli_lint_mode_project_exclude_arg_name` (+ `_arg_value` `{{DIR}}`/`{{WORKSPACE}}` template, `_separator` if a repeated flag overrides, `_seed_values` if the flag replaces the tool's built-in defaults, `_config_key` if it replaces a list in the tool's config file) - flag taking an ignore file → `cli_lint_mode_project_exclude_ignore_file_arg_name` (+ `_seed_files`, `_pass_existing`) - - ignore file only discovered inside the repo → `cli_lint_mode_project_exclude_workspace_file_name` - generated/merged config needed → `manage_excluded_directories_config()` override in the linter class + - tool that can only read an ignore file it discovers itself inside the repository → **no forwarding**: MegaLinter never writes in the analyzed sources, document the limitation in `linter_text` **Research the official docs first**: exact flag, value syntax (path/glob/regex, anchoring), repeatability, and whether it replaces config/built-in defaults — a wrong choice silently drops exclusions or clobbers user configuration. - **Poison fixture** — after declaring forwarding, add a deliberately failing file in `.automation/test//good/.wireit/` so `test_success_project_lint_mode` guards the forwarding against regressions (only if all project-capable linters sharing the folder have forwarding) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24bbe47f182..48752b088c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-l - `EXCLUDED_DIRECTORIES` are forwarded in project lint mode through a generated configuration extending the workspace one - Disabled linters + - **COFFEE_COFFEELINT** is disabled: CoffeeScript tooling is discontinued, and coffeelint can not receive `EXCLUDED_DIRECTORIES` in project lint mode (it has no exclusion option and reads `.coffeelintignore` only from its working directory). The linter will be removed in a future version - Re-enabled linters - **[spectral](https://megalinter.io/latest/descriptors/api_spectral/)** is back as **API_SPECTRAL**, together with the **API** descriptor, to lint your **OpenAPI**, **AsyncAPI** and **Arazzo** specifications ([#8717](https://github.com/oxsecurity/megalinter/issues/8717)) @@ -32,12 +33,16 @@ Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-l - Media - Linters enhancements + - **CLOJURE_CLJSTYLE** now forwards `EXCLUDED_DIRECTORIES` through its native repeatable `--ignore` argument, instead of a temporary `.cljstyle` written in your repository. Exclusions are now also applied when your repository already has a `.cljstyle` config, whose own ignore patterns are preserved + - **SQL_SQLFLUFF** now receives `EXCLUDED_DIRECTORIES` through the `ignore_paths` key of a generated sqlfluff configuration, instead of a temporary `.sqlfluffignore` written in your repository - **SARIF output** is now available for 13 more linters: **zizmor**, **bicep_linter**, **cppcheck**, **clj-kondo**, **roslynator**, **htmlhint**, **protolint**, **sqlfluff**, **swiftlint**, **osv-scanner**, **trufflehog**, **jscpd** and **lintr**. Enable it the same way as any other SARIF-capable linter, with `SARIF_REPORTER: true` (optionally scoped with `SARIF_REPORTER_LINTERS`) - The 4 **Salesforce Code Analyzer** engines (`SALESFORCE_CODE_ANALYZER_APEX`, `_AURA`, `_LWC`, `_FLOW`) also gained SARIF output: their report switches from CSV to SARIF automatically when SARIF reporting is requested - `csharp_roslynator` is bumped from 0.12.0 to **0.13.0**, the first release including its SARIF output support - `clj-kondo`'s upstream SARIF output currently nests the `region` property one level too deep, which may affect line/column display in strict SARIF consumers (clj-kondo/clj-kondo#2345) - Fixes + - Fixed **random crashes of project-mode linters** (`REPOSITORY_TRIVY`, `REPOSITORY_GRYPE`, `REPOSITORY_SYFT`…) caused by MegaLinter writing temporary ignore files inside the analyzed sources: a file appearing then disappearing while another linter walked the repository aborted its scan (`walk dir error: ... no such file or directory`). **MegaLinter now writes only in REPORT_OUTPUT_FOLDER**, never in your sources + - **REPORT_OUTPUT_FOLDER** is now always excluded from what linters analyze, even when you override `EXCLUDED_DIRECTORIES`, and even when the folder does not exist yet when a linter starts - The **API reporter** variables (`API_REPORTER`, `API_REPORTER_URL`…) are not flagged as **deprecated** anymore in the configuration JSON schema: they were collateral damage of the removal of the `API` descriptor in v10.0.0, and IDEs displayed them as obsolete - Reporters @@ -64,6 +69,7 @@ Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-l - **megalinter-setup** in upgrade mode now also updates the **installed skills and sub-agents** (`npx skills update`), so the guidance you run matches the MegaLinter version you just upgraded to - Dev + - Retired the `cli_lint_mode_project_exclude_workspace_file_name` descriptor property and the `write_workspace_generated_file()` helper, and removed the property from the descriptor JSON schema so a future descriptor can not silently reintroduce a write inside the analyzed sources. Exclusion forwarding now offers three mechanisms only: native CLI flag, generated ignore file in the report folder, generated config via `manage_excluded_directories_config()` - **Deprecation flags of removed linters are now reversible** in the configuration JSON schema: `build.py` clears the `deprecated` flag and the `(deprecated)` title prefix of variables whose linter or descriptor is back, instead of only ever adding them - **spectral is installed in its own `node_modules` tree** (`/node-deps-spectral`) instead of the shared `/node-deps` one, which is what made it crash: `@prantlf/jsonlint` pins `ajv` to exactly `8.17.1` and so owns the hoisted root copy, while `@stoplight/spectral-core` requires `ajv >= 8.18.0` and gets a nested one, so its hoisted `ajv-errors` bound to the other `ajv` instance and ajv generated invalid JavaScript (`SyntaxError: Unexpected token ':'` at `new Function`). Any npm linter sharing the tree with an exact-pinned transitive dependency can hit the same trap - **6 Python dependencies removed** from the MegaLinter runtime, replaced by standard library equivalents: `commentjson`, `terminaltables` and `multiprocessing_logging` (unmaintained), plus `termcolor`, `regex` and the obsolete `importlib-metadata` backport diff --git a/Dockerfile b/Dockerfile index ed8b3814da3..6224163ee14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -199,8 +199,6 @@ ARG CLJ_KONDO_VERSION=2025.01.16 ARG CLJ_STYLE_VERSION=0.17.642 # renovate: datasource=pypi depName=cfn-lint ARG PIP_CFN_LINT_VERSION=1.54.0 -# renovate: datasource=npm depName=@coffeelint/cli -ARG NPM_COFFEELINT_CLI_VERSION=5.2.11 # renovate: datasource=npm depName=jscpd ARG NPM_JSCPD_VERSION=5.0.14 # renovate: datasource=nuget depName=csharpier @@ -731,7 +729,6 @@ RUN npm config set prefix /usr/local \ && npm --no-cache install --ignore-scripts --omit=dev \ @salesforce/cli@${NPM_SALESFORCE_CLI_VERSION} \ typescript@${NPM_TYPESCRIPT_VERSION} \ - @coffeelint/cli@${NPM_COFFEELINT_CLI_VERSION} \ jscpd@${NPM_JSCPD_VERSION} \ stylelint@${NPM_STYLELINT_VERSION} \ stylelint-config-standard@${NPM_STYLELINT_CONFIG_STANDARD_VERSION} \ @@ -946,7 +943,6 @@ esac \ && chmod +x install-cljstyle \ && ./install-cljstyle --static --version "$CLJ_STYLE_VERSION" \ # cfn-lint installation -# coffeelint installation # jscpd installation # cpplint installation # csharpier installation diff --git a/flavors/dotnetweb/Dockerfile b/flavors/dotnetweb/Dockerfile index f01de118095..0b1b59409cc 100644 --- a/flavors/dotnetweb/Dockerfile +++ b/flavors/dotnetweb/Dockerfile @@ -140,8 +140,6 @@ ARG BICEP_EXE='bicep' ARG BICEP_DIR='/usr/local/bin' # renovate: datasource=pypi depName=cpplint ARG PIP_CPPLINT_VERSION=2.0.2 -# renovate: datasource=npm depName=@coffeelint/cli -ARG NPM_COFFEELINT_CLI_VERSION=5.2.11 # renovate: datasource=npm depName=jscpd ARG NPM_JSCPD_VERSION=5.0.14 # renovate: datasource=nuget depName=csharpier @@ -462,7 +460,6 @@ WORKDIR /node-deps RUN npm config set prefix /usr/local \ && npm --no-cache install --ignore-scripts --omit=dev \ typescript@${NPM_TYPESCRIPT_VERSION} \ - @coffeelint/cli@${NPM_COFFEELINT_CLI_VERSION} \ jscpd@${NPM_JSCPD_VERSION} \ stylelint@${NPM_STYLELINT_VERSION} \ stylelint-config-standard@${NPM_STYLELINT_CONFIG_STANDARD_VERSION} \ @@ -596,7 +593,6 @@ esac \ && chmod +x "${BICEP_EXE}" \ && mv "${BICEP_EXE}" "${BICEP_DIR}" \ # cpplint installation -# coffeelint installation # jscpd installation # cpplint installation # csharpier installation diff --git a/flavors/dotnetweb/flavor.json b/flavors/dotnetweb/flavor.json index 14efbabb5b6..7040abafd26 100644 --- a/flavors/dotnetweb/flavor.json +++ b/flavors/dotnetweb/flavor.json @@ -21,7 +21,6 @@ "BICEP_BICEP_LINTER", "C_CPPCHECK", "C_CPPLINT", - "COFFEE_COFFEELINT", "COPYPASTE_JSCPD", "CPP_CPPCHECK", "CPP_CPPLINT", diff --git a/flavors/javascript/Dockerfile b/flavors/javascript/Dockerfile index 410ceccc028..ed487138817 100644 --- a/flavors/javascript/Dockerfile +++ b/flavors/javascript/Dockerfile @@ -125,8 +125,6 @@ ARG NPM_TYPESCRIPT_VERSION=6.0.3 ARG PIP_ANSIBLE_LINT_VERSION=26.6.0 # renovate: datasource=npm depName=@stoplight/spectral-cli ARG NPM_SPECTRAL_CLI_VERSION=6.16.3 -# renovate: datasource=npm depName=@coffeelint/cli -ARG NPM_COFFEELINT_CLI_VERSION=5.2.11 # renovate: datasource=npm depName=jscpd ARG NPM_JSCPD_VERSION=5.0.14 # renovate: datasource=npm depName=stylelint @@ -432,7 +430,6 @@ WORKDIR /node-deps RUN npm config set prefix /usr/local \ && npm --no-cache install --ignore-scripts --omit=dev \ typescript@${NPM_TYPESCRIPT_VERSION} \ - @coffeelint/cli@${NPM_COFFEELINT_CLI_VERSION} \ jscpd@${NPM_JSCPD_VERSION} \ stylelint@${NPM_STYLELINT_VERSION} \ stylelint-config-standard@${NPM_STYLELINT_CONFIG_STANDARD_VERSION} \ @@ -520,7 +517,6 @@ RUN npm --no-cache install --ignore-scripts --omit=dev --prefix /node-deps-spect # # COPY --link --from=shellcheck /bin/shellcheck /usr/bin/shellcheck # shfmt installation # Managed with COPY --link --from=shfmt /bin/shfmt /usr/bin/ -# coffeelint installation # jscpd installation # stylelint installation # biome installation diff --git a/flavors/javascript/flavor.json b/flavors/javascript/flavor.json index 9d167bb627e..c3f40281489 100644 --- a/flavors/javascript/flavor.json +++ b/flavors/javascript/flavor.json @@ -13,7 +13,6 @@ "BASH_EXEC", "BASH_SHELLCHECK", "BASH_SHFMT", - "COFFEE_COFFEELINT", "COPYPASTE_JSCPD", "CSS_STYLELINT", "CSS_BIOME", diff --git a/megalinter/Linter.py b/megalinter/Linter.py index cbc1b3ff8bc..94c81ce3991 100644 --- a/megalinter/Linter.py +++ b/megalinter/Linter.py @@ -165,10 +165,6 @@ def __init__(self, params=None, linter_config=None): self.cli_lint_mode_project_exclude_ignore_file_arg_name = None self.cli_lint_mode_project_exclude_ignore_file_seed_files = [] self.cli_lint_mode_project_exclude_ignore_file_pass_existing = [] - # When set, the generated ignore file is written at the workspace root - # under this name (only if absent, removed after the run), for linters - # that only discover ignore files inside the analyzed repository - self.cli_lint_mode_project_exclude_workspace_file_name = None self.cli_lint_errors_count = None self.cli_lint_errors_regex = None self.cli_lint_warnings_count = None @@ -188,9 +184,6 @@ def __init__(self, params=None, linter_config=None): self.log_lines_pre: list[str] = [] self.log_lines_post: list[str] = [] - # Files temporarily written inside the workspace (e.g. generated ignore - # files for linters that only discover them there), removed after run - self.workspace_generated_files: list[str] = [] self.report_folder = "" self.reporters = [] @@ -1295,7 +1288,6 @@ def execute_lint_command(self, command): return_code, return_stdout ) self.manage_sarif_output(return_stdout) - self.cleanup_workspace_generated_files() # Return linter result return return_code, return_stdout @@ -1761,7 +1753,10 @@ def log_project_exclude_forwarding(self, message): # prefixes. Only directories existing at the workspace root are kept, so # linter arguments and generated ignore/config files stay minimal. No # filesystem walk: a nested-only directory or a regex too complex to trim - # into an existing directory is simply skipped + # into an existing directory is simply skipped. + # REPORT_OUTPUT_FOLDER is the exception: MegaLinter writes into it while + # linters run, so it is always forwarded, even when it does not exist yet + # when the command line is built def get_project_exclude_directories(self): cached = getattr(self, "project_exclude_directories", None) if cached is not None: @@ -1778,6 +1773,7 @@ def get_project_exclude_directories(self): for candidate in utils.extract_dir_candidates_from_regex(exclude_regex): excluded.add(candidate) workspace_abs = os.path.abspath(self.workspace) + always_forwarded = utils.get_report_output_folder_name(self.request_id) existing = set() for excl_dir in excluded: if not excl_dir: @@ -1792,7 +1788,9 @@ def get_project_exclude_directories(self): if rel_dir == "." or rel_dir.startswith(".."): continue excl_dir = rel_dir.replace("\\", "/") - if os.path.isdir(os.path.join(self.workspace, excl_dir)): + if excl_dir == always_forwarded or os.path.isdir( + os.path.join(self.workspace, excl_dir) + ): existing.add(excl_dir) self.project_exclude_directories = sorted(existing) return self.project_exclude_directories @@ -1840,61 +1838,24 @@ def build_project_exclude_arguments(self): ) return exclude_args - # Write a temporary file inside the workspace, for linters that only - # discover their ignore/config files there. Removed after the lint run - def write_workspace_generated_file(self, file_name, content_lines): - file_path = os.path.join(self.workspace, file_name) - with open(file_path, "w", encoding="utf-8") as file_handler: - file_handler.write("\n".join(content_lines) + "\n") - self.workspace_generated_files += [file_path] - self.log_project_exclude_forwarding( - f"Temporarily generated {file_path} in the workspace to forward " - f"EXCLUDED_DIRECTORIES to {self.linter_name}, removed after the run " - f"(disable with {self.name}_FORWARD_EXCLUDED_DIRECTORIES: false)" - ) - return file_path - - def cleanup_workspace_generated_files(self): - for generated_file in self.workspace_generated_files: - if os.path.isfile(generated_file): - os.remove(generated_file) - self.workspace_generated_files = [] - # Generic forwarding of excluded directories through an ignore file, driven - # by the cli_lint_mode_project_exclude_ignore_file_* descriptor properties + # by the cli_lint_mode_project_exclude_ignore_file_* descriptor properties. + # The generated file always lands in the report folder: MegaLinter never + # writes inside the analyzed sources, as a file appearing then disappearing + # there crashes the project-mode linters walking the tree at the same time def build_project_exclude_ignore_file_arguments(self, cmd): arg_name = self.cli_lint_mode_project_exclude_ignore_file_arg_name - workspace_file_name = self.cli_lint_mode_project_exclude_workspace_file_name - if arg_name is None and workspace_file_name is None: + if arg_name is None: return [] if len(self.get_project_exclude_directories()) == 0: return [] - if arg_name is not None and arg_name in cmd: + if arg_name in cmd: return [] seed_lines = [] for seed_file in self.cli_lint_mode_project_exclude_ignore_file_seed_files: seed_lines = self.read_workspace_file_lines(seed_file) if len(seed_lines) > 0: break - if workspace_file_name is not None: - # Existing workspace file stays authoritative - if os.path.isfile(os.path.join(self.workspace, workspace_file_name)): - return [] - lines = list(seed_lines) - for excluded_dir in self.get_project_exclude_directories(): - # Line syntax follows the value template (gitignore-style bare - # name by default, glob like {{DIR}}/** for tools such as v8r) - line = self.cli_lint_mode_project_exclude_arg_value.replace( - "{{DIR}}", excluded_dir - ) - if line not in lines: - lines.append(line) - self.write_workspace_generated_file(workspace_file_name, lines) - # Pass the base name so linters resolving ignore files inside the - # scanned tree (ex: secretlint) discover the generated file - if arg_name is None: - return [] - return self.build_ignore_file_argument(arg_name, workspace_file_name) ignore_args = [] for ( existing_file diff --git a/megalinter/descriptors/all_flavors.json b/megalinter/descriptors/all_flavors.json index 4e9cf6cc035..694bd457b74 100644 --- a/megalinter/descriptors/all_flavors.json +++ b/megalinter/descriptors/all_flavors.json @@ -377,7 +377,6 @@ "BICEP_BICEP_LINTER", "C_CPPCHECK", "C_CPPLINT", - "COFFEE_COFFEELINT", "COPYPASTE_JSCPD", "CPP_CPPCHECK", "CPP_CPPLINT", @@ -631,7 +630,6 @@ "BASH_EXEC", "BASH_SHELLCHECK", "BASH_SHFMT", - "COFFEE_COFFEELINT", "COPYPASTE_JSCPD", "CSS_STYLELINT", "CSS_BIOME", diff --git a/megalinter/descriptors/clojure.megalinter-descriptor.yml b/megalinter/descriptors/clojure.megalinter-descriptor.yml index 828be377fce..7152c98f83f 100644 --- a/megalinter/descriptors/clojure.megalinter-descriptor.yml +++ b/megalinter/descriptors/clojure.megalinter-descriptor.yml @@ -87,8 +87,7 @@ linters: url: https://github.com/marcomorain/clojure-lint # cljstyle - - class: CljstyleLinter - linter_name: cljstyle + - linter_name: cljstyle linter_text: | **cljstyle** is a comprehensive tool for formatting Clojure code that transforms messy, inconsistent code into beautifully formatted, readable source files. It's a rewrite of the original cljfmt tool with enhanced capabilities and native compilation. @@ -117,6 +116,9 @@ linters: - project cli_lint_extra_args: - "check" + # Repeatable, values are compiled as regexes matched against the full path, + # and they are added to the ignore set of the user .cljstyle config + cli_lint_mode_project_exclude_arg_name: "--ignore" cli_lint_mode_project_extra_args_after: - . cli_lint_fix_arg_name: "fix" diff --git a/megalinter/descriptors/coffee.megalinter-descriptor.yml b/megalinter/descriptors/coffee.megalinter-descriptor.yml index 4f3920f9aaa..72210b91492 100644 --- a/megalinter/descriptors/coffee.megalinter-descriptor.yml +++ b/megalinter/descriptors/coffee.megalinter-descriptor.yml @@ -10,6 +10,11 @@ test_folder: coffeescript linters: # Coffee Lint - linter_name: coffeelint + disabled: true + disabled_reason: CoffeeScript tooling is discontinued, and coffeelint can not + honor EXCLUDED_DIRECTORIES (it has no exclusion option and only reads + .coffeelintignore from its working directory). The linter will be removed + in a future version linter_text: | **CoffeeLint** is a style checker that helps keep CoffeeScript code clean and consistent. While CoffeeScript insulates programmers from many of JavaScript's problematic features, it doesn't enforce consistent style across a codebase - that's where CoffeeLint comes in. @@ -34,7 +39,6 @@ linters: - file - list_of_files - project - cli_lint_mode_project_exclude_workspace_file_name: ".coffeelintignore" cli_lint_mode_project_extra_args_after: - . examples: diff --git a/megalinter/descriptors/repository.megalinter-descriptor.yml b/megalinter/descriptors/repository.megalinter-descriptor.yml index 845466b6d50..29e7194b08e 100644 --- a/megalinter/descriptors/repository.megalinter-descriptor.yml +++ b/megalinter/descriptors/repository.megalinter-descriptor.yml @@ -680,7 +680,6 @@ linters: - "--no-gitignore" cli_lint_mode_project_exclude_ignore_file_arg_name: "--secretlintignore" cli_lint_mode_project_exclude_ignore_file_seed_files: [".secretlintignore", ".gitignore"] - cli_lint_mode_project_exclude_workspace_file_name: ".megalinter-secretlintignore" cli_lint_mode_project_extra_args_after: - "**/*" cli_sarif_args: diff --git a/megalinter/descriptors/schemas/megalinter-descriptor.jsonschema.json b/megalinter/descriptors/schemas/megalinter-descriptor.jsonschema.json index 1b53edf1908..a432450b778 100644 --- a/megalinter/descriptors/schemas/megalinter-descriptor.jsonschema.json +++ b/megalinter/descriptors/schemas/megalinter-descriptor.jsonschema.json @@ -867,16 +867,6 @@ "title": "CLI project lint mode excluded directories separator", "type": "string" }, - "cli_lint_mode_project_exclude_workspace_file_name": { - "$id": "#/properties/linters/items/properties/cli_lint_mode_project_exclude_workspace_file_name", - "description": "When set, the generated ignore file is written under this name at the workspace root (only if absent, removed after the run), for linters that only discover ignore files inside the analyzed repository. Combined with cli_lint_mode_project_exclude_ignore_file_arg_name, the file base name is passed as the argument value", - "examples": [ - ".sqlfluffignore", - ".megalinter-secretlintignore" - ], - "title": "CLI project lint mode workspace-written ignore file name", - "type": "string" - }, "cli_lint_mode_project_extra_args_after": { "$id": "#/properties/linters/items/properties/cli_lint_mode_project_extra_args_after", "default": [], diff --git a/megalinter/descriptors/sql.megalinter-descriptor.yml b/megalinter/descriptors/sql.megalinter-descriptor.yml index 207334e76dc..37d14272829 100644 --- a/megalinter/descriptors/sql.megalinter-descriptor.yml +++ b/megalinter/descriptors/sql.megalinter-descriptor.yml @@ -38,7 +38,6 @@ linters: - file - list_of_files - project - cli_lint_mode_project_exclude_workspace_file_name: ".sqlfluffignore" cli_lint_fix_arg_name: fix cli_lint_fix_remove_args: - lint diff --git a/megalinter/linters/CljstyleLinter.py b/megalinter/linters/CljstyleLinter.py deleted file mode 100644 index d5b442886c7..00000000000 --- a/megalinter/linters/CljstyleLinter.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 -""" -Use cljstyle to check Clojure code formatting -""" - -import os - -from megalinter import Linter - - -class CljstyleLinter(Linter): - # Forward excluded directories: cljstyle only discovers .cljstyle files - # hierarchically in the scanned tree, so one is temporarily written at the - # workspace root when the repository has none (removed after the run). - # Ignore entries are exact name matches - def manage_excluded_directories_config(self, cmd): - if not os.path.isfile(os.path.join(self.workspace, ".cljstyle")): - ignore_names = " ".join( - f'"{excluded_dir}"' - for excluded_dir in self.get_project_exclude_directories() - ) - self.write_workspace_generated_file( - ".cljstyle", ["{:files {:ignore #{" + ignore_names + "}}}"] - ) - return cmd diff --git a/megalinter/linters/SqlFluffLinter.py b/megalinter/linters/SqlFluffLinter.py index 2980118c9f6..0621aa1fd68 100644 --- a/megalinter/linters/SqlFluffLinter.py +++ b/megalinter/linters/SqlFluffLinter.py @@ -3,10 +3,15 @@ Use SqlFluff to lint any type of file according to local config """ +import configparser import logging from megalinter import Linter +IGNORE_PATHS_KEY = "ignore_paths" +CORE_SECTION = "sqlfluff" +GENERATED_CONFIG_NAME = "sqlfluff-megalinter.cfg" + class SqlFluffLinter(Linter): @@ -27,3 +32,49 @@ def build_lint_command(self, file=None): ) logging.debug("[SqlFluffLinter] Added --show-lint-violations argument") return cmd + + # sqlfluff has no CLI exclusion argument, and its .sqlfluffignore files are + # only discovered inside the analyzed sources, where MegaLinter must not + # write. Excluded directories are forwarded through the ignore_paths key of + # a copy of the resolved configuration, generated in the report folder + def manage_excluded_directories_config(self, cmd): + excluded_dirs = self.get_project_exclude_directories() + if len(excluded_dirs) == 0: + return cmd + parser = configparser.ConfigParser() + if self.final_config_file is not None: + parser.read(self.final_config_file, encoding="utf-8") + if not parser.has_section(CORE_SECTION): + parser.add_section(CORE_SECTION) + # Patterns follow .sqlfluffignore syntax: a trailing / matches the + # directory at any level of the analyzed tree + ignore_paths = [ + value.strip() + for value in parser.get(CORE_SECTION, IGNORE_PATHS_KEY, fallback="").split( + "," + ) + if value.strip() != "" + ] + for excluded_dir in excluded_dirs: + pattern = f"{excluded_dir}/" + if pattern not in ignore_paths: + ignore_paths.append(pattern) + parser.set(CORE_SECTION, IGNORE_PATHS_KEY, ",".join(ignore_paths)) + config_lines = [] + for section in parser.sections(): + config_lines += [f"[{section}]"] + config_lines += [f"{key} = {value}" for key, value in parser.items(section)] + config_lines += [""] + generated_config_file = self.write_report_generated_file( + GENERATED_CONFIG_NAME, config_lines + ) + value_index = self.find_cli_argument_value_index(cmd, ["--config"]) + cmd = self.replace_or_append_cli_argument( + cmd, value_index, "--config", generated_config_file + ) + self.log_project_exclude_forwarding( + f"Forwarded EXCLUDED_DIRECTORIES to {self.linter_name} through the " + f"{IGNORE_PATHS_KEY} key of {generated_config_file} " + f"(disable with {self.name}_FORWARD_EXCLUDED_DIRECTORIES: false)" + ) + return cmd diff --git a/megalinter/tests/test_megalinter/linter_test.py b/megalinter/tests/test_megalinter/linter_test.py index 78fcb5b8e8a..0996584a8e1 100644 --- a/megalinter/tests/test_megalinter/linter_test.py +++ b/megalinter/tests/test_megalinter/linter_test.py @@ -4,10 +4,13 @@ """ +import os +import tempfile import unittest import uuid from unittest import mock +from megalinter import config from megalinter.Linter import Linter from megalinter.linters.StyleLintLinter import StyleLintLinter @@ -62,6 +65,33 @@ def test_activation_unknown_priority_falls_back_to_enable(self): self.run_activation(["JAVASCRIPT_ES"], ["JAVASCRIPT_ES"], "WHATEVER") ) + def get_forwarded_exclude_directories(self, existing_directories): + linter = Linter.__new__(Linter) + linter.name = "REPOSITORY_TRIVY" + linter.request_id = str(uuid.uuid1()) + linter.filter_regex_exclude_descriptor = None + linter.filter_regex_exclude_linter = None + config.init_config(linter.request_id, None, {}) + try: + with tempfile.TemporaryDirectory() as workspace: + linter.workspace = workspace + for directory in existing_directories: + os.makedirs(os.path.join(workspace, directory)) + return linter.get_project_exclude_directories() + finally: + config.delete(linter.request_id) + + def test_report_folder_is_forwarded_even_when_it_does_not_exist(self): + # Reporters write in the report folder while linters run: a project-mode + # linter must skip it even when it is not created yet at command build time + excluded = self.get_forwarded_exclude_directories([]) + self.assertIn("megalinter-reports", excluded) + + def test_other_excluded_directories_are_forwarded_only_when_existing(self): + excluded = self.get_forwarded_exclude_directories(["node_modules"]) + self.assertIn("node_modules", excluded) + self.assertNotIn(".venv", excluded) + def test_replace_vars_with_default_variables(self): linter = Linter.__new__(Linter) linter.sarif_output_file = "test_sarif_output_file" diff --git a/megalinter/tests/test_megalinter/linter_timeout_test.py b/megalinter/tests/test_megalinter/linter_timeout_test.py index d718c2fa177..3f997c739dd 100644 --- a/megalinter/tests/test_megalinter/linter_timeout_test.py +++ b/megalinter/tests/test_megalinter/linter_timeout_test.py @@ -33,7 +33,6 @@ def build_linter(request_id, workspace="."): linter.output_sarif = False linter.sarif_output_file = None linter.sarif_default_output_file = None - linter.workspace_generated_files = [] linter.unsecured_env_variables = [] linter.timeout_seconds = None linter.timeout_config_var = None diff --git a/megalinter/tests/test_megalinter/sqlfluff_linter_test.py b/megalinter/tests/test_megalinter/sqlfluff_linter_test.py new file mode 100644 index 00000000000..4ae8421a879 --- /dev/null +++ b/megalinter/tests/test_megalinter/sqlfluff_linter_test.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +""" +Unit tests for SqlFluffLinter excluded directories forwarding + +""" + +import os +import tempfile +import unittest +import uuid + +from megalinter.linters.SqlFluffLinter import SqlFluffLinter + + +class sqlfluff_linter_test(unittest.TestCase): + def build_linter(self, workspace, report_folder, user_config_content=None): + linter = SqlFluffLinter.__new__(SqlFluffLinter) + linter.name = "SQL_SQLFLUFF" + linter.linter_name = "sqlfluff" + linter.request_id = str(uuid.uuid1()) + linter.workspace = workspace + linter.report_folder = report_folder + linter.log_lines_pre = [] + linter.project_exclude_directories = ["megalinter-reports", "node_modules"] + linter.final_config_file = None + if user_config_content is not None: + user_config_file = os.path.join(workspace, ".sqlfluff") + with open(user_config_file, "w", encoding="utf-8") as file_handler: + file_handler.write(user_config_content) + linter.final_config_file = user_config_file + return linter + + def run_forwarding(self, user_config_content=None): + with tempfile.TemporaryDirectory() as workspace: + report_folder = os.path.join(workspace, "megalinter-reports") + linter = self.build_linter(workspace, report_folder, user_config_content) + cmd = linter.manage_excluded_directories_config( + ["sqlfluff", "lint", "--config", ".sqlfluff", "."] + ) + generated_file = os.path.join(report_folder, "sqlfluff-megalinter.cfg") + with open(generated_file, encoding="utf-8") as file_handler: + generated_content = file_handler.read() + workspace_entries = sorted(os.listdir(workspace)) + return cmd, generated_content, workspace_entries + + def test_excluded_directories_are_added_to_ignore_paths(self): + cmd, generated_content, _ = self.run_forwarding("[sqlfluff]\ndialect = ansi\n") + self.assertIn( + "ignore_paths = megalinter-reports/,node_modules/", generated_content + ) + # The user configuration is preserved in the generated copy + self.assertIn("dialect = ansi", generated_content) + self.assertIn("--config", cmd) + self.assertTrue( + cmd[cmd.index("--config") + 1].endswith("sqlfluff-megalinter.cfg") + ) + + def test_user_ignore_paths_are_kept(self): + _, generated_content, _ = self.run_forwarding( + "[sqlfluff]\ndialect = ansi\nignore_paths = target/\n" + ) + self.assertIn( + "ignore_paths = target/,megalinter-reports/,node_modules/", + generated_content, + ) + + def test_nothing_is_written_in_the_workspace(self): + # MegaLinter must never create files in the analyzed sources: a file + # appearing there crashes the project-mode linters walking the tree + _, _, workspace_entries = self.run_forwarding("[sqlfluff]\ndialect = ansi\n") + self.assertEqual([".sqlfluff", "megalinter-reports"], workspace_entries) + + def test_generated_config_without_user_config(self): + cmd, generated_content, workspace_entries = self.run_forwarding() + self.assertIn("[sqlfluff]", generated_content) + self.assertIn( + "ignore_paths = megalinter-reports/,node_modules/", generated_content + ) + self.assertEqual(["megalinter-reports"], workspace_entries) diff --git a/megalinter/tests/test_megalinter/utils_test.py b/megalinter/tests/test_megalinter/utils_test.py index 1ed8e27ed7b..bd12702bb52 100644 --- a/megalinter/tests/test_megalinter/utils_test.py +++ b/megalinter/tests/test_megalinter/utils_test.py @@ -6,13 +6,35 @@ import re import unittest +import uuid import warnings +from megalinter import config from megalinter.logger import fetch_betterleaks_regexes, sanitize_string -from megalinter.utils import fix_regex_pattern +from megalinter.utils import fix_regex_pattern, get_excluded_directories class utils_test(unittest.TestCase): + def test_report_folder_excluded_even_when_excluded_directories_overridden(self): + # MegaLinter writes its reports there while linters run, so analyzing it + # makes project-mode linters fail on files created or deleted mid-run + request_id = str(uuid.uuid1()) + config.init_config( + request_id, + None, + { + "EXCLUDED_DIRECTORIES": "custom_dir", + "REPORT_OUTPUT_FOLDER": "my-reports", + }, + ) + try: + excluded = get_excluded_directories(request_id) + finally: + config.delete(request_id) + self.assertIn("my-reports", excluded) + self.assertIn("custom_dir", excluded) + self.assertNotIn("node_modules", excluded) + def test_sanitize_string(self): input_string = "AWS Key: AKIAIOSFODNN7EXAMPLE and GitHub Token: ghp_abcdEFGHijklMNOPqrstUVWXyz1234567890" sanitized = sanitize_string(input_string) diff --git a/megalinter/utils.py b/megalinter/utils.py index 39a5d0add28..e36019c47c0 100644 --- a/megalinter/utils.py +++ b/megalinter/utils.py @@ -118,6 +118,13 @@ def get_prebuilt_linter_version(linter_name): _excluded_directories_cache: dict[str, set[str]] = {} +# MegaLinter writes its own files there while linters run, so it must never be +# analyzed: a report file created or deleted mid-run makes project-mode linters +# walking the workspace fail +def get_report_output_folder_name(request_id): + return config.get(request_id, "REPORT_OUTPUT_FOLDER", "megalinter-reports") + + def get_excluded_directories(request_id): cache_key = str(request_id) cached = _excluded_directories_cache.get(cache_key) @@ -143,12 +150,13 @@ def get_excluded_directories(request_id): ".wireit", ".yarn/cache", "node_modules", - config.get(request_id, "REPORT_OUTPUT_FOLDER", "megalinter-reports"), ] excluded_dirs = config.get_list( request_id, "EXCLUDED_DIRECTORIES", default_excluded_dirs ) excluded_dirs += config.get_list(request_id, "ADDITIONAL_EXCLUDED_DIRECTORIES", []) + # Always excluded, even when EXCLUDED_DIRECTORIES is overridden + excluded_dirs += [get_report_output_folder_name(request_id)] result = set(excluded_dirs) _excluded_directories_cache[cache_key] = result return result From c83619ec6c642069c881a1d3b5b8443efc7162e8 Mon Sep 17 00:00:00 2001 From: nvuillam Date: Wed, 12 Aug 2026 00:20:26 +0200 Subject: [PATCH 2/2] Fix CI: sqlfluff can not receive excluded directories, drop its forwarding sqlfluff reads ignore_paths only from a config file discovered between the working directory and the analyzed path (core/linter/discovery.py _iter_config_files), so a generated configuration passed with --config is never consulted for path exclusions and the .wireit poison fixture was still linted. Like coffeelint, sqlfluff simply gets no forwarding: its poison fixture is removed and the limitation is documented in the descriptor. --- .automation/test/sql/good/.wireit/poison.sql | 3 - .claude/rules/descriptors.md | 4 +- .claude/rules/testing.md | 1 + CHANGELOG.md | 2 +- .../descriptors/sql.megalinter-descriptor.yml | 2 + megalinter/linters/SqlFluffLinter.py | 51 ------------ .../test_megalinter/sqlfluff_linter_test.py | 79 ------------------- 7 files changed, 6 insertions(+), 136 deletions(-) delete mode 100644 .automation/test/sql/good/.wireit/poison.sql delete mode 100644 megalinter/tests/test_megalinter/sqlfluff_linter_test.py diff --git a/.automation/test/sql/good/.wireit/poison.sql b/.automation/test/sql/good/.wireit/poison.sql deleted file mode 100644 index e785450d1bb..00000000000 --- a/.automation/test/sql/good/.wireit/poison.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Deliberately failing fixture inside .wireit, a default excluded directory: --- project lint mode success passes only if exclusions are forwarded (#8645). -SELCT broken FROM; diff --git a/.claude/rules/descriptors.md b/.claude/rules/descriptors.md index 22d544dcbb8..e9709af2067 100644 --- a/.claude/rules/descriptors.md +++ b/.claude/rules/descriptors.md @@ -67,13 +67,13 @@ One more property preserves config-file lists that the CLI flag would replace: cli_lint_mode_project_exclude_config_key: "scan.skip-dirs" # dotted key path of the resolved config list the flag REPLACES: its entries are re-emitted first ``` -**Never write inside the analyzed sources.** Every file MegaLinter generates goes to `REPORT_OUTPUT_FOLDER` (`write_report_generated_file`). A file created then deleted at the workspace root during the run crashes the project-mode linters walking the tree at the same moment (trivy aborts with `walk dir error … no such file or directory`), and the failure is timing-dependent, so it surfaces as a random red build. A tool that can only read an ignore file it discovers itself inside the repository gets **no** exclusion forwarding — say so in `disabled_reason` or in `linter_text`, do not write the file. +**Never write inside the analyzed sources.** Every file MegaLinter generates goes to `REPORT_OUTPUT_FOLDER` (`write_report_generated_file`). A file created then deleted at the workspace root during the run crashes the project-mode linters walking the tree at the same moment (trivy aborts with `walk dir error … no such file or directory`), and the failure is timing-dependent, so it surfaces as a random red build. A tool that can only read exclusions from a file it discovers itself inside the repository gets **no** exclusion forwarding (coffeelint, and sqlfluff whose `ignore_paths` is read only from a config file located between the working directory and the analyzed path) — say so in `disabled_reason` or in `linter_text`, do not write the file. **Choosing the mechanism** (exactly ONE per linter — the base class applies all declared mechanisms, so declaring two forwards twice): 1. Native CLI exclusion flag → `..._exclude_arg_name` (+ value template / separator / seed values / config key). 2. Flag taking an ignore FILE → `..._exclude_ignore_file_*` (generated in the report folder, merged with seeds). -3. Anything needing a generated/merged CONFIG (yamllint extends, rubocop inherit_from, phpstan includes, sqlfluff ignore_paths, TOML/PHP configs…) → override `manage_excluded_directories_config(cmd)` in the linter class. It is called only in project mode when forwarding is active (single gate: `is_project_exclude_forwarding_active`, overridable via `FORWARD_EXCLUDED_DIRECTORIES` / `_FORWARD_EXCLUDED_DIRECTORIES`). Use the base helpers `find_cli_argument_value_index`, `replace_or_append_cli_argument`, `write_report_generated_file`, `read_workspace_file_lines`, and call `log_project_exclude_forwarding` so the action shows in the console log. +3. Anything needing a generated/merged CONFIG (yamllint extends, rubocop inherit_from, phpstan includes, TOML/PHP configs…) → override `manage_excluded_directories_config(cmd)` in the linter class. It is called only in project mode when forwarding is active (single gate: `is_project_exclude_forwarding_active`, overridable via `FORWARD_EXCLUDED_DIRECTORIES` / `_FORWARD_EXCLUDED_DIRECTORIES`). Use the base helpers `find_cli_argument_value_index`, `replace_or_append_cli_argument`, `write_report_generated_file`, `read_workspace_file_lines`, and call `log_project_exclude_forwarding` so the action shows in the console log. Rules and known traps (each was hit for real — verify against official docs/source before filling anything): diff --git a/.claude/rules/testing.md b/.claude/rules/testing.md index 2e49c2d22c2..927db81628f 100644 --- a/.claude/rules/testing.md +++ b/.claude/rules/testing.md @@ -24,6 +24,7 @@ globs: ["megalinter/tests/**/*.py", ".automation/test/**"] - `.wireit` is a default excluded directory that almost no tool skips natively, so `test_success_project_lint_mode` passes only if the forwarding actually excludes it — the fixture is a regression test for the forwarding, not for the linter rules - When adding forwarding to a linter, add its poison fixture; when a project success test fails on a file under `.wireit/`, the forwarding is broken, not the fixture - Constraints: only in folders with a `good/` subfolder, and only when every project-capable linter sharing the test folder has forwarding (otherwise the poison legitimately fails the non-forwarding tenant) +- No poison fixture for a linter that can not receive exclusions at all (coffeelint, sqlfluff): its `good/` folder must stay clean, and the limitation is documented in the descriptor `linter_text` instead - Poisons may be vacuous for tools that natively skip the directory (dot-folder-skipping globs, verified-only secret scanners): they never false-fail, so that is acceptable ## Running Tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 48752b088c6..0ed4a14c221 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-l - Linters enhancements - **CLOJURE_CLJSTYLE** now forwards `EXCLUDED_DIRECTORIES` through its native repeatable `--ignore` argument, instead of a temporary `.cljstyle` written in your repository. Exclusions are now also applied when your repository already has a `.cljstyle` config, whose own ignore patterns are preserved - - **SQL_SQLFLUFF** now receives `EXCLUDED_DIRECTORIES` through the `ignore_paths` key of a generated sqlfluff configuration, instead of a temporary `.sqlfluffignore` written in your repository + - **SQL_SQLFLUFF** does not receive `EXCLUDED_DIRECTORIES` in `project` lint mode anymore: sqlfluff reads path exclusions only from a `.sqlfluffignore`, `.sqlfluff` or `pyproject.toml` located inside the analyzed sources, where MegaLinter used to write a temporary file. List the directories to skip in your own `.sqlfluffignore`, or keep the default `list_of_files` lint mode where MegaLinter filters the files itself - **SARIF output** is now available for 13 more linters: **zizmor**, **bicep_linter**, **cppcheck**, **clj-kondo**, **roslynator**, **htmlhint**, **protolint**, **sqlfluff**, **swiftlint**, **osv-scanner**, **trufflehog**, **jscpd** and **lintr**. Enable it the same way as any other SARIF-capable linter, with `SARIF_REPORTER: true` (optionally scoped with `SARIF_REPORTER_LINTERS`) - The 4 **Salesforce Code Analyzer** engines (`SALESFORCE_CODE_ANALYZER_APEX`, `_AURA`, `_LWC`, `_FLOW`) also gained SARIF output: their report switches from CSV to SARIF automatically when SARIF reporting is requested - `csharp_roslynator` is bumped from 0.12.0 to **0.13.0**, the first release including its SARIF output support diff --git a/megalinter/descriptors/sql.megalinter-descriptor.yml b/megalinter/descriptors/sql.megalinter-descriptor.yml index 37d14272829..2d56dce2982 100644 --- a/megalinter/descriptors/sql.megalinter-descriptor.yml +++ b/megalinter/descriptors/sql.megalinter-descriptor.yml @@ -23,6 +23,8 @@ linters: - **Detailed Reporting**: Comprehensive error reporting with line numbers, positions, and fix suggestions - **Custom Rules**: Extensible architecture allowing custom rule development for specific requirements - **Performance Optimized**: Fast parsing and analysis even for large SQL codebases + + Note: in `project` lint mode, `EXCLUDED_DIRECTORIES` can not be forwarded to sqlfluff. It reads path exclusions only from a `.sqlfluffignore`, a `.sqlfluff` or a `pyproject.toml` located inside the analyzed sources, where MegaLinter never writes. Add the directories to ignore in your own `.sqlfluffignore`, or keep the default `list_of_files` lint mode, where MegaLinter filters the files itself. linter_url: https://www.sqlfluff.com/ linter_repo: https://github.com/sqlfluff/sqlfluff linter_spdx_license: MIT diff --git a/megalinter/linters/SqlFluffLinter.py b/megalinter/linters/SqlFluffLinter.py index 0621aa1fd68..2980118c9f6 100644 --- a/megalinter/linters/SqlFluffLinter.py +++ b/megalinter/linters/SqlFluffLinter.py @@ -3,15 +3,10 @@ Use SqlFluff to lint any type of file according to local config """ -import configparser import logging from megalinter import Linter -IGNORE_PATHS_KEY = "ignore_paths" -CORE_SECTION = "sqlfluff" -GENERATED_CONFIG_NAME = "sqlfluff-megalinter.cfg" - class SqlFluffLinter(Linter): @@ -32,49 +27,3 @@ def build_lint_command(self, file=None): ) logging.debug("[SqlFluffLinter] Added --show-lint-violations argument") return cmd - - # sqlfluff has no CLI exclusion argument, and its .sqlfluffignore files are - # only discovered inside the analyzed sources, where MegaLinter must not - # write. Excluded directories are forwarded through the ignore_paths key of - # a copy of the resolved configuration, generated in the report folder - def manage_excluded_directories_config(self, cmd): - excluded_dirs = self.get_project_exclude_directories() - if len(excluded_dirs) == 0: - return cmd - parser = configparser.ConfigParser() - if self.final_config_file is not None: - parser.read(self.final_config_file, encoding="utf-8") - if not parser.has_section(CORE_SECTION): - parser.add_section(CORE_SECTION) - # Patterns follow .sqlfluffignore syntax: a trailing / matches the - # directory at any level of the analyzed tree - ignore_paths = [ - value.strip() - for value in parser.get(CORE_SECTION, IGNORE_PATHS_KEY, fallback="").split( - "," - ) - if value.strip() != "" - ] - for excluded_dir in excluded_dirs: - pattern = f"{excluded_dir}/" - if pattern not in ignore_paths: - ignore_paths.append(pattern) - parser.set(CORE_SECTION, IGNORE_PATHS_KEY, ",".join(ignore_paths)) - config_lines = [] - for section in parser.sections(): - config_lines += [f"[{section}]"] - config_lines += [f"{key} = {value}" for key, value in parser.items(section)] - config_lines += [""] - generated_config_file = self.write_report_generated_file( - GENERATED_CONFIG_NAME, config_lines - ) - value_index = self.find_cli_argument_value_index(cmd, ["--config"]) - cmd = self.replace_or_append_cli_argument( - cmd, value_index, "--config", generated_config_file - ) - self.log_project_exclude_forwarding( - f"Forwarded EXCLUDED_DIRECTORIES to {self.linter_name} through the " - f"{IGNORE_PATHS_KEY} key of {generated_config_file} " - f"(disable with {self.name}_FORWARD_EXCLUDED_DIRECTORIES: false)" - ) - return cmd diff --git a/megalinter/tests/test_megalinter/sqlfluff_linter_test.py b/megalinter/tests/test_megalinter/sqlfluff_linter_test.py deleted file mode 100644 index 4ae8421a879..00000000000 --- a/megalinter/tests/test_megalinter/sqlfluff_linter_test.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 -""" -Unit tests for SqlFluffLinter excluded directories forwarding - -""" - -import os -import tempfile -import unittest -import uuid - -from megalinter.linters.SqlFluffLinter import SqlFluffLinter - - -class sqlfluff_linter_test(unittest.TestCase): - def build_linter(self, workspace, report_folder, user_config_content=None): - linter = SqlFluffLinter.__new__(SqlFluffLinter) - linter.name = "SQL_SQLFLUFF" - linter.linter_name = "sqlfluff" - linter.request_id = str(uuid.uuid1()) - linter.workspace = workspace - linter.report_folder = report_folder - linter.log_lines_pre = [] - linter.project_exclude_directories = ["megalinter-reports", "node_modules"] - linter.final_config_file = None - if user_config_content is not None: - user_config_file = os.path.join(workspace, ".sqlfluff") - with open(user_config_file, "w", encoding="utf-8") as file_handler: - file_handler.write(user_config_content) - linter.final_config_file = user_config_file - return linter - - def run_forwarding(self, user_config_content=None): - with tempfile.TemporaryDirectory() as workspace: - report_folder = os.path.join(workspace, "megalinter-reports") - linter = self.build_linter(workspace, report_folder, user_config_content) - cmd = linter.manage_excluded_directories_config( - ["sqlfluff", "lint", "--config", ".sqlfluff", "."] - ) - generated_file = os.path.join(report_folder, "sqlfluff-megalinter.cfg") - with open(generated_file, encoding="utf-8") as file_handler: - generated_content = file_handler.read() - workspace_entries = sorted(os.listdir(workspace)) - return cmd, generated_content, workspace_entries - - def test_excluded_directories_are_added_to_ignore_paths(self): - cmd, generated_content, _ = self.run_forwarding("[sqlfluff]\ndialect = ansi\n") - self.assertIn( - "ignore_paths = megalinter-reports/,node_modules/", generated_content - ) - # The user configuration is preserved in the generated copy - self.assertIn("dialect = ansi", generated_content) - self.assertIn("--config", cmd) - self.assertTrue( - cmd[cmd.index("--config") + 1].endswith("sqlfluff-megalinter.cfg") - ) - - def test_user_ignore_paths_are_kept(self): - _, generated_content, _ = self.run_forwarding( - "[sqlfluff]\ndialect = ansi\nignore_paths = target/\n" - ) - self.assertIn( - "ignore_paths = target/,megalinter-reports/,node_modules/", - generated_content, - ) - - def test_nothing_is_written_in_the_workspace(self): - # MegaLinter must never create files in the analyzed sources: a file - # appearing there crashes the project-mode linters walking the tree - _, _, workspace_entries = self.run_forwarding("[sqlfluff]\ndialect = ansi\n") - self.assertEqual([".sqlfluff", "megalinter-reports"], workspace_entries) - - def test_generated_config_without_user_config(self): - cmd, generated_content, workspace_entries = self.run_forwarding() - self.assertIn("[sqlfluff]", generated_content) - self.assertIn( - "ignore_paths = megalinter-reports/,node_modules/", generated_content - ) - self.assertEqual(["megalinter-reports"], workspace_entries)