Ruff Linting (#1314)

* Disable all linters for now and slowly add them back

* Disable more unneeded linter rules

* Enabled linters with 0 issues

* Fixes for flake8-async linter

* Add ignored variable names

* Fixes for refurb linter

* Fixes for flake8-implicit-str-concat

* Fixes for flake8-import-conventions

* Fixes for flake8-future-annotations

* Fixes for flake8-pyi

* Remove blanket noqa statements to fix (pygrep-hooks)

* Remove large batch of unused noqa F401

* Fixes for flake8-builtins

* Misc rule fixes

* Fixes for flake8-logging-format

* Fixes for flake8-raise

* Fixes for Perflint

* Sort imports with isort

* Format with ruff

* Reorganize disabled rules list

* Update megalinter in CI (to update ruff)

* Update MegaLinter workflow

Author: TimPansino

.github/workflows/mega-linter.yml

@@ -1,14 +1,14 @@
 ---
-# Mega-Linter GitHub Action configuration file
-# More info at https://oxsecurity.github.io/megalinter
-name: Mega-Linter
+# MegaLinter GitHub Action configuration file
+# More info at https://megalinter.io
+name: MegaLinter
 
 on:
   # Trigger mega-linter at every push. Action will also be visible from Pull Requests to main
   # push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions)
   pull_request:
 
-env: # Comment env block if you do not want to apply fixes
+env: # Comment env block if you don't want to apply fixes
   # Apply linter fixes configuration
   APPLY_FIXES: all # When active, APPLY_FIXES must also be defined as environment variable (in github/workflows/mega-linter.yml or other CI tool)
   APPLY_FIXES_EVENT: pull_request # Decide which event triggers application of fixes in a commit or a PR (pull_request, push, all)
@@ -19,66 +19,56 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build:
-    name: Mega-Linter
+  megalinter:
+    name: MegaLinter
     runs-on: ubuntu-22.04
+    permissions:
+      # Give the default GITHUB_TOKEN write permission to commit and push, comment issues & post new PR
+      # Remove the ones you do not need
+      contents: write
+      issues: write
+      pull-requests: write
     steps:
       # Git Checkout
       - name: Checkout Code
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
         with:
           token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
-          fetch-depth: 0
 
-      # Mega-Linter
-      - name: Mega-Linter
+      # MegaLinter
+      - name: MegaLinter
         id: ml
-        # You can override Mega-Linter flavor used to have faster performances
-        # More info at https://oxsecurity.github.io/megalinter/flavors/
-        uses: oxsecurity/megalinter/flavors/python@a7a0163b6c8ff7474a283d99a706e27483ddd80f # 7.10.0
+        # You can override MegaLinter flavor used to have faster performances
+        # More info at https://megalinter.io/flavors/
+        uses: oxsecurity/megalinter/flavors/python@ec124f7998718d79379a3c5b39f5359952baf21d # 8.4.2
         env:
           # All available variables are described in documentation
-          # https://oxsecurity.github.io/megalinter/configuration/
-          VALIDATE_ALL_CODEBASE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} # Validates all source when push on main, else just the git diff with main. Set 'true' if you always want to lint all sources
-          DEFAULT_BRANCH: ${{ github.event_name == 'pull_request' && github.base_ref || 'main' }}
+          # https://megalinter.io/configuration/
+          VALIDATE_ALL_CODEBASE: "true"
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          # ADD YOUR CUSTOM ENV VARIABLES HERE TO OVERRIDE VALUES OF .mega-linter.yml AT THE ROOT OF YOUR REPOSITORY
-          PYTHON_RUFF_ARGUMENTS: '--config=''output-format="github"'''
-          PYTHON_RUFF_FORMAT_ARGUMENTS: '--config=''output-format="github"'''
+          # ADD YOUR CUSTOM ENV VARIABLES HERE OR DEFINE THEM IN A FILE .mega-linter.yml AT THE ROOT OF YOUR REPOSITORY
+          PYTHON_RUFF_ARGUMENTS: --config pyproject.toml --config 'output-format="github"'
+          PYTHON_RUFF_FORMAT_ARGUMENTS: --config pyproject.toml --config 'output-format="github"'
 
-      # Upload Mega-Linter artifacts
+      # Upload MegaLinter artifacts
       - name: Archive production artifacts
-        if: ${{ success() }} || ${{ failure() }}
-        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1
+        if: success() || failure()
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # 4.6.1
         with:
-          name: Mega-Linter reports
+          name: MegaLinter reports
           path: |
             megalinter-reports
             mega-linter.log
 
-      # Create pull request if applicable (for now works only on PR from same repository, not from forks)
-      - name: Create Pull Request with applied fixes
-        id: cpr
-        if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES_EVENT == 'all' || env.APPLY_FIXES_EVENT == github.event_name) && env.APPLY_FIXES_MODE == 'pull_request' && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && !contains(github.event.head_commit.message, 'skip fix')
-        uses: peter-evans/create-pull-request@c55203cfde3e5c11a452d352b4393e68b85b4533 # 6.0.3
-        with:
-          token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
-          commit-message: "[Mega-Linter] Apply linters automatic fixes"
-          title: "[Mega-Linter] Apply linters automatic fixes"
-          labels: bot
-      - name: Create PR output
-        if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES_EVENT == 'all' || env.APPLY_FIXES_EVENT == github.event_name) && env.APPLY_FIXES_MODE == 'pull_request' && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && !contains(github.event.head_commit.message, 'skip fix')
-        run: |
-          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
-          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
-
       # Push new commit if applicable (for now works only on PR from same repository, not from forks)
       - name: Prepare commit
         if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES_EVENT == 'all' || env.APPLY_FIXES_EVENT == github.event_name) && env.APPLY_FIXES_MODE == 'commit' && github.ref != 'refs/heads/main' && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && !contains(github.event.head_commit.message, 'skip fix')
         run: sudo chown -Rc $UID .git/
       - name: Commit and push applied linter fixes
         if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES_EVENT == 'all' || env.APPLY_FIXES_EVENT == github.event_name) && env.APPLY_FIXES_MODE == 'commit' && github.ref != 'refs/heads/main' && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && !contains(github.event.head_commit.message, 'skip fix')
-        uses: stefanzweifel/git-auto-commit-action@8621497c8c39c72f3e2a999a26b4ca1b5058a842 # 5.0.1
+        uses: stefanzweifel/git-auto-commit-action@e348103e9026cc0eee72ae06630dbe30c8bf7a79 # 5.1.0
         with:
           branch: ${{ github.event.pull_request.head.ref || github.head_ref || github.ref }}
-          commit_message: "[Mega-Linter] Apply linters fixes"
+          commit_message: "[MegaLinter] Apply linters fixes"
+          commit_user_name: newrelic-python-agent-team
+          commit_user_email: [email protected]

.mega-linter.yml

@@ -6,7 +6,7 @@ DEFAULT_BRANCH: main # Usually master or main
 SHOW_ELAPSED_TIME: true
 FILEIO_REPORTER: false
 PRINT_ALPACA: false
-VALIDATE_ALL_CODEBASE: false # only lint new and edited files
+VALIDATE_ALL_CODEBASE: true
 IGNORE_GITIGNORED_FILES: true
 FILTER_REGEX_EXCLUDE: "(.*/?packages/.*)" # Ignore packages directories
 
@@ -23,4 +23,6 @@ ENABLE_LINTERS: # If you use ENABLE_LINTERS variable, all other linters will be
   - YAML_YAMLLINT
 
 PYTHON_RUFF_CONFIG_FILE: pyproject.toml
+PYTHON_RUFF_CLI_LINT_MODE: project
 PYTHON_RUFF_FORMAT_CONFIG_FILE: pyproject.toml
+PYTHON_RUFF_FORMAT_CLI_LINT_MODE: project

newrelic/admin/__init__.py

@@ -53,7 +53,7 @@ def usage(name):
 
 
 @command("help", "[command]", hidden=True)
-def help(args):
+def help_(args):
     if not args:
         print("Usage: newrelic-admin command [options]")
         print()
@@ -125,7 +125,7 @@ def load_external_plugins():
         if sys.version_info >= (3, 10):
             from importlib.metadata import entry_points
         # Introduced in Python 3.8
-        elif sys.version_info >= (3, 8) and sys.version_info <= (3, 9):
+        elif sys.version_info >= (3, 8) and sys.version_info < (3, 9):
             from importlib_metadata import entry_points
         # Removed in Python 3.12
         else:

newrelic/admin/local_config.py

@@ -22,9 +22,9 @@
 from <config_file>.""",
 )
 def local_config(args):
+    import logging
     import os
     import sys
-    import logging
 
     if len(args) == 0:
         usage("local-config")

newrelic/admin/network_config.py

@@ -22,9 +22,9 @@
 from <config_file>.""",
 )
 def network_config(args):
+    import logging
     import os
     import sys
-    import logging
 
     if len(args) == 0:
         usage("network-config")

newrelic/admin/run_python.py

@@ -61,7 +61,8 @@ def log_message(text, *args):
         if name.startswith("NEW_RELIC_") or name.startswith("PYTHON"):
             log_message("%s = %r", name, os.environ.get(name))
 
-    from newrelic import version, __file__ as root_directory
+    from newrelic import __file__ as root_directory
+    from newrelic import version
 
     root_directory = os.path.dirname(root_directory)
     boot_directory = os.path.join(root_directory, "bootstrap")

newrelic/admin/server_config.py

@@ -24,9 +24,9 @@
 configuration file is used.""",
 )
 def server_config(args):
+    import logging
     import os
     import sys
-    import logging
     import time
 
     if len(args) == 0:

newrelic/api/cat_header_mixin.py

@@ -13,12 +13,12 @@
 # limitations under the License.
 
 from newrelic.common.encoding_utils import (
-    obfuscate,
+    base64_decode,
+    base64_encode,
     deobfuscate,
-    json_encode,
     json_decode,
-    base64_encode,
-    base64_decode,
+    json_encode,
+    obfuscate,
 )
 
 

newrelic/api/generator_trace.py

@@ -13,13 +13,13 @@
 # limitations under the License.
 
 import functools
-import types
 import sys
+import types
 
-from newrelic.api.time_trace import current_trace
 from newrelic.api.function_trace import FunctionTrace
-from newrelic.common.object_wrapper import FunctionWrapper, wrap_object
+from newrelic.api.time_trace import current_trace
 from newrelic.common.object_names import callable_name
+from newrelic.common.object_wrapper import FunctionWrapper, wrap_object
 
 
 def GeneratorTraceWrapper(wrapped, name=None, group=None, label=None, params=None):

newrelic/api/html_insertion.py

@@ -16,7 +16,7 @@
 
 _head_re = re.compile(b"<head[^>]*>", re.IGNORECASE)
 
-_xua_meta_re = re.compile(b"""<\\s*meta[^>]+http-equiv\\s*=\\s*['"]""" b"""x-ua-compatible['"][^>]*>""", re.IGNORECASE)
+_xua_meta_re = re.compile(b"""<\\s*meta[^>]+http-equiv\\s*=\\s*['"]x-ua-compatible['"][^>]*>""", re.IGNORECASE)
 
 _charset_meta_re = re.compile(b"""<\\s*meta[^>]+charset\\s*=[^>]*>""", re.IGNORECASE)
 

newrelic/api/import_hook.py

@@ -14,10 +14,8 @@
 
 import logging
 import sys
-
 from importlib.util import find_spec
 
-
 _logger = logging.getLogger(__name__)
 
 
@@ -57,7 +55,7 @@
 _uninstrumented_modules = set()
 
 
-def register_import_hook(name, callable):  # pylint: disable=redefined-builtin
+def register_import_hook(name, callable):  # noqa: A002
     hooks = _import_hooks.get(name, None)
 
     if name not in _import_hooks or hooks is None:

newrelic/api/in_function.py

@@ -14,4 +14,4 @@
 
 # Use of these from this module will be deprecated.
 
-from newrelic.common.object_wrapper import in_function, InFunctionWrapper, wrap_in_function
+from newrelic.common.object_wrapper import InFunctionWrapper, in_function, wrap_in_function

newrelic/api/lambda_handler.py

@@ -14,14 +14,14 @@
 
 import functools
 import warnings
-from newrelic.common.object_wrapper import FunctionWrapper
+
+from newrelic.api.application import application_instance
 from newrelic.api.transaction import current_transaction
 from newrelic.api.web_transaction import WebTransaction
-from newrelic.api.application import application_instance
+from newrelic.common.object_wrapper import FunctionWrapper
 from newrelic.core.attribute import truncate
 from newrelic.core.config import global_settings
 
-
 COLD_START_RECORDED = False
 MEGABYTE_IN_BYTES = 2**20
 

newrelic/api/ml_model.py

@@ -116,7 +116,8 @@ def _wrap_callback(model, content):
 
         if not isinstance(token_count_val, int) or token_count_val < 0:
             _logger.warning(
-                f"llm_token_count_callback returned an invalid value of {token_count_val}. This value must be a positive integer and will not be recorded for the token_count."
+                "llm_token_count_callback returned an invalid value of %s. This value must be a positive integer and will not be recorded for the token_count.",
+                token_count_val,
             )
             return None
 

newrelic/api/object_wrapper.py

@@ -17,7 +17,6 @@
 # These have been moved. They are retained here until all references to
 # them are moved at which point will mark as deprecated to ensure users
 # weren't using them directly.
-
 from newrelic.common.object_names import callable_name
 from newrelic.common.object_wrapper import ObjectWrapper, wrap_object
 

newrelic/api/out_function.py

@@ -14,4 +14,4 @@
 
 # Use of these from this module will be deprecated.
 
-from newrelic.common.object_wrapper import out_function, OutFunctionWrapper, wrap_out_function
+from newrelic.common.object_wrapper import OutFunctionWrapper, out_function, wrap_out_function

newrelic/api/post_function.py

@@ -14,4 +14,4 @@
 
 # Use of these from this module will be deprecated.
 
-from newrelic.common.object_wrapper import post_function, PostFunctionWrapper, wrap_post_function
+from newrelic.common.object_wrapper import PostFunctionWrapper, post_function, wrap_post_function

newrelic/api/pre_function.py

@@ -14,4 +14,4 @@
 
 # Use of these from this module will be deprecated.
 
-from newrelic.common.object_wrapper import pre_function, PreFunctionWrapper, wrap_pre_function
+from newrelic.common.object_wrapper import PreFunctionWrapper, pre_function, wrap_pre_function

newrelic/api/profile_trace.py

@@ -70,7 +70,7 @@ def _callable():
             except Exception:
                 pass
 
-            for name, obj in frame.f_globals.items():
+            for obj in frame.f_globals.values():
                 try:
                     if obj.__dict__[func_name].func_code is co:
                         return obj.__dict__[func_name]

newrelic/api/time_trace.py

@@ -211,7 +211,9 @@ def add_code_level_metrics(self, source):
                 node.add_attrs(self._add_agent_attribute)
             except Exception as exc:
                 _logger.debug(
-                    f"Failed to extract source code context from callable {source}. Report this issue to newrelic support. Exception: {exc}"
+                    "Failed to extract source code context from callable %s. Report this issue to newrelic support. Exception: %s",
+                    source,
+                    exc,
                 )
 
     def _observe_exception(self, exc_info=None, ignore=None, expected=None, status_code=None):

newrelic/api/transaction.py

@@ -1871,7 +1871,7 @@ def capture_request_params(flag=True):
     transaction = current_transaction()
     if transaction and transaction.settings:
         if transaction.settings.high_security:
-            _logger.warn("Cannot modify capture_params in High Security Mode.")
+            _logger.warning("Cannot modify capture_params in High Security Mode.")
         else:
             transaction.capture_params = flag
 

newrelic/api/transaction_name.py

@@ -15,8 +15,8 @@
 import functools
 
 from newrelic.api.transaction import current_transaction
-from newrelic.common.object_wrapper import FunctionWrapper, wrap_object
 from newrelic.common.object_names import callable_name
+from newrelic.common.object_wrapper import FunctionWrapper, wrap_object
 
 
 def TransactionNameWrapper(wrapped, name=None, group=None, priority=None):

newrelic/api/web_transaction.py

@@ -15,9 +15,8 @@
 import functools
 import logging
 import time
-import warnings
-
 import urllib.parse as urlparse
+import warnings
 
 from newrelic.api.application import Application, application_instance
 from newrelic.api.transaction import Transaction, current_transaction

newrelic/api/wsgi_application.py

@@ -96,7 +96,7 @@ def close(self):
 
 
 class _WSGIInputWrapper:
-    def __init__(self, transaction, input):
+    def __init__(self, transaction, input):  # noqa: A002
         self.__transaction = transaction
         self.__input = input
 

newrelic/common/async_proxy.py

@@ -15,7 +15,7 @@
 import logging
 import time
 
-from newrelic.common.coroutine import is_coroutine_callable, is_asyncio_coroutine, is_generator_function
+from newrelic.common.coroutine import is_asyncio_coroutine, is_coroutine_callable, is_generator_function
 from newrelic.common.object_wrapper import ObjectProxy
 from newrelic.core.trace_cache import trace_cache
 

newrelic/common/async_wrapper.py

@@ -13,11 +13,12 @@
 # limitations under the License.
 
 import functools
+
 from newrelic.common.coroutine import (
-    is_coroutine_callable,
+    is_async_generator_function,
     is_asyncio_coroutine,
+    is_coroutine_callable,
     is_generator_function,
-    is_async_generator_function,
 )