Merge pull request #281 from dclong/dev

Merge dev into main

Author: dclong

dsutil/__init__.py

@@ -3,4 +3,4 @@
 from . import git
 from . import poetry
 
-__version__ = "0.61.3"
+__version__ = "0.61.4"

dsutil/hadoop/log.py

@@ -65,6 +65,7 @@ class LogFilter:
         "error",
         "Exception",
         "exception",
+        "failures",
     )
     PATTERNS = (
         (r"\d{2,}[-/]\d{2,}[-/]\d{2,}\s\d+:\d+:\d+", "YYYY/MM/DD HH:MM:SS"),
@@ -290,6 +291,11 @@ def _error_priority(line: str) -> tuple[int, str, str]:
                 1,
                 "http://www.legendu.net/misc/blog/spark-issue-InsertOperationConflictException-failed-to-hold-insert-operation-lock"
             ),
+            (
+                r"(?i)max number of executor failures",
+                1,
+                "http://www.legendu.net/misc/blog/spark-issue:-max-number-of-executor-failures-reached"
+            ),
             (
                 "(?i)serialized results is bigger than spark.driver.maxResultSize", 2,
                 "http://www.legendu.net/misc/blog/spark-issues-total-size-bigger-than-maxresultsize/"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dsutil"
-version = "0.61.3"
+version = "0.61.4"
 description = "A utils Python package for data scientists."
 authors = ["Benjamin Du <[email protected]>"]
 

readme.md

@@ -25,7 +25,7 @@ Currently, Python 3.7 and 3.8 are supported.
 
 You can download a copy of the latest release and install it using pip.
 ```
-pip3 install --user -U https://github.com/dclong/dsutil/releases/download/v0.61.3/dsutil-0.61.3-py3-none-any.whl
+pip3 install --user -U https://github.com/dclong/dsutil/releases/download/v0.61.4/dsutil-0.61.4-py3-none-any.whl
 ```
 Or you can use the following command to install the latest master branch
 if you have pip 20.0+.