tox: Add formatting for all Python files

Add a tox entry to format all Python files using black and run this
tox command with doit. Also add a check about format as part of
lint-python

- Set the 'max-line-length' to 88 in pylintrc since it's the default for
  Black
- Do not longer ignore 'multiple-spaces-before-operator' in Salt
  pylintrc since we use Black to format all the code
- Disable error 'whitespace-before-colon' in pylintrc since it's a known
  issue between Flake8 and Black
  (Sees: https://black.readthedocs.io/en/stable/the_black_code_style.html#slices)

Refs: #2650

Author: TeddyAndrieux

.pylintrc

@@ -237,7 +237,7 @@ indent-after-paren=4
 # String used as indentation unit: 4 spaces (no tab).
 indent-string='    '
 
-max-line-length=80    # Maximum number of characters on a single line.
+max-line-length=88    # Maximum number of characters on a single line.
 max-module-lines=1000 # Maximum number of lines in a module.
 
 # List of optional constructs for which whitespace checking is disabled.

buildchain/buildchain/format.py

@@ -39,9 +39,24 @@ def format_go() -> types.TaskDict:
     }
 
 
+def format_python() -> types.TaskDict:
+    """Format Python code using black."""
+    python_files = [
+        filepath for filepath in utils.git_ls() if '.py' in filepath.suffixes
+    ]
+    return {
+        'name': 'python',
+        'title': utils.title_with_subtask_name('FORMAT'),
+        'doc': format_python.__doc__,
+        'actions': [['tox', '-e', 'format-python']],
+        'file_dep': python_files,
+    }
+
+
 # List of available formatting tasks.
 FORMATTERS: Tuple[Callable[[], types.TaskDict], ...] = (
     format_go,
+    format_python
 )
 
 

buildchain/requirements-dev.in

@@ -2,3 +2,4 @@ mypy == 0.740
 pylint ~= 2.4
 pyenchant ~= 2.0
 saltpylint == 2020.9.28
+black == 20.8b1

buildchain/requirements-dev.txt

@@ -4,10 +4,25 @@
 #
 #    tox -e pip-compile
 #
+appdirs==1.4.4 \
+    --hash=sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41 \
+    --hash=sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128 \
+    # via black
 astroid==2.4.2 \
     --hash=sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703 \
     --hash=sha256:bc58d83eb610252fd8de6363e39d4f1d0619c894b0ed24603b881c02e64c7386 \
     # via pylint
+black==20.8b1 \
+    --hash=sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea \
+    # via -r buildchain/requirements-dev.in
+click==7.1.2 \
+    --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \
+    --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc \
+    # via black
+dataclasses==0.8 \
+    --hash=sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf \
+    --hash=sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97 \
+    # via black
 isort==5.7.0 \
     --hash=sha256:c729845434366216d320e936b8ad6f9d681aab72dc7cbc2d51bedc3582f3ad1e \
     --hash=sha256:fff4f0c04e1825522ce6949973e83110a6e907750cd92d128b0d14aaaadbffdc \
@@ -45,7 +60,7 @@ modernize==0.5 \
 mypy-extensions==0.4.3 \
     --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \
     --hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8 \
-    # via mypy
+    # via black, mypy
 mypy==0.740 \
     --hash=sha256:1521c186a3d200c399bd5573c828ea2db1362af7209b2adb1bb8532cea2fb36f \
     --hash=sha256:31a046ab040a84a0fc38bc93694876398e62bc9f35eca8ccbf6418b7297f4c00 \
@@ -62,6 +77,10 @@ mypy==0.740 \
     --hash=sha256:dc889c84241a857c263a2b1cd1121507db7d5b5f5e87e77147097230f374d10b \
     --hash=sha256:f4748697b349f373002656bf32fede706a0e713d67bfdcf04edf39b1f61d46eb \
     # via -r buildchain/requirements-dev.in
+pathspec==0.8.1 \
+    --hash=sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd \
+    --hash=sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d \
+    # via black
 pycodestyle==2.6.0 \
     --hash=sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367 \
     --hash=sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e \
@@ -75,6 +94,49 @@ pylint==2.6.0 \
     --hash=sha256:bb4a908c9dadbc3aac18860550e870f58e1a02c9f2c204fdf5693d73be061210 \
     --hash=sha256:bfe68f020f8a0fece830a22dd4d5dddb4ecc6137db04face4c3420a46a52239f \
     # via -r buildchain/requirements-dev.in, saltpylint
+regex==2020.11.13 \
+    --hash=sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538 \
+    --hash=sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4 \
+    --hash=sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc \
+    --hash=sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa \
+    --hash=sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444 \
+    --hash=sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1 \
+    --hash=sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af \
+    --hash=sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8 \
+    --hash=sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9 \
+    --hash=sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88 \
+    --hash=sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba \
+    --hash=sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364 \
+    --hash=sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e \
+    --hash=sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7 \
+    --hash=sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0 \
+    --hash=sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31 \
+    --hash=sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683 \
+    --hash=sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee \
+    --hash=sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b \
+    --hash=sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884 \
+    --hash=sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c \
+    --hash=sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e \
+    --hash=sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562 \
+    --hash=sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85 \
+    --hash=sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c \
+    --hash=sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6 \
+    --hash=sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d \
+    --hash=sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b \
+    --hash=sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70 \
+    --hash=sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b \
+    --hash=sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b \
+    --hash=sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f \
+    --hash=sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0 \
+    --hash=sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5 \
+    --hash=sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5 \
+    --hash=sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f \
+    --hash=sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e \
+    --hash=sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512 \
+    --hash=sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d \
+    --hash=sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917 \
+    --hash=sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f \
+    # via black
 saltpylint==2020.9.28 \
     --hash=sha256:40fed07b859d28307d7ff02ca5b5df12763420127997016acbac7a8a32dbc80e \
     --hash=sha256:e34b8a221f3951b23469f1abb19507f70891330954965f91bbdb858e38dd987d \
@@ -86,7 +148,7 @@ six==1.15.0 \
 toml==0.10.2 \
     --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
     --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f \
-    # via pylint
+    # via black, pylint
 typed-ast==1.4.2 \
     --hash=sha256:07d49388d5bf7e863f7fa2f124b1b1d89d8aa0e2f7812faff0a5658c01c59aa1 \
     --hash=sha256:14bf1522cdee369e8f5581238edac09150c765ec1cb33615855889cf33dcb92d \
@@ -118,12 +180,12 @@ typed-ast==1.4.2 \
     --hash=sha256:d175297e9533d8d37437abc14e8a83cbc68af93cc9c1c59c2c292ec59a0697a3 \
     --hash=sha256:d746a437cdbca200622385305aedd9aef68e8a645e385cc483bdc5e488f07166 \
     --hash=sha256:e683e409e5c45d5c9082dc1daf13f6374300806240719f95dc783d1fc942af10 \
-    # via astroid, mypy
+    # via astroid, black, mypy
 typing-extensions==3.7.4.3 \
     --hash=sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918 \
     --hash=sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c \
     --hash=sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f \
-    # via mypy
+    # via black, mypy
 wrapt==1.12.1 \
     --hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7 \
     # via astroid

salt/.pylintrc

@@ -109,7 +109,7 @@ disable=R,
   repr-flag-used-in-string,
   keyword-arg-before-vararg,
   incompatible-py3-code,
-  multiple-spaces-before-operator
+  whitespace-before-colon
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

tox.ini

@@ -95,6 +95,16 @@ commands =
            --cov-fail-under=100 \
           {posargs} salt/tests/unit
 
+[testenv:format-python]
+description =
+    Format of all Python files using black.
+basepython = python3.6
+deps =
+    -r{toxinidir}/buildchain/requirements.txt
+    -r{toxinidir}/buildchain/requirements-dev.txt
+commands =
+    bash -c "black {posargs} $(git ls-files | grep -P '\.py$')"
+
 [testenv:lint-python]
 description =
     Lint Python files using pylint and mypy.
@@ -127,8 +137,9 @@ commands =
         pylint $\{untyped_files[@]\} $\{typed_files[@]\}; PYLINT_RC=$?; \
         pylint --rcfile=salt/.pylintrc --ignore=metalk8s_package_manager_apt.py $\{salt_files[@]\}; PYLINT_SALT_RC=$?; \
         mypy --strict $\{typed_files[@]\}; MYPY_RC=$?; \
-        if [[ $PYLINT_RC -gt 0 || $PYLINT_SALT_RC -gt 0 || $MYPY_RC -gt 0 ]]; then \
-            echo Failed: pylint [$PYLINT_RC] - pylint salt [$PYLINT_SALT_RC] - mypy [$MYPY_RC]; exit 1; \
+        black --check --diff $(git ls-files | grep -P '\.py$'); BLACK_RC=$?; \
+        if [[ $PYLINT_RC -gt 0 || $PYLINT_SALT_RC -gt 0 || $MYPY_RC -gt 0 || $BLACK_RC -gt 0 ]]; then \
+            echo Failed: pylint [$PYLINT_RC] - pylint salt [$PYLINT_SALT_RC] - mypy [$MYPY_RC] - black [$BLACK_RC]; exit 1; \
         else \
             echo Success!; \
         fi \