add type checks (#694)

* pin dependencies

* add doctest support

* add mypy

* adjust lint and format scripts

* add types

* bump min python 3.7 -> 3.8

* update type_check flags

Author: AndreyNikiforov

.github/workflows/quality-checks.yml

@@ -13,11 +13,11 @@ on:
 
 jobs:
 
-  check_quality:
+  lint:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, '3.10', 3.11]
+        python-version: [3.8, 3.9, '3.10', 3.11]
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
@@ -31,7 +31,42 @@ jobs:
     - name: Lint
       run: |
         scripts/lint
-        
+
+  type_check: 
+    runs-on: ubuntu-20.04 
+    strategy: 
+      matrix: 
+        python-version: [3.8, 3.9, '3.10', 3.11]
+    steps: 
+    - uses: actions/checkout@v3 
+    - name: Set up Python ${{ matrix.python-version }} 
+      uses: actions/setup-python@v4 
+      with: 
+        python-version: ${{ matrix.python-version }} 
+        cache: 'pip' 
+    - name: Install Test dependencies 
+      run: | 
+        pip install -e .[test] 
+    - name: Type Check 
+      run: | 
+        scripts/type_check 
+
+  test: 
+    runs-on: ubuntu-20.04 
+    strategy: 
+      matrix: 
+        python-version: [3.8, 3.9, '3.10', 3.11]
+    steps: 
+    - uses: actions/checkout@v3 
+    - name: Set up Python ${{ matrix.python-version }} 
+      uses: actions/setup-python@v4 
+      with: 
+        python-version: ${{ matrix.python-version }} 
+        cache: 'pip' 
+    - name: Install Test dependencies 
+      run: | 
+        pip install -e .[test] 
+ 
     - name: Test
       run: |
         scripts/test

pyproject.toml

@@ -1,7 +1,7 @@
 [build-system]
 requires = [
-    "setuptools>=68.0.0,<69",
-    "wheel>=0.40.0,<0.41",
+    "setuptools==68.0.0",
+    "wheel==0.41.1",
 ]
 build-backend = "setuptools.build_meta"
 
@@ -10,7 +10,7 @@ version="1.16.1"
 name = "icloudpd"
 description = "icloudpd is a command-line tool to download photos and videos from iCloud."
 readme = "README_PYPI.md"
-requires-python = ">=3.7,<3.12"
+requires-python = ">=3.8,<3.12"
 keywords = ["icloud", "photo"]
 license = {file="LICENSE.md"}
 authors=[
@@ -23,41 +23,49 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
 ]
 dependencies = [
-    "requests>=2.28.2,<3",
-    "schema>=0.7.5,<0.8",
-    "click>=8.1.3,<9",
-    "python-dateutil>=2.8.2,<3",
-    "tqdm>=4.64.1,<5",
-    "piexif>=1.1.3,<2",
-    "urllib3>=1.26.14,<2",
+    "requests==2.31.0",
+    "schema==0.7.5",
+    "click==8.1.6",
+    "python-dateutil==2.8.2",
+    "tqdm==4.66.0",
+    "piexif==1.1.3",
+    "urllib3==1.26.16",
     # from pyicloud_ipd
-    "six>=1.16.0,<2",
-    "tzlocal>=4.2,<5",
-    "pytz>=2022.7.1,<2023",
-    "certifi>=2022.12.7,<2023",
-    "future>=0.18.3,<0.19",
-    "keyring>=23.13.1,<24",
-    "keyrings-alt>=4.2.0,<5"
+    "six==1.16.0",
+    "tzlocal==4.3.1",
+    "pytz==2022.7.1",
+    "certifi==2022.12.7",
+    "future==0.18.3",
+    "keyring==23.13.1",
+    "keyrings-alt==4.2.0"
 ]
 
 [project.optional-dependencies]
 dev = [
-    "twine>=4.0.0,<5",
-    "pyinstaller>=5.7.0,<6",
-    "wheel>=0.40.0,<0.41",
-    "auditwheel>=5.4.0,<5.5"
+    "twine==4.0.2",
+    "pyinstaller==5.13.0",
+    "wheel==0.41.1",
+    "auditwheel==5.4.0"
 ]
 test = [
-    "pytest>=7.2.1,<8",
-    "mock>=5.0.1,<6",
-    "freezegun>=1.2.2,<2",
-    "vcrpy>=4.2.1,<5",
-    "pytest-cov>=4.0.0,<5",
-    "pylint>=2.15.10,<3",
-    "coveralls>=3.3.1,<4",
-    "autopep8>=2.0.1,<3",
-    "pytest-timeout>=2.1.0,<3",
-    "pytest-xdist>=3.1.0,<4"
+    "pytest==7.4.0",
+    "mock==5.1.0",
+    "freezegun==1.2.2",
+    "vcrpy==4.4.0",
+    "pytest-cov==4.1.0",
+    "pylint==2.17.5",
+    "coveralls==3.3.1",
+    "autopep8==2.0.2",
+    "pytest-timeout==2.1.0",
+    "pytest-xdist==3.3.1",
+    "mypy==1.5.0",
+    "types-pytz==2022.7.1.2",
+    "types-tzlocal==4.3.0.0",
+    "types-requests==2.31.0.2",
+    "types-six==1.16.0",
+    "types-urllib3==1.26.16",
+    "types-tqdm==4.66.0.1",
+    "types-mock==5.1.0.1"
 ]
 
 [project.urls]
@@ -72,12 +80,18 @@ log_format = "%(levelname)-8s %(message)s"
 log_date_format = "%Y-%m-%d %H:%M:%S"
 timeout = 300
 testpaths = [
-    "tests"
+    "tests",
+    "src" # needed for doctests    
 ]
 pythonpath = [
     "src"
 ]
+addopts = "--doctest-modules"
 
 [tool.setuptools.packages.find]
 where = ["src"]  # list of folders that contain the packages (["."] by default)
 exclude = ["starters"]
+
+[[tool.mypy.overrides]]
+module = ['piexif.*', 'future.*', 'vcr.*']
+ignore_missing_imports = true

scripts/format

@@ -1,3 +1,4 @@
 #!/bin/bash
 set -euo pipefail
-autopep8 -r --in-place --aggressive --aggressive icloudpd/
+echo "Running autopep8..."
+autopep8 -r --in-place --aggressive --aggressive src/ --exclude src/pyicloud_ipd

scripts/lint

@@ -1,4 +1,4 @@
 #!/bin/bash
 set -euo pipefail
 echo "Running pylint..."
-python3 -m pylint src/icloudpd
+python3 -m pylint src --ignore-paths src/icloud,src/pyicloud_ipd,src/starters

scripts/run_all_checks

@@ -11,4 +11,5 @@ ROOT_DIR="$(realpath $(dirname "$0")/..)"
 
 $CURRENT_DIR/format
 $CURRENT_DIR/lint
+$CURRENT_DIR/type_check
 $CURRENT_DIR/test

scripts/type_check

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -euo pipefail
+echo "Running mypy..."
+python3 -m mypy src tests 
+# too strict now: --disallow-any-generics --disallow-untyped-defs --strict-equality --disallow-untyped-calls --warn-return-any

src/icloudpd/authentication.py

@@ -1,5 +1,6 @@
 """Handles username/password authentication and two-step authentication"""
 
+import logging
 import sys
 import click
 import pyicloud_ipd
@@ -12,15 +13,15 @@ class TwoStepAuthRequiredError(Exception):
     """
 
 
-def authenticator(logger, domain):
+def authenticator(logger: logging.Logger, domain: str):
     """Wraping authentication with domain context"""
     def authenticate_(
             username,
             password,
             cookie_directory=None,
             raise_error_on_2sa=False,
             client_id=None,
-    ):
+    ) -> pyicloud_ipd.PyiCloudService:
         """Authenticate with iCloud username and password"""
         logger.debug("Authenticating...")
         while True:
@@ -49,7 +50,7 @@ def authenticate_(
     return authenticate_
 
 
-def request_2sa(icloud, logger):
+def request_2sa(icloud: pyicloud_ipd.PyiCloudService, logger: logging.Logger):
     """Request two-step authentication. Prompts for SMS or device"""
     devices = icloud.trusted_devices
     devices_count = len(devices)

src/icloudpd/autodelete.py

@@ -1,30 +1,32 @@
 """
 Delete any files found in "Recently Deleted"
 """
+import logging
 import os
 from tzlocal import get_localzone
 from icloudpd.paths import local_download_path
+import pyicloud_ipd
 
 
-def delete_file(logger, path) -> bool:
+def delete_file(logger: logging.Logger, path: str) -> bool:
     """ Actual deletion of files """
     os.remove(path)
     logger.info("Deleted %s", path)
     return True
 
 
-def delete_file_dry_run(logger, path) -> bool:
+def delete_file_dry_run(logger: logging.Logger, path: str) -> bool:
     """ Dry run deletion of files """
     logger.info("[DRY RUN] Would delete %s", path)
     return True
 
 
 def autodelete_photos(
-        logger,
-        dry_run,
+        logger: logging.Logger,
+        dry_run: bool,
         library_object,
-        folder_structure,
-        directory):
+        folder_structure: str,
+        directory: str):
     """
     Scans the "Recently Deleted" folder and deletes any matching files
     from the download directory.
@@ -46,7 +48,7 @@ def autodelete_photos(
         date_path = folder_structure.format(created_date)
         download_dir = os.path.join(directory, date_path)
 
-        for size in [None, "original", "medium", "thumb"]:
+        for size in ["small", "original", "medium", "thumb"]:
             path = os.path.normpath(
                 local_download_path(
                     media, size, download_dir))