Merge pull request #109 from mkurnikov/new-semanal

New semanal support, rewrite to use Django app registry

Author: mkurnikov

Diff for: .gitignore

@@ -5,7 +5,7 @@ out/
 /django
 .idea/
 .mypy_cache/
-django-sources
 build/
 dist/
-pip-wheel-metadata/
+pip-wheel-metadata/
+.pytest_cache/

Diff for: .gitmodules

@@ -0,0 +1,4 @@
+[submodule "django-sources"]
+	path = django-sources
+	url = https://github.com/django/django.git
+	branch = stable/2.2.x

Diff for: .travis.yml

@@ -10,15 +10,17 @@ jobs:
         set -e
         pytest
 
-    - name: Run plugin test suite with python 3.6
+    - name: Typecheck Django test suite with python 3.7
+      python: 3.7
+      script: 'python ./scripts/typecheck_tests.py'
+
+    - name: Typecheck Django test suite with python 3.6
       python: 3.6
-      script: |
-        set -e
-        pytest
+      script: 'python ./scripts/typecheck_tests.py'
 
-    - name: Typecheck Django test suite
+    - name: Mypy for plugin code
       python: 3.7
-      script: 'python ./scripts/typecheck_tests.py'
+      script: 'mypy ./mypy_django_plugin'
 
     - name: Lint with black
       python: 3.7
@@ -36,13 +38,4 @@ before_install: |
   # Upgrade pip, setuptools, and wheel
   pip install -U pip setuptools wheel
 install: |
-  pip install -r ./dev-requirements.txt
-  pip install -r ./scripts/typecheck-tests-requirements.txt
-
-#deploy:
-#  provider: pypi
-#  user: "mkurnikov"
-#  password:
-#    secure: 0E+hkaIdtpEtyL1KZeglunZ5/PKjouFfa8ljakAwoig7VNUL+2sO/bTyg38wRQl0NvzDzEHSMEt1bzg4Tq7b7Zp6nLuewG/w7mGLzqaOlTySiPEfRsg8s6uO2KrTn7g9VhlXH6UtyTXoQdMt6aE8+bt/GmEesanS57NB2mhwmylFgQwlJFu4LfIv/+aGmc4eLeGI2Qhvs9QYf7qvYlLQldgFh8mAckQEEvaBg35sf+puypZgf4nkx1k/dfG9wnFWZU8PJ41LbMw/Wj+k/9NpF8ePwiAr0fvRMErZd8nvoiWjQQjhzgrLVHhXEP5pTHh3zjDuGFMWyKuBhC6WLsG4qOQz/HvxeYvNI+jaTp15BgxtefG/pCNDUl/8GlCde7xVt7xzEcYNJSRaZPY2oofEFSd9qDnr4kqmyCXpNsaHRHvkL61bFjXUcfOsMMYvQCC6N2Jjb7S97RbnDdkOZO/lnFhVANT2rigsaXlSlWyN6f7ApxDNvu6Ehu5yrx6IjlPZJ0sI9vvY3IoS6Fik7w9E6zjNVjbmUn1D4MKFP4v5ppNASOqYcZeLd42j8rjEp0gIc3ccz9aUIT9q8VqSXSdUbqA6SVwvHXIVPxJMXj0bqWBG1iKs0cPBuzRVpRrwkENWCSWElDAewM1qFEnK0LppyoYFbqoQ8F5FG0+re7QttKQ=
-#  on:
-#    tags: true
+  pip install -r ./dev-requirements.txt

Diff for: README.md

@@ -17,6 +17,10 @@ Could be run on earlier versions of Django, but expect some missing imports warn
 pip install django-stubs
 ```
 
+### WARNING: All configuration from pre-1.0.0 versions is dropped, use one below.
+
+### WARNING: 1.0.0 breaks `dmypy`, if you need it, stay on the 0.12.x series. 
+
 To make mypy aware of the plugin, you need to add
 
 ```
@@ -27,27 +31,33 @@ plugins =
 
 in your `mypy.ini` file.
 
+Plugin requires Django settings module (what you put into `DJANGO_SETTINGS_MODULE` variable) to be specified inside `mypy.ini` file.
+```
+[mypy]
+strict_optional = True
 
-## Configuration
-
-In order to specify config file, set `MYPY_DJANGO_CONFIG` environment variable with path to the config file. Default is `./mypy_django.ini`
-
-Config file format (.ini):
+; this one is new
+[mypy.plugins.django-stubs]
+django_settings_module = mysettings
 ```
-[mypy_django_plugin]
+where `mysettings` is a value of `DJANGO_SETTINGS_MODULE` (with or without quotes)
+
+New implementation uses Django runtime to extract models information, so it will crash, if your installed apps `models.py` is not correct. For this same reason, you cannot use `reveal_type` inside global scope of any Python file that will be executed for `django.setup()`. 
 
-# specify settings module to use for django.conf.settings, this setting
-# could also be specified with DJANGO_SETTINGS_MODULE environment variable
-# (it also takes priority over config file)
-django_settings = mysettings.local
+In other words, if your `manage.py runserver` crashes, mypy will crash too. 
 
-# if True, all unknown settings in django.conf.settings will fallback to Any,
-# specify it if your settings are loaded dynamically to avoid false positives
-ignore_missing_settings = True
+## Notes
 
-# if True, unknown attributes on Model instances won't produce errors
-ignore_missing_model_attributes = True
+Implementation monkey-patches Django to add `__class_getitem__` to the `Manager` class. If you'd use Python3.7 and do that too in your code, you can make things like
 ```
+class MyUserManager(models.Manager['MyUser']):
+    pass
+class MyUser(models.Model):
+    objects = UserManager()
+```
+work, which should make a error messages a bit better. 
+
+Otherwise, custom type will be created in mypy, named `MyUser__MyUserManager`, which will rewrite base manager as `models.Manager[User]` to make methods like `get_queryset()` and others return properly typed `QuerySet`. 
 
 ## To get help
 

Diff for: dev-requirements.txt

@@ -1,5 +1,5 @@
 black
-pytest-mypy-plugins
+pytest-mypy-plugins==1.0.3
 flake8
 isort==4.3.4
 -e .

Diff for: django-sources

@@ -1,30 +1,31 @@
-import collections
 import threading
-from typing import Any, Callable, List, Optional, Tuple, Type, Union, Iterable, DefaultDict
+from collections import OrderedDict
+from typing import Any, Callable, List, Optional, Tuple, Type, Union, Iterable, DefaultDict, Dict
 
 from django.db.migrations.state import AppConfigStub
 from django.db.models.base import Model
 
 from .config import AppConfig
 
 class Apps:
-    all_models: collections.defaultdict = ...
-    app_configs: collections.OrderedDict = ...
+    all_models: "Dict[str, OrderedDict[str, Type[Model]]]" = ...
+    app_configs: "OrderedDict[str, AppConfig]" = ...
     stored_app_configs: List[Any] = ...
     apps_ready: bool = ...
     ready_event: threading.Event = ...
     loading: bool = ...
     _pending_operations: DefaultDict[Tuple[str, str], List]
-    def __init__(self, installed_apps: Optional[Union[List[AppConfigStub], List[str], Tuple]] = ...) -> None: ...
     models_ready: bool = ...
     ready: bool = ...
+    def __init__(self, installed_apps: Optional[Union[List[AppConfigStub], List[str], Tuple]] = ...) -> None: ...
     def populate(self, installed_apps: Union[List[AppConfigStub], List[str], Tuple] = ...) -> None: ...
     def check_apps_ready(self) -> None: ...
     def check_models_ready(self) -> None: ...
     def get_app_configs(self) -> Iterable[AppConfig]: ...
     def get_app_config(self, app_label: str) -> AppConfig: ...
-    def get_models(self, include_auto_created: bool = ..., include_swapped: bool = ...) -> List[Type[Model]]: ...
-    def get_model(self, app_label: str, model_name: Optional[str] = ..., require_ready: bool = ...) -> Type[Model]: ...
+    # it's not possible to support it in plugin properly now
+    def get_models(self, include_auto_created: bool = ..., include_swapped: bool = ...) -> List[Type[Any]]: ...
+    def get_model(self, app_label: str, model_name: Optional[str] = ..., require_ready: bool = ...) -> Type[Any]: ...
     def register_model(self, app_label: str, model: Type[Model]) -> None: ...
     def is_installed(self, app_name: str) -> bool: ...
     def get_containing_app_config(self, object_name: str) -> Optional[AppConfig]: ...

Diff for: django-stubs/apps/registry.pyi

@@ -1,12 +1,16 @@
-from typing import Any, Optional, Tuple, List, overload
+from typing import Any, Optional, Tuple, List, overload, TypeVar
+
+from django.db.models.base import Model
 
 from django.db import models
 
-class BaseUserManager(models.Manager):
+_T = TypeVar("_T", bound=Model)
+
+class BaseUserManager(models.Manager[_T]):
     @classmethod
     def normalize_email(cls, email: Optional[str]) -> str: ...
     def make_random_password(self, length: int = ..., allowed_chars: str = ...) -> str: ...
-    def get_by_natural_key(self, username: Optional[str]) -> AbstractBaseUser: ...
+    def get_by_natural_key(self, username: Optional[str]) -> _T: ...
 
 class AbstractBaseUser(models.Model):
     password: models.CharField = ...

Diff for: django-stubs/contrib/auth/base_user.pyi

@@ -1,10 +1,11 @@
-from typing import Any, Collection, Optional, Set, Tuple, Type, Union
+from typing import Any, Collection, Optional, Set, Tuple, Type, TypeVar
 
 from django.contrib.auth.base_user import AbstractBaseUser as AbstractBaseUser, BaseUserManager as BaseUserManager
+from django.contrib.auth.validators import UnicodeUsernameValidator
 from django.contrib.contenttypes.models import ContentType
+from django.db.models.base import Model
 from django.db.models.manager import EmptyManager
 
-from django.contrib.auth.validators import UnicodeUsernameValidator
 from django.db import models
 
 def update_last_login(sender: Type[AbstractBaseUser], user: AbstractBaseUser, **kwargs: Any) -> None: ...
@@ -27,13 +28,15 @@ class Group(models.Model):
     permissions: models.ManyToManyField = models.ManyToManyField(Permission)
     def natural_key(self): ...
 
-class UserManager(BaseUserManager):
+_T = TypeVar("_T", bound=Model)
+
+class UserManager(BaseUserManager[_T]):
     def create_user(
         self, username: str, email: Optional[str] = ..., password: Optional[str] = ..., **extra_fields: Any
-    ) -> AbstractUser: ...
+    ) -> _T: ...
     def create_superuser(
         self, username: str, email: Optional[str], password: Optional[str], **extra_fields: Any
-    ) -> AbstractBaseUser: ...
+    ) -> _T: ...
 
 class PermissionsMixin(models.Model):
     is_superuser: models.BooleanField = ...

Diff for: django-stubs/contrib/auth/models.pyi

@@ -7,13 +7,18 @@ from django.db.models.fields.related import ForeignObject
 from django.db.models.fields.related_descriptors import ReverseManyToOneDescriptor
 from django.db.models.fields.reverse_related import ForeignObjectRel
 
+from django.db.models.expressions import Combinable
 from django.db.models.fields import Field, PositiveIntegerField
 from django.db.models.fields.mixins import FieldCacheMixin
 from django.db.models.query import QuerySet
 from django.db.models.query_utils import FilteredRelation, PathInfo
 from django.db.models.sql.where import WhereNode
 
 class GenericForeignKey(FieldCacheMixin):
+    # django-stubs implementation only fields
+    _pyi_private_set_type: Union[Any, Combinable]
+    _pyi_private_get_type: Any
+    # attributes
     auto_created: bool = ...
     concrete: bool = ...
     editable: bool = ...
@@ -44,36 +49,21 @@ class GenericForeignKey(FieldCacheMixin):
     def get_prefetch_queryset(
         self, instances: Union[List[Model], QuerySet], queryset: Optional[QuerySet] = ...
     ) -> Tuple[List[Model], Callable, Callable, bool, str, bool]: ...
-    def __get__(
-        self, instance: Optional[Model], cls: Type[Model] = ...
-    ) -> Optional[Union[GenericForeignKey, Model]]: ...
-    def __set__(self, instance: Model, value: Optional[Model]) -> None: ...
+    def __get__(self, instance: Optional[Model], cls: Type[Model] = ...) -> Optional[Any]: ...
+    def __set__(self, instance: Model, value: Optional[Any]) -> None: ...
 
 class GenericRel(ForeignObjectRel):
     field: GenericRelation
-    limit_choices_to: Optional[Union[Dict[str, Any], Callable[[], Any]]]
-    model: Type[Model]
-    multiple: bool
-    on_delete: Callable
-    parent_link: bool
-    related_name: str
-    related_query_name: None
-    symmetrical: bool
     def __init__(
         self,
         field: GenericRelation,
         to: Union[Type[Model], str],
-        related_name: None = ...,
+        related_name: Optional[str] = ...,
         related_query_name: Optional[str] = ...,
         limit_choices_to: Optional[Union[Dict[str, Any], Callable[[], Any]]] = ...,
     ) -> None: ...
 
 class GenericRelation(ForeignObject):
-    auto_created: bool = ...
-    many_to_many: bool = ...
-    many_to_one: bool = ...
-    one_to_many: bool = ...
-    one_to_one: bool = ...
     rel_class: Any = ...
     mti_inherited: bool = ...
     object_id_field_name: Any = ...
@@ -90,23 +80,16 @@ class GenericRelation(ForeignObject):
         limit_choices_to: Optional[Union[Dict[str, Any], Callable[[], Any]]] = ...,
         **kwargs: Any
     ) -> None: ...
-    def check(self, **kwargs: Any) -> List[Error]: ...
     def resolve_related_fields(self) -> List[Tuple[PositiveIntegerField, Field]]: ...
     def get_path_info(self, filtered_relation: Optional[FilteredRelation] = ...) -> List[PathInfo]: ...
     def get_reverse_path_info(self, filtered_relation: None = ...) -> List[PathInfo]: ...
     def value_to_string(self, obj: Model) -> str: ...
-    model: Any = ...
-    def set_attributes_from_rel(self) -> None: ...
-    def get_internal_type(self) -> str: ...
     def get_content_type(self) -> ContentType: ...
     def get_extra_restriction(
         self, where_class: Type[WhereNode], alias: Optional[str], remote_alias: str
     ) -> WhereNode: ...
     def bulk_related_objects(self, objs: List[Model], using: str = ...) -> QuerySet: ...
 
-class ReverseGenericManyToOneDescriptor(ReverseManyToOneDescriptor):
-    field: GenericRelation
-    rel: GenericRel
-    def related_manager_cls(self): ...
+class ReverseGenericManyToOneDescriptor(ReverseManyToOneDescriptor): ...
 
 def create_generic_related_manager(superclass: Any, rel: Any): ...

Diff for: django-stubs/contrib/contenttypes/fields.pyi

@@ -5,7 +5,7 @@ from django.db.models import Field
 from django.db.models.lookups import Transform
 from .mixins import CheckFieldDefaultMixin
 
-class JsonAdapter(object):
+class JsonAdapter:
     encoder: Any = ...
     def __init__(self, adapted: Any, dumps: Optional[Any] = ..., encoder: Optional[Any] = ...) -> None: ...
     def dumps(self, obj: Any): ...
@@ -22,12 +22,7 @@ class JSONField(CheckFieldDefaultMixin, Field):
         encoder: Optional[Type[JSONEncoder]] = ...,
         **kwargs: Any
     ) -> None: ...
-    def db_type(self, connection: Any): ...
-    def get_transform(self, name: Any): ...
-    def get_prep_value(self, value: Any): ...
-    def validate(self, value: Any, model_instance: Any) -> None: ...
     def value_to_string(self, obj: Any): ...
-    def formfield(self, **kwargs: Any): ...
 
 class KeyTransform(Transform):
     operator: str = ...

Diff for: django-stubs/contrib/postgres/fields/jsonb.pyi

@@ -1,15 +1,6 @@
-from typing import Any, List, Optional
-
-from django.core.checks.messages import Error
-from django.db.models.query import QuerySet
+from typing import Optional
 
 from django.db import models
 
 class CurrentSiteManager(models.Manager):
-    creation_counter: int
-    model: None
-    name: None
-    use_in_migrations: bool = ...
     def __init__(self, field_name: Optional[str] = ...) -> None: ...
-    def check(self, **kwargs: Any) -> List[Error]: ...
-    def get_queryset(self) -> QuerySet: ...

Diff for: django-stubs/contrib/sites/managers.pyi

@@ -1,11 +1,11 @@
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Tuple, Union
 
-from django.core.management.base import BaseCommand as BaseCommand, CommandError as CommandError
+from .base import BaseCommand as BaseCommand, CommandError as CommandError
 
 def find_commands(management_dir: str) -> List[str]: ...
 def load_command_class(app_name: str, name: str) -> BaseCommand: ...
 def get_commands() -> Dict[str, str]: ...
-def call_command(command_name: Union[Tuple[str], BaseCommand, str], *args: Any, **options: Any) -> Optional[str]: ...
+def call_command(command_name: Union[Tuple[str], BaseCommand, str], *args: Any, **options: Any) -> str: ...
 
 class ManagementUtility:
     argv: List[str] = ...