|
| 1 | +import datetime |
| 2 | +import logging |
| 3 | +import time |
| 4 | +from typing import Callable, Dict, Optional, Union |
| 5 | + |
| 6 | +from django.conf import settings |
| 7 | +from django.db import DEFAULT_DB_ALIAS, connections, models |
| 8 | +from django.db.models import QuerySet |
| 9 | +from django.db.models.base import ModelBase |
| 10 | + |
| 11 | +__all__ = [ |
| 12 | + "MaterializedViewModel", |
| 13 | + "DBViewsRegistry", |
| 14 | +] |
| 15 | + |
| 16 | +from django_materialized_view.models import MaterializedViewRefreshLog |
| 17 | + |
| 18 | +logger = logging.getLogger(__name__) |
| 19 | + |
| 20 | +DBViewsRegistry: Dict[str, "MaterializedViewModel"] = {} |
| 21 | + |
| 22 | + |
| 23 | +class DBViewModelBase(ModelBase): |
| 24 | + def __new__(mcs, *args, **kwargs): |
| 25 | + new_class = super().__new__(mcs, *args, **kwargs) |
| 26 | + assert new_class._meta.managed is False, "For DB View managed must be se to false" # noqa |
| 27 | + if new_class._meta.abstract is False: # noqa |
| 28 | + DBViewsRegistry[new_class._meta.db_table] = new_class # noqa |
| 29 | + return new_class |
| 30 | + |
| 31 | + |
| 32 | +class DBMaterializedView(models.Model, metaclass=DBViewModelBase): |
| 33 | + """ |
| 34 | + Children should define: |
| 35 | + view_definition - define the view, can be callable or attribute (string) |
| 36 | + """ |
| 37 | + |
| 38 | + view_definition: Union[Callable, str, dict] |
| 39 | + |
| 40 | + class Meta: |
| 41 | + managed = False |
| 42 | + abstract = True |
| 43 | + |
| 44 | + @classmethod |
| 45 | + def refresh(cls, using: Optional[str] = None, concurrently: bool = False): |
| 46 | + """ |
| 47 | + concurrently option requires an index and postgres db |
| 48 | + """ |
| 49 | + using = using or DEFAULT_DB_ALIAS |
| 50 | + with connections[using].cursor() as cursor: |
| 51 | + if concurrently: |
| 52 | + cursor.execute("REFRESH MATERIALIZED VIEW CONCURRENTLY %s;" % cls._meta.db_table) |
| 53 | + else: |
| 54 | + cursor.execute("REFRESH MATERIALIZED VIEW %s;" % cls._meta.db_table) |
| 55 | + |
| 56 | + |
| 57 | +class MaterializedViewModel(DBMaterializedView): |
| 58 | + """ |
| 59 | + 1. create class and inherit from `MaterializedViewModel` |
| 60 | + EXAMPLE: |
| 61 | + ---------------------------------------------------------------- |
| 62 | + from django.db import models |
| 63 | + from core.models.materialized_view import MaterializedViewModel |
| 64 | +
|
| 65 | + class JiraTimeestimateChangelog(MaterializedViewModel): |
| 66 | + create_pkey_index = True # if you need add uniq field as a primary kay and create indexes |
| 67 | +
|
| 68 | + class Meta: |
| 69 | + managed = False |
| 70 | +
|
| 71 | + # if create_pkey_index = True you must identify primary_key=True |
| 72 | + item = models.OneToOneField("jira.Item", on_delete=models.DO_NOTHING, primary_key=True, db_column="id") |
| 73 | + from_seconds = models.IntegerField() |
| 74 | + to_seconds = models.IntegerField() |
| 75 | + type = models.CharField(max_length=255) |
| 76 | + ---------------------------------------------------------------- |
| 77 | +
|
| 78 | + 2. create materialized view query .sql file |
| 79 | + 1. run migrate_with_views command for getting your new sql file name and path |
| 80 | + ./manage.py migrate_with_views |
| 81 | + 2. you will get file path in your console |
| 82 | + [Errno 2] No such file or directory: |
| 83 | + '..../models/materialized_view/sql_files/jiradetailedstatusitem.sql' |
| 84 | + - please create SQL file and put it to this directory |
| 85 | + 3. create file on suggested path with suggested name |
| 86 | + 4. run again `./manage.py migrate_with_views` - |
| 87 | + this command will run default migrate command and apply materialized views |
| 88 | +
|
| 89 | + 3. add your materialized view to update cron job |
| 90 | + - `JiraTimeestimateChangelog.refresh()` |
| 91 | + """ |
| 92 | + |
| 93 | + create_pkey_index = False |
| 94 | + |
| 95 | + class Meta: |
| 96 | + managed = False |
| 97 | + abstract = True |
| 98 | + |
| 99 | + @classmethod |
| 100 | + def refresh(cls, using: Optional[str] = None, concurrently: bool = True) -> None: |
| 101 | + log = MaterializedViewRefreshLog( |
| 102 | + view_name=cls.get_tablename(), |
| 103 | + ) |
| 104 | + try: |
| 105 | + start_time = time.monotonic() |
| 106 | + super().refresh(using=using, concurrently=concurrently) |
| 107 | + end_time = time.monotonic() |
| 108 | + log.duration = datetime.timedelta(seconds=end_time - start_time) |
| 109 | + except Exception: # noqa |
| 110 | + log.failed = True |
| 111 | + logging.exception(f"failed to refresh materialized view {cls.get_tablename()}") |
| 112 | + log.save() |
| 113 | + |
| 114 | + @classmethod |
| 115 | + def view_definition(cls): |
| 116 | + return cls.__get_query() |
| 117 | + |
| 118 | + @classmethod |
| 119 | + def __create_index_for_primary_key(cls) -> str: |
| 120 | + try: |
| 121 | + if not cls.create_pkey_index: |
| 122 | + return "" |
| 123 | + if cls._meta.pk.db_column: |
| 124 | + primary_key_field = cls._meta.pk.db_column |
| 125 | + else: |
| 126 | + primary_key_field = cls._meta.pk.attname |
| 127 | + return f"CREATE UNIQUE INDEX {cls.get_tablename()}_pkey ON {cls.get_tablename()} ({primary_key_field})" |
| 128 | + except Exception as exc: |
| 129 | + print(exc) |
| 130 | + exit(-1) |
| 131 | + |
| 132 | + @classmethod |
| 133 | + def __get_sql_file_path(cls) -> str: |
| 134 | + return ( |
| 135 | + f"{settings.SRC_DIR}/{cls.__get_app_label()}" |
| 136 | + f"/models/materialized_views/sql_files/{cls.__get_class_name()}.sql" |
| 137 | + ) |
| 138 | + |
| 139 | + @staticmethod |
| 140 | + def get_query_from_queryset() -> QuerySet: |
| 141 | + """ |
| 142 | + redefine this method if you need create materialized view based on queryset instead sql file |
| 143 | + example: |
| 144 | + @staticmethod |
| 145 | + def get_query_from_queryset() -> QuerySet: |
| 146 | + return User.objects.all() |
| 147 | + """ |
| 148 | + pass |
| 149 | + |
| 150 | + @classmethod |
| 151 | + def __get_query(cls) -> str: |
| 152 | + queryset = cls.get_query_from_queryset() |
| 153 | + |
| 154 | + if isinstance(queryset, QuerySet): |
| 155 | + sql_query = f"{queryset.query}; {cls.__create_index_for_primary_key()}" |
| 156 | + return sql_query |
| 157 | + try: |
| 158 | + with open(cls.__get_sql_file_path(), "r") as sql_file: |
| 159 | + sql_query = f"{sql_file.read()}; {cls.__create_index_for_primary_key()}" |
| 160 | + except FileNotFoundError as exc: |
| 161 | + raise FileNotFoundError(f"{exc}, - please create SQL file and put it to this directory") |
| 162 | + return sql_query |
| 163 | + |
| 164 | + @classmethod |
| 165 | + def __get_class_name(cls) -> str: |
| 166 | + return cls.__name__.casefold() |
| 167 | + |
| 168 | + @classmethod |
| 169 | + def __get_app_label(cls) -> str: |
| 170 | + return cls._meta.app_label |
| 171 | + |
| 172 | + @classmethod |
| 173 | + def get_tablename(cls) -> str: |
| 174 | + return f"{cls.__get_app_label()}_{cls.__get_class_name()}" |
| 175 | + |
| 176 | + |
0 commit comments