Merge pull request #427 from OSLL/352_DB_limitation

352 DB limitation

Author: HadronCollider

app/api/files.py

@@ -145,7 +145,15 @@ def upload_presentation() -> (dict, int):
             'message': 'Presentation file should not exceed {}MB.'.format(
                 Config.c.constants.presentation_file_max_size_in_megabytes
             )
-        }, 404
+        }, 413
+    # check if file can be stored (*2.5 is an estimate of pdf and preview)
+    if not DBManager().check_storage_limit(request.content_length * 2.5):
+        return {
+            'message': "Not enough space in database to store file. "
+                        "Please contact at email: "
+                        f"<a href='mailto:{Config.c.bugreport.report_mail}'>{Config.c.bugreport.report_mail}</a>"
+        }, 413
+        
     presentation_file = request.files['presentation']
 
     # check extension and mimetype of file 

app/mongo_models.py

@@ -156,3 +156,6 @@ class Logs(MongoModel):
     filename = fields.CharField()
     funcName = fields.CharField()
     lineno = fields.IntegerField()
+
+class StorageMeta(MongoModel):
+    used_size = fields.IntegerField()

app/mongo_odm.py

@@ -11,7 +11,7 @@
 from gridfs import GridFSBucket, NoFile
 from pymodm import connect
 from pymodm.connection import _get_db
-from pymodm.errors import ValidationError
+from pymodm.errors import ValidationError, DoesNotExist
 from pymodm.files import GridFSStorage
 from pymongo import ReturnDocument
 from pymongo.errors import CollectionInvalid
@@ -23,26 +23,35 @@
                               RecognizedAudioToProcess,
                               RecognizedPresentationsToProcess, Sessions,
                               TaskAttempts, TaskAttemptsToPassBack, Tasks,
-                              Trainings, TrainingsToProcess)
+                              Trainings, TrainingsToProcess, StorageMeta)
 from app.status import (AudioStatus, PassBackStatus, PresentationStatus,
                         TrainingStatus)
 from app.utils import remove_blank_and_none
 
 logger = get_root_logger()
 
-
+BYTES_PER_MB = 1024*1024    
 
 class DBManager:
     def __new__(cls):
         if not hasattr(cls, 'init_done'):
             cls.instance = super(DBManager, cls).__new__(cls)
             connect(Config.c.mongodb.url + Config.c.mongodb.database_name)
             cls.instance.storage = GridFSStorage(GridFSBucket(_get_db()))
+            cls.instance.max_size = float(Config.c.constants.storage_max_size_mbytes) * BYTES_PER_MB
             cls.init_done = True
         return cls.instance
 
     def add_file(self, file, filename=uuid.uuid4()):
-        return self.storage.save(name=filename, content=file)
+        try:
+            file.seek(0, os.SEEK_END)
+            size = file.tell()
+            file.seek(0)
+        except:
+            size = len(file)
+        _id = self.storage.save(name=filename, content=file)
+        self.update_storage_size(size)
+        return _id
 
     def read_and_add_file(self, path, filename=None):
         if filename is None:
@@ -67,7 +76,50 @@ def get_file(self, file_id):
         except (NoFile, ValidationError, InvalidId) as e:
             logger.warning('file_id = {}, {}.'.format(file_id, e))
             return None
-
+    
+    def _get_or_create_storage_meta(self):
+        try:
+            return StorageMeta.objects.get({})
+        except DoesNotExist:
+            meta = StorageMeta(used_size=0).save()
+            return meta
+    
+    def get_used_storage_size(self):
+        return self._get_or_create_storage_meta().used_size
+    
+    def set_used_storage_size(self, size):
+        meta = self._get_or_create_storage_meta()
+        meta.used_size = size
+        meta.save()
+        
+    def update_storage_size(self, deltasize):
+        meta = self._get_or_create_storage_meta()
+        meta.used_size += deltasize
+        meta.save()
+        
+    def get_max_size(self):
+        return self.max_size
+    
+    # returns Bool variable - True if file can be stored else False
+    def check_storage_limit(self, new_file_size):
+        current_size = self.get_used_storage_size()
+        inf_msg = (
+            f"Check for ability to add file: "
+            f"Current: {current_size/BYTES_PER_MB:.2f} MB, "
+            f"New file: {new_file_size/BYTES_PER_MB:.2f} MB, "
+            f"Storage size: {self.max_size/BYTES_PER_MB} MB"
+        )
+        logger.info(inf_msg)
+        return False if current_size + new_file_size > self.max_size else True
+    
+    def recalculate_used_storage_data(self):
+        total_size = 0
+        db = _get_db()
+        for file_doc in db.fs.files.find():
+            total_size += file_doc['length']
+        self.set_used_storage_size(total_size)
+        logger.info(f"Storage size recalculated: {total_size/BYTES_PER_MB:.2f} MB")
+        
 
 class TrainingsDBManager:
     def __new__(cls):

app/routes/capacity.py

@@ -0,0 +1,32 @@
+from flask import Blueprint, render_template
+from app.mongo_odm import DBManager
+from app.root_logger import get_root_logger
+from app.lti_session_passback.auth_checkers import is_admin
+
+logger = get_root_logger()
+
+BYTES_PER_MB = 1024*1024
+
+routes_capacity = Blueprint(
+    'routes_capacity', __name__, url_prefix='/capacity')
+
+@routes_capacity.route('/', methods=['GET'])
+def storage_capacity():
+    if not is_admin():
+        return {}, 404
+    current_size = DBManager().get_used_storage_size()
+    max_size = DBManager().get_max_size()
+    ratio = current_size / max_size
+    return render_template(
+        'capacity.html',
+        size=round(current_size / BYTES_PER_MB, 2),
+        max_size=round(max_size / BYTES_PER_MB, 2),
+        ratio=round(ratio * 100, 1)
+    )
+    
+@routes_capacity.route('/refresh_capacity', methods=['POST'])
+def refresh_capacity():
+    if not is_admin():
+        return {}, 404
+    DBManager().recalculate_used_storage_data()
+    return {'message': 'OK'}

app/static/js/upload.js

@@ -26,6 +26,17 @@ $(function(){
                 return;
             }
             $("#spinner").hide();
+
+            if (response.status == 413) {
+                response.json().then(responseJson => {
+                    $("#alert").show();
+                    $("#error-text").html(responseJson["message"] || "Файл слишком большой или превышает лимит хранилища");
+                });
+                $('#button-submit')[0].value = button_value;
+                
+                return;
+            }
+
             response.json().then(responseJson => {
                 if (responseJson["message"] !== "OK"){
                     $("#alert").show();

app/templates/admin.html

@@ -13,5 +13,7 @@
     <a href="{{ url_for("routes_admin.view_dumps")}}" target="_blank">{{ t("Архивы БД") }}</a>
     <br>
     <a href="{{ url_for("routes_version.view_version")}}" target="_blank">{{ t("Версия") }}</a>
+    <br>
+    <a href="{{ url_for("routes_capacity.storage_capacity")}}" target="_blank">{{ t("Заполненность БД") }}</a>
 </div>
 {% endblock %}

app/templates/capacity.html

@@ -0,0 +1,29 @@
+{% extends 'base.html' %}
+
+{% block content %}
+<div class="container mt-4">
+    <div class="card">
+        <div class="card-body">
+            <h4 class="card-title">Загруженность Базы Данных</h4>
+            <p><strong>Использовано:</strong> {{ size }} Мбайт</p>
+            <p><strong>Максимум:</strong> {{ max_size }} Мбайт</p>
+            <p><strong>Заполнено:</strong> {{ ratio }}%</p>
+            <div class="progress">
+                <div class="progress-bar" role="progressbar" style="width: {{ ratio }}%;" aria-valuenow="{{ ratio }}" aria-valuemin="0" aria-valuemax="100">
+                    {{ ratio }}
+                </div>
+            </div>
+        </div>
+        <button type="button" id="refresh_button">Обновить данные</button>
+
+        <script src="/static/js/libraries/jquery.min.js"></script>
+        <script>
+            $('#refresh_button').click(function() {
+                $.post("{{ url_for('routes_capacity.refresh_capacity') }}", {}, function(data) {
+                    location.reload();
+                });
+            });
+        </script>
+    </div>
+</div>
+{% endblock %}

app/web_speech_trainer.py

@@ -31,6 +31,7 @@
 from app.routes.trainings import routes_trainings
 from app.routes.task_attempts import routes_task_attempts
 from app.routes.version import routes_version
+from app.routes.capacity import routes_capacity
 from app.status import PassBackStatus, TrainingStatus
 from app.training_manager import TrainingManager
 from app.utils import ALLOWED_EXTENSIONS, DEFAULT_EXTENSION
@@ -55,6 +56,7 @@
 app.register_blueprint(routes_trainings)
 app.register_blueprint(routes_task_attempts)
 app.register_blueprint(routes_version)
+app.register_blueprint(routes_capacity)
 
 logger = get_root_logger(service_name='web')
 

app_conf/config.ini

@@ -5,6 +5,7 @@ lti_consumer_key=secretconsumerkey
 lti_consumer_secret=supersecretconsumersecret
 version_file=VERSION.json
 backup_path=../dump/database-dump/
+storage_max_size_mbytes=20000
 
 [mongodb]
 url=mongodb://db:27017/

app_conf/testing.ini

@@ -5,6 +5,7 @@ lti_consumer_key=testing_lti_consumer_key
 lti_consumer_secret=testing_lti_consumer_secret
 version_file=VERSION.json
 backup_path=../dump/database-dump/
+storage_max_size_mbytes=20000
 
 [mongodb]
 url=mongodb://db:27017/