Merge pull request #503 from moevm/494_selenium+docker

494 selenium+docker

Author: HadronCollider

.env_example

@@ -10,11 +10,11 @@ CONSUMER_SECRET=secret
 WEB_PORT=8080
 
 REDIS_URL=redis://redis:6379/0
-CELERY_QUERIES=query
+CELERY_QUERIES=check-solution
 
 FLOWER_SECRET_KEY=123
 FLOWER_PORT=5555
 FLOWER_AUTH=user:user
 FLOWER_PREFIX=/
 
-MONGODB_CACHE_SIZE=1
+MONGODB_CACHE_SIZE=1

.github/workflows/selenium_tests.yml

@@ -0,0 +1,45 @@
+name: Selenium_tests
+
+on: [push]
+
+jobs:
+  check_tests:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build docker-compose with docker-compose-selenium (tests)
+        run: |
+          cp .env_example .env
+          cp app/VERSION_example.json app/VERSION.json
+          docker-compose -f docker-compose.yml -f docker-compose-selenium.yml build
+      
+      - name: Run docker-compose with docker-compose-selenium (tests)
+        run: |
+          docker-compose -f docker-compose.yml -f docker-compose-selenium.yml up -d
+          chmod +x tests/scripts/docker_check_tests.sh
+          ./tests/scripts/docker_check_tests.sh
+
+
+# jobs:
+#   check_tests:
+#     runs-on: ubuntu-20.04
+
+#     steps:
+#       - uses: actions/checkout@v2
+
+#       - name: Simplify docker-compose file name
+#         run: mv docker-compose-tests.yml docker-compose.yml
+
+#       - name: Build system images (non-pulling)
+#         run: |
+#           # build base image
+#           docker build -f Dockerfile_base -t osll/slides-base:20230202 .
+
+#       - name: Run docker-compose tests
+#         run: |
+#           cp .env_example .env
+#           docker-compose up -d
+#           chmod +x tests/scripts/docker_check_tests.sh
+#           ./tests/scripts/docker_check_tests.sh

Dockerfile_selenium

@@ -0,0 +1,17 @@
+FROM selenium/standalone-chrome:121.0-chromedriver-121.0-grid-4.18.0-20240220
+
+ENV LOGIN=""
+ENV PASSWORD=""
+
+WORKDIR /usr/src/project
+
+USER root
+RUN apt-get update && \
+    apt-get install -y python3 python3-pip && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY tests/requirements.txt tests/requirements.txt
+RUN pip install -r tests/requirements.txt
+COPY tests ./tests
+
+ENTRYPOINT python3 tests/main.py --login ${LOGIN} --password ${PASSWORD}

app/VERSION_example.json

@@ -0,0 +1,6 @@
+{
+"commit":  "",
+"message":  "",
+"date":    "",
+"version": ""
+}

app/main/checks/report_checks/needed_headers_check.py

@@ -23,8 +23,8 @@ def late_init(self):
         self.headers = self.file.make_chapters(self.file_type['report_type'])
         self.headers_page = self.file.find_header_page(self.file_type['report_type'])
         self.chapters_str = self.file.show_chapters(self.file_type['report_type'])
-        self.headers_main = self.file.get_main_headers()
         # TODO: change
+        self.headers_main = self.file.get_main_headers(self.file_type['report_type'])
         if self.headers_main == "Задание 1":
             self.patterns = StyleCheckSettings.CONFIGS.get(self.config)[0]["headers"]
         elif self.headers_main == "Задание 2":

app/main/reports/document_uploader.py

@@ -48,3 +48,6 @@ def find_literature_page(self, work_type):
     @abstractmethod
     def show_chapters(self, work_type):
         pass
+
+    def get_main_headers(self):
+        pass

app/main/reports/docx_uploader/docx_uploader.py

@@ -18,6 +18,7 @@ def __init__(self):
         self.inline_shapes = []
         self.core_properties = None
         self.headers = []
+        self.headers_main = ''
         self.file = None
         self.special_paragraph_indices = {}
         self.headers_page = 0
@@ -90,6 +91,12 @@ def make_headers(self, work_type):
                                 break
                 self.headers = headers
         return self.headers
+    
+    def get_main_headers(self, work_type): #this method helps to avoid mistake in "needed_headers_check" (because of structure checks for md)
+        if not self.headers_main:
+            if work_type == 'VKR':
+                self.headers_main = self.make_headers(work_type)[1]['name']
+        return self.headers_main
 
     def __make_table(self, tables):
         for i in range(len(tables)):

app/main/reports/md_uploader/md_uploader.py

@@ -95,7 +95,7 @@ def page_counter(self): # we need this just to find a last page and make link to
             self.page_count = 5 
         return self.page_count
 
-    def get_main_headers(self):
+    def get_main_headers(self, work_type):
         if not self.headers_main:
             header_main_regex = "<h1>(.*?)<\/h1>"
             self.headers_main = re.findall(header_main_regex, self.html_text)[0]

app/nlp/stemming.py

@@ -1,12 +1,8 @@
 import itertools
-
-import nltk
 from nltk.corpus import stopwords
 from nltk.tokenize import word_tokenize, sent_tokenize
 from pymorphy2 import MorphAnalyzer
 
-nltk.download('stopwords')
-nltk.download('punkt')
 
 MORPH_ANALYZER = MorphAnalyzer()
 TASKS = 'задачи:'

app/tasks.py

@@ -1,18 +1,19 @@
-import configparser
+from configparser import ConfigParser
 import os
 from os.path import join, exists
 
 from celery import Celery
+from celery.signals import worker_ready
 
-import passback_grades
+from passback_grades import run_passback
 from db import db_methods
 from db.db_types import Check
 from main.checker import check
 from main.parser import parse
 from main.check_packs import BASE_PACKS
 from root_logger import get_root_logger
 
-config = configparser.ConfigParser()
+config = ConfigParser()
 config.read('app/config.ini')
 
 TASK_RETRY_COUNTDOWN = 60  # default = 3 * 60
@@ -33,6 +34,16 @@
 celery.conf.timezone = 'Europe/Moscow'  # todo: get from env
 
 
+@worker_ready.connect
+def at_start(sender, **k):
+    from nltk import download
+    download('stopwords')
+    download('punkt')
+    
+    from language_tool_python.download_lt import download_lt
+    download_lt()
+
+
 @celery.task(name="create_task", queue='check-solution', bind=True)
 def create_task(self, check_info):
     check_obj = Check(check_info)
@@ -67,12 +78,11 @@ def create_task(self, check_info):
         self.retry(countdown=TASK_RETRY_COUNTDOWN)  # Retry the task, adding it to the back of the queue.
 
 
+@celery.task(name="passback-task", queue='passback-grade')
+def passback_task():
+    return run_passback()
+
+
 def remove_files(filepaths):
     for filepath in filepaths:
         if exists(filepath): os.remove(filepath)
-
-
-@celery.task(name="passback-task", queue='passback-grade')
-def passback_task():
-    print('Run passback')
-    return passback_grades.run_passback()