Merge pull request #810 from rix1337/dev

Improve RegEx search for BY

Author: rix1337

feedcrawler/external_sites/feed_search/sites/content_all_by.py

@@ -30,11 +30,15 @@ def __init__(self, filename):
         self.url = self.hostnames.get('by')
         self.password = self.url.split('.')[0]
 
-        if "List_ContentAll_Seasons" not in filename:
+        if "_Regex" in filename:
+            self.URL = 'https://' + self.url + "/?cat="
+            self.FEED_URLS = [self.URL + "1", self.URL + "2"]
+        elif "List_ContentAll_Seasons" not in filename:
             self.URL = 'https://' + self.url + "/?cat=1"
+            self.FEED_URLS = [self.URL]
         else:
             self.URL = 'https://' + self.url + "/?cat=2"
-        self.FEED_URLS = [self.URL]
+            self.FEED_URLS = [self.URL]
 
         self.config = CrawlerConfig("ContentAll")
         self.feedcrawler = CrawlerConfig("FeedCrawler")
@@ -49,9 +53,17 @@ def __init__(self, filename):
         search = int(CrawlerConfig("ContentAll").get("search"))
         i = 2
         while i <= search:
-            page_url = self.URL + "&start=" + str(i)
-            if page_url not in self.FEED_URLS:
-                self.FEED_URLS.append(page_url)
+            if "_Regex" in filename:
+                page_url_1 = self.URL + "1&start=" + str(i)
+                page_url_2 = self.URL + "2&start=" + str(i)
+                if page_url_1 not in self.FEED_URLS:
+                    self.FEED_URLS.append(page_url_1)
+                if page_url_2 not in self.FEED_URLS:
+                    self.FEED_URLS.append(page_url_2)
+            else:
+                page_url = self.URL + "&start=" + str(i)
+                if page_url not in self.FEED_URLS:
+                    self.FEED_URLS.append(page_url)
             i += 1
         self.cdc = FeedDb('cdc')
 

feedcrawler/providers/common_functions.py

@@ -620,7 +620,7 @@ def keep_alphanumeric_with_special_characters(string):
 
 def keep_alphanumeric_with_regex_characters(string):
     string = replace_with_stripped_ascii(string)
-    return re.sub(r'[^0-9a-zA-Z\s\-.*+()|\[\]?!]', '', string)
+    return re.sub(r'[^0-9a-zA-Z\s\-.*+()|\[\]\\{},?!]', '', string)
 
 
 def keep_numbers(string):

feedcrawler/providers/version.py

@@ -8,7 +8,7 @@
 
 
 def get_version():
-    return "20.1.8"
+    return "20.1.9"
 
 
 def create_version_file():

feedcrawler/web_interface/vuejs_frontend/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "feedcrawler-web",
-  "version": "20.1.8",
+  "version": "20.1.9",
   "type": "module",
   "scripts": {
     "dev": "vite",

feedcrawler/web_interface/vuejs_frontend/package.json

@@ -112,7 +112,7 @@ function submitLists() {
                               @click="showRegExHelp">RegEx-Suche</span>
                     </h5><!-- Setting variables in label is unsupported -->
                     <FormKit v-model="store.lists.mb.regex"
-                             :validation="[['?matches', /^[a-zA-Z0-9ÄäÖöÜüß\-\s.*+()|\[\]?!]+$/]]"
+                             :validation="[['?matches', /^[a-zA-Z0-9ÄäÖöÜüß\-\s.*+()|\[\]\\{},?!]+$/]]"
                              :validation-messages="{
                                 matches: 'Bitte nur Buchstaben, Zahlen, Leerzeichen oder folgende Sonderzeichen eingeben: . * + ( ) | [ ] ? !'
                              }"