Working with Image Downloading

Author: digitaldreams

Image/Download.py

@@ -1,51 +1,44 @@
 import errno
 import os.path
-
+import Models.Queue.Image
+import Models.Complete.Image
 from Image import Save
 
 
 class Download():
-    def __init__(self, file_name, path=''):
-        self.links = set()
-        self.completed = set()
-        self.file_name = file_name
-        self.file_to_set()
-        self.path = path
+    path = 'storage/images'
 
-    def file_to_set(self) -> object:
-        """
-        Load links from file and set to Set()
-        :return: object
-        """
-        if not os.path.exists(self.file_name):
-            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), self.file_name)
-        with open(self.file_name, 'rt') as f:
-            for line in f:
-                self.links.add(line.replace('\n', ''))
-        return sorted(self.links)
+    def __init__(self, take=None):
+        self.queue = Models.Queue.Image.Image()
+        self.complete = Models.Complete.Image.Image()
+        self.limit = take
+        if not os.path.exists(self.path):
+            os.makedirs(self.path)
 
     def start(self) -> object:
         """
         Start Downloading file
         :rtype: object
         """
-        for file in self.links:
+        if isinstance(self.limit, int) and len(self.queue.links) >= self.limit:
+            links = sorted(self.queue.links)[0:self.limit]
+        else:
+            links = self.queue.links
+
+        for file in links:
             try:
-                img = Save.SaveFile(file, self.path)
+                img = Save.Save(file, self.path)
                 img.save()
-            except:
-                continue
-            self.completed.add(file)
-        self.set_to_file()
+            except Exception as e:
+                print(str(e))
+            self.complete.add(file)
+        self.save()
 
-    def set_to_file(self) -> object:
+    def save(self) -> object:
         """
         Update links txt file
         :return : None
         """
-        remaining = self.links.difference(self.completed)
-        with open(self.file_name, 'w') as f:
-            if len(remaining) > 0:
-                for line in self.links:
-                    f.write(line + "\n")
-            f.write("")
+        self.queue.links = self.queue.links.difference(self.complete.links)
+        self.queue.save()
+        self.complete.save()

Image/__pycache__/Download.cpython-36.pyc

@@ -5,6 +5,5 @@ class Image(Model):
     file_path = 'storage/complete/images.txt'
 
     def __init__(self):
-        self.links.clear()
-        self.fetch()
         Model.__init__(self)
+        self.fetch()

Models/Complete/Image.py

@@ -5,7 +5,6 @@ class Link(Model):
     file_path = 'storage/complete/links.txt'
 
     def __init__(self):
-        self.links.clear()
-        self.fetch()
         Model.__init__(self)
+        self.fetch()
 

Models/Complete/Link.py

@@ -3,11 +3,14 @@
 import os.path;
 from Crawler.Page import Page
 from Crawler.Image import Image
+from Image.Download import Download
 
 if __name__ == '__main__':
+    # page=Page('https://gopostie.com')
+    # page.save_links()
 
-    #page=Page('https://gopostie.com')
-    #page.save_links()
+    # img = Image('https://gopostie.com')
+    # img.save_links()
 
-    img = Image('https://gopostie.com')
-    img.save_links()
+    down = Download()
+    down.start()