spidr

Author: immisterio

AppInit.cs

@@ -74,6 +74,8 @@ public static AppInit conf
 
         public bool syncsport = true;
 
+        public bool syncspidr = false;
+
         public int maxreadfile = 200;
 
         public Evercache evercache = new Evercache() { enable = true, validHour = 1, maxOpenWriteTask = 2000 };

Controllers/SyncController.cs

@@ -20,6 +20,7 @@ public JsonResult Configuration()
             return Json(new 
             {
                 fbd = true,
+                spidr = true,
                 version = 2
             });
         }
@@ -41,7 +42,7 @@ public ActionResult FdbKey(string key)
         }
 
         [Route("/sync/fdb/torrents")]
-        public ActionResult FdbTorrents(long time, long start = -1)
+        public ActionResult FdbTorrents(long time, long start = -1, bool spidr = false)
         {
             if (!AppInit.conf.opensync || time == 0)
                 return Json(new { nextread = false, collections = new List<Collection>() });
@@ -62,7 +63,7 @@ public ActionResult FdbTorrents(long time, long start = -1)
 
                 foreach (var t in FileDB.OpenRead(item.Key))
                 {
-                    if (start != -1 && start > t.Value.updateTime.ToFileTimeUtc())
+                    if (spidr || (start != -1 && start > t.Value.updateTime.ToFileTimeUtc()))
                     {
                         torrent.TryAdd(t.Key, new TorrentDetails() 
                         {
@@ -115,7 +116,7 @@ public ActionResult FdbTorrents(long time, long start = -1)
                 }
             }
 
-            return Json(new { nextread, collections });
+            return Json(new { nextread, countread, take, collections });
         }
 
 

Engine/SyncCron.cs

@@ -14,7 +14,8 @@ public static class SyncCron
     {
         static long lastsync = -1, starsync = -1;
 
-        async public static Task Run()
+        #region Torrents
+        async public static Task Torrents()
         {
             await Task.Delay(20_000);
 
@@ -55,19 +56,16 @@ async public static Task Run()
                             else if (root.collections.Count > 0)
                             {
                                 reset = true;
-                                var torrents = new List<TorrentBaseDetails>();
+                                var torrents = new List<TorrentBaseDetails>(root.countread);
 
                                 foreach (var collection in root.collections)
                                 {
                                     foreach (var torrent in collection.Value.torrents)
                                     {
-                                        if (torrent.Value.types == null)
+                                        if (AppInit.conf.synctrackers != null && torrent.Value.trackerName != null && !AppInit.conf.synctrackers.Contains(torrent.Value.trackerName))
                                             continue;
 
-                                        if (AppInit.conf.synctrackers != null && !AppInit.conf.synctrackers.Contains(torrent.Value.trackerName))
-                                            continue;
-
-                                        if (!AppInit.conf.syncsport && torrent.Value.types.Contains("sport"))
+                                        if (!AppInit.conf.syncsport && torrent.Value.types != null && torrent.Value.types.Contains("sport"))
                                             continue;
 
                                         torrents.Add(torrent.Value);
@@ -118,14 +116,16 @@ async public static Task Run()
 
                         FileDB.SaveChangesToFile();
                         File.WriteAllText("lastsync.txt", lastsync.ToString());
+
+                        Console.WriteLine("sync: end");
                     }
                     else
                     {
                         await Task.Delay(TimeSpan.FromMinutes(1));
                         continue;
                     }
                 }
-                catch
+                catch (Exception ex)
                 {
                     try
                     {
@@ -136,12 +136,62 @@ async public static Task Run()
                         }
                     }
                     catch { }
+
+                    Console.WriteLine("sync: error / " + ex.Message);
                 }
 
-                Console.WriteLine("sync: end");
                 await Task.Delay(1000 * Random.Shared.Next(60, 300));
                 await Task.Delay(1000 * 60 * (20 > AppInit.conf.timeSync ? 20 : AppInit.conf.timeSync));
             }
         }
+        #endregion
+
+        #region Spidr
+        async public static Task Spidr()
+        {
+            while (true)
+            {
+                await Task.Delay(1000 * Random.Shared.Next(60, 300));
+                await Task.Delay(1000 * 60 * 120);
+
+                try
+                {
+                    if (!string.IsNullOrWhiteSpace(AppInit.conf.syncapi) && AppInit.conf.syncspidr)
+                    {
+                        long lastsync_spidr = -1;
+
+                        var conf = await HttpClient.Get<JObject>($"{AppInit.conf.syncapi}/sync/conf");
+                        if (conf != null && conf.ContainsKey("spidr") && conf.Value<bool>("spidr"))
+                        {
+                            Console.WriteLine($"\n\nsync_spidr: start / {DateTime.Now}");
+
+                            next: var root = await HttpClient.Get<Models.Sync.v2.RootObject>($"{AppInit.conf.syncapi}/sync/fdb/torrents?time={lastsync_spidr}&spidr=true", timeoutSeconds: 300, MaxResponseContentBufferSize: 100_000_000);
+
+                            Console.WriteLine($"sync_spidr: time={lastsync_spidr}");
+
+                            if (root?.collections != null && root.collections.Count > 0)
+                            {
+                                foreach (var collection in root.collections)
+                                    FileDB.AddOrUpdate(collection.Value.torrents.Values);
+
+                                lastsync_spidr = root.collections.Last().Value.fileTime;
+
+                                if (root.nextread)
+                                    goto next;
+                            }
+
+                            Console.WriteLine("sync_spidr: end");
+                        }
+                    }
+                    else
+                    {
+                        await Task.Delay(TimeSpan.FromMinutes(1));
+                        continue;
+                    }
+                }
+                catch (Exception ex) { Console.WriteLine("sync_spidr: error / " + ex.Message); }
+            }
+        }
+        #endregion
     }
 }

Models/Sync/v2/RootObject.cs

@@ -6,6 +6,10 @@ public class RootObject
     {
         public bool nextread { get; set; }
 
+        public int take { get; set; }
+
+        public int countread { get; set; }
+
         public List<Collection> collections { get; set; }
     }
 }

Program.cs

@@ -12,7 +12,9 @@ public class Program
     {
         public static void Main(string[] args)
         {
-            ThreadPool.QueueUserWorkItem(async _ => await SyncCron.Run());
+            ThreadPool.QueueUserWorkItem(async _ => await SyncCron.Torrents());
+            ThreadPool.QueueUserWorkItem(async _ => await SyncCron.Spidr());
+
             ThreadPool.QueueUserWorkItem(async _ => await StatsCron.Run());
 
             ThreadPool.QueueUserWorkItem(async _ => await FileDB.Cron());