ticket queue extended

shukriadams · shukriadams · commit 25b4a8badb1f · 2025-01-30T22:16:24.000+01:00
- priority tickets
- improved keepalive
diff --git a/docs/development.md b/docs/development.md
@@ -16,9 +16,9 @@ If you want to develop Linux on Windows, Vagrant is an excellent and convenient
 - Dotnet 6.0 SDK
 - Visual Studio 2019 (Windows) or Visual Studio Code with C# extension. If you're using Visual Studio make sure you've update to the latest version. Visual Studio 2017 isn't supported.
 
-Unit test coverage is done with https://github.com/SteveGilham/altcover, install ReportGenerator with
-
-    dotnet tool install --global dotnet-reportgenerator-globaltool
+Unit test coverage is done with https://github.com/SteveGilham/altcover, install ReportGenerator. To get a version compatible with this project run
+    
+    dotnet tool install --global dotnet-reportgenerator-globaltool --version 5.3.11
 
 ## Running from Visual Studio
 
diff --git a/src/Tetrifact.Core/IProcessManager.cs b/src/Tetrifact.Core/IProcessManager.cs
@@ -14,7 +14,9 @@ public interface IProcessManager
         bool AnyOfKeyExists(string key);
 
         void AddUnique(ProcessCategories category, string key);
-        
+
+        void AddUnique(ProcessCategories category, string key, string metadata);
+
         void AddUnique(ProcessCategories category, string key, TimeSpan timespan);
 
         void AddUnique(ProcessCategories category, string key, string metadata, TimeSpan timespan);
diff --git a/src/Tetrifact.Core/ISettings.cs b/src/Tetrifact.Core/ISettings.cs
@@ -120,6 +120,11 @@ public interface ISettings
         /// </summary>
         int DownloadQueueTicketLifespan { get; set; }
 
+        /// <summary>
+        /// Static tickets which can be used to bypass the regular download queue.
+        /// </summary>
+        IEnumerable<string> DownloadQueuePriorityTickets { get; set; }
+
         /// <summary>
         /// Minimum amount of free space (megabytes) on storage drive - if less is available, new uploads will fail.
         /// </summary>
diff --git a/src/Tetrifact.Core/ProcessItem.cs b/src/Tetrifact.Core/ProcessItem.cs
@@ -20,10 +20,15 @@ public class ProcessItem
         public ProcessCategories Category { get; set; }
 
         /// <summary>
-        /// If set, the time lock item was created. If not set, lock never times out
+        /// If set, the time item was created. 
         /// </summary>
         public DateTime? AddedUTC { get; set; }
 
+        /// <summary>
+        /// If set, the time item was kept alive. If not set, lock never times out.
+        /// </summary>
+        public DateTime? KeepAliveUtc { get; set; }
+
         /// <summary>
         /// If set, the max age lock item can have before being auto purged.
         /// </summary>
@@ -44,7 +49,7 @@ public ProcessItem Clone()
         /// <returns></returns>
         public override string ToString()
         {
-            return $"{{{this.GetType().Name} Id:{Id}, MetaData:{Metadata}, Category:{Category}, AddedUtc:{AddedUTC}, MaxLifespace:{MaxLifespan} }}";
+            return $"{{{this.GetType().Name} Id:{Id}, MetaData:{Metadata}, Category:{Category}, AddedUtc:{AddedUTC}, KeepAliveUtc:{KeepAliveUtc}, MaxLifespace:{MaxLifespan} }}";
         }
     }
 }
diff --git a/src/Tetrifact.Core/ProcessManager.cs b/src/Tetrifact.Core/ProcessManager.cs
@@ -69,7 +69,28 @@ public void AddUnique(ProcessCategories category, string key)
                 if (_items.ContainsKey(key))
                     return;
 
-                _items.Add(key, new ProcessItem { Id = key, Category = category });
+                _items.Add(key, new ProcessItem { 
+                    Id = key, 
+                    AddedUTC = DateTime.UtcNow,
+                    Category = category });
+
+                _log.LogInformation($"Created process, category {category}, id {key}, no lifespan limit.");
+            }
+        }
+
+        public void AddUnique(ProcessCategories category, string key, string metadata)
+        {
+            lock (_items)
+            {
+                if (_items.ContainsKey(key))
+                    return;
+
+                _items.Add(key, new ProcessItem { 
+                    Id = key, 
+                    AddedUTC = DateTime.UtcNow,
+                    Category = category, 
+                    Metadata = metadata });
+
                 _log.LogInformation($"Created process, category {category}, id {key}, no lifespan limit.");
             }
         }
@@ -81,7 +102,14 @@ public void AddUnique(ProcessCategories category, string key, string metadata, T
                 if (_items.ContainsKey(key))
                     return;
 
-                _items.Add(key, new ProcessItem { Id = key, Metadata = metadata, AddedUTC = DateTime.UtcNow, MaxLifespan = timespan, Category = category });
+                _items.Add(key, new ProcessItem { 
+                    Id = key, 
+                    Metadata = metadata, 
+                    AddedUTC = DateTime.UtcNow, 
+                    KeepAliveUtc = DateTime.UtcNow, 
+                    MaxLifespan = timespan, 
+                    Category = category });
+
                 _log.LogInformation($"Created process, category {category}, id {key}, metadata {metadata}, forced lifespan {timespan}.");
             }
         }
@@ -93,7 +121,13 @@ public void AddUnique(ProcessCategories category, string key, TimeSpan timespan)
                 if (_items.ContainsKey(key))
                     return;
 
-                _items.Add(key, new ProcessItem{ Id = key, AddedUTC = DateTime.UtcNow, MaxLifespan = timespan, Category = category });
+                _items.Add(key, new ProcessItem{ 
+                    Id = key,
+                    AddedUTC = DateTime.UtcNow,
+                    KeepAliveUtc = DateTime.UtcNow,
+                    MaxLifespan = timespan,
+                    Category = category });
+
                 _log.LogInformation($"Created process, category {category}, id {key}, forced lifespan {timespan}.");
             }
         }
@@ -133,25 +167,28 @@ public void KeepAlive(string key)
                 if (!_items.ContainsKey(key))
                     return;
 
-                _items[key].AddedUTC = DateTime.UtcNow;
+                _items[key].KeepAliveUtc = DateTime.UtcNow;
             }
         }
 
         public void ClearExpired()
         {
-            for (int i = 0 ; i < _items.Count; i ++)
+            lock (_items) 
             {
-                string key = _items.Keys.ElementAt(_items.Count - 1 - i);
-                if (!_items[key].AddedUTC.HasValue || !_items[key].MaxLifespan.HasValue)
-                    continue;
+                for (int i = 0; i < _items.Count; i++)
+                {
+                    string key = _items.Keys.ElementAt(_items.Count - 1 - i);
+                    if (!_items[key].KeepAliveUtc.HasValue || !_items[key].MaxLifespan.HasValue)
+                        continue;
 
-                TimeSpan ts = _items[key].MaxLifespan.Value;
-                DateTime dt = _items[key].AddedUTC.Value;
-                if (DateTime.UtcNow - dt < ts)
-                    continue;
+                    TimeSpan ts = _items[key].MaxLifespan.Value;
+                    DateTime dt = _items[key].KeepAliveUtc.Value;
+                    if (DateTime.UtcNow - dt < ts)
+                        continue;
 
-                _items.Remove(key);
-                _log.LogInformation($"Processes id {key} timed out and removed.");
+                    _items.Remove(key);
+                    _log.LogInformation($"Processes id {key} timed out and removed.");
+                }
             }
         }
 
diff --git a/src/Tetrifact.Core/Settings.cs b/src/Tetrifact.Core/Settings.cs
@@ -96,6 +96,8 @@ public class Settings : ISettings
 
         public int DownloadQueueTicketLifespan { get; set; }
 
+        public IEnumerable<string> DownloadQueuePriorityTickets { get; set; }
+
         /// <summary>
         /// 
         /// </summary>
@@ -112,6 +114,7 @@ public Settings()
             // defaults
             this.DownloadQueueTicketLifespan = 10; // seconds
             this.AccessTokens = new List<string>();
+            this.DownloadQueuePriorityTickets = new List<string>();
             this.PackageDeleteEnabled = true;
             this.PackageCreateEnabled = true;
             this.ArchiveCPUThreads = 4;                 // for compression solutions that support multithreading.
diff --git a/src/Tetrifact.Web/Controllers/ArchivesController.cs b/src/Tetrifact.Web/Controllers/ArchivesController.cs
@@ -2,7 +2,6 @@
 using Microsoft.AspNetCore.Http.Headers;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
-using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
 using System.IO;
@@ -99,19 +98,28 @@ public ActionResult GetArchive(string packageId, [FromQuery(Name = "ticket")] st
                 RequestHeaders headers = Request.GetTypedHeaders();
                 bool isLocal = headers.Host.ToString().ToLower() == "localhost";
 
+                // local (this website) downloads always allowed.
                 if (!isLocal && _settings.MaximumSimultaneousDownloads.HasValue) 
                 {
                     if (string.IsNullOrEmpty(ticket))
                         return Responses.NoTicket();
 
-                    IEnumerable<ProcessItem> userTickets = _processManager.GetByCategory(ProcessCategories.ArchiveQueueSlot);
-                    ProcessItem userTicket = userTickets.FirstOrDefault(i => i.Id == ticket);
-                    if (userTicket == null)
-                        return Responses.NoTicket();
-
-                    int count = userTickets.Where(t => t.AddedUTC < userTicket.AddedUTC).Count();
-                    if (count > _settings.MaximumSimultaneousDownloads)
-                        return Responses.QueueFull(count, _settings.MaximumSimultaneousDownloads.Value);
+                    // check for priority tickets
+                    if (_settings.DownloadQueuePriorityTickets.Contains(ticket))
+                    {
+                        _log.LogInformation($"Priority ticket {ticket} used from host {headers.Host}");
+                    }
+                    else 
+                    {
+                        IEnumerable<ProcessItem> userTickets = _processManager.GetByCategory(ProcessCategories.ArchiveQueueSlot);
+                        ProcessItem userTicket = userTickets.FirstOrDefault(i => i.Id == ticket);
+                        if (userTicket == null)
+                            return Responses.NoTicket();
+
+                        int count = userTickets.Where(t => t.AddedUTC < userTicket.AddedUTC).Count();
+                        if (count > _settings.MaximumSimultaneousDownloads)
+                            return Responses.QueueFull(count, _settings.MaximumSimultaneousDownloads.Value);
+                    }
                 }
 
                 string archivePath = _archiveService.GetPackageArchivePath(packageId);
@@ -135,7 +143,11 @@ public ActionResult GetArchive(string packageId, [FromQuery(Name = "ticket")] st
                     _processManager.KeepAlive(ticket);
                 };
 
-                return File(progressableStream, "application/octet-stream", $"{packageId}.zip", enableRangeProcessing: true);
+                return File(
+                    progressableStream, 
+                    "application/octet-stream", 
+                    $"{packageId}.zip", 
+                    enableRangeProcessing: true);
             }
             catch (PackageNotFoundException ex)
             {
diff --git a/src/Tetrifact.Web/Controllers/TicketsController.cs b/src/Tetrifact.Web/Controllers/TicketsController.cs
@@ -59,13 +59,16 @@ public ActionResult CreateQueueTicket(string requestIdentifier)
                     });
 
                 string ticket = Guid.NewGuid().ToString();
-                _processManager.AddUnique(ProcessCategories.ArchiveQueueSlot, ticket, requestIdentifier, new TimeSpan(0, 0, _settings.DownloadQueueTicketLifespan));
+                _processManager.AddUnique(
+                    ProcessCategories.ArchiveQueueSlot, 
+                    ticket, 
+                    requestIdentifier);
 
                 return new JsonResult(new
                 {
                     success = new
                     {
-                        ticket = ticket,
+                        ticket,
                         clientIdentifier = requestIdentifier,
                         required = true
                     }
diff --git a/src/Tetrifact.Web/Core/Daemons/ProcessManagerCron.cs b/src/Tetrifact.Web/Core/Daemons/ProcessManagerCron.cs
@@ -1,6 +1,5 @@
 ﻿using Microsoft.Extensions.Logging;
 using System.Threading.Tasks;
-using System;
 using Tetrifact.Core;
 
 namespace Tetrifact.Web
diff --git a/src/Tetrifact.Web/Views/Home/Processes.cshtml b/src/Tetrifact.Web/Views/Home/Processes.cshtml
@@ -27,6 +27,13 @@
                                 at @process.AddedUTC.ToHumanString()
                             </text>
                         }
+
+                        @process.KeepAliveUtc.HasValue
+                        {
+                            <text>
+                                at @process.KeepAliveUtc.ToHumanString()
+                            </text>
+                        }
                     </span>
                 </li>
             }
diff --git a/src/cover.bat b/src/cover.bat
@@ -1,6 +1,5 @@
 :: Convenient single-click script to run cover. Works on Windows only.
 
 dotnet test /p:AltCover=true
-pause
-:: reportgenerator -reports:./Tetrifact.Tests/coverage.xml -targetdir:./Tetrifact.Tests/coverage -assemblyfilters:+Tetrifact.*;-Tetrifact.Tests -classfilters:-Tetrifact.Core.ThreadDefault;-Tetrifact.Web.DaemonProcessRunner;-Tetrifact.Web.Pager;-Tetrifact.Web.Program;-Tetrifact.Web.Startup;-Tetrifact.Web.ReadLevel;-Tetrifact.Web.WriteLevel;-*f__*
-:: start Tetrifact.Tests\coverage\index.html
+reportgenerator -reports:./Tetrifact.Tests/coverage.xml -targetdir:./Tetrifact.Tests/coverage -assemblyfilters:+Tetrifact.*;-Tetrifact.Tests -classfilters:-Tetrifact.Core.ThreadDefault;-Tetrifact.Web.DaemonProcessRunner;-Tetrifact.Web.Pager;-Tetrifact.Web.Program;-Tetrifact.Web.Startup;-Tetrifact.Web.ReadLevel;-Tetrifact.Web.WriteLevel;-*f__*
+start Tetrifact.Tests\coverage\index.html

Original file line number	Diff line number	Diff line change
`@@ -20,10 +20,15 @@ public class ProcessItem`
`20`	`20`	`public ProcessCategories Category { get; set; }`
`21`	`21`
`22`	`22`	`/// <summary>`
`23`		`- /// If set, the time lock item was created. If not set, lock never times out`
	`23`	`+ /// If set, the time item was created.`
`24`	`24`	`/// </summary>`
`25`	`25`	`public DateTime? AddedUTC { get; set; }`
`26`	`26`
	`27`	`+ /// <summary>`
	`28`	`+ /// If set, the time item was kept alive. If not set, lock never times out.`
	`29`	`+ /// </summary>`
	`30`	`+ public DateTime? KeepAliveUtc { get; set; }`
	`31`	`+`
`27`	`32`	`/// <summary>`
`28`	`33`	`/// If set, the max age lock item can have before being auto purged.`
`29`	`34`	`/// </summary>`
`@@ -44,7 +49,7 @@ public ProcessItem Clone()`
`44`	`49`	`/// <returns></returns>`
`45`	`50`	`public override string ToString()`
`46`	`51`	`{`
`47`		`- return $"{{{this.GetType().Name} Id:{Id}, MetaData:{Metadata}, Category:{Category}, AddedUtc:{AddedUTC}, MaxLifespace:{MaxLifespan} }}";`
	`52`	`+ return $"{{{this.GetType().Name} Id:{Id}, MetaData:{Metadata}, Category:{Category}, AddedUtc:{AddedUTC}, KeepAliveUtc:{KeepAliveUtc}, MaxLifespace:{MaxLifespan} }}";`
`48`	`53`	`}`
`49`	`54`	`}`
`50`	`55`	`}`
Original file line number	Diff line number	Diff line change
`@@ -59,13 +59,16 @@ public ActionResult CreateQueueTicket(string requestIdentifier)`
`59`	`59`	`});`
`60`	`60`
`61`	`61`	`string ticket = Guid.NewGuid().ToString();`
`62`		`- _processManager.AddUnique(ProcessCategories.ArchiveQueueSlot, ticket, requestIdentifier, new TimeSpan(0, 0, _settings.DownloadQueueTicketLifespan));`
	`62`	`+ _processManager.AddUnique(`
	`63`	`+ ProcessCategories.ArchiveQueueSlot,`
	`64`	`+ ticket,`
	`65`	`+ requestIdentifier);`
`63`	`66`
`64`	`67`	`return new JsonResult(new`
`65`	`68`	`{`
`66`	`69`	`success = new`
`67`	`70`	`{`
`68`		`- ticket = ticket,`
	`71`	`+ ticket,`
`69`	`72`	`clientIdentifier = requestIdentifier,`
`70`	`73`	`required = true`
`71`	`74`	`}`
Original file line number	Diff line number	Diff line change
`@@ -27,6 +27,13 @@`
`27`	`27`	`at @process.AddedUTC.ToHumanString()`
`28`	`28`	`</text>`
`29`	`29`	`}`
	`30`	`+`
	`31`	`+ @process.KeepAliveUtc.HasValue`
	`32`	`+ {`
	`33`	`+ <text>`
	`34`	`+ at @process.KeepAliveUtc.ToHumanString()`
	`35`	`+ </text>`
	`36`	`+ }`
`30`	`37`	`</span>`
`31`	`38`	`</li>`
`32`	`39`	`}`