improved prune report

Author: shukriadams

.gitignore

@@ -42,3 +42,4 @@ src/packages
 utils/upload/packages
 utils/upload/__pycache__
 utils/upload/.generate
+utils/upload/.setDateSpread

src/Tetrifact.Core/DateTimeSpanExtensions.cs

@@ -25,6 +25,22 @@ public static string ToIso(this DateTime date)
 
             return iso
                 .Substring(0, iso.Length - 3); // remove sec
+        }
+
+        /// <summary>
+        /// Converts to yyyy-mm-dd ISO
+        /// </summary>
+        /// <param name="date"></param>
+        /// <returns></returns>
+        public static string ToIsoShort(this DateTime date)
+        {
+            string iso = date
+                .ToLocalTime()
+                .ToString("s") // convert to ymdhms
+                .Replace("T", " "); // replace T after ymd
+
+            return iso
+                .Substring(0, iso.Length - 9); // remove all time data
         }
 
         public static string ToIsoFSFriendly(this DateTime date)

src/Tetrifact.Core/PruneBracket.cs

@@ -1,7 +1,9 @@
 namespace Tetrifact.Core
 {
     public class PruneBracket
-    {
+    {
+        #region PROPERTIES
+
         /// <summary>
         /// Number of packages to keep for the given period
         /// </summary>
@@ -12,16 +14,29 @@ public class PruneBracket
         /// </summary>
         public int Days { get; set; }
 
-        public PruneBracketGrouping Grouping { get; set; }
-
+        /// <summary>
+        /// 
+        /// </summary>
+        public PruneBracketGrouping Grouping { get; set; }
+
+        #endregion
+
+        #region CTORS
+
         public PruneBracket() 
         {
             Grouping = PruneBracketGrouping.Grouped;
-        }
+        }
+
+        #endregion
+
+        #region METHODS
 
         public override string ToString()
         {
-            return $"{Days} days {Amount} packages";
-        }
+            return $"Covers {Days} day(s), allows {Amount} package(s), {Grouping}";
+        }
+
+        #endregion
     }
 }

src/Tetrifact.Core/PruneBracketProcess.cs

@@ -1,15 +1,35 @@
-using System;
+using Newtonsoft.Json.Bson;
+using System;
 using System.Collections.Generic;
-
+
 namespace Tetrifact.Core
 {
     public class PruneBracketProcess : PruneBracket
     {
-        public IList<Manifest> Keep {get ; set; } = new List<Manifest>();
+        public IList<Manifest> Keep { get ; set; } = new List<Manifest>();
 
         public IList<Manifest> Prune { get; set; }  = new List<Manifest>();
 
-        public DateTime Floor { get; set; }
+        public TimeSpan Coverage { get; set; }
+
+        /// <summary>
+        /// Days back in time from Now that bracket covers. Calculated at start of a given prune run.
+        /// </summary>
+        public DateTime Floor { get; set; }
+
+        public DateTime Ceiling { get; set; }
+
+        public override string ToString()
+        {
+            return $"Ceiling:{this.Ceiling.ToIsoShort()} floor:{this.Floor.ToIsoShort()} (from {this.Ceiling.Ago()} ago to {this.Floor.Ago()} ago), {base.ToString()}";
+        }
+
+        public bool Contains(DateTime date) 
+        {
+            // note that we use compare only, not equal. Still trying to trace issues with packages being aggressively deleted by landing
+            // in multiple brackets, for erring on side of caution. Packages that fail to match a bracket will always be kept.
+            return date < this.Ceiling && date > this.Floor;
+        }
 
         public static PruneBracketProcess FromPruneBracket(PruneBracket pruneBracket) 
         { 

src/Tetrifact.Core/PruneService.cs

@@ -92,64 +92,67 @@ public PrunePlan GeneratePrunePlan()
                 .ToList();
 
             IList<string> taggedKeep = new List<string>();
-            IList<string> newKeep = new List<string>();
             IList<string> report = new List<string>();
 
-            int unhandled = 0;
-
-            report.Add(" ******************************** Prune audit start **********************************");
+            int ignoringNoBracketCount = 0;
 
             IList<string> packageIds = _indexReader.GetAllPackageIds().ToList();
             packageIds = packageIds.OrderBy(n => Guid.NewGuid()).ToList(); // randomize collection order
 
             DateTime utcNow = _timeprovider.GetUtcNow();
-            foreach(PruneBracketProcess pruneBracketProcess in processBrackets)
-                pruneBracketProcess.Floor = utcNow.AddDays(-1 * pruneBracketProcess.Days);
+            DateTime ceiling = utcNow;
+            foreach(PruneBracketProcess pruneBracketProcess in processBrackets) 
+            {
+                pruneBracketProcess.Ceiling = ceiling;
+                pruneBracketProcess.Floor = ceiling.AddDays(-1 * pruneBracketProcess.Days);
+                ceiling = pruneBracketProcess.Floor;
+            }
 
             int startingPackageCount = packageIds.Count;
 
-            report.Add($"Server currently contains {packageIds.Count} packages.");
+            report.Add($"Server currently holds {packageIds.Count} packages.");
 
             foreach (string packageId in packageIds)
             {
                 Manifest manifest = _indexReader.GetManifestHead(packageId);
+                report.Add(string.Empty);
 
                 if (manifest == null)
                 {
                     _log.LogWarning($"Expected manifest for package {packageId} was not found, skipping.");
                     continue;
-                }
-
+                }
+
                 string flattenedTags = manifest.Tags.Count == 0 ? string.Empty : $"Tags : {string.Join(",", manifest.Tags)}";
+                flattenedTags = string.IsNullOrEmpty(flattenedTags) ? "Package is untagged": $"Tagged with : {flattenedTags}";
+
                 int ageInDays = (int)Math.Round((utcNow - manifest.CreatedUtc).TotalDays, 0);
-                report.Add($"Analysing {packageId}, added {manifest.CreatedUtc.ToIso()} ({ageInDays} days ago). Tagged with: {flattenedTags}");
+                report.Add($"Analysing package \"{packageId}\", added {manifest.CreatedUtc.ToIso()} ({ageInDays} days ago). {flattenedTags}.");
 
-                PruneBracketProcess matchingBracket = processBrackets.FirstOrDefault(b => manifest.CreatedUtc < b.Floor);
+                PruneBracketProcess matchingBracket = processBrackets.FirstOrDefault(bracket => bracket.Contains(manifest.CreatedUtc));
                 if (matchingBracket == null)
                 {
-                    report.Add($"{packageId}, created {manifest.CreatedUtc.ToIso()}, does not land in any prune bracket, will be kept.");
-                    unhandled ++;
+                    report.Add($"Package \"{packageId}\" doesn't fit into any prune brackets, will be kept.");
+                    ignoringNoBracketCount ++;
                     continue;
                 }
 
-                report.Add($"{packageId}, created {manifest.CreatedUtc.ToIso()}, lands in prune bracket {matchingBracket.Days}Days.");
-
                 // try to find reasons to keep package
 
                 // packages can be tagged to never be deleted. This ignores keep count, but will push out packages that are not tagged
-                bool isTaggedKeep = manifest.Tags.Any(tag => _settings.PruneIgnoreTags.Any(protectedTag => protectedTag.Equals(tag)));
-                if (isTaggedKeep)
+                IEnumerable<string> keepTagsOnPackage = manifest.Tags.Where(tag => _settings.PruneIgnoreTags.Any(protectedTag => protectedTag.Equals(tag)));
+                if (keepTagsOnPackage.Any())
                 {
                     taggedKeep.Add(packageId);
                     matchingBracket.Keep.Add(manifest);
-                    report.Add($"{packageId} marked for keep based on tag.");
+                    report.Add($"Package \"{packageId}\" marked for keep based on tag(s) {string.Join(",", keepTagsOnPackage)}.");
                     continue;
                 }
 
                 // "group strategy" - the entire bracket is treated as one big bag
                 if (matchingBracket.Grouping == PruneBracketGrouping.Grouped && matchingBracket.Keep.Count < matchingBracket.Amount)
                 {
-                    report.Add($"{packageId} marked for keep, {matchingBracket.Keep.Count} packages kept so far.");
+                    report.Add($"Package \"{packageId}\" marked for keep based on its date grouping (bracket {matchingBracket}) {matchingBracket.Amount - matchingBracket.Keep.Count} slots left in bracket.");
                     matchingBracket.Keep.Add(manifest);
                     continue;
                 }
@@ -158,8 +161,10 @@ public PrunePlan GeneratePrunePlan()
                 if (matchingBracket.Grouping == PruneBracketGrouping.Daily) 
                 {
                     int code = manifest.CreatedUtc.ToDayCode();
-                    if (matchingBracket.Keep.Count(m => m.CreatedUtc.ToDayCode() == code) < matchingBracket.Amount) 
+                    int kept = matchingBracket.Keep.Count(m => m.CreatedUtc.ToDayCode() == code);
+                    if (kept < matchingBracket.Amount) 
                     {
+                        report.Add($"Package \"{packageId}\" marked for keep, bracket {matchingBracket.Days} had {matchingBracket.Amount - kept} slots left for day {code}.");
                         matchingBracket.Keep.Add(manifest);
                         continue;
                     }
@@ -169,8 +174,10 @@ public PrunePlan GeneratePrunePlan()
                 if (matchingBracket.Grouping == PruneBracketGrouping.Weekly)
                 {
                     int code = manifest.CreatedUtc.ToWeekCode();
-                    if (matchingBracket.Keep.Count(m => m.CreatedUtc.ToWeekCode() == code) < matchingBracket.Amount)
+                    int kept = matchingBracket.Keep.Count(m => m.CreatedUtc.ToWeekCode() == code);
+                    if (kept < matchingBracket.Amount)
                     {
+                        report.Add($"Package \"{packageId}\" marked for keep, bracket {matchingBracket.Days} had {matchingBracket.Amount - kept} slots left for week {code}.");
                         matchingBracket.Keep.Add(manifest);
                         continue;
                     }
@@ -180,53 +187,78 @@ public PrunePlan GeneratePrunePlan()
                 if (matchingBracket.Grouping == PruneBracketGrouping.Monthly)
                 {
                     int code = manifest.CreatedUtc.ToMonthCode();
-                    if (matchingBracket.Keep.Count(m => m.CreatedUtc.ToMonthCode() == code) < matchingBracket.Amount)
+                    int kept = matchingBracket.Keep.Count(m => m.CreatedUtc.ToMonthCode() == code);
+                    if (kept < matchingBracket.Amount)
                     {
+                        report.Add($"Package \"{packageId}\" marked for keep, bracket {matchingBracket.Days} had {matchingBracket.Amount - kept} slots left for month {code}.");
                         matchingBracket.Keep.Add(manifest);
                         continue;
                     }
                 }
 
-                // bracket is on x packages per month basis
+                // bracket is on x packages per year basis
                 if (matchingBracket.Grouping == PruneBracketGrouping.Yearly)
                 {
-                    int code = manifest.CreatedUtc.ToMonthCode();
-                    if (matchingBracket.Keep.Count(m => m.CreatedUtc.ToMonthCode() == code) < matchingBracket.Amount)
+                    int code = manifest.CreatedUtc.Year;
+                    int kept = matchingBracket.Keep.Count(m => m.CreatedUtc.Year == code);
+                    if (kept < matchingBracket.Amount)
                     {
+                        report.Add($"Package \"{packageId}\" marked for keep, bracket {matchingBracket.Days} had {matchingBracket.Amount - kept} slots left for year {code}.");
                         matchingBracket.Keep.Add(manifest);
                         continue;
                     }
                 }
 
-
                 // no reasons found, prune package
                 matchingBracket.Prune.Add(manifest);
-                report.Add($"{packageId} marked for prune, {matchingBracket.Keep.Count} packages already kept.");
+                report.Add($"Package \"{packageId}\" failed to pass any keep tests, marked for prune.");
+
             } // for each
 
             string pruneIdList = string.Empty;
             if (packageIds.Count > 0)
                 pruneIdList = $" ({string.Join(",", packageIds)})";
 
             report.Add(string.Empty);
-            report.Add($"Pre-weekly ignore count is {newKeep.Count()} - {string.Join(",", newKeep)}");
-            report.Add($"Unhandled: {unhandled}");
-            
-            if (taggedKeep.Count > 0)
-                report.Add($"Kept due to tagging - {string.Join(",", taggedKeep)}.");
-    
-            foreach(PruneBracketProcess p in processBrackets)
-                report.Add($"Bracket {p}, keeping {p.Keep.Count} packages ({string.Join(",",p.Keep)}), pruning {p.Prune.Count} packages ({string.Join(",", p.Prune)})");
-
-            report.Add(string.Empty);
-            report.Add(" ******************************** Prune audit end **********************************");
+
+            int totalKeep = 0;
+            int totalPrune = 0;
+
+            foreach (PruneBracketProcess p in processBrackets)
+            {
+                totalKeep += p.Keep.Count;
+                totalPrune += p.Prune.Count;
+                report.Add($"Bracket {p}.");
+                report.Add($"Keeping {p.Keep.Count}{FlattenList(p.Keep.Select(p => p.Id))}.");
+                report.Add($"Pruning {p.Prune.Count}{FlattenList(p.Prune.Select(p => p.Id))}.");
+                report.Add(string.Empty);
+            }
+
+            if (taggedKeep.Any())
+                report.Add($"Kept {taggedKeep.Count} packages because of tag matches{FlattenList(taggedKeep)}.");
+            else
+                report.Add("No packages were kept due to tag matching. Note that packages need to fall into a bracket first before keep tagging rules are applied.");
+
+            report.Add($"Total packages in system:{packageIds.Count}, no bracket match:{ignoringNoBracketCount}, pruning:{totalPrune}, keeping:{totalKeep}.");
+
+            int totalHandled = totalKeep + totalPrune + ignoringNoBracketCount;
+            if (packageIds.Count != totalHandled) 
+                report.Add($"ERROR : Package handling count error, expected {packageIds.Count}, got {totalHandled}.");
 
             return new PrunePlan{ 
                 Report = report, 
                 Brackets = processBrackets
             };
         }
 
+        private string FlattenList(IEnumerable<object> packages) 
+        {
+            if (!packages.Any())
+                return string.Empty;
+
+            return $" ({string.Join(",", packages)})";
+        }
+
         #endregion
     }
 }

utils/upload/setDateSpread.py

@@ -3,9 +3,18 @@
 import glob
 import json
 import datetime
+import argparse
+from importlib.machinery import SourceFileLoader
 
-work_dir='./../../src/Tetrifact.Web/bin/Debug/net6.0/data/'
-packagesPerDay=5
+loader = SourceFileLoader('loader', './vars.py').load_module()
+
+argParser = argparse.ArgumentParser()
+argParser.add_argument('--work_dir', default='./../../src/Tetrifact.Web/bin/Debug/net6.0/data/')
+argParser.add_argument('--packages_per_day', default=5)
+args = loader.mergeFromFile('.setDateSpread', vars(argParser.parse_args()))
+
+work_dir = args['work_dir']
+packages_per_day = args['packages_per_day']
 
 def loadJson(filepath):
     import json
@@ -38,9 +47,12 @@ def writeJson(filepath, dataObject):
 date = datetime.datetime.now()
 
 for package in packages:
-    date = date + datetime.timedelta(days = daysBack)
 
-    for day in range(packagesPerDay):
+    date = datetime.datetime.now() + datetime.timedelta(days = -1*daysBack)
+    daysBack = daysBack + 1
+    print(f'Setting package date back to {str(date)}')
+
+    for day in range(packages_per_day):
         manifestPath = os.path.join(package, 'manifest.json')
         manifest = loadJson(manifestPath)
         manifest['CreatedUtc'] = str(date)
@@ -53,4 +65,3 @@ def writeJson(filepath, dataObject):
 
         print(f'Updated package {package}')
 
-    daysBack += 1

utils/upload/uploadAll.py

@@ -9,7 +9,7 @@
 import sys
 from pathlib import Path
 
-pause=1 # seconds
+pause=0 # seconds
 zipPath = './content.zip'
 packages = glob.glob(f'./packages/*.zip')
 server_address='http://localhost:5000'