Periodically check for new stars and remove old stars

Author: skalahonza

Components/Pages/Home.razor

@@ -241,16 +241,7 @@ else
             Logger.LogInformation("Indexing {Count} repositories starred by {Username}", starred.Count, GithubUsername);
             await SearchService.IndexRepositories(starred
                 .Where(x => !x.Private) // skip private repositories, the token used should not have access anyway, but just in case
-                .Select(x => new Repository
-            {
-                Id = Repository.ComputeRepositoryId(GithubUsername, x.Id),
-                Slug = x.Name,
-                Owner = x.Owner.Login,
-                Url = x.HtmlUrl,
-                UpdatedAt = x.UpdatedAt,
-                StarredBy = GithubUsername,
-                Description = x.Description,
-            }));
+                .Select(x => Repository.FromGithubRepository(x, GithubUsername)));
             Logger.LogInformation("User {Username} onboarded successfully", GithubUsername);
             Snackbar.Add("Repositories indexed successfully", Severity.Success);
         }

Program.cs

@@ -3,6 +3,7 @@
 using GithubStarSearch.Components;
 using GithubStarSearch.Searching;
 using MudBlazor.Services;
+using Octokit;
 using Serilog;
 
 var builder = WebApplication.CreateBuilder(args);
@@ -21,9 +22,28 @@
     .AddInteractiveServerComponents();
 
 // Application logic services
+builder.Services.AddScoped<GitHubClient>(x =>
+{
+    var configuration = x.GetRequiredService<IConfiguration>();
+    var logger = x.GetRequiredService<ILogger<Program>>();
+    var token = configuration["Github:FineGrainedToken"];
+
+    var client = new GitHubClient(new ProductHeaderValue("GithubStarSearch"));
+    if (string.IsNullOrEmpty(token))
+    {
+        logger.LogCritical("No Github:FineGrainedToken found in configuration");
+    }
+    else
+    {
+        client.Credentials = new Credentials(token);
+    }
+
+    return client;
+});
 builder.AddMeilisearch();
 builder.Services.AddBlazoredLocalStorage();
-builder.Services.AddHostedService<Indexer>();
+builder.Services.AddHostedService<RepositoryUpdater>();
+builder.Services.AddHostedService<StarsUpdater>();
 
 var app = builder.Build();
 

Indexer.cs RepositoryUpdater.csIndexer.cs renamed to RepositoryUpdater.cs

@@ -9,7 +9,7 @@ namespace GithubStarSearch;
 /// <summary>
 /// Background worker that periodically updates repositories.
 /// </summary>
-public class Indexer(ILogger<Indexer> logger, IServiceProvider serviceProvider) : BackgroundService
+public class RepositoryUpdater(ILogger<RepositoryUpdater> logger, IServiceProvider serviceProvider) : BackgroundService
 {
     private const int RequestLimit = 200;
     private readonly PeriodicTimer _timer = new(TimeSpan.FromMinutes(5));
@@ -66,7 +66,7 @@ private async Task<ResourceResults<IEnumerable<Repository>>> DoWork(int offset)
 
     private async Task UpdateRepository(Repository repository, GitHubClient github)
     {
-        var details = await github.Repository.Get(repository.Owner, repository.Slug);
+        var details = await GetDetails(repository, github);
         if (details is null)
         {
             logger.LogWarning("Repository {Owner}/{Slug} not found", repository.Owner, repository.Slug);
@@ -80,6 +80,25 @@ private async Task UpdateRepository(Repository repository, GitHubClient github)
         repository.Readme = readme;
     }
 
+    private async Task<Octokit.Repository?> GetDetails(Repository repository, GitHubClient github)
+    {
+        try
+        {
+            return await github.Repository.Get(repository.Owner, repository.Slug);
+        }
+        catch (ForbiddenException e)
+        {
+            logger.LogWarning(e, "Forbidden while fetching details for {Owner}/{Slug}", repository.Owner,
+                repository.Slug);
+            return null;
+        }
+        catch (NotFoundException e)
+        {
+            logger.LogWarning(e, "Repository {Owner}/{Slug} not found", repository.Owner, repository.Slug);
+            return null;
+        }
+    }
+
     private GitHubClient CreateClient()
     {
         using var scope = serviceProvider.CreateScope();

Searching/Repository.cs

@@ -17,7 +17,18 @@ public class Repository
 
     public string Readme { get; set; } = "";
 
-    public static string ComputeRepositoryId(string starredBy, long id)
+    public static Repository FromGithubRepository(Octokit.Repository repository, string starredBy) => new()
+    {
+        Id = ComputeRepositoryId(starredBy, repository.Id),
+        Slug = repository.Name,
+        Owner = repository.Owner.Login,
+        Url = repository.HtmlUrl,
+        UpdatedAt = repository.UpdatedAt,
+        StarredBy = starredBy,
+        Description = repository.Description,
+    };
+
+    private static string ComputeRepositoryId(string starredBy, long id)
     {
         // having unique id composed of owner and slug is not enough 
         // because the same repository can be starred by multiple users
@@ -26,4 +37,8 @@ public static string ComputeRepositoryId(string starredBy, long id)
         // A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_).
         return $"{starredBy}-{id}";
     }
+
+    public override int GetHashCode() => Id.GetHashCode();
+
+    public override bool Equals(object? obj) => obj is Repository other && other.Id == Id;
 }

Searching/SearchService.cs

@@ -42,6 +42,28 @@ public async Task IndexRepositories(IEnumerable<Repository> repositories)
         }
     }
 
+    public async Task RemoveRepositories(IEnumerable<Repository> repositories)
+    {
+        var index = client.Index(searchOptions.Value.RepositoriesIndexName);
+        var task = await index.DeleteDocumentsAsync(repositories.Select(x => x.Id));
+
+        var info = await index.WaitForTaskAsync(task.TaskUid);
+        if (info.Status != TaskInfoStatus.Succeeded)
+        {
+            var builder = new StringBuilder();
+            foreach (var (key, value) in info.Error)
+            {
+                builder.AppendLine($"{key}: {value}");
+            }
+
+            logger.LogError("Failed to remove repositories: {Error}", builder.ToString());
+        }
+        else
+        {
+            logger.LogInformation("Removing succeeded in {Duration}", info.Duration);
+        }
+    }
+
     public async Task<IReadOnlyCollection<Repository>> SearchRepositories(string starredBy,
         string term,
         SearchOptions options)
@@ -91,6 +113,23 @@ public Task<ResourceResults<IEnumerable<Repository>>> GetRepositories(int limit,
         });
     }
 
+    public async IAsyncEnumerable<Repository> GetAllRepositories()
+    {
+        var index = client.Index(searchOptions.Value.RepositoriesIndexName);
+        var offset = 0;
+        ResourceResults<IEnumerable<Repository>>? result;
+        do
+        {
+            result = await index.GetDocumentsAsync<Repository>(new DocumentsQuery { Offset = offset });
+            foreach (var repository in result.Results)
+            {
+                yield return repository;
+            }
+
+            offset += result.Results.Count();
+        } while (result.Results.Any());
+    }
+
     public async Task<bool> IsIndexed(string githubUsername)
     {
         var index = client.Index(searchOptions.Value.RepositoriesIndexName);

StarsUpdater.cs

@@ -0,0 +1,52 @@
+using GithubStarSearch.Searching;
+using Octokit;
+using Repository = GithubStarSearch.Searching.Repository;
+
+namespace GithubStarSearch;
+
+/// <summary>
+/// Periodically checks stars of monitored users.
+/// Adds newly starred repositories to the full-text search index.
+/// Removes un-starred repositories from the full-text search index.
+/// </summary>
+public class StarsUpdater(ILogger<StarsUpdater> logger, IServiceProvider serviceProvider) : BackgroundService
+{
+    private const int RequestLimit = 200;
+    private readonly PeriodicTimer _timer = new(TimeSpan.FromMinutes(5));
+
+    protected override async Task ExecuteAsync(CancellationToken ct)
+    {
+        do
+        {
+            try
+            {
+                using var scope = serviceProvider.CreateScope();
+                var service = scope.ServiceProvider.GetRequiredService<SearchService>();
+                var github = scope.ServiceProvider.GetRequiredService<GitHubClient>();
+
+                var allRepositories = new Dictionary<string, HashSet<Repository>>();
+                await foreach (var repository in service.GetAllRepositories().WithCancellation(ct))
+                {
+                    allRepositories.TryAdd(repository.StarredBy, []);
+                    allRepositories[repository.StarredBy].Add(repository);
+                }
+
+                foreach (var (user, currentStars) in allRepositories)
+                {
+                    var starred = await github.Activity.Starring.GetAllForUser(user) ?? [];
+                    var githubStars = starred.Select(x => Repository.FromGithubRepository(x, user)).ToHashSet();
+                    var unstarred = currentStars.Except(githubStars).ToList();
+                    var newlyStarred = githubStars.Except(currentStars).ToList();
+                    await service.IndexRepositories(newlyStarred);
+                    await service.RemoveRepositories(unstarred);
+                }
+
+                logger.LogInformation("Waiting {Period} for next tick", _timer.Period);
+            }
+            catch (Exception e)
+            {
+                logger.LogError(e, "Error while updating stars");
+            }
+        } while (!ct.IsCancellationRequested && await _timer.WaitForNextTickAsync(ct));
+    }
+}

github-star-search.csproj

@@ -17,5 +17,6 @@
     <PackageReference Include="Octokit" Version="14.0.0"/>
     <PackageReference Include="MeiliSearch" Version="0.15.5"/>
     <PackageReference Include="Serilog.AspNetCore" Version="9.0.0"/>
+    <PackageReference Include="System.Linq.Async" Version="6.0.1"/>
   </ItemGroup>
 </Project>