-
Notifications
You must be signed in to change notification settings - Fork 0
Development: Cron job for user data deletion after inactivity
#1704
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
sachmii
wants to merge
25
commits into
main
Choose a base branch
from
feat/delete-user-data-after-inactivity
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
+632
−5
Open
Changes from 21 commits
Commits
Show all changes
25 commits
Select commit
Hold shift + click to select a range
4317af3
implement last activity at
sachmii eccea16
add missing comment
sachmii 3b0c16d
Merge branch 'main' into feat/1696-delete-user-data-after-inactivity
sachmii bb36eec
chore: update OpenAPI spec and generated client
github-actions[bot] 20acce7
use localdatetime instead of instant
sachmii ba0a9de
Merge branch 'feat/1696-delete-user-data-after-inactivity' of https:/…
sachmii fb0e379
feat: 1. user retention properties in yaml file and java class 2. use…
sachmii 0693cb3
feat: implemented applicant data deletion and necessary repository me…
sachmii 2310815
merge
sachmii 2238585
user retention i guess implemented so far
sachmii 0044042
logic should be working
sachmii 5b66be2
pls work
sachmii cfa1b4c
just oncew insert
sachmii b49e111
fix
sachmii d3f0860
fix codacy
sachmii f4d43c7
fix
sachmii 8a9800c
nerv nicht
sachmii 666f93a
fix server test
sachmii 23acd57
fix test
sachmii 7722a6b
fix docs
sachmii 9151309
update docs
sachmii e7bdcbc
fix data getter
sachmii 0222b01
Merge branch 'main' of https://github.com/ls1intum/tum-apply into fea…
sachmii 956a5cd
fix batch delete
sachmii ea3b8a9
fix batch delete
sachmii File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
136 changes: 136 additions & 0 deletions
136
src/main/java/de/tum/cit/aet/core/config/UserRetentionProperties.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| package de.tum.cit.aet.core.config; | ||
|
|
||
| import java.util.UUID; | ||
| import org.springframework.boot.context.properties.ConfigurationProperties; | ||
|
|
||
| /** | ||
| * Configuration properties for user retention and scheduled cleanup behavior. | ||
| * <p> | ||
| * Includes settings for inactivity thresholds, applicant data deletion timing, | ||
| * job execution parameters (cron, batch size, max runtime), and placeholders for | ||
| * representing deleted users (ID, email, names, language). Supports enabling/disabling | ||
| * the feature and running in dry-run mode. | ||
| */ | ||
| @ConfigurationProperties(prefix = "user.retention", ignoreUnknownFields = false) | ||
| public class UserRetentionProperties { | ||
|
|
||
| private Integer inactiveDaysBeforeDeletion; | ||
|
|
||
| private Integer daysBeforeApplicantDataDeletion; | ||
|
|
||
| private Boolean enabled; | ||
|
|
||
| private Boolean dryRun; | ||
|
|
||
| private Integer batchSize; | ||
|
|
||
| private Integer maxRuntimeMinutes; | ||
|
|
||
| private String cron; | ||
|
|
||
| private UUID deletedUserId; | ||
|
|
||
| private String deletedUserEmail; | ||
|
|
||
| private String deletedUserFirstName; | ||
|
|
||
| private String deletedUserLastName; | ||
|
|
||
| private String deletedUserLanguage; | ||
|
|
||
| public Integer getBatchSize() { | ||
| return batchSize; | ||
| } | ||
|
|
||
| public void setBatchSize(Integer batchSize) { | ||
| this.batchSize = batchSize; | ||
| } | ||
|
|
||
| public Integer getDaysBeforeApplicantDataDeletion() { | ||
| return daysBeforeApplicantDataDeletion; | ||
| } | ||
|
|
||
| public void setDaysBeforeApplicantDataDeletion(Integer daysBeforeApplicantDataDeletion) { | ||
| this.daysBeforeApplicantDataDeletion = daysBeforeApplicantDataDeletion; | ||
| } | ||
|
|
||
| public Boolean getDryRun() { | ||
| return dryRun; | ||
| } | ||
|
|
||
| public void setDryRun(Boolean dryRun) { | ||
| this.dryRun = dryRun; | ||
| } | ||
|
|
||
| public Boolean getEnabled() { | ||
| return enabled; | ||
| } | ||
|
|
||
| public void setEnabled(Boolean enabled) { | ||
| this.enabled = enabled; | ||
| } | ||
|
|
||
| public Integer getInactiveDaysBeforeDeletion() { | ||
| return inactiveDaysBeforeDeletion; | ||
| } | ||
|
|
||
| public void setInactiveDaysBeforeDeletion(Integer inactiveDaysBeforeDeletion) { | ||
| this.inactiveDaysBeforeDeletion = inactiveDaysBeforeDeletion; | ||
| } | ||
|
|
||
| public Integer getMaxRuntimeMinutes() { | ||
| return maxRuntimeMinutes; | ||
| } | ||
|
|
||
| public void setMaxRuntimeMinutes(Integer maxRuntimeMinutes) { | ||
| this.maxRuntimeMinutes = maxRuntimeMinutes; | ||
| } | ||
|
|
||
| public String getCron() { | ||
| return cron; | ||
| } | ||
|
|
||
| public void setCron(String cron) { | ||
| this.cron = cron; | ||
| } | ||
|
|
||
| public UUID getDeletedUserId() { | ||
| return deletedUserId; | ||
| } | ||
|
|
||
| public void setDeletedUserId(UUID deletedUserId) { | ||
| this.deletedUserId = deletedUserId; | ||
| } | ||
|
|
||
| public String getDeletedUserEmail() { | ||
| return deletedUserEmail; | ||
| } | ||
|
|
||
| public void setDeletedUserEmail(String deletedUserEmail) { | ||
| this.deletedUserEmail = deletedUserEmail; | ||
| } | ||
|
|
||
| public String getDeletedUserFirstName() { | ||
| return deletedUserFirstName; | ||
| } | ||
|
|
||
| public void setDeletedUserFirstName(String deletedUserFirstName) { | ||
| this.deletedUserFirstName = deletedUserFirstName; | ||
| } | ||
|
|
||
| public String getDeletedUserLastName() { | ||
| return deletedUserLastName; | ||
| } | ||
|
|
||
| public void setDeletedUserLastName(String deletedUserLastName) { | ||
| this.deletedUserLastName = deletedUserLastName; | ||
| } | ||
|
|
||
| public String getDeletedUserLanguage() { | ||
| return deletedUserLanguage; | ||
| } | ||
|
|
||
| public void setDeletedUserLanguage(String deletedUserLanguage) { | ||
| this.deletedUserLanguage = deletedUserLanguage; | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
3 changes: 3 additions & 0 deletions
3
src/main/java/de/tum/cit/aet/core/repository/DocumentRepository.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,11 +1,14 @@ | ||
| package de.tum.cit.aet.core.repository; | ||
|
|
||
| import de.tum.cit.aet.core.domain.Document; | ||
| import de.tum.cit.aet.usermanagement.domain.User; | ||
| import java.util.Optional; | ||
| import java.util.UUID; | ||
| import org.springframework.stereotype.Repository; | ||
|
|
||
| @Repository | ||
| public interface DocumentRepository extends TumApplyJpaRepository<Document, UUID> { | ||
| Optional<Document> findBySha256Id(String sha256Id); | ||
|
|
||
| void deleteByUploadedBy(User uploadedBy); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
128 changes: 128 additions & 0 deletions
128
src/main/java/de/tum/cit/aet/core/retention/UserRetentionJob.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| package de.tum.cit.aet.core.retention; | ||
|
|
||
| import de.tum.cit.aet.core.config.UserRetentionProperties; | ||
| import de.tum.cit.aet.usermanagement.repository.UserRepository; | ||
| import java.time.Duration; | ||
| import java.time.Instant; | ||
| import java.time.LocalDateTime; | ||
| import java.time.ZoneOffset; | ||
| import java.util.List; | ||
| import java.util.UUID; | ||
| import lombok.RequiredArgsConstructor; | ||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.springframework.data.domain.Page; | ||
| import org.springframework.data.domain.PageRequest; | ||
| import org.springframework.scheduling.annotation.Scheduled; | ||
| import org.springframework.stereotype.Component; | ||
|
|
||
| @Component | ||
| @RequiredArgsConstructor | ||
| @Slf4j | ||
| public class UserRetentionJob { | ||
|
|
||
| private final UserRetentionProperties properties; | ||
| private final UserRepository userRepository; | ||
| private final UserRetentionService userRetentionService; | ||
|
|
||
| /** | ||
| * Scheduled job that performs user retention cleanup based on configured settings. | ||
| * <p> | ||
| * If retention is disabled or no valid run configuration is available, the method exits early. | ||
| * Otherwise, it builds a run configuration, processes candidate records within the maximum | ||
| * runtime window, and logs a summary including dry-run status, cutoff timestamp, number of | ||
| * candidates seen, actual runtime, and configured maximum runtime. | ||
| */ | ||
| // Runs daily at 03:17 UTC (override with user.retention.cron) | ||
| @Scheduled(cron = "${user.retention.cron:0 17 3 * * *}", zone = "UTC") | ||
sachmii marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| public void deleteUserData() { | ||
| if (!Boolean.TRUE.equals(properties.getEnabled())) { | ||
| return; | ||
| } | ||
|
|
||
| RetentionRunConfig config = buildRunConfig(); | ||
| if (config == null) { | ||
| return; | ||
| } | ||
|
|
||
| Instant start = Instant.now(); | ||
| Instant deadline = start.plus(Duration.ofMinutes(config.maxRuntimeMinutes())); | ||
|
|
||
| RetentionRunResult result = processCandidates(config, deadline); | ||
|
|
||
| Duration runtime = Duration.between(start, Instant.now()); | ||
| log.info( | ||
| "User retention run finished: enabled=true dryRun={} cutoff={} candidatesSeen={} runtimeMs={} maxRuntimeMinutes={}", | ||
| config.dryRun(), | ||
| config.cutoff(), | ||
| result.totalCandidatesSeen(), | ||
| runtime.toMillis(), | ||
| config.maxRuntimeMinutes() | ||
| ); | ||
| } | ||
|
|
||
| private RetentionRunConfig buildRunConfig() { | ||
| Integer inactiveDays = properties.getInactiveDaysBeforeDeletion(); | ||
| Integer batchSize = properties.getBatchSize(); | ||
| Integer maxRuntimeMinutes = properties.getMaxRuntimeMinutes(); | ||
|
|
||
| if (inactiveDays == null || inactiveDays <= 0) { | ||
| log.warn("User retention enabled, but inactiveDaysBeforeDeletion is not configured (value={})", inactiveDays); | ||
| return null; | ||
| } | ||
| if (batchSize == null || batchSize <= 0) { | ||
| log.warn("User retention enabled, but batchSize is not configured (value={})", batchSize); | ||
| return null; | ||
| } | ||
| if (maxRuntimeMinutes == null || maxRuntimeMinutes <= 0) { | ||
| log.warn("User retention enabled, but maxRuntimeMinutes is not configured (value={})", maxRuntimeMinutes); | ||
| return null; | ||
| } | ||
|
|
||
| LocalDateTime nowUtc = LocalDateTime.now(ZoneOffset.UTC); | ||
| LocalDateTime cutoff = nowUtc.minusDays(inactiveDays); | ||
| boolean dryRun = Boolean.TRUE.equals(properties.getDryRun()); | ||
|
|
||
| return new RetentionRunConfig(inactiveDays, batchSize, maxRuntimeMinutes, cutoff, dryRun); | ||
| } | ||
|
|
||
| private RetentionRunResult processCandidates(RetentionRunConfig config, Instant deadline) { | ||
| long totalCandidatesSeen = 0; | ||
| int pageNumber = 0; | ||
|
|
||
| while (Instant.now().isBefore(deadline)) { | ||
| PageRequest pageRequest = config.dryRun() | ||
| ? PageRequest.of(pageNumber, config.batchSize()) | ||
| : PageRequest.of(0, config.batchSize()); | ||
|
|
||
| Page<UUID> userIds = userRepository.findInactiveNonAdminUserIdsForRetention(config.cutoff(), pageRequest); | ||
| List<UUID> ids = userIds.getContent(); | ||
|
|
||
| if (ids.isEmpty()) { | ||
| break; | ||
| } | ||
|
|
||
| totalCandidatesSeen += ids.size(); | ||
|
|
||
| userRetentionService.processUserIdsList(ids, config.cutoff(), config.dryRun()); | ||
|
|
||
| if (config.dryRun()) { | ||
| if (!userIds.hasNext()) { | ||
| break; | ||
| } | ||
| pageNumber++; | ||
| } | ||
| } | ||
|
|
||
| return new RetentionRunResult(totalCandidatesSeen); | ||
| } | ||
|
|
||
| private record RetentionRunConfig( | ||
| Integer inactiveDays, | ||
| Integer batchSize, | ||
| Integer maxRuntimeMinutes, | ||
| LocalDateTime cutoff, | ||
| boolean dryRun | ||
| ) {} | ||
|
|
||
| private record RetentionRunResult(long totalCandidatesSeen) {} | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.