diff --git a/pom-dependency-tree.txt b/pom-dependency-tree.txt index becfc3364..5ff96512c 100644 --- a/pom-dependency-tree.txt +++ b/pom-dependency-tree.txt @@ -1,4 +1,4 @@ -ai.elimu:webapp:war:2.6.88-SNAPSHOT +ai.elimu:webapp:war:2.6.89-SNAPSHOT +- ai.elimu:model:jar:model-2.0.114:compile | \- com.google.code.gson:gson:jar:2.13.1:compile | \- com.google.errorprone:error_prone_annotations:jar:2.38.0:compile diff --git a/src/main/java/ai/elimu/entity/analytics/NumberAssessmentEvent.java b/src/main/java/ai/elimu/entity/analytics/NumberAssessmentEvent.java index 2fa894256..03f240b5e 100644 --- a/src/main/java/ai/elimu/entity/analytics/NumberAssessmentEvent.java +++ b/src/main/java/ai/elimu/entity/analytics/NumberAssessmentEvent.java @@ -1,6 +1,8 @@ package ai.elimu.entity.analytics; +import ai.elimu.entity.content.Number; import jakarta.persistence.Entity; +import jakarta.persistence.ManyToOne; import jakarta.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; @@ -27,4 +29,7 @@ public class NumberAssessmentEvent extends AssessmentEvent { * In that case, this field will be {@code null}. */ private Long numberId; + + @ManyToOne + private Number number; } diff --git a/src/main/java/ai/elimu/tasks/analytics/NumberAssessmentEventImportScheduler.java b/src/main/java/ai/elimu/tasks/analytics/NumberAssessmentEventImportScheduler.java new file mode 100644 index 000000000..84fb73032 --- /dev/null +++ b/src/main/java/ai/elimu/tasks/analytics/NumberAssessmentEventImportScheduler.java @@ -0,0 +1,127 @@ +package ai.elimu.tasks.analytics; + +import ai.elimu.dao.StudentDao; +import ai.elimu.dao.NumberAssessmentEventDao; +import ai.elimu.dao.NumberDao; +import ai.elimu.entity.analytics.NumberAssessmentEvent; +import ai.elimu.entity.analytics.students.Student; +import ai.elimu.model.v2.enums.Language; +import ai.elimu.rest.v2.analytics.NumberAssessmentEventsRestController; +import ai.elimu.util.ConfigHelper; +import ai.elimu.util.DiscordHelper; +import ai.elimu.util.DiscordHelper.Channel; +import ai.elimu.util.DomainHelper; +import ai.elimu.util.csv.CsvAnalyticsExtractionHelper; +import java.io.File; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +/** + * Extracts assessment events from CSV files previously received by the {@link NumberAssessmentEventsRestController}, and imports them into the database. + *

+ *

+ * Expected folder structure: + *

+ * ├── lang-ENG
+ * │   ├── analytics
+ * │   │   ├── android-id-e387e38700000001
+ * │   │   │   └── number-assessment-events
+ * │   │   │       ├── e387e38700000001_3001018_number-assessment-events_2024-10-09.csv
+ * │   │   │       ├── e387e38700000001_3001018_number-assessment-events_2024-10-10.csv
+ * │   │   │       ├── e387e38700000001_3001018_number-assessment-events_2024-10-11.csv
+ * │   │   │       ├── e387e38700000001_3001018_number-assessment-events_2024-10-14.csv
+ * │   │   │       ├── e387e38700000001_3001018_number-assessment-events_2024-10-18.csv
+ * │   │   │       └── e387e38700000001_3001018_number-assessment-events_2024-10-20.csv
+ * │   │   ├── android-id-e387e38700000002
+ * │   │   │   └── number-assessment-events
+ * │   │   │       ├── e387e38700000002_3001018_number-assessment-events_2024-10-09.csv
+ * │   │   │       ├── e387e38700000002_3001018_number-assessment-events_2024-10-10.csv
+ * │   │   │       ├── e387e38700000002_3001018_number-assessment-events_2024-10-11.csv
+ * 
+ */ +@Service +@RequiredArgsConstructor +@Slf4j +public class NumberAssessmentEventImportScheduler { + + private final NumberAssessmentEventDao numberAssessmentEventDao; + private final NumberDao numberDao; + private final StudentDao studentDao; + + @Scheduled(cron = "00 25 * * * *") // 25 minutes past every hour + public synchronized void execute() { + log.info("execute"); + + try { + // Lookup CSV files stored on the filesystem + File elimuAiDir = new File(System.getProperty("user.home"), ".elimu-ai"); + File languageDir = new File(elimuAiDir, "lang-" + Language.valueOf(ConfigHelper.getProperty("content.language"))); + File analyticsDir = new File(languageDir, "analytics"); + log.info("analyticsDir: " + analyticsDir); + analyticsDir.mkdirs(); + for (File analyticsDirFile : analyticsDir.listFiles()) { + if (analyticsDirFile.getName().startsWith("android-id-")) { + File androidIdDir = new File(analyticsDir, analyticsDirFile.getName()); + for (File androidIdDirFile : androidIdDir.listFiles()) { + Long studentId = null; + Integer eventImportCount = 0; + if (androidIdDirFile.getName().equals("number-assessment-events")) { + File numberAssessmentEventsDir = new File(androidIdDir, androidIdDirFile.getName()); + for (File csvFile : numberAssessmentEventsDir.listFiles()) { + log.info("csvFile: " + csvFile); + + // Convert from CSV to Java + List events = CsvAnalyticsExtractionHelper.extractNumberAssessmentEvents(csvFile); + log.info("events.size(): " + events.size()); + + // Store in database + for (NumberAssessmentEvent event : events) { + // Check if the event has already been stored in the database + NumberAssessmentEvent existingNumberAssessmentEvent = numberAssessmentEventDao.read(event.getTimestamp(), event.getAndroidId(), event.getPackageName()); + if (existingNumberAssessmentEvent != null) { + log.warn("The event has already been stored in the database. Skipping data import."); + continue; + } + + // Generate Student ID + Student existingStudent = studentDao.read(event.getAndroidId()); + if (existingStudent == null) { + Student student = new Student(); + student.setAndroidId(event.getAndroidId()); + studentDao.create(student); + log.info("Stored Student in database with ID " + student.getId()); + studentId = student.getId(); + } else { + studentId = existingStudent.getId(); + } + + // If content ID has been provided, look for match in the database + if (event.getNumberId() != null) { + event.setNumber(numberDao.read(event.getNumberId())); + } + + // Store the event in the database + numberAssessmentEventDao.create(event); + log.info("Stored event in database with ID " + event.getId()); + eventImportCount++; + } + } + } + if ((studentId != null) && (eventImportCount > 0)) { + String contentUrl = DomainHelper.getBaseUrl() + "/analytics/students/" + studentId; + DiscordHelper.postToChannel(Channel.ANALYTICS, "Imported " + eventImportCount + " number assessment events: " + contentUrl); + } + } + } + } + } catch (Exception e) { + log.error("Error during data import:", e); + DiscordHelper.postToChannel(Channel.ANALYTICS, "Error during import of number assessment events: `" + e.getClass() + ": " + e.getMessage() + "`"); + } + + log.info("execute complete"); + } +} diff --git a/src/main/java/ai/elimu/tasks/analytics/StoryBookLearningEventImportScheduler.java b/src/main/java/ai/elimu/tasks/analytics/StoryBookLearningEventImportScheduler.java index 6d4602747..59a805cea 100644 --- a/src/main/java/ai/elimu/tasks/analytics/StoryBookLearningEventImportScheduler.java +++ b/src/main/java/ai/elimu/tasks/analytics/StoryBookLearningEventImportScheduler.java @@ -51,7 +51,7 @@ public class StoryBookLearningEventImportScheduler { private final StoryBookDao storyBookDao; private final StudentDao studentDao; - @Scheduled(cron = "00 45 * * * *") // 35 minutes past every hour + @Scheduled(cron = "00 45 * * * *") // 45 minutes past every hour public synchronized void execute() { log.info("execute"); diff --git a/src/main/java/ai/elimu/tasks/analytics/VideoLearningEventImportScheduler.java b/src/main/java/ai/elimu/tasks/analytics/VideoLearningEventImportScheduler.java index 5e092e4d8..f127e4dac 100644 --- a/src/main/java/ai/elimu/tasks/analytics/VideoLearningEventImportScheduler.java +++ b/src/main/java/ai/elimu/tasks/analytics/VideoLearningEventImportScheduler.java @@ -51,7 +51,7 @@ public class VideoLearningEventImportScheduler { private final VideoDao videoDao; private final StudentDao studentDao; - @Scheduled(cron = "00 50 * * * *") // 40 minutes past every hour + @Scheduled(cron = "00 50 * * * *") // 50 minutes past every hour public synchronized void execute() { log.info("execute"); diff --git a/src/main/java/ai/elimu/tasks/analytics/WordAssessmentEventImportScheduler.java b/src/main/java/ai/elimu/tasks/analytics/WordAssessmentEventImportScheduler.java index 9eb437122..ae8012bd4 100644 --- a/src/main/java/ai/elimu/tasks/analytics/WordAssessmentEventImportScheduler.java +++ b/src/main/java/ai/elimu/tasks/analytics/WordAssessmentEventImportScheduler.java @@ -51,7 +51,7 @@ public class WordAssessmentEventImportScheduler { private final WordDao wordDao; private final StudentDao studentDao; - @Scheduled(cron = "00 35 * * * *") // 25 minutes past every hour + @Scheduled(cron = "00 35 * * * *") // 35 minutes past every hour public synchronized void execute() { log.info("execute"); diff --git a/src/main/java/ai/elimu/tasks/analytics/WordLearningEventImportScheduler.java b/src/main/java/ai/elimu/tasks/analytics/WordLearningEventImportScheduler.java index d8b2f2821..9afca5f6c 100644 --- a/src/main/java/ai/elimu/tasks/analytics/WordLearningEventImportScheduler.java +++ b/src/main/java/ai/elimu/tasks/analytics/WordLearningEventImportScheduler.java @@ -51,7 +51,7 @@ public class WordLearningEventImportScheduler { private final WordDao wordDao; private final StudentDao studentDao; - @Scheduled(cron = "00 40 * * * *") // 30 minutes past every hour + @Scheduled(cron = "00 40 * * * *") // 40 minutes past every hour public synchronized void execute() { log.info("execute"); diff --git a/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java b/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java index 6ec8ee256..0be28267a 100644 --- a/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java +++ b/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java @@ -2,6 +2,7 @@ import ai.elimu.entity.analytics.LetterSoundAssessmentEvent; import ai.elimu.entity.analytics.LetterSoundLearningEvent; +import ai.elimu.entity.analytics.NumberAssessmentEvent; import ai.elimu.entity.analytics.NumberLearningEvent; import ai.elimu.entity.analytics.StoryBookLearningEvent; import ai.elimu.entity.analytics.VideoLearningEvent; @@ -198,7 +199,79 @@ public static List extractLetterSoundLearningEvents(Fi } - // TODO: number assessment events + public static List extractNumberAssessmentEvents(File csvFile) { + log.info("extractNumberAssessmentEvents"); + + Integer versionCode = AnalyticsHelper.extractVersionCodeFromCsvFilename(csvFile.getName()); + log.info("versionCode: " + versionCode); + + List numberAssessmentEvents = new ArrayList<>(); + + // Iterate each row in the CSV file + Path csvFilePath = Paths.get(csvFile.toURI()); + log.info("csvFilePath: " + csvFilePath); + try { + Reader reader = Files.newBufferedReader(csvFilePath); + CSVFormat csvFormat = CSVFormat.DEFAULT.withFirstRecordAsHeader(); + log.info("header: " + Arrays.toString(csvFormat.getHeader())); + CSVParser csvParser = new CSVParser(reader, csvFormat); + for (CSVRecord csvRecord : csvParser) { + log.info("csvRecord: " + csvRecord); + + // Convert from CSV to Java + + NumberAssessmentEvent numberAssessmentEvent = new NumberAssessmentEvent(); + + long timestampInMillis = Long.valueOf(csvRecord.get("timestamp").substring(0, 10)) * 1_000; + Calendar timestamp = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + timestamp.setTimeInMillis(timestampInMillis); + numberAssessmentEvent.setTimestamp(timestamp); + + String androidId = AnalyticsHelper.extractAndroidIdFromCsvFilename(csvFile.getName()); + numberAssessmentEvent.setAndroidId(androidId); + + String packageName = csvRecord.get("package_name"); + numberAssessmentEvent.setPackageName(packageName); + + Float masteryScore = Float.valueOf(csvRecord.get("mastery_score")); + numberAssessmentEvent.setMasteryScore(masteryScore); + + Long timeSpentMs = Long.valueOf(csvRecord.get("time_spent_ms")); + numberAssessmentEvent.setTimeSpentMs(timeSpentMs); + + String additionalData = csvRecord.get("additional_data"); + if (StringUtils.isNotBlank(additionalData)) { + numberAssessmentEvent.setAdditionalData(additionalData); + } + + int researchExperimentOrdinal = Integer.valueOf(csvRecord.get("research_experiment")); + ResearchExperiment researchExperiment = ResearchExperiment.values()[researchExperimentOrdinal]; + numberAssessmentEvent.setResearchExperiment(researchExperiment); + + int experimentGroupOrdinal = Integer.valueOf(csvRecord.get("experiment_group")); + ExperimentGroup experimentGroup = ExperimentGroup.values()[experimentGroupOrdinal]; + numberAssessmentEvent.setExperimentGroup(experimentGroup); + + Integer numberValue = Integer.valueOf(csvRecord.get("number_value")); + numberAssessmentEvent.setNumberValue(numberValue); + + // String numberSymbol = csvRecord.get("number_symbol"); + // numberAssessmentEvent.setNumberSymbol(numberSymbol); + + if (StringUtils.isNotBlank(csvRecord.get("number_id"))) { + Long numberId = Long.valueOf(csvRecord.get("number_id")); + numberAssessmentEvent.setNumberId(numberId); + } + + numberAssessmentEvents.add(numberAssessmentEvent); + } + csvParser.close(); + } catch (IOException ex) { + log.error(ex.getMessage()); + } + + return numberAssessmentEvents; + } public static List extractNumberLearningEvents(File csvFile) { log.info("extractNumberLearningEvents"); diff --git a/src/main/java/ai/elimu/web/analytics/MainAnalyticsController.java b/src/main/java/ai/elimu/web/analytics/MainAnalyticsController.java index 35c04d2e2..00f7e479a 100644 --- a/src/main/java/ai/elimu/web/analytics/MainAnalyticsController.java +++ b/src/main/java/ai/elimu/web/analytics/MainAnalyticsController.java @@ -2,6 +2,7 @@ import ai.elimu.dao.LetterSoundAssessmentEventDao; import ai.elimu.dao.LetterSoundLearningEventDao; +import ai.elimu.dao.NumberAssessmentEventDao; import ai.elimu.dao.NumberLearningEventDao; import ai.elimu.dao.StoryBookLearningEventDao; import ai.elimu.dao.StudentDao; @@ -30,6 +31,7 @@ public class MainAnalyticsController { private final WordAssessmentEventDao wordAssessmentEventDao; private final WordLearningEventDao wordLearningEventDao; + private final NumberAssessmentEventDao numberAssessmentEventDao; private final NumberLearningEventDao numberLearningEventDao; private final StoryBookLearningEventDao storyBookLearningEventDao; @@ -49,7 +51,7 @@ public String handleRequest(Model model) { model.addAttribute("wordAssessmentEventCount", wordAssessmentEventDao.readCount()); model.addAttribute("wordLearningEventCount", wordLearningEventDao.readCount()); - // TODO: number assessment events + model.addAttribute("numberAssessmentEventCount", numberAssessmentEventDao.readCount()); model.addAttribute("numberLearningEventCount", numberLearningEventDao.readCount()); model.addAttribute("storyBookLearningEventCount", storyBookLearningEventDao.readCount()); diff --git a/src/main/java/ai/elimu/web/analytics/students/NumberAssessmentEventsCsvExportController.java b/src/main/java/ai/elimu/web/analytics/students/NumberAssessmentEventsCsvExportController.java new file mode 100644 index 000000000..e78e3526e --- /dev/null +++ b/src/main/java/ai/elimu/web/analytics/students/NumberAssessmentEventsCsvExportController.java @@ -0,0 +1,96 @@ +package ai.elimu.web.analytics.students; + +import ai.elimu.dao.NumberAssessmentEventDao; +import ai.elimu.dao.StudentDao; +import ai.elimu.entity.analytics.NumberAssessmentEvent; +import ai.elimu.entity.analytics.students.Student; +import ai.elimu.util.DiscordHelper; +import ai.elimu.util.DiscordHelper.Channel; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.OutputStream; +import java.io.StringWriter; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; + +@Controller +@RequestMapping("/analytics/students/{studentId}/number-assessment-events.csv") +@RequiredArgsConstructor +@Slf4j +public class NumberAssessmentEventsCsvExportController { + + private final StudentDao studentDao; + + private final NumberAssessmentEventDao numberAssessmentEventDao; + + @GetMapping + public void handleRequest( + @PathVariable Long studentId, + HttpServletResponse response, + OutputStream outputStream + ) throws IOException { + log.info("handleRequest"); + + try { + Student student = studentDao.read(studentId); + log.info("student.getAndroidId(): " + student.getAndroidId()); + + List numberAssessmentEvents = numberAssessmentEventDao.readAll(student.getAndroidId()); + log.info("numberAssessmentEvents.size(): " + numberAssessmentEvents.size()); + + CSVFormat csvFormat = CSVFormat.DEFAULT.builder() + .setHeader( + "id", + "timestamp", + "package_name", + "mastery_score", + "time_spent_ms", + "additional_data", + "research_experiment", + "experiment_group", + "number_value", + "number_id" + ).build(); + StringWriter stringWriter = new StringWriter(); + CSVPrinter csvPrinter = new CSVPrinter(stringWriter, csvFormat); + for (NumberAssessmentEvent event : numberAssessmentEvents) { + log.info("event.getId(): " + event.getId()); + csvPrinter.printRecord( + event.getId(), + event.getTimestamp().getTimeInMillis() / 1_000, + event.getPackageName(), + event.getMasteryScore(), + event.getTimeSpentMs(), + event.getAdditionalData(), + (event.getResearchExperiment() != null) ? event.getResearchExperiment().ordinal() : null, + (event.getExperimentGroup() != null) ? event.getExperimentGroup().ordinal() : null, + event.getNumberValue(), + event.getNumberId() + ); + } + csvPrinter.flush(); + csvPrinter.close(); + + String csvFileContent = stringWriter.toString(); + response.setContentType("text/csv"); + byte[] bytes = csvFileContent.getBytes(); + response.setContentLength(bytes.length); + + outputStream.write(bytes); + outputStream.flush(); + outputStream.close(); + } catch (Exception ex) { + log.error(ex.getMessage()); + response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value()); + DiscordHelper.postToChannel(Channel.ANALYTICS, "Error during CSV export of number assessment events: `" + ex.getClass() + ": " + ex.getMessage() + "`"); + } + } +} diff --git a/src/main/java/ai/elimu/web/analytics/students/StudentController.java b/src/main/java/ai/elimu/web/analytics/students/StudentController.java index 5781dbb09..9dfb12046 100644 --- a/src/main/java/ai/elimu/web/analytics/students/StudentController.java +++ b/src/main/java/ai/elimu/web/analytics/students/StudentController.java @@ -2,6 +2,7 @@ import ai.elimu.dao.LetterSoundAssessmentEventDao; import ai.elimu.dao.LetterSoundLearningEventDao; +import ai.elimu.dao.NumberAssessmentEventDao; import ai.elimu.dao.NumberLearningEventDao; import ai.elimu.dao.StoryBookChapterDao; import ai.elimu.dao.StoryBookDao; @@ -13,6 +14,7 @@ import ai.elimu.dao.WordLearningEventDao; import ai.elimu.entity.analytics.LetterSoundAssessmentEvent; import ai.elimu.entity.analytics.LetterSoundLearningEvent; +import ai.elimu.entity.analytics.NumberAssessmentEvent; import ai.elimu.entity.analytics.NumberLearningEvent; import ai.elimu.entity.analytics.StoryBookLearningEvent; import ai.elimu.entity.analytics.VideoLearningEvent; @@ -58,6 +60,7 @@ public class StudentController { private final WordAssessmentEventDao wordAssessmentEventDao; private final WordLearningEventDao wordLearningEventDao; + private final NumberAssessmentEventDao numberAssessmentEventDao; private final NumberLearningEventDao numberLearningEventDao; private final StoryBookLearningEventDao storyBookLearningEventDao; @@ -237,6 +240,33 @@ public String handleRequest(@PathVariable Long studentId, Model model) { model.addAttribute("wordEventCountList", wordEventCountList); + // Prepare chart data - NumberAssessmentEvents + List numberAssessmentEvents = numberAssessmentEventDao.readAll(student.getAndroidId()); + model.addAttribute("numberAssessmentEvents", numberAssessmentEvents); + List numberAssessmentEventCorrectCountList = new ArrayList<>(); + List numberAssessmentEventIncorrectCountList = new ArrayList<>(); + if (!numberAssessmentEvents.isEmpty()) { + Map eventCorrectCountByWeekMap = new HashMap<>(); + Map eventIncorrectCountByWeekMap = new HashMap<>(); + for (NumberAssessmentEvent event : numberAssessmentEvents) { + String eventWeek = simpleDateFormat.format(event.getTimestamp().getTime()); + if (event.getMasteryScore() < 0.5) { + eventIncorrectCountByWeekMap.put(eventWeek, eventIncorrectCountByWeekMap.getOrDefault(eventWeek, 0) + 1); + } else { + eventCorrectCountByWeekMap.put(eventWeek, eventCorrectCountByWeekMap.getOrDefault(eventWeek, 0) + 1); + } + } + week = (Calendar) calendar6MonthsAgo.clone(); + while (!week.after(calendarNow)) { + String weekAsString = simpleDateFormat.format(week.getTime()); + numberAssessmentEventCorrectCountList.add(eventCorrectCountByWeekMap.getOrDefault(weekAsString, 0)); + numberAssessmentEventIncorrectCountList.add(eventIncorrectCountByWeekMap.getOrDefault(weekAsString, 0)); + week.add(Calendar.WEEK_OF_YEAR, 1); + } + } + model.addAttribute("numberAssessmentEventCorrectCountList", numberAssessmentEventCorrectCountList); + model.addAttribute("numberAssessmentEventIncorrectCountList", numberAssessmentEventIncorrectCountList); + // Prepare chart data - NumberLearningEvents List numberLearningEvents = numberLearningEventDao.readAll(student.getAndroidId()); model.addAttribute("numberLearningEvents", numberLearningEvents); diff --git a/src/main/java/ai/elimu/web/servlet/CustomDispatcherServlet.java b/src/main/java/ai/elimu/web/servlet/CustomDispatcherServlet.java index c5eb0158c..343227052 100644 --- a/src/main/java/ai/elimu/web/servlet/CustomDispatcherServlet.java +++ b/src/main/java/ai/elimu/web/servlet/CustomDispatcherServlet.java @@ -7,6 +7,7 @@ import ai.elimu.dao.LetterDao; import ai.elimu.dao.LetterSoundDao; import ai.elimu.dao.LetterSoundLearningEventDao; +import ai.elimu.dao.NumberAssessmentEventDao; import ai.elimu.dao.NumberDao; import ai.elimu.dao.NumberLearningEventDao; import ai.elimu.dao.SoundDao; @@ -21,6 +22,7 @@ import ai.elimu.dao.WordDao; import ai.elimu.dao.WordLearningEventDao; import ai.elimu.entity.analytics.LetterSoundLearningEvent; +import ai.elimu.entity.analytics.NumberAssessmentEvent; import ai.elimu.entity.analytics.NumberLearningEvent; import ai.elimu.entity.analytics.StoryBookLearningEvent; import ai.elimu.entity.analytics.VideoLearningEvent; @@ -358,6 +360,7 @@ private void populateDatabase(WebApplicationContext webApplicationContext) { LetterSoundLearningEventDao letterSoundLearningEventDao = (LetterSoundLearningEventDao) webApplicationContext.getBean("letterSoundLearningEventDao"); WordAssessmentEventDao wordAssessmentEventDao = (WordAssessmentEventDao) webApplicationContext.getBean("wordAssessmentEventDao"); WordLearningEventDao wordLearningEventDao = (WordLearningEventDao) webApplicationContext.getBean("wordLearningEventDao"); + NumberAssessmentEventDao numberAssessmentEventDao = (NumberAssessmentEventDao) webApplicationContext.getBean("numberAssessmentEventDao"); NumberLearningEventDao numberLearningEventDao = (NumberLearningEventDao) webApplicationContext.getBean("numberLearningEventDao"); StoryBookLearningEventDao storyBookLearningEventDao = (StoryBookLearningEventDao) webApplicationContext.getBean("storyBookLearningEventDao"); VideoLearningEventDao videoLearningEventDao = (VideoLearningEventDao) webApplicationContext.getBean("videoLearningEventDao"); @@ -387,14 +390,14 @@ private void populateDatabase(WebApplicationContext webApplicationContext) { wordAssessmentEvent.setTimestamp(week); wordAssessmentEvent.setAndroidId(student.getAndroidId()); wordAssessmentEvent.setPackageName("ai.elimu.kukariri"); + wordAssessmentEvent.setMasteryScore((float) (int) (Math.random() * 2)); + wordAssessmentEvent.setTimeSpentMs((long) (Math.random() * 20_000)); if (weekCount > 26/2) { wordAssessmentEvent.setResearchExperiment(ResearchExperiment.EXP_0_WORD_EMOJIS); wordAssessmentEvent.setExperimentGroup(ExperimentGroup.values()[(int) (Math.random() * 2)]); } wordAssessmentEvent.setWordText(wordMAA.getText()); wordAssessmentEvent.setWordId(wordMAA.getId()); - wordAssessmentEvent.setMasteryScore((float) (int) (Math.random() * 2)); - wordAssessmentEvent.setTimeSpentMs((long) (Math.random() * 20_000)); wordAssessmentEventDao.create(wordAssessmentEvent); } @@ -419,6 +422,23 @@ private void populateDatabase(WebApplicationContext webApplicationContext) { wordLearningEventDao.create(wordLearningEvent); } + int randomNumberOfNumberAssessmentEvents = (int) (Math.random() * 10); + for (int i = 0; i < randomNumberOfNumberAssessmentEvents; i++) { + NumberAssessmentEvent numberAssessmentEvent = new NumberAssessmentEvent(); + numberAssessmentEvent.setTimestamp(week); + numberAssessmentEvent.setAndroidId(student.getAndroidId()); + numberAssessmentEvent.setPackageName("ai.elimu.learndigits"); + numberAssessmentEvent.setMasteryScore((float) (int) (Math.random() * 2)); + numberAssessmentEvent.setTimeSpentMs((long) (Math.random() * 10_000)); + if (weekCount > 26/2) { + numberAssessmentEvent.setResearchExperiment(ResearchExperiment.EXP_0_WORD_EMOJIS); + numberAssessmentEvent.setExperimentGroup(ExperimentGroup.values()[(int) (Math.random() * 2)]); + } + numberAssessmentEvent.setNumberValue(number3.getValue()); + numberAssessmentEvent.setNumberId(number3.getId()); + numberAssessmentEventDao.create(numberAssessmentEvent); + } + int randomNumberOfNumberLearningEvents = (int) (Math.random() * 10); for (int i = 0; i < randomNumberOfNumberLearningEvents; i++) { NumberLearningEvent numberLearningEvent = new NumberLearningEvent(); diff --git a/src/main/resources/META-INF/jpa-schema-export.sql b/src/main/resources/META-INF/jpa-schema-export.sql index d2ae54608..0a8ff6589 100644 --- a/src/main/resources/META-INF/jpa-schema-export.sql +++ b/src/main/resources/META-INF/jpa-schema-export.sql @@ -393,6 +393,7 @@ numberSymbol varchar(255), numberValue integer, application_id bigint, + number_id bigint, primary key (id) ) type=MyISAM; @@ -887,6 +888,11 @@ foreign key (application_id) references Application (id); + alter table NumberAssessmentEvent + add constraint FK8xswt8aljh7hfnqsg4iyot3gj + foreign key (number_id) + references Number (id); + alter table NumberContributionEvent add constraint FK8tr84kkqmavan1jxfmc1pq8h6 foreign key (contributor_id) diff --git a/src/main/webapp/WEB-INF/jsp/analytics/students/id.jsp b/src/main/webapp/WEB-INF/jsp/analytics/students/id.jsp index 0cb1ad673..b975f9602 100644 --- a/src/main/webapp/WEB-INF/jsp/analytics/students/id.jsp +++ b/src/main/webapp/WEB-INF/jsp/analytics/students/id.jsp @@ -481,8 +481,94 @@
🔢 Numbers
+ + Export to CSVvertical_align_bottom + +
Number assessment events (${fn:length(numberAssessmentEvents)})
- ... + + + + + + + + + + + + + + + + + + + + + + +
timestamppackage_namemastery_scoretime_spent_msnumber_value
+ + + ${numberAssessmentEvent.packageName} + + + + + + + + + + + ms + + ${numberAssessmentEvent.numberValue} +
diff --git a/src/test/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelperTest.java b/src/test/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelperTest.java index 996bb68ab..fea9826f2 100644 --- a/src/test/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelperTest.java +++ b/src/test/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelperTest.java @@ -2,17 +2,21 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.io.IOException; import java.util.List; +import org.json.JSONObject; + import org.junit.jupiter.api.Test; import org.springframework.core.io.ClassRelativeResourceLoader; import org.springframework.core.io.Resource; import org.springframework.core.io.ResourceLoader; import ai.elimu.entity.analytics.LetterSoundLearningEvent; +import ai.elimu.entity.analytics.NumberAssessmentEvent; import ai.elimu.entity.analytics.StoryBookLearningEvent; import ai.elimu.entity.analytics.VideoLearningEvent; import ai.elimu.entity.analytics.WordAssessmentEvent; @@ -89,11 +93,6 @@ public void testExtractLetterSoundLearningEvents_v3005013() throws IOException { assertEquals(1, letterSoundLearningEvent.getLetterSoundId()); } - - // TODO: number assessment events - - // TODO: number learning events - /** * Test extraction of data from CSV files generated by version 3001030 of the Analytics app: @@ -189,6 +188,37 @@ public void testExtractWordLearningEvents_v3005009() throws IOException { assertEquals("ดี", wordLearningEvent.getWordText()); } + + /** + * Test extraction of data from CSV files generated by version 4000028 of the Analytics app: + * https://github.com/elimu-ai/analytics/releases/tag/4.0.28 + */ + @Test + public void testExtractNumberAssessmentEvents_v4000028() throws IOException { + ResourceLoader resourceLoader = new ClassRelativeResourceLoader(CsvAnalyticsExtractionHelper.class); + Resource resource = resourceLoader.getResource("5b7c682a12ecbe2e_4000028_number-assessment-events_2025-07-03.csv"); + File csvFile = resource.getFile(); + + List numberAssessmentEvents = CsvAnalyticsExtractionHelper.extractNumberAssessmentEvents(csvFile); + assertEquals(3, numberAssessmentEvents.size()); + + NumberAssessmentEvent numberAssessmentEvent = numberAssessmentEvents.get(0); + assertEquals(1751536157 * 1_000L, numberAssessmentEvent.getTimestamp().getTimeInMillis()); + assertEquals("5b7c682a12ecbe2e", numberAssessmentEvent.getAndroidId()); + assertEquals("ai.elimu.learndigits.debug", numberAssessmentEvent.getPackageName()); + assertEquals(0.0F, numberAssessmentEvent.getMasteryScore()); + assertEquals(1_782, numberAssessmentEvent.getTimeSpentMs()); + JSONObject additionalData = new JSONObject(numberAssessmentEvent.getAdditionalData()); + assertTrue(additionalData.has("numberSelected")); + assertEquals(6, additionalData.getInt("numberSelected")); + assertEquals(ResearchExperiment.EXP_0_WORD_EMOJIS, numberAssessmentEvent.getResearchExperiment()); + assertEquals(ExperimentGroup.TREATMENT, numberAssessmentEvent.getExperimentGroup()); + assertEquals(9, numberAssessmentEvent.getNumberValue()); + assertNull(numberAssessmentEvent.getNumberId()); + } + + // TODO: number learning events + /** * Test extraction of data from CSV files generated by version 3001030 of the Analytics app: diff --git a/src/test/resources/ai/elimu/util/csv/5b7c682a12ecbe2e_4000028_number-assessment-events_2025-07-03.csv b/src/test/resources/ai/elimu/util/csv/5b7c682a12ecbe2e_4000028_number-assessment-events_2025-07-03.csv new file mode 100644 index 000000000..7ff4663bf --- /dev/null +++ b/src/test/resources/ai/elimu/util/csv/5b7c682a12ecbe2e_4000028_number-assessment-events_2025-07-03.csv @@ -0,0 +1,4 @@ +id,timestamp,package_name,mastery_score,time_spent_ms,additional_data,research_experiment,experiment_group,number_value,number_id +21,1751536157,ai.elimu.learndigits.debug,0.0,1782,"{""numberSelected"":6}",0,1,9, +22,1751536167,ai.elimu.learndigits.debug,0.0,6274,"{""numberSelected"":7}",0,1,0, +23,1751536171,ai.elimu.learndigits.debug,1.0,417,,0,1,2,