@@ -389,9 +389,10 @@ public function lotw_users() {
389389 // Only truncate table AFTER we've validated the remote file
390390 $ this ->db ->query ("TRUNCATE TABLE lotw_users " );
391391
392- $ i = 0 ;
392+ $ i = 0 ; // raw rows read
393393 $ batch_count = 0 ;
394- $ lotwdata = array ();
394+ // Use a map to deduplicate by callsign and keep the latest timestamp
395+ $ lotw_map = array ();
395396 $ batch_size = 500 ; // Smaller batch size for better performance and memory usage
396397
397398 // Skip CSV header row
@@ -403,36 +404,42 @@ public function lotw_users() {
403404 $ callsign = strtoupper ($ data [0 ]);
404405 // Validate callsign format (basic check)
405406 if (preg_match ('/^[A-Z0-9\/]+$/ ' , $ callsign )) {
406- $ lotwdata [] = array (
407- 'callsign ' => $ callsign ,
408- 'lastupload ' => $ data [1 ] . ' ' . $ data [2 ]
409- );
410407 $ i ++;
411-
412- // Insert batch when we reach batch_size
413- if (count ($ lotwdata ) >= $ batch_size ) {
414- if (!$ this ->db ->insert_batch ('lotw_users ' , $ lotwdata )) {
415- echo "FAILED: Database error during batch insert " ;
416- log_message ('error ' , 'Database error during LoTW batch insert ' );
417- fclose ($ handle );
418- return ;
419- }
420- $ batch_count ++;
421- $ lotwdata = array (); // Reset array
408+ // Compose timestamp string and compare; keep the latest per callsign
409+ $ ts = $ data [1 ] . ' ' . $ data [2 ];
410+ // If we haven't seen this callsign, or this row is newer, store it
411+ if (!isset ($ lotw_map [$ callsign ]) || strtotime ($ ts ) > strtotime ($ lotw_map [$ callsign ])) {
412+ $ lotw_map [$ callsign ] = $ ts ;
422413 }
423414 }
424415 }
425416 }
426417 fclose ($ handle );
427418
428- // Insert any remaining records in final batch
429- if (!empty ($ lotwdata )) {
430- if (!$ this ->db ->insert_batch ('lotw_users ' , $ lotwdata )) {
431- echo "FAILED: Database error during final batch insert " ;
432- log_message ('error ' , 'Database error during LoTW final batch insert ' );
433- return ;
419+ // Insert deduplicated records in batches
420+ if (!empty ($ lotw_map )) {
421+ $ lotwdata = array ();
422+ foreach ($ lotw_map as $ cs => $ lu ) {
423+ $ lotwdata [] = array ('callsign ' => $ cs , 'lastupload ' => $ lu );
424+ if (count ($ lotwdata ) >= $ batch_size ) {
425+ if (!$ this ->db ->insert_batch ('lotw_users ' , $ lotwdata )) {
426+ echo "FAILED: Database error during batch insert " ;
427+ log_message ('error ' , 'Database error during LoTW batch insert while inserting deduped map ' );
428+ return ;
429+ }
430+ $ batch_count ++;
431+ $ lotwdata = array ();
432+ }
433+ }
434+ // Final batch
435+ if (!empty ($ lotwdata )) {
436+ if (!$ this ->db ->insert_batch ('lotw_users ' , $ lotwdata )) {
437+ echo "FAILED: Database error during final batch insert " ;
438+ log_message ('error ' , 'Database error during LoTW final batch insert while inserting deduped map ' );
439+ return ;
440+ }
441+ $ batch_count ++;
434442 }
435- $ batch_count ++;
436443 }
437444
438445 // Verify we actually imported data
@@ -454,7 +461,8 @@ public function lotw_users() {
454461 $ endtime = $ mtime ;
455462 $ totaltime = ($ endtime - $ starttime );
456463 echo "This page was created in " .$ totaltime ." seconds <br /> " ;
457- echo "Records inserted: " . $ i . " <br/> " ;
464+ echo "Records read: " . $ i . " <br/> " ;
465+ echo "Unique callsigns inserted: " . $ final_count . " <br/> " ;
458466 }
459467
460468 public function lotw_check () {
0 commit comments