fix: declare synchronized entity classes on native writes DHIS2-21963 (#24803) (#24845)

* fix: declare synchronized entity classes on native writes

Native queries that run executeUpdate() without declaring which tables
they touch make Hibernate conservatively evict EVERY L2 entity region,
not just the affected one.

NativeSQLQueryPlan.performExecuteUpdate calls coordinateSharedCacheCleanup
with getCustomQuery().getQuerySpaces(). SQLCustomQuery leaves that set
empty unless addSynchronizedEntityClass/QuerySpace was called, and
BulkOperationCleanupAction.affectedEntity treats an empty set as "affects
everything", so every persister that can write to cache gets an
EntityCleanup. EntityCleanup.release() calls unlockRegion(), which is
evictAll() -> evictData() on the whole region.

Note AbstractReadWriteAccess.removeAll IS a no-op under read-write, which
is why this is easy to miss; the eviction comes from unlockRegion, not
removeAll. Both stores here use stateless sessions, where
isEventSource() is false, so the cleanup runs immediately rather than
being deferred to the ActionQueue.

Draft to confirm the mechanism: measured on a Sierra Leone instance, a
single data value write with a new period emptied the whole DataElement
region (0 hits / 3037 logical misses on the next read), while the same
write with an existing period left it fully warm. No eviction or removal
counter moves and nothing is logged, so this is invisible without
instrumenting Hibernate.

HousekeepingJob runs every 20s by default and drives the jobconfiguration
writes here, so on a stock instance every cached entity region is dropped
on that cadence.

* fix: declare query spaces on the remaining native writes

* fix: synchronize entities not join tables so cached collections are evicted

* docs: explain query spaces in nativeSynchronizedQuery javadoc

* refactor: add stateless session variant of nativeSynchronizedQuery

* refactor: use nativeSynchronizedQuery in complete data set registration store

* refactor: run audit trigger DDL via jdbcTemplate

* fix: declare element entities for native writes to collection tables

A write to a collection table needs the collection's element entity
declared, not the owning entity. An entity's query spaces are its own
table(s) only (SingleTableEntityPersister), never the tables of the
collections it owns, and BulkOperationCleanupAction resolves collection
regions via getCollectionRolesByEntityParticipant, which is keyed by the
collection's element entity.

So synchronizing on the owner leaves its cached collection stale:

* categories_categoryoptions needs CategoryOption, not Category
* users_catdimensionconstraints needs Category, not User

Confirmed against a running instance. With a category collection warm and
in steady state (+1 hit, 0 misses per read), a merge of an unrelated pair
of categories left a bystander category's cached collection reloading cold
(+2 misses, +1 put), while an unrelated write left it fully warm.

Also correct the comments in the category combo and data set stores, which
reached the right regions but described the wrong reason, and expand the
nativeSynchronizedQuery javadoc with the element-entity rule.

Author: teleivo

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/analytics/hibernate/HibernateCategoryDimensionStore.java

@@ -79,8 +79,7 @@ public int updateCatDimensionCategoryRefs(
         set categoryid = :targetCategoryId
         where cd.categoryid in :sourceCategoryIds
         """;
-    return getSession()
-        .createNativeQuery(sql)
+    return nativeSynchronizedQuery(sql)
         .setParameter("targetCategoryId", targetCategoryId)
         .setParameter("sourceCategoryIds", sourceCategoryIds)
         .setLockOptions(new LockOptions(PESSIMISTIC_WRITE).setTimeOut(5000))

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/category/hibernate/HibernateCategoryComboStore.java

@@ -37,6 +37,7 @@
 import java.util.List;
 import java.util.Set;
 import org.hibernate.LockOptions;
+import org.hisp.dhis.category.Category;
 import org.hisp.dhis.category.CategoryCombo;
 import org.hisp.dhis.category.CategoryComboStore;
 import org.hisp.dhis.category.CategoryOptionCombo;
@@ -97,8 +98,11 @@ public int updateCatComboCategoryRefs(Set<Long> sourceCategoryIds, long targetCa
         set categoryid = :targetCategoryId
         where cc_c.categoryid in :sourceCategoryIds
         """;
-    return getSession()
-        .createNativeQuery(sql)
+    return nativeSynchronizedQuery(sql)
+        // categorycombos_categories backs both the cached CategoryCombo.categories and
+        // Category.categoryCombos collections. Collection regions are keyed by the collection's
+        // element entity, so naming both entities is what reaches both regions.
+        .addSynchronizedEntityClass(Category.class)
         .setParameter("targetCategoryId", targetCategoryId)
         .setParameter("sourceCategoryIds", sourceCategoryIds)
         .setLockOptions(new LockOptions(PESSIMISTIC_WRITE).setTimeOut(5000))

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/category/hibernate/HibernateCategoryStore.java

@@ -39,6 +39,7 @@
 import javax.annotation.Nonnull;
 import org.hibernate.LockOptions;
 import org.hisp.dhis.category.Category;
+import org.hisp.dhis.category.CategoryOption;
 import org.hisp.dhis.category.CategoryStore;
 import org.hisp.dhis.common.DataDimensionType;
 import org.hisp.dhis.common.hibernate.HibernateIdentifiableObjectStore;
@@ -118,8 +119,11 @@ public int removeCatOptionCategoryRefs(Set<Long> sourceCategoryIds) {
               delete from categories_categoryoptions c_co
               where c_co.categoryid in :sourceCategoryIds
               """;
-    return getSession()
-        .createNativeQuery(sql)
+    return nativeSynchronizedQuery(sql)
+        // categories_categoryoptions is the Category.categoryOptions collection table. Collection
+        // regions are keyed by the collection's element entity, so CategoryOption is what reaches
+        // that region, not the owning Category.
+        .addSynchronizedEntityClass(CategoryOption.class)
         .setParameter("sourceCategoryIds", sourceCategoryIds)
         .setLockOptions(new LockOptions(PESSIMISTIC_WRITE).setTimeOut(5000))
         .executeUpdate();

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataapproval/hibernate/HibernateDataApprovalWorkflowStore.java

@@ -66,8 +66,7 @@ public int updateCategoryComboRefs(Set<Long> sourceCategoryComboIds, long target
         set categorycomboid = :targetCategoryComboId
         where categorycomboid in :sourceCategoryComboIds
         """;
-    return getSession()
-        .createNativeQuery(sql)
+    return nativeSynchronizedQuery(sql)
         .setParameter("targetCategoryComboId", targetCategoryComboId)
         .setParameter("sourceCategoryComboIds", sourceCategoryComboIds)
         .setLockOptions(new LockOptions(PESSIMISTIC_WRITE).setTimeOut(5000))

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataset/hibernate/HibernateCompleteDataSetRegistrationStore.java

@@ -135,8 +135,7 @@ public void deleteCompleteDataSetRegistration(
           AND c.sourceid = (SELECT organisationunitid FROM organisationunit ou WHERE ou.uid = :ou)
           AND c.attributeoptioncomboid = (SELECT categoryoptioncomboid FROM categoryoptioncombo aoc WHERE aoc.uid = :aoc)
         """;
-    entityManager
-        .createNativeQuery(sql)
+    nativeSynchronizedQuery(sql)
         .setParameter("ds", dataSet.getValue())
         .setParameter("pe", period.getIsoDate())
         .setParameter("ou", orgUnit.getValue())

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataset/hibernate/HibernateDataSetStore.java

@@ -171,8 +171,11 @@ public int updateDataSetElementCategoryComboRefs(
         set categorycomboid = :targetCategoryComboId
         where categorycomboid in :sourceCategoryComboIds
         """;
-    return getSession()
-        .createNativeQuery(sql)
+    return nativeSynchronizedQuery(sql)
+        // datasetelement is DataSetElement's own table. It is also the DataSet.dataSetElements
+        // collection table, and since collection regions are keyed by the collection's element
+        // entity, naming DataSetElement reaches that cached collection too.
+        .addSynchronizedEntityClass(DataSetElement.class)
         .setParameter("targetCategoryComboId", targetCategoryComboId)
         .setParameter("sourceCategoryComboIds", sourceCategoryComboIds)
         .setLockOptions(new LockOptions(PESSIMISTIC_WRITE).setTimeOut(5000))

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/datavalue/hibernate/HibernateDataValueChangelogStore.java

@@ -73,7 +73,7 @@ public void deleteByOrgUnit(@Nonnull UID orgUnit) {
       DELETE FROM datavalueaudit dva
       WHERE dva.organisationunitid = (SELECT ou.organisationunitid FROM organisationunit ou WHERE ou.uid = :ou)""";
 
-    entityManager.createNativeQuery(sql).setParameter("ou", orgUnit.getValue()).executeUpdate();
+    nativeSynchronizedQuery(sql).setParameter("ou", orgUnit.getValue()).executeUpdate();
   }
 
   @Override
@@ -83,7 +83,7 @@ public void deleteByDataElement(@Nonnull UID dataElement) {
       DELETE FROM datavalueaudit dva
       WHERE dva.dataelementid = (SELECT de.dataelementid FROM dataelement de WHERE de.uid = :de)""";
 
-    entityManager.createNativeQuery(sql).setParameter("de", dataElement.getValue()).executeUpdate();
+    nativeSynchronizedQuery(sql).setParameter("de", dataElement.getValue()).executeUpdate();
   }
 
   @Override
@@ -93,8 +93,7 @@ public void deleteByOptionCombo(@Nonnull UID categoryOptionCombo) {
         DELETE FROM datavalueaudit dva
         WHERE dva.categoryoptioncomboid = (SELECT categoryoptioncomboid FROM categoryoptioncombo WHERE uid = :coc)
            OR dva.attributeoptioncomboid = (SELECT categoryoptioncomboid FROM categoryoptioncombo WHERE uid = :coc);""";
-    entityManager
-        .createNativeQuery(sql)
+    nativeSynchronizedQuery(sql)
         .setParameter("coc", categoryOptionCombo.getValue())
         .executeUpdate();
   }
@@ -258,14 +257,16 @@ IF NOT EXISTS (
           END IF;
       END;
       $$""";
-    getSession().createNativeQuery(sql).executeUpdate();
+    // DDL, not entity persistence, so it skips Hibernate and its cache invalidation
+    jdbcTemplate.execute(sql);
   }
 
   @Override
   public void disableAudit() {
     String sql =
         """
       DROP TRIGGER IF EXISTS trg_datavalue_audit ON datavalue""";
-    getSession().createNativeQuery(sql).executeUpdate();
+    // DDL, not entity persistence, so it skips Hibernate and its cache invalidation
+    jdbcTemplate.execute(sql);
   }
 }

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/datavalue/hibernate/HibernateDataValueTrimStore.java

@@ -87,8 +87,7 @@ public int updateDeletedIfNotZeroIsSignificant() {
           AND de.zeroissignificant = false
           AND dv.dataelementid = de.dataelementid
           AND (dv.value IS NULL OR dv.value = '')""";
-    return getSession()
-        .createNativeQuery(sql)
+    return nativeSynchronizedQuery(sql)
         .setLockOptions(new LockOptions(PESSIMISTIC_WRITE).setTimeOut(1000))
         .executeUpdate();
   }

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/period/hibernate/HibernatePeriodStore.java

@@ -124,8 +124,7 @@ INSERT INTO period (periodid, periodtypeid, startdate, enddate, iso)
               if (pk != null) {
                 return pk;
               }
-              session
-                  .createNativeQuery(sql2)
+              nativeSynchronizedQuery(session, sql2)
                   .setParameter("type", period.getPeriodType().getName())
                   .setParameter("start", period.getStartDate())
                   .setParameter("end", period.getEndDate())
@@ -144,8 +143,7 @@ INSERT INTO period (periodid, periodtypeid, startdate, enddate, iso)
   public void delete(@Nonnull Period period) {
     String isoDate = period.getIsoDate();
     int deleted =
-        getSession()
-            .createNativeQuery("DELETE FROM period where iso = :iso")
+        nativeSynchronizedQuery("DELETE FROM period where iso = :iso")
             .setParameter("iso", isoDate)
             .executeUpdate();
     if (deleted > 0) {
@@ -261,7 +259,12 @@ INSERT INTO periodtype (periodtypeid, name)
               if (pk != null) {
                 return pk;
               }
-              session.createNativeQuery(sql2).setParameter("name", name).executeUpdate();
+              session
+                  .createNativeQuery(sql2)
+                  // PeriodType, not the store's own Period
+                  .addSynchronizedEntityClass(PeriodType.class)
+                  .setParameter("name", name)
+                  .executeUpdate();
               return session.createNativeQuery("SELECT lastval()").uniqueResult();
             });
     if (id instanceof Number n) {
@@ -286,6 +289,7 @@ public void updatePeriodType(PeriodType periodType) {
         session ->
             session
                 .createNativeQuery(sql)
+                .addSynchronizedEntityClass(PeriodType.class)
                 .setParameter("name", name)
                 .setParameter("label", label)
                 .executeUpdate());

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/hibernate/HibernateProgramIndicatorStore.java

@@ -118,8 +118,7 @@ WHEN attributecomboid IN (:sourceCategoryComboIds)
             categorycomboid IN (:sourceCategoryComboIds)
             OR attributecomboid IN (:sourceCategoryComboIds);
         """;
-    return getSession()
-        .createNativeQuery(sql)
+    return nativeSynchronizedQuery(sql)
         .setParameter("targetCategoryComboId", targetCategoryComboId)
         .setParameter("sourceCategoryComboIds", sourceCategoryComboIds)
         .setLockOptions(new LockOptions(PESSIMISTIC_WRITE).setTimeOut(5000))