Skip to content

Commit 9902717

Browse files
nickgrosnickgros
authored
Merge pull request #5410 from marcomarasca/PLFM-9187 
2 parents 81f474b + a6f1da5 commit 9902717

File tree

20 files changed

+876
-46
lines changed

20 files changed

+876
-46
lines changed

‎lib/jdomodels/src/main/java/org/sagebionetworks/repo/model/dbo/asynch/AsynchJobType.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import org.sagebionetworks.repo.model.doi.v2.DoiResponse;
1010
import org.sagebionetworks.repo.model.download.AddToDownloadListRequest;
1111
import org.sagebionetworks.repo.model.download.AddToDownloadListResponse;
12+
import org.sagebionetworks.repo.model.download.AddToDownloadListStatsRequest;
13+
import org.sagebionetworks.repo.model.download.AddToDownloadListStatsResponse;
1214
import org.sagebionetworks.repo.model.download.DownloadListManifestRequest;
1315
import org.sagebionetworks.repo.model.download.DownloadListManifestResponse;
1416
import org.sagebionetworks.repo.model.download.DownloadListPackageRequest;
@@ -90,6 +92,8 @@ public enum AsynchJobType {
9092

9193
ADD_TO_DOWNLOAD_LIST(AddToDownloadListRequest.class, AddToDownloadListResponse.class),
9294

95+
ADD_TO_DOWNLOAD_LIST_STATS(AddToDownloadListStatsRequest.class, AddToDownloadListStatsResponse.class),
96+
9397
DOWNLOAD_LIST_PACKAGE(DownloadListPackageRequest.class, DownloadListPackageResponse.class),
9498

9599
DOWNLOAD_LIST_MANIFEST(DownloadListManifestRequest.class, DownloadListManifestResponse.class),

‎lib/jdomodels/src/main/java/org/sagebionetworks/repo/model/dbo/dao/NodeDAOImpl.java‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
import java.util.LinkedList;
7070
import java.util.List;
7171
import java.util.Map;
72+
import java.util.Objects;
7273
import java.util.Optional;
7374
import java.util.Set;
7475
import java.util.UUID;
@@ -450,7 +451,7 @@ public class NodeDAOImpl implements NodeDAO, InitializingBean {
450451
private static final String UPDATE_REVISION_FILE_HANDLE = "UPDATE " + TABLE_REVISION + " SET " + COL_REVISION_FILE_HANDLE_ID
451452
+ " = ? WHERE " + COL_REVISION_OWNER_NODE + " = ? AND " + COL_REVISION_NUMBER + " = ?";
452453

453-
private static final String SELECT_FILE_SUMMARY_FOR_ID_AND_VERSION = "SELECT COUNT(*) AS COUNT, " +
454+
private static final String SELECT_FILE_SUMMARY = "SELECT COUNT(*) AS COUNT, " +
454455
"MD5(GROUP_CONCAT(F." + COL_FILES_CONTENT_MD5 + " ORDER BY F." + COL_FILES_CONTENT_MD5 + " ASC SEPARATOR '')) AS CHECKSUM, " +
455456
"SUM(F." + COL_FILES_CONTENT_SIZE + ") AS SIZE " +
456457
" FROM " + TABLE_REVISION + " R JOIN " + TABLE_FILES + " F ON R." + COL_REVISION_FILE_HANDLE_ID + " = F." + COL_FILES_ID +
@@ -459,7 +460,7 @@ public class NodeDAOImpl implements NodeDAO, InitializingBean {
459460
private static final RowMapper<FileSummary> FILE_SUMMARY_ROW_MAPPER = (rs, rowNum) -> {
460461
String checksum = rs.getString("CHECKSUM");
461462
long size = rs.getLong("SIZE");
462-
int count = rs.getInt("COUNT");
463+
long count = rs.getLong("COUNT");
463464
return new FileSummary(checksum, size, count);
464465
};
465466

@@ -2309,33 +2310,31 @@ public boolean isSearchEnabled(Long nodeId, Long versionNumber) {
23092310
@Override
23102311
@WriteTransaction
23112312
public FileSummary getFileSummary(List<EntityRef> entityRefs) {
2312-
List<Long[]> specificIdVersionPairs = new ArrayList<>(entityRefs.size());
2313-
for (EntityRef ref : entityRefs) {
2314-
if (ref.getEntityId() != null) {
2315-
Long entityId = KeyFactory.stringToKey(ref.getEntityId());
2316-
specificIdVersionPairs.add(new Long[]{entityId, ref.getVersionNumber()});
2317-
}
2318-
}
2319-
2320-
if (specificIdVersionPairs.isEmpty()) {
2321-
return new FileSummary(null, 0, 0);
2313+
List<Long[]> idAndVersionPairs = entityRefs.stream()
2314+
.filter(Objects::nonNull)
2315+
.filter(ref -> ref.getEntityId() != null)
2316+
.map(ref -> new Long[]{ KeyFactory.stringToKey(ref.getEntityId()), ref.getVersionNumber() })
2317+
.collect(Collectors.toList());
2318+
2319+
if (idAndVersionPairs.isEmpty()) {
2320+
return new FileSummary(0, 0);
23222321
}
23232322

2324-
Map<String, List<Long[]>> namedParameters = Collections.singletonMap("pairs", specificIdVersionPairs);
2323+
Map<String, List<Long[]>> namedParameters = Collections.singletonMap("pairs", idAndVersionPairs);
23252324

23262325
Long currentGroupConcatMax = jdbcTemplate.queryForObject("SHOW SESSION VARIABLES LIKE ?", (rs, i) -> rs.getLong("Value"), "group_concat_max_len");
23272326

23282327
try {
23292328
// We temporarily increase the group_concat length to allow computing the correct MD5 with bigger lists
23302329
jdbcTemplate.execute("SET SESSION group_concat_max_len=" + FILE_SUMMARY_GROUP_CONCAT_LENGTH);
23312330

2332-
return namedParameterJdbcTemplate.queryForObject(SELECT_FILE_SUMMARY_FOR_ID_AND_VERSION, namedParameters, FILE_SUMMARY_ROW_MAPPER);
2331+
return namedParameterJdbcTemplate.queryForObject(SELECT_FILE_SUMMARY, namedParameters, FILE_SUMMARY_ROW_MAPPER);
23332332
} finally {
23342333
// Restores the group_concat length for the connection/session
23352334
jdbcTemplate.execute("SET SESSION group_concat_max_len=" + currentGroupConcatMax);
23362335
}
23372336
}
2338-
2337+
23392338
@Override
23402339
public Optional<String> getDefiningSql(IdAndVersion id) {
23412340
return selectRevisionColumnValue(id, COL_REVISION_DEFINING_SQL, String.class);

‎lib/jdomodels/src/main/java/org/sagebionetworks/repo/model/dbo/file/download/v2/DownloadListDAO.java‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import org.json.JSONObject;
66
import org.sagebionetworks.repo.model.EntityRef;
77
import org.sagebionetworks.repo.model.download.ActionRequiredCount;
8+
import org.sagebionetworks.repo.model.download.AddToDownloadListStatsResponse;
89
import org.sagebionetworks.repo.model.download.AvailableFilter;
910
import org.sagebionetworks.repo.model.download.DownloadListItem;
1011
import org.sagebionetworks.repo.model.download.DownloadListItemResult;
@@ -143,6 +144,12 @@ List<ActionRequiredCount> getActionsRequiredFromDownloadList(EntityActionRequire
143144
*/
144145
Long addChildrenToDownloadList(Long userId, Long parentId, boolean useVersion, long limit);
145146

147+
/**
148+
* @param parentId
149+
* @return The count and size of files that are children of the container with the given parentId
150+
*/
151+
AddToDownloadListStatsResponse getAddChildrenToDownloadListStats(Long parentId);
152+
146153
/**
147154
* Add all the files in the tree rooted in the given parentId to the user's download list.
148155
* @param userId
@@ -153,6 +160,14 @@ List<ActionRequiredCount> getActionsRequiredFromDownloadList(EntityActionRequire
153160
* @return
154161
*/
155162
Long addDescendantsToDownloadList(Long userId, Long parentId, boolean useVersion, long limit);
163+
164+
/**
165+
* @param parentId
166+
* @param maxContainers The max number of containers to use in the computation, if the container size is exceeded will
167+
* set the {@link AddToDownloadListStatsResponse#getIsFileCountAndSizeEstimate()} to true.
168+
* @return The count and size of files contained in the given parent container
169+
*/
170+
AddToDownloadListStatsResponse getAddDescendantsToDownloadListStats(Long parentId, int maxContainers);
156171

157172
/**
158173
* For the given item load all of the details needed to write to a manifest
@@ -161,7 +176,6 @@ List<ActionRequiredCount> getActionsRequiredFromDownloadList(EntityActionRequire
161176
*/
162177
JSONObject getItemManifestDetails(DownloadListItem item);
163178

164-
165179
/**
166180
* Adds all of the files referenced in the given list of {@link EntityRef} to the user's download list.
167181
*
@@ -172,13 +186,26 @@ List<ActionRequiredCount> getActionsRequiredFromDownloadList(EntityActionRequire
172186
*/
173187
Long addFileEntityRefToDownloadList(Long userId, List<EntityRef> fileRefs, long limit);
174188

189+
/**
190+
* @param fileRefs
191+
* @return The total count and size of files referenced by the given list of {@link EntityRef}
192+
*/
193+
AddToDownloadListStatsResponse getAddFileEntityRefToDownloadListStats(List<EntityRef> fileRefs);
194+
175195
/**
176196
* Add all the files that are reference by each of the data set referenced in the given list of {@link EntityRef} to the user's download list.
197+
*
177198
* @param userId
178199
* @param datasetRefs
179200
* @param limit Limit the number of files that can be added.
180201
* @return
181202
*/
182203
Long addDatasetEntityRefFilesToDownloadList(Long userId, List<EntityRef> datasetRefs, long limit);
183204

205+
/**
206+
* @param datasets
207+
* @return The total count and size of files referenced by each dataset in the given list of {@link EntityRef}
208+
*/
209+
AddToDownloadListStatsResponse getAddDatasetEntityRefFilesToDownloadListStats(List<EntityRef> datasetRefs);
210+
184211
}

‎lib/jdomodels/src/main/java/org/sagebionetworks/repo/model/dbo/file/download/v2/DownloadListDAOImpl.java‎

Lines changed: 97 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717
import static org.sagebionetworks.repo.model.query.jdo.SqlConstants.COL_NODE_PARENT_ID;
1818
import static org.sagebionetworks.repo.model.query.jdo.SqlConstants.COL_NODE_TYPE;
1919
import static org.sagebionetworks.repo.model.query.jdo.SqlConstants.COL_REVISION_FILE_HANDLE_ID;
20-
import static org.sagebionetworks.repo.model.query.jdo.SqlConstants.COL_REVISION_NUMBER;
2120
import static org.sagebionetworks.repo.model.query.jdo.SqlConstants.COL_REVISION_ITEMS;
21+
import static org.sagebionetworks.repo.model.query.jdo.SqlConstants.COL_REVISION_NUMBER;
2222
import static org.sagebionetworks.repo.model.query.jdo.SqlConstants.COL_REVISION_OWNER_NODE;
2323
import static org.sagebionetworks.repo.model.query.jdo.SqlConstants.COL_REVISION_USER_ANNOS_JSON;
2424
import static org.sagebionetworks.repo.model.query.jdo.SqlConstants.TABLE_DERIVED_ANNOTATIONS;
@@ -52,6 +52,7 @@
5252
import org.sagebionetworks.repo.model.dao.FileHandleMetadataType;
5353
import org.sagebionetworks.repo.model.dbo.DDLUtilsImpl;
5454
import org.sagebionetworks.repo.model.download.ActionRequiredCount;
55+
import org.sagebionetworks.repo.model.download.AddToDownloadListStatsResponse;
5556
import org.sagebionetworks.repo.model.download.AvailableFilter;
5657
import org.sagebionetworks.repo.model.download.DownloadListItem;
5758
import org.sagebionetworks.repo.model.download.DownloadListItemResult;
@@ -95,6 +96,9 @@ public class DownloadListDAOImpl implements DownloadListDAO {
9596

9697
public static final String TEMP_ACTION_REQUIRED_TEMPLATE = DDLUtilsImpl
9798
.loadSQLFromClasspath("sql/TempActionRequired-ddl.sql");
99+
100+
public static final String GET_DATASET_COLLECTION_FILE_STATS = DDLUtilsImpl
101+
.loadSQLFromClasspath("sql/GetDatasetFileStats.sql");
98102

99103
private static final int BATCH_SIZE = 10000;
100104

@@ -647,11 +651,33 @@ public Long addChildrenToDownloadList(Long userId, Long parentId, boolean useVer
647651
return (long) jdbcTemplate.update(sql, userId, parentId, limit);
648652
}
649653

654+
@Override
655+
public AddToDownloadListStatsResponse getAddChildrenToDownloadListStats(Long parentId) {
656+
List<String> fileTypeNames = EntityTypeUtils.getFileTypes().stream().map(EntityType::name).collect(Collectors.toList());
657+
658+
String sql = "SELECT COUNT(N." + COL_NODE_ID + ") AS count, COALESCE(SUM(F." + COL_FILES_CONTENT_SIZE + "), 0) as size "
659+
+ "FROM " + TABLE_NODE + " N "
660+
+ "JOIN " + TABLE_REVISION + " R ON (N." + COL_NODE_ID + "=R." + COL_REVISION_OWNER_NODE + " AND N." + COL_NODE_CURRENT_REV + "=R." + COL_REVISION_NUMBER + ") "
661+
+ "JOIN " + TABLE_FILES + " F ON (R." + COL_REVISION_FILE_HANDLE_ID + "=F." + COL_FILES_ID + ")"
662+
+ "WHERE N." + COL_NODE_PARENT_ID + " = ? AND N."
663+
+ COL_NODE_TYPE + " IN ('" + String.join("','", fileTypeNames) + "')";
664+
665+
return jdbcTemplate.queryForObject(sql, (rs, i) ->
666+
new AddToDownloadListStatsResponse()
667+
.setFileCount(rs.getLong("count"))
668+
.setFileSize(rs.getLong("size"))
669+
.setIsFileCountAndSizeEstimate(false)
670+
, parentId);
671+
672+
}
673+
650674
@WriteTransaction
651675
@Override
652676
public Long addDescendantsToDownloadList(Long userId, Long parentId, boolean useVersion, long limit) {
653677
String versionString = useVersion ? "N." + COL_NODE_CURRENT_REV : "-1";
654678

679+
List<String> fileTypeNames = EntityTypeUtils.getFileTypes().stream().map(EntityType::name).collect(Collectors.toList());
680+
655681
String sql = String.format("INSERT IGNORE INTO " + TABLE_DOWNLOAD_LIST_ITEM_V2 + " ("
656682
+ COL_DOWNLOAD_LIST_ITEM_V2_PRINCIPAL_ID + ","
657683
+ COL_DOWNLOAD_LIST_ITEM_V2_ENTITY_ID + ","
@@ -662,13 +688,37 @@ public Long addDescendantsToDownloadList(Long userId, Long parentId, boolean use
662688
+ " UNION DISTINCT"
663689
+ " SELECT N." + COL_NODE_ID + ", CONTAINERS.DIST + 1 FROM CONTAINERS JOIN " + TABLE_NODE + " AS N ON (CONTAINERS.ID = N." + COL_NODE_PARENT_ID + " AND N."+COL_NODE_TYPE + " IN ('project', 'folder'))"
664690
+ ") "
665-
+ "SELECT ?, N." + COL_NODE_ID + ", %s, NOW(3) FROM " + TABLE_NODE + " N JOIN CONTAINERS ON (N." + COL_NODE_PARENT_ID + " = CONTAINERS.ID AND N." + COL_NODE_TYPE + "= 'file') "
691+
+ "SELECT ?, N." + COL_NODE_ID + ", %s, NOW(3) FROM " + TABLE_NODE + " N JOIN CONTAINERS ON (N." + COL_NODE_PARENT_ID + " = CONTAINERS.ID AND N." + COL_NODE_TYPE + " IN ('" + String.join("','", fileTypeNames) + "')) "
666692
+ "ORDER BY CONTAINERS.DIST, CONTAINERS.ID, N." + COL_NODE_ID + " LIMIT ?", versionString);
667693

668694
createOrUpdateDownloadList(userId);
669695

670696
return (long) jdbcTemplate.update(sql, parentId, userId, limit);
671697
}
698+
699+
@Override
700+
public AddToDownloadListStatsResponse getAddDescendantsToDownloadListStats(Long parentId, int maxContainers) {
701+
702+
List<String> fileTypeNames = EntityTypeUtils.getFileTypes().stream().map(EntityType::name).collect(Collectors.toList());
703+
704+
String sql = "WITH RECURSIVE CONTAINERS (ID, DIST) AS ("
705+
+ " SELECT " + COL_NODE_ID + ", 0 FROM " + TABLE_NODE + " WHERE " + COL_NODE_ID + "=? AND " + COL_NODE_TYPE + " IN ('project', 'folder')"
706+
+ " UNION ALL"
707+
+ " SELECT N." + COL_NODE_ID + ", CONTAINERS.DIST + 1 FROM CONTAINERS JOIN " + TABLE_NODE + " AS N ON (CONTAINERS.ID = N." + COL_NODE_PARENT_ID + " AND N."+COL_NODE_TYPE + " IN ('project', 'folder'))"
708+
// We limit the number of containers that we do the computation against to avoid overloading the database
709+
+ " LIMIT ?)"
710+
+ "SELECT COUNT(N." + COL_NODE_ID + ") AS count, COALESCE(SUM(F." + COL_FILES_CONTENT_SIZE + "), 0) as size, (SELECT COUNT(ID) FROM CONTAINERS) >= ? as isEstimate "
711+
+ "FROM CONTAINERS "
712+
+ "JOIN " + TABLE_NODE + " N ON (N." + COL_NODE_PARENT_ID + " = CONTAINERS.ID AND N." + COL_NODE_TYPE + " IN ('" + String.join("','", fileTypeNames) + "')) "
713+
+ "JOIN " + TABLE_REVISION + " R ON (N." + COL_NODE_ID + "=R." + COL_REVISION_OWNER_NODE + " AND N." + COL_NODE_CURRENT_REV + "=R." + COL_REVISION_NUMBER + ") "
714+
+ "JOIN " + TABLE_FILES + " F ON (R." + COL_REVISION_FILE_HANDLE_ID + "=F." + COL_FILES_ID + ")";
715+
716+
return jdbcTemplate.queryForObject(sql, (rs, i) -> new AddToDownloadListStatsResponse()
717+
.setFileCount(rs.getLong("count"))
718+
.setFileSize(rs.getLong("size"))
719+
.setIsFileCountAndSizeEstimate(rs.getBoolean("isEstimate"))
720+
, parentId, maxContainers, maxContainers);
721+
}
672722

673723
@Override
674724
public JSONObject getItemManifestDetails(DownloadListItem item) {
@@ -721,6 +771,29 @@ public int getBatchSize() {
721771
return (long)IntStream.of(updates).sum();
722772
}
723773

774+
@Override
775+
public AddToDownloadListStatsResponse getAddFileEntityRefToDownloadListStats(List<EntityRef> fileRefs) {
776+
String sql = "SELECT COUNT(R." + COL_REVISION_OWNER_NODE + ") AS count, SUM(F." + COL_FILES_CONTENT_SIZE + ") as size "
777+
+ "FROM " + TABLE_REVISION + " R "
778+
+ "JOIN " + TABLE_FILES + " F ON (R." + COL_REVISION_FILE_HANDLE_ID + "=F." + COL_FILES_ID + ") "
779+
+ "WHERE (R." + COL_REVISION_OWNER_NODE + ", R." + COL_REVISION_NUMBER + ") IN (:fileRefs)";
780+
781+
List<Long[]> fileRefsParam = fileRefs.stream().map( fileRef -> new Long[] {
782+
KeyFactory.stringToKey(fileRef.getEntityId()), fileRef.getVersionNumber()
783+
}).collect(Collectors.toList());
784+
785+
Map<String, ?> params = Map.of(
786+
"fileRefs", fileRefsParam
787+
);
788+
789+
return namedJdbcTemplate.queryForObject(sql, params, (rs, i) ->
790+
new AddToDownloadListStatsResponse()
791+
.setFileCount(rs.getLong("count"))
792+
.setFileSize(rs.getLong("size"))
793+
.setIsFileCountAndSizeEstimate(false)
794+
);
795+
}
796+
724797
@Override
725798
@WriteTransaction
726799
public Long addDatasetEntityRefFilesToDownloadList(Long userId, List<EntityRef> datasetRefs, long limit) {
@@ -762,4 +835,26 @@ public Long addDatasetEntityRefFilesToDownloadList(Long userId, List<EntityRef>
762835
return (long) namedJdbcTemplate.update(sql, params);
763836
}
764837

838+
@Override
839+
public AddToDownloadListStatsResponse getAddDatasetEntityRefFilesToDownloadListStats(List<EntityRef> datasetRefs) {
840+
841+
List<Long[]> datasetParam = datasetRefs
842+
.stream()
843+
.map( dataset -> new Long[] {
844+
KeyFactory.stringToKey(dataset.getEntityId()), dataset.getVersionNumber()
845+
})
846+
.collect(Collectors.toList());
847+
848+
Map<String, ?> params = Map.of(
849+
"datasetRefs", datasetParam
850+
);
851+
852+
return namedJdbcTemplate.queryForObject(GET_DATASET_COLLECTION_FILE_STATS, params, (rs, i) ->
853+
new AddToDownloadListStatsResponse()
854+
.setFileCount(rs.getLong("count"))
855+
.setFileSize(rs.getLong("size"))
856+
.setIsFileCountAndSizeEstimate(false)
857+
);
858+
}
859+
765860
}

‎lib/jdomodels/src/main/resources/schema/AsynchJobStatus-ddl.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ CREATE TABLE IF NOT EXISTS `ASYNCH_JOB_STATUS` (
22
`JOB_ID` BIGINT NOT NULL,
33
`ETAG` VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL,
44
`JOB_STATE` ENUM('PROCESSING','FAILED','COMPLETE') NOT NULL,
5-
`JOB_TYPE` ENUM('TABLE_UPDATE_TRANSACTION','UPLOAD_CSV_TO_TABLE_PREVIEW','DOWNLOAD_CSV_FROM_TABLE','QUERY','QUERY_NEXT_PAGE','BULK_FILE_DOWNLOAD','MIGRATION','DOI','ADD_FILES_TO_DOWNLOAD_LIST','STORAGE_REPORT','JSON_SCHEMA_CREATE','VIEW_COLUMN_MODEL_REQUEST','GET_VALIDATION_SCHEMA','QUERY_DOWNLOAD_LIST', 'ADD_TO_DOWNLOAD_LIST', 'DOWNLOAD_LIST_PACKAGE', 'DOWNLOAD_LIST_MANIFEST', 'FILE_HANDLE_ARCHIVAL_REQUEST', 'FILE_HANDLE_RESTORE_REQUEST', 'AGENT_CHAT', 'QUERY_AS_PFB', 'GRID_CREATE', 'DOWNLOAD_CSV_FROM_GRID', 'GRID_EXPORT_RECORDSET', 'GRID_IMPORT_CSV') NOT NULL,
5+
`JOB_TYPE` ENUM('TABLE_UPDATE_TRANSACTION','UPLOAD_CSV_TO_TABLE_PREVIEW','DOWNLOAD_CSV_FROM_TABLE','QUERY','QUERY_NEXT_PAGE','BULK_FILE_DOWNLOAD','MIGRATION','DOI','ADD_FILES_TO_DOWNLOAD_LIST','STORAGE_REPORT','JSON_SCHEMA_CREATE','VIEW_COLUMN_MODEL_REQUEST','GET_VALIDATION_SCHEMA','QUERY_DOWNLOAD_LIST', 'ADD_TO_DOWNLOAD_LIST', 'ADD_TO_DOWNLOAD_LIST_STATS', 'DOWNLOAD_LIST_PACKAGE', 'DOWNLOAD_LIST_MANIFEST', 'FILE_HANDLE_ARCHIVAL_REQUEST', 'FILE_HANDLE_RESTORE_REQUEST', 'AGENT_CHAT', 'QUERY_AS_PFB', 'GRID_CREATE', 'DOWNLOAD_CSV_FROM_GRID', 'GRID_EXPORT_RECORDSET', 'GRID_IMPORT_CSV') NOT NULL,
66
`CANCELING` BIT(1) NOT NULL,
77
`EXCEPTION` VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL,
88
`ERROR_MESSAGE` VARCHAR(3000) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL,

‎lib/jdomodels/src/main/resources/sql/GetDatasetFileStats.sql‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
WITH ALL_FILES AS (
2+
SELECT D_FILES.OWNER_NODE_ID AS ID, FILES.CONTENT_SIZE AS SIZE
3+
FROM NODE_REVISION AS D
4+
-- Unpack the JSON array of file references from each dataset
5+
JOIN JSON_TABLE(
6+
D.items,
7+
'$[*]' COLUMNS (
8+
id VARCHAR(255) PATH '$.entityId',
9+
version BIGINT PATH '$.versionNumber'
10+
)
11+
) AS D_FILES_REF
12+
JOIN NODE_REVISION AS D_FILES ON (
13+
-- The items array might contain entity ids with the syn prefix
14+
D_FILES.OWNER_NODE_ID = CAST(REPLACE(D_FILES_REF.id, 'syn', '') AS UNSIGNED)
15+
AND D_FILES.NUMBER = D_FILES_REF.version
16+
)
17+
JOIN FILES ON (D_FILES.FILE_HANDLE_ID = FILES.ID)
18+
WHERE (D.OWNER_NODE_ID, D.NUMBER) IN (:datasetRefs)
19+
-- De-duplicate the id/version pairs in case datasets overlap
20+
GROUP BY D_FILES.OWNER_NODE_ID, D_FILES.NUMBER
21+
) SELECT COUNT(ALL_FILES.ID) AS count, COALESCE(SUM(ALL_FILES.SIZE), 0) AS size FROM ALL_FILES;

‎lib/jdomodels/src/test/java/org/sagebionetworks/repo/model/dbo/dao/NodeDAOImplTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5625,7 +5625,7 @@ public void testGetFileSummaryLarge() {
56255625

56265626
@Test
56275627
public void testGetFileSummaryWithEmptyList() {
5628-
FileSummary expectedFileSummary = new FileSummary(null, 0, 0);
5628+
FileSummary expectedFileSummary = new FileSummary(0, 0);
56295629
FileSummary fileSummary = nodeDao.getFileSummary(Collections.emptyList());
56305630
assertEquals(expectedFileSummary, fileSummary);
56315631
}
@@ -5638,14 +5638,14 @@ public void testGetFileSummaryWithEntityVersionNull() {
56385638
file1.setFileHandleId(fileHandle.getId());
56395639
String file1Id = nodeDao.createNew(file1);
56405640
toDelete.add(file1Id);
5641-
FileSummary expectedFileSummary = new FileSummary(null, 0, 0);
5641+
FileSummary expectedFileSummary = new FileSummary(0, 0);
56425642
FileSummary fileSummary = nodeDao.getFileSummary(Collections.singletonList(new EntityRef().setEntityId(file1Id).setVersionNumber(null)));
56435643
assertEquals(expectedFileSummary, fileSummary);
56445644
}
56455645

56465646
@Test
56475647
public void testGetFileSummaryWithEntityRefIdNull() {
5648-
FileSummary expectedFileSummary = new FileSummary(null, 0, 0);
5648+
FileSummary expectedFileSummary = new FileSummary(0, 0);
56495649
FileSummary fileSummary = nodeDao.getFileSummary(Collections.singletonList(new EntityRef().setEntityId(null).setVersionNumber(1L)));
56505650
assertEquals(expectedFileSummary, fileSummary);
56515651
}

0 commit comments

Comments
 (0)