Skip to content

Commit 8345a74

Browse files
MichaelBrimMichaelBrim
committed
more extent caching rework
1 parent 462f2e0 commit 8345a74

File tree

5 files changed

+72
-62
lines changed

5 files changed

+72
-62
lines changed

‎server/src/unifyfs_group_rpc.c‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1113,7 +1113,7 @@ int unifyfs_invoke_broadcast_extents(int gfid)
11131113

11141114
size_t n_extents = 0;
11151115
struct extent_metadata* extents = NULL;
1116-
ret = unifyfs_inode_get_extents(gfid, 1, &n_extents, &extents, NULL);
1116+
ret = unifyfs_inode_get_extents(gfid, &n_extents, &extents, NULL);
11171117
if (ret != UNIFYFS_SUCCESS) {
11181118
LOGERR("failed to get extents for gfid=%d", gfid);
11191119
return ret;
@@ -1249,21 +1249,12 @@ int unifyfs_invoke_broadcast_extents_cache(int gfid)
12491249
/* assuming success */
12501250
int ret = UNIFYFS_SUCCESS;
12511251

1252-
struct timespec cache_ts, bcast_ts;
1253-
ret = unifyfs_inode_get_cache_times(gfid, &cache_ts, &bcast_ts);
1254-
if (UNIFYFS_SUCCESS == ret) {
1255-
if (0 == compare_timespec(&cache_ts, &bcast_ts)) {
1256-
/* last cache was broadcast already */
1257-
return UNIFYFS_SUCCESS;
1258-
}
1259-
}
1260-
12611252
LOGDBG("BCAST_RPC: starting cache extents for gfid=%d", gfid);
12621253

12631254
size_t n_extents = 0;
12641255
struct extent_metadata* extents = NULL;
12651256
struct timespec ts;
1266-
ret = unifyfs_inode_get_extents(gfid, 1, &n_extents, &extents, &ts);
1257+
ret = unifyfs_inode_get_extents(gfid, &n_extents, &extents, &ts);
12671258
if (ret != UNIFYFS_SUCCESS) {
12681259
LOGERR("failed to get extents for gfid=%d", gfid);
12691260
return ret;
@@ -1520,7 +1511,7 @@ int unifyfs_invoke_broadcast_laminate(int gfid)
15201511

15211512
size_t n_extents = 0;
15221513
struct extent_metadata* extents = NULL;
1523-
ret = unifyfs_inode_get_extents(gfid, 1, &n_extents, &extents, NULL);
1514+
ret = unifyfs_inode_get_extents(gfid, &n_extents, &extents, NULL);
15241515
if (ret != UNIFYFS_SUCCESS) {
15251516
LOGERR("failed to get extents for gfid=%d", gfid);
15261517
return ret;

‎server/src/unifyfs_inode.c‎

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -540,6 +540,7 @@ int unifyfs_inode_cache_extents(int gfid,
540540
int ret = UNIFYFS_SUCCESS;
541541
unifyfs_inode_wrlock(ino);
542542
{
543+
/* update cache if inode and parameter timestamps differ */
543544
if (0 != compare_timespec(cache_time, &(ino->cache_time))) {
544545
invalidate_extents_cache(ino);
545546
if (num_extents > 0) {
@@ -553,11 +554,19 @@ int unifyfs_inode_cache_extents(int gfid,
553554
cache_time->tv_nsec);
554555
}
555556
} else {
556-
LOGDBG("local cache time for gfid=%d already matches", gfid);
557+
LOGDBG("inode cache time matches for gfid=%d", gfid);
557558
if (NULL != extents) {
558559
free(extents);
559560
}
560561
}
562+
563+
/* set cache valid time */
564+
int owner_rank = hash_gfid_to_server(gfid);
565+
if (owner_rank != glb_pmi_rank) {
566+
struct timespec now;
567+
clock_gettime(CLOCK_REALTIME, &now);
568+
ino->valid_time = now;
569+
}
561570
}
562571
unifyfs_inode_unlock(ino);
563572

@@ -566,7 +575,7 @@ int unifyfs_inode_cache_extents(int gfid,
566575

567576
int unifyfs_inode_get_cache_times(int gfid,
568577
struct timespec* cache_time,
569-
struct timespec* bcast_time)
578+
struct timespec* valid_time)
570579
{
571580
struct unifyfs_inode* ino = unifyfs_inode_lookup(gfid);
572581
if (NULL == ino) {
@@ -578,8 +587,8 @@ int unifyfs_inode_get_cache_times(int gfid,
578587
{
579588
if (NULL != cache_time)
580589
*cache_time = ino->cache_time;
581-
if (NULL != bcast_time)
582-
*bcast_time = ino->bcast_time;
590+
if (NULL != valid_time)
591+
*valid_time = ino->valid_time;
583592
}
584593
unifyfs_inode_unlock(ino);
585594

@@ -630,12 +639,11 @@ int unifyfs_inode_laminate(int gfid)
630639
}
631640

632641
int unifyfs_inode_get_extents(int gfid,
633-
int for_bcast,
634-
size_t* n,
642+
size_t* num_extents,
635643
extent_metadata** extents,
636644
struct timespec* timestamp)
637645
{
638-
if ((NULL == n) || (NULL == extents)) {
646+
if ((NULL == num_extents) || (NULL == extents)) {
639647
return EINVAL;
640648
}
641649

@@ -651,8 +659,11 @@ int unifyfs_inode_get_extents(int gfid,
651659
if ((NULL != ino->extents_cache) &&
652660
(ino->attr.mtime.tv_sec == ino->cache_time.tv_sec) &&
653661
(ino->attr.mtime.tv_nsec == ino->cache_time.tv_nsec)) {
662+
struct timespec now;
663+
clock_gettime(CLOCK_REALTIME, &now);
664+
ino->valid_time = now;
654665
*extents = ino->extents_cache;
655-
*n = ino->extents_cache_count;
666+
*num_extents = ino->extents_cache_count;
656667
} else {
657668
struct extent_tree* tree = ino->extents;
658669
n_extents = tree->count;
@@ -667,7 +678,7 @@ int unifyfs_inode_get_extents(int gfid,
667678
i++;
668679
}
669680

670-
*n = n_extents;
681+
*num_extents = n_extents;
671682
*extents = extarr;
672683

673684
if (NULL != ino->extents_cache) {
@@ -677,11 +688,10 @@ int unifyfs_inode_get_extents(int gfid,
677688
ino->extents_cache = extarr;
678689
ino->extents_cache_count = n_extents;
679690
ino->cache_time = ino->attr.mtime;
691+
ino->valid_time = ino->cache_time;
680692
}
681693
}
682-
if (for_bcast) {
683-
ino->bcast_time = ino->cache_time;
684-
}
694+
685695
if (NULL != timestamp) {
686696
*timestamp = ino->cache_time;
687697
}
@@ -786,7 +796,7 @@ int unifyfs_inode_get_extent_chunks(unifyfs_extent_t* extent,
786796
cache_expire.tv_sec += UNIFYFS_METADATA_CACHE_SECONDS;
787797
clock_gettime(CLOCK_REALTIME, &now);
788798
if (compare_timespec(&now, &cache_expire) <= 0) {
789-
LOGDBG("using cached extent metadata - stamp=(%lu.%09lu)",
799+
LOGDBG("using cached extents - timestamp=%lu.%09lu",
790800
ino->cache_time.tv_sec, ino->cache_time.tv_nsec);
791801
ret = get_extent_cache_chunks(extent,
792802
ino->extents_cache,
@@ -797,7 +807,7 @@ int unifyfs_inode_get_extent_chunks(unifyfs_extent_t* extent,
797807
done = 1;
798808
}
799809
} else {
800-
LOGDBG("NOT using cached extent metadata - stamp=(%lu.%09lu)",
810+
LOGDBG("NOT using cached extents - timestamp=%lu.%09lu",
801811
ino->cache_time.tv_sec, ino->cache_time.tv_nsec);
802812
}
803813
}
@@ -851,7 +861,6 @@ int compare_data_chunks(const void* _c1, const void* _c2)
851861
}
852862
}
853863

854-
855864
int unifyfs_inode_resolve_extent_chunks(unsigned int n_extents,
856865
unifyfs_extent_t* extents,
857866
unsigned int* n_locs,

‎server/src/unifyfs_inode.h‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ struct unifyfs_inode {
3939
extent_metadata* extents_cache; /* cached serialized array of extents */
4040
size_t extents_cache_count; /* number of entries in cached array */
4141
struct timespec cache_time; /* mtime of file at last cache update */
42-
struct timespec bcast_time; /* cache_time used for last broadcast */
42+
struct timespec valid_time; /* cache_time used for last broadcast */
4343

4444
ABT_rwlock rwlock; /* reader-writer lock */
4545
};
@@ -124,17 +124,15 @@ int unifyfs_inode_truncate(int gfid, unsigned long size);
124124
/**
125125
* @brief get the local extent array from the target inode
126126
*
127-
* @param gfid the global file identifier
128-
* @param for_bcast if non-zero, set the bcast_time on the inode
129-
* @param n pointer to size of the extents array
130-
* @param extents pointer to extents array (caller should NOT free)
131-
* @param timestamp pointer to struct timespec to fill with mtime
127+
* @param gfid the global file identifier
128+
* @param num_extents pointer to size of the extents array
129+
* @param extents pointer to extents array (caller should NOT free)
130+
* @param timestamp pointer to struct timespec to fill with mtime
132131
*
133132
* @return 0 on success, errno otherwise
134133
*/
135134
int unifyfs_inode_get_extents(int gfid,
136-
int for_bcast,
137-
size_t* n,
135+
size_t* num_extents,
138136
extent_metadata** extents,
139137
struct timespec* timestamp);
140138

@@ -208,14 +206,14 @@ int unifyfs_inode_cache_extents(int gfid,
208206
*
209207
* @param gfid the global file identifier
210208
*
211-
* @param[out] cache_time timestamp associated with cache
212-
* @param[out] bcast_time timestamp of cache for last broadcast
209+
* @param[out] cache_time timestamp associated with cache contents
210+
* @param[out] valid_time timestamp when cache was last validated
213211
*
214212
* @return 0 on success, errno otherwise
215213
*/
216214
int unifyfs_inode_get_cache_times(int gfid,
217215
struct timespec* cache_time,
218-
struct timespec* bcast_time);
216+
struct timespec* valid_time);
219217

220218
/**
221219
* @brief get the maximum file size from the local extent tree of given file

‎server/src/unifyfs_p2p_rpc.c‎

Lines changed: 35 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -895,16 +895,28 @@ int unifyfs_find_extent_chunks(unifyfs_fops_ctx_t* ctx,
895895
if (ret == UNIFYFS_SUCCESS) {
896896
int file_laminated = (attrs.is_shared && attrs.is_laminated);
897897
if (!(is_owner || use_server_local_extents || file_laminated)) {
898-
/* send request for updated extents to owner */
899-
struct timespec cache_ts;
900-
ret = unifyfs_inode_get_cache_times(gfid, &cache_ts, NULL);
901-
if (UNIFYFS_SUCCESS != ret) {
898+
int send_request = 1;
899+
struct timespec cache_ts, valid_ts;
900+
ret = unifyfs_inode_get_cache_times(gfid, &cache_ts, &valid_ts);
901+
if (UNIFYFS_SUCCESS == ret) {
902+
struct timespec now;
903+
struct timespec cache_expire = valid_ts;
904+
cache_expire.tv_sec += UNIFYFS_METADATA_CACHE_SECONDS;
905+
clock_gettime(CLOCK_REALTIME, &now);
906+
if (compare_timespec(&now, &cache_expire) <= 0) {
907+
/* cached extent metadata is still valid */
908+
send_request = 0;
909+
}
910+
} else {
902911
cache_ts = (struct timespec) {0};
903912
}
904-
ret = unifyfs_invoke_get_extents_rpc(gfid, &cache_ts);
905-
if (ret != UNIFYFS_SUCCESS) {
906-
LOGERR("request to get extents for gfid=%d failed (ret=%d)",
907-
gfid, ret);
913+
if (send_request) {
914+
/* send request for updated extents to owner */
915+
ret = unifyfs_invoke_get_extents_rpc(gfid, &cache_ts);
916+
if (ret != UNIFYFS_SUCCESS) {
917+
LOGERR("request to get extents for gfid=%d failed (ret=%d)",
918+
gfid, ret);
919+
}
908920
}
909921
}
910922
}
@@ -952,15 +964,15 @@ int unifyfs_invoke_get_extents_rpc(int gfid,
952964
LOGERR("failed to add pending remote get_extents");
953965
return UNIFYFS_FAILURE;
954966
} else if (rc == UNIFYFS_PENDING) {
955-
/* wait up to 5 seconds for completion of pending */
956-
struct timespec timeout;
957-
clock_gettime(CLOCK_REALTIME, &timeout);
958-
timeout.tv_sec += 5;
967+
/* wait for completion of pending get_extents request */
968+
struct timespec pendwait;
969+
clock_gettime(CLOCK_REALTIME, &pendwait);
970+
pendwait.tv_sec += UNIFYFS_METADATA_CACHE_SECONDS;
959971
ABT_mutex_lock(preq->pending_sync);
960972
preq->pending_waiters++;
961973
LOGDBG("waiting on pending get_extents condition for preq(%p)", preq);
962974
rc = ABT_cond_timedwait(preq->pending_cond, preq->pending_sync,
963-
&timeout);
975+
&pendwait);
964976
if (ABT_ERR_COND_TIMEDOUT == rc) {
965977
LOGERR("wait for pending get_extents timed-out");
966978
ret = UNIFYFS_ERROR_TIMEOUT;
@@ -983,7 +995,6 @@ int unifyfs_invoke_get_extents_rpc(int gfid,
983995
get_extents_out_t* out = preq->req_state->outputs;
984996

985997
/* fill rpc input struct and forward request to owner */
986-
987998
in->src_rank = (int32_t) glb_pmi_rank;
988999
in->gfid = (int32_t) gfid;
9891000
in->src_timestamp = *timestamp;
@@ -1004,24 +1015,23 @@ int unifyfs_invoke_get_extents_rpc(int gfid,
10041015
ret = out->ret;
10051016
if (ret == UNIFYFS_SUCCESS) {
10061017
/* get number of extents */
1007-
unsigned int n_ext = (unsigned int) out->num_extents;
1018+
extent_metadata* em_arr = NULL;
1019+
size_t n_ext = (size_t) out->num_extents;
1020+
struct timespec owner_stamp = (struct timespec) out->owner_timestamp;
10081021
if ((n_ext > 0) && (out->extents != HG_BULK_NULL)) {
10091022
/* get bulk buffer with extent locations */
1010-
size_t buf_sz = (size_t)n_ext * sizeof(extent_metadata);
1023+
size_t buf_sz = n_ext * sizeof(extent_metadata);
10111024
void* buf = pull_margo_bulk(preq->req_state->handle,
10121025
out->extents, buf_sz, NULL);
10131026
if (NULL == buf) {
10141027
LOGERR("failed to pull extent metadata array");
10151028
ret = UNIFYFS_ERROR_MARGO;
10161029
} else {
10171030
/* replace local cache with received extents */
1018-
extent_metadata* em_arr = (extent_metadata*) buf;
1019-
struct timespec owner_stamp =
1020-
(struct timespec) out->owner_timestamp;
1021-
ret = unifyfs_inode_cache_extents(gfid, (int) n_ext, em_arr,
1022-
&owner_stamp);
1031+
em_arr = (extent_metadata*) buf;
10231032
}
10241033
}
1034+
ret = sm_cache_extents(gfid, n_ext, em_arr, &owner_stamp);
10251035
}
10261036

10271037
clear_pending_extents_get:
@@ -1067,7 +1077,7 @@ static void process_get_extents_rpc(server_rpc_req_t* sreq)
10671077
int send_extents = 0;
10681078
int owner_rank = hash_gfid_to_server(gfid);
10691079
if (owner_rank == glb_pmi_rank) {
1070-
ret = unifyfs_inode_get_extents(gfid, 0, &num_extents, &extents,
1080+
ret = unifyfs_inode_get_extents(gfid, &num_extents, &extents,
10711081
&owner_stamp);
10721082
if (ret == UNIFYFS_SUCCESS) {
10731083
/* Compare source timestamp to owner's and do:
@@ -1080,7 +1090,7 @@ static void process_get_extents_rpc(server_rpc_req_t* sreq)
10801090
LOGDBG("owner has newer extents metadata");
10811091
send_extents = 1;
10821092
if (src_stamp.tv_sec == 0) {
1083-
/* zero source timestamp, time to broadcast */
1093+
/* source timestamp is zero, time to broadcast */
10841094
LOGDBG("broadcasting extents metadata to cache");
10851095
ret = unifyfs_invoke_broadcast_extents_cache(gfid);
10861096
}
@@ -1090,6 +1100,8 @@ static void process_get_extents_rpc(server_rpc_req_t* sreq)
10901100
send_extents = 1;
10911101
LOGWARN("source cache timestamp is newer than owner?!?");
10921102
}
1103+
// else, the timestamps are the same, so don't send
1104+
10931105
if (send_extents && (num_extents > 0)) {
10941106
/* define a bulk handle to transfer extent_metadata array */
10951107
margo_instance_id mid =

‎server/src/unifyfs_transfer.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ int create_local_transfers(int gfid,
122122

123123
size_t n_extents = 0;
124124
extent_metadata* extents = NULL;
125-
int rc = unifyfs_inode_get_extents(gfid, 0, &n_extents, &extents, NULL);
125+
int rc = unifyfs_inode_get_extents(gfid, &n_extents, &extents, NULL);
126126
if (rc != UNIFYFS_SUCCESS) {
127127
if (rc != ENOENT) {
128128
LOGERR("failed to get extents from inode for gfid=%d", gfid);

0 commit comments

Comments
 (0)