Skip to content

Commit d46fad0

Browse files
author
Aakash Arayambeth
committed
generic sharding: organize llmeta and create/drop views
Signed-off-by: Aakash Arayambeth <[email protected]>
1 parent 2f9ab0e commit d46fad0

24 files changed

+786
-131
lines changed

Diff for: ‎bdb/bdb_api.h

+3
Original file line numberDiff line numberDiff line change
@@ -2496,3 +2496,6 @@ int bdb_keylen(bdb_state_type *bdb_state, int ixnum);
24962496

24972497
void llmeta_collect_tablename_alias(void);
24982498
#endif
2499+
int bdb_set_genshard(tran_type *, const char *, char *, int *);
2500+
int bdb_get_genshard(tran_type *, const char *, char **, int *, int *);
2501+
int bdb_get_genshard_names(tran_type *t, char **names, int *num);

Diff for: ‎bdb/bdb_schemachange.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ int handle_scdone(DB_ENV *dbenv, u_int32_t rectype, llog_scdone_args *scdoneop,
190190
}
191191

192192
if (sctype == rename_table || sctype == alias_table ||
193-
(sctype == add && (strlen(table) + 1 < scdoneop->table.size))) {
193+
((sctype == add || sctype == genshard_add || sctype == genshard_drop) && (strlen(table) + 1 < scdoneop->table.size))) {
194194
strarg = &table[strlen(table) + 1];
195195
}
196196

Diff for: ‎bdb/bdb_schemachange.h

+3-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,9 @@ typedef enum scdone {
5151
add_queue_file, // 22
5252
del_queue_file, // 23
5353
alias_table, // 24
54-
alias // 25
54+
alias, // 25
55+
genshard_add, // 26
56+
genshard_drop // 27
5557
} scdone_t;
5658

5759
#define BDB_BUMP_DBOPEN_GEN(type, msg) \

Diff for: ‎bdb/llmeta.c

+237
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,7 @@ typedef enum {
179179
LLMETA_SCHEMACHANGE_STATUS_V2 = 56,
180180
LLMETA_SCHEMACHANGE_LIST = 57, /* list of all sc-s in a uuid txh */
181181
LLMETA_SCHEMACHANGE_STATUS_PROTOBUF = 58, /* Indicate protobuf sc */
182+
LLMETA_GEN_SHARD = 59,
182183
} llmetakey_t;
183184

184185
struct llmeta_file_type_key {
@@ -7354,6 +7355,15 @@ int bdb_llmeta_print_record(bdb_state_type *bdb_state, void *key, int keylen,
73547355
if (osql_scl_print((uint8_t*)p_buf_key, p_buf_end_key, (uint8_t*)p_buf_data, p_buf_end_data))
73557356
logmsg(LOGMSG_USER, " failed to deserialize object\n");
73567357
} break;
7358+
case LLMETA_GEN_SHARD: {
7359+
char tblname[LLMETA_TBLLEN + 1];
7360+
buf_no_net_get(&(tblname), sizeof(tblname), p_buf_key + sizeof(int),
7361+
p_buf_end_key);
7362+
7363+
logmsg(LOGMSG_USER,
7364+
"LLMETA_GEN_SHARD table=\"%s\" value=\"%s\"\n",
7365+
tblname, (char *)data);
7366+
} break;
73577367
default:
73587368
logmsg(LOGMSG_USER, "Todo (type=%d)\n", type);
73597369
break;
@@ -11372,3 +11382,230 @@ void *buf_get_schemachange(struct schema_change_type *s, void *p_buf, void *p_bu
1137211382
}
1137311383
return NULL;
1137411384
}
11385+
11386+
struct llmeta_gen_shard_key {
11387+
int file_type;
11388+
char tablename[LLMETA_TBLLEN];
11389+
};
11390+
11391+
11392+
static int make_gen_shard_key(char *key, const char *table)
11393+
{
11394+
11395+
const uint8_t *p_gen_shard_key = NULL;
11396+
const uint8_t *p_gen_shard_key_end = NULL;
11397+
uint32_t key_type = 0;
11398+
assert(sizeof(int) + strlen(table) <= LLMETA_IXLEN);
11399+
key_type = LLMETA_GEN_SHARD;
11400+
p_gen_shard_key = (uint8_t *)key;
11401+
p_gen_shard_key_end = p_gen_shard_key + sizeof(int) + LLMETA_TBLLEN;
11402+
11403+
if (!llmeta_bdb_state) {
11404+
logmsg(LOGMSG_ERROR, "%s: low level meta table not yet "
11405+
"open, you must run bdb_llmeta_open\n",
11406+
__func__);
11407+
return -1;
11408+
}
11409+
11410+
/* put key type */
11411+
if (!(p_gen_shard_key =
11412+
buf_put(&key_type, sizeof(key_type), (uint8_t *)p_gen_shard_key,
11413+
p_gen_shard_key_end))) {
11414+
logmsg(LOGMSG_ERROR, "%s:%d error converting to correct "
11415+
"endianess\n",
11416+
__func__, __LINE__);
11417+
return -1;
11418+
}
11419+
11420+
/* put tablename of generic shard */
11421+
if (!(p_gen_shard_key =
11422+
buf_put(table, strlen(table) , (uint8_t *)p_gen_shard_key, p_gen_shard_key_end))) {
11423+
logmsg(LOGMSG_ERROR, "%s:%d error converting to correct "
11424+
"endianess\n",
11425+
__func__, __LINE__);
11426+
return -1;
11427+
}
11428+
return 0;
11429+
}
11430+
11431+
int bdb_set_genshard(tran_type *input_tran, const char *table, char *str, int *bdberr) {
11432+
uint32_t retries = 0;
11433+
int rc=0;
11434+
char key[LLMETA_IXLEN] = {0};
11435+
tran_type *tran;
11436+
if (make_gen_shard_key(key, table)) {
11437+
*bdberr = BDBERR_BADARGS;
11438+
return -1;
11439+
}
11440+
/*const uint8_t *p_gen_shard_key = NULL;
11441+
const uint8_t *p_gen_shard_key_end = NULL;
11442+
11443+
assert(sizeof(int) + strlen(table) <= LLMETA_IXLEN);
11444+
key_type = LLMETA_GEN_SHARD;
11445+
p_gen_shard_key = (uint8_t *)key;
11446+
p_gen_shard_key_end = p_gen_shard_key + sizeof(int) + LLMETA_TBLLEN;
11447+
11448+
if (!llmeta_bdb_state) {
11449+
logmsg(LOGMSG_ERROR, "%s: low level meta table not yet "
11450+
"open, you must run bdb_llmeta_open\n",
11451+
__func__);
11452+
*bdberr = BDBERR_MISC;
11453+
return -1;
11454+
}
11455+
11456+
if (!(p_gen_shard_key =
11457+
buf_put(&key_type, sizeof(key_type), (uint8_t *)p_gen_shard_key,
11458+
p_gen_shard_key_end))) {
11459+
logmsg(LOGMSG_ERROR, "%s:%d error converting to correct "
11460+
"endianess\n",
11461+
__func__, __LINE__);
11462+
*bdberr = BDBERR_MISC;
11463+
return -1;
11464+
}
11465+
11466+
if (!(p_gen_shard_key =
11467+
buf_put(table, strlen(table) , (uint8_t *)p_gen_shard_key, p_gen_shard_key_end))) {
11468+
logmsg(LOGMSG_ERROR, "%s:%d error converting to correct "
11469+
"endianess\n",
11470+
__func__, __LINE__);
11471+
*bdberr = BDBERR_MISC;
11472+
return -1;
11473+
}*/
11474+
retry:
11475+
if (++retries >= gbl_maxretries) {
11476+
logmsg(LOGMSG_ERROR, "%s: giving up after %d retries\n", __func__, retries);
11477+
return -1;
11478+
}
11479+
/*if the user didn't give us a transaction, create our own*/
11480+
if (!input_tran) {
11481+
tran = bdb_tran_begin(llmeta_bdb_state, NULL, bdberr);
11482+
if (!tran) {
11483+
if (*bdberr == BDBERR_DEADLOCK) {
11484+
int dp = gbl_llmeta_deadlock_poll;
11485+
if (dp > 1)
11486+
poll(NULL, 0, rand() % dp);
11487+
goto retry;
11488+
}
11489+
11490+
logmsg(LOGMSG_ERROR, "%s: failed to get "
11491+
"transaction\n",
11492+
__func__);
11493+
goto cleanup;
11494+
}
11495+
} else {
11496+
tran = input_tran;
11497+
}
11498+
11499+
/* write the shard string */
11500+
if (str) {
11501+
rc = bdb_lite_add(llmeta_bdb_state, tran, str,
11502+
strlen(str) + 1, key, bdberr);
11503+
} else {
11504+
rc = bdb_lite_exact_del(llmeta_bdb_state, tran, key, bdberr);
11505+
}
11506+
if (rc && *bdberr != BDBERR_NOERROR)
11507+
goto backout;
11508+
11509+
/*commit if we created our own transaction*/
11510+
if (!input_tran) {
11511+
rc = bdb_tran_commit(llmeta_bdb_state, tran, bdberr);
11512+
if (rc && *bdberr != BDBERR_NOERROR)
11513+
goto cleanup;
11514+
}
11515+
11516+
*bdberr = BDBERR_NOERROR;
11517+
return 0;
11518+
11519+
backout:
11520+
/*if we created the transaction*/
11521+
if (!input_tran) {
11522+
/*kill the transaction*/
11523+
rc = bdb_tran_abort(llmeta_bdb_state, tran, bdberr);
11524+
if (rc && !BDBERR_NOERROR) {
11525+
logmsg(LOGMSG_ERROR, "%s: trans abort failed with "
11526+
"bdberr %d\n",
11527+
__func__, *bdberr);
11528+
return -1;
11529+
}
11530+
if (*bdberr == BDBERR_DEADLOCK) {
11531+
int dp = gbl_llmeta_deadlock_poll;
11532+
if (dp > 1)
11533+
poll(NULL, 0, rand() % dp);
11534+
goto retry;
11535+
}
11536+
11537+
logmsg(LOGMSG_ERROR, "%s:%d failed with bdberr %d\n", __func__,__LINE__, *bdberr);
11538+
}
11539+
cleanup:
11540+
return -1;
11541+
}
11542+
11543+
11544+
int bdb_get_genshard(tran_type *tran, const char *table, char **str, int *size, int *bdberr) {
11545+
char key[LLMETA_IXLEN] = {0};
11546+
if (make_gen_shard_key(key, table)) {
11547+
*bdberr = BDBERR_BADARGS;
11548+
return -1;
11549+
}
11550+
/*const uint8_t *p_gen_shard_key = NULL;
11551+
const uint8_t *p_gen_shard_key_end = NULL;
11552+
11553+
assert(sizeof(int) + strlen(table) <= LLMETA_IXLEN);
11554+
key_type = LLMETA_GEN_SHARD;
11555+
p_gen_shard_key = (uint8_t *)key;
11556+
p_gen_shard_key_end = p_gen_shard_key + sizeof(int) + LLMETA_TBLLEN;
11557+
11558+
if (!llmeta_bdb_state) {
11559+
logmsg(LOGMSG_ERROR, "%s: low level meta table not yet "
11560+
"open, you must run bdb_llmeta_open\n",
11561+
__func__);
11562+
*bdberr = BDBERR_MISC;
11563+
return -1;
11564+
}
11565+
11566+
if (!(p_gen_shard_key =
11567+
buf_put(&key_type, sizeof(key_type), (uint8_t *)p_gen_shard_key,
11568+
p_gen_shard_key_end))) {
11569+
logmsg(LOGMSG_ERROR, "%s:%d error converting to correct "
11570+
"endianess\n",
11571+
__func__, __LINE__);
11572+
*bdberr = BDBERR_MISC;
11573+
return -1;
11574+
}
11575+
11576+
if (!(p_gen_shard_key =
11577+
buf_put(table, strlen(table) , (uint8_t *)p_gen_shard_key, p_gen_shard_key_end))) {
11578+
logmsg(LOGMSG_ERROR, "%s:%d error converting to correct "
11579+
"endianess\n",
11580+
__func__, __LINE__);
11581+
*bdberr = BDBERR_MISC;
11582+
return -1;
11583+
}*/
11584+
return bdb_lite_exact_var_fetch_tran(llmeta_bdb_state, tran, key, (void **)str, size, bdberr);
11585+
}
11586+
11587+
/* Fetch all generic shard names */
11588+
int bdb_get_genshard_names(tran_type *t, char **names, int *num)
11589+
{
11590+
union {
11591+
struct llmeta_gen_shard_key key;
11592+
uint8_t buf[LLMETA_IXLEN];
11593+
} * *shards;
11594+
int rc, n, bdberr;
11595+
llmetakey_t k;
11596+
11597+
k = htonl(LLMETA_GEN_SHARD);
11598+
rc = kv_get_keys(t, &k, sizeof(k), (void ***)&shards, &n, &bdberr);
11599+
if (rc || (n == 0)) {
11600+
*num = 0;
11601+
return rc;
11602+
}
11603+
11604+
for (int i = 0; i < n; ++i) {
11605+
names[i] = strdup(shards[i]->key.tablename);
11606+
free(shards[i]);
11607+
}
11608+
free(shards);
11609+
*num = n;
11610+
return rc;
11611+
}

0 commit comments

Comments
 (0)