Skip to content

Commit b1d13fe

Browse files
author
Aakash Arayambeth
committed
generic sharding : setup metadata and remote tables. Create/delete
Signed-off-by: Aakash Arayambeth <[email protected]>
1 parent 7f65be4 commit b1d13fe

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+2351
-51
lines changed

Diff for: ‎bbinc/cdb2_constants.h

+2
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@
3636
#define MAX_SPNAME MAXTABLELEN
3737
#define MAX_SPVERSION_LEN 80
3838
#define MAXTABLELEN 32
39+
#define MAXPARTITIONS 32
40+
#define MAXPARTITIONLEN (1 + (MAX_DBNAME_LENGTH + MAXTABLELEN)) /* partitions of the form <database>.<table> */
3941
#define MAXTAGLEN 64
4042
#define REPMAX 32
4143
/* Maximum buffer length for generated key name. */

Diff for: ‎bbinc/consistent_hash.h

+5
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@ enum ch_err {
1515
CH_ERR_DUP = 4
1616
};
1717

18+
enum ch_hash_func_type {
19+
CH_HASH_SHA = 1,
20+
CH_HASH_MD5 = 2,
21+
CH_HASH_CRC = 3
22+
};
1823
struct consistent_hash_node {
1924
uint8_t *data;
2025
size_t data_len;

Diff for: ‎bdb/bdb_schemachange.c

+6
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,12 @@ int bdb_llog_partition(bdb_state_type *bdb_state, tran_type *tran, char *name,
353353
return bdb_llog_scdone_tran(bdb_state, views, tran, name, strlen(name) + 1,
354354
bdberr);
355355
}
356+
357+
int bdb_llog_hash_partition(bdb_state_type *bdb_state, tran_type *tran, char *name, int *bdberr)
358+
{
359+
return bdb_llog_scdone_tran(bdb_state, hash_views, tran, name, strlen(name) + 1, bdberr);
360+
}
361+
356362
int bdb_llog_luareload(bdb_state_type *bdb_state, int wait, int *bdberr)
357363
{
358364
return bdb_llog_scdone(bdb_state, luareload, NULL, 0, wait, bdberr);

Diff for: ‎bdb/bdb_schemachange.h

+3-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ typedef enum scdone {
5151
add_queue_file, // 22
5252
del_queue_file, // 23
5353
alias_table, // 24
54-
alias // 25
54+
alias, // 25
55+
hash_views // 25
5556
} scdone_t;
5657

5758
#define BDB_BUMP_DBOPEN_GEN(type, msg) \
@@ -73,6 +74,7 @@ int bdb_llog_views(bdb_state_type *bdb_state, char *name, int wait,
7374
int *bdberr);
7475
int bdb_llog_partition(bdb_state_type *bdb_state, tran_type *tran, char *name,
7576
int *bdberr);
77+
int bdb_llog_hash_partition(bdb_state_type *bdb_state, tran_type *tran, char *name, int *bdberr);
7678
int bdb_llog_rowlocks(bdb_state_type *, scdone_t, int *bdberr);
7779
int bdb_llog_genid_format(bdb_state_type *, scdone_t, int *bdberr);
7880
int bdb_reload_rowlocks(bdb_state_type *, scdone_t, int *bdberr);

Diff for: ‎db/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ set(src
109109
machclass.c
110110
osqluprec.c
111111
macc_glue.c
112+
hash_partition.c
112113
${PROJECT_BINARY_DIR}/protobuf/bpfunc.pb-c.c
113114
${PROJECT_SOURCE_DIR}/tools/cdb2_dump/cdb2_dump.c
114115
${PROJECT_SOURCE_DIR}/tools/cdb2_load/cdb2_load.c

Diff for: ‎db/comdb2.c

+21-2
Original file line numberDiff line numberDiff line change
@@ -494,7 +494,7 @@ double gbl_sql_cost_error_threshold = -1;
494494

495495
int gbl_parallel_recovery_threads = 0;
496496

497-
int gbl_fdb_resolve_local = 0;
497+
int gbl_fdb_resolve_local = 1;
498498
int gbl_fdb_allow_cross_classes = 0;
499499
uint64_t gbl_sc_headroom = 10;
500500
/*---COUNTS---*/
@@ -818,6 +818,7 @@ int gbl_hostname_refresh_time = 60;
818818
int gbl_pstack_self = 1;
819819

820820
char *gbl_cdb2api_policy_override = NULL;
821+
int gbl_create_remote_tables = 0;
821822

822823
int close_all_dbs_tran(tran_type *tran);
823824

@@ -2412,6 +2413,15 @@ int llmeta_load_timepart(struct dbenv *dbenv)
24122413
return thedb->timepart_views ? 0 : -1;
24132414
}
24142415

2416+
int llmeta_load_hash_partitions(struct dbenv *dbenv)
2417+
{
2418+
logmsg(LOGMSG_INFO, "Loading hash-based partitions\n");
2419+
Pthread_rwlock_init(&hash_partition_lk, NULL);
2420+
dbenv->hash_partition_views = hash_create_all_views();
2421+
2422+
return dbenv->hash_partition_views ? 0 : -1;
2423+
}
2424+
24152425
/* replace the table names and dbnums saved in the low level meta table with the
24162426
* ones in the dbenv. returns 0 on success and anything else otherwise */
24172427
int llmeta_set_tables(tran_type *tran, struct dbenv *dbenv)
@@ -4144,6 +4154,12 @@ static int init(int argc, char **argv)
41444154
unlock_schema_lk();
41454155
return -1;
41464156
}
4157+
4158+
if (llmeta_load_hash_partitions(thedb)) {
4159+
logmsg(LOGMSG_ERROR, "could not load hash partitions\n");
4160+
unlock_schema_lk();
4161+
return -1;
4162+
}
41474163

41484164
if (llmeta_load_queues(thedb)) {
41494165
logmsg(LOGMSG_FATAL, "could not load queues from the low level meta "
@@ -6343,7 +6359,10 @@ int comdb2_reload_schemas(void *dbenv, void *inlsn)
63436359
logmsg(LOGMSG_ERROR, "could not load time partitions\n");
63446360
abort();
63456361
}
6346-
6362+
if (llmeta_load_hash_partitions(thedb)) {
6363+
logmsg(LOGMSG_FATAL, "could not load mod based shards\n");
6364+
abort();
6365+
}
63476366
if ((rc = bdb_get_rowlocks_state(&rlstate, tran, &bdberr)) != 0) {
63486367
logmsg(LOGMSG_ERROR, "Get rowlocks llmeta failed, rc=%d bdberr=%d\n",
63496368
rc, bdberr);

Diff for: ‎db/comdb2.h

+7-6
Original file line numberDiff line numberDiff line change
@@ -969,11 +969,11 @@ struct dbenv {
969969
int incoh_notcoherent;
970970
uint32_t incoh_file, incoh_offset;
971971
timepart_views_t *timepart_views;
972-
973-
struct time_metric *service_time;
974-
struct time_metric *queue_depth;
975-
struct time_metric *concurrent_queries;
976-
struct time_metric *connections;
972+
hash_t *hash_partition_views;
973+
struct time_metric* service_time;
974+
struct time_metric* queue_depth;
975+
struct time_metric* concurrent_queries;
976+
struct time_metric* connections;
977977
struct time_metric *sql_queue_time;
978978
struct time_metric *handle_buf_queue_time;
979979
struct time_metric *watchdog_time;
@@ -1908,7 +1908,7 @@ extern int gbl_dohsql_pool_thr_slack;
19081908
extern int gbl_dohsql_sc_max_threads;
19091909
extern int gbl_sockbplog;
19101910
extern int gbl_sockbplog_sockpool;
1911-
1911+
extern int gbl_sharding_ddl_verbose;
19121912
extern int gbl_logical_live_sc;
19131913

19141914
extern int gbl_test_io_errors;
@@ -3739,4 +3739,5 @@ void get_disable_skipscan_all();
37393739

37403740
void get_client_origin(char *out, size_t outlen, struct sqlclntstate *clnt);
37413741

3742+
extern pthread_rwlock_t hash_partition_lk;
37423743
#endif /* !INCLUDED_COMDB2_H */

Diff for: ‎db/db_tunables.c

-1
Original file line numberDiff line numberDiff line change
@@ -528,7 +528,6 @@ int gbl_old_column_names = 1;
528528
int gbl_enable_sq_flattening_optimization = 1;
529529
int gbl_mask_internal_tunables = 1;
530530
int gbl_allow_readonly_runtime_mod = 0;
531-
532531
size_t gbl_cached_output_buffer_max_bytes = 8 * 1024 * 1024; /* 8 MiB */
533532
int gbl_sqlite_sorterpenalty = 5;
534533
int gbl_file_permissions = 0660;

Diff for: ‎db/db_tunables.h

+4
Original file line numberDiff line numberDiff line change
@@ -2483,4 +2483,8 @@ REGISTER_TUNABLE("sc_status_max_rows", "Max number of rows returned in comdb2_sc
24832483
REGISTER_TUNABLE("rep_process_pstack_time", "pstack the server if rep_process runs longer than time specified in secs (Default: 30s)",
24842484
TUNABLE_INTEGER, &gbl_rep_process_pstack_time, 0, NULL, NULL, NULL, NULL);
24852485
REGISTER_TUNABLE("sql_recover_time", "Number of msec before checking if SQL has waiters. 0 will disable. (Default: 10ms)", TUNABLE_INTEGER, &gbl_sql_recover_time, 0, NULL, NULL, NULL, NULL);
2486+
REGISTER_TUNABLE("sharding_ddl_verbose",
2487+
"Print debug information for sharded DDL operations",
2488+
TUNABLE_BOOLEAN, &gbl_sharding_ddl_verbose, 0, NULL, NULL, NULL,
2489+
NULL);
24862490
#endif /* _DB_TUNABLES_H */

Diff for: ‎db/dohast.h

Whitespace-only changes.

Diff for: ‎db/fdb_fend.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -1136,7 +1136,7 @@ static int check_table_fdb(fdb_t *fdb, fdb_tbl_t *tbl, int initial,
11361136
return rc;
11371137
}
11381138

1139-
static enum mach_class get_fdb_class(const char **p_dbname, int *local,
1139+
enum mach_class get_fdb_class(const char **p_dbname, int *local,
11401140
int *lvl_override)
11411141
{
11421142
const char *dbname = *p_dbname;

Diff for: ‎db/fdb_fend.h

+2
Original file line numberDiff line numberDiff line change
@@ -485,5 +485,7 @@ void fdb_init_disttxn(sqlclntstate *clnt);
485485
*/
486486
int fdb_2pc_set(sqlclntstate *clnt, fdb_t *fdb, cdb2_hndl_tp *hndl);
487487

488+
enum mach_class get_fdb_class(const char **p_dbname, int *local,
489+
int *lvl_override);
488490
#endif
489491

0 commit comments

Comments
 (0)