diff --git a/src/brain/catalog_sync_brain_job.cpp b/src/brain/catalog_sync_brain_job.cpp new file mode 100644 index 00000000000..190a88ff7f3 --- /dev/null +++ b/src/brain/catalog_sync_brain_job.cpp @@ -0,0 +1,102 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// catalog_sync_brain_job.cpp +// +// Identification: src/brain/catalog_sync_brain_job.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// +#include "brain/catalog_sync_brain_job.h" + +namespace peloton { +namespace brain { +void CatalogSyncBrainJob::OnJobInvocation(BrainEnvironment *env) { + auto &manager = concurrency::TransactionManagerFactory::GetInstance(); + for (auto *catalog : catalog::Catalog::GetInstance()->AvailableCatalogs()) + SyncCatalog(catalog, env, manager); +} + +time_t CatalogSyncBrainJob::TimeFromString(const char *str) { + struct tm tm; + PELOTON_MEMSET(&tm, 0, sizeof(struct tm)); + strptime(str, "%Y-%m-%d %H:%M:%S", &tm); + return mktime(&tm); +} + +std::string CatalogSyncBrainJob::FetchCatalogQuery(catalog::AbstractCatalog *catalog) { + // We need to special cast these two tables because we cannot put a reasonable + // primary key on them without sequences + if (catalog->GetName() == QUERY_HISTORY_CATALOG_NAME) + return "SELECT * FROM pg_catalog." + std::string(QUERY_HISTORY_CATALOG_NAME) + + " WHERE timestamp > " + std::to_string(last_history_timestamp_); + else + return "SELECT * FROM pg_catalog." + catalog->GetName(); +} + +void CatalogSyncBrainJob::UpdateTimestamp(catalog::AbstractCatalog *catalog, + pqxx::field field) { + if (catalog->GetName() == QUERY_HISTORY_CATALOG_NAME + && field.name() == std::string("timestamp")) + last_history_timestamp_ = + std::max(last_history_timestamp_, field.as()); +} + +void CatalogSyncBrainJob::SyncCatalog(catalog::AbstractCatalog *catalog, + BrainEnvironment *env, + concurrency::TransactionManager &manager) { + pqxx::result r = env->ExecuteQuery(FetchCatalogQuery(catalog)); + for (auto row : r) { + concurrency::TransactionContext *txn = + manager.BeginTransaction(IsolationLevelType::REPEATABLE_READS); + catalog::Schema *catalog_schema = catalog->GetDataTable()->GetSchema(); + std::unique_ptr tuple( + new storage::Tuple(catalog_schema, true)); + for (auto field : row) { + oid_t column_id = catalog_schema->GetColumnID(field.name()); + tuple->SetValue(column_id, PqxxFieldToPelotonValue(field)); + UpdateTimestamp(catalog, field); + } + catalog->InsertTuple(std::move(tuple), txn); + // We know this will always succeed on the brain side + manager.CommitTransaction(txn); + } +} + +type::Value CatalogSyncBrainJob::PqxxFieldToPelotonValue(pqxx::field &f) { + type::TypeId type = PostgresValueTypeToPelotonValueType( + static_cast(f.type())); + if (f.is_null()) { + return type == peloton::type::TypeId::VARCHAR + ? type::ValueFactory::GetVarcharValue("") + : type::ValueFactory::GetNullValueByType(type); + } + switch (type) { + case type::TypeId::BOOLEAN: + return type::ValueFactory::GetBooleanValue(f.as()); + case type::TypeId::TINYINT: + return type::ValueFactory::GetTinyIntValue(static_cast(f.as< + int32_t>())); + case type::TypeId::SMALLINT: + return type::ValueFactory::GetSmallIntValue(static_cast(f.as< + int32_t>())); + case type::TypeId::INTEGER: + return type::ValueFactory::GetIntegerValue(f.as()); + case type::TypeId::BIGINT: + return type::ValueFactory::GetBigIntValue(f.as()); + case type::TypeId::TIMESTAMP: + return type::ValueFactory::GetTimestampValue(TimeFromString(f.c_str())); + case type::TypeId::DECIMAL: + return type::ValueFactory::GetDecimalValue(f.as()); + case type::TypeId::VARCHAR:return type::ValueFactory::GetVarcharValue(f.c_str()); + default: + throw ConversionException(StringUtil::Format( + "No corresponding c++ type for postgres type %d", + static_cast(type))); + } +} + +} // namespace brain +} // nanespace peloton \ No newline at end of file diff --git a/src/brain/query_logger.cpp b/src/brain/query_logger.cpp index 6aa512e8e95..28ec3793d83 100644 --- a/src/brain/query_logger.cpp +++ b/src/brain/query_logger.cpp @@ -40,7 +40,7 @@ void QueryLogger::LogQuery(std::string query_string, uint64_t timestamp) { Fingerprint fingerprint{query_string}; // Log query + fingerprint - auto &query_history_catalog = catalog::QueryHistoryCatalog::GetInstance(); + auto &query_history_catalog = catalog::QueryHistoryCatalog::GetInstance(txn); query_history_catalog.InsertQueryHistory( query_string, fingerprint.GetFingerprint(), timestamp, nullptr, txn); diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index 90b9d13a62a..ae22271d1fe 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -20,12 +20,12 @@ #include "catalog/language_catalog.h" #include "catalog/proc_catalog.h" #include "catalog/query_history_catalog.h" -#include "catalog/query_metrics_catalog.h" #include "catalog/settings_catalog.h" #include "catalog/system_catalogs.h" #include "catalog/table_catalog.h" #include "catalog/table_metrics_catalog.h" #include "catalog/trigger_catalog.h" +#include "catalog/tuple_access_metrics_catalog.h" #include "concurrency/transaction_manager_factory.h" #include "function/date_functions.h" #include "function/decimal_functions.h" @@ -33,6 +33,7 @@ #include "function/timestamp_functions.h" #include "index/index_factory.h" #include "settings/settings_manager.h" +#include "storage/database.h" #include "storage/storage_manager.h" #include "storage/table_factory.h" #include "type/ephemeral_pool.h" @@ -191,11 +192,12 @@ void Catalog::Bootstrap() { DatabaseMetricsCatalog::GetInstance(txn); SettingsCatalog::GetInstance(txn); LanguageCatalog::GetInstance(txn); + TupleAccessMetricsCatalog::GetInstance(txn); // TODO: change pg_proc to per database ProcCatalog::GetInstance(txn); - if (settings::SettingsManager::GetBool(settings::SettingId::brain)) { + if (settings::SettingsManager::GetBool(settings::SettingId::brain_data_collection)) { QueryHistoryCatalog::GetInstance(txn); } diff --git a/src/catalog/column_stats_catalog.cpp b/src/catalog/column_stats_catalog.cpp index bbe94340cdb..7a38f2f4cea 100644 --- a/src/catalog/column_stats_catalog.cpp +++ b/src/catalog/column_stats_catalog.cpp @@ -41,13 +41,9 @@ ColumnStatsCatalog::ColumnStatsCatalog(concurrency::TransactionContext *txn) "most_common_freqs VARCHAR, " "histogram_bounds VARCHAR, " "column_name VARCHAR, " - "has_index BOOLEAN);", + "has_index BOOLEAN," + "PRIMARY KEY(database_id, table_id, column_id));", txn) { - // unique key: (database_id, table_id, column_id) - Catalog::GetInstance()->CreateIndex( - CATALOG_DATABASE_NAME, CATALOG_SCHEMA_NAME, COLUMN_STATS_CATALOG_NAME, - {0, 1, 2}, COLUMN_STATS_CATALOG_NAME "_skey0", true, IndexType::BWTREE, - txn); // non-unique key: (database_id, table_id) Catalog::GetInstance()->CreateIndex( CATALOG_DATABASE_NAME, CATALOG_SCHEMA_NAME, COLUMN_STATS_CATALOG_NAME, diff --git a/src/catalog/database_metrics_catalog.cpp b/src/catalog/database_metrics_catalog.cpp index 4fb04fbbb88..7c71cd1c9ce 100644 --- a/src/catalog/database_metrics_catalog.cpp +++ b/src/catalog/database_metrics_catalog.cpp @@ -4,7 +4,7 @@ // // database_metrics_catalog.cpp // -// Identification: src/catalog/query_metrics_catalog.cpp +// Identification: src/catalog/database_metrics_catalog.cpp // // Copyright (c) 2015-17, Carnegie Mellon University Database Group // @@ -19,6 +19,23 @@ namespace peloton { namespace catalog { +DatabaseMetricsCatalogObject::DatabaseMetricsCatalogObject( + executor::LogicalTile *tile, int tupleId) + : database_oid_( + tile->GetValue(tupleId, + DatabaseMetricsCatalog::ColumnId::DATABASE_OID) + .GetAs()), + txn_committed_( + tile->GetValue(tupleId, + DatabaseMetricsCatalog::ColumnId::TXN_COMMITTED) + .GetAs()), + txn_aborted_( + tile->GetValue(tupleId, DatabaseMetricsCatalog::ColumnId::TXN_ABORTED) + .GetAs()), + time_stamp_( + tile->GetValue(tupleId, DatabaseMetricsCatalog::ColumnId::TIME_STAMP) + .GetAs()) {} + DatabaseMetricsCatalog *DatabaseMetricsCatalog::GetInstance( concurrency::TransactionContext *txn) { static DatabaseMetricsCatalog database_metrics_catalog{txn}; @@ -30,7 +47,7 @@ DatabaseMetricsCatalog::DatabaseMetricsCatalog( : AbstractCatalog("CREATE TABLE " CATALOG_DATABASE_NAME "." CATALOG_SCHEMA_NAME "." DATABASE_METRICS_CATALOG_NAME " (" - "database_oid INT NOT NULL, " + "database_oid INT NOT NULL PRIMARY KEY, " "txn_committed INT NOT NULL, " "txn_aborted INT NOT NULL, " "time_stamp INT NOT NULL);", @@ -71,5 +88,54 @@ bool DatabaseMetricsCatalog::DeleteDatabaseMetrics( return DeleteWithIndexScan(index_offset, values, txn); } +bool DatabaseMetricsCatalog::UpdateDatabaseMetrics( + oid_t database_oid, oid_t txn_committed, oid_t txn_aborted, + oid_t time_stamp, concurrency::TransactionContext *txn) { + std::vector update_columns(all_column_ids_); + std::vector update_values; + + update_values.push_back( + type::ValueFactory::GetIntegerValue(database_oid).Copy()); + update_values.push_back( + type::ValueFactory::GetIntegerValue(txn_committed).Copy()); + update_values.push_back( + type::ValueFactory::GetIntegerValue(txn_aborted).Copy()); + update_values.push_back( + type::ValueFactory::GetIntegerValue(time_stamp).Copy()); + + std::vector scan_values; + scan_values.push_back(type::ValueFactory::GetIntegerValue(database_oid)); + oid_t index_offset = IndexId::PRIMARY_KEY; + + return UpdateWithIndexScan(update_columns, update_values, scan_values, + index_offset, txn); +} + +std::shared_ptr +DatabaseMetricsCatalog::GetDatabaseMetricsObject( + oid_t database_oid, concurrency::TransactionContext *txn) { + if (txn == nullptr) { + throw CatalogException("Transaction is invalid!"); + } + + // set up query + std::vector column_ids(all_column_ids_); + oid_t index_offset = IndexId::PRIMARY_KEY; + std::vector values; + values.push_back(type::ValueFactory::GetIntegerValue(database_oid).Copy()); + + auto result_tiles = + GetResultWithIndexScan(column_ids, index_offset, values, txn); + + if (result_tiles->size() == 1 && (*result_tiles)[0]->GetTupleCount() == 1) { + auto database_metric_object = + std::make_shared( + (*result_tiles)[0].get()); + return database_metric_object; + } + + return nullptr; +} + } // namespace catalog } // namespace peloton diff --git a/src/catalog/index_metrics_catalog.cpp b/src/catalog/index_metrics_catalog.cpp index e744765ef96..f4b0c5ed9d9 100644 --- a/src/catalog/index_metrics_catalog.cpp +++ b/src/catalog/index_metrics_catalog.cpp @@ -19,17 +19,45 @@ namespace peloton { namespace catalog { +IndexMetricsCatalogObject::IndexMetricsCatalogObject( + executor::LogicalTile *tile, int tupleId) + : index_oid_( + tile->GetValue(tupleId, IndexMetricsCatalog::ColumnId::INDEX_OID) + .GetAs()), + table_oid_( + tile->GetValue(tupleId, IndexMetricsCatalog::ColumnId::TABLE_OID) + .GetAs()), + reads_(tile->GetValue(tupleId, IndexMetricsCatalog::ColumnId::READS) + .GetAs()), + updates_(tile->GetValue(tupleId, IndexMetricsCatalog::ColumnId::UPDATES) + .GetAs()), + inserts_(tile->GetValue(tupleId, IndexMetricsCatalog::ColumnId::INSERTS) + .GetAs()), + deletes_(tile->GetValue(tupleId, IndexMetricsCatalog::ColumnId::DELETES) + .GetAs()), + memory_alloc_( + tile->GetValue(tupleId, IndexMetricsCatalog::ColumnId::MEMORY_ALLOC) + .GetAs()), + memory_usage_( + tile->GetValue(tupleId, IndexMetricsCatalog::ColumnId::MEMORY_USAGE) + .GetAs()) {} + + IndexMetricsCatalog::IndexMetricsCatalog(const std::string &database_name, concurrency::TransactionContext *txn) : AbstractCatalog("CREATE TABLE " + database_name + "." CATALOG_SCHEMA_NAME "." INDEX_METRICS_CATALOG_NAME " (" - "table_oid INT NOT NULL, " "index_oid INT NOT NULL, " - "reads INT NOT NULL, " - "deletes INT NOT NULL, " - "inserts INT NOT NULL, " - "time_stamp INT NOT NULL);", + "table_oid INT NOT NULL, " + "reads BIGINT NOT NULL, " + "updates BIGINT NOT NULL, " + "inserts BIGINT NOT NULL, " + "deletes BIGINT NOT NULL, " + "memory_alloc BIGINT NOT NULL, " + "memory_usage BIGINT NOT NULL, " + "time_stamp BIGINT NOT NULL," + "PRIMARY KEY(index_oid));", txn) { // Add secondary index here if necessary } @@ -37,25 +65,32 @@ IndexMetricsCatalog::IndexMetricsCatalog(const std::string &database_name, IndexMetricsCatalog::~IndexMetricsCatalog() {} bool IndexMetricsCatalog::InsertIndexMetrics( - oid_t table_oid, oid_t index_oid, int64_t reads, int64_t deletes, - int64_t inserts, int64_t time_stamp, type::AbstractPool *pool, + oid_t index_oid, oid_t table_oid, int64_t reads, int64_t updates, + int64_t inserts, int64_t deletes, int64_t memory_alloc, + int64_t memory_usage, int64_t time_stamp, type::AbstractPool *pool, concurrency::TransactionContext *txn) { std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); + auto val0 = type::ValueFactory::GetIntegerValue(index_oid); auto val1 = type::ValueFactory::GetIntegerValue(table_oid); - auto val2 = type::ValueFactory::GetIntegerValue(index_oid); - auto val3 = type::ValueFactory::GetIntegerValue(reads); - auto val4 = type::ValueFactory::GetIntegerValue(deletes); - auto val5 = type::ValueFactory::GetIntegerValue(inserts); - auto val6 = type::ValueFactory::GetIntegerValue(time_stamp); + auto val2 = type::ValueFactory::GetBigIntValue(reads); + auto val3 = type::ValueFactory::GetBigIntValue(updates); + auto val4 = type::ValueFactory::GetBigIntValue(inserts); + auto val5 = type::ValueFactory::GetBigIntValue(deletes); + auto val6 = type::ValueFactory::GetBigIntValue(memory_alloc); + auto val7 = type::ValueFactory::GetBigIntValue(memory_usage); + auto val8 = type::ValueFactory::GetBigIntValue(time_stamp); + tuple->SetValue(ColumnId::INDEX_OID, val0, pool); tuple->SetValue(ColumnId::TABLE_OID, val1, pool); - tuple->SetValue(ColumnId::INDEX_OID, val2, pool); - tuple->SetValue(ColumnId::READS, val3, pool); - tuple->SetValue(ColumnId::DELETES, val4, pool); - tuple->SetValue(ColumnId::INSERTS, val5, pool); - tuple->SetValue(ColumnId::TIME_STAMP, val6, pool); + tuple->SetValue(ColumnId::READS, val2, pool); + tuple->SetValue(ColumnId::UPDATES, val3, pool); + tuple->SetValue(ColumnId::INSERTS, val4, pool); + tuple->SetValue(ColumnId::DELETES, val5, pool); + tuple->SetValue(ColumnId::MEMORY_ALLOC, val6, pool); + tuple->SetValue(ColumnId::MEMORY_USAGE, val7, pool); + tuple->SetValue(ColumnId::TIME_STAMP, val8, pool); // Insert the tuple return InsertTuple(std::move(tuple), txn); @@ -71,5 +106,62 @@ bool IndexMetricsCatalog::DeleteIndexMetrics( return DeleteWithIndexScan(index_offset, values, txn); } +bool IndexMetricsCatalog::UpdateIndexMetrics( + oid_t index_oid, oid_t table_oid, int64_t reads, int64_t updates, + int64_t inserts, int64_t deletes, int64_t memory_alloc, + int64_t memory_usage, int64_t time_stamp, + concurrency::TransactionContext *txn) { + std::vector update_columns(all_column_ids_); + std::vector update_values; + + update_values.push_back( + type::ValueFactory::GetIntegerValue(index_oid).Copy()); + update_values.push_back( + type::ValueFactory::GetIntegerValue(table_oid).Copy()); + update_values.push_back(type::ValueFactory::GetBigIntValue(reads).Copy()); + update_values.push_back(type::ValueFactory::GetBigIntValue(updates).Copy()); + update_values.push_back(type::ValueFactory::GetBigIntValue(inserts).Copy()); + update_values.push_back(type::ValueFactory::GetBigIntValue(deletes).Copy()); + update_values.push_back( + type::ValueFactory::GetBigIntValue(memory_alloc).Copy()); + update_values.push_back( + type::ValueFactory::GetBigIntValue(memory_usage).Copy()); + update_values.push_back( + type::ValueFactory::GetBigIntValue(time_stamp).Copy()); + + std::vector scan_values; + scan_values.push_back(type::ValueFactory::GetIntegerValue(index_oid)); + oid_t index_offset = IndexId::PRIMARY_KEY; + + // Update the tuple + return UpdateWithIndexScan(update_columns, update_values, scan_values, + index_offset, txn); +} + +std::shared_ptr +IndexMetricsCatalog::GetIndexMetricsObject( + oid_t index_oid, concurrency::TransactionContext *txn) { + if (txn == nullptr) { + throw CatalogException("Transaction is invalid!"); + } + + // set up read query + std::vector column_ids(all_column_ids_); + oid_t index_offset = IndexId::PRIMARY_KEY; + std::vector values; + values.push_back(type::ValueFactory::GetIntegerValue(index_oid).Copy()); + + auto result_tiles = + GetResultWithIndexScan(column_ids, index_offset, values, txn); + + if (result_tiles->size() == 1 && (*result_tiles)[0]->GetTupleCount() == 1) { + auto index_metric_object = + std::make_shared((*result_tiles)[0].get()); + return index_metric_object; + } + + return nullptr; +} + } // namespace catalog } // namespace peloton diff --git a/src/catalog/proc_catalog.cpp b/src/catalog/proc_catalog.cpp index 6da75db67fc..a477588ac30 100644 --- a/src/catalog/proc_catalog.cpp +++ b/src/catalog/proc_catalog.cpp @@ -21,8 +21,6 @@ namespace peloton { namespace catalog { -#define PROC_CATALOG_NAME "pg_proc" - ProcCatalogObject::ProcCatalogObject(executor::LogicalTile *tile, concurrency::TransactionContext *txn) : oid_(tile->GetValue(0, 0).GetAs()), diff --git a/src/catalog/query_history_catalog.cpp b/src/catalog/query_history_catalog.cpp index 4433197ba28..86edcd8d1a4 100644 --- a/src/catalog/query_history_catalog.cpp +++ b/src/catalog/query_history_catalog.cpp @@ -31,7 +31,8 @@ QueryHistoryCatalog::QueryHistoryCatalog(concurrency::TransactionContext *txn) " (" "query_string VARCHAR NOT NULL, " "fingerprint VARCHAR NOT NULL, " - "timestamp TIMESTAMP NOT NULL);", + "timestamp BIGINT NOT NULL," + "PRIMARY KEY(query_string, timestamp));", txn) {} QueryHistoryCatalog::~QueryHistoryCatalog() = default; @@ -45,7 +46,7 @@ bool QueryHistoryCatalog::InsertQueryHistory( auto val0 = type::ValueFactory::GetVarcharValue(query_string); auto val1 = type::ValueFactory::GetVarcharValue(fingerprint); - auto val2 = type::ValueFactory::GetTimestampValue(timestamp); + auto val2 = type::ValueFactory::GetBigIntValue(timestamp); tuple->SetValue(ColumnId::QUERY_STRING, val0, pool != nullptr ? pool : &pool_); diff --git a/src/catalog/query_metrics_catalog.cpp b/src/catalog/query_metrics_catalog.cpp deleted file mode 100644 index 0fd6b7c44d7..00000000000 --- a/src/catalog/query_metrics_catalog.cpp +++ /dev/null @@ -1,163 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// query_metrics_catalog.cpp -// -// Identification: src/catalog/query_metrics_catalog.cpp -// -// Copyright (c) 2015-17, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "catalog/query_metrics_catalog.h" - -#include "catalog/catalog.h" -#include "executor/logical_tile.h" -#include "storage/data_table.h" -#include "type/value_factory.h" - -namespace peloton { -namespace catalog { - -QueryMetricsCatalog::QueryMetricsCatalog(const std::string &database_name, - concurrency::TransactionContext *txn) - : AbstractCatalog("CREATE TABLE " + database_name + - "." CATALOG_SCHEMA_NAME "." QUERY_METRICS_CATALOG_NAME - " (" - "query_name VARCHAR NOT NULL PRIMARY KEY, " - "database_oid INT NOT NULL PRIMARY KEY, " - "num_params INT NOT NULL, " - "param_types VARBINARY, " - "param_formats VARBINARY, " - "param_values VARBINARY, " - "reads INT NOT NULL, " - "updates INT NOT NULL, " - "deletes INT NOT NULL, " - "inserts INT NOT NULL, " - "latency INT NOT NULL, " - "cpu_time INT NOT NULL, " - "time_stamp INT NOT NULL);", - txn) { - // Add secondary index here if necessary -} - -QueryMetricsCatalog::~QueryMetricsCatalog() {} - -bool QueryMetricsCatalog::InsertQueryMetrics( - const std::string &name, oid_t database_oid, int64_t num_params, - const stats::QueryMetric::QueryParamBuf &type_buf, - const stats::QueryMetric::QueryParamBuf &format_buf, - const stats::QueryMetric::QueryParamBuf &value_buf, int64_t reads, - int64_t updates, int64_t deletes, int64_t inserts, int64_t latency, - int64_t cpu_time, int64_t time_stamp, type::AbstractPool *pool, - concurrency::TransactionContext *txn) { - std::unique_ptr tuple( - new storage::Tuple(catalog_table_->GetSchema(), true)); - - auto val0 = type::ValueFactory::GetVarcharValue(name, pool); - auto val1 = type::ValueFactory::GetIntegerValue(database_oid); - auto val2 = type::ValueFactory::GetIntegerValue(num_params); - - auto val3 = type::ValueFactory::GetNullValueByType(type::TypeId::VARBINARY); - auto val4 = type::ValueFactory::GetNullValueByType(type::TypeId::VARBINARY); - auto val5 = type::ValueFactory::GetNullValueByType(type::TypeId::VARBINARY); - - if (num_params != 0) { - val3 = - type::ValueFactory::GetVarbinaryValue(type_buf.buf, type_buf.len, true); - val4 = type::ValueFactory::GetVarbinaryValue(format_buf.buf, format_buf.len, - true); - val5 = type::ValueFactory::GetVarbinaryValue(value_buf.buf, value_buf.len, - true); - } - - auto val6 = type::ValueFactory::GetIntegerValue(reads); - auto val7 = type::ValueFactory::GetIntegerValue(updates); - auto val8 = type::ValueFactory::GetIntegerValue(deletes); - auto val9 = type::ValueFactory::GetIntegerValue(inserts); - auto val10 = type::ValueFactory::GetIntegerValue(latency); - auto val11 = type::ValueFactory::GetIntegerValue(cpu_time); - auto val12 = type::ValueFactory::GetIntegerValue(time_stamp); - - tuple->SetValue(ColumnId::NAME, val0, pool); - tuple->SetValue(ColumnId::DATABASE_OID, val1, pool); - tuple->SetValue(ColumnId::NUM_PARAMS, val2, pool); - tuple->SetValue(ColumnId::PARAM_TYPES, val3, pool); - tuple->SetValue(ColumnId::PARAM_FORMATS, val4, pool); - tuple->SetValue(ColumnId::PARAM_VALUES, val5, pool); - tuple->SetValue(ColumnId::READS, val6, pool); - tuple->SetValue(ColumnId::UPDATES, val7, pool); - tuple->SetValue(ColumnId::DELETES, val8, pool); - tuple->SetValue(ColumnId::INSERTS, val9, pool); - tuple->SetValue(ColumnId::LATENCY, val10, pool); - tuple->SetValue(ColumnId::CPU_TIME, val11, pool); - tuple->SetValue(ColumnId::TIME_STAMP, val12, pool); - - // Insert the tuple - return InsertTuple(std::move(tuple), txn); -} - -bool QueryMetricsCatalog::DeleteQueryMetrics( - const std::string &name, concurrency::TransactionContext *txn) { - oid_t index_offset = IndexId::PRIMARY_KEY; // Primary key index - - std::vector values; - values.push_back(type::ValueFactory::GetVarcharValue(name, nullptr).Copy()); - values.push_back(type::ValueFactory::GetIntegerValue(database_oid).Copy()); - - return DeleteWithIndexScan(index_offset, values, txn); -} - -stats::QueryMetric::QueryParamBuf QueryMetricsCatalog::GetParamTypes( - const std::string &name, concurrency::TransactionContext *txn) { - std::vector column_ids({ColumnId::PARAM_TYPES}); // param_types - oid_t index_offset = IndexId::PRIMARY_KEY; // Primary key index - std::vector values; - values.push_back(type::ValueFactory::GetVarcharValue(name, nullptr).Copy()); - values.push_back(type::ValueFactory::GetIntegerValue(database_oid).Copy()); - - auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); - - stats::QueryMetric::QueryParamBuf param_types; - PELOTON_ASSERT(result_tiles->size() <= 1); // unique - if (result_tiles->size() != 0) { - PELOTON_ASSERT((*result_tiles)[0]->GetTupleCount() <= 1); - if ((*result_tiles)[0]->GetTupleCount() != 0) { - auto param_types_value = (*result_tiles)[0]->GetValue(0, 0); - param_types.buf = const_cast( - reinterpret_cast(param_types_value.GetData())); - param_types.len = param_types_value.GetLength(); - } - } - - return param_types; -} - -int64_t QueryMetricsCatalog::GetNumParams( - const std::string &name, concurrency::TransactionContext *txn) { - std::vector column_ids({ColumnId::NUM_PARAMS}); // num_params - oid_t index_offset = IndexId::PRIMARY_KEY; // Primary key index - std::vector values; - values.push_back(type::ValueFactory::GetVarcharValue(name, nullptr).Copy()); - - auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); - - int64_t num_params = 0; - PELOTON_ASSERT(result_tiles->size() <= 1); // unique - if (result_tiles->size() != 0) { - PELOTON_ASSERT((*result_tiles)[0]->GetTupleCount() <= 1); - if ((*result_tiles)[0]->GetTupleCount() != 0) { - num_params = (*result_tiles)[0] - ->GetValue(0, 0) - .GetAs(); // After projection left 1 column - } - } - - return num_params; -} - -} // namespace catalog -} // namespace peloton diff --git a/src/catalog/settings_catalog.cpp b/src/catalog/settings_catalog.cpp index bcdc518af24..d19524b9d01 100644 --- a/src/catalog/settings_catalog.cpp +++ b/src/catalog/settings_catalog.cpp @@ -16,8 +16,6 @@ #include "storage/data_table.h" #include "type/value_factory.h" -#define SETTINGS_CATALOG_NAME "pg_settings" - namespace peloton { namespace catalog { @@ -31,7 +29,7 @@ SettingsCatalog::SettingsCatalog(concurrency::TransactionContext *txn) : AbstractCatalog("CREATE TABLE " CATALOG_DATABASE_NAME "." CATALOG_SCHEMA_NAME "." SETTINGS_CATALOG_NAME " (" - "name VARCHAR NOT NULL, " + "name VARCHAR NOT NULL PRIMARY KEY, " "value VARCHAR NOT NULL, " "value_type VARCHAR NOT NULL, " "description VARCHAR, " diff --git a/src/catalog/system_catalogs.cpp b/src/catalog/system_catalogs.cpp index b1371ddd379..b9054aab2ea 100644 --- a/src/catalog/system_catalogs.cpp +++ b/src/catalog/system_catalogs.cpp @@ -31,8 +31,7 @@ SystemCatalogs::SystemCatalogs(storage::Database *database, concurrency::TransactionContext *txn) : pg_trigger_(nullptr), pg_table_metrics_(nullptr), - pg_index_metrics_(nullptr), - pg_query_metrics_(nullptr) { + pg_index_metrics_(nullptr) { oid_t database_oid = database->GetOid(); pg_attribute_ = new ColumnCatalog(database, pool, txn); pg_namespace_ = new SchemaCatalog(database, pool, txn); @@ -72,7 +71,6 @@ SystemCatalogs::~SystemCatalogs() { // if (pg_proc) delete pg_proc; if (pg_table_metrics_) delete pg_table_metrics_; if (pg_index_metrics_) delete pg_index_metrics_; - if (pg_query_metrics_) delete pg_query_metrics_; } /*@brief using sql create statement to create secondary catalog tables @@ -98,10 +96,6 @@ void SystemCatalogs::Bootstrap(const std::string &database_name, if (!pg_index_metrics_) { pg_index_metrics_ = new IndexMetricsCatalog(database_name, txn); } - - if (!pg_query_metrics_) { - pg_query_metrics_ = new QueryMetricsCatalog(database_name, txn); - } } } // namespace catalog diff --git a/src/catalog/table_metrics_catalog.cpp b/src/catalog/table_metrics_catalog.cpp index f1f91f46474..bd1491b1a8e 100644 --- a/src/catalog/table_metrics_catalog.cpp +++ b/src/catalog/table_metrics_catalog.cpp @@ -19,17 +19,54 @@ namespace peloton { namespace catalog { +TableMetricsCatalogObject::TableMetricsCatalogObject( + executor::LogicalTile *tile, int tupleId) + : table_oid_( + tile->GetValue(tupleId, TableMetricsCatalog::ColumnId::TABLE_OID) + .GetAs()), + reads_(tile->GetValue(tupleId, TableMetricsCatalog::ColumnId::READS) + .GetAs()), + updates_(tile->GetValue(tupleId, TableMetricsCatalog::ColumnId::UPDATES) + .GetAs()), + inserts_(tile->GetValue(tupleId, TableMetricsCatalog::ColumnId::INSERTS) + .GetAs()), + deletes_(tile->GetValue(tupleId, TableMetricsCatalog::ColumnId::DELETES) + .GetAs()), + inline_memory_alloc_( + tile->GetValue(tupleId, + TableMetricsCatalog::ColumnId::INLINE_MEMORY_ALLOC) + .GetAs()), + inline_memory_usage_( + tile->GetValue(tupleId, + TableMetricsCatalog::ColumnId::INLINE_MEMORY_USAGE) + .GetAs()), + varlen_memory_alloc_( + tile->GetValue(tupleId, + TableMetricsCatalog::ColumnId::VARLEN_MEMORY_ALLOC) + .GetAs()), + varlen_memory_usage_( + tile->GetValue(tupleId, + TableMetricsCatalog::ColumnId::VARLEN_MEMORY_USAGE) + .GetAs()), + time_stamp_( + tile->GetValue(tupleId, TableMetricsCatalog::ColumnId::TIME_STAMP) + .GetAs()) {} + TableMetricsCatalog::TableMetricsCatalog(const std::string &database_name, concurrency::TransactionContext *txn) : AbstractCatalog("CREATE TABLE " + database_name + "." CATALOG_SCHEMA_NAME "." TABLE_METRICS_CATALOG_NAME " (" - "table_oid INT NOT NULL, " - "reads INT NOT NULL, " - "updates INT NOT NULL, " - "deletes INT NOT NULL, " - "inserts INT NOT NULL, " - "time_stamp INT NOT NULL);", + "table_oid INT NOT NULL PRIMARY KEY, " + "reads BIGINT NOT NULL, " + "updates BIGINT NOT NULL, " + "inserts BIGINT NOT NULL, " + "deletes BIGINT NOT NULL, " + "inline_memory_alloc BIGINT NOT NULL, " + "inline_memory_usage BIGINT NOT NULL, " + "varlen_memory_alloc BIGINT NOT NULL, " + "varlen_memory_usage BIGINT NOT NULL, " + "time_stamp BIGINT NOT NULL);", txn) { // Add secondary index here if necessary } @@ -37,25 +74,35 @@ TableMetricsCatalog::TableMetricsCatalog(const std::string &database_name, TableMetricsCatalog::~TableMetricsCatalog() {} bool TableMetricsCatalog::InsertTableMetrics( - oid_t table_oid, int64_t reads, int64_t updates, int64_t deletes, - int64_t inserts, int64_t time_stamp, type::AbstractPool *pool, + oid_t table_oid, int64_t reads, int64_t updates, int64_t inserts, + int64_t deletes, int64_t inline_memory_alloc, int64_t inline_memory_usage, + int64_t varlen_memory_alloc, int64_t varlen_memory_usage, + int64_t time_stamp, type::AbstractPool *pool, concurrency::TransactionContext *txn) { std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); auto val1 = type::ValueFactory::GetIntegerValue(table_oid); - auto val2 = type::ValueFactory::GetIntegerValue(reads); - auto val3 = type::ValueFactory::GetIntegerValue(updates); - auto val4 = type::ValueFactory::GetIntegerValue(deletes); - auto val5 = type::ValueFactory::GetIntegerValue(inserts); - auto val6 = type::ValueFactory::GetIntegerValue(time_stamp); + auto val2 = type::ValueFactory::GetBigIntValue(reads); + auto val3 = type::ValueFactory::GetBigIntValue(updates); + auto val4 = type::ValueFactory::GetBigIntValue(inserts); + auto val5 = type::ValueFactory::GetBigIntValue(deletes); + auto val6 = type::ValueFactory::GetBigIntValue(inline_memory_alloc); + auto val7 = type::ValueFactory::GetBigIntValue(inline_memory_usage); + auto val8 = type::ValueFactory::GetBigIntValue(varlen_memory_alloc); + auto val9 = type::ValueFactory::GetBigIntValue(varlen_memory_usage); + auto val10 = type::ValueFactory::GetBigIntValue(time_stamp); tuple->SetValue(ColumnId::TABLE_OID, val1, pool); tuple->SetValue(ColumnId::READS, val2, pool); tuple->SetValue(ColumnId::UPDATES, val3, pool); - tuple->SetValue(ColumnId::DELETES, val4, pool); - tuple->SetValue(ColumnId::INSERTS, val5, pool); - tuple->SetValue(ColumnId::TIME_STAMP, val6, pool); + tuple->SetValue(ColumnId::INSERTS, val4, pool); + tuple->SetValue(ColumnId::DELETES, val5, pool); + tuple->SetValue(ColumnId::INLINE_MEMORY_ALLOC, val6, pool); + tuple->SetValue(ColumnId::INLINE_MEMORY_USAGE, val7, pool); + tuple->SetValue(ColumnId::VARLEN_MEMORY_ALLOC, val8, pool); + tuple->SetValue(ColumnId::VARLEN_MEMORY_USAGE, val9, pool); + tuple->SetValue(ColumnId::TIME_STAMP, val10, pool); // Insert the tuple return InsertTuple(std::move(tuple), txn); @@ -71,5 +118,64 @@ bool TableMetricsCatalog::DeleteTableMetrics( return DeleteWithIndexScan(index_offset, values, txn); } +bool TableMetricsCatalog::UpdateTableMetrics( + oid_t table_oid, int64_t reads, int64_t updates, int64_t inserts, + int64_t deletes, int64_t inline_memory_alloc, int64_t inline_memory_usage, + int64_t varlen_memory_alloc, int64_t varlen_memory_usage, + int64_t time_stamp, concurrency::TransactionContext *txn) { + std::vector update_columns(all_column_ids_); + std::vector update_values; + + update_values.push_back( + type::ValueFactory::GetIntegerValue(table_oid).Copy()); + update_values.push_back(type::ValueFactory::GetBigIntValue(reads).Copy()); + update_values.push_back(type::ValueFactory::GetBigIntValue(updates).Copy()); + update_values.push_back(type::ValueFactory::GetBigIntValue(inserts).Copy()); + update_values.push_back(type::ValueFactory::GetBigIntValue(deletes).Copy()); + update_values.push_back( + type::ValueFactory::GetBigIntValue(inline_memory_alloc).Copy()); + update_values.push_back( + type::ValueFactory::GetBigIntValue(inline_memory_usage).Copy()); + update_values.push_back( + type::ValueFactory::GetBigIntValue(varlen_memory_alloc).Copy()); + update_values.push_back( + type::ValueFactory::GetBigIntValue(varlen_memory_usage).Copy()); + update_values.push_back( + type::ValueFactory::GetBigIntValue(time_stamp).Copy()); + + std::vector scan_values; + scan_values.push_back(type::ValueFactory::GetIntegerValue(table_oid)); + oid_t index_offset = IndexId::PRIMARY_KEY; + + // Update the tuple + return UpdateWithIndexScan(update_columns, update_values, scan_values, + index_offset, txn); +} + +std::shared_ptr +TableMetricsCatalog::GetTableMetricsObject( + oid_t table_oid, concurrency::TransactionContext *txn) { + if (txn == nullptr) { + throw CatalogException("Transaction is invalid!"); + } + + // set up read query + std::vector column_ids(all_column_ids_); + oid_t index_offset = IndexId::PRIMARY_KEY; + std::vector values; + values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); + + auto result_tiles = + GetResultWithIndexScan(column_ids, index_offset, values, txn); + + if (result_tiles->size() == 1 && (*result_tiles)[0]->GetTupleCount() == 1) { + auto table_metric_object = + std::make_shared((*result_tiles)[0].get()); + return table_metric_object; + } + + return nullptr; +} + } // namespace catalog } // namespace peloton diff --git a/src/catalog/tuple_access_metrics_catalog.cpp b/src/catalog/tuple_access_metrics_catalog.cpp new file mode 100644 index 00000000000..9cbc4f4f599 --- /dev/null +++ b/src/catalog/tuple_access_metrics_catalog.cpp @@ -0,0 +1,118 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// table_metrics_catalog.cpp +// +// Identification: src/catalog/tuple_access_catalog.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "catalog/tuple_access_metrics_catalog.h" + +#include +#include "executor/logical_tile.h" +#include "storage/data_table.h" +#include "type/value_factory.h" + +namespace peloton { +namespace catalog { + +TupleAccessMetricsCatalogObject::TupleAccessMetricsCatalogObject(executor::LogicalTile *tile, + oid_t tupleId) + : tid_(tile->GetValue(tupleId, TupleAccessMetricsCatalog::ColumnId::TXN_ID).GetAs()), + reads_(tile->GetValue(tupleId, + TupleAccessMetricsCatalog::ColumnId::READS) + .GetAs()), + valid_( + tile->GetValue(tupleId, TupleAccessMetricsCatalog::ColumnId::VALID) + .GetAs()), + committed_( + tile->GetValue(tupleId, + TupleAccessMetricsCatalog::ColumnId::COMMITTED) + .GetAs()) {} + +TupleAccessMetricsCatalog::TupleAccessMetricsCatalog(concurrency::TransactionContext *txn) + : AbstractCatalog("CREATE TABLE " CATALOG_DATABASE_NAME + "." CATALOG_SCHEMA_NAME "." TUPLE_ACCESS_METRICS_CATALOG_NAME + " (" + "txn_id BIGINT NOT NULL PRIMARY KEY, " + "reads BIGINT NOT NULL, " + "valid BOOL NOT NULL, " + "committed BOOL NOT NULL);", + txn) {} + +bool TupleAccessMetricsCatalog::InsertAccessMetric(peloton::txn_id_t tid, + uint64_t reads, + bool valid, + bool committed, + peloton::type::AbstractPool *pool, + peloton::concurrency::TransactionContext *txn) { + std::unique_ptr + tuple(new storage::Tuple(catalog_table_->GetSchema(), true)); + + tuple->SetValue(ColumnId::TXN_ID, + type::ValueFactory::GetBigIntValue(tid), + pool); + tuple->SetValue(ColumnId::READS, + type::ValueFactory::GetBigIntValue(reads), + pool); + tuple->SetValue(ColumnId::VALID, + type::ValueFactory::GetBooleanValue(valid), + pool); + tuple->SetValue(ColumnId::COMMITTED, + type::ValueFactory::GetBooleanValue(committed), + pool); + return InsertTuple(std::move(tuple), txn); +} + +bool TupleAccessMetricsCatalog::DeleteAccessMetrics(peloton::txn_id_t tid, + peloton::concurrency::TransactionContext *txn) { + oid_t index_offset = IndexId::PRIMARY_KEY; + std::vector values; + values.push_back(type::ValueFactory::GetBigIntValue(tid).Copy()); + return DeleteWithIndexScan(index_offset, values, txn); +} + +bool TupleAccessMetricsCatalog::UpdateAccessMetrics(peloton::txn_id_t tid, + uint64_t reads, + bool valid, + bool committed, + peloton::concurrency::TransactionContext *txn) { + std::vector update_columns(all_column_ids_); + std::vector update_values; + update_values.push_back(type::ValueFactory::GetBigIntValue(tid).Copy()); + update_values.push_back(type::ValueFactory::GetBigIntValue(reads).Copy()); + update_values.push_back(type::ValueFactory::GetBooleanValue(valid).Copy()); + update_values.push_back(type::ValueFactory::GetBooleanValue(committed).Copy()); + + std::vector scan_values; + scan_values.push_back(type::ValueFactory::GetBigIntValue(tid).Copy()); + oid_t index_offset = IndexId::PRIMARY_KEY; + return UpdateWithIndexScan(update_columns, + update_values, + scan_values, + index_offset, + txn); +} + +std::shared_ptr TupleAccessMetricsCatalog::GetTupleAccessMetricsCatalogObject( + txn_id_t tid, + concurrency::TransactionContext *txn) { + if (txn == nullptr) throw CatalogException("Invalid Transaction"); + + std::vector column_ids(all_column_ids_); + oid_t index_offset = IndexId::PRIMARY_KEY; + std::vector values; + values.push_back(type::ValueFactory::GetBigIntValue(tid).Copy()); + + auto result_tiles = + GetResultWithIndexScan(column_ids, index_offset, values, txn); + if (result_tiles->size() == 1 && (*result_tiles)[0]->GetTupleCount() == 1) + return std::make_shared((*result_tiles)[0].get()); + return nullptr; +} +} // namespace catalog +} // namespace peloton diff --git a/src/catalog/zone_map_catalog.cpp b/src/catalog/zone_map_catalog.cpp index ec59dd24d82..751f00fd75b 100644 --- a/src/catalog/zone_map_catalog.cpp +++ b/src/catalog/zone_map_catalog.cpp @@ -42,7 +42,8 @@ ZoneMapCatalog::ZoneMapCatalog(concurrency::TransactionContext *txn) "column_id INT NOT NULL, " "minimum VARCHAR, " "maximum VARCHAR, " - "type VARCHAR);", + "type VARCHAR, " + "PRIMARY KEY(database_id, table_id, tile_group_id, column_id));", txn) { Catalog::GetInstance()->CreateIndex( CATALOG_DATABASE_NAME, CATALOG_SCHEMA_NAME, ZONE_MAP_CATALOG_NAME, diff --git a/src/common/container/circular_buffer.cpp b/src/common/container/circular_buffer.cpp index 775d63e5a14..7ae605317fc 100644 --- a/src/common/container/circular_buffer.cpp +++ b/src/common/container/circular_buffer.cpp @@ -22,7 +22,7 @@ void CIRCULAR_BUFFER_TYPE::PushBack(ValueType value) { // Set the container capaciry CIRCULAR_BUFFER_TEMPLATE_ARGUMENTS -void CIRCULAR_BUFFER_TYPE::SetCapaciry(size_t new_capacity) { +void CIRCULAR_BUFFER_TYPE::SetCapacity(size_t new_capacity) { circular_buffer_.set_capacity(new_capacity); } diff --git a/src/common/container/cuckoo_map.cpp b/src/common/container/cuckoo_map.cpp index c89e7a945e7..acdde8de8b4 100644 --- a/src/common/container/cuckoo_map.cpp +++ b/src/common/container/cuckoo_map.cpp @@ -27,7 +27,7 @@ class TileGroup; namespace stats { class BackendStatsContext; -class IndexMetric; +class IndexMetricOld; } // namespace stats class StatementCache; @@ -110,7 +110,7 @@ template class CuckooMap>; template class CuckooMap>; -template class CuckooMap>; +template class CuckooMap>; // Used in SharedPointerKeyTest template class CuckooMap, std::shared_ptr>; @@ -122,4 +122,7 @@ template class CuckooMap; template class CuckooMap; +// Used in StatsCollector +template class CuckooMap *>; + } // namespace peloton diff --git a/src/common/container/lock_free_array.cpp b/src/common/container/lock_free_array.cpp index cde4466f213..625868d762c 100644 --- a/src/common/container/lock_free_array.cpp +++ b/src/common/container/lock_free_array.cpp @@ -13,9 +13,9 @@ #include #include "common/container/lock_free_array.h" +#include "common/internal_types.h" #include "common/logger.h" #include "common/macros.h" -#include "common/internal_types.h" namespace peloton { @@ -27,7 +27,7 @@ namespace storage { class TileGroup; class Database; class IndirectionArray; -} +} // namespace storage template LOCK_FREE_ARRAY_TYPE::LockFreeArray() { @@ -35,7 +35,10 @@ LOCK_FREE_ARRAY_TYPE::LockFreeArray() { } template -LOCK_FREE_ARRAY_TYPE::~LockFreeArray() { lock_free_array.clear(); } +LOCK_FREE_ARRAY_TYPE::~LockFreeArray() { + lock_free_array.clear(); + lock_free_array.shrink_to_fit(); +} template bool LOCK_FREE_ARRAY_TYPE::Update(const std::size_t &offset, ValueType value) { @@ -93,13 +96,19 @@ ValueType LOCK_FREE_ARRAY_TYPE::FindValid( } template -size_t LOCK_FREE_ARRAY_TYPE::GetSize() const { return lock_free_array.size(); } +size_t LOCK_FREE_ARRAY_TYPE::GetSize() const { + return lock_free_array.size(); +} template -bool LOCK_FREE_ARRAY_TYPE::IsEmpty() const { return lock_free_array.empty(); } +bool LOCK_FREE_ARRAY_TYPE::IsEmpty() const { + return lock_free_array.empty(); +} template -void LOCK_FREE_ARRAY_TYPE::Clear() { lock_free_array.clear(); } +void LOCK_FREE_ARRAY_TYPE::Clear() { + lock_free_array.clear(); +} template bool LOCK_FREE_ARRAY_TYPE::Contains(const ValueType &value) { @@ -107,7 +116,7 @@ bool LOCK_FREE_ARRAY_TYPE::Contains(const ValueType &value) { for (std::size_t array_itr = 0; array_itr < lock_free_array.size(); array_itr++) { - auto array_value = lock_free_array.at(array_itr); + auto &array_value = lock_free_array.at(array_itr); // Check array value if (array_value == value) { exists = true; diff --git a/src/common/init.cpp b/src/common/init.cpp index c4ab64adc51..ddc1efff100 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -22,6 +22,7 @@ #include "gc/gc_manager_factory.h" #include "index/index.h" #include "settings/settings_manager.h" +#include "storage/data_table.h" #include "threadpool/mono_queue_pool.h" #include "tuning/index_tuner.h" #include "tuning/layout_tuner.h" @@ -56,7 +57,8 @@ void PelotonInit::Initialize() { concurrency::EpochManagerFactory::GetInstance().StartEpoch(); // start GC. - gc::GCManagerFactory::Configure(settings::SettingsManager::GetInt(settings::SettingId::gc_num_threads)); + gc::GCManagerFactory::Configure( + settings::SettingsManager::GetInt(settings::SettingId::gc_num_threads)); gc::GCManagerFactory::GetInstance().StartGC(); // start index tuner diff --git a/src/common/portal.cpp b/src/common/portal.cpp index 77de6522f50..a5aa7754ba4 100644 --- a/src/common/portal.cpp +++ b/src/common/portal.cpp @@ -18,12 +18,10 @@ namespace peloton { Portal::Portal(const std::string& portal_name, std::shared_ptr statement, - std::vector bind_parameters, - std::shared_ptr param_stat) + std::vector bind_parameters) : portal_name_(portal_name), statement_(statement), - bind_parameters_(std::move(bind_parameters)), - param_stat_(param_stat) {} + bind_parameters_(std::move(bind_parameters)) {} Portal::~Portal() { statement_.reset(); } diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index c86c53292e0..5710482f23f 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -22,6 +22,7 @@ #include "gc/gc_manager_factory.h" #include "logging/log_manager_factory.h" #include "settings/settings_manager.h" +#include "statistics/thread_level_stats_collector.h" namespace peloton { namespace concurrency { @@ -180,6 +181,9 @@ bool TimestampOrderingTransactionManager::PerformRead( ////////////////////////////////////////////////////////// if (current_txn->GetIsolationLevel() == IsolationLevelType::READ_ONLY) { // do not update read set for read-only transactions. + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectTupleRead( + current_txn, + location.block); return true; } // end READ ONLY @@ -227,25 +231,17 @@ bool TimestampOrderingTransactionManager::PerformRead( PELOTON_ASSERT(IsOwner(current_txn, tile_group_header, tuple_id) == true); // Increment table read op stats - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTableReads( - location.block); - } - + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectTupleRead(current_txn, tile_group_id); return true; - } else { // if it's not select for update, then update read set and return true. current_txn->RecordRead(location); // Increment table read op stats - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTableReads( - location.block); - } + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectTupleRead(current_txn, tile_group_id); return true; } @@ -283,12 +279,10 @@ bool TimestampOrderingTransactionManager::PerformRead( } // if we have already owned the version. PELOTON_ASSERT(IsOwner(current_txn, tile_group_header, tuple_id) == true); + // Increment table read op stats - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTableReads( - location.block); - } + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectTupleRead(current_txn, location.block); return true; } else { @@ -298,11 +292,8 @@ bool TimestampOrderingTransactionManager::PerformRead( current_txn->RecordRead(location); // Increment table read op stats - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTableReads( - location.block); - } + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectTupleRead(current_txn, location.block); return true; } else { @@ -318,11 +309,8 @@ bool TimestampOrderingTransactionManager::PerformRead( // current_txn->RecordRead(location); // Increment table read op stats - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTableReads( - location.block); - } + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectTupleRead(current_txn, location.block); return true; } } @@ -380,11 +368,8 @@ bool TimestampOrderingTransactionManager::PerformRead( current_txn->GetCommitId() || GetLastReaderCommitId(tile_group_header, tuple_id) == 0); // Increment table read op stats - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTableReads( - location.block); - } + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectTupleRead(current_txn, location.block); return true; } else { @@ -397,11 +382,8 @@ bool TimestampOrderingTransactionManager::PerformRead( current_txn->RecordRead(location); // Increment table read op stats - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTableReads( - location.block); - } + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectTupleRead(current_txn, location.block); return true; } else { // if the tuple has been owned by some concurrent transactions, @@ -422,11 +404,8 @@ bool TimestampOrderingTransactionManager::PerformRead( // current_txn->RecordRead(location); // Increment table read op stats - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTableReads( - location.block); - } + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectTupleRead(current_txn, location.block); return true; } } @@ -467,11 +446,9 @@ void TimestampOrderingTransactionManager::PerformInsert( tile_group_header->SetIndirection(tuple_id, index_entry_ptr); // Increment table insert op stats - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTableInserts( - location.block); - } + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectTupleInsert( + current_txn, + location.block); } void TimestampOrderingTransactionManager::PerformUpdate( @@ -499,9 +476,7 @@ void TimestampOrderingTransactionManager::PerformUpdate( // version. PELOTON_ASSERT(tile_group_header->GetTransactionId(old_location.offset) == transaction_id); - PELOTON_ASSERT( - tile_group_header->GetPrevItemPointer(old_location.offset).IsNull() == - true); + PELOTON_ASSERT(tile_group_header->GetPrevItemPointer(old_location.offset).IsNull()); // check whether the new version is empty. PELOTON_ASSERT(new_tile_group_header->GetTransactionId(new_location.offset) == @@ -552,11 +527,9 @@ void TimestampOrderingTransactionManager::PerformUpdate( current_txn->RecordUpdate(old_location); // Increment table update op stats - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTableUpdates( - new_location.block); - } + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectTupleUpdate( + current_txn, + location.block); } void TimestampOrderingTransactionManager::PerformUpdate( @@ -586,11 +559,9 @@ void TimestampOrderingTransactionManager::PerformUpdate( // in this case, nothing needs to be performed. // Increment table update op stats - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTableUpdates( - location.block); - } + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectTupleUpdate( + current_txn, + location.block); } void TimestampOrderingTransactionManager::PerformDelete( @@ -624,9 +595,7 @@ void TimestampOrderingTransactionManager::PerformDelete( PELOTON_ASSERT(tile_group_header->GetTransactionId(old_location.offset) == transaction_id); // we must be deleting the latest version. - PELOTON_ASSERT( - tile_group_header->GetPrevItemPointer(old_location.offset).IsNull() == - true); + PELOTON_ASSERT(tile_group_header->GetPrevItemPointer(old_location.offset).IsNull()); // check whether the new version is empty. PELOTON_ASSERT(new_tile_group_header->GetTransactionId(new_location.offset) == @@ -675,11 +644,9 @@ void TimestampOrderingTransactionManager::PerformDelete( current_txn->RecordDelete(old_location); // Increment table delete op stats - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTableDeletes( - old_location.block); - } + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectTupleDelete( + current_txn, + old_location.block); } void TimestampOrderingTransactionManager::PerformDelete( @@ -710,11 +677,9 @@ void TimestampOrderingTransactionManager::PerformDelete( } // Increment table delete op stats - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTableDeletes( - location.block); - } + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectTupleDelete( + current_txn, + old_location.block); } ResultType TimestampOrderingTransactionManager::CommitTransaction( @@ -744,7 +709,6 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( auto &rw_set = current_txn->GetReadWriteSet(); auto &rw_object_set = current_txn->GetCreateDropSet(); - auto gc_set = current_txn->GetGCSetPtr(); auto gc_object_set = current_txn->GetGCObjectSetPtr(); @@ -757,19 +721,10 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( gc_object_set->emplace_back(database_oid, table_oid, index_oid); } - oid_t database_id = 0; - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - for (const auto &tuple_entry : rw_set.GetConstIterator()) { - // Call the GetConstIterator() function to explicitly lock the cuckoohash - // and initilaize the iterator - const auto tile_group_id = tuple_entry.first.block; - database_id = manager.GetTileGroup(tile_group_id)->GetDatabaseId(); - if (database_id != CATALOG_DATABASE_OID) { - break; - } - } - } + // Sine we have to iterate rw_set in the function anyway, it does not + // matter to peek the first element of rw_set. + oid_t stats_tile_group_id = INVALID_OID; + // install everything. // 1. install a new version for update operations; @@ -782,6 +737,7 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( for (const auto &tuple_entry : rw_set.GetConstIterator()) { ItemPointer item_ptr = tuple_entry.first; oid_t tile_group_id = item_ptr.block; + stats_tile_group_id = tile_group_id; oid_t tuple_slot = item_ptr.offset; auto tile_group_header = manager.GetTileGroup(tile_group_id)->GetHeader(); @@ -898,16 +854,11 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( ResultType result = current_txn->GetResult(); log_manager.LogEnd(); - - EndTransaction(current_txn); - // Increment # txns committed metric - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTxnCommitted( - database_id); - } + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectTransactionCommit(current_txn, stats_tile_group_id); + EndTransaction(current_txn); return result; } @@ -936,19 +887,7 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( gc_object_set->emplace_back(database_oid, table_oid, index_oid); } - oid_t database_id = 0; - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - for (const auto &tuple_entry : rw_set.GetConstIterator()) { - // Call the GetConstIterator() function to explicitly lock the cuckoohash - // and initilaize the iterator - const auto tile_group_id = tuple_entry.first.block; - database_id = manager.GetTileGroup(tile_group_id)->GetDatabaseId(); - if (database_id != CATALOG_DATABASE_OID) { - break; - } - } - } + oid_t stats_tile_group_id = INVALID_OID; // Iterate through each item pointer in the read write set // TODO (Pooja): This might be inefficient since we will have to get the @@ -956,6 +895,7 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( for (const auto &tuple_entry : rw_set.GetConstIterator()) { ItemPointer item_ptr = tuple_entry.first; oid_t tile_group_id = item_ptr.block; + stats_tile_group_id = tile_group_id; oid_t tuple_slot = item_ptr.offset; auto tile_group_header = manager.GetTileGroup(tile_group_id)->GetHeader(); @@ -1089,14 +1029,10 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( } current_txn->SetResult(ResultType::ABORTED); + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectTransactionAbort(current_txn, stats_tile_group_id); EndTransaction(current_txn); - // Increment # txns aborted metric - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementTxnAborted(database_id); - } - return ResultType::ABORTED; } diff --git a/src/concurrency/transaction_manager.cpp b/src/concurrency/transaction_manager.cpp index 93c2c6e8f1f..2f086a18a0d 100644 --- a/src/concurrency/transaction_manager.cpp +++ b/src/concurrency/transaction_manager.cpp @@ -18,8 +18,8 @@ #include "gc/gc_manager_factory.h" #include "logging/log_manager.h" #include "settings/settings_manager.h" -#include "statistics/stats_aggregator.h" #include "storage/tile_group.h" +#include "statistics/thread_level_stats_collector.h" namespace peloton { namespace concurrency { @@ -66,15 +66,7 @@ TransactionContext *TransactionManager::BeginTransaction( txn = new TransactionContext(thread_id, type, read_id); } - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance() - ->GetTxnLatencyMetric() - .StartTimer(); - } - txn->SetTimestamp(function::DateFunctions::Now()); - return txn; } @@ -84,21 +76,13 @@ void TransactionManager::EndTransaction(TransactionContext *current_txn) { current_txn->ExecOnCommitTriggers(); } - if(gc::GCManagerFactory::GetGCType() == GarbageCollectionType::ON) { + if (gc::GCManagerFactory::GetGCType() == GarbageCollectionType::ON) { gc::GCManagerFactory::GetInstance().RecycleTransaction(current_txn); } else { delete current_txn; } current_txn = nullptr; - - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance() - ->GetTxnLatencyMetric() - .RecordLatency(); - - } } // this function checks whether a concurrent transaction is inserting the same @@ -106,7 +90,7 @@ void TransactionManager::EndTransaction(TransactionContext *current_txn) { // that is to-be-inserted by the current transaction. bool TransactionManager::IsOccupied(TransactionContext *const current_txn, const void *position_ptr) { - ItemPointer &position = *((ItemPointer *)position_ptr); + ItemPointer &position = *((ItemPointer *) position_ptr); auto tile_group_header = catalog::Manager::GetInstance().GetTileGroup(position.block)->GetHeader(); @@ -219,7 +203,7 @@ VisibilityType TransactionManager::IsVisible( // the only version that is visible is the newly inserted/updated one. return VisibilityType::OK; } else if (current_txn->GetRWType(ItemPointer(tile_group_id, tuple_id)) == - RWType::READ_OWN) { + RWType::READ_OWN) { // the ownership is from a select-for-update read operation return VisibilityType::OK; } else if (tuple_end_cid == INVALID_CID) { diff --git a/src/executor/plan_executor.cpp b/src/executor/plan_executor.cpp index 104aff1351c..3e8d6774933 100644 --- a/src/executor/plan_executor.cpp +++ b/src/executor/plan_executor.cpp @@ -64,7 +64,7 @@ static void CompileAndExecutePlan( } auto on_query_result = - [&on_complete, &consumer, plan](executor::ExecutionResult result) { + [&on_complete, &consumer, plan](executor::ExecutionResult result) { std::vector values; for (const auto &tuple : consumer.GetOutputTuples()) { for (uint32_t i = 0; i < tuple.tuple_.size(); i++) { diff --git a/src/gc/transaction_level_gc_manager.cpp b/src/gc/transaction_level_gc_manager.cpp index c2274a330e0..79a661b9c79 100644 --- a/src/gc/transaction_level_gc_manager.cpp +++ b/src/gc/transaction_level_gc_manager.cpp @@ -131,7 +131,7 @@ int TransactionLevelGCManager::Unlink(const int &thread_id, } // Log the query into query_history_catalog - if (settings::SettingsManager::GetBool(settings::SettingId::brain)) { + if (settings::SettingsManager::GetBool(settings::SettingId::brain_data_collection)) { std::vector query_strings = txn_ctx->GetQueryStrings(); if (query_strings.size() != 0) { uint64_t timestamp = txn_ctx->GetTimestamp(); diff --git a/src/include/brain/brain.h b/src/include/brain/brain.h index 6614767423b..4a55e6b5ee0 100644 --- a/src/include/brain/brain.h +++ b/src/include/brain/brain.h @@ -16,6 +16,9 @@ #include #include #include +#include +#include "settings/setting_id.h" +#include "settings/settings_manager.h" #include "capnp/ez-rpc.h" #include "peloton/capnp/peloton_service.capnp.h" #include "common/notifiable_task.h" @@ -28,7 +31,28 @@ namespace brain { * the brain, such as RPC and Catalog. */ class BrainEnvironment { - // TODO(tianyu): fill in as needed + public: + BrainEnvironment() : + rpc_client_{ + settings::SettingsManager::GetString(settings::SettingId::peloton_rpc_address)}, + sql_connection_{ + settings::SettingsManager::GetString(settings::SettingId::peloton_address)} {} + + inline capnp::EzRpcClient &GetPelotonClient() { + return rpc_client_; + } + + inline pqxx::result ExecuteQuery(const std::string &query) { + pqxx::work w(sql_connection_); + pqxx::result result = w.exec(query); + w.commit(); + return result; + } + + private: + capnp::EzRpcClient rpc_client_; + // TODO(tianyu): eventually change this into rpc + pqxx::connection sql_connection_; }; /** @@ -83,11 +107,21 @@ class Brain { Brain() : scheduler_(0) {} ~Brain() { - for (auto entry : jobs_) + for (auto &entry : jobs_) delete entry.second; } - template + /** + * Registers a Job of type BrainJob (given as template parameter) to be run + * periodically on Brain. + * + * @tparam BrainJob The typename of the brain job being submitted, must subclass BrainJob + * @tparam Args Arguments to pass to the BrainJob constructor, in addition to BrainEnvironment + * @param period The time period between each run of the job + * @param name name of the job + * @param args arguments to BrainJob constructor + */ + template inline void RegisterJob(const struct timeval *period, std::string name, Args... args) { auto *job = new BrainJob(&env_, args...); @@ -99,18 +133,25 @@ class Brain { scheduler_.RegisterPeriodicEvent(period, callback, job); } - inline void Run() { - scheduler_.EventLoop(); - } + /** + * Executes the main eventloop on the brain. Tasks will begin executing periodically. + * Does not return unless there is an exception, or the loop is terminated. + */ + inline void Run() { scheduler_.EventLoop(); } - inline void Terminate() { - scheduler_.ExitLoop(); - } + /** + * Terminate the main event loop. + */ + inline void Terminate() { scheduler_.ExitLoop(); } private: + // Main Event loop NotifiableTask scheduler_; + // collection of all the jobs registered std::unordered_map jobs_; + // mapping of jobs to their event structs (used to trigger the job) std::unordered_map job_handles_; + // Shared environment for all the tasks BrainEnvironment env_; }; } // namespace brain diff --git a/src/include/brain/catalog_sync_brain_job.h b/src/include/brain/catalog_sync_brain_job.h new file mode 100644 index 00000000000..1188ce58c04 --- /dev/null +++ b/src/include/brain/catalog_sync_brain_job.h @@ -0,0 +1,58 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// catalog_sync_brain_job.h +// +// Identification: src/include/brain/catalog_sync_brain_job.h +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// +#pragma once +#include +#include +#include +#include +#include "concurrency/transaction_manager_factory.h" +#include "brain/brain.h" +#include "catalog/catalog.h" +#include "catalog/table_catalog.h" +#include "storage/data_table.h" +#include "catalog/schema.h" +#include "type/value_factory.h" +#include "catalog/query_history_catalog.h" + +namespace peloton { +namespace brain { +/** + * Brain Job that fetches catalog updates from Peloton + */ +class CatalogSyncBrainJob : public BrainJob { + public: + inline CatalogSyncBrainJob(BrainEnvironment *env) : BrainJob(env) {} + + // TODO(tianyu): Eventually use Log for replication + void OnJobInvocation(BrainEnvironment *env) override; + + private: + static time_t TimeFromString(const char *str); + + // TODO(tianyu): Switch to Sequence when we have them + // Returns the SQL string for fetching entries from a catalog table + std::string FetchCatalogQuery(catalog::AbstractCatalog *catalog); + + // Logs the last entry we have seen in the last_history and last_metric + void UpdateTimestamp(catalog::AbstractCatalog *catalog, pqxx::field field); + + // Fetches all new entries from the catalog + void SyncCatalog(catalog::AbstractCatalog *catalog, BrainEnvironment *env, + concurrency::TransactionManager &manager); + + // turns a pqxx field into a peloton value + type::Value PqxxFieldToPelotonValue(pqxx::field &f); + + int64_t last_history_timestamp_ = 0, last_metric_timestamp_ = 0; +}; +} // namespace brain +} // nanespace peloton diff --git a/src/include/catalog/abstract_catalog.h b/src/include/catalog/abstract_catalog.h index e0c8d81df53..eb51fad5fa7 100644 --- a/src/include/catalog/abstract_catalog.h +++ b/src/include/catalog/abstract_catalog.h @@ -13,6 +13,7 @@ #pragma once #include +#include #include "catalog/catalog_defaults.h" #include "catalog/schema.h" @@ -43,6 +44,15 @@ class AbstractCatalog { public: virtual ~AbstractCatalog() {} + bool InsertTuple(std::unique_ptr tuple, + concurrency::TransactionContext *txn); + + inline const storage::DataTable *GetDataTable() const { + return catalog_table_; + } + + virtual std::string GetName() const = 0; + protected: /* For pg_database, pg_table, pg_index, pg_column */ AbstractCatalog(oid_t catalog_table_oid, std::string catalog_table_name, @@ -56,8 +66,6 @@ class AbstractCatalog { //===--------------------------------------------------------------------===// // Helper Functions //===--------------------------------------------------------------------===// - bool InsertTuple(std::unique_ptr tuple, - concurrency::TransactionContext *txn); bool DeleteWithIndexScan(oid_t index_offset, std::vector values, concurrency::TransactionContext *txn); diff --git a/src/include/catalog/catalog.h b/src/include/catalog/catalog.h index cd19d7fe72b..b1e6208a6e3 100644 --- a/src/include/catalog/catalog.h +++ b/src/include/catalog/catalog.h @@ -24,6 +24,7 @@ class Schema; class DatabaseCatalogObject; class TableCatalogObject; class IndexCatalogObject; +class AbstractCatalog; class SystemCatalogs; } // namespace catalog @@ -225,6 +226,8 @@ class Catalog { const FunctionData GetFunction( const std::string &name, const std::vector &argument_types); + inline const std::vector &AvailableCatalogs() const { return catalogs_; } + private: Catalog(); @@ -239,6 +242,7 @@ class Catalog { // The pool for new varlen tuple fields std::unique_ptr pool_; + std::vector catalogs_; std::mutex catalog_mutex; // key: database oid // value: SystemCatalog object(including pg_table, pg_index and pg_attribute) diff --git a/src/include/catalog/catalog_defaults.h b/src/include/catalog/catalog_defaults.h index 2cfcacbda70..f866d4f5e29 100644 --- a/src/include/catalog/catalog_defaults.h +++ b/src/include/catalog/catalog_defaults.h @@ -35,7 +35,7 @@ namespace catalog { // Local oids from START_OID = 0 to START_OID + OID_OFFSET are reserved #define OID_OFFSET 100 -#define CATALOG_TABLES_COUNT 8 +#define CATALOG_TABLES_COUNT 7 // Oid mask for each type #define DATABASE_OID_MASK (static_cast(catalog::CatalogType::DATABASE)) diff --git a/src/include/catalog/column_catalog.h b/src/include/catalog/column_catalog.h index 7dcdf96e4a5..fce814e77a8 100644 --- a/src/include/catalog/column_catalog.h +++ b/src/include/catalog/column_catalog.h @@ -74,6 +74,8 @@ class ColumnCatalog : public AbstractCatalog { ~ColumnCatalog(); + inline std::string GetName() const override { return COLUMN_CATALOG_NAME; } + // No use inline oid_t GetNextOid() { return INVALID_OID; } diff --git a/src/include/catalog/column_stats_catalog.h b/src/include/catalog/column_stats_catalog.h index d409a9da338..fc672e210d2 100644 --- a/src/include/catalog/column_stats_catalog.h +++ b/src/include/catalog/column_stats_catalog.h @@ -53,6 +53,8 @@ class ColumnStatsCatalog : public AbstractCatalog { static ColumnStatsCatalog *GetInstance( concurrency::TransactionContext *txn = nullptr); + inline std::string GetName() const override { return COLUMN_STATS_CATALOG_NAME; } + //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// diff --git a/src/include/catalog/database_catalog.h b/src/include/catalog/database_catalog.h index cc097414931..8849ec06865 100644 --- a/src/include/catalog/database_catalog.h +++ b/src/include/catalog/database_catalog.h @@ -109,6 +109,8 @@ class DatabaseCatalog : public AbstractCatalog { type::AbstractPool *pool = nullptr, concurrency::TransactionContext *txn = nullptr); + inline std::string GetName() const override { return DATABASE_CATALOG_NAME; } + inline oid_t GetNextOid() { return oid_++ | DATABASE_OID_MASK; } //===--------------------------------------------------------------------===// diff --git a/src/include/catalog/database_metrics_catalog.h b/src/include/catalog/database_metrics_catalog.h index a035063a1a5..afa538686ac 100644 --- a/src/include/catalog/database_metrics_catalog.h +++ b/src/include/catalog/database_metrics_catalog.h @@ -27,14 +27,32 @@ #pragma once #include "catalog/abstract_catalog.h" -#include "statistics/database_metric.h" #define DATABASE_METRICS_CATALOG_NAME "pg_database_metrics" namespace peloton { namespace catalog { +class DatabaseMetricsCatalogObject { + public: + // construct object from logical tile + DatabaseMetricsCatalogObject(executor::LogicalTile *tile, int tupleID = 0); + + inline oid_t GetDatabaseOid() { return database_oid_; } + inline int64_t GetTxnCommitted() { return txn_committed_; } + inline int64_t GetTxnAborted() { return txn_aborted_; } + inline int64_t GetTimeStamp() { return time_stamp_; } + + private: + oid_t database_oid_; + int64_t txn_committed_; + int64_t txn_aborted_; + int64_t time_stamp_; +}; + class DatabaseMetricsCatalog : public AbstractCatalog { + friend class DatabaseMetricsCatalogObject; + public: ~DatabaseMetricsCatalog(); @@ -42,6 +60,10 @@ class DatabaseMetricsCatalog : public AbstractCatalog { static DatabaseMetricsCatalog *GetInstance( concurrency::TransactionContext *txn = nullptr); + inline std::string GetName() const override { + return DATABASE_METRICS_CATALOG_NAME; + } + //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// @@ -49,14 +71,21 @@ class DatabaseMetricsCatalog : public AbstractCatalog { oid_t txn_aborted, oid_t time_stamp, type::AbstractPool *pool, concurrency::TransactionContext *txn); - bool DeleteDatabaseMetrics(oid_t database_oid, concurrency::TransactionContext *txn); + + bool DeleteDatabaseMetrics(oid_t database_oid, + concurrency::TransactionContext *txn); + + bool UpdateDatabaseMetrics(oid_t database_oid, oid_t txn_committed, + oid_t txn_aborted, oid_t time_stamp, + concurrency::TransactionContext *txn); //===--------------------------------------------------------------------===// // Read-only Related API //===--------------------------------------------------------------------===// - oid_t GetTimeStamp(oid_t database_oid, concurrency::TransactionContext *txn); - // TODO: add more if needed + std::shared_ptr GetDatabaseMetricsObject( + oid_t database_oid, concurrency::TransactionContext *txn); + private: enum ColumnId { DATABASE_OID = 0, TXN_COMMITTED = 1, @@ -64,14 +93,14 @@ class DatabaseMetricsCatalog : public AbstractCatalog { TIME_STAMP = 3, // Add new columns here in creation order }; - - private: - DatabaseMetricsCatalog(concurrency::TransactionContext *txn); + std::vector all_column_ids_ = {0, 1, 2, 3}; enum IndexId { PRIMARY_KEY = 0, // Add new indexes here in creation order }; + + DatabaseMetricsCatalog(concurrency::TransactionContext *txn); }; } // namespace catalog diff --git a/src/include/catalog/index_catalog.h b/src/include/catalog/index_catalog.h index 3ece01952b9..b24f6946005 100644 --- a/src/include/catalog/index_catalog.h +++ b/src/include/catalog/index_catalog.h @@ -78,6 +78,8 @@ class IndexCatalog : public AbstractCatalog { ~IndexCatalog(); + inline std::string GetName() const override { return INDEX_CATALOG_NAME; } + inline oid_t GetNextOid() { return oid_++ | INDEX_OID_MASK; } /** Write Related API */ diff --git a/src/include/catalog/index_metrics_catalog.h b/src/include/catalog/index_metrics_catalog.h index 7b806623565..1c859eed881 100644 --- a/src/include/catalog/index_metrics_catalog.h +++ b/src/include/catalog/index_metrics_catalog.h @@ -6,7 +6,7 @@ // // Identification: src/include/catalog/index_metrics_catalog.h // -// Copyright (c) 2015-17, Carnegie Mellon University Database Group +// Copyright (c) 2015-18, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// @@ -14,12 +14,15 @@ // pg_index_metrics // // Schema: (column offset: column_name) +// 0: index_oid // 1: table_oid -// 2: index_oid -// 3: reads -// 4: deletes -// 5: inserts -// 6: time_stamp +// 2: reads +// 3: updates +// 4: inserts +// 5: deletes +// 6: memory_alloc +// 7: memory_usage +// 8: time_stamp // // Indexes: (index offset: indexed columns) // 0: index_oid (unique & primary key) @@ -36,37 +39,83 @@ namespace peloton { namespace catalog { +// helper class for reading tuples from catalog +class IndexMetricsCatalogObject { + public: + IndexMetricsCatalogObject(executor::LogicalTile *tile, int tupleId = 0); + + inline oid_t GetIndexOid() { return index_oid_; } + inline oid_t GetTableOid() { return table_oid_; } + inline int64_t GetReads() { return reads_; } + inline int64_t GetUpdates() { return updates_; } + inline int64_t GetInserts() { return inserts_; } + inline int64_t GetDeletes() { return deletes_; } + inline int64_t GetMemoryAlloc() { return memory_alloc_; } + inline int64_t GetMemoryUsage() { return memory_usage_; } + inline int64_t GetTimeStamp() { return time_stamp_; } + + private: + oid_t index_oid_; + oid_t table_oid_; + int64_t reads_; + int64_t updates_; + int64_t inserts_; + int64_t deletes_; + int64_t memory_alloc_; + int64_t memory_usage_; + int64_t time_stamp_; +}; + class IndexMetricsCatalog : public AbstractCatalog { + friend class IndexMetricsCatalogObject; + public: IndexMetricsCatalog(const std::string &database_name, concurrency::TransactionContext *txn); ~IndexMetricsCatalog(); + inline std::string GetName() const override { + return INDEX_METRICS_CATALOG_NAME; + } + //===--------------------------------------------------------------------===// // Write Related API //===--------------------------------------------------------------------===// - bool InsertIndexMetrics(oid_t table_oid, oid_t index_oid, int64_t reads, - int64_t deletes, int64_t inserts, int64_t time_stamp, - type::AbstractPool *pool, + bool InsertIndexMetrics(oid_t index_oid, oid_t table_oid, int64_t reads, + int64_t updates, int64_t inserts, int64_t deletes, + int64_t memory_alloc, int64_t memory_usage, + int64_t time_stamp, type::AbstractPool *pool, concurrency::TransactionContext *txn); + bool DeleteIndexMetrics(oid_t index_oid, concurrency::TransactionContext *txn); + bool UpdateIndexMetrics(oid_t index_oid, oid_t table_oid, int64_t reads, + int64_t updates, int64_t inserts, int64_t deletes, + int64_t memory_alloc, int64_t memory_usage, + int64_t time_stamp, + concurrency::TransactionContext *txn); + //===--------------------------------------------------------------------===// // Read-only Related API //===--------------------------------------------------------------------===// - // TODO: add if needed + std::shared_ptr GetIndexMetricsObject( + oid_t index_oid, concurrency::TransactionContext *txn); private: enum ColumnId { - TABLE_OID = 0, - INDEX_OID = 1, + INDEX_OID = 0, + TABLE_OID = 1, READS = 2, - DELETES = 3, + UPDATES = 3, INSERTS = 4, - TIME_STAMP = 5, + DELETES = 5, + MEMORY_ALLOC = 6, + MEMORY_USAGE = 7, + TIME_STAMP = 8, // Add new columns here in creation order }; + std::vector all_column_ids_ = {0, 1, 2, 3, 4, 5, 6, 7, 8}; enum IndexId { PRIMARY_KEY = 0, diff --git a/src/include/catalog/language_catalog.h b/src/include/catalog/language_catalog.h index 139f51eda21..3746ef7f8d1 100644 --- a/src/include/catalog/language_catalog.h +++ b/src/include/catalog/language_catalog.h @@ -56,6 +56,8 @@ class LanguageCatalog : public AbstractCatalog { // Global Singleton static LanguageCatalog &GetInstance(concurrency::TransactionContext *txn = nullptr); + inline std::string GetName() const override { return LANGUAGE_CATALOG_NAME; } + //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// diff --git a/src/include/catalog/proc_catalog.h b/src/include/catalog/proc_catalog.h index 0445daecc9d..f1cc273366d 100644 --- a/src/include/catalog/proc_catalog.h +++ b/src/include/catalog/proc_catalog.h @@ -30,6 +30,8 @@ #include "catalog/abstract_catalog.h" +#define PROC_CATALOG_NAME "pg_proc" + namespace peloton { namespace catalog { @@ -82,6 +84,8 @@ class ProcCatalog : public AbstractCatalog { // Global Singleton static ProcCatalog &GetInstance(concurrency::TransactionContext *txn = nullptr); + inline std::string GetName() const override { return PROC_CATALOG_NAME; } + //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// diff --git a/src/include/catalog/query_history_catalog.h b/src/include/catalog/query_history_catalog.h index 3f004508d02..6d0881ed6fd 100644 --- a/src/include/catalog/query_history_catalog.h +++ b/src/include/catalog/query_history_catalog.h @@ -38,6 +38,8 @@ class QueryHistoryCatalog : public AbstractCatalog { static QueryHistoryCatalog &GetInstance( concurrency::TransactionContext *txn = nullptr); + inline std::string GetName() const override { return QUERY_HISTORY_CATALOG_NAME; } + //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// diff --git a/src/include/catalog/query_metrics_catalog.h b/src/include/catalog/query_metrics_catalog.h deleted file mode 100644 index a8045fd94e1..00000000000 --- a/src/include/catalog/query_metrics_catalog.h +++ /dev/null @@ -1,99 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// query_metrics_catalog.h -// -// Identification: src/include/catalog/query_metrics_catalog.h -// -// Copyright (c) 2015-17, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -//===----------------------------------------------------------------------===// -// pg_query(per database, name become primary key) -// -// Schema: (column offset: column_name) -// 0: name (pkey) -// 1: database_id(pkey) -// 2: num_params -// 3: param_types -// 4: param_formats -// 5: param_values -// 6: reads -// 7: updates -// 8: deletes -// 9: inserts -// 10: latency -// 11: cpu_time -// 12: time_stamp -// -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "catalog/abstract_catalog.h" -#include "statistics/query_metric.h" - -#define QUERY_METRICS_CATALOG_NAME "pg_query_metrics" - -namespace peloton { -namespace catalog { - -class QueryMetricsCatalog : public AbstractCatalog { - public: - QueryMetricsCatalog(const std::string &database_name, - concurrency::TransactionContext *txn); - ~QueryMetricsCatalog(); - - //===--------------------------------------------------------------------===// - // write Related API - //===--------------------------------------------------------------------===// - bool InsertQueryMetrics(const std::string &name, oid_t database_oid, - int64_t num_params, - const stats::QueryMetric::QueryParamBuf &type_buf, - const stats::QueryMetric::QueryParamBuf &format_buf, - const stats::QueryMetric::QueryParamBuf &value_buf, - int64_t reads, int64_t updates, int64_t deletes, - int64_t inserts, int64_t latency, int64_t cpu_time, - int64_t time_stamp, type::AbstractPool *pool, - concurrency::TransactionContext *txn); - bool DeleteQueryMetrics(const std::string &name, - concurrency::TransactionContext *txn); - - //===--------------------------------------------------------------------===// - // Read-only Related API - //===--------------------------------------------------------------------===// - stats::QueryMetric::QueryParamBuf GetParamTypes( - const std::string &name, concurrency::TransactionContext *txn); - int64_t GetNumParams(const std::string &name, - concurrency::TransactionContext *txn); - // TODO: In theory, we don't need database_oid - // but now we store all the query metrics under default database "peloton" - enum ColumnId { - NAME = 0, - DATABASE_OID = 1, - NUM_PARAMS = 2, - PARAM_TYPES = 3, - PARAM_FORMATS = 4, - PARAM_VALUES = 5, - READS = 6, - UPDATES = 7, - DELETES = 8, - INSERTS = 9, - LATENCY = 10, - CPU_TIME = 11, - TIME_STAMP = 12, - // Add new columns here in creation order - }; - - private: - enum IndexId { - PRIMARY_KEY = 0, - // Add new indexes here in creation order - }; -}; - -} // namespace catalog -} // namespace peloton diff --git a/src/include/catalog/schema_catalog.h b/src/include/catalog/schema_catalog.h index 2d9e06693d7..4e900bb5c0a 100644 --- a/src/include/catalog/schema_catalog.h +++ b/src/include/catalog/schema_catalog.h @@ -24,6 +24,7 @@ //===----------------------------------------------------------------------===// #pragma once +#define SCHEMA_CATALOG_NAME "pg_namespace" #include "catalog/abstract_catalog.h" #include "executor/logical_tile.h" @@ -62,6 +63,8 @@ class SchemaCatalog : public AbstractCatalog { inline oid_t GetNextOid() { return oid_++ | SCHEMA_OID_MASK; } + inline std::string GetName() const override { return SCHEMA_CATALOG_NAME; } + //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// diff --git a/src/include/catalog/settings_catalog.h b/src/include/catalog/settings_catalog.h index a3c8b1bf6df..d7ec11fe6ca 100644 --- a/src/include/catalog/settings_catalog.h +++ b/src/include/catalog/settings_catalog.h @@ -14,6 +14,8 @@ #include "catalog/abstract_catalog.h" +#define SETTINGS_CATALOG_NAME "pg_settings" + namespace peloton { namespace catalog { @@ -24,6 +26,8 @@ class SettingsCatalog : public AbstractCatalog { // Global Singleton static SettingsCatalog &GetInstance(concurrency::TransactionContext *txn = nullptr); + inline std::string GetName() const override { return SETTINGS_CATALOG_NAME; } + //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// diff --git a/src/include/catalog/system_catalogs.h b/src/include/catalog/system_catalogs.h index 7791c019097..8a6b819adb7 100644 --- a/src/include/catalog/system_catalogs.h +++ b/src/include/catalog/system_catalogs.h @@ -16,7 +16,6 @@ #include "catalog/database_catalog.h" #include "catalog/index_metrics_catalog.h" -#include "catalog/query_metrics_catalog.h" #include "catalog/schema_catalog.h" #include "catalog/table_catalog.h" #include "catalog/table_metrics_catalog.h" @@ -100,13 +99,6 @@ class SystemCatalogs { return pg_index_metrics_; } - QueryMetricsCatalog *GetQueryMetricsCatalog() { - if (!pg_query_metrics_) { - throw CatalogException("Query metrics catalog has not been initialized"); - } - return pg_query_metrics_; - } - private: ColumnCatalog *pg_attribute_; SchemaCatalog *pg_namespace_; @@ -117,7 +109,6 @@ class SystemCatalogs { // ProcCatalog *pg_proc; TableMetricsCatalog *pg_table_metrics_; IndexMetricsCatalog *pg_index_metrics_; - QueryMetricsCatalog *pg_query_metrics_; }; } // namespace catalog diff --git a/src/include/catalog/table_catalog.h b/src/include/catalog/table_catalog.h index 0dfc3f51fa9..203b343d59a 100644 --- a/src/include/catalog/table_catalog.h +++ b/src/include/catalog/table_catalog.h @@ -129,6 +129,8 @@ class TableCatalog : public AbstractCatalog { inline oid_t GetNextOid() { return oid_++ | TABLE_OID_MASK; } + inline std::string GetName() const override { return TABLE_CATALOG_NAME; } + //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// diff --git a/src/include/catalog/table_metrics_catalog.h b/src/include/catalog/table_metrics_catalog.h index 38c59a76857..cc7d80295f4 100644 --- a/src/include/catalog/table_metrics_catalog.h +++ b/src/include/catalog/table_metrics_catalog.h @@ -19,7 +19,11 @@ // 3: updates // 4: deletes // 5: inserts -// 6: time_stamp +// 6: inline_memory_alloc +// 7: inline_memory_usage +// 8: varlen_memory_alloc +// 9: varlen_memory_usage +// 10: time_stamp // // Indexes: (index offset: indexed columns) // 0: index_oid (unique & primary key) @@ -29,48 +33,102 @@ #pragma once #include "catalog/abstract_catalog.h" -#include "statistics/index_metric.h" #define TABLE_METRICS_CATALOG_NAME "pg_table_metrics" namespace peloton { namespace catalog { +// helper class for reading tuples from catalog +class TableMetricsCatalogObject { + public: + // construct object from logical tile + TableMetricsCatalogObject(executor::LogicalTile *tile, int tupleId = 0); + + inline oid_t GetTableOid() { return table_oid_; } + inline int64_t GetReads() { return reads_; } + inline int64_t GetUpdates() { return updates_; } + inline int64_t GetInserts() { return inserts_; } + inline int64_t GetDeletes() { return deletes_; } + inline int64_t GetInlineMemoryAlloc() { return inline_memory_alloc_; } + inline int64_t GetInlineMemoryUsage() { return inline_memory_usage_; } + inline int64_t GetVarlenMemoryAlloc() { return varlen_memory_alloc_; } + inline int64_t GetVarlenMemoryUsage() { return varlen_memory_usage_; } + inline int64_t GetTimeStamp() { return time_stamp_; } + + private: + oid_t table_oid_; + int64_t reads_; + int64_t updates_; + int64_t inserts_; + int64_t deletes_; + int64_t inline_memory_alloc_; + int64_t inline_memory_usage_; + int64_t varlen_memory_alloc_; + int64_t varlen_memory_usage_; + int64_t time_stamp_; +}; + class TableMetricsCatalog : public AbstractCatalog { + friend class TableMetricsCatalogObject; + public: TableMetricsCatalog(const std::string &database_name, concurrency::TransactionContext *txn); ~TableMetricsCatalog(); + inline std::string GetName() const override { + return TABLE_METRICS_CATALOG_NAME; + } + //===--------------------------------------------------------------------===// // Write Related API //===--------------------------------------------------------------------===// bool InsertTableMetrics(oid_t table_oid, int64_t reads, int64_t updates, - int64_t deletes, int64_t inserts, int64_t time_stamp, + int64_t inserts, int64_t deletes, + int64_t inline_memory_alloc, + int64_t inline_memory_usage, + int64_t varlen_memory_alloc, + int64_t varlen_memory_usage, int64_t time_stamp, type::AbstractPool *pool, concurrency::TransactionContext *txn); bool DeleteTableMetrics(oid_t table_oid, concurrency::TransactionContext *txn); + bool UpdateTableMetrics(oid_t table_oid, int64_t reads, int64_t updates, + int64_t inserts, int64_t deletes, + int64_t inline_memory_alloc, + int64_t inline_memory_usage, + int64_t varlen_memory_alloc, + int64_t varlen_memory_usage, int64_t time_stamp, + concurrency::TransactionContext *txn); + //===--------------------------------------------------------------------===// // Read-only Related API //===--------------------------------------------------------------------===// - // TODO: add if needed + std::shared_ptr GetTableMetricsObject( + oid_t table_oid, concurrency::TransactionContext *txn); private: enum ColumnId { TABLE_OID = 0, READS = 1, UPDATES = 2, - DELETES = 3, - INSERTS = 4, - TIME_STAMP = 5, + INSERTS = 3, + DELETES = 4, + INLINE_MEMORY_ALLOC = 5, + INLINE_MEMORY_USAGE = 6, + VARLEN_MEMORY_ALLOC = 7, + VARLEN_MEMORY_USAGE = 8, + TIME_STAMP = 9, // Add new columns here in creation order }; + std::vector all_column_ids_ = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; enum IndexId { PRIMARY_KEY = 0, - // Add new indexes here in creation order + // under new hierarchical catalog design, each database has its own table + // catalogs, so table_oid is a valid primary key }; }; diff --git a/src/include/catalog/trigger_catalog.h b/src/include/catalog/trigger_catalog.h index 6b71792e42c..39333e00bdd 100644 --- a/src/include/catalog/trigger_catalog.h +++ b/src/include/catalog/trigger_catalog.h @@ -2,7 +2,7 @@ // // Peloton // -// query_metrics_catalog.h +// trigger_catalog.h // // Identification: src/include/catalog/trigger_catalog.h // @@ -51,6 +51,11 @@ class TriggerCatalog : public AbstractCatalog { concurrency::TransactionContext *txn); ~TriggerCatalog(); + // Global Singleton + static TriggerCatalog &GetInstance(concurrency::TransactionContext *txn = nullptr); + + inline std::string GetName() const override { return TRIGGER_CATALOG_NAME; } + //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// diff --git a/src/include/catalog/tuple_access_metrics_catalog.h b/src/include/catalog/tuple_access_metrics_catalog.h new file mode 100644 index 00000000000..9c19287e648 --- /dev/null +++ b/src/include/catalog/tuple_access_metrics_catalog.h @@ -0,0 +1,109 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// table_metrics_catalog.h +// +// Identification: src/include/catalog/table_metrics_catalog.h +// +// Copyright (c) 2015-17, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// tuple_access_metrics +// +// Schema: (column offset: column_name) +// 1: txn_id +// 2: reads +// 3: valid +// 4: committed +// Indexes: (index offset: indexed columns) +// 0: index_oid (unique & primary key) +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "catalog/abstract_catalog.h" +#include "concurrency/transaction_context.h" + +#define TUPLE_ACCESS_METRICS_CATALOG_NAME "tuple_access_metrics" + +namespace peloton { +namespace catalog { + +class TupleAccessMetricsCatalogObject { + public: + TupleAccessMetricsCatalogObject(executor::LogicalTile *tile, oid_t tupleId = 0); + + inline txn_id_t GetTxnId() { return tid_; } + inline uint64_t GetReads() { return reads_; } + inline bool IsValid() { return valid_; } + inline bool IsCommitted() { return committed_; } + + private: + txn_id_t tid_; + uint64_t reads_; + bool valid_; + bool committed_; +}; + +class TupleAccessMetricsCatalog : public AbstractCatalog { + friend class TupleAccessMetricsCatalogObject; + public: + + static TupleAccessMetricsCatalog *GetInstance(concurrency::TransactionContext *txn) { + static TupleAccessMetricsCatalog catalog{txn}; + return &catalog; + } + TupleAccessMetricsCatalog(concurrency::TransactionContext *txn); + + ~TupleAccessMetricsCatalog() = default; + + inline std::string GetName() const override { + return TUPLE_ACCESS_METRICS_CATALOG_NAME; + } + + //===--------------------------------------------------------------------===// + // Write Related API + //===--------------------------------------------------------------------===// + bool InsertAccessMetric(txn_id_t tid, + uint64_t reads, + bool valid, + bool committed, + type::AbstractPool *pool, + concurrency::TransactionContext *txn); + bool DeleteAccessMetrics(txn_id_t tid, + concurrency::TransactionContext *txn); + + bool UpdateAccessMetrics(txn_id_t tid, + uint64_t reads, + bool valid, + bool committed, + concurrency::TransactionContext *txn); + //===--------------------------------------------------------------------===// + // Read-only Related API + //===--------------------------------------------------------------------===// + std::shared_ptr GetTupleAccessMetricsCatalogObject( + txn_id_t tid, + concurrency::TransactionContext *txn); + + private: + enum ColumnId { + TXN_ID = 0, + READS = 1, + VALID = 2, + COMMITTED = 3 + }; + std::vector all_column_ids_ = {0, 1, 2, 3}; + + enum IndexId { + PRIMARY_KEY = 0, + // under new hierarchical catalog design, each database has its own table + // catalogs, so table_oid is a valid primary key + }; +}; + +} // namespace catalog +} // namespace peloton diff --git a/src/include/catalog/zone_map_catalog.h b/src/include/catalog/zone_map_catalog.h index a38d26d0597..7e64272147c 100644 --- a/src/include/catalog/zone_map_catalog.h +++ b/src/include/catalog/zone_map_catalog.h @@ -33,6 +33,8 @@ class ZoneMapCatalog : public AbstractCatalog { static ZoneMapCatalog *GetInstance(concurrency::TransactionContext *txn = nullptr); + inline std::string GetName() const override { return ZONE_MAP_CATALOG_NAME; } + //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// diff --git a/src/include/common/container/circular_buffer.h b/src/include/common/container/circular_buffer.h index 92324dd4bb0..16b92e47391 100644 --- a/src/include/common/container/circular_buffer.h +++ b/src/include/common/container/circular_buffer.h @@ -31,7 +31,7 @@ class CircularBuffer { void PushBack(ValueType value); // Set the container capaciry - void SetCapaciry(size_t new_capacity); + void SetCapacity(size_t new_capacity); // Returns item count in the circular_buffer size_t GetSize() const; diff --git a/src/include/common/container/lock_free_array.h b/src/include/common/container/lock_free_array.h index 10f25edc266..54ee5319a42 100644 --- a/src/include/common/container/lock_free_array.h +++ b/src/include/common/container/lock_free_array.h @@ -11,14 +11,14 @@ //===----------------------------------------------------------------------===// #pragma once -#include "tbb/concurrent_vector.h" -#include "tbb/tbb_allocator.h" -#include -#include -#include #include #include +#include +#include +#include #include +#include "tbb/concurrent_vector.h" +#include "tbb/tbb_allocator.h" namespace peloton { @@ -63,6 +63,35 @@ class LockFreeArray { // Exists ? bool Contains(const ValueType &value); + /** + * Wrapper iterator class around tbb_iterator + */ + class Iterator { + using tbb_iterator = tbb::internal::vector_iterator< + tbb::concurrent_vector>, + ValueType>; + + public: + Iterator(tbb_iterator iter) : iter_(iter) {} + + inline Iterator &operator++() { + ++iter_; + return *this; + } + + inline ValueType &operator*() { return *iter_; } + + inline bool operator==(const Iterator &rhs) { return iter_ == rhs.iter_; } + inline bool operator!=(const Iterator &rhs) { return iter_ != rhs.iter_; } + + private: + tbb_iterator iter_; + }; + + Iterator Begin() { return Iterator(lock_free_array.begin()); } + + Iterator End() { return Iterator(lock_free_array.end()); } + private: // lock free array tbb::concurrent_vector> diff --git a/src/include/common/dedicated_thread_registry.h b/src/include/common/dedicated_thread_registry.h index a725048edc9..4ca34a9ed12 100644 --- a/src/include/common/dedicated_thread_registry.h +++ b/src/include/common/dedicated_thread_registry.h @@ -35,7 +35,6 @@ class DedicatedThreadRegistry { // Note that if registry is shutting down, it doesn't matter whether // owners are notified as this class should have the same life cycle // as the entire peloton process. - for (auto &entry : thread_owners_table_) { for (auto &task : entry.second) { task->Terminate(); diff --git a/src/include/common/internal_types.h b/src/include/common/internal_types.h index 17020512944..4ed3500410f 100644 --- a/src/include/common/internal_types.h +++ b/src/include/common/internal_types.h @@ -1026,15 +1026,15 @@ const int TRIGGER_TYPE_MAX = TRIGGER_TYPE_ROW | TRIGGER_TYPE_STATEMENT | // Statistics Types //===--------------------------------------------------------------------===// -// Statistics Collection Type -// Disable or enable -// TODO: This should probably be a collection level and not a boolean -// (enable/disable) -enum class StatsType { +// Statistics Collection Mode Type +// (enable/disable/test) +enum class StatsModeType { // Disable statistics collection - INVALID = INVALID_TYPE_ID, + DISABLE = INVALID_TYPE_ID, // Enable statistics collection ENABLE = 1, + // Enable Test stats only + TEST = 2, }; enum class MetricType { @@ -1060,6 +1060,41 @@ enum class MetricType { QUERY = 9, // Statistics for CPU PROCESSOR = 10, + // Statictics for memory usage and allocation + MEMORY = 11, +}; + +enum class CollectionPointType { + // invalid collection + INVALID = INVALID_TYPE_ID, + // tuple read + ON_TUPLE_READ, + // tuple update + ON_TUPLE_UPDATE, + // tuple insert + ON_TUPLE_INSERT, + // tuple delete + ON_TUPLE_DELETE, + // index read + ON_INDEX_READ, + // index update + ON_INDEX_UPDATE, + // index insert + ON_INDEX_INSERT, + // index delete + ON_INDEX_DELETE, + // txn commit - for point metric + ON_COMMIT, + // txn abort - for point metric + ON_ABORT, + // query start + ON_QUERY_START, + // query end + ON_QUERY_END, + // txn start + ON_TXN_START, + // txn end - for interval metric + ON_TXN_END, }; // All builtin operators we currently support @@ -1216,8 +1251,7 @@ typedef CuckooMap ReadWriteSet; typedef tbb::concurrent_unordered_set - WriteSet; + ItemPointerComparator> WriteSet; // this enum is to identify why the version should be GC'd. enum class GCVersionType { diff --git a/src/include/common/portal.h b/src/include/common/portal.h index f4017904583..f35e1cac685 100644 --- a/src/include/common/portal.h +++ b/src/include/common/portal.h @@ -16,7 +16,6 @@ #include #include #include "type/value.h" -#include "statistics/query_metric.h" namespace peloton { @@ -31,8 +30,7 @@ class Portal { Portal &operator=(Portal &&) = delete; Portal(const std::string &portal_name, std::shared_ptr statement, - std::vector bind_parameters, - std::shared_ptr param_stat); + std::vector bind_parameters); ~Portal(); @@ -40,10 +38,6 @@ class Portal { const std::vector &GetParameters() const; - inline std::shared_ptr GetParamStat() const { - return param_stat_; - } - // Portal name std::string portal_name_; @@ -53,8 +47,6 @@ class Portal { // Values bound to the statement of this portal std::vector bind_parameters_; - // The serialized params for stats collection - std::shared_ptr param_stat_; }; } // namespace peloton diff --git a/src/include/concurrency/timestamp_ordering_transaction_manager.h b/src/include/concurrency/timestamp_ordering_transaction_manager.h index 6e24b10b4c8..ae2e3419132 100644 --- a/src/include/concurrency/timestamp_ordering_transaction_manager.h +++ b/src/include/concurrency/timestamp_ordering_transaction_manager.h @@ -15,7 +15,6 @@ #include "concurrency/transaction_manager.h" #include "storage/tile_group.h" -#include "statistics/stats_aggregator.h" #include "common/synchronization/spin_latch.h" namespace peloton { @@ -48,7 +47,7 @@ class TimestampOrderingTransactionManager : public TransactionManager { */ static TimestampOrderingTransactionManager &GetInstance( const ProtocolType protocol, - const IsolationLevelType isolation, + const IsolationLevelType isolation, const ConflictAvoidanceType conflict); /** @@ -209,7 +208,7 @@ class TimestampOrderingTransactionManager : public TransactionManager { const ItemPointer &location); /** - * @brief Perform a delete operation. Used when the transaction is not + * @brief Perform a delete operation. Used when the transaction is not * the owner of the tuple. * * @param current_txn The current transaction @@ -243,7 +242,7 @@ class TimestampOrderingTransactionManager : public TransactionManager { /** * @brief Gets the spin latch field. - * + * * Timestamp ordering requires a spinlock field for protecting the atomic access * to txn_id field and last_reader_cid field. * @@ -258,7 +257,7 @@ class TimestampOrderingTransactionManager : public TransactionManager { /** * @brief Gets the last reader commit identifier. - * + * * In timestamp ordering, the last_reader_cid records the timestamp of the last * transaction that reads the tuple. * @@ -283,8 +282,8 @@ class TimestampOrderingTransactionManager : public TransactionManager { */ bool SetLastReaderCommitId( const storage::TileGroupHeader *const tile_group_header, - const oid_t &tuple_id, - const cid_t ¤t_cid, + const oid_t &tuple_id, + const cid_t ¤t_cid, const bool is_owner); /** diff --git a/src/include/executor/copy_executor.h b/src/include/executor/copy_executor.h index 31d65adaa1b..0794cb1539b 100644 --- a/src/include/executor/copy_executor.h +++ b/src/include/executor/copy_executor.h @@ -13,7 +13,6 @@ #pragma once #include "executor/abstract_executor.h" -#include "catalog/query_metrics_catalog.h" #define COPY_BUFFER_SIZE 65536 #define INVALID_COL_ID -1 @@ -81,15 +80,14 @@ class CopyExecutor : public AbstractExecutor { // Total number of bytes written size_t total_bytes_written = 0; - // The special column ids in query_metric table - unsigned int num_param_col_id = - catalog::QueryMetricsCatalog::ColumnId::NUM_PARAMS; - unsigned int param_type_col_id = - catalog::QueryMetricsCatalog::ColumnId::PARAM_TYPES; - unsigned int param_format_col_id = - catalog::QueryMetricsCatalog::ColumnId::PARAM_FORMATS; - unsigned int param_val_col_id = - catalog::QueryMetricsCatalog::ColumnId::PARAM_VALUES; + // TODO(Tianyi) These value are set to zero when + // clearing the old query metric. As COPY is not + // correct anyway, I am just going to set them to some default value + // (Old Comment) The special column ids in query_metric table + unsigned int num_param_col_id = 0; + unsigned int param_type_col_id = 0; + unsigned int param_format_col_id = 0; + unsigned int param_val_col_id = 0; }; } // namespace executor diff --git a/src/include/index/art_index.h b/src/include/index/art_index.h index 29852bb2811..b4c4d277007 100644 --- a/src/include/index/art_index.h +++ b/src/include/index/art_index.h @@ -91,9 +91,6 @@ class ArtIndex : public Index { return IndexTypeToString(GetIndexMethodType()); } - // TODO(pmenon): Implement me - size_t GetMemoryFootprint() override { return 0; } - // TODO(pmenon): Implement me bool NeedGC() override { return false; } diff --git a/src/include/index/bwtree_index.h b/src/include/index/bwtree_index.h index d6423af5851..e9ee0916eda 100644 --- a/src/include/index/bwtree_index.h +++ b/src/include/index/bwtree_index.h @@ -23,17 +23,13 @@ #include "index/bwtree.h" -#define BWTREE_INDEX_TYPE BWTreeIndex +#define BWTREE_INDEX_TYPE \ + BWTreeIndex namespace peloton { namespace index { - + /** * BW tree-based index implementation. * @@ -44,23 +40,14 @@ namespace index { * * @see Index */ -template +template class BWTreeIndex : public Index { friend class IndexFactory; - using MapType = BwTree; + using MapType = BwTree; public: BWTreeIndex(IndexMetadata *metadata); @@ -71,24 +58,21 @@ class BWTreeIndex : public Index { bool DeleteEntry(const storage::Tuple *key, ItemPointer *value) override; - bool CondInsertEntry(const storage::Tuple *key, - ItemPointer *value, + bool CondInsertEntry(const storage::Tuple *key, ItemPointer *value, std::function predicate) override; void Scan(const std::vector &values, const std::vector &key_column_ids, const std::vector &expr_types, - ScanDirectionType scan_direction, - std::vector &result, + ScanDirectionType scan_direction, std::vector &result, const ConjunctionScanPredicate *csp_p) override; - + void ScanLimit(const std::vector &values, const std::vector &key_column_ids, const std::vector &expr_types, ScanDirectionType scan_direction, std::vector &result, - const ConjunctionScanPredicate *csp_p, - uint64_t limit, + const ConjunctionScanPredicate *csp_p, uint64_t limit, uint64_t offset) override; void ScanAllKeys(std::vector &result) override; @@ -98,17 +82,12 @@ class BWTreeIndex : public Index { std::string GetTypeName() const override; - // TODO: Implement this - size_t GetMemoryFootprint() override { return 0; } - - bool NeedGC() override { - return container.NeedGarbageCollection(); - } + bool NeedGC() override { return container.NeedGarbageCollection(); } void PerformGC() override { LOG_INFO("Bw-Tree Garbage Collection!"); container.PerformGarbageCollection(); - + return; } @@ -117,7 +96,7 @@ class BWTreeIndex : public Index { KeyComparator comparator; KeyEqualityChecker equals; KeyHashFunc hash_func; - + // container MapType container; }; diff --git a/src/include/index/index.h b/src/include/index/index.h index 824f2f49f81..b1c41a348e7 100644 --- a/src/include/index/index.h +++ b/src/include/index/index.h @@ -352,40 +352,6 @@ class Index : public Printable { */ virtual void PerformGC() = 0; - ////////////////////////////////////////////////////////////////////////////// - /// Stats - ////////////////////////////////////////////////////////////////////////////// - - /** - * @brief Increase the number of tuples in the index by the provided amount - */ - void IncreaseNumberOfTuplesBy(size_t amount); - - /** - * @brief Decrease the number of tuples in the index by the provided amount - */ - void DecreaseNumberOfTuplesBy(size_t amount); - - /** - * @brief Set the number of tuples in the index - */ - void SetNumberOfTuples(size_t num_tuples); - - /** - * @brief Return the number of tuples in the index - */ - size_t GetNumberOfTuples() const; - - /** - * @brief Returns if the index is dirty. WTF that means is unknown. - */ - bool IsDirty() const; - - /** - * @brief No idea. Reset some dirty flag? - */ - void ResetDirty(); - ////////////////////////////////////////////////////////////////////////////// /// Utilities ////////////////////////////////////////////////////////////////////////////// @@ -426,13 +392,6 @@ class Index : public Printable { type::AbstractPool *GetPool() const { return pool; } - /** - * @brief Calculate the total number of bytes used by this index - * - * @return The number of bytes this index occupies - */ - virtual size_t GetMemoryFootprint() = 0; - // Get the indexed tile group offset virtual size_t GetIndexedTileGroupOff() { return indexed_tile_group_offset.load(); @@ -453,12 +412,6 @@ class Index : public Printable { oid_t index_oid = INVALID_OID; - // access counters - int lookup_counter; - int insert_counter; - int delete_counter; - int update_counter; - // number of tuples size_t number_of_tuples = 0; diff --git a/src/include/network/peloton_rpc_handler_task.h b/src/include/network/peloton_rpc_handler_task.h index 8abfa510af4..a5474ffe278 100644 --- a/src/include/network/peloton_rpc_handler_task.h +++ b/src/include/network/peloton_rpc_handler_task.h @@ -11,10 +11,12 @@ //===----------------------------------------------------------------------===// #pragma once +#include "concurrency/transaction_manager_factory.h" #include "capnp/ez-rpc.h" #include "capnp/message.h" #include "common/dedicated_thread_task.h" #include "common/logger.h" +#include "catalog/catalog.h" #include "kj/debug.h" #include "peloton/capnp/peloton_service.capnp.h" @@ -23,7 +25,6 @@ namespace network { class PelotonRpcServerImpl final : public PelotonService::Server { protected: kj::Promise createIndex(CreateIndexContext) override { - // TODO(tianyu) Write actual index code LOG_DEBUG("Received rpc to create index"); return kj::READY_NOW; } @@ -32,7 +33,7 @@ class PelotonRpcServerImpl final : public PelotonService::Server { class PelotonRpcHandlerTask : public DedicatedThreadTask { public: - explicit PelotonRpcHandlerTask(const char *address) : address_(address) {} + explicit PelotonRpcHandlerTask(std::string address) : address_(address) {} void Terminate() override { // TODO(tianyu): Not implemented. See: @@ -40,13 +41,13 @@ class PelotonRpcHandlerTask : public DedicatedThreadTask { } void RunTask() override { - capnp::EzRpcServer server(kj::heap(), address_); - LOG_DEBUG("Server listening on %s", address_); + capnp::EzRpcServer server(kj::heap(), address_.c_str()); + LOG_DEBUG("Server listening on %s", address_.c_str()); kj::NEVER_DONE.wait(server.getWaitScope()); } private: - const char *address_; + std::string address_; }; } // namespace network } // namespace peloton diff --git a/src/include/network/postgres_protocol_handler.h b/src/include/network/postgres_protocol_handler.h index ef75f0a4cb1..ac8f33450c0 100644 --- a/src/include/network/postgres_protocol_handler.h +++ b/src/include/network/postgres_protocol_handler.h @@ -218,16 +218,6 @@ class PostgresProtocolHandler : public ProtocolHandler { // packets ready for read size_t pkt_cntr_; - // Manage parameter types for unnamed statement - stats::QueryMetric::QueryParamBuf unnamed_stmt_param_types_; - - // Parameter types for statements - // Warning: the data in the param buffer becomes invalid when the value - // stored - // in stat table is destroyed - std::unordered_map - statement_param_types_; - std::unordered_map cmdline_options_; //===--------------------------------------------------------------------===// diff --git a/src/include/settings/settings.h b/src/include/settings/settings.h index e90cb78b7da..b0ecee0d770 100644 --- a/src/include/settings/settings.h +++ b/src/include/settings/settings.h @@ -18,18 +18,19 @@ // CONNECTIONS //===----------------------------------------------------------------------===// // Peloton port +// clang-format off SETTING_int(port, - "Peloton port (default: 15721)", - 15721, - 1024, 65535, - false, false) + "Peloton port (default: 15721)", + 15721, + 1024, 65535, + false, false) // Maximum number of connections SETTING_int(max_connections, - "Maximum number of connections (default: 64)", - 64, - 1, 512, - true, true) + "Maximum number of connections (default: 64)", + 64, + 1, 512, + true, true) SETTING_int(rpc_port, "Peloton rpc port (default: 15445)", @@ -44,68 +45,74 @@ SETTING_int(rpc_port, // See: https://groups.google.com/forum/#!topic/capnproto/bgxCdqGD6oE SETTING_bool(rpc_enabled, "Enable rpc, this should be turned off when testing", - false, false, false) + false, + false, false) // Socket family SETTING_string(socket_family, - "Socket family (default: AF_INET)", - "AF_INET", - false, false) + "Socket family (default: AF_INET)", + "AF_INET", + false, false) // Added for SSL only begins // Enables SSL connection. The default value is false -SETTING_bool(ssl, "Enable SSL connection (default: true)", true, false, false) +SETTING_bool(ssl, + "Enable SSL connection (default: true)", + true, + false, false) // Peloton private key file // Currently use hardcoded private key path, may need to change // to generate file dynamically at runtime // The same applies to certificate file SETTING_string(private_key_file, - "path to private key file", - "peloton_insecure_server.key", - false, false) + "path to private key file", + "peloton_insecure_server.key", + false, false) // Peloton certificate file SETTING_string(certificate_file, - "path to certificate file", - "peloton_insecure_server.crt", - false, false) + "path to certificate file", + "peloton_insecure_server.crt", + false, false) // Peloton root certificate file SETTING_string(root_cert_file, "path to root certificate file", "root.crt", - false, false) + false, false) //===----------------------------------------------------------------------===// // RESOURCE USAGE //===----------------------------------------------------------------------===// SETTING_double(bnlj_buffer_size, - "The default buffer size to use for blockwise nested loop joins (default: 1 MB)", - 1.0 * 1024.0 * 1024.0, - 1.0 * 1024, - 1.0 * 1024.0 * 1024.0 * 1024, - true, true) + "The default buffer size to use for blockwise nested loop " + "joins (default: 1 MB)", + 1.0 * 1024.0 * 1024.0, + 1.0 * 1024, + 1.0 * 1024.0 * 1024.0 * 1024, + true, true) // Size of the MonoQueue task queue SETTING_int(monoqueue_task_queue_size, "MonoQueue Task Queue Size (default: 32)", - 32, + 32, 8, 128, false, false) // Size of the MonoQueue worker pool SETTING_int(monoqueue_worker_pool_size, "MonoQueue Worker Pool Size (default: 4)", - 4, + 4, 1, 32, false, false) // Number of connection threads used by peloton SETTING_int(connection_thread_count, - "Number of connection threads (default: std::hardware_concurrency())", + "Number of connection threads " + "(default: std::hardware_concurrency())", std::thread::hardware_concurrency(), 1, 64, false, false) @@ -130,9 +137,9 @@ SETTING_int(gc_num_threads, // Display configuration SETTING_bool(display_settings, - "Display settings (default: false)", - false, - true, true) + "Display settings (default: false)", + false, + true, true) //===----------------------------------------------------------------------===// // STATISTICS @@ -140,10 +147,10 @@ SETTING_bool(display_settings, // Enable or disable statistics collection SETTING_int(stats_mode, - "Enable statistics collection (default: 0)", - static_cast(peloton::StatsType::INVALID), - 0, 16, - true, true) + "Enable statistics collection (default: 0)", + static_cast(peloton::StatsModeType::DISABLE), + 0, 16, + true, true) //===----------------------------------------------------------------------===// // AI @@ -151,15 +158,15 @@ SETTING_int(stats_mode, // Enable or disable index tuner SETTING_bool(index_tuner, - "Enable index tuner (default: false)", - false, - true, true) + "Enable index tuner (default: false)", + false, + true, true) // Enable or disable layout tuner SETTING_bool(layout_tuner, - "Enable layout tuner (default: false)", - false, - true, true) + "Enable layout tuner (default: false)", + false, + true, true) //===----------------------------------------------------------------------===// // BRAIN @@ -167,14 +174,15 @@ SETTING_bool(layout_tuner, // Enable or disable brain SETTING_bool(brain, - "Enable brain (default: false)", - false, - true, true) + "Enable brain (default: false)", + false, + true, true) -SETTING_string(peloton_address, - "ip and port of the peloton rpc service, address:port", - "127.0.0.1:15445", - false, false) +// Enable or disable data collection +SETTING_bool(brain_data_collection, + "Enable data collection for the brain (default false)", + false, + true, true) // Size of the brain task queue SETTING_int(brain_task_queue_size, @@ -190,15 +198,24 @@ SETTING_int(brain_worker_pool_size, 1, 16, false, false) +SETTING_string(peloton_rpc_address, + "ip and port of the peloton rpc service, address:port", + "127.0.0.1:15445", + false, false) + +SETTING_string(peloton_address, "dbstring to peloton", + "host=127.0.0.1 port=15721 user=default_database " + "sslmode=disable application_name=psql", + false, false) + //===----------------------------------------------------------------------===// // CODEGEN //===----------------------------------------------------------------------===// SETTING_bool(codegen, - "Enable code-generation for query execution (default: true)", - true, - true, true) - + "Enable code-generation for query execution (default: true)", + true, + true, true) //===----------------------------------------------------------------------===// // Optimizer @@ -215,12 +232,13 @@ SETTING_bool(hash_join_bloom_filter, SETTING_int(task_execution_timeout, "Maximum allowed length of time (in ms) for task " - "execution step of optimizer, " - "assuming one plan has been found (default 5000)", + "execution step of optimizer, " + "assuming one plan has been found (default 5000)", 5000, - 1000, 60000, - true, true) + 1000, 60000, + true, true) //===----------------------------------------------------------------------===// // GENERAL //===----------------------------------------------------------------------===// +// clang-format on diff --git a/src/include/statistics/abstract_metric.h b/src/include/statistics/abstract_metric.h index 827d4138688..6f578b21524 100644 --- a/src/include/statistics/abstract_metric.h +++ b/src/include/statistics/abstract_metric.h @@ -4,48 +4,282 @@ // // abstract_metric.h // -// Identification: src/statistics/abstract_metric.h +// Identification: src/include/statistics/abstract_metric.h // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// #pragma once +#include +#include #include #include "common/internal_types.h" -#include "common/printable.h" +#include "common/platform.h" +#include "statistics/abstract_raw_data.h" +#include "statistics/stats_event_type.h" namespace peloton { -namespace stats { +namespace concurrency { +class TransactionContext; +} // namespace concurrency + +namespace stats { /** - * Abstract class for metrics - * A metric should be able to: - * (1) identify its type; - * (2) print itself (ToString); - * (3) reset itself; - * (4) aggregate itself with another source - * of the same type. + * @brief Interface representing a metric. + * A metric is defined to be some piece of logic that processes events generated + * by the database. @see StatsEventType for a list of available events. + * It is guaranteed that the appropriate callback method (identified by the + * naming pattern On[event name]) is invoked and the args filled out with + * relevant information. To enable safe and efficient collection of data, + * it is required that all data be collected be written to a RawData + * object, @see AbstractRawData. + * + * While you could write your own metric directly extending from this class, + * it is recommended that you use @see AbstractMetric class, which takes in + * an AbstractRawData class a a template argument and implements the tricky + * concurrent code for you. + * + * To write a new Metric, first write your own RawData class, extending from + * AbstractRawData, and extend from AbstractMetric with your RawData class as + * template argument. Then, override the event callbacks that you wish to know + * about. @see AbstractMetric on how to deal with concurrency. */ -class AbstractMetric : public Printable { +class Metric { public: - AbstractMetric(MetricType type_); - virtual ~AbstractMetric(); + virtual ~Metric() = default; + + // TODO(tianyu): Add more parameters for events as needed + /** + * @param Context of the transaction beginning + */ + virtual void OnTransactionBegin(const concurrency::TransactionContext *) {}; + + /** + * @param Context of the transaction committing + * @param Tile Group ID that used to track database where the txn happens. + */ + virtual void OnTransactionCommit(const concurrency::TransactionContext *, + oid_t) {}; + + /** + * @param Context of the transaction aborting + * @param Tile Group ID that used to track database where the txn happens. + */ + virtual void OnTransactionAbort(const concurrency::TransactionContext *, + oid_t) {}; + + /** + * @param Context of the transaction performing read + * @param Tile Group ID that used to track database and table + * where the read happens. + */ + virtual void OnTupleRead(const concurrency::TransactionContext *, oid_t) {}; + + /** + * @param Context of the transaction performing update + * @param Tile Group ID that used to track database and table + * where the update happens. + */ + virtual void OnTupleUpdate(const concurrency::TransactionContext *, oid_t) {}; + + /** + * @param Context of the transaction performing insert + * @param Tile Group ID that used to track database and table + * where the insert happens. + */ + virtual void OnTupleInsert(const concurrency::TransactionContext *, oid_t) {}; + + /** + * @param Context of the transaction performing delete + * @param Tile Group ID that used to track database and table + * where the delete happens. + */ + virtual void OnTupleDelete(const concurrency::TransactionContext *, oid_t) {}; + + /** + * @param Database and index id pair that the index read happens + * @param Number of read happening + */ + virtual void OnIndexRead(std::pair, size_t) {}; + + /** + * @param Database and index id pair that the index update happens + */ + virtual void OnIndexUpdate(std::pair) {}; + + /** + * @param Database and index id pair that the index insert happens + */ + virtual void OnIndexInsert(std::pair) {}; + + /** + * @param Database and index id pair that the index delete happens + */ + virtual void OnIndexDelete(std::pair) {}; + + /** + * @param Database and index/table id pair that the memory allocation happens + * @param Number of bytes being allocated + */ + virtual void OnMemoryAlloc(std::pair, size_t) {}; + + /** + * @param Database and index/table id pair that the memory free happens + * @param Number of bytes being freed + */ + virtual void OnMemoryFree(std::pair, size_t) {}; + + /** + * @param Database and index/table id pair that the memory usage happens + * @param Number of bytes being used + */ + virtual void OnMemoryUsage(std::pair, size_t) {}; + + /** + * @param Database and index/table id pair that the memory reclaim happens + * @param Number of bytes being reclaim + */ + virtual void OnMemoryReclaim(std::pair, size_t) {}; - const inline MetricType& GetType() const { return type_; } + /** + * @brief collect the signal of query begin + */ + virtual void OnQueryBegin() {}; - virtual void Reset() = 0; + /** + * @brief collect the signal of query end + */ + virtual void OnQueryEnd() {}; - virtual const std::string GetInfo() const = 0; + /** + * @brief Event used to test the framework + */ + virtual void OnTest(int) {}; - virtual void Aggregate(AbstractMetric& source) = 0; + /** + * @brief Replace RawData with an empty one and return the old one. + * + * Data from a metric is collected first into a thread-local storage to + * ensure efficiency and safety, and periodically aggregated by an aggregator + * thread into meaningful statistics. However, new statistics can still come + * in when we aggregate, resulting in race conditions. To avoid this, every + * time the aggregator wishes to aggregate data, the RawData object is + * extracted and a fresh one swapped in, so collection continues seamlessly + * while the aggregator is working. + * + * Unless you know what you are doing, you should probably just use the one + * implemented for you(@see AbstractMetric). Otherwise, it is guaranteed that + * this method is only called from the aggregator thread, so it is okay to + * block in this method. As soon as this method returns, the aggregator + * assumes that it is safe to start reading from the data and discards the + * data after it's done. Therefore, it is essential that any implementation + * ensures this method does not return if the collecting thread still can + * write to the old raw data. + * + * @return a shared pointer to the old AbstractRawData + */ + virtual std::shared_ptr Swap() = 0; +}; + +/* Forward Declaration */ +template +class AbstractMetric; + +/** + * @brief Wraps around a pointer to an AbstractRawData to allow safe access. + * + * This class is always handed out by an AbstractMetric and would prevent an + * Aggregator from reading or deleting the AbstractRawData it holds. When the + * object goes out of scope, its destructor will unblock the aggregator. Access + * to its underlying pointer is always non-blocking. + * + * @tparam DataType the type of AbstractRawData this Wrapper holds + */ +template +class RawDataWrapper { + friend class AbstractMetric; + + public: + inline RawDataWrapper(RawDataWrapper &&other) = default; + + inline ~RawDataWrapper() { safe_ = true; } // Unblock aggregator + + DISALLOW_COPY(RawDataWrapper); + + /** + * @return the underlying pointer + */ + inline DataType *operator->() const { return ptr_; } private: - // The type this metric belongs to - MetricType type_; + /** + * Constructs a new Wrapper instance + * @param ptr the pointer it wraps around + * @param safe the boolean variable it uses to signal its lifetime + */ + inline RawDataWrapper(DataType *ptr, std::atomic &safe) + : ptr_(ptr), safe_(safe) {} + DataType *ptr_; + std::atomic &safe_; }; +/** + * @brief General purpose implementation to Metric that you should inherit from. + * + * This class implements the tricky Swap method and exposes an interface for + * children class. @see Metric for detail + * + * @tparam DataType the type of AbstractRawData this Metric holds + */ +template +class AbstractMetric : public Metric { + public: + AbstractMetric() : raw_data_(new DataType()), safe_{true} {} + + ~AbstractMetric() { delete raw_data_.load(); } + /** + * @see Metric + * + * To ensure this method works as intended, be sure to use GetRawData() to + * access the underlying raw data + * @return a shared pointer to the old AbstractRawData + */ + std::shared_ptr Swap() override { + // After this point, the collector thread can not see old data on new + // events, but will still be able to write to it, if they loaded the + // pointer before this operation but haven't written to it yet. + DataType *old_data = raw_data_.exchange(new DataType()); + // We will need to wait for last writer to finish before it's safe + // to start reading the content. It is okay to block since this + // method should only be called from the aggregator thread. + while (!safe_) _mm_pause(); + return std::shared_ptr(old_data); + } + + protected: + /** + * @see RawDataWrapper + * + * Always use this method to access the raw data within an AbstractMetric. + * @return a RawDataWrapper object to access raw_data_ + */ + inline RawDataWrapper GetRawData() { + // safe_ should first be flipped to false before loading the raw_data_ so + // that the aggregator would always be blocked when it tries to swap out if + // there is a reader. At most one instance of this should be live at any + // given time. + PELOTON_ASSERT(safe_); + safe_ = false; + return {raw_data_.load(), safe_}; + } + + private: + std::atomic raw_data_; + std::atomic safe_; +}; } // namespace stats } // namespace peloton diff --git a/src/include/statistics/abstract_raw_data.h b/src/include/statistics/abstract_raw_data.h new file mode 100644 index 00000000000..3baa199fee6 --- /dev/null +++ b/src/include/statistics/abstract_raw_data.h @@ -0,0 +1,59 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// abstract_raw_data.h +// +// Identification: src/include/statistics/abstract_raw_data.h +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once +#include +#include "common/printable.h" + +namespace peloton { +namespace stats { +/** + * @brief An always-consistent storage unit for intermediate stats results. + * + * These objects hold raw data points processed by a metric on the thread-local + * level. Entries into this object must be always consistent. + * (i.e. future entries should not rely on some early entries being in this + * object) + * This is because an aggregator can come at any time and swap out + * the object for aggregation. + * + * @see Metric for detailed description of how this would work. + */ +class AbstractRawData : public Printable { + public: + /** + * Given another AbstractRawData classes, combine the other's content with the + * content of this one. It is guaranteed that nobody will have access to the + * other object at this point or after. + * @param other The other AbstractRawData to be merged + */ + virtual void Aggregate(AbstractRawData &other) = 0; + /** + * Make necessary updates to the metric raw data and persist the content of + * this RawData into the Catalog. Expect this object + * to be garbage-collected after this method is called. + */ + virtual void UpdateAndPersist() = 0; + + protected: + struct pair_hash { + template + inline std::size_t operator()(const std::pair &p) const { + size_t seed = 0; + boost::hash_combine(seed, p.first); + boost::hash_combine(seed, p.second); + return seed; + } + }; +}; +} // namespace stats +} // namespace peloton diff --git a/src/include/statistics/access_metric.h b/src/include/statistics/access_metric.h deleted file mode 100644 index 701dbc5834e..00000000000 --- a/src/include/statistics/access_metric.h +++ /dev/null @@ -1,151 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// access_metric.h -// -// Identification: src/statistics/access_metric.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include - -#include "common/internal_types.h" -#include "statistics/abstract_metric.h" -#include "statistics/counter_metric.h" - -namespace peloton { -namespace stats { - -/** - * Metric that counts the number of reads, updates, - * inserts, and deletes for a given storage type - * (e.g. index or table). - */ -class AccessMetric : public AbstractMetric { - public: - AccessMetric(MetricType type) : AbstractMetric(type) {} - - //===--------------------------------------------------------------------===// - // ACCESSORS - //===--------------------------------------------------------------------===// - - inline void IncrementReads() { access_counters_[READ_COUNTER].Increment(); } - - inline void IncrementUpdates() { - access_counters_[UPDATE_COUNTER].Increment(); - } - - inline void IncrementInserts() { - access_counters_[INSERT_COUNTER].Increment(); - } - - inline void IncrementDeletes() { - access_counters_[DELETE_COUNTER].Increment(); - } - - inline void IncrementReads(int64_t count) { - access_counters_[READ_COUNTER].Increment(count); - } - - inline void IncrementUpdates(int64_t count) { - access_counters_[UPDATE_COUNTER].Increment(count); - } - - inline void IncrementInserts(int64_t count) { - access_counters_[INSERT_COUNTER].Increment(count); - } - - inline void IncrementDeletes(int64_t count) { - access_counters_[DELETE_COUNTER].Increment(count); - } - - inline int64_t GetReads() { - return access_counters_[READ_COUNTER].GetCounter(); - } - - inline int64_t GetUpdates() { - return access_counters_[UPDATE_COUNTER].GetCounter(); - } - - inline int64_t GetInserts() { - return access_counters_[INSERT_COUNTER].GetCounter(); - } - - inline int64_t GetDeletes() { - return access_counters_[DELETE_COUNTER].GetCounter(); - } - - inline CounterMetric &GetAccessCounter(size_t counter_type) { - return access_counters_[counter_type]; - } - - //===--------------------------------------------------------------------===// - // HELPER METHODS - //===--------------------------------------------------------------------===// - - inline bool operator==(const AccessMetric &other) { - for (size_t access_counter_itr = 0; - access_counter_itr < access_counters_.size(); ++access_counter_itr) { - if (access_counters_[access_counter_itr] != - other.access_counters_[access_counter_itr]) { - return false; - } - } - return true; - } - - inline bool operator!=(const AccessMetric &other) { - return !(*this == other); - } - - // Resets all access counters to zero - inline void Reset() { - for (auto &counter : access_counters_) { - counter.Reset(); - } - } - - // Returns a string representation of this access metric - inline const std::string GetInfo() const { - std::stringstream ss; - ss << "[ reads=" << access_counters_[READ_COUNTER].GetInfo() - << ", updates=" << access_counters_[UPDATE_COUNTER].GetInfo() - << ", inserts=" << access_counters_[INSERT_COUNTER].GetInfo() - << ", deletes=" << access_counters_[DELETE_COUNTER].GetInfo() << " ]"; - return ss.str(); - } - - // Adds the counters from the source access metric - // to the counters in this access metric - void Aggregate(AbstractMetric &source); - - private: - //===--------------------------------------------------------------------===// - // MEMBERS - //===--------------------------------------------------------------------===// - - // Vector containing all access types - std::vector access_counters_{ - CounterMetric(MetricType::COUNTER), // READ_COUNTER - CounterMetric(MetricType::COUNTER), // UPDATE_COUNTER - CounterMetric(MetricType::COUNTER), // INSERT_COUNTER - CounterMetric(MetricType::COUNTER) // DELETE_COUNTER - }; - - // The different types of accesses. These also - // serve as indexes into the access_counters_ - // vector. - static const size_t READ_COUNTER = 0; - static const size_t UPDATE_COUNTER = 1; - static const size_t INSERT_COUNTER = 2; - static const size_t DELETE_COUNTER = 3; - static const size_t NUM_COUNTERS = 4; -}; - -} // namespace stats -} // namespace peloton diff --git a/src/include/statistics/backend_stats_context.h b/src/include/statistics/backend_stats_context.h deleted file mode 100644 index 2abce1d93f4..00000000000 --- a/src/include/statistics/backend_stats_context.h +++ /dev/null @@ -1,195 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// backend_stats_context.h -// -// Identification: src/statistics/backend_stats_context.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include -#include -#include - -#include "common/container/cuckoo_map.h" -#include "common/container/lock_free_queue.h" -#include "common/platform.h" -#include "common/synchronization/spin_latch.h" -#include "statistics/database_metric.h" -#include "statistics/index_metric.h" -#include "statistics/latency_metric.h" -#include "statistics/query_metric.h" -#include "statistics/table_metric.h" - -#define QUERY_METRIC_QUEUE_SIZE 100000 - -namespace peloton { - -class Statement; - -namespace index { -class IndexMetadata; -} // namespace index - -namespace stats { - -class CounterMetric; - -/** - * Context of backend stats as a singleton per thread - */ -class BackendStatsContext { - public: - static BackendStatsContext *GetInstance(); - - BackendStatsContext(size_t max_latency_history, bool regiser_to_aggregator); - ~BackendStatsContext(); - - //===--------------------------------------------------------------------===// - // ACCESSORS - //===--------------------------------------------------------------------===// - - inline std::thread::id GetThreadId() { return thread_id_; } - - // Returns the table metric with the given database ID and table ID - TableMetric *GetTableMetric(oid_t database_id, oid_t table_id); - - // Returns the database metric with the given database ID - DatabaseMetric *GetDatabaseMetric(oid_t database_id); - - // Returns the index metric with the given database ID, table ID, and - // index ID - IndexMetric *GetIndexMetric(oid_t database_id, oid_t table_id, - oid_t index_id); - - // Returns the metrics for completed queries - LockFreeQueue> &GetCompletedQueryMetrics() { - return completed_query_metrics_; - }; - - // Returns the metric for the on going query - QueryMetric *GetOnGoingQueryMetric() { return ongoing_query_metric_.get(); } - - // Returns the latency metric - LatencyMetric &GetTxnLatencyMetric(); - - // Increment the read stat for given tile group - void IncrementTableReads(oid_t tile_group_id); - - // Increment the insert stat for given tile group - void IncrementTableInserts(oid_t tile_group_id); - - // Increment the update stat for given tile group - void IncrementTableUpdates(oid_t tile_group_id); - - // Increment the delete stat for given tile group - void IncrementTableDeletes(oid_t tile_group_id); - - // Increment the read stat for given index by read_count - void IncrementIndexReads(size_t read_count, index::IndexMetadata *metadata); - - // Increment the insert stat for index - void IncrementIndexInserts(index::IndexMetadata *metadata); - - // Increment the update stat for index - void IncrementIndexUpdates(index::IndexMetadata *metadata); - - // Increment the delete stat for index - void IncrementIndexDeletes(size_t delete_count, - index::IndexMetadata *metadata); - - // Increment the commit stat for given database - void IncrementTxnCommitted(oid_t database_id); - - // Increment the abortion stat for given database - void IncrementTxnAborted(oid_t database_id); - - // Initialize the query stat - void InitQueryMetric(const std::shared_ptr statement, - const std::shared_ptr params); - - //===--------------------------------------------------------------------===// - // HELPER FUNCTIONS - //===--------------------------------------------------------------------===// - - /** - * Aggregate another BackendStatsContext to myself - */ - void Aggregate(BackendStatsContext &source); - - // Resets all metrics (and sub-metrics) to their starting state - // (e.g., sets all counters to zero) - void Reset(); - - std::string ToString() const; - - // Returns the total number of query aggregated so far - oid_t GetQueryCount() { return aggregated_query_count_; } - - // Resets the total number of query aggregated to zero - void ResetQueryCount() { aggregated_query_count_ = 0; } - - //===--------------------------------------------------------------------===// - // MEMBERS - //===--------------------------------------------------------------------===// - - // Database metrics - std::unordered_map> - database_metrics_{}; - - // Table metrics - std::unordered_map> table_metrics_{}; - - // Index metrics - CuckooMap> index_metrics_{}; - - // Index oids - std::unordered_set index_ids_; - - // Metrics for completed queries - LockFreeQueue> completed_query_metrics_{ - QUERY_METRIC_QUEUE_SIZE}; - - private: - //===--------------------------------------------------------------------===// - // MEMBERS - //===--------------------------------------------------------------------===// - - // The query metric for the on going metric - std::shared_ptr ongoing_query_metric_ = nullptr; - - // The thread ID of this worker - std::thread::id thread_id_; - - // Latencies recorded by this worker - LatencyMetric txn_latencies_; - - // Whether this context is registered to the global aggregator - bool is_registered_to_aggregator_; - - // The total number of queries aggregated - oid_t aggregated_query_count_ = 0; - - // Index oid spin lock - common::synchronization::SpinLatch index_id_lock; - - //===--------------------------------------------------------------------===// - // HELPER FUNCTIONS - //===--------------------------------------------------------------------===// - - // Mark the on going query as completed and move it to completed query queue - void CompleteQueryMetric(); - - // Get the mapping table of backend stat context for each thread - static CuckooMap> - &GetBackendContextMap(void); -}; - -} // namespace stats -} // namespace peloton diff --git a/src/include/statistics/counter_metric.h b/src/include/statistics/counter_metric.h deleted file mode 100644 index 6d03045fcf8..00000000000 --- a/src/include/statistics/counter_metric.h +++ /dev/null @@ -1,79 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// counter_metric.h -// -// Identification: src/statistics/counter_metric.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include - -#include "common/internal_types.h" -#include "statistics/abstract_metric.h" - -namespace peloton { -namespace stats { - -/** - * Metric as a counter. E.g. # txns committed, # tuples read, etc. - */ -class CounterMetric : public AbstractMetric { - public: - CounterMetric(MetricType type); - - //===--------------------------------------------------------------------===// - // ACCESSORS - //===--------------------------------------------------------------------===// - - inline void Increment() { count_++; } - - inline void Increment(int64_t count) { count_ += count; } - - inline void Decrement() { count_--; } - - inline void Decrement(int64_t count) { count_ -= count; } - - //===--------------------------------------------------------------------===// - // HELPER METHODS - //===--------------------------------------------------------------------===// - - inline void Reset() { count_ = 0; } - - inline int64_t GetCounter() { return count_; } - - inline bool operator==(const CounterMetric &other) { - return count_ == other.count_; - } - - inline bool operator!=(const CounterMetric &other) { - return !(*this == other); - } - - // Adds the source counter to this counter - void Aggregate(AbstractMetric &source); - - // Returns a string representation of this counter - inline const std::string GetInfo() const { - std::stringstream ss; - ss << count_; - return ss.str(); - } - - private: - //===--------------------------------------------------------------------===// - // MEMBERS - //===--------------------------------------------------------------------===// - - // The current count - int64_t count_; -}; - -} // namespace stats -} // namespace peloton diff --git a/src/include/statistics/database_metric.h b/src/include/statistics/database_metric.h index 23899373586..0c682ac6343 100644 --- a/src/include/statistics/database_metric.h +++ b/src/include/statistics/database_metric.h @@ -4,9 +4,9 @@ // // database_metric.h // -// Identification: src/statistics/database_metric.h +// Identification: src/include/statistics/database_metric.h // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// @@ -15,70 +15,82 @@ #include #include +#include "catalog/manager.h" +#include "catalog/database_metrics_catalog.h" #include "common/internal_types.h" -#include "statistics/counter_metric.h" #include "statistics/abstract_metric.h" +#include "storage/tile_group.h" +#include "type/ephemeral_pool.h" namespace peloton { -namespace stats { - -/** - * Database-specific metrics, including the number of committed/aborted txns. - */ -class DatabaseMetric : public AbstractMetric { - public: - DatabaseMetric(MetricType type, oid_t database_id); - - //===--------------------------------------------------------------------===// - // ACCESSORS - //===--------------------------------------------------------------------===// - - inline void IncrementTxnCommitted() { txn_committed_.Increment(); } - - inline void IncrementTxnAborted() { txn_aborted_.Increment(); } - - inline CounterMetric &GetTxnCommitted() { return txn_committed_; } - - inline CounterMetric &GetTxnAborted() { return txn_aborted_; } - - inline oid_t GetDatabaseId() { return database_id_; } - //===--------------------------------------------------------------------===// - // HELPER METHODS - //===--------------------------------------------------------------------===// +namespace concurrency { +class TransactionContext; +} // namespace concurrency - inline void Reset() { - txn_committed_.Reset(); - txn_aborted_.Reset(); +namespace stats { +class DatabaseMetricRawData : public AbstractRawData { + public: + inline void IncrementTxnCommited(oid_t database_id) { + counters_[database_id].first++; } - inline bool operator==(const DatabaseMetric &other) { - return database_id_ == other.database_id_ && - txn_committed_ == other.txn_committed_ && - txn_aborted_ == other.txn_aborted_; + inline void IncrementTxnAborted(oid_t database_id) { + counters_[database_id].second++; } - inline bool operator!=(const DatabaseMetric &other) { - return !(*this == other); + void Aggregate(AbstractRawData &other) override { + auto &other_db_metric = dynamic_cast(other); + for (auto &entry : other_db_metric.counters_) { + auto &this_counter = counters_[entry.first]; + auto &other_counter = entry.second; + this_counter.first += other_counter.first; + this_counter.second += other_counter.second; + } } - void Aggregate(AbstractMetric &source); + void UpdateAndPersist() override; - const std::string GetInfo() const; + // TODO(Tianyu): Pretty Print + const std::string GetInfo() const override { return ""; } private: - //===--------------------------------------------------------------------===// - // MEMBERS - //===--------------------------------------------------------------------===// + inline static std::pair GetDBTableIdFromTileGroupOid( + oid_t tile_group_id) { + auto tile_group = + catalog::Manager::GetInstance().GetTileGroup(tile_group_id); + if (tile_group == nullptr) return {INVALID_OID, INVALID_OID}; + return {tile_group->GetDatabaseId(), tile_group->GetTableId()}; + } + /** + * Maps from database id to a pair of counters. + * + * First counter represents number of transactions committed and the second + * one represents the number of transactions aborted. + */ + std::unordered_map> counters_; +}; - // The ID of this database - oid_t database_id_; +class DatabaseMetric : public AbstractMetric { + public: + inline void OnTransactionCommit(const concurrency::TransactionContext *, oid_t tile_group_id) override { + oid_t database_id = GetDBTableIdFromTileGroupOid(tile_group_id).first; + GetRawData()->IncrementTxnCommited(database_id); + } - // Count of the number of transactions committed - CounterMetric txn_committed_{MetricType::COUNTER}; + inline void OnTransactionAbort(const concurrency::TransactionContext *, oid_t tile_group_id) override { + oid_t database_id = GetDBTableIdFromTileGroupOid(tile_group_id).first; + GetRawData()->IncrementTxnAborted(database_id); + } - // Count of the number of transactions aborted - CounterMetric txn_aborted_{MetricType::COUNTER}; + private: + inline static std::pair GetDBTableIdFromTileGroupOid( + oid_t tile_group_id) { + auto tile_group = + catalog::Manager::GetInstance().GetTileGroup(tile_group_id); + if (tile_group == nullptr) return {INVALID_OID, INVALID_OID}; + return {tile_group->GetDatabaseId(), tile_group->GetTableId()}; + } }; } // namespace stats diff --git a/src/include/statistics/index_metric.h b/src/include/statistics/index_metric.h index 9ab8d9c666b..20bb4f98aa5 100644 --- a/src/include/statistics/index_metric.h +++ b/src/include/statistics/index_metric.h @@ -4,9 +4,9 @@ // // index_metric.h // -// Identification: src/statistics/index_metric.h +// Identification: src/include/statistics/index_metric.h // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// @@ -18,81 +18,116 @@ #include "common/internal_types.h" #include "statistics/abstract_metric.h" -#include "statistics/access_metric.h" #include "util/string_util.h" namespace peloton { namespace stats { +class IndexMetricRawData : public AbstractRawData { + // this serves as an index into each table's counter vector + enum CounterType { + READ = 0, + UPDATE, + INSERT, + DELETE, + MEMORY_ALLOC, + MEMORY_USAGE + }; -/** - * Metric of index accesses and other index-specific metrics. - */ -class IndexMetric : public AbstractMetric { public: - typedef std::string IndexKey; - - IndexMetric(MetricType type, oid_t database_id, oid_t table_id, - oid_t index_id); - - //===--------------------------------------------------------------------===// - // ACCESSORS - //===--------------------------------------------------------------------===// - - // Returns a metric containing the counts of all - // accesses to this index - inline AccessMetric &GetIndexAccess() { return index_access_; } + inline void IncrementIndexReads(std::pair db_index_id, + size_t num_read) { + GetCounter(db_index_id, READ) += num_read; + } - inline std::string GetName() { return index_name_; } + inline void IncrementIndexUpdates(std::pair db_index_id) { + GetCounter(db_index_id, UPDATE)++; + } - inline oid_t GetDatabaseId() { return database_id_; } + inline void IncrementIndexInserts(std::pair db_index_id) { + GetCounter(db_index_id, INSERT)++; + } - inline oid_t GetTableId() { return table_id_; } + inline void IncrementIndexDeletes(std::pair db_index_id) { + GetCounter(db_index_id, DELETE)++; + } - inline oid_t GetIndexId() { return index_id_; } + inline void IncrementIndexMemoryAlloc(std::pair db_index_id, + size_t bytes) { + GetCounter(db_index_id, MEMORY_ALLOC) += bytes; + } - //===--------------------------------------------------------------------===// - // HELPER METHODS - //===--------------------------------------------------------------------===// + inline void DecrementIndexMemoryAlloc(std::pair db_index_id, + size_t bytes) { + GetCounter(db_index_id, MEMORY_ALLOC) -= bytes; + } - inline void Reset() { index_access_.Reset(); } + inline void IncrementIndexMemoryUsage(std::pair db_index_id, + size_t bytes) { + GetCounter(db_index_id, MEMORY_USAGE) += bytes; + } - inline bool operator==(const IndexMetric &other) { - return database_id_ == other.database_id_ && table_id_ == other.table_id_ && - index_id_ == other.index_id_ && index_name_ == other.index_name_ && - index_access_ == other.index_access_; + inline void DecrementIndexMemoryUsage(std::pair db_index_id, + size_t bytes) { + GetCounter(db_index_id, MEMORY_USAGE) -= bytes; } - inline bool operator!=(const IndexMetric &other) { return !(*this == other); } + void Aggregate(AbstractRawData &other) override; - void Aggregate(AbstractMetric &source); + void UpdateAndPersist() override; - inline const std::string GetInfo() const { - std::stringstream ss; - ss << "INDEXES: " << std::endl; - ss << index_name_ << "(OID=" << index_id_ << "): "; - ss << index_access_.GetInfo(); - return ss.str(); - } + const std::string GetInfo() const override { return "index metric"; } private: - //===--------------------------------------------------------------------===// - // MEMBERS - //===--------------------------------------------------------------------===// + inline int64_t &GetCounter(std::pair db_index_id, + CounterType type) { + auto entry = counters_.find(db_index_id); + if (entry == counters_.end()) + counters_[db_index_id] = std::vector(NUM_COUNTERS); + return counters_[db_index_id][type]; + } - // The database ID of this index - oid_t database_id_; + // map from (database_oid, index_oid) pair to vector of counters, + // where CounterType enum describes what kind of counter is kept at what index + std::unordered_map, std::vector, pair_hash> + counters_; - // The table ID of this index - oid_t table_id_; + // list of counter types (used by Aggregate) + static const std::vector COUNTER_TYPES; + // should be equal to COUNTER_TYPES.size() + static const int NUM_COUNTERS = 6; +}; - // The ID of this index - oid_t index_id_; +class IndexMetric : public AbstractMetric { + public: + inline void OnIndexRead(std::pair db_index_id, + size_t num_read) override { + GetRawData()->IncrementIndexReads(db_index_id, num_read); + } - // The name of this index - std::string index_name_; + inline void OnIndexUpdate(std::pair db_index_id) override { + GetRawData()->IncrementIndexUpdates(db_index_id); + } + + inline void OnIndexInsert(std::pair db_index_id) override { + GetRawData()->IncrementIndexInserts(db_index_id); + } - // Counts the number of index entries accessed - AccessMetric index_access_{MetricType::ACCESS}; + inline void OnIndexDelete(std::pair db_index_id) override { + GetRawData()->IncrementIndexDeletes(db_index_id); + } + inline void OnMemoryAlloc(std::pair db_index_id, size_t bytes) override { + GetRawData()->IncrementIndexMemoryAlloc(db_index_id, bytes); + }; + inline void OnMemoryFree(std::pair db_index_id, size_t bytes) override { + GetRawData()->DecrementIndexMemoryAlloc(db_index_id, bytes); + }; + inline void OnMemoryUsage(std::pair db_index_id, size_t bytes) override { + GetRawData()->IncrementIndexMemoryUsage(db_index_id, bytes); + }; + inline void OnMemoryReclaim(std::pair db_index_id, + size_t bytes) override { + GetRawData()->DecrementIndexMemoryUsage(db_index_id, bytes); + }; }; } // namespace stats diff --git a/src/include/statistics/latency_metric.h b/src/include/statistics/latency_metric.h index 34bd8f75af1..d4344c40075 100644 --- a/src/include/statistics/latency_metric.h +++ b/src/include/statistics/latency_metric.h @@ -4,9 +4,9 @@ // // latency_metric.h // -// Identification: src/statistics/latency_metric.h +// Identification: src/include/statistics/latency_metric.h // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// @@ -37,83 +37,58 @@ struct LatencyMeasurements { double perc_99th_ = 0.0; }; -/** - * Metric for storing raw latency values and computing - * latency measurements. - */ -class LatencyMetric : public AbstractMetric { +class LatencyMetricRawData : public AbstractRawData { public: - LatencyMetric(MetricType type, size_t max_history); + // TODO (Justin): remove hard-coded constant + // Probably want agg structure to have more capacity + LatencyMetricRawData(size_t max_history = 100) { + latencies_.SetCapacity(max_history); + } - //===--------------------------------------------------------------------===// - // HELPER METHODS - //===--------------------------------------------------------------------===// + inline void RecordLatency(const double val) { latencies_.PushBack(val); } - inline void Reset() { - latencies_.Clear(); - timer_ms_.Reset(); + void Aggregate(AbstractRawData &other) { + auto &other_latency_metric = dynamic_cast(other); + for (double next_latency : other_latency_metric.latencies_) { + latencies_.PushBack(next_latency); + } } - // Starts the timer for the next latency measurement - inline void StartTimer() { + // Computes descriptive statistics on the aggregated latencies, + // then writes these computed values to the catalog. + void UpdateAndPersist(); + + private: + /** + * @brief Calculate descriptive statistics on raw latency measurements. + * + * Should only be called by aggregator thread, after it has aggregated + * latencies from all worker threads. + * Only then does it make sense to calculate stats such as min, max, and + *percentiles. + */ + LatencyMeasurements DescriptiveFromRaw(); + + // Circular buffer with capacity N that stores up to the N + // most recent latencies collected + CircularBuffer latencies_; +}; + +class LatencyMetric : public AbstractMetric { + public: + inline void OnQueryBegin() { timer_ms_.Reset(); timer_ms_.Start(); } - // Stops the latency timer and records the total time elapsed - inline void RecordLatency() { + inline void OnQueryEnd() { timer_ms_.Stop(); - double latency_value = timer_ms_.GetDuration(); - // Record this latency only if we can do so without blocking. - // Occasionally losing single latency measurements is fine. - { - std::unique_lock lock(latency_mutex_, std::defer_lock); - if (lock.try_lock()) { - latencies_.PushBack(latency_value); - } - } - } - - // Returns the first latency value recorded - inline double GetFirstLatencyValue() { - PELOTON_ASSERT(latencies_.begin() != latencies_.end()); - return *(latencies_.begin()); + GetRawData()->RecordLatency(timer_ms_.GetDuration()); } - // Computes the latency measurements using the latencies - // collected so far. - void ComputeLatencies(); - - // Combines the source latency metric with this latency metric - void Aggregate(AbstractMetric &source); - - // Returns a string representation of this latency metric - const std::string GetInfo() const; - - // Returns a copy of the latencies collected - CircularBuffer Copy(); - private: - //===--------------------------------------------------------------------===// - // MEMBERS - //===--------------------------------------------------------------------===// - - // Circular buffer with capacity N that stores the <= N - // most recent latencies collected - CircularBuffer latencies_; - // Timer for timing individual latencies Timer> timer_ms_; - - // Stores result of last call to ComputeLatencies() - LatencyMeasurements latency_measurements_; - - // The maximum number of latencies that can be stored - // (the capacity size N of the circular buffer) - size_t max_history_; - - // Protects accesses to the circular buffer - std::mutex latency_mutex_; }; } // namespace stats diff --git a/src/include/statistics/processor_metric.h b/src/include/statistics/processor_metric.h deleted file mode 100644 index 1c3fdaf2781..00000000000 --- a/src/include/statistics/processor_metric.h +++ /dev/null @@ -1,97 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// processor_metric.h -// -// Identification: src/statistics/processor_metric.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include - -#include "common/internal_types.h" -#include "common/exception.h" -#include "common/macros.h" - -#include -#include "statistics/abstract_metric.h" -#include "statistics/access_metric.h" - -namespace peloton { -namespace stats { - -/** - * Metric for storing raw processor execution time values. - */ -class ProcessorMetric : public AbstractMetric { - public: - ProcessorMetric(MetricType type); - - // Reset the metric - inline void Reset() { - user_time_begin_ = 0; - user_time_end_ = 0; - sys_time_begin_ = 0; - sys_time_end_ = 0; - } - - // Starts the timer - void StartTimer(); - - // Stops the timer and records the total time of execution - void RecordTime(); - - // Get the CPU time for user execution (ms) - inline double GetUserDuration() const { - PELOTON_ASSERT(user_time_end_ - user_time_begin_ >= 0); - return user_time_end_ - user_time_begin_; - } - - // Get the CPU time for system execution (ms) - inline double GetSystemDuration() const { - PELOTON_ASSERT(sys_time_end_ - sys_time_begin_ >= 0); - return sys_time_end_ - sys_time_begin_; - } - - // Returns a string representation of this latency metric - const std::string GetInfo() const; - - // Combines the source processor metric with this processor metric - void Aggregate(AbstractMetric &source); - - private: - //===--------------------------------------------------------------------===// - // HELPER METHODS - //===--------------------------------------------------------------------===// - - // Utility function to convert struc timeval to millisecond - inline double GetMilliSec(struct timeval time) const; - - // Internal function to update CPU time values - void UpdateTimeInt(double &user_time, double &system_time); - - //===--------------------------------------------------------------------===// - // MEMBERS - //===--------------------------------------------------------------------===// - - // Begin CPU time (ms) for user execution - double user_time_begin_ = 0; - - // End CPU time (ms) for user execution - double user_time_end_ = 0; - - // Begin CPU time (ms) for system execution - double sys_time_begin_ = 0; - - // End CPU time (ms) for system execution - double sys_time_end_ = 0; -}; - -} // namespace stats -} // namespace peloton diff --git a/src/include/statistics/query_metric.h b/src/include/statistics/query_metric.h deleted file mode 100644 index 27e90bc7979..00000000000 --- a/src/include/statistics/query_metric.h +++ /dev/null @@ -1,130 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// query_metric.h -// -// Identification: src/statistics/query_metric.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include -#include -#include "common/internal_types.h" -#include "statistics/abstract_metric.h" -#include "statistics/access_metric.h" -#include "statistics/latency_metric.h" -#include "statistics/processor_metric.h" -#include "util/string_util.h" - -namespace peloton { - -namespace stats { - -// Same type defined in network/marshal.h -typedef unsigned char uchar; - -/** - * Metric for the access of a query - */ -class QueryMetric : public AbstractMetric { - public: - // A wrapper of the query param buffer copy - struct QueryParamBuf { - uchar *buf = nullptr; - int len = 0; - - // Default constructor - QueryParamBuf() {} - QueryParamBuf(uchar *buf, int len) : buf(buf), len(len) {} - }; - - // A wrapper of the query params used in prepared statement - struct QueryParams { - - QueryParams(QueryParamBuf format_buf_copy, QueryParamBuf type_buf_copy, - QueryParamBuf val_buf_copy, int num_params); - - // A copy of paramter format buffer - QueryParamBuf format_buf_copy; - - // A copy of the types of the params - QueryParamBuf type_buf_copy; - - // A copy of paramter value buffer - QueryParamBuf val_buf_copy; - - // Number of parameters - int num_params = 0; - }; - - QueryMetric(MetricType type, const std::string &query_name, - std::shared_ptr query_params, - const oid_t database_id); - - //===--------------------------------------------------------------------===// - // ACCESSORS - //===--------------------------------------------------------------------===// - - inline AccessMetric &GetQueryAccess() { return query_access_; } - - inline LatencyMetric &GetQueryLatency() { return latency_metric_; } - - inline ProcessorMetric &GetProcessorMetric() { return processor_metric_; } - - inline std::string GetName() const { return query_name_; } - - inline oid_t GetDatabaseId() const { return database_id_; } - - inline std::shared_ptr const GetQueryParams() { - return query_params_; - } - - //===--------------------------------------------------------------------===// - // HELPER FUNCTIONS - //===--------------------------------------------------------------------===// - - inline void Reset() { query_access_.Reset(); } - - void Aggregate(AbstractMetric &source); - - inline const std::string GetInfo() const { - std::stringstream ss; - ss << peloton::GETINFO_SINGLE_LINE << std::endl; - ss << " QUERY " << query_name_ << std::endl; - ss << peloton::GETINFO_SINGLE_LINE << std::endl; - ss << query_access_.GetInfo(); - return ss.str(); - } - - private: - //===--------------------------------------------------------------------===// - // MEMBERS - //===--------------------------------------------------------------------===// - - // The database ID of this query - oid_t database_id_; - - // The name of this query - std::string query_name_; - - // The parameter of this query - std::shared_ptr query_params_; - - // The number of tuple accesses - AccessMetric query_access_{MetricType::ACCESS}; - - // Latency metric - LatencyMetric latency_metric_{MetricType::LATENCY, 2}; - - // Processor metric - ProcessorMetric processor_metric_{MetricType::PROCESSOR}; -}; - -} // namespace stats -} // namespace peloton diff --git a/src/include/statistics/stats_aggregator.h b/src/include/statistics/stats_aggregator.h index 5f5fcdd1911..2f1fbd1c41e 100644 --- a/src/include/statistics/stats_aggregator.h +++ b/src/include/statistics/stats_aggregator.h @@ -4,9 +4,9 @@ // // stats_aggregator.h // -// Identification: src/statistics/stats_aggregator.h +// Identification: src/include/statistics/stats_aggregator.h // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// @@ -22,148 +22,46 @@ #include "common/logger.h" #include "common/macros.h" -#include "concurrency/transaction_context.h" -#include "statistics/backend_stats_context.h" -#include "storage/data_table.h" #include "storage/database.h" +#include "storage/data_table.h" +#include "concurrency/transaction_context.h" +#include "common/dedicated_thread_task.h" +#include "thread_level_stats_collector.h" +#include "type/ephemeral_pool.h" //===--------------------------------------------------------------------===// // GUC Variables //===--------------------------------------------------------------------===// #define STATS_AGGREGATION_INTERVAL_MS 1000 -#define STATS_LOG_INTERVALS 10 -#define LATENCY_MAX_HISTORY_THREAD 100 -#define LATENCY_MAX_HISTORY_AGGREGATOR 10000 - -class BackendStatsContext; namespace peloton { namespace stats { -//===--------------------------------------------------------------------===// -// Stats Aggregator -//===--------------------------------------------------------------------===// - -// One singleton stats aggregator over the whole DBMS. Worker threads register -// their BackendStatsContext pointer to this aggregator. And this singleton -// aggregator call Aggregate() periodically to aggregate stats from all worker -// threads. Then print them out or log them into a file. - -/** - * Global Stats Aggregator - */ -class StatsAggregator { +class StatsAggregator : public DedicatedThreadTask { public: - StatsAggregator(const StatsAggregator &) = delete; - StatsAggregator &operator=(const StatsAggregator &) = delete; - StatsAggregator(StatsAggregator &&) = delete; - StatsAggregator &operator=(StatsAggregator &&) = delete; + StatsAggregator(int64_t aggregation_interval) + : aggregation_interval_ms_(aggregation_interval) {} - StatsAggregator(int64_t aggregation_interval_ms); - ~StatsAggregator(); + void Terminate() override; - //===--------------------------------------------------------------------===// - // ACCESSORS - //===--------------------------------------------------------------------===// + void RunTask() override; - // Global singleton - static StatsAggregator &GetInstance( - int64_t aggregation_interval_ms = STATS_AGGREGATION_INTERVAL_MS); + /** + * Aggregate metrics from all threads which have collected stats, + * combine with what was previously in catalog + * and insert new total into catalog + */ + void Aggregate(); - // Get the aggregated stats history of all exited threads - inline BackendStatsContext &GetStatsHistory() { return stats_history_; } - - // Get the current aggregated stats of all threads (including history) - inline BackendStatsContext &GetAggregatedStats() { return aggregated_stats_; } - - //===--------------------------------------------------------------------===// - // HELPER FUNCTIONS - //===--------------------------------------------------------------------===// - - // Register the BackendStatsContext of a worker thread to global Stats - // Aggregator - void RegisterContext(std::thread::id id_, BackendStatsContext *context_); - - // Unregister a BackendStatsContext. Currently we directly reuse the thread id - // instead of explicitly unregistering it. - void UnregisterContext(std::thread::id id); - - // Aggregate the stats of current living threads - void Aggregate(int64_t &interval_cnt, double &alpha, - double &weighted_avg_throughput); - - // Launch aggregator thread - void LaunchAggregator(); - - // Terminate aggregator thread - void ShutdownAggregator(); + std::vector> AggregateRawData(); private: - //===--------------------------------------------------------------------===// - // MEMBERS - //===--------------------------------------------------------------------===// - - // Stores stats of exited threads - BackendStatsContext stats_history_; - - // Stores all aggregated stats - BackendStatsContext aggregated_stats_; - - // Protect register and unregister of BackendStatsContext* - std::mutex stats_mutex_{}; - - // Map the thread id to the pointer of its BackendStatsContext - std::unordered_map backend_stats_{}; - - // How often to aggregate all worker thread stats int64_t aggregation_interval_ms_; - - // Number of threads registered - int thread_number_; - - int64_t total_prev_txn_committed_; - - // Stats aggregator background thread - std::thread aggregator_thread_; - - // CV to signal aggregator if finished + // mutex for aggregate task scheduling. No conflict generally + std::mutex mutex_; std::condition_variable exec_finished_; - - // Output path of the stats log - std::string peloton_stats_directory_ = "./stats_log"; - - // Output file stream - std::ofstream ofs_; - - // Whether the aggregator is running - bool is_aggregating_ = false; - - // Abstract Pool to hold query strings - std::unique_ptr pool_; - - //===--------------------------------------------------------------------===// - // HELPER FUNCTIONS - //===--------------------------------------------------------------------===// - - // Write all metrics to metric tables - void UpdateMetrics(); - - // Update the table metrics with a given database - void UpdateTableMetrics(storage::Database *database, int64_t time_stamp, - concurrency::TransactionContext *txn); - - // Update the index metrics with a given table - void UpdateIndexMetrics(storage::Database *database, - storage::DataTable *table, int64_t time_stamp, - concurrency::TransactionContext *txn); - - // Write all query metrics to a metric table - void UpdateQueryMetrics(int64_t time_stamp, - concurrency::TransactionContext *txn); - - // Aggregate stats periodically - void RunAggregator(); + bool exiting_ = false; }; } // namespace stats diff --git a/src/include/statistics/stats_event_type.h b/src/include/statistics/stats_event_type.h new file mode 100644 index 00000000000..a031855f7a4 --- /dev/null +++ b/src/include/statistics/stats_event_type.h @@ -0,0 +1,39 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// stats_event_type.h +// +// Identification: src/include/statistics/stats_event_type.h +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once +namespace peloton { +namespace stats { +enum class StatsEventType { + TXN_BEGIN, + TXN_COMMIT, + TXN_ABORT, + TUPLE_READ, + TUPLE_UPDATE, + TUPLE_INSERT, + TUPLE_DELETE, + INDEX_READ, + INDEX_UPDATE, + INDEX_INSERT, + INDEX_DELETE, + TABLE_MEMORY_ALLOC, + TABLE_MEMORY_FREE, + INDEX_MEMORY_ALLOC, + INDEX_MEMORY_FREE, + INDEX_MEMORY_USAGE, + INDEX_MEMORY_RECLAIM, + QUERY_BEGIN, + QUERY_END, + TEST // Testing event +}; +}; +} diff --git a/src/include/statistics/table_metric.h b/src/include/statistics/table_metric.h index 98c7d476253..66461feaabf 100644 --- a/src/include/statistics/table_metric.h +++ b/src/include/statistics/table_metric.h @@ -4,89 +4,157 @@ // // table_metric.h // -// Identification: src/statistics/table_metric.h +// Identification: src/include/statistics/table_metric.h // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// #pragma once -#include #include +#include +#include "catalog/manager.h" #include "common/internal_types.h" #include "statistics/abstract_metric.h" -#include "statistics/access_metric.h" +#include "storage/tile_group.h" #include "util/string_util.h" namespace peloton { namespace stats { +class TableMetricRawData : public AbstractRawData { + // this serves as an index into each table's counter vector + enum CounterType { + READ = 0, + UPDATE, + INSERT, + DELETE, + INLINE_MEMORY_ALLOC, + INLINE_MEMORY_USAGE, + VARLEN_MEMORY_ALLOC, + VARLEN_MEMORY_USAGE + }; -/** - * Metric for the access of a table - */ -class TableMetric : public AbstractMetric { public: - typedef std::string TableKey; + inline void IncrementTableReads(std::pair db_table_id) { + GetCounter(db_table_id, READ)++; + } - TableMetric(MetricType type, oid_t database_id, oid_t table_id); + inline void IncrementTableUpdates(std::pair db_table_id) { + GetCounter(db_table_id, UPDATE)++; + } - //===--------------------------------------------------------------------===// - // ACCESSORS - //===--------------------------------------------------------------------===// + inline void IncrementTableInserts(std::pair db_table_id) { + GetCounter(db_table_id, INSERT)++; + } - inline AccessMetric &GetTableAccess() { return table_access_; } + inline void IncrementTableDeletes(std::pair db_table_id) { + GetCounter(db_table_id, DELETE)++; + } - inline std::string GetName() { return table_name_; } + inline void IncrementTableMemAlloc(std::pair db_table_id, + int64_t bytes) { + GetCounter(db_table_id, INLINE_MEMORY_ALLOC) += bytes; + } - inline oid_t GetDatabaseId() { return database_id_; } + inline void DecrementTableMemAlloc(std::pair db_table_id, + int64_t bytes) { + GetCounter(db_table_id, INLINE_MEMORY_ALLOC) -= bytes; + } + + inline void AddModifiedTileGroup(std::pair db_table_id, + oid_t tile_group_id) { + auto tile_group_set = modified_tile_group_id_set_.find(db_table_id); + if (tile_group_set == modified_tile_group_id_set_.end()) + modified_tile_group_id_set_[db_table_id] = std::unordered_set(); + + modified_tile_group_id_set_[db_table_id].insert(tile_group_id); + } - inline oid_t GetTableId() { return table_id_; } + void Aggregate(AbstractRawData &other) override; - //===--------------------------------------------------------------------===// - // HELPER FUNCTIONS - //===--------------------------------------------------------------------===// + void UpdateAndPersist() override; - inline void Reset() { table_access_.Reset(); } + const std::string GetInfo() const override { return "table metric"; } - inline bool operator==(const TableMetric &other) { - return database_id_ == other.database_id_ && table_id_ == other.table_id_ && - table_name_ == other.table_name_ && - table_access_ == other.table_access_; + private: + inline int64_t &GetCounter(std::pair db_table_id, + CounterType type) { + auto entry = counters_.find(db_table_id); + if (entry == counters_.end()) + counters_[db_table_id] = std::vector(NUM_COUNTERS); + return counters_[db_table_id][type]; } - inline bool operator!=(const TableMetric &other) { return !(*this == other); } + /** + * Fetch Usage for inlined tile memory and both allocation and usage for + * varlen pool + */ + void FetchMemoryStats(); + + std::unordered_map, std::vector, pair_hash> + counters_; - void Aggregate(AbstractMetric &source); + // list of counter types (used by Aggregate) + static const std::vector COUNTER_TYPES; + // should be number of possible CounterType values + static const size_t NUM_COUNTERS = 8; - inline const std::string GetInfo() const { - std::stringstream ss; - ss << peloton::GETINFO_SINGLE_LINE << std::endl; - ss << " TABLE " << table_name_ << "(OID="; - ss << table_id_ << ")" << std::endl; - ; - ss << peloton::GETINFO_SINGLE_LINE << std::endl; - ss << table_access_.GetInfo(); - return ss.str(); + std::unordered_map, std::unordered_set, + pair_hash> modified_tile_group_id_set_; +}; + +class TableMetric : public AbstractMetric { + public: + inline void OnTupleRead(const concurrency::TransactionContext *, oid_t tile_group_id) override { + auto db_table_id = GetDBTableIdFromTileGroupOid(tile_group_id); + if (db_table_id.second == INVALID_OID) return; + GetRawData()->IncrementTableReads(db_table_id); } - private: - //===--------------------------------------------------------------------===// - // MEMBERS - //===--------------------------------------------------------------------===// + inline void OnTupleUpdate(const concurrency::TransactionContext *, oid_t tile_group_id) override { + auto db_table_id = GetDBTableIdFromTileGroupOid(tile_group_id); + if (db_table_id.second == INVALID_OID) return; + GetRawData()->AddModifiedTileGroup(db_table_id, tile_group_id); + GetRawData()->IncrementTableUpdates(db_table_id); + } + + inline void OnTupleInsert(const concurrency::TransactionContext *, oid_t tile_group_id) override { + auto db_table_id = GetDBTableIdFromTileGroupOid(tile_group_id); + if (db_table_id.second == INVALID_OID) return; + GetRawData()->AddModifiedTileGroup(db_table_id, tile_group_id); + GetRawData()->IncrementTableInserts(db_table_id); + } - // The database ID of this table - oid_t database_id_; + inline void OnTupleDelete(const concurrency::TransactionContext *, oid_t tile_group_id) override { + auto db_table_id = GetDBTableIdFromTileGroupOid(tile_group_id); + if (db_table_id.second == INVALID_OID) return; + GetRawData()->AddModifiedTileGroup(db_table_id, tile_group_id); + GetRawData()->IncrementTableDeletes(db_table_id); + } - // The ID of this table - oid_t table_id_; + inline void OnMemoryAlloc(std::pair db_table_id, + size_t bytes) override { + GetRawData()->IncrementTableMemAlloc(db_table_id, bytes); + } - // The name of this table - std::string table_name_; + inline void OnMemoryFree(std::pair db_table_id, + size_t bytes) override { + GetRawData()->DecrementTableMemAlloc(db_table_id, bytes); + } - // The number of tuple accesses - AccessMetric table_access_{MetricType::ACCESS}; + private: + inline static std::pair GetDBTableIdFromTileGroupOid( + oid_t tile_group_id) { + auto tile_group = + catalog::Manager::GetInstance().GetTileGroup(tile_group_id); + if (tile_group == nullptr) { + return std::pair(INVALID_OID, INVALID_OID); + } + return std::pair(tile_group->GetDatabaseId(), + tile_group->GetTableId()); + } }; } // namespace stats diff --git a/src/include/statistics/test_metric.h b/src/include/statistics/test_metric.h new file mode 100644 index 00000000000..ef2896146a0 --- /dev/null +++ b/src/include/statistics/test_metric.h @@ -0,0 +1,51 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// test_metric.h +// +// Identification: src/include/statistics/test_metric.h +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "abstract_metric.h" + +namespace peloton { +namespace stats { +/** + * @brief raw data type for testing purpose + */ +class TestMetricRawData : public AbstractRawData { + public: + /** + * @brief integrate the count with the number specified + * @param num number to be integrate + */ + inline void Integrate(int num) { count_ += num; } + + /** + * @brief aggregate the counts + * @param other + */ + inline void Aggregate(AbstractRawData &other) override { + auto &other_test = dynamic_cast(other); + count_ += other_test.count_; + } + + void UpdateAndPersist() override {} + + const std::string GetInfo() const override { return "test metric"; } + + int count_; +}; + +class TestMetric : public AbstractMetric { + public: + inline void OnTest(int num) override { GetRawData()->Integrate(num); } +}; +} +} \ No newline at end of file diff --git a/src/include/statistics/thread_level_stats_collector.h b/src/include/statistics/thread_level_stats_collector.h new file mode 100644 index 00000000000..b16c5cec91f --- /dev/null +++ b/src/include/statistics/thread_level_stats_collector.h @@ -0,0 +1,206 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// stats_collector.h +// +// Identification: src/include/statistics/thread_level_stats_collector.h +// +// Copyright (c) 2017-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include +#include +#include "common/internal_types.h" +#include "settings/settings_manager.h" +#include "statistics/abstract_metric.h" +#include "tbb/concurrent_unordered_map.h" + +namespace peloton { + +namespace concurrency { +class TransactionContext; +} // namespace concurrency + +namespace stats { +/** + * @brief Class responsible for collecting raw data on a single thread. + * + * Each thread will be assigned one collector that is globally unique. This is + * to ensure that we can collect raw data in an non-blocking way as the + *collection + * code runs on critical query path. Periodically a dedicated aggregator thread + * will put the data from all collectors together into a meaningful form. + */ +class ThreadLevelStatsCollector { + public: + using CollectorsMap = + tbb::concurrent_unordered_map>; + /** + * @return the Collector for the calling thread + */ + static ThreadLevelStatsCollector &GetCollectorForThread() { + std::thread::id tid = std::this_thread::get_id(); + return collector_map_[tid]; + } + + /** + * @return A mapping from each thread to their assigned Collector + */ + static CollectorsMap &GetAllCollectors() { return collector_map_; }; + + /** + * @brief Constructor of collector + */ + ThreadLevelStatsCollector(); + + /** + * @brief Destructor of collector + */ + ~ThreadLevelStatsCollector(); + + /* See Metric for documentation on the following methods. They should correspond + * to the "OnXxx" methods one-to-one */ + inline void CollectTransactionBegin(const concurrency::TransactionContext *txn) { + for (auto &metric : metric_dispatch_[StatsEventType::TXN_BEGIN]) + metric->OnTransactionBegin(txn); + }; + + inline void CollectTransactionCommit(const concurrency::TransactionContext *txn, + oid_t tile_group_id) { + for (auto &metric : metric_dispatch_[StatsEventType::TXN_COMMIT]) + metric->OnTransactionCommit(txn, tile_group_id); + }; + + inline void CollectTransactionAbort(const concurrency::TransactionContext *txn, + oid_t tile_group_id) { + for (auto &metric : metric_dispatch_[StatsEventType::TXN_ABORT]) + metric->OnTransactionAbort(txn, tile_group_id); + }; + + inline void CollectTupleRead(const concurrency::TransactionContext *current_txn, + oid_t tile_group_id) { + for (auto &metric : metric_dispatch_[StatsEventType::TUPLE_READ]) + metric->OnTupleRead(current_txn, tile_group_id); + }; + inline void CollectTupleUpdate(const concurrency::TransactionContext *current_txn, + oid_t tile_group_id) { + for (auto &metric : metric_dispatch_[StatsEventType::TUPLE_UPDATE]) + metric->OnTupleUpdate(current_txn, tile_group_id); + }; + inline void CollectTupleInsert(const concurrency::TransactionContext *current_txn, + oid_t tile_group_id) { + for (auto &metric : metric_dispatch_[StatsEventType::TUPLE_INSERT]) + metric->OnTupleInsert(current_txn, tile_group_id); + }; + inline void CollectTupleDelete(const concurrency::TransactionContext *current_txn, + oid_t tile_group_id) { + for (auto &metric : metric_dispatch_[StatsEventType::TUPLE_DELETE]) + metric->OnTupleDelete(current_txn, tile_group_id); + }; + inline void CollectTableMemoryAlloc(oid_t database_id, oid_t table_id, + size_t bytes) { + if (table_id == INVALID_OID || database_id == INVALID_OID) return; + + for (auto &metric : metric_dispatch_[StatsEventType::TABLE_MEMORY_ALLOC]) + metric->OnMemoryAlloc({database_id, table_id}, bytes); + }; + inline void CollectTableMemoryFree(oid_t database_id, oid_t table_id, + size_t bytes) { + if (table_id == INVALID_OID || database_id == INVALID_OID) return; + for (auto &metric : metric_dispatch_[StatsEventType::TABLE_MEMORY_FREE]) + metric->OnMemoryFree({database_id, table_id}, bytes); + }; + inline void CollectIndexRead(oid_t database_id, oid_t index_id, + size_t num_read) { + for (auto &metric : metric_dispatch_[StatsEventType::INDEX_READ]) + metric->OnIndexRead({database_id, index_id}, num_read); + }; + inline void CollectIndexUpdate(oid_t database_id, oid_t index_id) { + for (auto &metric : metric_dispatch_[StatsEventType::INDEX_UPDATE]) + metric->OnIndexUpdate({database_id, index_id}); + }; + inline void CollectIndexInsert(oid_t database_id, oid_t index_id) { + for (auto &metric : metric_dispatch_[StatsEventType::INDEX_INSERT]) + metric->OnIndexInsert({database_id, index_id}); + }; + inline void CollectIndexDelete(oid_t database_id, oid_t index_id) { + for (auto &metric : metric_dispatch_[StatsEventType::INDEX_DELETE]) + metric->OnIndexDelete({database_id, index_id}); + }; + inline void CollectIndexMemoryAlloc(oid_t database_id, oid_t index_id, + size_t bytes) { + for (auto &metric : metric_dispatch_[StatsEventType::INDEX_MEMORY_ALLOC]) + metric->OnMemoryAlloc({database_id, index_id}, bytes); + }; + inline void CollectIndexMemoryUsage(oid_t database_id, oid_t index_id, + size_t bytes) { + for (auto &metric : metric_dispatch_[StatsEventType::INDEX_MEMORY_USAGE]) + metric->OnMemoryUsage({database_id, index_id}, bytes); + }; + inline void CollectIndexMemoryFree(oid_t database_id, oid_t index_id, + size_t bytes) { + for (auto &metric : metric_dispatch_[StatsEventType::INDEX_MEMORY_FREE]) + metric->OnMemoryFree({database_id, index_id}, bytes); + }; + inline void CollectIndexMemoryReclaim(oid_t database_id, oid_t index_id, + size_t bytes) { + for (auto &metric : metric_dispatch_[StatsEventType::INDEX_MEMORY_RECLAIM]) + metric->OnMemoryReclaim({database_id, index_id}, bytes); + }; + inline void CollectQueryBegin() { + for (auto &metric : metric_dispatch_[StatsEventType::QUERY_BEGIN]) + metric->OnQueryBegin(); + }; + inline void CollectQueryEnd() { + for (auto &metric : metric_dispatch_[StatsEventType::QUERY_END]) + metric->OnQueryEnd(); + }; + inline void CollectTestNum(int number) { + for (auto &metric : metric_dispatch_[StatsEventType::TEST]) + metric->OnTest(number); + } + + /** + * @return A vector of raw data, for each registered metric. Each piece of + * data is guaranteed to be safe to read and remove, and the same type of + * metric is guaranteed to be in the same positopn in the returned vector + * for different instances of Collector. + */ + std::vector> GetDataToAggregate(); + + private: + /** + * Registers a Metric so that its callbacks are invoked. + * Use this only in the constructor. + * @tparam metric type of Metric to register + * @param types A list of event types to receive updates about. + */ + template + void RegisterMetric(std::vector types) { + auto m = std::make_shared(); + metrics_.push_back(m); + for (StatsEventType type : types) metric_dispatch_[type].push_back(m); + } + + using MetricList = std::vector>; + /** + * List of all registered metrics + */ + MetricList metrics_; + /** + * Mapping from each type of event to a list of metrics registered to + * receive updates from that type of event. + */ + std::unordered_map> + metric_dispatch_; + + static CollectorsMap collector_map_; +}; + +} // namespace stats +} // namespace peloton diff --git a/src/include/statistics/tuple_access_metric.h b/src/include/statistics/tuple_access_metric.h new file mode 100644 index 00000000000..1c476ee3035 --- /dev/null +++ b/src/include/statistics/tuple_access_metric.h @@ -0,0 +1,84 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// database_metric.h +// +// Identification: src/include/statistics/tuple_access_metric.h +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once +#include +#include +#include "statistics/abstract_metric.h" +#include "common/internal_types.h" +#include "concurrency/transaction_context.h" + +namespace peloton { +namespace stats { +// TODO(tianyu): This is a hack to not log internal transactions. Fix this eventually +class TupleAccessRawData: public AbstractRawData { + public: + inline void LogTupleRead(txn_id_t tid) { + if (begins_.find(tid) != begins_.end()) + tuple_access_counters_[tid]++; + } + inline void LogCommit(txn_id_t tid) { + if (begins_.find(tid) != begins_.end()) + commits_.insert(tid); + } + + inline void LogAbort(txn_id_t tid) { + if (begins_.find(tid) != begins_.end()) + aborts_.insert(tid); + } + + inline void LogTxnBegin(txn_id_t tid) { + begins_.insert(tid); + } + + inline void Aggregate(AbstractRawData &other) override { + auto &other_db_metric = dynamic_cast(other); + for (auto &entry : other_db_metric.tuple_access_counters_) + tuple_access_counters_[entry.first] += entry.second; + for (auto &txn : other_db_metric.commits_) + commits_.insert(txn); + for (auto &txn : other_db_metric.aborts_) + aborts_.insert(txn); + } + + void UpdateAndPersist() override; + + // TODO(Tianyu): Pretty Print + const std::string GetInfo() const override { return "TupleAccessRawData"; }; + private: + void WriteToCatalog(txn_id_t tid, bool complete, bool commit, concurrency::TransactionContext *txn); + std::unordered_map tuple_access_counters_; + std::unordered_set begins_, commits_, aborts_; +}; + +class TupleAccessMetric : public AbstractMetric { + public: + void OnTransactionBegin(const concurrency::TransactionContext *context) override { + GetRawData()->LogTxnBegin(context->GetTransactionId()); + } + void OnTransactionCommit(const concurrency::TransactionContext *context, + oid_t) override { + GetRawData()->LogCommit(context->GetTransactionId()); + } + + void OnTransactionAbort(const concurrency::TransactionContext *context, + oid_t) override { + GetRawData()->LogAbort(context->GetTransactionId()); + } + + void OnTupleRead(const concurrency::TransactionContext *context, + oid_t) override { + GetRawData()->LogTupleRead(context->GetTransactionId()); + } +}; +} // namespace stats +} // namespace peloton \ No newline at end of file diff --git a/src/include/storage/tile.h b/src/include/storage/tile.h index edb210f868c..417346da807 100644 --- a/src/include/storage/tile.h +++ b/src/include/storage/tile.h @@ -18,6 +18,7 @@ #include "catalog/schema.h" #include "common/item_pointer.h" #include "common/printable.h" +#include "statistics/thread_level_stats_collector.h" #include "type/abstract_pool.h" #include "type/serializeio.h" #include "type/serializer.h" @@ -289,6 +290,10 @@ class TileFactory { TileFactory::InitCommon(tile, database_id, table_id, tile_group_id, tile_id, schema); + // Record memory allocation + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectTableMemoryAlloc(database_id, table_id, tile->tile_size); + return tile; } diff --git a/src/include/storage/tile_group.h b/src/include/storage/tile_group.h index 946c65f5f8c..491460a4fef 100644 --- a/src/include/storage/tile_group.h +++ b/src/include/storage/tile_group.h @@ -76,7 +76,8 @@ class TileGroup : public Printable { // Tile group constructor TileGroup(BackendType backend_type, TileGroupHeader *tile_group_header, AbstractTable *table, const std::vector &schemas, - const column_map_type &column_map, int tuple_count); + const column_map_type &column_map, int tuple_count, + oid_t database_id, oid_t table_id, oid_t tile_group_id); ~TileGroup(); diff --git a/src/include/storage/tile_group_header.h b/src/include/storage/tile_group_header.h index c7e8c010530..5b50f9b6d55 100644 --- a/src/include/storage/tile_group_header.h +++ b/src/include/storage/tile_group_header.h @@ -177,6 +177,11 @@ class TileGroupHeader : public Printable { return (char *)(TUPLE_HEADER_LOCATION + reserved_field_offset); } + // Return the allocated size of this tile group header + inline size_t GetHeaderSize() { + return header_size; + } + // Setters inline void SetTileGroup(TileGroup *tile_group) { diff --git a/src/include/storage/tuple.h b/src/include/storage/tuple.h index eb0041ee203..67e8d1da52c 100644 --- a/src/include/storage/tuple.h +++ b/src/include/storage/tuple.h @@ -252,7 +252,7 @@ inline Tuple::Tuple(char *data, catalog::Schema *schema) { tuple_data_ = data; tuple_schema_ = schema; - allocated_ = false; // ??? + allocated_ = false; // The data comes from argument, do not free it when this tuple is destroyed } inline Tuple &Tuple::operator=(const Tuple &rhs) { diff --git a/src/include/traffic_cop/traffic_cop.h b/src/include/traffic_cop/traffic_cop.h index e324b87fe82..5b9912a4eed 100644 --- a/src/include/traffic_cop/traffic_cop.h +++ b/src/include/traffic_cop/traffic_cop.h @@ -65,12 +65,12 @@ class TrafficCop { void Reset(); // Execute a statement - ResultType ExecuteStatement( - const std::shared_ptr &statement, - const std::vector ¶ms, const bool unnamed, - std::shared_ptr param_stats, - const std::vector &result_format, std::vector &result, - size_t thread_id = 0); + ResultType ExecuteStatement(const std::shared_ptr &statement, + const std::vector ¶ms, + const bool unnamed, + const std::vector &result_format, + std::vector &result, + size_t thread_id = 0); // Helper to handle txn-specifics for the plan-tree of a statement. executor::ExecutionResult ExecuteHelper( diff --git a/src/include/type/abstract_pool.h b/src/include/type/abstract_pool.h index 1b8246e3bbf..ebe51c293b6 100644 --- a/src/include/type/abstract_pool.h +++ b/src/include/type/abstract_pool.h @@ -19,10 +19,9 @@ namespace type { // Interface of a memory pool that can quickly allocate chunks of memory class AbstractPool { -public: - + public: // Empty virtual destructor for proper cleanup - virtual ~AbstractPool(){} + virtual ~AbstractPool() {} // Allocate a contiguous block of memory of the given size. If the allocation // is successful a non-null pointer is returned. If the allocation fails, a @@ -33,6 +32,17 @@ class AbstractPool { // Returns the provided chunk of memory back into the pool virtual void Free(void *ptr) = 0; + /** + * Get how many bytes are allocated in the pool + * @return bytes that are allocated + */ + virtual size_t GetMemoryAlloc() = 0; + + /** + * Get how many bytes are allocated and used in the pool + * @return bytes that are used + */ + virtual size_t GetMemoryUsage() = 0; }; } // namespace type diff --git a/src/include/type/ephemeral_pool.h b/src/include/type/ephemeral_pool.h index 9be1cf53ca0..92dc4bc5f35 100644 --- a/src/include/type/ephemeral_pool.h +++ b/src/include/type/ephemeral_pool.h @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include "common/macros.h" #include "common/synchronization/spin_latch.h" @@ -26,53 +26,64 @@ namespace type { // A memory pool that can quickly allocate chunks of memory to clients. class EphemeralPool : public AbstractPool { -public: - - EphemeralPool(){ - - } + public: + EphemeralPool() : mem_comsume_{0} {} // Destroy this pool, and all memory it owns. - ~EphemeralPool(){ - + ~EphemeralPool() { pool_lock_.Lock(); - for(auto location: locations_){ - delete[] location; + for (auto &entry : locations_) { + delete[] entry.first; } pool_lock_.Unlock(); - } // Allocate a contiguous block of memory of the given size. If the allocation // is successful a non-null pointer is returned. If the allocation fails, a // null pointer will be returned. - void *Allocate(size_t size){ + void *Allocate(size_t size) override { auto location = new char[size]; pool_lock_.Lock(); - locations_.insert(location); + locations_[location] = size; + mem_comsume_ += size; pool_lock_.Unlock(); return location; } // Returns the provided chunk of memory back into the pool - void Free(UNUSED_ATTRIBUTE void *ptr) { - char *cptr = (char *) ptr; + void Free(UNUSED_ATTRIBUTE void *ptr) override { + char *cptr = (char *)ptr; pool_lock_.Lock(); + size_t block_size = locations_[cptr]; + mem_comsume_ -= block_size; locations_.erase(cptr); pool_lock_.Unlock(); - delete [] cptr; + delete[] cptr; } -public: + /** + * @see AbstractPool + */ + inline size_t GetMemoryAlloc() override { return mem_comsume_.load(); }; + + /** + * @see AbstractPool + */ + inline size_t GetMemoryUsage() override { return mem_comsume_.load(); }; + public: // Location list - std::unordered_set locations_; + std::unordered_map locations_; // Spin lock protecting location list common::synchronization::SpinLatch pool_lock_; + /** + * Memory usage as well as allocation + */ + std::atomic mem_comsume_; }; } // namespace type diff --git a/src/index/art_index.cpp b/src/index/art_index.cpp index 18de5d02393..51f78598df4 100644 --- a/src/index/art_index.cpp +++ b/src/index/art_index.cpp @@ -14,8 +14,7 @@ #include "common/container_tuple.h" #include "index/scan_optimizer.h" -#include "settings/settings_manager.h" -#include "statistics/backend_stats_context.h" +#include "statistics/thread_level_stats_collector.h" #include "storage/data_table.h" #include "storage/storage_manager.h" #include "util/portable_endian.h" @@ -65,15 +64,19 @@ bool ArtIndex::InsertEntry(const storage::Tuple *key, ItemPointer *value) { auto thread_info = container_.getThreadInfo(); container_.insert(tree_key, reinterpret_cast(value), thread_info); - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementIndexInserts( - GetMetadata()); - } - // Update stats - IncreaseNumberOfTuplesBy(1); - + auto &stats_collector = + stats::ThreadLevelStatsCollector::GetCollectorForThread(); + stats_collector.CollectIndexInsert(metadata->GetDatabaseOid(), GetOid()); + + // TODO: The memory collection here is just an inaccurate estimation + // Those who is familiar with the code base for index implementation should + // insert these lines to accurate place with correct values + size_t memory = key->GetLength() + 8; // key size + item pointer size + stats_collector.CollectIndexMemoryAlloc(metadata->GetDatabaseOid(), GetOid(), + memory); + stats_collector.CollectIndexMemoryUsage(metadata->GetDatabaseOid(), GetOid(), + memory); return true; } @@ -89,12 +92,18 @@ bool ArtIndex::DeleteEntry(const storage::Tuple *key, ItemPointer *value) { if (removed) { // Update stats - DecreaseNumberOfTuplesBy(1); - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementIndexDeletes( - 1, GetMetadata()); - } + auto &stats_collector = + stats::ThreadLevelStatsCollector::GetCollectorForThread(); + stats_collector.CollectIndexInsert(metadata->GetDatabaseOid(), GetOid()); + + // TODO: The memory collection here is just an inaccurate estimation + // Those who is familiar with the code base for index implementation should + // insert these lines to accurate place with correct values + size_t memory = key->GetLength() + 8; // key size + item pointer size + stats_collector.CollectIndexMemoryFree(metadata->GetDatabaseOid(), + GetOid(), memory); + stats_collector.CollectIndexMemoryReclaim(metadata->GetDatabaseOid(), + GetOid(), memory); } return removed; @@ -113,12 +122,18 @@ bool ArtIndex::CondInsertEntry(const storage::Tuple *key, ItemPointer *value, if (inserted) { // Update stats - IncreaseNumberOfTuplesBy(1); - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementIndexInserts( - GetMetadata()); - } + auto &stats_collector = + stats::ThreadLevelStatsCollector::GetCollectorForThread(); + stats_collector.CollectIndexInsert(metadata->GetDatabaseOid(), GetOid()); + + // TODO: The memory collection here is just an inaccurate estimation + // Those who is familiar with the code base for index implementation should + // insert these lines to accurate place with correct values + size_t memory = key->GetLength() + 8; // key size + item pointer size + stats_collector.CollectIndexMemoryAlloc(metadata->GetDatabaseOid(), + GetOid(), memory); + stats_collector.CollectIndexMemoryUsage(metadata->GetDatabaseOid(), + GetOid(), memory); } return inserted; @@ -153,11 +168,8 @@ void ArtIndex::Scan( } // Update stats - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementIndexReads( - result.size(), GetMetadata()); - } + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectIndexRead( + metadata->GetDatabaseOid(), GetOid(), result.size()); } void ArtIndex::ScanLimit(const std::vector &values, @@ -225,11 +237,8 @@ void ArtIndex::ScanRange(const art::Key &start, const art::Key &end, } // Update stats - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementIndexReads( - result.size(), GetMetadata()); - } + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectIndexRead( + metadata->GetDatabaseOid(), GetOid(), result.size()); } void ArtIndex::SetLoadKeyFunc(art::Tree::LoadKeyFunction load_func, void *ctx) { diff --git a/src/index/bwtree_index.cpp b/src/index/bwtree_index.cpp index 74094843ef8..e785e6d2d8d 100755 --- a/src/index/bwtree_index.cpp +++ b/src/index/bwtree_index.cpp @@ -14,9 +14,8 @@ #include "index/index_key.h" #include "index/scan_optimizer.h" -#include "statistics/stats_aggregator.h" -#include "settings/settings_manager.h" - +#include "statistics/thread_level_stats_collector.h" + namespace peloton { namespace index { @@ -53,24 +52,25 @@ bool BWTREE_INDEX_TYPE::InsertEntry(const storage::Tuple *key, KeyType index_key; index_key.SetFromKey(key); - bool ret; - if(HasUniqueKeys() == true) { - ret = container.Insert(index_key, value, true); - } else { - ret = container.Insert(index_key, value, false); + bool inserted = container.Insert(index_key, value, HasUniqueKeys()); + if (inserted) { + auto &stats_collector = + stats::ThreadLevelStatsCollector::GetCollectorForThread(); + stats_collector.CollectIndexInsert(metadata->GetDatabaseOid(), GetOid()); + + // TODO: The memory collection here is just an inaccurate estimation + // Those who is familiar with the code base for index implementation should + // insert these lines to accurate place with correct values + size_t memory = key->GetLength() + 8; // key size + item pointer size + stats_collector.CollectIndexMemoryAlloc(metadata->GetDatabaseOid(), + GetOid(), memory); + stats_collector.CollectIndexMemoryUsage(metadata->GetDatabaseOid(), + GetOid(), memory); } - if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementIndexInserts(metadata); - } - - // NOTE: If I use index_key.GetInfo() here, I always get an empty key? - LOG_TRACE("InsertEntry(key=%s, val=%s) [%s]", - key->GetInfo().c_str(), - IndexUtil::GetInfo(value).c_str(), - (ret ? "SUCCESS" : "FAIL")); - - return ret; + LOG_TRACE("InsertEntry(key=%s, val=%s) [%s]", index_key.GetInfo().c_str(), + IndexUtil::GetInfo(value).c_str(), (ret ? "SUCCESS" : "FAIL")); + return inserted; } /* @@ -84,23 +84,27 @@ bool BWTREE_INDEX_TYPE::DeleteEntry(const storage::Tuple *key, KeyType index_key; index_key.SetFromKey(key); - size_t delete_count = 0; - // In Delete() since we just use the value for comparison (i.e. read-only) // it is unnecessary for us to allocate memory - bool ret = container.Delete(index_key, value); - - if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementIndexDeletes( - delete_count, metadata); + bool removed = container.Delete(index_key, value); + if (removed) { + auto &stats_collector = + stats::ThreadLevelStatsCollector::GetCollectorForThread(); + stats_collector.CollectIndexDelete(metadata->GetDatabaseOid(), GetOid()); + + // TODO: The memory collection here is just an inaccurate estimation + // Those who is familiar with the code base for index implementation should + // insert these lines to accurate place with correct values + size_t memory = key->GetLength() + 8; // key size + item pointer size + stats_collector.CollectIndexMemoryFree(metadata->GetDatabaseOid(), GetOid(), + memory); + stats_collector.CollectIndexMemoryReclaim(metadata->GetDatabaseOid(), + GetOid(), memory); } - LOG_TRACE("DeleteEntry(key=%s, val=%s) [%s]", - key->GetInfo().c_str(), - IndexUtil::GetInfo(value).c_str(), - (ret ? "SUCCESS" : "FAIL")); - - return ret; + LOG_TRACE("DeleteEntry(key=%s, val=%s) [%s]", index_key.GetInfo().c_str(), + IndexUtil::GetInfo(value).c_str(), (ret ? "SUCCESS" : "FAIL")); + return removed; } BWTREE_TEMPLATE_ARGUMENTS @@ -115,22 +119,32 @@ bool BWTREE_INDEX_TYPE::CondInsertEntry( // This function will complete them in one step // predicate will be set to nullptr if the predicate // returns true for some value - bool ret = container.ConditionalInsert(index_key, value, predicate, - &predicate_satisfied); + bool inserted = container.ConditionalInsert(index_key, value, predicate, + &predicate_satisfied); // If predicate is not satisfied then we know insertion successes if (predicate_satisfied == false) { // So it should always succeed? - assert(ret == true); + assert(inserted == true); } else { - assert(ret == false); + assert(inserted == false); } - - if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementIndexInserts(metadata); + if (inserted) { + auto &stats_collector = + stats::ThreadLevelStatsCollector::GetCollectorForThread(); + stats_collector.CollectIndexInsert(metadata->GetDatabaseOid(), GetOid()); + + // TODO: The memory collection here is just an inaccurate estimation + // Those who is familiar with the code base for index implementation should + // insert these lines to accurate place with correct values + size_t memory = key->GetLength() + 8; // key size + item pointer size + stats_collector.CollectIndexMemoryAlloc(metadata->GetDatabaseOid(), + GetOid(), memory); + stats_collector.CollectIndexMemoryUsage(metadata->GetDatabaseOid(), + GetOid(), memory); } - return ret; + return inserted; } /* @@ -200,12 +214,8 @@ void BWTREE_INDEX_TYPE::Scan( } } // if is full scan - if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementIndexReads( - result.size(), metadata); - } - - return; + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectIndexRead( + metadata->GetDatabaseOid(), GetOid(), result.size()); } /* @@ -253,8 +263,6 @@ void BWTREE_INDEX_TYPE::ScanLimit( Scan(value_list, tuple_column_id_list, expr_list, scan_direction, result, csp_p); } - - return; } BWTREE_TEMPLATE_ARGUMENTS @@ -267,11 +275,8 @@ void BWTREE_INDEX_TYPE::ScanAllKeys(std::vector &result) { it++; } - if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementIndexReads( - result.size(), metadata); - } - return; + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectIndexRead( + metadata->GetDatabaseOid(), GetOid(), result.size()); } BWTREE_TEMPLATE_ARGUMENTS @@ -283,12 +288,8 @@ void BWTREE_INDEX_TYPE::ScanKey(const storage::Tuple *key, // This function in BwTree fills a given vector container.GetValue(index_key, result); - if (static_cast(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->IncrementIndexReads( - result.size(), metadata); - } - - return; + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectIndexRead( + metadata->GetDatabaseOid(), GetOid(), result.size()); } BWTREE_TEMPLATE_ARGUMENTS diff --git a/src/index/index.cpp b/src/index/index.cpp index a7e999e484e..aa839ca0c7e 100644 --- a/src/index/index.cpp +++ b/src/index/index.cpp @@ -124,9 +124,6 @@ Index::Index(IndexMetadata *metadata) // This is redundant index_oid = metadata->GetOid(); - // initialize counters - lookup_counter = insert_counter = delete_counter = update_counter = 0; - // initialize pool pool = new type::EphemeralPool(); @@ -327,32 +324,5 @@ const std::string Index::GetInfo() const { return os.str(); } -// Increase the number of tuples in this table -void Index::IncreaseNumberOfTuplesBy(const size_t amount) { - number_of_tuples += amount; - dirty = true; -} - -// Decrease the number of tuples in this table -void Index::DecreaseNumberOfTuplesBy(const size_t amount) { - number_of_tuples -= amount; - dirty = true; -} - -// Set the number of tuples in this table -void Index::SetNumberOfTuples(const size_t num_tuples) { - number_of_tuples = num_tuples; - dirty = true; -} - -// Get the number of tuples in this table -size_t Index::GetNumberOfTuples() const { return number_of_tuples; } - -// Return dirty flag -bool Index::IsDirty() const { return dirty; } - -// Reset dirty flag -void Index::ResetDirty() { dirty = false; } - } // namespace index } // namespace peloton diff --git a/src/index/skiplist_index.cpp b/src/index/skiplist_index.cpp index 4361c2b808c..712abdf0443 100644 --- a/src/index/skiplist_index.cpp +++ b/src/index/skiplist_index.cpp @@ -14,7 +14,6 @@ #include "common/logger.h" #include "index/index_key.h" #include "index/scan_optimizer.h" -#include "statistics/stats_aggregator.h" #include "storage/tuple.h" namespace peloton { diff --git a/src/main/peloton/peloton.cpp b/src/main/peloton/peloton.cpp index 8c5e0b204c6..3a5a589f76f 100644 --- a/src/main/peloton/peloton.cpp +++ b/src/main/peloton/peloton.cpp @@ -13,11 +13,13 @@ #include #include +#include "brain/catalog_sync_brain_job.h" #include "common/init.h" #include "common/logger.h" #include "network/peloton_server.h" #include "settings/settings_manager.h" #include "brain/brain.h" +#include "catalog/catalog.h" // For GFlag's built-in help message flag DECLARE_bool(help); @@ -50,14 +52,20 @@ int RunPelotonBrain() { // TODO(tianyu): boot up other peloton resources as needed here peloton::brain::Brain brain; evthread_use_pthreads(); + auto catalog = peloton::catalog::Catalog::GetInstance(); + catalog->Bootstrap(); + peloton::settings::SettingsManager::GetInstance().InitializeCatalog(); // TODO(tianyu): register jobs here struct timeval one_second; one_second.tv_sec = 1; one_second.tv_usec = 0; - auto example_task = [](peloton::brain::BrainEnvironment *) { - // TODO(tianyu): Replace with real address - capnp::EzRpcClient client("localhost:15445"); + struct timeval one_minute; + one_minute.tv_sec = 60; + one_minute.tv_usec = 0; + + auto example_task = [](peloton::brain::BrainEnvironment *env) { + capnp::EzRpcClient &client = env->GetPelotonClient(); PelotonService::Client peloton_service = client.getMain(); auto request = peloton_service.createIndexRequest(); request.getRequest().setIndexKeys(42); @@ -65,6 +73,7 @@ int RunPelotonBrain() { }; brain.RegisterJob(&one_second, "test", example_task); + brain.RegisterJob(&one_minute, "sync"); brain.Run(); return 0; } diff --git a/src/network/peloton_server.cpp b/src/network/peloton_server.cpp index 6e72caefbbc..fcfd719fa76 100644 --- a/src/network/peloton_server.cpp +++ b/src/network/peloton_server.cpp @@ -10,7 +10,6 @@ // //===----------------------------------------------------------------------===// - #include #include #include "common/utility.h" @@ -20,6 +19,7 @@ #include "network/peloton_rpc_handler_task.h" #include "network/peloton_server.h" #include "settings/settings_manager.h" +#include "statistics/stats_aggregator.h" #include "peloton_config.h" @@ -80,16 +80,16 @@ void PelotonServer::SSLLockingFunction(int mode, int n, } unsigned long PelotonServer::SSLIdFunction(void) { - return ((unsigned long) THREAD_ID); + return ((unsigned long)THREAD_ID); } void PelotonServer::LoadSSLFileSettings() { private_key_file_ = DATA_DIR + settings::SettingsManager::GetString( - settings::SettingId::private_key_file); + settings::SettingId::private_key_file); certificate_file_ = DATA_DIR + settings::SettingsManager::GetString( - settings::SettingId::certificate_file); + settings::SettingId::certificate_file); root_cert_file_ = DATA_DIR + settings::SettingsManager::GetString( - settings::SettingId::root_cert_file); + settings::SettingId::root_cert_file); } void PelotonServer::SSLInit() { @@ -111,7 +111,8 @@ void PelotonServer::SSLInit() { // TODO(Yuchen): deal with returned error 0? SSLMutexSetup(); // set general-purpose version, actual protocol will be negotiated to the - // highest version mutually support between client and server during handshake + // highest version mutually support between client and server during + // handshake ssl_context = SSL_CTX_new(SSLv23_method()); if (ssl_context == nullptr) { SetSSLLevel(SSLLevel::SSL_DISABLE); @@ -162,10 +163,10 @@ void PelotonServer::SSLInit() { // automatically. set routine to filter the return status of the default // verification and returns new verification status. SSL_VERIFY_PEER: send // certificate request to client. Client may ignore the request. If the - // client sends back the certificate, it will be verified. Handshake will be - // terminated if the verification fails. SSL_VERIFY_FAIL_IF_NO_PEER_CERT: use - // with SSL_VERIFY_PEER, if client does not send back the certificate, - // terminate the handshake. + // client sends back the certificate, it will be verified. Handshake will + // be terminated if the verification fails. + // SSL_VERIFY_FAIL_IF_NO_PEER_CERT: use with SSL_VERIFY_PEER, if client does + // not send back the certificate, terminate the handshake. SSL_CTX_set_verify(ssl_context, SSL_VERIFY_PEER, VerifyCallback); SSL_CTX_set_verify_depth(ssl_context, 4); } else { @@ -223,7 +224,7 @@ int PelotonServer::VerifyCallback(int ok, X509_STORE_CTX *store) { return ok; } -template +template void PelotonServer::TrySslOperation(int (*func)(Ts...), Ts... arg) { if (func(arg...) < 0) { auto error_message = peloton_error_message(); @@ -238,7 +239,7 @@ PelotonServer &PelotonServer::SetupServer() { // This line is critical to performance for some reason evthread_use_pthreads(); if (settings::SettingsManager::GetString( - settings::SettingId::socket_family) != "AF_INET") + settings::SettingId::socket_family) != "AF_INET") throw ConnectionException("Unsupported socket family"); struct sockaddr_in sin; @@ -258,13 +259,13 @@ PelotonServer &PelotonServer::SetupServer() { setsockopt(listen_fd_, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); TrySslOperation( - bind, listen_fd_, (struct sockaddr *) &sin, sizeof(sin)); + bind, listen_fd_, (struct sockaddr *)&sin, sizeof(sin)); TrySslOperation(listen, listen_fd_, conn_backlog); dispatcher_task_ = std::make_shared( CONNECTION_THREAD_COUNT, listen_fd_); - LOG_INFO("Listening on port %llu", (unsigned long long) port_); + LOG_INFO("Listening on port %llu", (unsigned long long)port_); return *this; } @@ -277,6 +278,15 @@ void PelotonServer::ServerLoop() { DedicatedThreadRegistry::GetInstance() .RegisterDedicatedThread(this, rpc_task); } + + if (static_cast(settings::SettingsManager::GetInt( + settings::SettingId::stats_mode)) == StatsModeType::ENABLE) { + auto stats_aggregate_task = + std::make_shared(STATS_AGGREGATION_INTERVAL_MS); + DedicatedThreadRegistry::GetInstance() + .RegisterDedicatedThread(this, + stats_aggregate_task); + } dispatcher_task_->EventLoop(); peloton_close(listen_fd_); diff --git a/src/network/postgres_protocol_handler.cpp b/src/network/postgres_protocol_handler.cpp index ffbb786b88e..2bdf232803e 100644 --- a/src/network/postgres_protocol_handler.cpp +++ b/src/network/postgres_protocol_handler.cpp @@ -29,7 +29,6 @@ #include "planner/insert_plan.h" #include "planner/plan_util.h" #include "planner/update_plan.h" -#include "settings/settings_manager.h" #include "traffic_cop/traffic_cop.h" #include "type/value.h" #include "type/value_factory.h" @@ -215,7 +214,7 @@ ProcessResult PostgresProtocolHandler::ExecQueryMessage( bool unnamed = false; auto status = traffic_cop_->ExecuteStatement( traffic_cop_->GetStatement(), traffic_cop_->GetParamVal(), unnamed, - nullptr, result_format_, traffic_cop_->GetResult(), thread_id); + result_format_, traffic_cop_->GetResult(), thread_id); if (traffic_cop_->GetQueuing()) { return ProcessResult::PROCESSING; } @@ -247,7 +246,7 @@ ProcessResult PostgresProtocolHandler::ExecQueryMessage( traffic_cop_->GetStatement()->GetTupleDescriptor().size(), 0); auto status = traffic_cop_->ExecuteStatement( traffic_cop_->GetStatement(), traffic_cop_->GetParamVal(), unnamed, - nullptr, result_format_, traffic_cop_->GetResult(), thread_id); + result_format_, traffic_cop_->GetResult(), thread_id); if (traffic_cop_->GetQueuing()) { return ProcessResult::PROCESSING; } @@ -262,8 +261,8 @@ ResultType PostgresProtocolHandler::ExecQueryExplain( std::unique_ptr unnamed_sql_stmt_list( new parser::SQLStatementList()); unnamed_sql_stmt_list->PassInStatement(std::move(explain_stmt.real_sql_stmt)); - auto stmt = traffic_cop_->PrepareStatement( - "explain", query, std::move(unnamed_sql_stmt_list)); + auto stmt = traffic_cop_->PrepareStatement("explain", query, + std::move(unnamed_sql_stmt_list)); ResultType status = ResultType::UNKNOWN; if (stmt != nullptr) { traffic_cop_->SetStatement(stmt); @@ -379,29 +378,11 @@ void PostgresProtocolHandler::ExecParseMessage(InputPacket *pkt) { // Read param types std::vector param_types(num_params); - auto type_buf_begin = pkt->Begin() + pkt->ptr; - auto type_buf_len = ReadParamType(pkt, num_params, param_types); + ReadParamType(pkt, num_params, param_types); // Cache the received query - bool unnamed_query = statement_name.empty(); statement->SetParamTypes(param_types); - // Stat - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - // Make a copy of param types for stat collection - stats::QueryMetric::QueryParamBuf query_type_buf; - query_type_buf.len = type_buf_len; - query_type_buf.buf = PacketCopyBytes(type_buf_begin, type_buf_len); - - // Unnamed statement - if (unnamed_query) { - unnamed_stmt_param_types_ = query_type_buf; - } else { - statement_param_types_[statement_name] = query_type_buf; - } - } - // Cache the statement statement_cache_.AddStatement(statement); @@ -429,8 +410,7 @@ void PostgresProtocolHandler::ExecBindMessage(InputPacket *pkt) { int num_params_format = PacketGetInt(pkt, 2); std::vector formats(num_params_format); - auto format_buf_begin = pkt->Begin() + pkt->ptr; - auto format_buf_len = ReadParamFormat(pkt, num_params_format, formats); + ReadParamFormat(pkt, num_params_format, formats); int num_params = PacketGetInt(pkt, 2); // error handling @@ -444,7 +424,6 @@ void PostgresProtocolHandler::ExecBindMessage(InputPacket *pkt) { // Get statement info generated in PARSE message std::shared_ptr statement; - stats::QueryMetric::QueryParamBuf param_type_buf; statement = statement_cache_.GetStatement(statement_name); @@ -471,14 +450,6 @@ void PostgresProtocolHandler::ExecBindMessage(InputPacket *pkt) { return; } - // UNNAMED STATEMENT - if (statement_name.empty()) { - param_type_buf = unnamed_stmt_param_types_; - // NAMED STATEMENT - } else { - param_type_buf = statement_param_types_[statement_name]; - } - const auto &query_string = statement->GetQueryString(); const auto &query_type = statement->GetQueryType(); @@ -500,9 +471,8 @@ void PostgresProtocolHandler::ExecBindMessage(InputPacket *pkt) { auto param_types = statement->GetParamTypes(); - auto val_buf_begin = pkt->Begin() + pkt->ptr; - auto val_buf_len = ReadParamValue(pkt, num_params, param_types, - bind_parameters, param_values, formats); + ReadParamValue(pkt, num_params, param_types, bind_parameters, param_values, + formats); int format_codes_number = PacketGetInt(pkt, 2); LOG_TRACE("format_codes_number: %d", format_codes_number); @@ -532,30 +502,10 @@ void PostgresProtocolHandler::ExecBindMessage(InputPacket *pkt) { // executor context. } - std::shared_ptr param_stat(nullptr); - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID && - num_params > 0) { - // Make a copy of format for stat collection - stats::QueryMetric::QueryParamBuf param_format_buf; - param_format_buf.len = format_buf_len; - param_format_buf.buf = PacketCopyBytes(format_buf_begin, format_buf_len); - PELOTON_ASSERT(format_buf_len > 0); - - // Make a copy of value for stat collection - stats::QueryMetric::QueryParamBuf param_val_buf; - param_val_buf.len = val_buf_len; - param_val_buf.buf = PacketCopyBytes(val_buf_begin, val_buf_len); - PELOTON_ASSERT(val_buf_len > 0); - - param_stat.reset(new stats::QueryMetric::QueryParams( - param_format_buf, param_type_buf, param_val_buf, num_params)); - } - // Construct a portal. // Notice that this will move param_values so no value will be left there. auto portal = - new Portal(portal_name, statement, std::move(param_values), param_stat); + new Portal(portal_name, statement, std::move(param_values)); std::shared_ptr portal_reference(portal); auto itr = portals_.find(portal_name); @@ -635,7 +585,8 @@ size_t PostgresProtocolHandler::ReadParamValue( .CastAs(PostgresValueTypeToPelotonValueType( (PostgresValueType)param_types[param_idx])); } - PELOTON_ASSERT(param_values[param_idx].GetTypeId() != type::TypeId::INVALID); + PELOTON_ASSERT(param_values[param_idx].GetTypeId() != + type::TypeId::INVALID); } else { // BINARY mode PostgresValueType pg_value_type = @@ -715,7 +666,8 @@ size_t PostgresProtocolHandler::ReadParamValue( break; } } - PELOTON_ASSERT(param_values[param_idx].GetTypeId() != type::TypeId::INVALID); + PELOTON_ASSERT(param_values[param_idx].GetTypeId() != + type::TypeId::INVALID); } } } @@ -797,7 +749,6 @@ ProcessResult PostgresProtocolHandler::ExecExecuteMessage( traffic_cop_->SetStatement(portal->GetStatement()); - auto param_stat = portal->GetParamStat(); if (traffic_cop_->GetStatement().get() == nullptr) { LOG_ERROR("Did not find statement in portal : %s", portal_name.c_str()); SendErrorResponse( @@ -812,7 +763,7 @@ ProcessResult PostgresProtocolHandler::ExecExecuteMessage( auto status = traffic_cop_->ExecuteStatement( traffic_cop_->GetStatement(), traffic_cop_->GetParamVal(), unnamed, - param_stat, result_format_, traffic_cop_->GetResult(), thread_id); + result_format_, traffic_cop_->GetResult(), thread_id); if (traffic_cop_->GetQueuing()) { return ProcessResult::PROCESSING; } diff --git a/src/optimizer/stats/stats_storage.cpp b/src/optimizer/stats/stats_storage.cpp index d1b2fed6b12..13717920a6c 100644 --- a/src/optimizer/stats/stats_storage.cpp +++ b/src/optimizer/stats/stats_storage.cpp @@ -17,6 +17,7 @@ #include "concurrency/transaction_manager_factory.h" #include "optimizer/stats/column_stats.h" #include "optimizer/stats/table_stats.h" +#include "storage/database.h" #include "storage/storage_manager.h" #include "type/ephemeral_pool.h" diff --git a/src/optimizer/stats/tuple_samples_storage.cpp b/src/optimizer/stats/tuple_samples_storage.cpp index 9ae85256758..15ba706ae75 100644 --- a/src/optimizer/stats/tuple_samples_storage.cpp +++ b/src/optimizer/stats/tuple_samples_storage.cpp @@ -19,6 +19,7 @@ #include "executor/seq_scan_executor.h" #include "optimizer/stats/tuple_sampler.h" #include "planner/insert_plan.h" +#include "storage/data_table.h" namespace peloton { namespace optimizer { diff --git a/src/optimizer/util.cpp b/src/optimizer/util.cpp index 0d01e35e8ac..1587d173e5d 100644 --- a/src/optimizer/util.cpp +++ b/src/optimizer/util.cpp @@ -12,7 +12,6 @@ #include "optimizer/util.h" -#include "catalog/query_metrics_catalog.h" #include "concurrency/transaction_manager_factory.h" #include "expression/expression_util.h" #include "planner/copy_plan.h" @@ -147,12 +146,6 @@ std::unique_ptr CreateCopyPlan( std::string table_name(copy_stmt->cpy_table->GetTableName()); bool deserialize_parameters = false; - // If we're copying the query metric table, then we need to handle the - // deserialization of prepared stmt parameters - if (table_name == QUERY_METRICS_CATALOG_NAME) { - LOG_DEBUG("Copying the query_metric table."); - deserialize_parameters = true; - } std::unique_ptr copy_plan( new planner::CopyPlan(copy_stmt->file_path, deserialize_parameters)); diff --git a/src/statistics/abstract_metric.cpp b/src/statistics/abstract_metric.cpp deleted file mode 100644 index 48d176938b9..00000000000 --- a/src/statistics/abstract_metric.cpp +++ /dev/null @@ -1,23 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// abstract_metric.cpp -// -// Identification: src/statistics/abstract_metric.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "statistics/abstract_metric.h" - -namespace peloton { -namespace stats { - -AbstractMetric::AbstractMetric(MetricType type) { type_ = type; } - -AbstractMetric::~AbstractMetric() {} - -} // namespace stats -} // namespace peloton diff --git a/src/statistics/access_metric.cpp b/src/statistics/access_metric.cpp deleted file mode 100644 index 1eee210fcee..00000000000 --- a/src/statistics/access_metric.cpp +++ /dev/null @@ -1,30 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// access_metric.cpp -// -// Identification: src/statistics/access_metric.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "statistics/access_metric.h" -#include "common/macros.h" - -namespace peloton { -namespace stats { - -void AccessMetric::Aggregate(AbstractMetric &source) { - PELOTON_ASSERT(source.GetType() == MetricType::ACCESS); - - auto access_metric = static_cast(source); - for (size_t i = 0; i < NUM_COUNTERS; ++i) { - access_counters_[i].Aggregate( - static_cast(access_metric.GetAccessCounter(i))); - } -} - -} // namespace stats -} // namespace peloton diff --git a/src/statistics/backend_stats_context.cpp b/src/statistics/backend_stats_context.cpp deleted file mode 100644 index d10a1912831..00000000000 --- a/src/statistics/backend_stats_context.cpp +++ /dev/null @@ -1,382 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// backend_stats_context.cpp -// -// Identification: src/statistics/backend_stats_context.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "statistics/backend_stats_context.h" - -#include - -#include "catalog/catalog.h" -#include "catalog/manager.h" -#include "common/internal_types.h" -#include "common/statement.h" -#include "index/index.h" -#include "statistics/stats_aggregator.h" -#include "storage/storage_manager.h" -#include "storage/tile_group.h" - -namespace peloton { -namespace stats { - -CuckooMap> & -BackendStatsContext::GetBackendContextMap() { - static CuckooMap> - stats_context_map; - return stats_context_map; -} - -BackendStatsContext *BackendStatsContext::GetInstance() { - // Each thread gets a backend stats context - std::thread::id this_id = std::this_thread::get_id(); - std::shared_ptr result(nullptr); - auto &stats_context_map = GetBackendContextMap(); - if (stats_context_map.Find(this_id, result) == false) { - result.reset(new BackendStatsContext(LATENCY_MAX_HISTORY_THREAD, true)); - stats_context_map.Insert(this_id, result); - } - return result.get(); -} - -BackendStatsContext::BackendStatsContext(size_t max_latency_history, - bool regiser_to_aggregator) - : txn_latencies_(MetricType::LATENCY, max_latency_history) { - std::thread::id this_id = std::this_thread::get_id(); - thread_id_ = this_id; - - is_registered_to_aggregator_ = regiser_to_aggregator; - - // Register to the global aggregator - if (regiser_to_aggregator == true) - StatsAggregator::GetInstance().RegisterContext(thread_id_, this); -} - -BackendStatsContext::~BackendStatsContext() {} - -//===--------------------------------------------------------------------===// -// ACCESSORS -//===--------------------------------------------------------------------===// - -// Returns the table metric with the given database ID and table ID -TableMetric *BackendStatsContext::GetTableMetric(oid_t database_id, - oid_t table_id) { - // combine database_id and table_id to form unique key - uint64_t key = ((uint64_t)database_id << 32) | table_id; - if (table_metrics_.find(key) == table_metrics_.end()) { - table_metrics_[key] = std::unique_ptr( - new TableMetric{MetricType::TABLE, database_id, table_id}); - } - return table_metrics_[key].get(); -} - -// Returns the database metric with the given database ID -DatabaseMetric *BackendStatsContext::GetDatabaseMetric(oid_t database_id) { - if (database_metrics_.find(database_id) == database_metrics_.end()) { - database_metrics_[database_id] = std::unique_ptr( - new DatabaseMetric{MetricType::DATABASE, database_id}); - } - return database_metrics_[database_id].get(); -} - -// Returns the index metric with the given database ID, table ID, and -// index ID -IndexMetric *BackendStatsContext::GetIndexMetric(oid_t database_id, - oid_t table_id, - oid_t index_id) { - std::shared_ptr index_metric; - uint64_t key = ((uint64_t)database_id << 32) | index_id; - // Index metric doesn't exist yet - if (index_metrics_.Contains(key) == false) { - index_metric.reset( - new IndexMetric{MetricType::INDEX, database_id, table_id, index_id}); - index_metrics_.Insert(key, index_metric); - index_id_lock.Lock(); - index_ids_.insert(key); - index_id_lock.Unlock(); - } - // Get index metric from map - index_metrics_.Find(key, index_metric); - return index_metric.get(); -} - -LatencyMetric &BackendStatsContext::GetTxnLatencyMetric() { - return txn_latencies_; -} - -void BackendStatsContext::IncrementTableReads(oid_t tile_group_id) { - oid_t table_id = - catalog::Manager::GetInstance().GetTileGroup(tile_group_id)->GetTableId(); - oid_t database_id = catalog::Manager::GetInstance() - .GetTileGroup(tile_group_id) - ->GetDatabaseId(); - auto table_metric = GetTableMetric(database_id, table_id); - PELOTON_ASSERT(table_metric != nullptr); - table_metric->GetTableAccess().IncrementReads(); - if (ongoing_query_metric_ != nullptr) { - ongoing_query_metric_->GetQueryAccess().IncrementReads(); - } -} - -void BackendStatsContext::IncrementTableInserts(oid_t tile_group_id) { - oid_t table_id = - catalog::Manager::GetInstance().GetTileGroup(tile_group_id)->GetTableId(); - oid_t database_id = catalog::Manager::GetInstance() - .GetTileGroup(tile_group_id) - ->GetDatabaseId(); - auto table_metric = GetTableMetric(database_id, table_id); - PELOTON_ASSERT(table_metric != nullptr); - table_metric->GetTableAccess().IncrementInserts(); - if (ongoing_query_metric_ != nullptr) { - ongoing_query_metric_->GetQueryAccess().IncrementInserts(); - } -} - -void BackendStatsContext::IncrementTableUpdates(oid_t tile_group_id) { - oid_t table_id = - catalog::Manager::GetInstance().GetTileGroup(tile_group_id)->GetTableId(); - oid_t database_id = catalog::Manager::GetInstance() - .GetTileGroup(tile_group_id) - ->GetDatabaseId(); - auto table_metric = GetTableMetric(database_id, table_id); - PELOTON_ASSERT(table_metric != nullptr); - table_metric->GetTableAccess().IncrementUpdates(); - if (ongoing_query_metric_ != nullptr) { - ongoing_query_metric_->GetQueryAccess().IncrementUpdates(); - } -} - -void BackendStatsContext::IncrementTableDeletes(oid_t tile_group_id) { - oid_t table_id = - catalog::Manager::GetInstance().GetTileGroup(tile_group_id)->GetTableId(); - oid_t database_id = catalog::Manager::GetInstance() - .GetTileGroup(tile_group_id) - ->GetDatabaseId(); - auto table_metric = GetTableMetric(database_id, table_id); - PELOTON_ASSERT(table_metric != nullptr); - table_metric->GetTableAccess().IncrementDeletes(); - if (ongoing_query_metric_ != nullptr) { - ongoing_query_metric_->GetQueryAccess().IncrementDeletes(); - } -} - -void BackendStatsContext::IncrementIndexReads(size_t read_count, - index::IndexMetadata *metadata) { - oid_t index_id = metadata->GetOid(); - oid_t table_id = metadata->GetTableOid(); - oid_t database_id = metadata->GetDatabaseOid(); - auto index_metric = GetIndexMetric(database_id, table_id, index_id); - PELOTON_ASSERT(index_metric != nullptr); - index_metric->GetIndexAccess().IncrementReads(read_count); -} - -void BackendStatsContext::IncrementIndexInserts( - index::IndexMetadata *metadata) { - oid_t index_id = metadata->GetOid(); - oid_t table_id = metadata->GetTableOid(); - oid_t database_id = metadata->GetDatabaseOid(); - auto index_metric = GetIndexMetric(database_id, table_id, index_id); - PELOTON_ASSERT(index_metric != nullptr); - index_metric->GetIndexAccess().IncrementInserts(); -} - -void BackendStatsContext::IncrementIndexUpdates( - index::IndexMetadata *metadata) { - oid_t index_id = metadata->GetOid(); - oid_t table_id = metadata->GetTableOid(); - oid_t database_id = metadata->GetDatabaseOid(); - auto index_metric = GetIndexMetric(database_id, table_id, index_id); - PELOTON_ASSERT(index_metric != nullptr); - index_metric->GetIndexAccess().IncrementUpdates(); -} - -void BackendStatsContext::IncrementIndexDeletes( - size_t delete_count, index::IndexMetadata *metadata) { - oid_t index_id = metadata->GetOid(); - oid_t table_id = metadata->GetTableOid(); - oid_t database_id = metadata->GetDatabaseOid(); - auto index_metric = GetIndexMetric(database_id, table_id, index_id); - PELOTON_ASSERT(index_metric != nullptr); - index_metric->GetIndexAccess().IncrementDeletes(delete_count); -} - -void BackendStatsContext::IncrementTxnCommitted(oid_t database_id) { - auto database_metric = GetDatabaseMetric(database_id); - PELOTON_ASSERT(database_metric != nullptr); - database_metric->IncrementTxnCommitted(); - CompleteQueryMetric(); -} - -void BackendStatsContext::IncrementTxnAborted(oid_t database_id) { - auto database_metric = GetDatabaseMetric(database_id); - PELOTON_ASSERT(database_metric != nullptr); - database_metric->IncrementTxnAborted(); - CompleteQueryMetric(); -} - -void BackendStatsContext::InitQueryMetric( - const std::shared_ptr statement, - const std::shared_ptr params) { - CompleteQueryMetric(); - // TODO currently all queries belong to DEFAULT_DB - ongoing_query_metric_.reset(new QueryMetric(MetricType::QUERY, - statement->GetQueryString(), - params, CATALOG_DATABASE_OID)); -} - -//===--------------------------------------------------------------------===// -// HELPER FUNCTIONS -//===--------------------------------------------------------------------===// - -void BackendStatsContext::Aggregate(BackendStatsContext &source) { - // Aggregate all global metrics - txn_latencies_.Aggregate(source.txn_latencies_); - txn_latencies_.ComputeLatencies(); - - // Aggregate all per-database metrics - for (auto &database_item : source.database_metrics_) { - GetDatabaseMetric(database_item.first)->Aggregate(*database_item.second); - } - - // Aggregate all per-table metrics - for (auto &table_item : source.table_metrics_) { - GetTableMetric(table_item.second->GetDatabaseId(), - table_item.second->GetTableId()) - ->Aggregate(*table_item.second); - } - - // Aggregate all per-index metrics - for (auto id : index_ids_) { - std::shared_ptr index_metric; - index_metrics_.Find(id, index_metric); - auto database_oid = index_metric->GetDatabaseId(); - auto table_oid = index_metric->GetTableId(); - index_metric->Aggregate( - *(source.GetIndexMetric(database_oid, table_oid, id))); - } - - // Aggregate all per-query metrics - std::shared_ptr query_metric; - while (source.completed_query_metrics_.Dequeue(query_metric)) { - completed_query_metrics_.Enqueue(query_metric); - LOG_TRACE("Found a query metric to aggregate"); - aggregated_query_count_++; - } -} - -void BackendStatsContext::Reset() { - txn_latencies_.Reset(); - - for (auto &database_item : database_metrics_) { - database_item.second->Reset(); - } - for (auto &table_item : table_metrics_) { - table_item.second->Reset(); - } - for (auto id : index_ids_) { - std::shared_ptr index_metric; - index_metrics_.Find(id, index_metric); - index_metric->Reset(); - } - - oid_t num_databases = - storage::StorageManager::GetInstance()->GetDatabaseCount(); - for (oid_t i = 0; i < num_databases; ++i) { - auto database = - storage::StorageManager::GetInstance()->GetDatabaseWithOffset(i); - oid_t database_id = database->GetOid(); - - // Reset database metrics - if (database_metrics_.find(database_id) == database_metrics_.end()) { - database_metrics_[database_id] = std::unique_ptr( - new DatabaseMetric{MetricType::DATABASE, database_id}); - } - - // Reset table metrics - oid_t num_tables = database->GetTableCount(); - for (oid_t j = 0; j < num_tables; ++j) { - auto table = database->GetTable(j); - oid_t table_id = table->GetOid(); - - uint64_t key = ((uint64_t)database_id << 32) | table_id; - if (table_metrics_.find(key) == table_metrics_.end()) { - table_metrics_[key] = std::unique_ptr( - new TableMetric{MetricType::TABLE, database_id, table_id}); - } - - // Reset indexes metrics - oid_t num_indexes = table->GetIndexCount(); - for (oid_t k = 0; k < num_indexes; ++k) { - auto index = table->GetIndex(k); - if (index == nullptr) continue; - oid_t index_id = index->GetOid(); - // combine database_id and index_id to form unique key - uint64_t index_key = ((uint64_t)database_id << 32) | index_id; - if (index_metrics_.Contains(index_key) == false) { - std::shared_ptr index_metric(new IndexMetric{ - MetricType::INDEX, database_id, table_id, index_id}); - index_metrics_.Insert(index_key, index_metric); - index_ids_.insert(index_key); - } - } - } - } -} - -std::string BackendStatsContext::ToString() const { - std::stringstream ss; - - ss << txn_latencies_.GetInfo() << std::endl; - - for (auto &database_item : database_metrics_) { - oid_t database_id = database_item.second->GetDatabaseId(); - ss << database_item.second->GetInfo(); - - for (auto &table_item : table_metrics_) { - if (table_item.second->GetDatabaseId() == database_id) { - ss << table_item.second->GetInfo(); - - oid_t table_id = table_item.second->GetTableId(); - for (auto id : index_ids_) { - std::shared_ptr index_metric; - index_metrics_.Find(id, index_metric); - if (index_metric->GetDatabaseId() == database_id && - index_metric->GetTableId() == table_id) { - ss << index_metric->GetInfo(); - } - } - if (!index_metrics_.IsEmpty()) { - ss << std::endl; - } - } - if (!table_metrics_.empty()) { - ss << std::endl; - } - } - if (!database_metrics_.empty()) { - ss << std::endl; - } - } - std::string info = ss.str(); - StringUtil::RTrim(info); - return info; -} - -void BackendStatsContext::CompleteQueryMetric() { - if (ongoing_query_metric_ != nullptr) { - ongoing_query_metric_->GetProcessorMetric().RecordTime(); - ongoing_query_metric_->GetQueryLatency().RecordLatency(); - completed_query_metrics_.Enqueue(ongoing_query_metric_); - ongoing_query_metric_.reset(); - LOG_TRACE("Ongoing query completed"); - } -} - -} // namespace stats -} // namespace peloton diff --git a/src/statistics/counter_metric.cpp b/src/statistics/counter_metric.cpp deleted file mode 100644 index 8cb5325139a..00000000000 --- a/src/statistics/counter_metric.cpp +++ /dev/null @@ -1,29 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// counter_metric.cpp -// -// Identification: src/statistics/counter_metric.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "statistics/counter_metric.h" -#include "common/macros.h" - -namespace peloton { -namespace stats { - -CounterMetric::CounterMetric(MetricType type) : AbstractMetric(type) { - count_ = 0; -} - -void CounterMetric::Aggregate(AbstractMetric &source) { - PELOTON_ASSERT(source.GetType() == MetricType::COUNTER); - count_ += static_cast(source).GetCounter(); -} - -} // namespace stats -} // namespace peloton diff --git a/src/statistics/database_metric.cpp b/src/statistics/database_metric.cpp index 211a19d4813..32a0647c583 100644 --- a/src/statistics/database_metric.cpp +++ b/src/statistics/database_metric.cpp @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +#include "concurrency/transaction_manager_factory.h" #include "util/string_util.h" #include "statistics/database_metric.h" #include "common/macros.h" @@ -17,25 +18,35 @@ namespace peloton { namespace stats { -DatabaseMetric::DatabaseMetric(MetricType type, oid_t database_id) - : AbstractMetric(type), database_id_(database_id) {} +void DatabaseMetricRawData::UpdateAndPersist() { + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + auto time_since_epoch = std::chrono::system_clock::now().time_since_epoch(); + auto time_stamp = std::chrono::duration_cast( + time_since_epoch).count(); -void DatabaseMetric::Aggregate(AbstractMetric& source) { - PELOTON_ASSERT(source.GetType() == MetricType::DATABASE); + auto database_metrics_catalog = + catalog::DatabaseMetricsCatalog::GetInstance(); + for (auto &entry : counters_) { + // one iteration per database + oid_t database_oid = entry.first; + auto &counts = entry.second; - DatabaseMetric& db_metric = static_cast(source); - txn_committed_.Aggregate(db_metric.GetTxnCommitted()); - txn_aborted_.Aggregate(db_metric.GetTxnAborted()); -} + auto old_metric = + database_metrics_catalog->GetDatabaseMetricsObject(database_oid, txn); + if (old_metric == nullptr) { + // no entry exists for this database yet + database_metrics_catalog->InsertDatabaseMetrics( + database_oid, counts.first, counts.second, time_stamp, nullptr, txn); + } else { + // update existing entry + database_metrics_catalog->UpdateDatabaseMetrics( + database_oid, old_metric->GetTxnCommitted() + counts.first, + old_metric->GetTxnAborted() + counts.second, time_stamp, txn); + } + } -const std::string DatabaseMetric::GetInfo() const { - std::stringstream ss; - ss << peloton::GETINFO_THICK_LINE << std::endl; - ss << "// DATABASE_ID " << database_id_ << std::endl; - ss << peloton::GETINFO_THICK_LINE << std::endl; - ss << "# transactions committed: " << txn_committed_.GetInfo() << std::endl; - ss << "# transactions aborted: " << txn_aborted_.GetInfo(); - return ss.str(); + txn_manager.CommitTransaction(txn); } } // namespace stats diff --git a/src/statistics/index_metric.cpp b/src/statistics/index_metric.cpp index f117eb48318..c44159ca435 100644 --- a/src/statistics/index_metric.cpp +++ b/src/statistics/index_metric.cpp @@ -6,10 +6,14 @@ // // Identification: src/statistics/index_metric.cpp // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// +#include "catalog/catalog.h" +#include "catalog/system_catalogs.h" +#include "catalog/index_metrics_catalog.h" +#include "concurrency/transaction_manager_factory.h" #include "statistics/index_metric.h" #include "storage/storage_manager.h" #include "index/index.h" @@ -17,29 +21,61 @@ namespace peloton { namespace stats { -IndexMetric::IndexMetric(MetricType type, oid_t database_id, oid_t table_id, - oid_t index_id) - : AbstractMetric(type), - database_id_(database_id), - table_id_(table_id), - index_id_(index_id) { - index_name_ = ""; - try { - auto index = storage::StorageManager::GetInstance()->GetIndexWithOid( - database_id, table_id, index_id); - index_name_ = index->GetName(); - for (auto& ch : index_name_) { - ch = toupper(ch); +const std::vector + IndexMetricRawData::COUNTER_TYPES = { + IndexMetricRawData::CounterType::READ, + IndexMetricRawData::CounterType::UPDATE, + IndexMetricRawData::CounterType::INSERT, + IndexMetricRawData::CounterType::DELETE, + IndexMetricRawData::CounterType::MEMORY_ALLOC, + IndexMetricRawData::CounterType::MEMORY_USAGE}; + +void IndexMetricRawData::Aggregate(AbstractRawData &other) { + auto &other_index_metric = dynamic_cast(other); + for (auto &entry : other_index_metric.counters_) { + for (auto &counter_type : COUNTER_TYPES) { + GetCounter(entry.first, counter_type) += + other_index_metric.GetCounter(entry.first, counter_type); } - } catch (CatalogException& e) { } } -void IndexMetric::Aggregate(AbstractMetric& source) { - assert(source.GetType() == MetricType::INDEX); +void IndexMetricRawData::UpdateAndPersist() { + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + auto time_since_epoch = std::chrono::system_clock::now().time_since_epoch(); + auto time_stamp = std::chrono::duration_cast( + time_since_epoch).count(); + + for (auto &entry : counters_) { + oid_t database_oid = entry.first.first; + oid_t index_oid = entry.first.second; + oid_t table_oid = 0; // FIXME!! + + auto &counts = entry.second; + auto system_catalogs = + catalog::Catalog::GetInstance()->GetSystemCatalogs(database_oid); + auto index_metrics_catalog = system_catalogs->GetIndexMetricsCatalog(); + auto old_metric = + index_metrics_catalog->GetIndexMetricsObject(index_oid, txn); + + if (old_metric == nullptr) { + index_metrics_catalog->InsertIndexMetrics( + index_oid, table_oid, counts[READ], counts[UPDATE], counts[INSERT], + counts[DELETE], counts[MEMORY_ALLOC], counts[MEMORY_USAGE], + time_stamp, nullptr, txn); + } else { + index_metrics_catalog->UpdateIndexMetrics( + index_oid, table_oid, old_metric->GetReads() + counts[READ], + old_metric->GetUpdates() + counts[UPDATE], + old_metric->GetInserts() + counts[INSERT], + old_metric->GetDeletes() + counts[DELETE], + old_metric->GetMemoryAlloc() + counts[MEMORY_ALLOC], + old_metric->GetMemoryUsage() + counts[MEMORY_USAGE], time_stamp, txn); + } + } - IndexMetric& index_metric = static_cast(source); - index_access_.Aggregate(index_metric.GetIndexAccess()); + txn_manager.CommitTransaction(txn); } } // namespace stats diff --git a/src/statistics/latency_metric.cpp b/src/statistics/latency_metric.cpp index 7a8b768f8ee..4cd780533af 100644 --- a/src/statistics/latency_metric.cpp +++ b/src/statistics/latency_metric.cpp @@ -6,7 +6,7 @@ // // Identification: src/statistics/latency_metric.cpp // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// @@ -18,67 +18,14 @@ namespace peloton { namespace stats { -LatencyMetric::LatencyMetric(MetricType type, size_t max_history) - : AbstractMetric(type) { - max_history_ = max_history; - latencies_.SetCapaciry(max_history_); -} - -void LatencyMetric::Aggregate(AbstractMetric& source) { - PELOTON_ASSERT(source.GetType() == MetricType::LATENCY); - - LatencyMetric& latency_metric = static_cast(source); - CircularBuffer source_latencies = latency_metric.Copy(); - { - // This method should only ever be called by the aggregator which - // is the only thread to access its own latencies_, but we lock - // here just to be safe. Either way the aggregator should never - // have to block here. - std::lock_guard lock(latency_mutex_); - for (double latency_value : source_latencies) { - latencies_.PushBack(latency_value); - } - } -} - -CircularBuffer LatencyMetric::Copy() { - CircularBuffer new_buffer; - { - // This method is only called by the aggregator to make - // a copy of a worker thread's latencies_. - std::lock_guard lock(latency_mutex_); - new_buffer = latencies_; - } - return new_buffer; -} - -const std::string LatencyMetric::GetInfo() const { - std::stringstream ss; - ss << "TXN LATENCY (ms): [ "; - ss << "average=" << latency_measurements_.average_; - ss << ", min=" << latency_measurements_.min_; - ss << ", 25th-%-tile=" << latency_measurements_.perc_25th_; - ss << ", median=" << latency_measurements_.median_; - ss << ", 75th-%-tile=" << latency_measurements_.perc_75th_; - ss << ", 99th-%-tile=" << latency_measurements_.perc_99th_; - ss << ", max=" << latency_measurements_.max_; - ss << " ]"; - return ss.str(); -} - -void LatencyMetric::ComputeLatencies() { - // LatencyMeasurements measurements; +LatencyMeasurements LatencyMetricRawData::DescriptiveFromRaw() { + LatencyMeasurements measurements; if (latencies_.IsEmpty()) { - return; + return measurements; } std::vector sorted_latencies; double latency_sum = 0.0; { - // This method is called only by the aggregator when - // after it's aggregated all worker threads latencies - // into its own latencies_ buffer. Still we lock here - // just in case. - std::lock_guard lock(latency_mutex_); for (double latency : latencies_) { sorted_latencies.push_back(latency); latency_sum += latency; @@ -89,24 +36,25 @@ void LatencyMetric::ComputeLatencies() { size_t latencies_size = sorted_latencies.size(); // Calculate average - latency_measurements_.average_ = latency_sum / latencies_size; + measurements.average_ = latency_sum / latencies_size; // Min, max, median, and percentiles are values at indexes - latency_measurements_.min_ = sorted_latencies[0]; - latency_measurements_.max_ = sorted_latencies[latencies_size - 1]; + measurements.min_ = sorted_latencies[0]; + measurements.max_ = sorted_latencies[latencies_size - 1]; size_t mid = latencies_size / 2; if (latencies_size % 2 == 0 || latencies_size == 1) { - latency_measurements_.median_ = sorted_latencies[mid]; + measurements.median_ = sorted_latencies[mid]; } else { - latency_measurements_.median_ = + measurements.median_ = (sorted_latencies[mid - 1] + sorted_latencies[mid]) / 2; } size_t index_25th = (size_t)(0.25 * latencies_size); size_t index_75th = (size_t)(0.75 * latencies_size); size_t index_99th = (size_t)(0.99 * latencies_size); - latency_measurements_.perc_25th_ = sorted_latencies[index_25th]; - latency_measurements_.perc_75th_ = sorted_latencies[index_75th]; - latency_measurements_.perc_99th_ = sorted_latencies[index_99th]; + measurements.perc_25th_ = sorted_latencies[index_25th]; + measurements.perc_75th_ = sorted_latencies[index_75th]; + measurements.perc_99th_ = sorted_latencies[index_99th]; + return measurements; } } // namespace stats diff --git a/src/statistics/processor_metric.cpp b/src/statistics/processor_metric.cpp deleted file mode 100644 index 0f8a807d861..00000000000 --- a/src/statistics/processor_metric.cpp +++ /dev/null @@ -1,79 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// processor_metric.cpp -// -// Identification: src/statistics/processor_metric.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "statistics/processor_metric.h" - -#ifndef RUSAGE_THREAD - -#include -#include -#include - -#endif - -namespace peloton { -namespace stats { - -ProcessorMetric::ProcessorMetric(MetricType type) : AbstractMetric(type) {} - -void ProcessorMetric::StartTimer() { - UpdateTimeInt(user_time_begin_, sys_time_begin_); -} - -void ProcessorMetric::RecordTime() { - UpdateTimeInt(user_time_end_, sys_time_end_); -} - -double ProcessorMetric::GetMilliSec(struct timeval time) const { - return time.tv_sec * 1000 + time.tv_usec / 1000.0; -} - -void ProcessorMetric::UpdateTimeInt(double &user_time, double &system_time) { -#ifdef RUSAGE_THREAD // RUSAGE_THREAD is Linux-specific. - struct rusage usage; - int ret = getrusage(RUSAGE_THREAD, &usage); - if (ret != 0) { - throw StatException("Error getting resource usage"); - } - user_time = GetMilliSec(usage.ru_utime); - system_time = GetMilliSec(usage.ru_stime); -#else // https://stackoverflow.com/questions/13893134/get-current-pthread-cpu-usage-mac-os-x - mach_port_t thread = mach_thread_self(); - thread_basic_info_data_t info; - mach_msg_type_number_t count = THREAD_BASIC_INFO_COUNT; - kern_return_t kr = thread_info(thread, THREAD_BASIC_INFO, (thread_info_t) &info, &count); - - if (kr == KERN_SUCCESS && (info.flags & TH_FLAGS_IDLE) == 0) { - user_time = ((double) info.user_time.microseconds) / 1000; - system_time = ((double) info.system_time.microseconds) / 1000; - } - else { - throw StatException("Error getting resource usage"); - } - mach_port_deallocate(mach_task_self(), thread); -#endif - -} - -const std::string ProcessorMetric::GetInfo() const { - std::stringstream ss; - ss << "Query CPU Time (ms): [ "; - ss << "system time=" << GetSystemDuration(); - ss << ", user time=" << GetUserDuration(); - ss << " ]"; - return ss.str(); -} - -void ProcessorMetric::Aggregate(AbstractMetric &source UNUSED_ATTRIBUTE) {} - -} // namespace stats -} // namespace peloton diff --git a/src/statistics/query_metric.cpp b/src/statistics/query_metric.cpp deleted file mode 100644 index b580d177820..00000000000 --- a/src/statistics/query_metric.cpp +++ /dev/null @@ -1,49 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// table_metric.cpp -// -// Identification: src/statistics/table_metric.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "common/macros.h" -#include "common/logger.h" -#include "statistics/query_metric.h" -#include "catalog/catalog.h" -#include "storage/data_table.h" - -namespace peloton { -namespace stats { - -QueryMetric::QueryMetric(MetricType type, const std::string& query_name, - std::shared_ptr query_params, - const oid_t database_id) - : AbstractMetric(type), - database_id_(database_id), - query_name_(query_name), - query_params_(query_params) { - latency_metric_.StartTimer(); - processor_metric_.StartTimer(); - LOG_TRACE("Query metric initialized"); -} - -QueryMetric::QueryParams::QueryParams(QueryParamBuf format_buf_copy, - QueryParamBuf type_buf_copy, - QueryParamBuf val_buf_copy, - int num_params) - : format_buf_copy(format_buf_copy), - type_buf_copy(type_buf_copy), - val_buf_copy(val_buf_copy), - num_params(num_params) { - LOG_TRACE("query param: %d, %d, %d", type_buf_copy.len, format_buf_copy.len, - val_buf_copy.len); -} - -void QueryMetric::Aggregate(AbstractMetric& source UNUSED_ATTRIBUTE) {} - -} // namespace stats -} // namespace peloton diff --git a/src/statistics/stats_aggregator.cpp b/src/statistics/stats_aggregator.cpp index 6d4a39847a3..53da60d7a92 100644 --- a/src/statistics/stats_aggregator.cpp +++ b/src/statistics/stats_aggregator.cpp @@ -6,337 +6,53 @@ // // Identification: src/statistics/stats_aggregator.cpp // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// #include "statistics/stats_aggregator.h" #include - -#include "catalog/catalog.h" -#include "catalog/database_metrics_catalog.h" -#include "catalog/system_catalogs.h" -#include "concurrency/transaction_manager_factory.h" -#include "index/index.h" -#include "storage/storage_manager.h" -#include "type/ephemeral_pool.h" - namespace peloton { namespace stats { -StatsAggregator::StatsAggregator(int64_t aggregation_interval_ms) - : stats_history_(0, false), - aggregated_stats_(LATENCY_MAX_HISTORY_AGGREGATOR, false), - aggregation_interval_ms_(aggregation_interval_ms), - thread_number_(0), - total_prev_txn_committed_(0) { - pool_.reset(new type::EphemeralPool()); - try { - ofs_.open(peloton_stats_directory_, std::ofstream::out); - } catch (std::ofstream::failure &e) { - LOG_ERROR("Couldn't open the stats log file %s", e.what()); - } - LaunchAggregator(); -} - -StatsAggregator::~StatsAggregator() { - LOG_DEBUG("StatsAggregator destruction"); - ShutdownAggregator(); - try { - ofs_.close(); - } catch (std::ofstream::failure &e) { - LOG_ERROR("Couldn't close the stats log file %s", e.what()); - } -} - -void StatsAggregator::LaunchAggregator() { - if (!is_aggregating_) { - aggregator_thread_ = std::thread(&StatsAggregator::RunAggregator, this); - is_aggregating_ = true; - } +void StatsAggregator::Terminate() { + std::unique_lock lock(mutex_); + exiting_ = true; + while (exiting_) exec_finished_.wait(lock); } -void StatsAggregator::ShutdownAggregator() { - if (is_aggregating_) { - is_aggregating_ = false; - exec_finished_.notify_one(); - LOG_DEBUG("notifying aggregator thread..."); - aggregator_thread_.join(); - LOG_DEBUG("aggregator thread joined"); - } -} - -void StatsAggregator::Aggregate(int64_t &interval_cnt, double &alpha, - double &weighted_avg_throughput) { - interval_cnt++; - LOG_TRACE( - "\n//////////////////////////////////////////////////////" - "//////////////////////////////////////////////////////\n"); - LOG_TRACE("TIME ELAPSED: %" PRId64 " sec", interval_cnt); - - aggregated_stats_.Reset(); - std::thread::id this_id = aggregator_thread_.get_id(); - - for (auto &val : backend_stats_) { - // Exclude the txn stats generated by the aggregator thread - if (val.first != this_id) { - aggregated_stats_.Aggregate((*val.second)); - } - } - aggregated_stats_.Aggregate(stats_history_); - LOG_TRACE("%s\n", aggregated_stats_.ToString().c_str()); - - int64_t current_txns_committed = 0; - // Traverse the metric of all threads to get the total number of committed - // txns. - for (auto &database_item : aggregated_stats_.database_metrics_) { - current_txns_committed += - database_item.second->GetTxnCommitted().GetCounter(); - } - int64_t txns_committed_this_interval = - current_txns_committed - total_prev_txn_committed_; - double throughput_ = (double)txns_committed_this_interval / 1000 * - STATS_AGGREGATION_INTERVAL_MS; - double avg_throughput_ = (double)current_txns_committed / interval_cnt / - STATS_AGGREGATION_INTERVAL_MS * 1000; - if (interval_cnt == 1) { - weighted_avg_throughput = throughput_; - } else { - weighted_avg_throughput = - alpha * throughput_ + (1 - alpha) * weighted_avg_throughput; - } - - total_prev_txn_committed_ = current_txns_committed; - LOG_TRACE("Average throughput: %lf txn/s", avg_throughput_); - LOG_TRACE("Moving avg. throughput: %lf txn/s", weighted_avg_throughput); - LOG_TRACE("Current throughput: %lf txn/s", throughput_); - - // Write the stats to metric tables - UpdateMetrics(); - - if (interval_cnt % STATS_LOG_INTERVALS == 0) { - try { - ofs_ << "At interval: " << interval_cnt << std::endl; - ofs_ << aggregated_stats_.ToString(); - ofs_ << "Weighted avg. throughput=" << weighted_avg_throughput - << std::endl; - ofs_ << "Average throughput=" << avg_throughput_ << std::endl; - ofs_ << "Current throughput=" << throughput_; - } catch (std::ofstream::failure &e) { - LOG_ERROR("Error when writing to the stats log file %s", e.what()); - } - } -} - -void StatsAggregator::UpdateQueryMetrics(int64_t time_stamp, - concurrency::TransactionContext *txn) { - // Get the target query metrics table - LOG_TRACE("Inserting Query Metric Tuples"); - std::shared_ptr query_metric; - auto &completed_query_metrics = aggregated_stats_.GetCompletedQueryMetrics(); - while (completed_query_metrics.Dequeue(query_metric)) { - // Get physical stats - auto table_access = query_metric->GetQueryAccess(); - auto reads = table_access.GetReads(); - auto updates = table_access.GetUpdates(); - auto deletes = table_access.GetDeletes(); - auto inserts = table_access.GetInserts(); - auto latency = query_metric->GetQueryLatency().GetFirstLatencyValue(); - auto cpu_system = query_metric->GetProcessorMetric().GetSystemDuration(); - auto cpu_user = query_metric->GetProcessorMetric().GetUserDuration(); - - // Get query params - auto query_params = query_metric->GetQueryParams(); - auto num_params = 0; - QueryMetric::QueryParamBuf value_buf; - QueryMetric::QueryParamBuf type_buf; - QueryMetric::QueryParamBuf format_buf; - - if (query_params != nullptr) { - value_buf = query_params->val_buf_copy; - num_params = query_params->num_params; - format_buf = query_params->format_buf_copy; - type_buf = query_params->type_buf_copy; - PELOTON_ASSERT(num_params > 0); - } - - // Generate and insert the tuple - catalog::Catalog::GetInstance() - ->GetSystemCatalogs(query_metric->GetDatabaseId()) - ->GetQueryMetricsCatalog() - ->InsertQueryMetrics( - query_metric->GetName(), query_metric->GetDatabaseId(), num_params, - type_buf, format_buf, value_buf, reads, updates, deletes, inserts, - (int64_t)latency, (int64_t)(cpu_system + cpu_user), time_stamp, - pool_.get(), txn); - - LOG_TRACE("Query Metric Tuple inserted"); - } -} - -void StatsAggregator::UpdateMetrics() { - // All tuples are inserted in a single txn - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto txn = txn_manager.BeginTransaction(); - - // Get the target table metrics table - LOG_TRACE("Inserting stat tuples into catalog database.."); - auto storage_manager = storage::StorageManager::GetInstance(); - - auto time_since_epoch = std::chrono::system_clock::now().time_since_epoch(); - auto time_stamp = - std::chrono::duration_cast(time_since_epoch) - .count(); - - auto database_count = storage_manager->GetDatabaseCount(); - for (oid_t database_offset = 0; database_offset < database_count; - database_offset++) { - auto database = storage_manager->GetDatabaseWithOffset(database_offset); - - // Update database metrics table - if (!database) continue; - oid_t database_oid = database->GetOid(); - std::string database_name; - try { - auto database_object = - catalog::Catalog::GetInstance()->GetDatabaseObject(database_oid, txn); - database_name = database_object->GetDatabaseName(); - } catch (CatalogException &e) { - continue; - } - - LOG_TRACE("Updating metrics for database: %s", database_name.c_str()); - - auto database_metric = aggregated_stats_.GetDatabaseMetric(database_oid); - auto txn_committed = database_metric->GetTxnCommitted().GetCounter(); - auto txn_aborted = database_metric->GetTxnAborted().GetCounter(); - - catalog::DatabaseMetricsCatalog::GetInstance()->InsertDatabaseMetrics( - database_oid, txn_committed, txn_aborted, time_stamp, pool_.get(), txn); - LOG_TRACE("DB Metric Tuple inserted"); - - // Update all the indices of this database - UpdateTableMetrics(database, time_stamp, txn); - } - - // Update all query metrics - UpdateQueryMetrics(time_stamp, txn); - - txn_manager.CommitTransaction(txn); -} - -void StatsAggregator::UpdateTableMetrics(storage::Database *database, - int64_t time_stamp, - concurrency::TransactionContext *txn) { - // Update table metrics table for each of the indices - auto database_oid = database->GetOid(); - auto table_count = database->GetTableCount(); - for (oid_t table_offset = 0; table_offset < table_count; table_offset++) { - auto table = database->GetTable(table_offset); - auto table_oid = table->GetOid(); - auto table_metrics = - aggregated_stats_.GetTableMetric(database_oid, table_oid); - auto table_access = table_metrics->GetTableAccess(); - auto reads = table_access.GetReads(); - auto updates = table_access.GetUpdates(); - auto deletes = table_access.GetDeletes(); - auto inserts = table_access.GetInserts(); - // insert record into table metrics catalog - auto table_metrics_catalog = catalog::Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) - ->GetTableMetricsCatalog(); - table_metrics_catalog->InsertTableMetrics(table_oid, reads, updates, - deletes, inserts, time_stamp, - pool_.get(), txn); - LOG_TRACE("Table Metric Tuple inserted"); - - UpdateIndexMetrics(database, table, time_stamp, txn); - } -} - -void StatsAggregator::UpdateIndexMetrics(storage::Database *database, - storage::DataTable *table, - int64_t time_stamp, - concurrency::TransactionContext *txn) { - // Update index metrics table for each of the indices - auto database_oid = database->GetOid(); - auto table_oid = table->GetOid(); - auto index_count = table->GetIndexCount(); - for (oid_t index_offset = 0; index_offset < index_count; index_offset++) { - auto index = table->GetIndex(index_offset); - if (index == nullptr) continue; - auto index_oid = index->GetOid(); - auto index_metric = - aggregated_stats_.GetIndexMetric(database_oid, table_oid, index_oid); - - auto index_access = index_metric->GetIndexAccess(); - auto reads = index_access.GetReads(); - auto deletes = index_access.GetDeletes(); - auto inserts = index_access.GetInserts(); - // insert record into index metrics catalog - auto index_metrics_catalog = catalog::Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) - ->GetIndexMetricsCatalog(); - index_metrics_catalog->InsertIndexMetrics(table_oid, index_oid, reads, - deletes, inserts, time_stamp, - pool_.get(), txn); - } -} - -void StatsAggregator::RunAggregator() { - LOG_DEBUG("Aggregator is now running."); - std::mutex mtx; - std::unique_lock lck(mtx); - int64_t interval_cnt = 0; - double alpha = 0.4; - double weighted_avg_throughput = 0.0; - +void StatsAggregator::RunTask() { + LOG_INFO("Aggregator is now running."); + std::unique_lock lock(mutex_); while (exec_finished_.wait_for( - lck, std::chrono::milliseconds(aggregation_interval_ms_)) == + lock, std::chrono::milliseconds(aggregation_interval_ms_)) == std::cv_status::timeout && - is_aggregating_) { - Aggregate(interval_cnt, alpha, weighted_avg_throughput); - } - LOG_DEBUG("Aggregator done!"); -} - -StatsAggregator &StatsAggregator::GetInstance(int64_t aggregation_interval_ms) { - static StatsAggregator stats_aggregator(aggregation_interval_ms); - return stats_aggregator; -} - -//===--------------------------------------------------------------------===// -// HELPER FUNCTIONS -//===--------------------------------------------------------------------===// - -// Register the BackendStatsContext of a worker thread to global Stats -// Aggregator -void StatsAggregator::RegisterContext(std::thread::id id_, - BackendStatsContext *context_) { - { - std::lock_guard lock(stats_mutex_); - - PELOTON_ASSERT(backend_stats_.find(id_) == backend_stats_.end()); - - thread_number_++; - backend_stats_[id_] = context_; - LOG_DEBUG("Stats aggregator hash map size: %ld", backend_stats_.size()); - } -} - -// Unregister a BackendStatsContext. Currently we directly reuse the thread id -// instead of explicitly unregistering it. -void StatsAggregator::UnregisterContext(std::thread::id id) { - { - std::lock_guard lock(stats_mutex_); - - if (backend_stats_.find(id) != backend_stats_.end()) { - stats_history_.Aggregate(*backend_stats_[id]); - backend_stats_.erase(id); - thread_number_--; - } else { - LOG_DEBUG("stats_context already deleted!"); - } + !exiting_) + Aggregate(); + exiting_ = false; + exec_finished_.notify_all(); + LOG_INFO("Aggregator done!"); +} + +using RawDataCollect = std::vector>; +RawDataCollect StatsAggregator::AggregateRawData() { + RawDataCollect acc; + for (auto &entry : ThreadLevelStatsCollector::GetAllCollectors()) { + auto data_block = entry.second.GetDataToAggregate(); + if (acc.empty()) + acc = data_block; + else + for (size_t i = 0; i < data_block.size(); i++) { + acc[i]->Aggregate(*data_block[i]); + } + }; + return acc; +} + +void StatsAggregator::Aggregate() { + auto acc = AggregateRawData(); + for (auto &raw_data : acc) { + raw_data->UpdateAndPersist(); } } diff --git a/src/statistics/table_metric.cpp b/src/statistics/table_metric.cpp index f5699c86a67..f443c168b52 100644 --- a/src/statistics/table_metric.cpp +++ b/src/statistics/table_metric.cpp @@ -6,34 +6,149 @@ // // Identification: src/statistics/table_metric.cpp // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// +#include "catalog/table_metrics_catalog.h" +#include "catalog/catalog.h" +#include "catalog/system_catalogs.h" +#include "concurrency/transaction_manager_factory.h" #include "statistics/table_metric.h" #include "storage/data_table.h" #include "storage/storage_manager.h" +#include "storage/tile.h" namespace peloton { namespace stats { -TableMetric::TableMetric(MetricType type, oid_t database_id, oid_t table_id) - : AbstractMetric(type), database_id_(database_id), table_id_(table_id) { - try { - auto table = storage::StorageManager::GetInstance()->GetTableWithOid( - database_id, table_id); - table_name_ = table->GetName(); - for (auto& ch : table_name_) ch = toupper(ch); - } catch (CatalogException& e) { - table_name_ = ""; +const std::vector + TableMetricRawData::COUNTER_TYPES = { + TableMetricRawData::CounterType::READ, + TableMetricRawData::CounterType::UPDATE, + TableMetricRawData::CounterType::INSERT, + TableMetricRawData::CounterType::DELETE, + TableMetricRawData::CounterType::INLINE_MEMORY_ALLOC, + TableMetricRawData::CounterType::INLINE_MEMORY_USAGE, + TableMetricRawData::CounterType::VARLEN_MEMORY_ALLOC, + TableMetricRawData::CounterType::VARLEN_MEMORY_USAGE}; + +void TableMetricRawData::Aggregate(AbstractRawData &other) { + auto &other_table_data = dynamic_cast(other); + // Collect counters + for (auto &entry : other_table_data.counters_) { + if (counters_.find(entry.first) == counters_.end()) + counters_[entry.first] = std::vector(NUM_COUNTERS); + + auto &this_counter = counters_[entry.first]; + auto &other_counter = entry.second; + for (size_t i = 0; i < NUM_COUNTERS; i++) { + this_counter[i] += other_counter[i]; + } + } + + // Collect referenced TileGroups + for (auto &tile_groups : other_table_data.modified_tile_group_id_set_) { + if (modified_tile_group_id_set_.find(tile_groups.first) == + modified_tile_group_id_set_.end()) + modified_tile_group_id_set_[tile_groups.first] = + std::unordered_set(); + + auto &this_set = modified_tile_group_id_set_[tile_groups.first]; + auto &other_set = tile_groups.second; + for (auto tile_group_id : other_set) { + this_set.insert(tile_group_id); + } + } +} + +void TableMetricRawData::FetchMemoryStats() { + auto &tile_group_manager = catalog::Manager::GetInstance(); + auto pg_catalog = catalog::Catalog::GetInstance(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto storage_manager = storage::StorageManager::GetInstance(); + for (auto &entry : modified_tile_group_id_set_) { + auto &db_table_id = entry.first; + auto &tile_group_ids = entry.second; + + // Begin a txn to avoid concurrency issue (i.e. Other people delete the + // table) + auto txn = txn_manager.BeginTransaction(); + try { + auto tb_object = pg_catalog->GetTableObject(db_table_id.first, + db_table_id.second, txn); + } catch (CatalogException &e) { + txn_manager.CommitTransaction(txn); + continue; + } + size_t inline_tuple_size = + storage_manager->GetTableWithOid(db_table_id.first, db_table_id.second) + ->GetSchema() + ->GetLength(); + txn_manager.CommitTransaction(txn); + + for (oid_t tile_group_id : tile_group_ids) { + auto tile_group = tile_group_manager.GetTileGroup(tile_group_id); + if (tile_group == nullptr) continue; + + // Collect inline table + counters_[db_table_id][INLINE_MEMORY_USAGE] += + tile_group->GetActiveTupleCount() * + (inline_tuple_size + storage::TileGroupHeader::header_entry_size); + + // Colelct Varlen Memory stats + for (size_t i = 0; i < tile_group->NumTiles(); i++) { + counters_[db_table_id][VARLEN_MEMORY_ALLOC] += + tile_group->GetTile(i)->GetPool()->GetMemoryAlloc(); + counters_[db_table_id][VARLEN_MEMORY_USAGE] += + tile_group->GetTile(i)->GetPool()->GetMemoryUsage(); + } + } } } -void TableMetric::Aggregate(AbstractMetric& source) { - assert(source.GetType() == MetricType::TABLE); +void TableMetricRawData::UpdateAndPersist() { + FetchMemoryStats(); + + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + auto time_since_epoch = std::chrono::system_clock::now().time_since_epoch(); + auto time_stamp = std::chrono::duration_cast( + time_since_epoch).count(); + + for (auto &entry : counters_) { + // one iteration per (database, table) pair + oid_t database_oid = entry.first.first; + oid_t table_oid = entry.first.second; + auto &counts = entry.second; + + // try and fetch existing data from catalog + auto system_catalogs = + catalog::Catalog::GetInstance()->GetSystemCatalogs(database_oid); + auto table_metrics_catalog = system_catalogs->GetTableMetricsCatalog(); + auto old_metric = + table_metrics_catalog->GetTableMetricsObject(table_oid, txn); + if (old_metric == nullptr) { + // no entry exists for this table yet + table_metrics_catalog->InsertTableMetrics( + table_oid, counts[READ], counts[UPDATE], counts[INSERT], + counts[DELETE], counts[INLINE_MEMORY_ALLOC], + counts[INLINE_MEMORY_USAGE], counts[VARLEN_MEMORY_ALLOC], + counts[VARLEN_MEMORY_USAGE], time_stamp, nullptr, txn); + } else { + // update existing entry + table_metrics_catalog->UpdateTableMetrics( + table_oid, old_metric->GetReads() + counts[READ], + old_metric->GetUpdates() + counts[UPDATE], + old_metric->GetInserts() + counts[INSERT], + old_metric->GetDeletes() + counts[DELETE], + old_metric->GetInlineMemoryAlloc() + counts[INLINE_MEMORY_ALLOC], + counts[INLINE_MEMORY_USAGE], counts[VARLEN_MEMORY_ALLOC], + counts[VARLEN_MEMORY_USAGE], time_stamp, txn); + } + } - TableMetric& table_metric = static_cast(source); - table_access_.Aggregate(table_metric.GetTableAccess()); + txn_manager.CommitTransaction(txn); } } // namespace stats diff --git a/src/statistics/thread_level_stats_collector.cpp b/src/statistics/thread_level_stats_collector.cpp new file mode 100644 index 00000000000..230189d8c9d --- /dev/null +++ b/src/statistics/thread_level_stats_collector.cpp @@ -0,0 +1,65 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// thread_level_stats_collector.cpp +// +// Identification: src/statistics/thread_level_stats_collector.cpp +// +// Copyright (c) 2017-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// +#include "statistics/thread_level_stats_collector.h" +#include "statistics/database_metric.h" +#include "statistics/index_metric.h" +#include "statistics/stats_event_type.h" +#include "statistics/table_metric.h" +#include "statistics/test_metric.h" +#include "statistics/tuple_access_metric.h" + +namespace peloton { +namespace stats { +using CollectorsMap = +tbb::concurrent_unordered_map>; + +CollectorsMap ThreadLevelStatsCollector::collector_map_ = CollectorsMap(); + +ThreadLevelStatsCollector::ThreadLevelStatsCollector() { + // TODO(tianyu): Write stats to register here + auto + stats_mode = static_cast(settings::SettingsManager::GetInt( + settings::SettingId::stats_mode)); + if (stats_mode == StatsModeType::ENABLE) { + RegisterMetric( + {StatsEventType::TUPLE_READ, StatsEventType::TUPLE_UPDATE, + StatsEventType::TUPLE_INSERT, StatsEventType::TUPLE_DELETE, + StatsEventType::TABLE_MEMORY_ALLOC, + StatsEventType::TABLE_MEMORY_FREE}); + RegisterMetric( + {StatsEventType::INDEX_READ, StatsEventType::INDEX_UPDATE, + StatsEventType::INDEX_INSERT, StatsEventType::INDEX_DELETE}); + + RegisterMetric({StatsEventType::TXN_BEGIN, + StatsEventType::TXN_COMMIT, + StatsEventType::TXN_ABORT}); + RegisterMetric({StatsEventType::TXN_BEGIN, + StatsEventType::TXN_ABORT, + StatsEventType::TXN_COMMIT, + StatsEventType::TUPLE_READ}); + } else if (stats_mode == StatsModeType::TEST) + RegisterMetric({StatsEventType::TEST}); +} + +ThreadLevelStatsCollector::~ThreadLevelStatsCollector() { + metrics_.clear(); + metric_dispatch_.clear(); +} + +std::vector> ThreadLevelStatsCollector::GetDataToAggregate() { + std::vector> result; + for (auto &metric : metrics_) result.push_back(metric->Swap()); + return result; +} +} // namespace stats +} // namespace peloton diff --git a/src/statistics/tuple_access_metric.cpp b/src/statistics/tuple_access_metric.cpp new file mode 100644 index 00000000000..867467cccfb --- /dev/null +++ b/src/statistics/tuple_access_metric.cpp @@ -0,0 +1,58 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// tuple_access_metric.cpp +// +// Identification: src/statistics/tuple_access_metric.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "catalog/catalog.h" +#include "concurrency/transaction_manager_factory.h" +#include "statistics/tuple_access_metric.h" +#include "catalog/tuple_access_metrics_catalog.h" + +namespace peloton { +namespace stats { +void TupleAccessRawData::WriteToCatalog(txn_id_t tid, + bool complete, + bool commit, + concurrency::TransactionContext *txn) { + auto catalog = catalog::TupleAccessMetricsCatalog::GetInstance(txn); + auto old = catalog->GetTupleAccessMetricsCatalogObject(tid, txn); + if (old == nullptr) + catalog->InsertAccessMetric(tid, + tuple_access_counters_[tid], + complete, + commit, + nullptr, + txn); + else + catalog->UpdateAccessMetrics(tid, + old->GetReads() + tuple_access_counters_[tid], + complete, + commit, + txn); +} + +void TupleAccessRawData::UpdateAndPersist() { + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + + for (auto &entry : tuple_access_counters_) { + auto tid = entry.first; + if (!(commits_.find(tid) == commits_.end())) + WriteToCatalog(tid, true, true, txn); + else if (!(aborts_.find(tid) == aborts_.end())) + WriteToCatalog(tid, true, false, txn); + else + WriteToCatalog(tid, false, false, txn); + } + txn_manager.CommitTransaction(txn); +} + +} // namespace stats +} // namespace peloton \ No newline at end of file diff --git a/src/storage/tile.cpp b/src/storage/tile.cpp index 7e30ff197ef..a5fcf1218d2 100644 --- a/src/storage/tile.cpp +++ b/src/storage/tile.cpp @@ -15,16 +15,16 @@ #include "catalog/schema.h" #include "common/exception.h" -#include "common/macros.h" -#include "type/serializer.h" #include "common/internal_types.h" -#include "type/ephemeral_pool.h" +#include "common/macros.h" #include "concurrency/transaction_manager_factory.h" #include "storage/backend_manager.h" #include "storage/tile.h" #include "storage/tile_group_header.h" #include "storage/tuple.h" #include "storage/tuple_iterator.h" +#include "type/ephemeral_pool.h" +#include "type/serializer.h" namespace peloton { namespace storage { @@ -70,6 +70,9 @@ Tile::Tile(BackendType backend_type, TileGroupHeader *tile_header, } Tile::~Tile() { + // Record memory deallocation + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectTableMemoryFree(database_id, table_id, this->tile_size); // reclaim the tile memory (INLINED data) // auto &storage_manager = storage::StorageManager::GetInstance(); // storage_manager.Release(backend_type, data); @@ -198,7 +201,7 @@ Tile *Tile::CopyTile(BackendType backend_type) { new_header, *schema, tile_group, allocated_tuple_count); PELOTON_MEMCPY(static_cast(new_tile->data), static_cast(data), - tile_size); + tile_size); // Do a deep copy if some column is uninlined, so that // the values in that column point to the new pool @@ -350,8 +353,9 @@ bool Tile::SerializeHeaderTo(SerializeOutput &output) { // Cache the column header column_header = new char[column_header_size]; - PELOTON_MEMCPY(column_header, static_cast(output.Data()) + start, - column_header_size); + PELOTON_MEMCPY(column_header, + static_cast(output.Data()) + start, + column_header_size); return true; } @@ -434,7 +438,8 @@ void Tile::DeserializeTuplesFrom(SerializeInput &input, for (oid_t column_itr = 0; column_itr < column_count; column_itr++) { message << "column " << column_itr << ": " << names[column_itr] - << ", type = " << static_cast(types[column_itr]) << std::endl; + << ", type = " << static_cast(types[column_itr]) + << std::endl; } throw SerializationException(message.str()); diff --git a/src/storage/tile_group.cpp b/src/storage/tile_group.cpp index 5a40f4ad5fd..72d5110db8d 100644 --- a/src/storage/tile_group.cpp +++ b/src/storage/tile_group.cpp @@ -31,10 +31,11 @@ namespace storage { TileGroup::TileGroup(BackendType backend_type, TileGroupHeader *tile_group_header, AbstractTable *table, const std::vector &schemas, - const column_map_type &column_map, int tuple_count) - : database_id(INVALID_OID), - table_id(INVALID_OID), - tile_group_id(INVALID_OID), + const column_map_type &column_map, int tuple_count, + oid_t database_id, oid_t table_id, oid_t tile_group_id) + : database_id(database_id), + table_id(table_id), + tile_group_id(tile_group_id), backend_type(backend_type), tile_schemas(schemas), tile_group_header(tile_group_header), @@ -57,8 +58,11 @@ TileGroup::TileGroup(BackendType backend_type, } TileGroup::~TileGroup() { + // Record memory deallocation for tile group header + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectTableMemoryFree(database_id, table_id, + tile_group_header->GetHeaderSize()); // Drop references on all tiles - // clean up tile group header delete tile_group_header; } @@ -78,9 +82,7 @@ type::AbstractPool *TileGroup::GetTilePool(const oid_t tile_id) const { return nullptr; } -oid_t TileGroup::GetTileGroupId() const { - return tile_group_id; -} +oid_t TileGroup::GetTileGroupId() const { return tile_group_id; } // TODO: check when this function is called. --Yingjun oid_t TileGroup::GetNextTupleSlot() const { @@ -93,7 +95,6 @@ oid_t TileGroup::GetActiveTupleCount() const { return tile_group_header->GetActiveTupleCount(); } - //===--------------------------------------------------------------------===// // Operations //===--------------------------------------------------------------------===// @@ -158,7 +159,7 @@ oid_t TileGroup::InsertTuple(const Tuple *tuple) { // Set MVCC info PELOTON_ASSERT(tile_group_header->GetTransactionId(tuple_slot_id) == - INVALID_TXN_ID); + INVALID_TXN_ID); PELOTON_ASSERT(tile_group_header->GetBeginCommitId(tuple_slot_id) == MAX_CID); PELOTON_ASSERT(tile_group_header->GetEndCommitId(tuple_slot_id) == MAX_CID); return tuple_slot_id; @@ -338,15 +339,13 @@ type::Value TileGroup::GetValue(oid_t tuple_id, oid_t column_id) { return GetTile(tile_offset)->GetValue(tuple_id, tile_column_id); } -void TileGroup::SetValue(type::Value &value, oid_t tuple_id, - oid_t column_id) { +void TileGroup::SetValue(type::Value &value, oid_t tuple_id, oid_t column_id) { PELOTON_ASSERT(tuple_id < GetNextTupleSlot()); oid_t tile_column_id, tile_offset; LocateTileAndColumn(column_id, tile_offset, tile_column_id); GetTile(tile_offset)->SetValue(value, tuple_id, tile_column_id); } - std::shared_ptr TileGroup::GetTileReference( const oid_t tile_offset) const { PELOTON_ASSERT(tile_offset < tile_count); @@ -396,12 +395,11 @@ const std::string TileGroup::GetInfo() const { for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { Tile *tile = GetTile(tile_itr); if (tile != nullptr) { - os << std::endl << (*tile); + os << std::endl + << (*tile); } } - // auto header = GetHeader(); - // if (header != nullptr) os << (*header); return peloton::StringUtil::Prefix(peloton::StringBoxUtil::Box(os.str()), GETINFO_SPACER); } diff --git a/src/storage/tile_group_factory.cpp b/src/storage/tile_group_factory.cpp index f050d58e6df..01992950d50 100644 --- a/src/storage/tile_group_factory.cpp +++ b/src/storage/tile_group_factory.cpp @@ -11,7 +11,8 @@ //===----------------------------------------------------------------------===// #include "storage/tile_group_factory.h" -// #include "logging/logging_util.h" +#include "settings/settings_manager.h" +#include "statistics/thread_level_stats_collector.h" #include "storage/tile_group_header.h" //===--------------------------------------------------------------------===// @@ -30,17 +31,20 @@ TileGroup *TileGroupFactory::GetTileGroup( const column_map_type &column_map, int tuple_count) { // Allocate the data on appropriate backend BackendType backend_type = BackendType::MM; - // logging::LoggingUtil::GetBackendType(peloton_logging_mode); + // logging::LoggingUtil::GetBackendType(peloton_logging_mode); TileGroupHeader *tile_header = new TileGroupHeader(backend_type, tuple_count); - TileGroup *tile_group = new TileGroup(backend_type, tile_header, table, - schemas, column_map, tuple_count); - tile_header->SetTileGroup(tile_group); + // Record memory allocation for tile group header + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectTableMemoryAlloc(database_id, table_id, + tile_header->GetHeaderSize()); + + TileGroup *tile_group = + new TileGroup(backend_type, tile_header, table, schemas, column_map, + tuple_count, database_id, table_id, tile_group_id); - tile_group->database_id = database_id; - tile_group->tile_group_id = tile_group_id; - tile_group->table_id = table_id; + tile_header->SetTileGroup(tile_group); return tile_group; } diff --git a/src/storage/tile_group_header.cpp b/src/storage/tile_group_header.cpp index 1e0b450144e..7ca92d30f76 100644 --- a/src/storage/tile_group_header.cpp +++ b/src/storage/tile_group_header.cpp @@ -239,7 +239,8 @@ oid_t TileGroupHeader::GetActiveTupleCount() const { tuple_slot_id++) { txn_id_t tuple_txn_id = GetTransactionId(tuple_slot_id); if (tuple_txn_id != INVALID_TXN_ID) { - PELOTON_ASSERT(tuple_txn_id == INITIAL_TXN_ID); + // TODO(Tianyu) Why the fuck is this thing here to begin with? +// PELOTON_ASSERT(tuple_txn_id == INITIAL_TXN_ID); active_tuple_slots++; } } diff --git a/src/traffic_cop/traffic_cop.cpp b/src/traffic_cop/traffic_cop.cpp index a87d99c0ac5..9c08629758c 100644 --- a/src/traffic_cop/traffic_cop.cpp +++ b/src/traffic_cop/traffic_cop.cpp @@ -22,6 +22,7 @@ #include "optimizer/optimizer.h" #include "planner/plan_util.h" #include "settings/settings_manager.h" +#include "statistics/thread_level_stats_collector.h" #include "threadpool/mono_queue_pool.h" namespace peloton { @@ -87,6 +88,8 @@ ResultType TrafficCop::BeginQueryHelper(size_t thread_id) { LOG_DEBUG("Begin txn failed"); return ResultType::FAILURE; } + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectTransactionBegin( + txn); // initialize the current result as success tcop_txn_state_.emplace(txn, ResultType::SUCCESS); } @@ -134,6 +137,9 @@ ResultType TrafficCop::AbortQueryHelper() { ResultType TrafficCop::ExecuteStatementGetResult() { LOG_TRACE("Statement executed. Result: %s", ResultTypeToString(p_status_.m_result).c_str()); + + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectQueryEnd(); + setRowsAffected(p_status_.m_processed); LOG_TRACE("rows_changed %d", p_status_.m_processed); is_queuing_ = false; @@ -166,6 +172,8 @@ executor::ExecutionResult TrafficCop::ExecuteHelper( single_statement_txn_ = true; txn = txn_manager.BeginTransaction(thread_id); tcop_txn_state_.emplace(txn, ResultType::SUCCESS); + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectTransactionBegin( + txn); } // skip if already aborted @@ -190,6 +198,10 @@ executor::ExecutionResult TrafficCop::ExecuteHelper( task_callback_(task_callback_arg_); }; + // start timer in tcop before submitting task to worker + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectQueryBegin(); + auto &pool = threadpool::MonoQueuePool::GetInstance(); pool.SubmitTask([plan, txn, ¶ms, &result_format, on_complete] { executor::PlanExecutor::ExecutePlan(plan, txn, params, result_format, @@ -301,11 +313,14 @@ std::shared_ptr TrafficCop::PrepareStatement( if (txn == nullptr) { LOG_TRACE("Begin txn failed"); } + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectTransactionBegin( + txn); // initialize the current result as success tcop_txn_state_.emplace(txn, ResultType::SUCCESS); } - if (settings::SettingsManager::GetBool(settings::SettingId::brain)) { + if (settings::SettingsManager::GetBool( + settings::SettingId::brain_data_collection)) { tcop_txn_state_.top().first->AddQueryString(query_string.c_str()); } @@ -377,6 +392,8 @@ bool TrafficCop::BindParamsForCachePlan( if (txn == nullptr) { LOG_ERROR("Begin txn failed"); } + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectTransactionBegin( + txn); // initialize the current result as success tcop_txn_state_.emplace(txn, ResultType::SUCCESS); } @@ -543,19 +560,12 @@ FieldInfo TrafficCop::GetColumnFieldForValueType(std::string column_name, field_size); } +// TODO(Tianyi) Further simplify this API ResultType TrafficCop::ExecuteStatement( const std::shared_ptr &statement, const std::vector ¶ms, UNUSED_ATTRIBUTE bool unnamed, - std::shared_ptr param_stats, const std::vector &result_format, std::vector &result, size_t thread_id) { - // TODO(Tianyi) Further simplify this API - if (static_cast(settings::SettingsManager::GetInt( - settings::SettingId::stats_mode)) != StatsType::INVALID) { - stats::BackendStatsContext::GetInstance()->InitQueryMetric( - statement, std::move(param_stats)); - } - LOG_TRACE("Execute Statement of name: %s", statement->GetStatementName().c_str()); LOG_TRACE("Execute Statement of query: %s", diff --git a/test/binder/binder_test.cpp b/test/binder/binder_test.cpp index b82df1ec72a..9e10a0e4de4 100644 --- a/test/binder/binder_test.cpp +++ b/test/binder/binder_test.cpp @@ -23,6 +23,7 @@ #include "optimizer/optimizer.h" #include "parser/postgresparser.h" #include "traffic_cop/traffic_cop.h" +#include "storage/database.h" #include "executor/testing_executor_util.h" #include "sql/testing_sql_util.h" diff --git a/test/brain/query_logger_test.cpp b/test/brain/query_logger_test.cpp index c3affa7fd40..89985c5d08e 100644 --- a/test/brain/query_logger_test.cpp +++ b/test/brain/query_logger_test.cpp @@ -22,7 +22,7 @@ namespace test { class QueryLoggerTests : public PelotonTest { protected: void SetUp() override { - settings::SettingsManager::SetBool(settings::SettingId::brain, true); + settings::SettingsManager::SetBool(settings::SettingId::brain_data_collection, true); PelotonInit::Initialize(); // query to check that logging is done @@ -33,7 +33,7 @@ class QueryLoggerTests : public PelotonTest { brain::QueryLogger::Fingerprint fingerprint{select_query_}; select_query_fingerprint_ = fingerprint.GetFingerprint(); - wait_time_ = 2; + wait_time_ = 5; } void TearDown() override { PelotonInit::Shutdown(); } diff --git a/test/catalog/catalog_test.cpp b/test/catalog/catalog_test.cpp index 3e6b5e15872..e8ad922f8e8 100644 --- a/test/catalog/catalog_test.cpp +++ b/test/catalog/catalog_test.cpp @@ -15,13 +15,13 @@ #include "catalog/database_catalog.h" #include "catalog/database_metrics_catalog.h" #include "catalog/index_catalog.h" -#include "catalog/query_metrics_catalog.h" #include "catalog/system_catalogs.h" #include "catalog/table_catalog.h" #include "common/harness.h" #include "common/logger.h" #include "concurrency/transaction_manager_factory.h" #include "storage/storage_manager.h" +#include "storage/database.h" #include "type/ephemeral_pool.h" #include "sql/testing_sql_util.h" @@ -90,27 +90,6 @@ TEST_F(CatalogTests, CreatingTable) { std::unique_ptr pool(new type::EphemeralPool()); catalog::DatabaseMetricsCatalog::GetInstance()->InsertDatabaseMetrics( 2, 3, 4, 5, pool.get(), txn); - - // inset meaningless tuple into QUERY_METRICS_CATALOG and check - stats::QueryMetric::QueryParamBuf param; - param.len = 1; - param.buf = (unsigned char *)pool->Allocate(1); - *param.buf = 'a'; - auto database_object = - catalog::Catalog::GetInstance()->GetDatabaseObject("emp_db", txn); - catalog::Catalog::GetInstance() - ->GetSystemCatalogs(database_object->GetDatabaseOid()) - ->GetQueryMetricsCatalog() - ->InsertQueryMetrics("a query", database_object->GetDatabaseOid(), 1, - param, param, param, 1, 1, 1, 1, 1, 1, 1, pool.get(), - txn); - auto param1 = catalog::Catalog::GetInstance() - ->GetSystemCatalogs(database_object->GetDatabaseOid()) - ->GetQueryMetricsCatalog() - ->GetParamTypes("a query", txn); - EXPECT_EQ(1, param1.len); - EXPECT_EQ('a', *param1.buf); - // check colum object EXPECT_EQ("name", catalog::Catalog::GetInstance() ->GetTableObject("emp_db", DEFUALT_SCHEMA_NAME, "department_table", txn) @@ -236,7 +215,7 @@ TEST_F(CatalogTests, DroppingTable) { auto catalog = catalog::Catalog::GetInstance(); // NOTE: everytime we create a database, there will be 8 catalog tables inside EXPECT_EQ( - 11, + 10, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); auto database_object = catalog::Catalog::GetInstance()->GetDatabaseObject("emp_db", txn); @@ -249,8 +228,7 @@ TEST_F(CatalogTests, DroppingTable) { EXPECT_NE(nullptr, database_object); auto department_table_object = database_object->GetTableObject("department_table", DEFUALT_SCHEMA_NAME); - EXPECT_EQ( - 10, + EXPECT_EQ(9, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); @@ -263,7 +241,7 @@ TEST_F(CatalogTests, DroppingTable) { CatalogException); // EXPECT_EQ( - 10, + 9, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); @@ -273,7 +251,7 @@ TEST_F(CatalogTests, DroppingTable) { "emp_db", DEFUALT_SCHEMA_NAME, "void_table", txn), CatalogException); EXPECT_EQ( - 10, + 9, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); @@ -282,7 +260,7 @@ TEST_F(CatalogTests, DroppingTable) { catalog::Catalog::GetInstance()->DropTable("emp_db", DEFUALT_SCHEMA_NAME, "emp_table", txn); EXPECT_EQ( - 9, + 8, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); } diff --git a/test/codegen/testing_codegen_util.cpp b/test/codegen/testing_codegen_util.cpp index fe8f6c26440..5ba8a91cc85 100644 --- a/test/codegen/testing_codegen_util.cpp +++ b/test/codegen/testing_codegen_util.cpp @@ -12,6 +12,8 @@ #include "codegen/testing_codegen_util.h" +#include + #include "catalog/table_catalog.h" #include "codegen/proxy/runtime_functions_proxy.h" #include "codegen/proxy/value_proxy.h" diff --git a/test/common/lock_free_array_test.cpp b/test/common/lock_free_array_test.cpp index c8b9f3cea68..59df5f44008 100644 --- a/test/common/lock_free_array_test.cpp +++ b/test/common/lock_free_array_test.cpp @@ -6,15 +6,15 @@ // // Identification: test/container/lock_free_array_test.cpp // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// - #include "common/container/lock_free_array.h" #include "common/harness.h" + namespace peloton { namespace test { @@ -26,14 +26,13 @@ class LockFreeArrayTests : public PelotonTest {}; // Test basic functionality TEST_F(LockFreeArrayTests, BasicTest) { - - typedef uint32_t value_type; + typedef uint32_t value_type; { LockFreeArray array; size_t const element_count = 3; - for (size_t element = 0; element < element_count; ++element ) { + for (size_t element = 0; element < element_count; ++element) { auto status = array.Append(element); EXPECT_TRUE(status); } @@ -41,19 +40,17 @@ TEST_F(LockFreeArrayTests, BasicTest) { auto array_size = array.GetSize(); EXPECT_EQ(array_size, element_count); } - } -//// Test shared pointers +// Test shared pointers TEST_F(LockFreeArrayTests, SharedPointerTest1) { - typedef std::shared_ptr value_type; { LockFreeArray array; size_t const element_count = 3; - for (size_t element = 0; element < element_count; ++element ) { + for (size_t element = 0; element < element_count; ++element) { std::shared_ptr entry(new oid_t); auto status = array.Append(entry); EXPECT_TRUE(status); @@ -62,29 +59,25 @@ TEST_F(LockFreeArrayTests, SharedPointerTest1) { auto array_size = array.GetSize(); EXPECT_EQ(array_size, element_count); } - } TEST_F(LockFreeArrayTests, SharedPointerTest2) { - typedef std::shared_ptr value_type; { LockFreeArray array; - std::thread t0([&] { size_t const element_count = 10000; - for (size_t element = 0; element < element_count; ++element ) { + for (size_t element = 0; element < element_count; ++element) { std::shared_ptr entry(new oid_t); auto status = array.Append(entry); EXPECT_TRUE(status); } }); - size_t const element_count = 10000; - for (size_t element = 0; element < element_count; ++element ) { + for (size_t element = 0; element < element_count; ++element) { std::shared_ptr entry(new oid_t); auto status = array.Append(entry); EXPECT_TRUE(status); @@ -92,12 +85,27 @@ TEST_F(LockFreeArrayTests, SharedPointerTest2) { t0.join(); auto array_size = array.GetSize(); - EXPECT_EQ(array_size, element_count*2); - + EXPECT_EQ(array_size, element_count * 2); + } +} +TEST_F(LockFreeArrayTests, IteratorTest) { + LockFreeArray array; + for (oid_t i = 0; i < 1000; i++) { + array.Append(i); + } + EXPECT_EQ(1000, array.GetSize()); + oid_t count = 0; + for (auto iter1 = array.Begin(), iter2 = array.Begin(); iter1 != array.End(); + ++iter1, ++iter2) { + EXPECT_EQ(count, *iter1); + EXPECT_EQ(count, *iter2); + EXPECT_TRUE(iter1 == iter2); + count++; } -} + EXPECT_EQ(1000, count); +} } // namespace test } // namespace peloton diff --git a/test/executor/create_index_test.cpp b/test/executor/create_index_test.cpp index 034eca553fa..bbefba9f5d8 100644 --- a/test/executor/create_index_test.cpp +++ b/test/executor/create_index_test.cpp @@ -32,6 +32,7 @@ #include "planner/insert_plan.h" #include "planner/plan_util.h" #include "planner/update_plan.h" +#include "storage/database.h" #include "traffic_cop/traffic_cop.h" #include "gtest/gtest.h" diff --git a/test/executor/drop_test.cpp b/test/executor/drop_test.cpp index 19e6c1293da..bad309b6438 100644 --- a/test/executor/drop_test.cpp +++ b/test/executor/drop_test.cpp @@ -25,6 +25,7 @@ #include "parser/postgresparser.h" #include "planner/drop_plan.h" #include "planner/plan_util.h" +#include "storage/database.h" namespace peloton { namespace test { @@ -106,19 +107,17 @@ TEST_F(DropTests, DroppingTable) { txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - // NOTE: everytime we create a database, there will be 8 catalog tables inside - EXPECT_EQ((int)catalog->GetDatabaseObject(TEST_DB_NAME, txn) - ->GetTableObjects() - .size(), - 10); + auto old_size = (int)catalog->GetDatabaseObject(TEST_DB_NAME, txn) + ->GetTableObjects() + .size(); // Now dropping the table using the executor catalog->DropTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); - EXPECT_EQ((int)catalog->GetDatabaseObject(TEST_DB_NAME, txn) - ->GetTableObjects() - .size(), - 9); + auto new_size = (int)catalog->GetDatabaseObject(TEST_DB_NAME, txn) + ->GetTableObjects() + .size(); + EXPECT_EQ(1, old_size - new_size); // free the database just created catalog->DropDatabaseWithName(TEST_DB_NAME, txn); @@ -214,7 +213,7 @@ TEST_F(DropTests, DroppingTrigger) { txn = txn_manager.BeginTransaction(); catalog->DropTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); - EXPECT_EQ(8, (int)catalog::Catalog::GetInstance() + EXPECT_EQ(7, (int)catalog::Catalog::GetInstance() ->GetDatabaseObject(TEST_DB_NAME, txn) ->GetTableObjects() .size()); diff --git a/test/executor/insert_test.cpp b/test/executor/insert_test.cpp index 4499eb6f53e..4ed90fb16db 100644 --- a/test/executor/insert_test.cpp +++ b/test/executor/insert_test.cpp @@ -22,6 +22,7 @@ #include "expression/constant_value_expression.h" #include "parser/insert_statement.h" #include "planner/insert_plan.h" +#include "storage/data_table.h" #include "type/value_factory.h" namespace peloton { diff --git a/test/executor/update_test.cpp b/test/executor/update_test.cpp index f8f3c12d0a2..041144b0010 100644 --- a/test/executor/update_test.cpp +++ b/test/executor/update_test.cpp @@ -44,6 +44,7 @@ #include "planner/plan_util.h" #include "planner/seq_scan_plan.h" #include "planner/update_plan.h" +#include "storage/database.h" #include "storage/data_table.h" #include "storage/tile_group_factory.h" #include "traffic_cop/traffic_cop.h" diff --git a/test/include/catalog/testing_constraints_util.h b/test/include/catalog/testing_constraints_util.h index 30a9b658223..f0bc24d478a 100644 --- a/test/include/catalog/testing_constraints_util.h +++ b/test/include/catalog/testing_constraints_util.h @@ -28,6 +28,18 @@ #include "concurrency/transaction_context.h" #include "concurrency/transaction_manager_factory.h" #include "executor/abstract_executor.h" +#include "executor/logical_tile.h" +#include "storage/tile_group.h" +#include "storage/tile_group_factory.h" +#include "storage/tuple.h" +#include "storage/database.h" +#include "storage/data_table.h" +#include "storage/table_factory.h" +#include "index/index_factory.h" +#include "planner/delete_plan.h" +#include "planner/insert_plan.h" +#include "planner/project_info.h" +#include "executor/executor_context.h" #include "executor/delete_executor.h" #include "executor/executor_context.h" #include "executor/insert_executor.h" diff --git a/test/include/statistics/testing_stats_util.h b/test/include/statistics/testing_stats_util.h index 2b0e5f31a80..3d82faf6566 100644 --- a/test/include/statistics/testing_stats_util.h +++ b/test/include/statistics/testing_stats_util.h @@ -6,65 +6,78 @@ // // Identification: tests/include/statistics/stats_tests_util.h // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// -#include "../concurrency/testing_transaction_util.h" -#include "common/harness.h" -#include "common/internal_types.h" -#include "type/value.h" -#include "type/value_factory.h" -#include "common/statement.h" +#pragma once + #include "catalog/catalog.h" #include "catalog/schema.h" #include "catalog/column.h" +#include "common/harness.h" +#include "common/internal_types.h" +#include "common/statement.h" +#include "../concurrency/testing_transaction_util.h" #include "concurrency/transaction_manager_factory.h" #include "concurrency/transaction_manager.h" #include "concurrency/transaction_context.h" +#include "executor/create_executor.h" +#include "executor/plan_executor.h" +#include "expression/comparison_expression.h" +#include "index/index.h" +#include "index/index_factory.h" +#include "parser/postgresparser.h" +#include "planner/insert_plan.h" +#include "storage/data_table.h" #include "storage/tile_group_factory.h" #include "storage/table_factory.h" #include "storage/tuple.h" #include "storage/tile.h" #include "storage/tile_group_header.h" -#include "index/index.h" -#include "index/index_factory.h" - -#include "expression/comparison_expression.h" -#include "planner/insert_plan.h" -#include "parser/postgresparser.h" - -#include "executor/plan_executor.h" -#include "storage/data_table.h" -#include "executor/create_executor.h" - -#pragma once +#include "type/value.h" +#include "type/value_factory.h" +#include "sql/testing_sql_util.h" namespace peloton { namespace test { class TestingStatsUtil { public: - static void ShowTable(std::string database_name, std::string table_name); - - static storage::Tuple PopulateTuple(const catalog::Schema *schema, - int first_col_val, int second_col_val, - int third_col_val, int fourth_col_val); - static void CreateTable(bool has_primary_key = true); - static std::shared_ptr GetQueryParams( - std::shared_ptr &type_buf, std::shared_ptr &format_buf, - std::shared_ptr &val_buf); - static std::shared_ptr GetInsertStmt(int id = 1, std::string val = "hello"); - static std::shared_ptr GetDeleteStmt(); + static void ParseAndPlan(Statement *statement, std::string sql); - static std::shared_ptr GetUpdateStmt(); + static int AggregateCounts(); - static void ParseAndPlan(Statement *statement, std::string sql); + static void Initialize(); + + static std::pair GetDbTableID(const std::string &table_name); + + static std::pair GetDbIndexID(const std::string &table_name); +}; + +class StatsWorkload { + public: + inline void DoQueries() { + for (auto &query : queries_) { + EXPECT_EQ(ResultType::SUCCESS, TestingSQLUtil::ExecuteSQLQuery(query)); + } + } + + inline void AddQuery(std::string query) { queries_.push_back(query); } + + inline StatsWorkload MakeCopy() { + StatsWorkload copy; + copy.queries_ = this->queries_; + return copy; + } + + private: + std::vector queries_; }; } // namespace test diff --git a/test/optimizer/old_optimizer_test.cpp b/test/optimizer/old_optimizer_test.cpp index 1069ab87909..fbf3a66a23c 100644 --- a/test/optimizer/old_optimizer_test.cpp +++ b/test/optimizer/old_optimizer_test.cpp @@ -21,6 +21,7 @@ #include "planner/plan_util.h" #include "planner/update_plan.h" #include "sql/testing_sql_util.h" +#include "storage/database.h" #include "traffic_cop/traffic_cop.h" namespace peloton { diff --git a/test/optimizer/optimizer_test.cpp b/test/optimizer/optimizer_test.cpp index dae410999d6..842658b1b70 100644 --- a/test/optimizer/optimizer_test.cpp +++ b/test/optimizer/optimizer_test.cpp @@ -36,6 +36,7 @@ #include "planner/update_plan.h" #include "sql/testing_sql_util.h" #include "traffic_cop/traffic_cop.h" +#include "storage/database.h" namespace peloton { namespace test { diff --git a/test/optimizer/selectivity_test.cpp b/test/optimizer/selectivity_test.cpp index 0f5e4a9ab8c..185af4dcdf4 100644 --- a/test/optimizer/selectivity_test.cpp +++ b/test/optimizer/selectivity_test.cpp @@ -24,6 +24,7 @@ #include "optimizer/stats/tuple_samples_storage.h" #include "optimizer/stats/value_condition.h" #include "sql/testing_sql_util.h" +#include "storage/database.h" #include "type/type.h" #include "type/value.h" #include "type/value_factory.h" diff --git a/test/planner/plan_util_test.cpp b/test/planner/plan_util_test.cpp index 8455e98efac..05855a807dc 100644 --- a/test/planner/plan_util_test.cpp +++ b/test/planner/plan_util_test.cpp @@ -20,6 +20,7 @@ #include "concurrency/transaction_manager_factory.h" #include "executor/testing_executor_util.h" #include "parser/postgresparser.h" +#include "storage/database.h" #include "storage/data_table.h" #include "planner/plan_util.h" diff --git a/test/sql/analyze_sql_test.cpp b/test/sql/analyze_sql_test.cpp index 16191ec000d..13b9fd9f8b9 100644 --- a/test/sql/analyze_sql_test.cpp +++ b/test/sql/analyze_sql_test.cpp @@ -20,6 +20,8 @@ #include "executor/create_executor.h" #include "optimizer/stats/stats_storage.h" #include "planner/create_plan.h" +#include "common/internal_types.h" +#include "storage/database.h" #include "sql/testing_sql_util.h" namespace peloton { diff --git a/test/sql/optimizer_sql_test.cpp b/test/sql/optimizer_sql_test.cpp index a81734c529d..ea88155591f 100644 --- a/test/sql/optimizer_sql_test.cpp +++ b/test/sql/optimizer_sql_test.cpp @@ -20,7 +20,9 @@ #include "planner/create_plan.h" #include "planner/order_by_plan.h" #include "sql/testing_sql_util.h" - +#include "storage/data_table.h" +using std::vector; +using std::unordered_set; using std::shared_ptr; using std::string; using std::unique_ptr; diff --git a/test/sql/testing_sql_util.cpp b/test/sql/testing_sql_util.cpp index 220fa558686..adf00ee4cda 100644 --- a/test/sql/testing_sql_util.cpp +++ b/test/sql/testing_sql_util.cpp @@ -81,7 +81,7 @@ ResultType TestingSQLUtil::ExecuteSQLQuery( // SetTrafficCopCounter(); counter_.store(1); auto status = traffic_cop_.ExecuteStatement(statement, param_values, unnamed, - nullptr, result_format, result); + result_format, result); if (traffic_cop_.GetQueuing()) { ContinueAfterComplete(); traffic_cop_.ExecuteStatementPlanGetResult(); @@ -182,7 +182,7 @@ ResultType TestingSQLUtil::ExecuteSQLQuery(const std::string query, // SetTrafficCopCounter(); counter_.store(1); auto status = traffic_cop_.ExecuteStatement(statement, param_values, unnamed, - nullptr, result_format, result); + result_format, result); if (traffic_cop_.GetQueuing()) { ContinueAfterComplete(); traffic_cop_.ExecuteStatementPlanGetResult(); @@ -221,7 +221,7 @@ ResultType TestingSQLUtil::ExecuteSQLQuery(const std::string query) { // SetTrafficCopCounter(); counter_.store(1); auto status = traffic_cop_.ExecuteStatement(statement, param_values, unnamed, - nullptr, result_format, result); + result_format, result); if (traffic_cop_.GetQueuing()) { ContinueAfterComplete(); traffic_cop_.ExecuteStatementPlanGetResult(); diff --git a/test/sql/update_sql_test.cpp b/test/sql/update_sql_test.cpp index f3584843918..9059ee2a35b 100644 --- a/test/sql/update_sql_test.cpp +++ b/test/sql/update_sql_test.cpp @@ -18,6 +18,7 @@ #include "executor/create_executor.h" #include "planner/create_plan.h" #include "sql/testing_sql_util.h" +#include "storage/data_table.h" namespace peloton { namespace test { diff --git a/test/statistics/database_metric_test.cpp b/test/statistics/database_metric_test.cpp new file mode 100644 index 00000000000..967026b9d6c --- /dev/null +++ b/test/statistics/database_metric_test.cpp @@ -0,0 +1,17 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// stats_tests_util.cpp +// +// Identification: tests/include/statistics/stats_tests_util.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +namespace peloton { +namespace test { + +} +} diff --git a/test/statistics/index_metric_test.cpp b/test/statistics/index_metric_test.cpp new file mode 100644 index 00000000000..74d5066c4a1 --- /dev/null +++ b/test/statistics/index_metric_test.cpp @@ -0,0 +1,93 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// stats_tests_util.cpp +// +// Identification: tests/include/statistics/index_metric_test.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include +#include "common/harness.h" +#include "statistics/index_metric.h" +#include "statistics/testing_stats_util.h" +#include "sql/testing_sql_util.h" +#include "catalog/index_metrics_catalog.h" +#include "concurrency/transaction_manager_factory.h" +#include "catalog/system_catalogs.h" +#include "storage/tile_group.h" +namespace peloton { +namespace test { + +class IndexMetricTests : public PelotonTest { + public: + static std::shared_ptr + GetIndexMetricObject(oid_t database_id, oid_t index_id) { + auto index_metrics_catalog = catalog::Catalog::GetInstance() + ->GetSystemCatalogs(database_id) + ->GetIndexMetricsCatalog(); + auto txn = concurrency::TransactionManagerFactory::GetInstance() + .BeginTransaction(); + auto index_metric_object = + index_metrics_catalog->GetIndexMetricsObject(index_id, txn); + concurrency::TransactionManagerFactory::GetInstance().CommitTransaction( + txn); + return index_metric_object; + } +}; + +TEST_F(IndexMetricTests, AccessTest) { + TestingStatsUtil::Initialize(); + + stats::StatsAggregator aggregator(1); + + EXPECT_EQ(ResultType::SUCCESS, + TestingSQLUtil::ExecuteSQLQuery( + "CREATE TABLE foo (ID INT PRIMARY KEY, YEAR INT);")); + auto db_index_ids = TestingStatsUtil::GetDbIndexID("foo"); + + aggregator.Aggregate(); + auto initial_index_metric = IndexMetricTests::GetIndexMetricObject( + db_index_ids.first, db_index_ids.second); + + // no catalog entry for this index metric until index first gets used + EXPECT_EQ(nullptr, initial_index_metric); + + size_t inserts = 0; + size_t reads = 0; + + StatsWorkload workload; + + // each insert also goes to index + workload.AddQuery("INSERT INTO foo VALUES (1, 2);"); + workload.AddQuery("INSERT INTO foo VALUES (2, 2);"); + workload.AddQuery("INSERT INTO foo VALUES (5, 2);"); + inserts += 3; + + // range scan -> no index use + workload.AddQuery("SELECT * FROM foo;"); + workload.AddQuery("SELECT * FROM foo;"); + + // do 3 index scans + workload.AddQuery("UPDATE foo SET year = 2018 WHERE id = 2"); + workload.AddQuery("UPDATE foo SET year = 2016 WHERE id = 1"); + workload.AddQuery("DELETE FROM foo WHERE id = 1"); + reads += 3; + + // not an index scan (year is not primary key) + workload.AddQuery("DELETE FROM foo WHERE year = 2018"); + + workload.DoQueries(); + aggregator.Aggregate(); + + auto final_index_metric = IndexMetricTests::GetIndexMetricObject( + db_index_ids.first, db_index_ids.second); + + EXPECT_EQ(inserts, final_index_metric->GetInserts()); + EXPECT_EQ(reads, final_index_metric->GetReads()); +} +} // namespace test +} // namespace peloton diff --git a/test/statistics/latency_metric_test.cpp b/test/statistics/latency_metric_test.cpp new file mode 100644 index 00000000000..967026b9d6c --- /dev/null +++ b/test/statistics/latency_metric_test.cpp @@ -0,0 +1,17 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// stats_tests_util.cpp +// +// Identification: tests/include/statistics/stats_tests_util.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +namespace peloton { +namespace test { + +} +} diff --git a/test/statistics/stats_framework_test.cpp b/test/statistics/stats_framework_test.cpp new file mode 100644 index 00000000000..a811b295bf6 --- /dev/null +++ b/test/statistics/stats_framework_test.cpp @@ -0,0 +1,126 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// stats_framework_test.cpp +// +// Identification: test/stats/stats_framework_test.cpp +// +// Copyright (c) 2016-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "common/harness.h" +#include "settings/settings_manager.h" +#include "statistics/stats_aggregator.h" +#include "statistics/testing_stats_util.h" + +namespace peloton { +namespace test { + +/** + * @brief Test the overall correctness of the stats framework + */ +class StatsFrameworkTests : public PelotonTest {}; + +/** + * @brief Single threaded test with few collection + */ +TEST_F(StatsFrameworkTests, BasicTest) { + settings::SettingsManager::SetInt(settings::SettingId::stats_mode, + static_cast(StatsModeType::TEST)); + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectTestNum(1); + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectTestNum(2); + + EXPECT_EQ(TestingStatsUtil::AggregateCounts(), 3); +}; + +/** + * @brief Single threaded test with a bulk collections + */ +TEST_F(StatsFrameworkTests, SingleThreadBulkTest) { + settings::SettingsManager::SetInt(settings::SettingId::stats_mode, + static_cast(StatsModeType::TEST)); + // Number of collection done in this test + const size_t trial = 10000; + // Aggregation is done once every aggr_step times of trial + const size_t aggr_step = 20; + int actual_sum = 0; + int aggreg_sum = 0; + for (size_t i = 0; i < trial; i++) { + int num = rand(); + stats::ThreadLevelStatsCollector::GetCollectorForThread().CollectTestNum( + num); + actual_sum += num; + + if (!(i % aggr_step)) { + aggreg_sum += TestingStatsUtil::AggregateCounts(); + ASSERT_EQ(actual_sum, aggreg_sum); + } + } + + aggreg_sum += TestingStatsUtil::AggregateCounts(); + ASSERT_EQ(actual_sum, aggreg_sum); +} + +/** + * @brief Multi threaded test running multiple collectors + */ +TEST_F(StatsFrameworkTests, MultiThreadTest) { + settings::SettingsManager::SetInt(settings::SettingId::stats_mode, + static_cast(StatsModeType::TEST)); + // Number of collector thread + const size_t num_of_collector = 10; + + // collecting interval in us + const size_t collect_interval = 1000; + + // aggregation interval in us + const size_t aggr_interval = 1000000; + + // Number of collection done by each collector + const size_t collect_tials = 5000; + + // Actual sum of collectors + std::atomic actual_sum(0); + + // Aggregated sum of aggregator + int aggreg_sum = 0; + + // Finish flag + std::atomic finish(false); + + // start the aggregator + std::thread aggregator([&] { + while (!finish) { + usleep(aggr_interval); + aggreg_sum += TestingStatsUtil::AggregateCounts(); + } + }); + + // Start the collectors; + std::vector collectors; + for (size_t i = 0; i < num_of_collector; i++) { + collectors.emplace_back([&]() { + for (size_t trial = 0; trial < collect_tials; trial++) { + int num = rand(); + stats::ThreadLevelStatsCollector::GetCollectorForThread() + .CollectTestNum(num); + actual_sum += num; + usleep(collect_interval); + } + }); + } + + for (auto &collector : collectors) { + collector.join(); + } + + finish = true; + + aggregator.join(); + + ASSERT_EQ(actual_sum, aggreg_sum); +} +} +} \ No newline at end of file diff --git a/test/statistics/stats_test.cpp b/test/statistics/stats_test.cpp deleted file mode 100644 index a53775e67a6..00000000000 --- a/test/statistics/stats_test.cpp +++ /dev/null @@ -1,451 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// stats_test.cpp -// -// Identification: test/statistics/stats_test.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include - -#include "common/harness.h" -#include "settings/settings_manager.h" - -#include -#include -#include "executor/testing_executor_util.h" -#include "statistics/testing_stats_util.h" - -#include "executor/executor_context.h" -#include "executor/insert_executor.h" -#include "statistics/backend_stats_context.h" -#include "statistics/stats_aggregator.h" -#include "traffic_cop/traffic_cop.h" - -#define NUM_ITERATION 50 -#define NUM_TABLE_INSERT 1 -#define NUM_TABLE_DELETE 2 -#define NUM_TABLE_UPDATE 3 -#define NUM_TABLE_READ 4 -#define NUM_INDEX_INSERT 1 -#define NUM_INDEX_DELETE 2 -#define NUM_INDEX_READ 4 -#define NUM_DB_COMMIT 1 -#define NUM_DB_ABORT 2 - -namespace peloton { -namespace test { - -class StatsTests : public PelotonTest {}; - -// Launch the aggregator thread manually -void LaunchAggregator(int64_t stat_interval) { - settings::SettingsManager::SetInt(settings::SettingId::stats_mode, - static_cast(StatsType::ENABLE)); - - auto &aggregator = - peloton::stats::StatsAggregator::GetInstance(stat_interval); - aggregator.GetAggregatedStats().ResetQueryCount(); - aggregator.ShutdownAggregator(); - aggregator.LaunchAggregator(); -} - -// Force a final aggregation -void ForceFinalAggregation(int64_t stat_interval) { - auto &aggregator = - peloton::stats::StatsAggregator::GetInstance(stat_interval); - int64_t interval_cnt = 0; - double alpha = 0; - double weighted_avg_throughput = 0; - aggregator.Aggregate(interval_cnt, alpha, weighted_avg_throughput); -} - -void TransactionTest(storage::Database *database, storage::DataTable *table, - UNUSED_ATTRIBUTE uint64_t thread_itr) { - uint64_t thread_id = TestingHarness::GetInstance().GetThreadId(); - auto tile_group_id = table->GetTileGroup(0)->GetTileGroupId(); - auto index_metadata = table->GetIndex(0)->GetMetadata(); - auto db_oid = database->GetOid(); - auto context = stats::BackendStatsContext::GetInstance(); - auto stmt = TestingStatsUtil::GetInsertStmt(); - - for (oid_t txn_itr = 1; txn_itr <= NUM_ITERATION; txn_itr++) { - context->InitQueryMetric(stmt, nullptr); - - if (thread_id % 2 == 0) { - std::chrono::microseconds sleep_time(1); - std::this_thread::sleep_for(sleep_time); - } - - // Record table stat - for (int i = 0; i < NUM_TABLE_READ; i++) { - context->IncrementTableReads(tile_group_id); - } - for (int i = 0; i < NUM_TABLE_UPDATE; i++) { - context->IncrementTableUpdates(tile_group_id); - } - for (int i = 0; i < NUM_TABLE_INSERT; i++) { - context->IncrementTableInserts(tile_group_id); - } - for (int i = 0; i < NUM_TABLE_DELETE; i++) { - context->IncrementTableDeletes(tile_group_id); - } - - // Record index stat - context->IncrementIndexReads(NUM_INDEX_READ, index_metadata); - context->IncrementIndexDeletes(NUM_INDEX_DELETE, index_metadata); - for (int i = 0; i < NUM_INDEX_INSERT; i++) { - context->IncrementIndexInserts(index_metadata); - } - - // Record database stat - for (int i = 0; i < NUM_DB_COMMIT; i++) { - context->GetOnGoingQueryMetric()->GetQueryLatency().RecordLatency(); - context->IncrementTxnCommitted(db_oid); - } - for (int i = 0; i < NUM_DB_ABORT; i++) { - context->IncrementTxnAborted(db_oid); - } - } -} - -TEST_F(StatsTests, MultiThreadStatsTest) { - auto catalog = catalog::Catalog::GetInstance(); - catalog->Bootstrap(); - - // Launch aggregator thread - int64_t aggregate_interval = 100; - LaunchAggregator(aggregate_interval); - auto &aggregator = stats::StatsAggregator::GetInstance(); - - // Create database, table and index - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto txn = txn_manager.BeginTransaction(); - auto id_column = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "dept_id", true); - catalog::Constraint constraint(ConstraintType::PRIMARY, "con_primary"); - id_column.AddConstraint(constraint); - auto name_column = - catalog::Column(type::TypeId::VARCHAR, 32, "dept_name", false); - - std::unique_ptr table_schema( - new catalog::Schema({id_column, name_column})); - catalog->CreateDatabase("emp_db", txn); - catalog::Catalog::GetInstance()->CreateTable( - "emp_db", DEFUALT_SCHEMA_NAME, "department_table", - std::move(table_schema), txn); - - // Create multiple stat worker threads - int num_threads = 8; - storage::Database *database = - catalog->GetDatabaseWithName("emp_db", txn); - storage::DataTable *table = catalog->GetTableWithName( - "emp_db", DEFUALT_SCHEMA_NAME, "department_table", txn); - txn_manager.CommitTransaction(txn); - LaunchParallelTest(num_threads, TransactionTest, database, table); - // Wait for aggregation to finish - std::chrono::microseconds sleep_time(aggregate_interval * 2 * 1000); - std::this_thread::sleep_for(sleep_time); - aggregator.ShutdownAggregator(); - // Force a final aggregation - ForceFinalAggregation(aggregate_interval); - - // Check query metrics - auto &aggregated_stats = aggregator.GetAggregatedStats(); - ASSERT_EQ(aggregated_stats.GetQueryCount(), num_threads * NUM_ITERATION); - - // Check database metrics - oid_t db_oid = database->GetOid(); - LOG_TRACE("db_oid is %u", db_oid); - auto db_metric = aggregated_stats.GetDatabaseMetric(db_oid); - ASSERT_GT(db_metric->GetTxnCommitted().GetCounter(), - num_threads * NUM_ITERATION * NUM_DB_COMMIT); - ASSERT_EQ(db_metric->GetTxnAborted().GetCounter(), - num_threads * NUM_ITERATION * NUM_DB_ABORT); - - // Check table metrics - auto table_oid = table->GetOid(); - auto table_metric = aggregated_stats.GetTableMetric(db_oid, table_oid); - auto table_access = table_metric->GetTableAccess(); - ASSERT_EQ(table_access.GetReads(), - num_threads * NUM_ITERATION * NUM_TABLE_READ); - ASSERT_EQ(table_access.GetUpdates(), - num_threads * NUM_ITERATION * NUM_TABLE_UPDATE); - ASSERT_EQ(table_access.GetDeletes(), - num_threads * NUM_ITERATION * NUM_TABLE_DELETE); - ASSERT_EQ(table_access.GetInserts(), - num_threads * NUM_ITERATION * NUM_TABLE_INSERT); - - // Check index metrics - auto index_oid = table->GetIndex(0)->GetOid(); - auto index_metric = - aggregated_stats.GetIndexMetric(db_oid, table_oid, index_oid); - auto index_access = index_metric->GetIndexAccess(); - ASSERT_EQ(index_access.GetReads(), - num_threads * NUM_ITERATION * NUM_INDEX_READ); - ASSERT_EQ(index_access.GetDeletes(), - num_threads * NUM_ITERATION * NUM_INDEX_DELETE); - ASSERT_EQ(index_access.GetInserts(), - num_threads * NUM_ITERATION * NUM_INDEX_INSERT); - - txn = txn_manager.BeginTransaction(); - catalog->DropDatabaseWithName("emp_db", txn); - txn_manager.CommitTransaction(txn); -} -// -// TEST_F(StatsTests, PerThreadStatsTest) { -// FLAGS_stats_mode = STATS_TYPE_ENABLE; -// -// // Register to StatsAggregator -// auto &aggregator = peloton::stats::StatsAggregator::GetInstance(1000000); -// -// // int tuple_count = 10; -// int tups_per_tile_group = 100; -// int num_rows = 10; -// -// // Create a table and wrap it in logical tiles -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// auto txn = txn_manager.BeginTransaction(); -// std::unique_ptr data_table( -// TestingExecutorUtil::CreateTable(tups_per_tile_group, true)); -// -// // Ensure that the tile group is as expected. -// const catalog::Schema *schema = data_table->GetSchema(); -// PELOTON_ASSERT(schema->GetColumnCount() == 4); -// -// // Insert tuples into tile_group. -// std::vector tuple_slot_ids; -// -// for (int rowid = 0; rowid < num_rows; rowid++) { -// int populate_value = rowid; -// -// storage::Tuple tuple = TestingStatsUtil::PopulateTuple( -// schema, TestingExecutorUtil::PopulatedValue(populate_value, 0), -// TestingExecutorUtil::PopulatedValue(populate_value, 1), -// TestingExecutorUtil::PopulatedValue(populate_value, 2), -// TestingExecutorUtil::PopulatedValue(populate_value, 3)); -// -// std::unique_ptr project_info{ -// TestingTransactionUtil::MakeProjectInfoFromTuple(&tuple)}; -// -// // Insert -// planner::InsertPlan node(data_table.get(), std::move(project_info)); -// std::unique_ptr context( -// new executor::ExecutorContext(txn)); -// executor::InsertExecutor executor(&node, context.get()); -// executor.Execute(); -// } -// txn_manager.CommitTransaction(txn); -// oid_t database_id = data_table->GetDatabaseOid(); -// oid_t table_id = data_table->GetOid(); -// -// // Check: # transactions committed = 1, # table inserts = 10 -// int64_t txn_commited = stats::BackendStatsContext::GetInstance() -// ->GetDatabaseMetric(database_id) -// ->GetTxnCommitted() -// .GetCounter(); -// int64_t inserts = stats::BackendStatsContext::GetInstance() -// ->GetTableMetric(database_id, table_id) -// ->GetTableAccess() -// .GetInserts(); -// EXPECT_EQ(1, txn_commited); -// EXPECT_EQ(num_rows, inserts); -// -// // Read every other tuple -// txn = txn_manager.BeginTransaction(); -// for (int i = 0; i < num_rows; i += 2) { -// int result; -// TestingTransactionUtil::ExecuteRead( -// txn, data_table.get(), TestingExecutorUtil::PopulatedValue(i, 0), -// result); -// } -// txn_manager.CommitTransaction(txn); -// -// // Check: # transactions committed = 2, # inserts = 10, # reads = 5 -// txn_commited = stats::BackendStatsContext::GetInstance() -// ->GetDatabaseMetric(database_id) -// ->GetTxnCommitted() -// .GetCounter(); -// inserts = stats::BackendStatsContext::GetInstance() -// ->GetTableMetric(database_id, table_id) -// ->GetTableAccess() -// .GetInserts(); -// int64_t reads = stats::BackendStatsContext::GetInstance() -// ->GetTableMetric(database_id, table_id) -// ->GetTableAccess() -// .GetReads(); -// EXPECT_EQ(2, txn_commited); -// EXPECT_EQ(num_rows, inserts); -// EXPECT_EQ(5, reads); -// -// // Do a single read and abort -// txn = txn_manager.BeginTransaction(); -// int result; -// TestingTransactionUtil::ExecuteRead( -// txn, data_table.get(), TestingExecutorUtil::PopulatedValue(0, 0), -// result); -// txn_manager.AbortTransaction(txn); -// -// // Check: # txns committed = 2, # txns aborted = 1, # reads = 6 -// txn_commited = stats::BackendStatsContext::GetInstance() -// ->GetDatabaseMetric(database_id) -// ->GetTxnCommitted() -// .GetCounter(); -// int64_t txn_aborted = stats::BackendStatsContext::GetInstance() -// ->GetDatabaseMetric(database_id) -// ->GetTxnAborted() -// .GetCounter(); -// reads = stats::BackendStatsContext::GetInstance() -// ->GetTableMetric(database_id, table_id) -// ->GetTableAccess() -// .GetReads(); -// EXPECT_EQ(2, txn_commited); -// EXPECT_EQ(1, txn_aborted); -// EXPECT_EQ(6, reads); -// -// // Read and update the first tuple -// txn = txn_manager.BeginTransaction(); -// TestingTransactionUtil::ExecuteUpdate(txn, data_table.get(), 0, 2); -// txn_manager.CommitTransaction(txn); -// -// // Check: # txns committed = 3, # updates = 1, # reads = 7 -// txn_commited = stats::BackendStatsContext::GetInstance() -// ->GetDatabaseMetric(database_id) -// ->GetTxnCommitted() -// .GetCounter(); -// reads = stats::BackendStatsContext::GetInstance() -// ->GetTableMetric(database_id, table_id) -// ->GetTableAccess() -// .GetReads(); -// int64_t updates = stats::BackendStatsContext::GetInstance() -// ->GetTableMetric(database_id, table_id) -// ->GetTableAccess() -// .GetUpdates(); -// EXPECT_EQ(3, txn_commited); -// EXPECT_EQ(7, reads); -// EXPECT_EQ(1, updates); -// -// // Delete the 6th tuple and read the 1st tuple -// txn = txn_manager.BeginTransaction(); -// TestingTransactionUtil::ExecuteDelete(txn, data_table.get(), -// TestingExecutorUtil::PopulatedValue(5, -// 0)); -// LOG_TRACE("before read"); -// TestingTransactionUtil::ExecuteRead( -// txn, data_table.get(), TestingExecutorUtil::PopulatedValue(1, 0), -// result); -// txn_manager.CommitTransaction(txn); -// -// // Check: # txns committed = 4, # deletes = 1, # reads = 8 -// txn_commited = stats::BackendStatsContext::GetInstance() -// ->GetDatabaseMetric(database_id) -// ->GetTxnCommitted() -// .GetCounter(); -// reads = stats::BackendStatsContext::GetInstance() -// ->GetTableMetric(database_id, table_id) -// ->GetTableAccess() -// .GetReads(); -// int64_t deletes = stats::BackendStatsContext::GetInstance() -// ->GetTableMetric(database_id, table_id) -// ->GetTableAccess() -// .GetDeletes(); -// EXPECT_EQ(4, txn_commited); -// EXPECT_EQ(9, reads); -// EXPECT_EQ(1, deletes); -// -// aggregator.ShutdownAggregator(); -//} -// -// TEST_F(StatsTests, PerQueryStatsTest) { -// int64_t aggregate_interval = 1000; -// LaunchAggregator(aggregate_interval); -// auto &aggregator = stats::StatsAggregator::GetInstance(); -// auto &traffic_cop = traffic_cop::TrafficCop::GetInstance(); -// -// // Create a table first -// auto catalog = catalog::Catalog::GetInstance(); -// catalog->CreateDatabase("emp_db", nullptr); -// TestingStatsUtil::CreateTable(); -// -// // Default database should include 4 metrics tables and the test table -// EXPECT_EQ(catalog::Catalog::GetInstance() -// ->GetDatabaseWithName(CATALOG_DATABASE_NAME) -// ->GetTableCount(), -// 6); -// LOG_TRACE("Table created!"); -// -// auto backend_context = stats::BackendStatsContext::GetInstance(); -// // Get a query param object -// std::shared_ptr type_buf; -// std::shared_ptr format_buf; -// std::shared_ptr val_buf; -// auto query_params = -// TestingStatsUtil::GetQueryParams(type_buf, format_buf, val_buf); -// -// // Inserting a tuple end-to-end -// auto statement = TestingStatsUtil::GetInsertStmt(); -// // Initialize the query metric, with prep stmt parameters -// backend_context->InitQueryMetric(statement, query_params); -// -// // Execute insert -// std::vector params; -// std::vector result; -// std::vector result_format(statement->GetTupleDescriptor().size(), 0); -// executor::ExecutionResult status = traffic_cop.ExecuteHelper( -// statement->GetPlanTree().get(), params, result, result_format); -// LOG_TRACE("Statement executed. Result: %s", -// ResultTypeToString(status.m_result).c_str()); -// LOG_TRACE("Tuple inserted!"); -// -// // Now Updating end-to-end -// statement = TestingStatsUtil::GetUpdateStmt(); -// // Initialize the query metric -// backend_context->InitQueryMetric(statement, nullptr); -// -// // Execute update -// params.clear(); -// result.clear(); -// result_format = -// std::move(std::vector(statement->GetTupleDescriptor().size(), 0)); -// status = traffic_cop.ExecuteHelper(statement->GetPlanTree().get(), -// params, result, result_format); -// LOG_TRACE("Statement executed. Result: %s", -// ResultTypeToString(status.m_result).c_str()); -// LOG_TRACE("Tuple updated!"); -// -// // Deleting end-to-end -// statement = std::move(TestingStatsUtil::GetDeleteStmt()); -// // Initialize the query metric -// backend_context->InitQueryMetric(statement, nullptr); -// -// // Execute delete -// params.clear(); -// result.clear(); -// result_format = -// std::move(std::vector(statement->GetTupleDescriptor().size(), 0)); -// status = traffic_cop.ExecuteHelper(statement->GetPlanTree().get(), -// params, result, result_format); -// LOG_TRACE("Statement executed. Result: %s", -// ResultTypeToString(status.m_result).c_str()); -// LOG_TRACE("Tuple deleted!"); -// -// // Wait for aggregation to finish -// std::chrono::microseconds sleep_time(aggregate_interval * 2 * 1000); -// std::this_thread::sleep_for(sleep_time); -// aggregator.ShutdownAggregator(); -// ForceFinalAggregation(aggregate_interval); -// -// EXPECT_EQ(aggregator.GetAggregatedStats().GetQueryCount(), 3); -// -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// auto txn = txn_manager.BeginTransaction(); -// catalog->DropDatabaseWithName("emp_db", txn); -// txn_manager.CommitTransaction(txn); -//} -} // namespace test -} // namespace peloton diff --git a/test/statistics/table_metric_test.cpp b/test/statistics/table_metric_test.cpp new file mode 100644 index 00000000000..99d38b0c8a1 --- /dev/null +++ b/test/statistics/table_metric_test.cpp @@ -0,0 +1,162 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// table_metric_test.cpp +// +// Identification: tests/include/statistics/table_metric_test.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include +#include "common/harness.h" +#include "statistics/table_metric.h" +#include "statistics/testing_stats_util.h" +#include "sql/testing_sql_util.h" +#include "catalog/table_metrics_catalog.h" +#include "concurrency/transaction_manager_factory.h" +#include "catalog/system_catalogs.h" +#include "storage/tile_group.h" + +namespace peloton { +namespace test { + +class TableMetricTests : public PelotonTest { + public: + static std::shared_ptr + GetTableMetricObject(oid_t database_id, oid_t table_id) { + auto table_metrics_catalog = catalog::Catalog::GetInstance() + ->GetSystemCatalogs(database_id) + ->GetTableMetricsCatalog(); + auto txn = concurrency::TransactionManagerFactory::GetInstance() + .BeginTransaction(); + auto table_metric_object = + table_metrics_catalog->GetTableMetricsObject(table_id, txn); + concurrency::TransactionManagerFactory::GetInstance().CommitTransaction( + txn); + return table_metric_object; + } +}; + +TEST_F(TableMetricTests, InitSetup) { TestingStatsUtil::Initialize(); }; + +TEST_F(TableMetricTests, AccessTest) { + stats::StatsAggregator aggregator(1); + + EXPECT_EQ(ResultType::SUCCESS, TestingSQLUtil::ExecuteSQLQuery( + "CREATE TABLE foo (ID INT, YEAR INT);")); + auto db_table_ids = TestingStatsUtil::GetDbTableID("foo"); + + aggregator.Aggregate(); + auto initial_table_metric = TableMetricTests::GetTableMetricObject( + db_table_ids.first, db_table_ids.second); + + size_t inserts = 0; + size_t reads = 0; + size_t updates = 0; + size_t deletes = 0; + + // no metrics logged initially + EXPECT_EQ(inserts, initial_table_metric->GetInserts()); + EXPECT_EQ(updates, initial_table_metric->GetUpdates()); + EXPECT_EQ(reads, initial_table_metric->GetReads()); + EXPECT_EQ(deletes, initial_table_metric->GetDeletes()); + + StatsWorkload workload; + + // no primary key, so all range scans + workload.AddQuery("INSERT INTO foo VALUES (1, 2);"); + workload.AddQuery("INSERT INTO foo VALUES (2, 2);"); + workload.AddQuery("INSERT INTO foo VALUES (5, 2);"); + inserts += 3; + + workload.AddQuery("SELECT * FROM foo;"); // 3 rows in table + reads += 3; + + workload.AddQuery("SELECT * FROM foo;"); // 3 rows in table + reads += 3; + + workload.AddQuery( + "UPDATE foo SET year = 2018 WHERE id = 2"); // 3 rows in table + reads += 3; + updates += 1; + + workload.AddQuery( + "UPDATE foo SET year = 2016 WHERE id = 1"); // 3 rows in table + reads += 3; + updates += 1; + + workload.AddQuery("DELETE FROM foo WHERE id = 1"); // 3 rows in table + reads += 3; + deletes += 1; + + workload.AddQuery("DELETE FROM foo WHERE year = 2018"); // 2 rows in table + reads += 2; + deletes += 1; + + // execute workload + workload.DoQueries(); + + aggregator.Aggregate(); + auto final_table_metric = TableMetricTests::GetTableMetricObject( + db_table_ids.first, db_table_ids.second); + + EXPECT_EQ(inserts, final_table_metric->GetInserts()); + EXPECT_EQ(updates, final_table_metric->GetUpdates()); + EXPECT_EQ(reads, final_table_metric->GetReads()); + EXPECT_EQ(deletes, final_table_metric->GetDeletes()); + + // Clean up + EXPECT_EQ(ResultType::SUCCESS, + TestingSQLUtil::ExecuteSQLQuery("DROP TABLE foo;")); +}; + +TEST_F(TableMetricTests, MemoryMetricTest) { + stats::StatsAggregator aggregator(1); + + EXPECT_EQ(ResultType::SUCCESS, + TestingSQLUtil::ExecuteSQLQuery( + "CREATE TABLE foo (ID INT, CONTENT TEXT);")); + auto db_table_ids = TestingStatsUtil::GetDbTableID("foo"); + + aggregator.Aggregate(); + auto origin_metric = TableMetricTests::GetTableMetricObject( + db_table_ids.first, db_table_ids.second); + + StatsWorkload workload; + + size_t varlen_alloc_inc = 30; + size_t varlen_usage_inc = 30; + size_t inline_tuple_size = + storage::TileGroupHeader::header_entry_size + 4 + 8; + size_t tuple_inc = 3; + size_t inline_alloc_inc = 0; + size_t inline_usage_inc = tuple_inc * inline_tuple_size; + + workload.AddQuery("INSERT INTO foo VALUES (1, \'test1\');"); + workload.AddQuery("INSERT INTO foo VALUES (2, \'test2\');"); + workload.AddQuery("INSERT INTO foo VALUES (5, \'test3\');"); + workload.DoQueries(); + + aggregator.Aggregate(); + auto table_metric_object = TableMetricTests::GetTableMetricObject( + db_table_ids.first, db_table_ids.second); + + EXPECT_EQ(origin_metric->GetInlineMemoryAlloc() + inline_alloc_inc, + table_metric_object->GetInlineMemoryAlloc()); + EXPECT_EQ(origin_metric->GetInlineMemoryUsage() + inline_usage_inc, + table_metric_object->GetInlineMemoryUsage()); + EXPECT_EQ(origin_metric->GetVarlenMemoryUsage() + varlen_usage_inc, + table_metric_object->GetVarlenMemoryUsage()); + EXPECT_EQ(origin_metric->GetVarlenMemoryAlloc() + varlen_alloc_inc, + table_metric_object->GetVarlenMemoryAlloc()); + + LOG_DEBUG("%lu", table_metric_object->GetInlineMemoryAlloc()); + LOG_DEBUG("%lu", table_metric_object->GetInlineMemoryUsage()); + LOG_DEBUG("%lu", table_metric_object->GetVarlenMemoryAlloc()); + LOG_DEBUG("%lu", table_metric_object->GetVarlenMemoryUsage()); +}; +} +} diff --git a/test/statistics/testing_stats_util.cpp b/test/statistics/testing_stats_util.cpp index 7b0f87852fe..5061e19066f 100644 --- a/test/statistics/testing_stats_util.cpp +++ b/test/statistics/testing_stats_util.cpp @@ -6,10 +6,11 @@ // // Identification: tests/include/statistics/stats_tests_util.cpp // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// +#include "statistics/test_metric.h" #include "statistics/testing_stats_util.h" #include "executor/delete_executor.h" @@ -26,86 +27,14 @@ #include "planner/delete_plan.h" #include "planner/insert_plan.h" #include "planner/plan_util.h" -#include "storage/tile.h" +#include "storage/storage_manager.h" #include "traffic_cop/traffic_cop.h" +#include "statistics/stats_aggregator.h" namespace peloton { namespace test { -void TestingStatsUtil::ShowTable(std::string database_name, - std::string table_name) { - std::unique_ptr statement; - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto &peloton_parser = parser::PostgresParser::GetInstance(); - auto &traffic_cop = tcop::TrafficCop::GetInstance(); - - std::vector params; - std::vector result; - std::string sql = "SELECT * FROM " + database_name + "." + table_name; - statement.reset(new Statement("SELECT", sql)); - // using transaction to optimize - auto txn = txn_manager.BeginTransaction(); - auto select_stmt = peloton_parser.BuildParseTree(sql); - statement->SetPlanTree( - optimizer::Optimizer().BuildPelotonPlanTree(select_stmt, txn)); - LOG_DEBUG("%s", - planner::PlanUtil::GetInfo(statement->GetPlanTree().get()).c_str()); - std::vector result_format(statement->GetTupleDescriptor().size(), 0); - traffic_cop.ExecuteHelper(statement->GetPlanTree(), params, result, - result_format); - txn_manager.CommitTransaction(txn); -} - -storage::Tuple TestingStatsUtil::PopulateTuple(const catalog::Schema *schema, - int first_col_val, - int second_col_val, - int third_col_val, - int fourth_col_val) { - auto testing_pool = TestingHarness::GetInstance().GetTestingPool(); - storage::Tuple tuple(schema, true); - tuple.SetValue(0, type::ValueFactory::GetIntegerValue(first_col_val), - testing_pool); - - tuple.SetValue(1, type::ValueFactory::GetIntegerValue(second_col_val), - testing_pool); - - tuple.SetValue(2, type::ValueFactory::GetDecimalValue(third_col_val), - testing_pool); - - type::Value string_value = - type::ValueFactory::GetVarcharValue(std::to_string(fourth_col_val)); - tuple.SetValue(3, string_value, testing_pool); - return tuple; -} - -std::shared_ptr -TestingStatsUtil::GetQueryParams(std::shared_ptr &type_buf, - std::shared_ptr &format_buf, - std::shared_ptr &val_buf) { - // Type - uchar *type_buf_data = new uchar[1]; - type_buf_data[0] = 'x'; - type_buf.reset(type_buf_data); - stats::QueryMetric::QueryParamBuf type(type_buf_data, 1); - - // Format - uchar *format_buf_data = new uchar[1]; - format_buf_data[0] = 'y'; - format_buf.reset(format_buf_data); - stats::QueryMetric::QueryParamBuf format(format_buf_data, 1); - - // Value - uchar *val_buf_data = new uchar[1]; - val_buf_data[0] = 'z'; - val_buf.reset(val_buf_data); - stats::QueryMetric::QueryParamBuf val(val_buf_data, 1); - - // Construct a query param object - std::shared_ptr query_params( - new stats::QueryMetric::QueryParams(format, type, val, 1)); - return query_params; -} - +// TODO(Tianyi) remove this thing when we fix COPY and change copy test void TestingStatsUtil::CreateTable(bool has_primary_key) { LOG_INFO("Creating a table..."); @@ -133,6 +62,7 @@ void TestingStatsUtil::CreateTable(bool has_primary_key) { txn_manager.CommitTransaction(txn); } +// TODO(Tianyi) remove this thing when we fix COPY and change copy test std::shared_ptr TestingStatsUtil::GetInsertStmt(int id, std::string val) { std::shared_ptr statement; @@ -146,26 +76,7 @@ std::shared_ptr TestingStatsUtil::GetInsertStmt(int id, return statement; } -std::shared_ptr TestingStatsUtil::GetDeleteStmt() { - std::shared_ptr statement; - std::string sql = "DELETE FROM emp_db.public.department_table"; - LOG_INFO("Query: %s", sql.c_str()); - statement.reset(new Statement("DELETE", sql)); - ParseAndPlan(statement.get(), sql); - return statement; -} - -std::shared_ptr TestingStatsUtil::GetUpdateStmt() { - std::shared_ptr statement; - std::string sql = - "UPDATE emp_db.public.department_table SET dept_name = 'CS' WHERE " - "dept_id = 1"; - LOG_INFO("Query: %s", sql.c_str()); - statement.reset(new Statement("UPDATE", sql)); - ParseAndPlan(statement.get(), sql); - return statement; -} - +// TODO(Tianyi) remove this thing when we fix COPY and change copy test void TestingStatsUtil::ParseAndPlan(Statement *statement, std::string sql) { auto &peloton_parser = parser::PostgresParser::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); @@ -180,5 +91,63 @@ void TestingStatsUtil::ParseAndPlan(Statement *statement, std::string sql) { txn_manager.CommitTransaction(txn); } +int TestingStatsUtil::AggregateCounts() { + stats::StatsAggregator aggregator(1); + auto result = aggregator.AggregateRawData(); + + // Only TestRawData should be collected + EXPECT_LE(result.size(), 1); + + if (result.empty()) return 0; + + return dynamic_cast(*result[0]).count_; +} + +void TestingStatsUtil::Initialize() { + // Setup Metric + settings::SettingsManager::SetInt(settings::SettingId::stats_mode, + static_cast(StatsModeType::ENABLE)); + // Initialize catalog + auto catalog = catalog::Catalog::GetInstance(); + catalog->Bootstrap(); + settings::SettingsManager::GetInstance().InitializeCatalog(); + EXPECT_EQ(1, storage::StorageManager::GetInstance()->GetDatabaseCount()); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + // begin a transaction + // initialize the catalog and add the default database, so we don't do this on + // the first query + catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + storage::Database *database = + catalog->GetDatabaseWithName(CATALOG_DATABASE_NAME, txn); + EXPECT_EQ(ResultType::SUCCESS, txn_manager.CommitTransaction(txn)); + EXPECT_NE(nullptr, database); +} + +std::pair TestingStatsUtil::GetDbTableID( + const std::string &table_name) { + auto txn = + concurrency::TransactionManagerFactory::GetInstance().BeginTransaction(); + auto table = catalog::Catalog::GetInstance()->GetTableWithName( + DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, table_name, txn); + auto table_id = table->GetOid(); + auto database_id = table->GetDatabaseOid(); + concurrency::TransactionManagerFactory::GetInstance().CommitTransaction(txn); + return {database_id, table_id}; +} + +std::pair TestingStatsUtil::GetDbIndexID( + const std::string &table_name) { + auto txn = + concurrency::TransactionManagerFactory::GetInstance().BeginTransaction(); + auto table = catalog::Catalog::GetInstance()->GetTableWithName( + DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, table_name, txn); + auto database_id = table->GetDatabaseOid(); + auto metadata = table->GetIndex(0)->GetMetadata(); + auto index_id = metadata->GetOid(); + concurrency::TransactionManagerFactory::GetInstance().CommitTransaction(txn); + return {database_id, index_id}; +} + } // namespace test } // namespace peloton diff --git a/test/statistics/tuple_access_metric_test.cpp b/test/statistics/tuple_access_metric_test.cpp new file mode 100644 index 00000000000..303e15852a7 --- /dev/null +++ b/test/statistics/tuple_access_metric_test.cpp @@ -0,0 +1,21 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// tuple_access_metric_test.cpp +// +// Identification: tests/include/statistics/tuple_access_metric_test.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "common/harness.h" + +namespace peloton { +namespace test { + +class TupleAccessMetricTest : public PelotonTest {}; + +} +} diff --git a/test/type/pool_test.cpp b/test/type/pool_test.cpp index a1c1521c46f..ea374bd5db7 100644 --- a/test/type/pool_test.cpp +++ b/test/type/pool_test.cpp @@ -59,8 +59,12 @@ TEST_F(PoolTests, AllocateOnceTest) { p = pool->Allocate(size); EXPECT_TRUE(p != nullptr); + EXPECT_EQ(size, pool->GetMemoryAlloc()); + EXPECT_EQ(size, pool->GetMemoryUsage()); pool->Free(p); + EXPECT_EQ(0, pool->GetMemoryAlloc()); + EXPECT_EQ(0, pool->GetMemoryUsage()); } } // namespace test