Merge remote-tracking branch 'upstream/master' into brain_rl_testing_framework

Author: saatviks

Diff for: cmake/Dependencies.cmake

@@ -86,6 +86,11 @@ llvm_map_components_to_libnames(LLVM_LIBRARIES core mcjit nativecodegen native)
 include_directories(SYSTEM ${LLVM_INCLUDE_DIRS})
 list(APPEND Peloton_LINKER_LIBS ${LLVM_LIBRARIES})
 
+# --[ FFI
+find_package(Libffi)
+include_directories(SYSTEM ${LIBFFI_INCLUDE_DIRS})
+list(APPEND Peloton_LINKER_LIBS ${LIBFFI_LIBRARIES})
+
 # --[ IWYU
 
 # Generate clang compilation database

Diff for: cmake/Modules/FindLibffi.cmake

@@ -0,0 +1,39 @@
+# - Try to find Libffi
+#
+#  A Portable Foreign Function Interface Library (https://sourceware.org/libffi)
+#
+# Usage:
+# LIBFFI_INCLUDE_DIRS, location of header files
+# LIBFFI_LIBRARIES, location of library
+# LIBFFI_FOUND, indicates if libffi was found
+
+# Look for the header file.
+execute_process(COMMAND brew --prefix libffi OUTPUT_VARIABLE LIBFFI_BREW_PREFIX)
+
+find_library(LIBFFI_LIBRARY NAMES ffi libffi
+        PATHS /usr /usr/local /opt/local
+        PATH_SUFFIXES lib lib64 x86_64-linux-gnu lib/x86_64-linux-gnu
+        )
+
+find_path(LIBFFI_INCLUDE_DIR ffi.h
+        PATHS /usr /usr/local /opt/local /usr/include/ffi
+        PATH_SUFFIXES include include/ffi include/x86_64-linux-gnu x86_64-linux-gnu
+        HINT LIBFFI_BREW_PREFIX
+        )
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(LIBFFI DEFAULT_MSG LIBFFI_LIBRARY LIBFFI_INCLUDE_DIR)
+
+
+# Copy the results to the output variables.
+IF(LIBFFI_FOUND)
+    SET(LIBFFI_LIBRARIES ${LIBFFI_LIBRARY})
+    SET(LIBFFI_INCLUDE_DIRS ${LIBFFI_INCLUDE_DIR})
+ELSE(LIBFFI_FOUND)
+    SET(LIBFFI_LIBRARIES)
+    SET(LIBFFI_INCLUDE_DIRS)
+ENDIF(LIBFFI_FOUND)
+
+MARK_AS_ADVANCED(LIBFFI_INCLUDE_DIRS LIBFFI_LIBRARIES)
+
+message(STATUS "Found Libffi (include: ${LIBFFI_INCLUDE_DIRS}, library: ${LIBFFI_LIBRARIES})")

Diff for: script/installation/packages.sh

@@ -47,6 +47,9 @@ TF_TYPE="cpu"
 function install_protobuf3.4.0() {
     # Install Relevant tooling
     # Remove any old versions of protobuf
+    # Note: Protobuf 3.5+ PPA available Ubuntu Bionic(18.04) onwards - Should be used
+    # when we retire 16.04 too: https://launchpad.net/~maarten-fonville/+archive/ubuntu/protobuf
+    # This PPA unfortunately doesnt have Protobuf 3.5 for 16.04, but does for 14.04/18.04+
     DISTRIB=$1 # ubuntu/fedora
     if [ "$DISTRIB" == "ubuntu" ]; then
         sudo apt-get --yes --force-yes remove --purge libprotobuf-dev protobuf-compiler
@@ -63,7 +66,7 @@ function install_protobuf3.4.0() {
     wget -O protobuf-cpp-3.4.0.tar.gz https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-cpp-3.4.0.tar.gz
     tar -xzf protobuf-cpp-3.4.0.tar.gz
     cd protobuf-3.4.0
-    ./autogen.sh && ./configure && make -j4 && sudo make install && sudo ldconfig
+    ./autogen.sh && ./configure && make -j4 && sudo make install && sudo ldconfig || exit 1
     cd ..
     # Cleanup
     rm -rf protobuf-3.4.0 protobuf-cpp-3.4.0.tar.gz
@@ -173,6 +176,8 @@ if [ "$DISTRO" = "UBUNTU" ]; then
         libedit-dev \
         libssl-dev \
         postgresql-client \
+        libffi6 \
+        libffi-dev \
         libtbb-dev \
         python3-pip \
         curl \
@@ -216,6 +221,7 @@ elif [ "$DISTRO" = "DARWIN" ]; then
     brew install libedit
     brew install [email protected]
     brew install postgresql
+    brew install libffi
     brew install tbb
     brew install curl
     brew install wget

Diff for: src/CMakeLists.txt

@@ -62,28 +62,6 @@ set_target_properties(peloton-bin PROPERTIES OUTPUT_NAME peloton)
 
 ##################################################################################
 
-# --[ Benchmarks
-
-# --[ tpcc
-file(GLOB_RECURSE tpcc_srcs ${PROJECT_SOURCE_DIR}/src/main/tpcc/*.cpp)
-add_executable(tpcc EXCLUDE_FROM_ALL ${tpcc_srcs})
-target_link_libraries(tpcc peloton)
-
-# --[ ycsb
-file(GLOB_RECURSE ycsb_srcs ${PROJECT_SOURCE_DIR}/src/main/ycsb/*.cpp)
-add_executable(ycsb EXCLUDE_FROM_ALL ${ycsb_srcs})
-target_link_libraries(ycsb peloton)
-
-# --[ sdbench
-file(GLOB_RECURSE sdbench_srcs ${PROJECT_SOURCE_DIR}/src/main/sdbench/*.cpp)
-add_executable(sdbench EXCLUDE_FROM_ALL ${sdbench_srcs})
-target_link_libraries(sdbench peloton)
-
-# --[ tpch
-file(GLOB_RECURSE tpch_srcs ${PROJECT_SOURCE_DIR}/src/main/tpch/*.cpp)
-add_executable(tpch EXCLUDE_FROM_ALL ${tpch_srcs})
-target_link_libraries(tpch peloton)
-
 # --[ logger
 #file(GLOB_RECURSE logger_srcs ${PROJECT_SOURCE_DIR}/src/main/logger/*.cpp)
 #list(APPEND logger_srcs ${ycsb_srcs})
@@ -96,16 +74,11 @@ target_link_libraries(tpch peloton)
 # --[ link to jemalloc
 set(EXE_LINK_LIBRARIES ${JEMALLOC_LIBRARIES})
 set(EXE_LINK_FLAGS "-Wl,--no-as-needed")
-set(EXE_LIST peloton-bin ycsb tpcc sdbench tpch)
+set(EXE_LIST peloton-bin)
 foreach(exe_name ${EXE_LIST})
     target_link_libraries(${exe_name} ${EXE_LINK_LIBRARIES})
     if (LINUX)
         set_target_properties(${exe_name} PROPERTIES LINK_FLAGS ${EXE_LINK_FLAGS})
     endif ()
 endforeach()
 
-# --[ benchmark
-
-add_custom_target(benchmark)
-add_dependencies(benchmark tpcc ycsb sdbench)
-

Diff for: src/binder/bind_node_visitor.cpp

@@ -208,7 +208,7 @@ void BindNodeVisitor::Visit(parser::AnalyzeStatement *node) {
 void BindNodeVisitor::Visit(expression::TupleValueExpression *expr) {
   if (!expr->GetIsBound()) {
     std::tuple<oid_t, oid_t, oid_t> col_pos_tuple;
-    std::shared_ptr<catalog::TableCatalogObject> table_obj = nullptr;
+    std::shared_ptr<catalog::TableCatalogEntry> table_obj = nullptr;
     type::TypeId value_type;
     int depth = -1;
 

Diff for: src/binder/binder_context.cpp

@@ -38,8 +38,10 @@ void BinderContext::AddRegularTable(const std::string db_name,
                                     const std::string table_alias,
                                     concurrency::TransactionContext *txn) {
   // using catalog object to retrieve meta-data
-  auto table_object = catalog::Catalog::GetInstance()->GetTableObject(
-      db_name, schema_name, table_name, txn);
+  auto table_object = catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn,
+                                                                            db_name,
+                                                                            schema_name,
+                                                                            table_name);
 
   if (regular_table_alias_map_.find(table_alias) !=
           regular_table_alias_map_.end() ||
@@ -79,9 +81,9 @@ void BinderContext::AddNestedTable(
 
 bool BinderContext::GetColumnPosTuple(
     const std::string &col_name,
-    std::shared_ptr<catalog::TableCatalogObject> table_obj,
+    std::shared_ptr<catalog::TableCatalogEntry> table_obj,
     std::tuple<oid_t, oid_t, oid_t> &col_pos_tuple, type::TypeId &value_type) {
-  auto column_object = table_obj->GetColumnObject(col_name);
+  auto column_object = table_obj->GetColumnCatalogEntry(col_name);
   if (column_object == nullptr) {
     return false;
   }
@@ -138,7 +140,7 @@ bool BinderContext::GetColumnPosTuple(
 
 bool BinderContext::GetRegularTableObj(
     std::shared_ptr<BinderContext> current_context, std::string &alias,
-    std::shared_ptr<catalog::TableCatalogObject> &table_obj, int &depth) {
+    std::shared_ptr<catalog::TableCatalogEntry> &table_obj, int &depth) {
   while (current_context != nullptr) {
     auto iter = current_context->regular_table_alias_map_.find(alias);
     if (iter != current_context->regular_table_alias_map_.end()) {
@@ -174,9 +176,9 @@ void BinderContext::GenerateAllColumnExpressions(
     std::vector<std::unique_ptr<expression::AbstractExpression>> &exprs) {
   for (auto &entry : regular_table_alias_map_) {
     auto &table_obj = entry.second;
-    auto col_cnt = table_obj->GetColumnObjects().size();
+    auto col_cnt = table_obj->GetColumnCatalogEntries().size();
     for (size_t i = 0; i < col_cnt; i++) {
-      auto col_obj = table_obj->GetColumnObject(i);
+      auto col_obj = table_obj->GetColumnCatalogEntry(i);
       auto tv_expr = new expression::TupleValueExpression(
           std::string(col_obj->GetColumnName()), std::string(entry.first));
       tv_expr->SetValueType(col_obj->GetColumnType());

Diff for: src/brain/index_selection_job.cpp

@@ -28,16 +28,16 @@ void IndexSelectionJob::OnJobInvocation(BrainEnvironment *env) {
 
   // Analyze stats for all the tables.
   // TODO: AnalyzeStatsForAllTables crashes sometimes.
-  optimizer::StatsStorage *stats_storage =
-      optimizer::StatsStorage::GetInstance();
-  ResultType stats_result = stats_storage->AnalyzeStatsForAllTables(txn);
-  if (stats_result != ResultType::SUCCESS) {
-    LOG_ERROR(
-        "Cannot generate stats for table columns. Not performing index "
-        "suggestion...");
-    txn_manager.AbortTransaction(txn);
-    return;
-  }
+//  optimizer::StatsStorage *stats_storage =
+//      optimizer::StatsStorage::GetInstance();
+//  ResultType stats_result = stats_storage->AnalyzeStatsForAllTables(txn);
+//  if (stats_result != ResultType::SUCCESS) {
+//    LOG_ERROR(
+//        "Cannot generate stats for table columns. Not performing index "
+//        "suggestion...");
+//    txn_manager.AbortTransaction(txn);
+//    return;
+//  }
 
   // Query the catalog for new SQL queries.
   // New SQL queries are the queries that were added to the system
@@ -71,12 +71,12 @@ void IndexSelectionJob::OnJobInvocation(BrainEnvironment *env) {
     }
 
     // Get the index objects from database.
-    auto database_object = catalog::Catalog::GetInstance()->GetDatabaseObject(
-        DEFAULT_DB_NAME, txn);
+    auto database_object = catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(
+        txn, DEFAULT_DB_NAME);
     auto pg_index = catalog::Catalog::GetInstance()
                         ->GetSystemCatalogs(database_object->GetDatabaseOid())
                         ->GetIndexCatalog();
-    auto cur_indexes = pg_index->GetIndexObjects(txn);
+    auto cur_indexes = pg_index->GetIndexCatalogEntries(txn, database_object->GetDatabaseOid());
     auto drop_indexes = GetIndexesToDrop(cur_indexes, best_config);
 
     // Drop useless indexes.
@@ -97,12 +97,12 @@ void IndexSelectionJob::OnJobInvocation(BrainEnvironment *env) {
   txn_manager.CommitTransaction(txn);
 }
 
-std::vector<std::shared_ptr<catalog::IndexCatalogObject>>
+std::vector<std::shared_ptr<catalog::IndexCatalogEntry>>
 IndexSelectionJob::GetIndexesToDrop(
-    std::unordered_map<oid_t, std::shared_ptr<catalog::IndexCatalogObject>>
+    std::unordered_map<oid_t, std::shared_ptr<catalog::IndexCatalogEntry>>
         &index_objects,
     brain::IndexConfiguration best_config) {
-  std::vector<std::shared_ptr<catalog::IndexCatalogObject>> ret_indexes;
+  std::vector<std::shared_ptr<catalog::IndexCatalogEntry>> ret_indexes;
   // Get the existing indexes and drop them.
   for (auto index : index_objects) {
     auto index_name = index.second->GetIndexName();
@@ -162,7 +162,7 @@ void IndexSelectionJob::CreateIndexRPC(brain::HypotheticalIndexObject *index) {
 }
 
 void IndexSelectionJob::DropIndexRPC(oid_t database_oid,
-                                     catalog::IndexCatalogObject *index) {
+                                     catalog::IndexCatalogEntry *index) {
   // TODO: Remove hardcoded database name and server end point.
   // TODO: Have to be removed when merged with tli's code.
   capnp::EzRpcClient client("localhost:15445");

Diff for: src/brain/indextune/compressed_index_config.cpp

@@ -35,9 +35,9 @@ CompressedIndexConfigContainer::CompressedIndexConfigContainer(
 
   auto txn = txn_manager_->BeginTransaction();
 
-  const auto db_obj = catalog_->GetDatabaseObject(database_name_, txn);
+  const auto db_obj = catalog_->GetDatabaseCatalogEntry(txn, database_name_);
   database_oid_ = db_obj->GetDatabaseOid();
-  const auto table_objs = db_obj->GetTableObjects();
+  const auto table_objs = db_obj->GetTableCatalogEntries();
 
   // Uniq identifier per index config
   size_t next_index_id = 0;
@@ -54,7 +54,7 @@ CompressedIndexConfigContainer::CompressedIndexConfigContainer(
     indexid_table_map_[table_oid] = {};
     auto &indexconf_id_map = table_indexid_map_[table_oid];
     auto &id_indexconf_map = indexid_table_map_[table_oid];
-    const auto col_objs = table_obj.second->GetColumnObjects();
+    const auto col_objs = table_obj.second->GetColumnCatalogEntries();
     std::vector<oid_t> null_conf;
     std::vector<oid_t> cols;
     for (const auto &col_obj : col_objs) {
@@ -79,7 +79,7 @@ CompressedIndexConfigContainer::CompressedIndexConfigContainer(
       continue;
     }
 
-    const auto index_objs = table_obj.second->GetIndexObjects();
+    const auto index_objs = table_obj.second->GetIndexCatalogEntries();
     if (index_objs.empty()) {
       SetBit(table_offset_map_.at(table_oid));
     } else {

Diff for: src/brain/indextune/compressed_index_config_util.cpp

@@ -41,7 +41,7 @@ void CompressedIndexConfigUtil::AddCandidates(
   } else if (cand_sel_type == CandidateSelectionType::Simple || cand_sel_type == CandidateSelectionType::Exhaustive) {
     auto sql_stmt_list = ToBindedSqlStmtList(container, query);
     auto txn = container.GetTransactionManager()->BeginTransaction();
-    container.GetCatalog()->GetDatabaseObject(container.GetDatabaseName(), txn);
+    container.GetCatalog()->GetDatabaseCatalogEntry(txn, container.GetDatabaseName());
 
     std::vector<planner::col_triplet> indexable_cols_vector =
         planner::PlanUtil::GetIndexableColumns(txn->catalog_cache,
@@ -106,7 +106,7 @@ void CompressedIndexConfigUtil::DropCandidates(
   auto sql_stmt = sql_stmt_list->GetStatement(0);
 
   auto txn = container.GetTransactionManager()->BeginTransaction();
-  container.GetCatalog()->GetDatabaseObject(container.GetDatabaseName(), txn);
+  container.GetCatalog()->GetDatabaseCatalogEntry(txn, container.GetDatabaseName());
   std::vector<planner::col_triplet> affected_indexes =
       planner::PlanUtil::GetAffectedIndexes(txn->catalog_cache, *sql_stmt,
                                             true);
@@ -126,9 +126,9 @@ CompressedIndexConfigUtil::ConvertIndexTriplet(
   const auto idx_oid = std::get<2>(idx_triplet);
 
   auto txn = container.GetTransactionManager()->BeginTransaction();
-  const auto db_obj = container.GetCatalog()->GetDatabaseObject(db_oid, txn);
-  const auto table_obj = db_obj->GetTableObject(table_oid);
-  const auto idx_obj = table_obj->GetIndexObject(idx_oid);
+  const auto db_obj = container.GetCatalog()->GetDatabaseCatalogEntry(txn, db_oid);
+  const auto table_obj = db_obj->GetTableCatalogEntry(table_oid);
+  const auto idx_obj = table_obj->GetIndexCatalogEntries(idx_oid);
   const auto col_oids = idx_obj->GetKeyAttrs();
   std::vector<oid_t> input_oids(col_oids);
 
@@ -224,8 +224,8 @@ void CompressedIndexConfigUtil::GetIgnoreTables(
 
   auto txn = txn_manager->BeginTransaction();
   const auto table_objs = catalog::Catalog::GetInstance()
-                              ->GetDatabaseObject(db_name, txn)
-                              ->GetTableObjects();
+                              ->GetDatabaseCatalogEntry(txn, db_name)
+                              ->GetTableCatalogEntries();
 
   for (const auto &it : table_objs) {
     ori_table_oids.insert(it.first);

Diff for: src/brain/query_logger.cpp

@@ -41,8 +41,11 @@ void QueryLogger::LogQuery(std::string query_string, uint64_t timestamp) {
 
   // Log query + fingerprint
   auto &query_history_catalog = catalog::QueryHistoryCatalog::GetInstance();
-  query_history_catalog.InsertQueryHistory(
-      query_string, fingerprint.GetFingerprint(), timestamp, nullptr, txn);
+  query_history_catalog.InsertQueryHistory(txn,
+                                           query_string,
+                                           fingerprint.GetFingerprint(),
+                                           timestamp,
+                                           nullptr);
 
   // We're done
   txn_manager.CommitTransaction(txn);

Diff for: src/brain/what_if_index.cpp

@@ -44,24 +44,24 @@ WhatIfIndex::GetCostAndBestPlanTree(
     // Load the tables into cache.
 
     // TODO: Hard coding the schema name for build to pass.
-    auto table_object = catalog::Catalog::GetInstance()->GetTableObject(
-        database_name, "public", table_name, txn);
+    auto table_object = catalog::Catalog::GetInstance()->GetTableCatalogEntry(
+        txn, database_name, "public", table_name);
 
     // Evict all the existing real indexes and
     // insert the what-if indexes into the cache.
-    table_object->EvictAllIndexObjects();
+    table_object->EvictAllIndexCatalogEntries();
 
     // Upon evict index objects, the index set becomes
     // invalid. Set it to valid so that we don't query
     // the catalog again while doing query optimization later.
-    table_object->SetValidIndexObjects(true);
+    table_object->SetValidIndexCatalogEntries(true);
 
     auto index_set = config.GetIndexes();
     for (auto it = index_set.begin(); it != index_set.end(); it++) {
       auto index = *it;
       if (index->table_oid == table_object->GetTableOid()) {
-        auto index_catalog_obj = CreateIndexCatalogObject(index.get());
-        table_object->InsertIndexObject(index_catalog_obj);
+        auto index_catalog_obj = CreateIndexCatalogEntry(index.get());
+        table_object->InsertIndexCatalogEntry(index_catalog_obj);
         LOG_TRACE("Created a new hypothetical index %d on table: %d",
                   index_catalog_obj->GetIndexOid(),
                   index_catalog_obj->GetTableOid());
@@ -84,8 +84,8 @@ WhatIfIndex::GetCostAndBestPlanTree(
   return opt_info_obj;
 }
 
-std::shared_ptr<catalog::IndexCatalogObject>
-WhatIfIndex::CreateIndexCatalogObject(HypotheticalIndexObject *index_obj) {
+std::shared_ptr<catalog::IndexCatalogEntry>
+WhatIfIndex::CreateIndexCatalogEntry(HypotheticalIndexObject *index_obj) {
   // Create an index name:
   // index_<database_name>_<table_name>_<col_oid1>_<col_oid2>_...
   std::ostringstream index_name_oss;
@@ -101,8 +101,8 @@ WhatIfIndex::CreateIndexCatalogObject(HypotheticalIndexObject *index_obj) {
   // Create a dummy catalog object.
   auto col_oids = std::vector<oid_t>(index_obj->column_oids.begin(),
                                      index_obj->column_oids.end());
-  auto index_cat_obj = std::shared_ptr<catalog::IndexCatalogObject>(
-      new catalog::IndexCatalogObject(
+  auto index_cat_obj = std::shared_ptr<catalog::IndexCatalogEntry>(
+      new catalog::IndexCatalogEntry(
           index_seq_no++, index_name_oss.str(), index_obj->table_oid,
           IndexType::BWTREE, IndexConstraintType::DEFAULT, false, col_oids));
   return index_cat_obj;