facebook · jialei-jialei · Apr 9, 2019 · Apr 12, 2019 · Apr 17, 2019 · Apr 18, 2019
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,4 @@
+[submodule "third-party/PhotonLibOS"]
+	path = third-party/PhotonLibOS
+	url = https://github.com/alibaba/PhotonLibOS.git
+	branch = release/0.8
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -44,6 +44,17 @@ endif()
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules/")
 
+# Photon
+set(PHOTON_ENABLE_URING OFF CACHE INTERNAL "Enable iouring")
+add_subdirectory(third-party/PhotonLibOS)
+if (PHOTON_ENABLE_URING)
+    add_compile_definitions("PHOTON_ENABLE_URING")
+endif ()
+option(INIT_PHOTON_IN_ENV "INIT PHOTON IN ROCKSDB" OFF)
+if(INIT_PHOTON_IN_ENV)
+    add_compile_definitions("INIT_PHOTON_IN_ENV")
+endif()
+
 option(WITH_JEMALLOC "build with JeMalloc" OFF)
 option(WITH_SNAPPY "build with SNAPPY" OFF)
 option(WITH_LZ4 "build with lz4" OFF)
@@ -174,11 +185,11 @@ if(MSVC)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324")
 else()
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing")
   if(MINGW)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
   endif()
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
   if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
     include(CheckCXXCompilerFlag)
@@ -458,7 +469,7 @@ endif()
 
 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_SOURCE_DIR}/include)
-include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src)
+include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src ${PROJECT_SOURCE_DIR}/third-party/PhotonLibOS/include)
 find_package(Threads REQUIRED)
 
 add_subdirectory(third-party/gtest-1.7.0/fused-src/gtest)
@@ -742,18 +753,18 @@ else()
 
   add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
   target_link_libraries(${ROCKSDB_SHARED_LIB}
-    ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
+    ${THIRDPARTY_LIBS} ${SYSTEM_LIBS} $<BUILD_INTERFACE:photon_static>)
   set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES
                         LINKER_LANGUAGE CXX
                         VERSION ${ROCKSDB_VERSION}
                         SOVERSION ${ROCKSDB_VERSION_MAJOR}
-                        CXX_STANDARD 11
+                        CXX_STANDARD 14
                         OUTPUT_NAME "rocksdb")
 endif()
 
 add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES})
 target_link_libraries(${ROCKSDB_STATIC_LIB}
-  ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
+  ${THIRDPARTY_LIBS} ${SYSTEM_LIBS} $<BUILD_INTERFACE:photon_static>)
 
 if(WIN32)
   add_library(${ROCKSDB_IMPORT_LIB} SHARED ${SOURCES})
@@ -1067,3 +1078,5 @@ option(WITH_TOOLS "build with tools" ON)
 if(WITH_TOOLS)
   add_subdirectory(tools)
 endif()
+
+add_subdirectory(examples)
diff --git a/HISTORY.md b/HISTORY.md
@@ -1,6 +1,9 @@
 # Rocksdb Change Log
+## 6.1.2 (6/4/2019)
+### Bug Fixes
+* Fix flush's/compaction's merge processing logic which allowed `Put`s covered by range tombstones to reappear. Note `Put`s may exist even if the user only ever called `Merge()` due to an internal conversion during compaction to the bottommost level.
 
-### Unreleased
+## 6.1.1 (4/9/2019)
 ### New Features
 * When reading from option file/string/map, customized comparators and/or merge operators can be filled according to object registry.
 ### Public API Change

diff --git a/db/db_range_del_test.cc b/db/db_range_del_test.cc
@@ -490,6 +490,30 @@ TEST_F(DBRangeDelTest, CompactionRemovesCoveredMergeOperands) {
   ASSERT_EQ(expected, actual);
 }
 
+TEST_F(DBRangeDelTest, PutDeleteRangeMergeFlush) {
+  // Test the sequence of operations: (1) Put, (2) DeleteRange, (3) Merge, (4)
+  // Flush. The `CompactionIterator` previously had a bug where we forgot to
+  // check for covering range tombstones when processing the (1) Put, causing
+  // it to reappear after the flush.
+  Options opts = CurrentOptions();
+  opts.merge_operator = MergeOperators::CreateUInt64AddOperator();
+  Reopen(opts);
+
+  std::string val;
+  PutFixed64(&val, 1);
+  ASSERT_OK(db_->Put(WriteOptions(), "key", val));
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
+                             "key", "key_"));
+  ASSERT_OK(db_->Merge(WriteOptions(), "key", val));
+  ASSERT_OK(db_->Flush(FlushOptions()));
+
+  ReadOptions read_opts;
+  std::string expected, actual;
+  ASSERT_OK(db_->Get(read_opts, "key", &actual));
+  PutFixed64(&expected, 1);
+  ASSERT_EQ(expected, actual);
+}
+
 // NumTableFilesAtLevel() is not supported in ROCKSDB_LITE
 #ifndef ROCKSDB_LITE
 TEST_F(DBRangeDelTest, ObsoleteTombstoneCleanup) {

diff --git a/db/db_test2.cc b/db/db_test2.cc
@@ -3661,6 +3661,48 @@ TEST_F(DBTest2, MultiDBParallelOpenTest) {
 }
 #endif  // OS_WIN
 
+namespace {
+class DummyOldStats : public Statistics {
+ public:
+  uint64_t getTickerCount(uint32_t /*ticker_type*/) const override { return 0; }
+  void recordTick(uint32_t /* ticker_type */, uint64_t /* count */) override {
+    num_rt++;
+  }
+  void setTickerCount(uint32_t /*ticker_type*/, uint64_t /*count*/) override {}
+  uint64_t getAndResetTickerCount(uint32_t /*ticker_type*/) override {
+    return 0;
+  }
+  void measureTime(uint32_t /*histogram_type*/, uint64_t /*count*/) override {
+    num_mt++;
+  }
+  void histogramData(uint32_t /*histogram_type*/,
+                     rocksdb::HistogramData* const /*data*/) const override {}
+  std::string getHistogramString(uint32_t /*type*/) const override {
+    return "";
+  }
+  bool HistEnabledForType(uint32_t /*type*/) const override { return false; }
+  std::string ToString() const override { return ""; }
+  int num_rt = 0;
+  int num_mt = 0;
+};
+}  // namespace
+
+TEST_F(DBTest2, OldStatsInterface) {
+  DummyOldStats* dos = new DummyOldStats();
+  std::shared_ptr<Statistics> stats(dos);
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.statistics = stats;
+  Reopen(options);
+
+  Put("foo", "bar");
+  ASSERT_EQ("bar", Get("foo"));
+  ASSERT_OK(Flush());
+  ASSERT_EQ("bar", Get("foo"));
+
+  ASSERT_GT(dos->num_rt, 0);
+  ASSERT_GT(dos->num_mt, 0);
+}
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

diff --git a/db/db_write_test.cc b/db/db_write_test.cc
@@ -61,6 +61,7 @@ TEST_P(DBWriteTest, IOErrorOnWALWritePropagateToWriteThreadFollower) {
           leader_count++;
           while (ready_count < kNumThreads) {
             // busy waiting
+            std::this_thread::yield();
           }
         }
       });

diff --git a/db/dbformat_test.cc b/db/dbformat_test.cc
@@ -202,6 +202,7 @@ TEST_F(FormatTest, RangeTombstoneSerializeEndKey) {
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {
+  rocksdb::PhotonEnv::Singleton();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }
diff --git a/db/merge_helper.cc b/db/merge_helper.cc
@@ -201,7 +201,15 @@ Status MergeHelper::MergeUntil(InternalIterator* iter,
       // want. Also if we're in compaction and it's a put, it would be nice to
       // run compaction filter on it.
       const Slice val = iter->value();
-      const Slice* val_ptr = (kTypeValue == ikey.type) ? &val : nullptr;
+      const Slice* val_ptr;
+      if (kTypeValue == ikey.type &&
+          (range_del_agg == nullptr ||
+           !range_del_agg->ShouldDelete(
+               ikey, RangeDelPositioningMode::kForwardTraversal))) {
+        val_ptr = &val;
+      } else {
+        val_ptr = nullptr;
+      }
       std::string merge_result;
       s = TimedFullMerge(user_merge_operator_, ikey.user_key, val_ptr,
                          merge_context_.GetOperands(), &merge_result, logger_,

diff --git a/db/prefix_test.cc b/db/prefix_test.cc
@@ -43,13 +43,17 @@ DEFINE_uint64(num_locks, 10001, "number of locks");
 DEFINE_bool(random_prefix, false, "randomize prefix");
 DEFINE_uint64(total_prefixes, 100000, "total number of prefixes");
 DEFINE_uint64(items_per_prefix, 1, "total number of values per prefix");
-DEFINE_int64(write_buffer_size, 33554432, "");
-DEFINE_int32(max_write_buffer_number, 2, "");
-DEFINE_int32(min_write_buffer_number_to_merge, 1, "");
+// DEFINE_int64(write_buffer_size, 33554432, "");
+DECLARE_int64(write_buffer_size);
+// DEFINE_int32(max_write_buffer_number, 2, "");
+DECLARE_int32(max_write_buffer_number);
+// DEFINE_int32(min_write_buffer_number_to_merge, 1, "");
+DECLARE_int32(min_write_buffer_number_to_merge);
 DEFINE_int32(skiplist_height, 4, "");
 DEFINE_double(memtable_prefix_bloom_size_ratio, 0.1, "");
 DEFINE_int32(memtable_huge_page_size, 2 * 1024 * 1024, "");
-DEFINE_int32(value_size, 40, "");
+// DEFINE_int32(value_size, 40, "");
+DECLARE_int32(value_size);
 DEFINE_bool(enable_print, false, "Print options generated to console.");
 
 // Path to the database on file system
@@ -876,6 +880,10 @@ TEST_F(PrefixTest, PrefixSeekModePrev3) {
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
   ParseCommandLineFlags(&argc, &argv, true);
+  FLAGS_write_buffer_size = 33554432;
+  FLAGS_max_write_buffer_number = 2;
+  FLAGS_min_write_buffer_number_to_merge = 1;
+  FLAGS_value_size = 40;
   return RUN_ALL_TESTS();
 }
 

diff --git a/db/range_del_aggregator.h b/db/range_del_aggregator.h
@@ -12,6 +12,7 @@
 #include <set>
 #include <string>
 #include <vector>
+#include <cstdint>
 
 #include "db/compaction_iteration_stats.h"
 #include "db/dbformat.h"

diff --git a/db/range_del_aggregator_test.cc b/db/range_del_aggregator_test.cc
@@ -704,6 +704,7 @@ TEST_F(RangeDelAggregatorTest,
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {
+  rocksdb::PhotonEnv::Singleton();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }
diff --git a/db/range_tombstone_fragmenter_test.cc b/db/range_tombstone_fragmenter_test.cc
@@ -547,6 +547,7 @@ TEST_F(RangeTombstoneFragmenterTest, SeekOutOfBounds) {
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {
+  rocksdb::PhotonEnv::Singleton();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }
diff --git a/db/version_edit.h b/db/version_edit.h
@@ -52,14 +52,14 @@ struct FileDescriptor {
         smallest_seqno(_smallest_seqno),
         largest_seqno(_largest_seqno) {}
 
-  FileDescriptor& operator=(const FileDescriptor& fd) {
-    table_reader = fd.table_reader;
-    packed_number_and_path_id = fd.packed_number_and_path_id;
-    file_size = fd.file_size;
-    smallest_seqno = fd.smallest_seqno;
-    largest_seqno = fd.largest_seqno;
-    return *this;
-  }
+  // FileDescriptor& operator=(const FileDescriptor& fd) {
+  //   table_reader = fd.table_reader;
+  //   packed_number_and_path_id = fd.packed_number_and_path_id;
+  //   file_size = fd.file_size;
+  //   smallest_seqno = fd.smallest_seqno;
+  //   largest_seqno = fd.largest_seqno;
+  //   return *this;
+  // }
 
   uint64_t GetNumber() const {
     return packed_number_and_path_id & kFileNumberMask;

diff --git a/db/version_set.cc b/db/version_set.cc
@@ -2235,7 +2235,7 @@ void VersionStorageInfo::GetOverlappingInputsRangeBinarySearch(
   // If there were no overlapping files, return immediately.
   if (!foundOverlap) {
     if (next_smallest) {
-      next_smallest = nullptr;
+      *next_smallest = nullptr;
     }
     return;
   }

diff --git a/db/write_callback_test.cc b/db/write_callback_test.cc
@@ -185,6 +185,7 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
                     // This allows us to confidently detect the first writer
                     // who increases threads_linked as the leader.
                     while (threads_linked.load() < cur_threads_joining) {
+                      std::this_thread::yield();
                     }
                   });
 
@@ -258,11 +259,13 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
 
                 // leaders gotta lead
                 while (i > 0 && threads_verified.load() < 1) {
+                  std::this_thread::yield();
                 }
 
                 // loser has to lose
                 while (i == write_group.size() - 1 &&
                        threads_verified.load() < write_group.size() - 1) {
+                  std::this_thread::yield();
                 }
 
                 auto& write_op = write_group.at(i);