Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test-for-compare #13167

Open
wants to merge 35 commits into
base: 6.8.fb
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
c931f3a
[rocksdb] 6.1.1 patch for internal version.
Apr 9, 2019
688a3c2
Still implement StatisticsImpl::measureTime() (#5181)
siying Apr 12, 2019
d98808f
Add back NewEmptyIterator (#5203)
riversand963 Apr 17, 2019
d81aedf
Fix a bug in GetOverlappingInputsRangeBinarySearch (#5211)
riversand963 Apr 18, 2019
386fa19
Fix merging range tombstone covering put during flush/compaction (#5406)
ajkr Jun 4, 2019
2b38e2d
Bump version to 6.1.2
ltamasi Jun 4, 2019
140fb17
photon + rocksdb
Sep 19, 2022
38151ed
upgrate photon to version 0.6
beef9999 Jul 28, 2023
f630466
Fix io_uring of io_posix.cc
beef9999 Aug 6, 2023
b63a654
disable BUILD_TESTING option in photon
Aug 10, 2023
86a415e
Temporarily use 0.4 backport
Aug 11, 2023
4cf7bfe
update to release/0.4
Aug 21, 2023
aef957e
update photon to 0.8
Sep 20, 2024
7e5773c
minor change
Sep 22, 2024
b5abf89
update
Sep 22, 2024
3cfd050
fix gcc 13 compile
Sep 23, 2024
d08b4d1
update readme
Sep 23, 2024
b50a1bf
add epoll/iouring option
Sep 24, 2024
d836578
db_compaction_test
Sep 24, 2024
6733e6b
inlineskiplist_test
Sep 24, 2024
cb7a21b
fix skiplist_test
Sep 24, 2024
8d097da
fix cassandra_functional_test stringappend_test write_prepared_transa…
Sep 25, 2024
e97c3a7
fix prefix_test
Sep 25, 2024
0a76da7
improve perf
Sep 25, 2024
0cfb18d
update readme
Sep 25, 2024
f5f9af2
update bench doc
Sep 27, 2024
f0c485b
build ok
Oct 3, 2024
6ce8866
change vcpu from 16 to 8
Oct 6, 2024
d149db7
update perf-server
Oct 6, 2024
f737090
update perf
Oct 6, 2024
d8f9132
update perf
Oct 6, 2024
bb3c780
update photon tag
Oct 6, 2024
f0204f9
photon fetch-content changes to submodule
Oct 20, 2024
f522fc5
Update photon-auto-convert.sh
Oct 20, 2024
1b651b8
update new_stub_pool API
Oct 21, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
[submodule "third-party/PhotonLibOS"]
path = third-party/PhotonLibOS
url = https://github.com/alibaba/PhotonLibOS.git
branch = release/0.8
25 changes: 19 additions & 6 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,17 @@ endif()

list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules/")

# Photon
set(PHOTON_ENABLE_URING OFF CACHE INTERNAL "Enable iouring")
add_subdirectory(third-party/PhotonLibOS)
if (PHOTON_ENABLE_URING)
add_compile_definitions("PHOTON_ENABLE_URING")
endif ()
option(INIT_PHOTON_IN_ENV "INIT PHOTON IN ROCKSDB" OFF)
if(INIT_PHOTON_IN_ENV)
add_compile_definitions("INIT_PHOTON_IN_ENV")
endif()

option(WITH_JEMALLOC "build with JeMalloc" OFF)
option(WITH_SNAPPY "build with SNAPPY" OFF)
option(WITH_LZ4 "build with lz4" OFF)
Expand Down Expand Up @@ -174,11 +185,11 @@ if(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324")
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing")
if(MINGW)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
include(CheckCXXCompilerFlag)
Expand Down Expand Up @@ -458,7 +469,7 @@ endif()

include_directories(${PROJECT_SOURCE_DIR})
include_directories(${PROJECT_SOURCE_DIR}/include)
include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src)
include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src ${PROJECT_SOURCE_DIR}/third-party/PhotonLibOS/include)
find_package(Threads REQUIRED)

add_subdirectory(third-party/gtest-1.7.0/fused-src/gtest)
Expand Down Expand Up @@ -742,18 +753,18 @@ else()

add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
target_link_libraries(${ROCKSDB_SHARED_LIB}
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
${THIRDPARTY_LIBS} ${SYSTEM_LIBS} $<BUILD_INTERFACE:photon_static>)
set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES
LINKER_LANGUAGE CXX
VERSION ${ROCKSDB_VERSION}
SOVERSION ${ROCKSDB_VERSION_MAJOR}
CXX_STANDARD 11
CXX_STANDARD 14
OUTPUT_NAME "rocksdb")
endif()

add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES})
target_link_libraries(${ROCKSDB_STATIC_LIB}
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
${THIRDPARTY_LIBS} ${SYSTEM_LIBS} $<BUILD_INTERFACE:photon_static>)

if(WIN32)
add_library(${ROCKSDB_IMPORT_LIB} SHARED ${SOURCES})
Expand Down Expand Up @@ -1067,3 +1078,5 @@ option(WITH_TOOLS "build with tools" ON)
if(WITH_TOOLS)
add_subdirectory(tools)
endif()

add_subdirectory(examples)
5 changes: 4 additions & 1 deletion HISTORY.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
# Rocksdb Change Log
## 6.1.2 (6/4/2019)
### Bug Fixes
* Fix flush's/compaction's merge processing logic which allowed `Put`s covered by range tombstones to reappear. Note `Put`s may exist even if the user only ever called `Merge()` due to an internal conversion during compaction to the bottommost level.

### Unreleased
## 6.1.1 (4/9/2019)
### New Features
* When reading from option file/string/map, customized comparators and/or merge operators can be filled according to object registry.
### Public API Change
Expand Down
24 changes: 24 additions & 0 deletions db/db_range_del_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,30 @@ TEST_F(DBRangeDelTest, CompactionRemovesCoveredMergeOperands) {
ASSERT_EQ(expected, actual);
}

TEST_F(DBRangeDelTest, PutDeleteRangeMergeFlush) {
// Test the sequence of operations: (1) Put, (2) DeleteRange, (3) Merge, (4)
// Flush. The `CompactionIterator` previously had a bug where we forgot to
// check for covering range tombstones when processing the (1) Put, causing
// it to reappear after the flush.
Options opts = CurrentOptions();
opts.merge_operator = MergeOperators::CreateUInt64AddOperator();
Reopen(opts);

std::string val;
PutFixed64(&val, 1);
ASSERT_OK(db_->Put(WriteOptions(), "key", val));
ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
"key", "key_"));
ASSERT_OK(db_->Merge(WriteOptions(), "key", val));
ASSERT_OK(db_->Flush(FlushOptions()));

ReadOptions read_opts;
std::string expected, actual;
ASSERT_OK(db_->Get(read_opts, "key", &actual));
PutFixed64(&expected, 1);
ASSERT_EQ(expected, actual);
}

// NumTableFilesAtLevel() is not supported in ROCKSDB_LITE
#ifndef ROCKSDB_LITE
TEST_F(DBRangeDelTest, ObsoleteTombstoneCleanup) {
Expand Down
42 changes: 42 additions & 0 deletions db/db_test2.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3661,6 +3661,48 @@ TEST_F(DBTest2, MultiDBParallelOpenTest) {
}
#endif // OS_WIN

namespace {
class DummyOldStats : public Statistics {
public:
uint64_t getTickerCount(uint32_t /*ticker_type*/) const override { return 0; }
void recordTick(uint32_t /* ticker_type */, uint64_t /* count */) override {
num_rt++;
}
void setTickerCount(uint32_t /*ticker_type*/, uint64_t /*count*/) override {}
uint64_t getAndResetTickerCount(uint32_t /*ticker_type*/) override {
return 0;
}
void measureTime(uint32_t /*histogram_type*/, uint64_t /*count*/) override {
num_mt++;
}
void histogramData(uint32_t /*histogram_type*/,
rocksdb::HistogramData* const /*data*/) const override {}
std::string getHistogramString(uint32_t /*type*/) const override {
return "";
}
bool HistEnabledForType(uint32_t /*type*/) const override { return false; }
std::string ToString() const override { return ""; }
int num_rt = 0;
int num_mt = 0;
};
} // namespace

TEST_F(DBTest2, OldStatsInterface) {
DummyOldStats* dos = new DummyOldStats();
std::shared_ptr<Statistics> stats(dos);
Options options = CurrentOptions();
options.create_if_missing = true;
options.statistics = stats;
Reopen(options);

Put("foo", "bar");
ASSERT_EQ("bar", Get("foo"));
ASSERT_OK(Flush());
ASSERT_EQ("bar", Get("foo"));

ASSERT_GT(dos->num_rt, 0);
ASSERT_GT(dos->num_mt, 0);
}
} // namespace rocksdb

int main(int argc, char** argv) {
Expand Down
1 change: 1 addition & 0 deletions db/db_write_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ TEST_P(DBWriteTest, IOErrorOnWALWritePropagateToWriteThreadFollower) {
leader_count++;
while (ready_count < kNumThreads) {
// busy waiting
std::this_thread::yield();
}
}
});
Expand Down
1 change: 1 addition & 0 deletions db/dbformat_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,7 @@ TEST_F(FormatTest, RangeTombstoneSerializeEndKey) {
} // namespace rocksdb

int main(int argc, char** argv) {
rocksdb::PhotonEnv::Singleton();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
10 changes: 9 additions & 1 deletion db/merge_helper.cc
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,15 @@ Status MergeHelper::MergeUntil(InternalIterator* iter,
// want. Also if we're in compaction and it's a put, it would be nice to
// run compaction filter on it.
const Slice val = iter->value();
const Slice* val_ptr = (kTypeValue == ikey.type) ? &val : nullptr;
const Slice* val_ptr;
if (kTypeValue == ikey.type &&
(range_del_agg == nullptr ||
!range_del_agg->ShouldDelete(
ikey, RangeDelPositioningMode::kForwardTraversal))) {
val_ptr = &val;
} else {
val_ptr = nullptr;
}
std::string merge_result;
s = TimedFullMerge(user_merge_operator_, ikey.user_key, val_ptr,
merge_context_.GetOperands(), &merge_result, logger_,
Expand Down
16 changes: 12 additions & 4 deletions db/prefix_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,17 @@ DEFINE_uint64(num_locks, 10001, "number of locks");
DEFINE_bool(random_prefix, false, "randomize prefix");
DEFINE_uint64(total_prefixes, 100000, "total number of prefixes");
DEFINE_uint64(items_per_prefix, 1, "total number of values per prefix");
DEFINE_int64(write_buffer_size, 33554432, "");
DEFINE_int32(max_write_buffer_number, 2, "");
DEFINE_int32(min_write_buffer_number_to_merge, 1, "");
// DEFINE_int64(write_buffer_size, 33554432, "");
DECLARE_int64(write_buffer_size);
// DEFINE_int32(max_write_buffer_number, 2, "");
DECLARE_int32(max_write_buffer_number);
// DEFINE_int32(min_write_buffer_number_to_merge, 1, "");
DECLARE_int32(min_write_buffer_number_to_merge);
DEFINE_int32(skiplist_height, 4, "");
DEFINE_double(memtable_prefix_bloom_size_ratio, 0.1, "");
DEFINE_int32(memtable_huge_page_size, 2 * 1024 * 1024, "");
DEFINE_int32(value_size, 40, "");
// DEFINE_int32(value_size, 40, "");
DECLARE_int32(value_size);
DEFINE_bool(enable_print, false, "Print options generated to console.");

// Path to the database on file system
Expand Down Expand Up @@ -876,6 +880,10 @@ TEST_F(PrefixTest, PrefixSeekModePrev3) {
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
ParseCommandLineFlags(&argc, &argv, true);
FLAGS_write_buffer_size = 33554432;
FLAGS_max_write_buffer_number = 2;
FLAGS_min_write_buffer_number_to_merge = 1;
FLAGS_value_size = 40;
return RUN_ALL_TESTS();
}

Expand Down
1 change: 1 addition & 0 deletions db/range_del_aggregator.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include <set>
#include <string>
#include <vector>
#include <cstdint>

#include "db/compaction_iteration_stats.h"
#include "db/dbformat.h"
Expand Down
1 change: 1 addition & 0 deletions db/range_del_aggregator_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -704,6 +704,7 @@ TEST_F(RangeDelAggregatorTest,
} // namespace rocksdb

int main(int argc, char** argv) {
rocksdb::PhotonEnv::Singleton();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
1 change: 1 addition & 0 deletions db/range_tombstone_fragmenter_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -547,6 +547,7 @@ TEST_F(RangeTombstoneFragmenterTest, SeekOutOfBounds) {
} // namespace rocksdb

int main(int argc, char** argv) {
rocksdb::PhotonEnv::Singleton();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
16 changes: 8 additions & 8 deletions db/version_edit.h
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,14 @@ struct FileDescriptor {
smallest_seqno(_smallest_seqno),
largest_seqno(_largest_seqno) {}

FileDescriptor& operator=(const FileDescriptor& fd) {
table_reader = fd.table_reader;
packed_number_and_path_id = fd.packed_number_and_path_id;
file_size = fd.file_size;
smallest_seqno = fd.smallest_seqno;
largest_seqno = fd.largest_seqno;
return *this;
}
// FileDescriptor& operator=(const FileDescriptor& fd) {
// table_reader = fd.table_reader;
// packed_number_and_path_id = fd.packed_number_and_path_id;
// file_size = fd.file_size;
// smallest_seqno = fd.smallest_seqno;
// largest_seqno = fd.largest_seqno;
// return *this;
// }

uint64_t GetNumber() const {
return packed_number_and_path_id & kFileNumberMask;
Expand Down
2 changes: 1 addition & 1 deletion db/version_set.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2235,7 +2235,7 @@ void VersionStorageInfo::GetOverlappingInputsRangeBinarySearch(
// If there were no overlapping files, return immediately.
if (!foundOverlap) {
if (next_smallest) {
next_smallest = nullptr;
*next_smallest = nullptr;
}
return;
}
Expand Down
3 changes: 3 additions & 0 deletions db/write_callback_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
// This allows us to confidently detect the first writer
// who increases threads_linked as the leader.
while (threads_linked.load() < cur_threads_joining) {
std::this_thread::yield();
}
});

Expand Down Expand Up @@ -258,11 +259,13 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {

// leaders gotta lead
while (i > 0 && threads_verified.load() < 1) {
std::this_thread::yield();
}

// loser has to lose
while (i == write_group.size() - 1 &&
threads_verified.load() < write_group.size() - 1) {
std::this_thread::yield();
}

auto& write_op = write_group.at(i);
Expand Down
Loading