|
| 1 | +/* |
| 2 | + * Copyright (c) Meta Platforms, Inc. and affiliates. |
| 3 | + * |
| 4 | + * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | + * you may not use this file except in compliance with the License. |
| 6 | + * You may obtain a copy of the License at |
| 7 | + * |
| 8 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | + * |
| 10 | + * Unless required by applicable law or agreed to in writing, software |
| 11 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | + * See the License for the specific language governing permissions and |
| 14 | + * limitations under the License. |
| 15 | + */ |
| 16 | + |
| 17 | +#include "cachelib/navy/common/ChecksumOffload.h" |
| 18 | + |
| 19 | +#include <folly/fibers/FiberManager.h> |
| 20 | +#include <folly/logging/xlog.h> |
| 21 | +#include <folly/portability/Asm.h> |
| 22 | + |
| 23 | +#include <cstring> |
| 24 | +#include <random> |
| 25 | +#include <vector> |
| 26 | + |
| 27 | +#include "cachelib/navy/common/Hash.h" |
| 28 | + |
| 29 | +#ifdef CACHELIB_BUILD_WITH_DTO |
| 30 | +#include <dto.h> |
| 31 | +#endif |
| 32 | + |
| 33 | +namespace facebook { |
| 34 | +namespace cachelib { |
| 35 | +namespace navy { |
| 36 | + |
| 37 | +#ifdef CACHELIB_BUILD_WITH_DTO |
| 38 | + |
| 39 | +static_assert(sizeof(dto_async_op) <= 192, |
| 40 | + "AsyncChecksumOp::opStorage_ too small for dto_async_op"); |
| 41 | +static_assert(alignof(dto_async_op) <= 64, |
| 42 | + "AsyncChecksumOp::opStorage_ under-aligned for dto_async_op"); |
| 43 | + |
| 44 | +bool checksumOffloadSupported() { return true; } |
| 45 | + |
| 46 | +AsyncChecksumOp::~AsyncChecksumOp() { |
| 47 | + // A submitted DSA operation writes to opStorage_ (completion record) and, |
| 48 | + // for copies, to dest_. It must be drained before this object dies. |
| 49 | + if (state_ == State::kDsaPending) { |
| 50 | + wait(); |
| 51 | + } |
| 52 | +} |
| 53 | + |
| 54 | +void AsyncChecksumOp::submitImpl(uint8_t* dest, |
| 55 | + BufferView src, |
| 56 | + bool cacheControl) { |
| 57 | + XDCHECK(state_ == State::kIdle); |
| 58 | + dest_ = dest; |
| 59 | + src_ = src; |
| 60 | + copy_ = dest != nullptr; |
| 61 | + |
| 62 | + auto* op = reinterpret_cast<dto_async_op*>(opStorage_); |
| 63 | + const int rc = copy_ ? dto_submit_memcpy_crc(op, dest, src.data(), |
| 64 | + src.size(), cacheControl) |
| 65 | + : dto_submit_crc(op, src.data(), src.size()); |
| 66 | + if (rc == DTO_ASYNC_SUBMITTED) { |
| 67 | + state_ = State::kDsaPending; |
| 68 | + return; |
| 69 | + } |
| 70 | + // DTO_ASYNC_FALLBACK: nothing submitted or copied; run on CPU now. |
| 71 | + if (copy_) { |
| 72 | + std::memcpy(dest, src.data(), src.size()); |
| 73 | + } |
| 74 | + crc_ = checksum(src); |
| 75 | + state_ = State::kCpuDone; |
| 76 | +} |
| 77 | + |
| 78 | +uint32_t AsyncChecksumOp::wait() { |
| 79 | + XDCHECK(state_ != State::kIdle); |
| 80 | + if (state_ == State::kCpuDone) { |
| 81 | + state_ = State::kIdle; |
| 82 | + return crc_; |
| 83 | + } |
| 84 | + |
| 85 | + auto* op = reinterpret_cast<dto_async_op*>(opStorage_); |
| 86 | + // Yielding suspends this fiber and lets other request fibers run on the |
| 87 | + // same NavyThread while the accelerator works; that is the actual |
| 88 | + // "free the writer thread" mechanism. Yield ONLY when another fiber is |
| 89 | + // ready to run: with a lone fiber, yield() returns immediately and the |
| 90 | + // loop would busy-spin through the FiberManager at full rate for the |
| 91 | + // whole accelerator operation, which is far more expensive than a pause |
| 92 | + // poll. Outside fiber context (plain thread-pool schedulers, tests) this |
| 93 | + // reduces to a pause-poll. |
| 94 | + auto* fm = folly::fibers::onFiber() |
| 95 | + ? folly::fibers::FiberManager::getFiberManagerUnsafe() |
| 96 | + : nullptr; |
| 97 | + int rc; |
| 98 | + while ((rc = dto_async_poll(op)) == DTO_ASYNC_PENDING) { |
| 99 | + if (fm && fm->hasReadyTasks()) { |
| 100 | + folly::fibers::yield(); |
| 101 | + } else { |
| 102 | + folly::asm_volatile_pause(); |
| 103 | + } |
| 104 | + } |
| 105 | + state_ = State::kIdle; |
| 106 | + if (rc == DTO_ASYNC_DONE) { |
| 107 | + return static_cast<uint32_t>(dto_async_crc_val(op)); |
| 108 | + } |
| 109 | + // Accelerator failure: destination contents are unspecified, so redo the |
| 110 | + // whole operation on the CPU. dest_ is not yet visible to readers per the |
| 111 | + // submit contract, so overwriting is safe. |
| 112 | + if (copy_) { |
| 113 | + std::memcpy(dest_, src_.data(), src_.size()); |
| 114 | + } |
| 115 | + return checksum(src_); |
| 116 | +} |
| 117 | + |
| 118 | +#else // !CACHELIB_BUILD_WITH_DTO |
| 119 | + |
| 120 | +bool checksumOffloadSupported() { return false; } |
| 121 | + |
| 122 | +AsyncChecksumOp::~AsyncChecksumOp() = default; |
| 123 | + |
| 124 | +void AsyncChecksumOp::submitImpl(uint8_t* dest, |
| 125 | + BufferView src, |
| 126 | + bool /* cacheControl */) { |
| 127 | + XDCHECK(state_ == State::kIdle); |
| 128 | + if (dest != nullptr) { |
| 129 | + std::memcpy(dest, src.data(), src.size()); |
| 130 | + } |
| 131 | + crc_ = checksum(src); |
| 132 | + state_ = State::kCpuDone; |
| 133 | +} |
| 134 | + |
| 135 | +uint32_t AsyncChecksumOp::wait() { |
| 136 | + XDCHECK(state_ == State::kCpuDone); |
| 137 | + state_ = State::kIdle; |
| 138 | + return crc_; |
| 139 | +} |
| 140 | + |
| 141 | +#endif // CACHELIB_BUILD_WITH_DTO |
| 142 | + |
| 143 | +void AsyncChecksumOp::submitCopyAndChecksum(uint8_t* dest, |
| 144 | + BufferView src, |
| 145 | + bool cacheControl) { |
| 146 | + XDCHECK(dest); |
| 147 | + submitImpl(dest, src, cacheControl); |
| 148 | +} |
| 149 | + |
| 150 | +void AsyncChecksumOp::submitChecksum(BufferView src) { |
| 151 | + submitImpl(nullptr, src, false); |
| 152 | +} |
| 153 | + |
| 154 | +uint32_t copyAndChecksum(uint8_t* dest, |
| 155 | + BufferView src, |
| 156 | + folly::FunctionRef<void()> overlap) { |
| 157 | + AsyncChecksumOp op; |
| 158 | + // Cache control: destinations of fused copies (write buffers) are read |
| 159 | + // again shortly, by lookups served from in-memory buffers and by the |
| 160 | + // device flush path. |
| 161 | + op.submitCopyAndChecksum(dest, src, true /* cacheControl */); |
| 162 | + overlap(); |
| 163 | + return op.wait(); |
| 164 | +} |
| 165 | + |
| 166 | +uint32_t checksumWithOverlap(BufferView src, |
| 167 | + folly::FunctionRef<void()> overlap) { |
| 168 | + AsyncChecksumOp op; |
| 169 | + op.submitChecksum(src); |
| 170 | + overlap(); |
| 171 | + return op.wait(); |
| 172 | +} |
| 173 | + |
| 174 | +bool checksumOffloadSelfCheck() { |
| 175 | + if (!checksumOffloadSupported()) { |
| 176 | + return false; |
| 177 | + } |
| 178 | + // Exercise both operations on a buffer large enough to exceed DTO's |
| 179 | + // minimum-size gates (DTO_CRC_MIN_BYTES / DTO_MIN_BYTES) so the DSA path |
| 180 | + // actually runs, and verify parity with navy::checksum() plus copy |
| 181 | + // fidelity. If DSA is unavailable, the CPU fallback must also match. |
| 182 | + constexpr size_t kSize = 1024 * 1024; |
| 183 | + std::vector<uint8_t> src(kSize); |
| 184 | + std::vector<uint8_t> dst(kSize, 0); |
| 185 | + std::mt19937 gen{12345}; |
| 186 | + for (auto& b : src) { |
| 187 | + b = static_cast<uint8_t>(gen()); |
| 188 | + } |
| 189 | + |
| 190 | + const BufferView view{src.size(), src.data()}; |
| 191 | + const uint32_t sw = checksum(view); |
| 192 | + const uint32_t viaCrc = checksumWithOverlap(view, [] {}); |
| 193 | + const uint32_t viaCopy = copyAndChecksum(dst.data(), view, [] {}); |
| 194 | + return viaCrc == sw && viaCopy == sw && |
| 195 | + std::memcmp(dst.data(), src.data(), kSize) == 0; |
| 196 | +} |
| 197 | + |
| 198 | +} // namespace navy |
| 199 | +} // namespace cachelib |
| 200 | +} // namespace facebook |
0 commit comments