Skip to content

Commit f99cbf3

Browse files
pashaboumeta-codesync[bot]
pashabou
authored and
meta-codesync[bot]
committed
F14 remove mask from tagMatchIter
Summary: `tagMatchIter` uses `kFullMask` to restrict matches to the 14 lowest bytes (the tag bytes) of the chunk. The result bitmask is consumed by `SparseMaskIter` which gives us the index of the first hit (using `__builtin_ctz`) if the bitmask is not zero. Since we already have a branch on the value of the bitmask, we can replace the "if bitmask is nonzero" branch with "if bitmask is less than 2^14", effectively folding the `kFullMask` into the branch and removing an AND from the critical path. I've done this by adding a new `BoundedMaskIter` which exploits the fact that x86's TZCNT intrinsic (unlike the compiler builtin `__builtin_ctz`) has a defined output for an input of zero. Because of this, `firstSet < BitCount` effectively captures both `mask_ != 0` from `SparseMaskIter` and `mask & kFullMask` from `tagMatchIter`. When running the following benchmark on a Bergamo server, I see geomean **+1.49%** Find performance and geomean **-0.51%** Insert performance: ``` taskset -c 37 buck2 run @//mode/opt fbcode//folly/container/test:hash_maps_bench -- --bm_regex="f14" --bm_mode=best-of ``` ``` Before: Find f14node<NonSSOString, a[128]>[352] 931.74ns 1.07M Find f14val<NonSSOString, a[128]>[352] 98.424% 946.66ns 1.06M Find f14vec<NonSSOString, a[128]>[352] 99.641% 935.09ns 1.07M Find f14node<NonSSOString, a[1]>[352] 936.37ns 1.07M Find f14val<NonSSOString, a[1]>[352] 99.943% 936.90ns 1.07M Find f14vec<NonSSOString, a[1]>[352] 98.427% 951.34ns 1.05M Find f14node<std::string, a[128]>[352] 1.31us 764.49K Find f14val<std::string, a[128]>[352] 92.947% 1.41us 710.57K Find f14vec<std::string, a[128]>[352] 120.19% 1.09us 918.83K Find f14node<std::string, a[1]>[352] 1.33us 750.68K Find f14val<std::string, a[1]>[352] 100.59% 1.32us 755.12K Find f14vec<std::string, a[1]>[352] 130.06% 1.02us 976.31K Find f14node<uint64_t, a[128]>[352] 423.43ns 2.36M Find f14val<uint64_t, a[128]>[352] 72.973% 580.26ns 1.72M Find f14vec<uint64_t, a[128]>[352] 94.439% 448.37ns 2.23M Find f14node<uint64_t, a[1]>[352] 416.32ns 2.40M Find f14val<uint64_t, a[1]>[352] 74.411% 559.49ns 1.79M Find f14vec<uint64_t, a[1]>[352] 102.82% 404.89ns 2.47M ManyFind f14node<NonSSOString, a[128]>[352] 1.01us 986.71K ManyFind f14val<NonSSOString, a[128]>[352] 93.297% 1.09us 920.57K ManyFind f14vec<NonSSOString, a[128]>[352] 92.049% 1.10us 908.26K ManyFind f14node<NonSSOString, a[1]>[352] 998.19ns 1.00M ManyFind f14val<NonSSOString, a[1]>[352] 95.642% 1.04us 958.15K ManyFind f14vec<NonSSOString, a[1]>[352] 95.958% 1.04us 961.32K ManyFind f14node<std::string, a[128]>[352] 1.09us 916.90K ManyFind f14val<std::string, a[128]>[352] 97.703% 1.12us 895.83K ManyFind f14vec<std::string, a[128]>[352] 133.70% 815.76ns 1.23M ManyFind f14node<std::string, a[1]>[352] 1.09us 916.90K ManyFind f14val<std::string, a[1]>[352] 100.37% 1.09us 920.26K ManyFind f14vec<std::string, a[1]>[352] 133.96% 814.12ns 1.23M ManyFind f14node<uint64_t, a[128]>[352] 490.69ns 2.04M ManyFind f14val<uint64_t, a[128]>[352] 92.379% 531.17ns 1.88M ManyFind f14vec<uint64_t, a[128]>[352] 95.346% 514.64ns 1.94M ManyFind f14node<uint64_t, a[1]>[352] 483.15ns 2.07M ManyFind f14val<uint64_t, a[1]>[352] 94.897% 509.13ns 1.96M ManyFind f14vec<uint64_t, a[1]>[352] 95.732% 504.69ns 1.98M SqBrFind f14node<NonSSOString, a[128]>[352] 2.52us 396.76K SqBrFind f14val<NonSSOString, a[128]>[352] 105.62% 2.39us 419.07K SqBrFind f14vec<NonSSOString, a[128]>[352] 93.042% 2.71us 369.15K SqBrFind f14node<NonSSOString, a[1]>[352] 2.44us 409.31K SqBrFind f14val<NonSSOString, a[1]>[352] 105.57% 2.31us 432.10K SqBrFind f14vec<NonSSOString, a[1]>[352] 95.380% 2.56us 390.40K SqBrFind f14node<std::string, a[128]>[352] 2.05us 488.28K SqBrFind f14val<std::string, a[128]>[352] 99.345% 2.06us 485.08K SqBrFind f14vec<std::string, a[128]>[352] 91.430% 2.24us 446.43K SqBrFind f14node<std::string, a[1]>[352] 1.95us 512.89K SqBrFind f14val<std::string, a[1]>[352] 102.48% 1.90us 525.61K SqBrFind f14vec<std::string, a[1]>[352] 94.815% 2.06us 486.30K SqBrFind f14node<uint64_t, a[128]>[352] 990.58ns 1.01M SqBrFind f14val<uint64_t, a[128]>[352] 149.60% 662.13ns 1.51M SqBrFind f14vec<uint64_t, a[128]>[352] 142.93% 693.04ns 1.44M SqBrFind f14node<uint64_t, a[1]>[352] 991.52ns 1.01M SqBrFind f14val<uint64_t, a[1]>[352] 141.21% 702.16ns 1.42M SqBrFind f14vec<uint64_t, a[1]>[352] 151.30% 655.35ns 1.53M After: Find f14node<NonSSOString, a[128]>[352] 914.26ns 1.09M Find f14val<NonSSOString, a[128]>[352] 98.862% 924.79ns 1.08M Find f14vec<NonSSOString, a[128]>[352] 96.057% 951.79ns 1.05M Find f14node<NonSSOString, a[1]>[352] 918.72ns 1.09M Find f14val<NonSSOString, a[1]>[352] 97.743% 939.93ns 1.06M Find f14vec<NonSSOString, a[1]>[352] 97.004% 947.10ns 1.06M Find f14node<std::string, a[128]>[352] 1.29us 777.23K Find f14val<std::string, a[128]>[352] 93.737% 1.37us 728.55K Find f14vec<std::string, a[128]>[352] 121.95% 1.06us 947.84K Find f14node<std::string, a[1]>[352] 1.29us 772.88K Find f14val<std::string, a[1]>[352] 100.01% 1.29us 772.92K Find f14vec<std::string, a[1]>[352] 135.22% 956.85ns 1.05M Find f14node<uint64_t, a[128]>[352] 423.17ns 2.36M Find f14val<uint64_t, a[128]>[352] 87.776% 482.11ns 2.07M Find f14vec<uint64_t, a[128]>[352] 77.054% 549.19ns 1.82M Find f14node<uint64_t, a[1]>[352] 402.80ns 2.48M Find f14val<uint64_t, a[1]>[352] 96.697% 416.56ns 2.40M Find f14vec<uint64_t, a[1]>[352] 78.848% 510.86ns 1.96M ManyFind f14node<NonSSOString, a[128]>[352] 999.36ns 1.00M ManyFind f14val<NonSSOString, a[128]>[352] 94.979% 1.05us 950.40K ManyFind f14vec<NonSSOString, a[128]>[352] 92.868% 1.08us 929.28K ManyFind f14node<NonSSOString, a[1]>[352] 1.00us 999.89K ManyFind f14val<NonSSOString, a[1]>[352] 95.283% 1.05us 952.72K ManyFind f14vec<NonSSOString, a[1]>[352] 96.906% 1.03us 968.95K ManyFind f14node<std::string, a[128]>[352] 1.09us 919.49K ManyFind f14val<std::string, a[128]>[352] 99.116% 1.10us 911.36K ManyFind f14vec<std::string, a[128]>[352] 132.72% 819.43ns 1.22M ManyFind f14node<std::string, a[1]>[352] 1.09us 917.27K ManyFind f14val<std::string, a[1]>[352] 99.856% 1.09us 915.95K ManyFind f14vec<std::string, a[1]>[352] 133.47% 816.79ns 1.22M ManyFind f14node<uint64_t, a[128]>[352] 506.19ns 1.98M ManyFind f14val<uint64_t, a[128]>[352] 100.77% 502.33ns 1.99M ManyFind f14vec<uint64_t, a[128]>[352] 80.661% 627.55ns 1.59M ManyFind f14node<uint64_t, a[1]>[352] 484.81ns 2.06M ManyFind f14val<uint64_t, a[1]>[352] 95.206% 509.23ns 1.96M ManyFind f14vec<uint64_t, a[1]>[352] 79.263% 611.65ns 1.63M SqBrFind f14node<NonSSOString, a[128]>[352] 2.44us 410.40K SqBrFind f14val<NonSSOString, a[128]>[352] 102.19% 2.38us 419.39K SqBrFind f14vec<NonSSOString, a[128]>[352] 91.086% 2.68us 373.81K SqBrFind f14node<NonSSOString, a[1]>[352] 2.32us 430.58K SqBrFind f14val<NonSSOString, a[1]>[352] 101.66% 2.28us 437.72K SqBrFind f14vec<NonSSOString, a[1]>[352] 92.840% 2.50us 399.75K SqBrFind f14node<std::string, a[128]>[352] 2.02us 495.78K SqBrFind f14val<std::string, a[128]>[352] 101.19% 1.99us 501.70K SqBrFind f14vec<std::string, a[128]>[352] 91.958% 2.19us 455.91K SqBrFind f14node<std::string, a[1]>[352] 1.94us 516.45K SqBrFind f14val<std::string, a[1]>[352] 104.00% 1.86us 537.12K SqBrFind f14vec<std::string, a[1]>[352] 95.854% 2.02us 495.03K SqBrFind f14node<uint64_t, a[128]>[352] 945.78ns 1.06M SqBrFind f14val<uint64_t, a[128]>[352] 147.86% 639.66ns 1.56M SqBrFind f14vec<uint64_t, a[128]>[352] 140.64% 672.49ns 1.49M SqBrFind f14node<uint64_t, a[1]>[352] 944.20ns 1.06M SqBrFind f14val<uint64_t, a[1]>[352] 158.89% 594.25ns 1.68M SqBrFind f14vec<uint64_t, a[1]>[352] 152.01% 621.14ns 1.61M ``` Reviewed By: yfeldblum Differential Revision: D94483961 fbshipit-source-id: ca31ec86c3fa9dae492ccc4e87c3e7511ca3f2bf
1 parent 29e92b8 commit f99cbf3

2 files changed

Lines changed: 30 additions & 3 deletions

File tree

‎third-party/folly/src/folly/container/detail/F14Mask.h‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,29 @@ class SparseMaskIter {
172172
}
173173
};
174174

175+
// A variant of SparseMaskIter but using tzcnt (x86-64/bmi1) which is more
176+
// efficient.
177+
template <unsigned BitCount>
178+
class BoundedMaskIter {
179+
MaskType mask_;
180+
181+
public:
182+
explicit BoundedMaskIter(MaskType mask) : mask_{mask} {}
183+
184+
bool hasNext() {
185+
unsigned firstSet =
186+
mask_ == 0 ? (sizeof(MaskType) * 8) : findFirstSetNonZero(mask_);
187+
return firstSet < BitCount;
188+
}
189+
190+
unsigned next() {
191+
FOLLY_SAFE_DCHECK(hasNext());
192+
unsigned i = findFirstSetNonZero(mask_);
193+
mask_ &= mask_ - 1;
194+
return i;
195+
}
196+
};
197+
175198
// Iterates a mask, optimized for the case that most bits are set
176199
class DenseMaskIter {
177200
MaskType mask_;

‎third-party/folly/src/folly/container/detail/F14Table.h‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -841,12 +841,16 @@ struct alignas(constexpr_max(kRequiredVectorAlignment, alignof(ItemType)))
841841
return static_cast<TagVector const*>(static_cast<void const*>(&tags_[0]));
842842
}
843843

844-
SparseMaskIter tagMatchIter(__m128i needleV) const {
844+
auto tagMatchIter(__m128i needleV) const {
845845
auto tagV = _mm_load_si128(tagVector());
846846

847847
auto eqV = _mm_cmpeq_epi8(tagV, needleV);
848-
auto mask = _mm_movemask_epi8(eqV) & kFullMask;
849-
return SparseMaskIter{mask};
848+
uint32_t mask = _mm_movemask_epi8(eqV);
849+
if constexpr (kIsArchAmd64) {
850+
return BoundedMaskIter<kCapacity>{mask};
851+
} else {
852+
return SparseMaskIter{mask & kFullMask};
853+
}
850854
}
851855

852856
MaskType occupiedMask() const {

0 commit comments

Comments
 (0)