Skip to content

Commit f234334

Browse files
SGSSGeneSGSSGene
authored
Merge pull request #107 from SGSSGene/feat/improved_ternary
Feat/improved ternary
2 parents bc4f9a0 + ed731b0 commit f234334

5 files changed

Lines changed: 89 additions & 46 deletions

File tree

‎src/fmindex-collection/fmindex/BiFMIndex.h‎

Lines changed: 21 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,10 @@ struct BiFMIndex {
2222
using NoDelim = BiFMIndex<TSigma, String, SparseArray, false, TReuseRev>;
2323
using ReuseRev = BiFMIndex<TSigma, String, SparseArray, TDelim, true>;
2424

25-
static size_t constexpr Sigma = TSigma;
26-
static size_t constexpr FirstSymb = TDelim?1:0;
25+
static size_t constexpr Sigma = TSigma;
26+
static size_t constexpr FirstSymb = TDelim?1:0;
27+
static size_t constexpr Delim_v = TDelim;
28+
static size_t constexpr ReuseRev_v = TReuseRev;
2729

2830
// Set RevBwtType to std::nullptr_t to indicate that it should not be used
2931
using RevBwtType = std::conditional_t<TReuseRev, std::nullptr_t, String<Sigma>>;
@@ -56,15 +58,18 @@ struct BiFMIndex {
5658
{}
5759

5860

59-
BiFMIndex(Sequence auto const& _sequence, SparseArray const& _annotatedSequence, size_t _threadNbr, bool mirrorInput = false) {
61+
/*
62+
* \param includeReversedInput assumes that the input data also has the reversed input data
63+
*/
64+
BiFMIndex(Sequence auto const& _sequence, SparseArray const& _annotatedSequence, size_t _threadNbr, bool includeReversedInput = false) {
6065
if (_sequence.size() >= std::numeric_limits<size_t>::max()/2) {
6166
throw std::runtime_error{"sequence is longer than what this system is capable of handling"};
6267
}
6368

6469
bool omegaSorting = !TDelim; // Use omega sorting if no delimiter is being used
6570

6671
// copy text into custom buffer
67-
auto inputText = createInputText(_sequence, omegaSorting, mirrorInput);
72+
auto inputText = createInputText(_sequence, omegaSorting, includeReversedInput);
6873

6974
// create bwt, bwtRev and annotatedArray
7075
auto [_bwt, _annotatedArray] = createBWTAndAnnotatedArray(inputText, _annotatedSequence, _threadNbr, omegaSorting);
@@ -97,15 +102,10 @@ struct BiFMIndex {
97102
*
98103
* \param _input a list of sequences
99104
* \param samplingRate rate of the sampling
105+
* \param includeReversedInput also adds all input and their reversed text
100106
*/
101-
BiFMIndex(Sequences auto const& _input, size_t samplingRate, size_t threadNbr, size_t seqOffset = 0, bool mirrorInput = false) {
102-
auto [totalSize, inputText, inputSizes] = [&]() {
103-
if (TDelim) {
104-
return createSequences(_input);
105-
} else {
106-
return createSequencesWithoutDelimiter(_input);
107-
}
108-
}();
107+
BiFMIndex(Sequences auto const& _input, size_t samplingRate, size_t threadNbr, size_t seqOffset = 0, bool includeReversedInput = false) {
108+
auto [totalSize, inputText, inputSizes] = createSequences(_input, /*._addReversed=*/includeReversedInput, /*._useDelimiters=*/TDelim);
109109

110110
size_t refId{0};
111111
size_t pos{0};
@@ -116,13 +116,10 @@ struct BiFMIndex {
116116
assert(inputSizes.size() < refId);
117117
}
118118

119-
auto size = totalSize;
120-
if (mirrorInput) size = size*2;
121-
122119

123120
auto annotatedSequence = SparseArray {
124-
std::views::iota(size_t{0}, size) | std::views::transform([&](size_t phase) -> std::optional<ADEntry> {
125-
if (phase < totalSize) { // going forward
121+
std::views::iota(size_t{0}, totalSize) | std::views::transform([&](size_t phase) -> std::optional<ADEntry> {
122+
if (!includeReversedInput || phase*2 < totalSize) { // going forward
126123
assert(refId < inputSizes.size());
127124
assert(pos < inputSizes[refId]);
128125

@@ -139,22 +136,20 @@ struct BiFMIndex {
139136
}
140137
return ret;
141138
} else { // going backwards
142-
if (phase == totalSize) { // reset values
143-
pos = 0;
144-
refId = 0;
145-
}
146-
147139
assert(refId < inputSizes.size());
148-
assert(pos < inputSizes[inputSizes.size() - refId - 1]);
140+
assert(pos < inputSizes[refId]);
149141

150142
auto ret = std::optional<ADEntry>{std::nullopt};
151143

152144
if (pos % samplingRate == 0) {
153-
ret = std::make_tuple(inputSizes.size()*2 - refId+seqOffset-1, inputSizes[refId] - pos-1);
145+
auto _refId = _input.size() + inputSizes.size() - refId-1+seqOffset;
146+
size_t extra = TDelim?1:0;
147+
auto _pos = (inputSizes[refId] - pos + inputSizes[refId] - 1 - extra) % inputSizes[refId];
148+
ret = std::make_tuple(_refId, _pos);
154149
}
155150

156151
++pos;
157-
if (inputSizes[inputSizes.size() - refId - 1] == pos) {
152+
if (inputSizes[refId] == pos) {
158153
refId += 1;
159154
pos = 0;
160155
}
@@ -163,7 +158,7 @@ struct BiFMIndex {
163158
})
164159
};
165160

166-
*this = BiFMIndex{inputText, annotatedSequence, threadNbr, mirrorInput};
161+
*this = BiFMIndex{inputText, annotatedSequence, threadNbr, /*includeReversedInput=*/false};
167162
}
168163

169164
auto operator=(BiFMIndex const&) -> BiFMIndex& = delete;

‎src/fmindex-collection/fmindex/VariableFMIndex.h‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ struct VariableFMIndex {
8383
query[i] = charToRankMapping[_query[i]];
8484
}
8585

86-
auto result = std::vector<std::tuple<ADEntry, size_t>>{};
86+
auto result = std::vector<std::tuple<size_t, size_t>>{};
8787

8888
// check for invalid mapping characters
8989
{
@@ -110,7 +110,8 @@ struct VariableFMIndex {
110110
} else {
111111
auto cursor = search_no_errors::search(index, query);
112112
for (auto [entry, offset] : LocateLinear{index, cursor}) {
113-
result.emplace_back(entry, offset);
113+
auto [docId, pos] = entry;
114+
result.emplace_back(docId, pos + offset);
114115
}
115116
}
116117
}, index);
@@ -122,7 +123,8 @@ struct VariableFMIndex {
122123
search_backtracking::search(index, query, k, [&](auto cursor, auto errors) {
123124
(void)errors;
124125
for (auto [entry, offset] : LocateLinear{index, cursor}) {
125-
result.emplace_back(entry, offset);
126+
auto [docId, pos] = entry;
127+
result.emplace_back(docId, pos + offset);
126128
}
127129
});
128130
}

‎src/fmindex-collection/string/WrappedBitvector.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ template <Bitvector_c Bitvector>
2222
struct WrappedBitvectorImpl<2, Bitvector> {
2323
static constexpr size_t Sigma = 2;
2424

25+
template <size_t Sigma2>
26+
using RmSigma = WrappedBitvectorImpl<Sigma2, Bitvector>;
27+
2528
Bitvector bitvector;
2629

2730
WrappedBitvectorImpl(std::span<uint8_t const> _symbols)

‎src/fmindex-collection/ternarylogic.h‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -638,6 +638,25 @@ auto mark_exact_v3(size_t value, std::bitset<N> const& _a, std::bitset<N> const&
638638
return {};
639639
};
640640

641+
/** Computes for each bit position (seen as spread over _a, _b and _c with _a being the most significant bit) if it
642+
* has the same bit value as Value
643+
*/
644+
template <size_t N>
645+
auto mark_exact_v3(size_t value, std::bitset<N> const& _b, std::bitset<N> const& _c) -> std::bitset<N> {
646+
assert(value < 4);
647+
648+
switch(value) {
649+
case 0: return ~_b & ~_c;
650+
case 1: return ~_b & _c;
651+
case 2: return _b & ~_c;
652+
case 3: return _b & _c;
653+
};
654+
// unreachable();
655+
// __builtin_unreachable();
656+
return {};
657+
};
658+
659+
641660
template <size_t N>
642661
auto mark_exact_all(std::bitset<N> const& _a, std::bitset<N> const& _b, std::bitset<N> const& _c) -> std::array<std::bitset<N>, 8> {
643662
auto r = std::array<std::bitset<N>, 8>{};
@@ -709,7 +728,9 @@ static std::array<std::bitset<N>, 2> mask_positive_or_negative = []() {
709728
*/
710729
template <size_t N1, size_t N2>
711730
auto mark_exact_large(size_t value, std::array<std::bitset<N1>, N2> const& _arr) -> std::bitset<N1> {
712-
if constexpr (N2 == 3) {
731+
if constexpr (N2 == 2) {
732+
return mark_exact_v3(value, _arr[1], _arr[0]);
733+
} else if constexpr (N2 == 3) {
713734
return mark_exact_v3(value, _arr[2], _arr[1], _arr[0]);
714735
} else {
715736
auto const& mask = mask_positive_or_negative<N1>;

‎src/fmindex-collection/utils.h‎

Lines changed: 38 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -237,29 +237,38 @@ inline auto createBWT(std::span<uint8_t const> inputText, size_t _threadNbr, boo
237237
}
238238
}
239239

240-
inline auto createInputText(std::span<uint8_t const> _input, bool _omegaSorting, bool _mirrorInput=false) -> std::vector<uint8_t> {
240+
inline auto createInputText(std::span<uint8_t const> _input, bool _omegaSorting, bool _includeReversedInput=false) -> std::vector<uint8_t> {
241241
auto output = std::vector<uint8_t>{};
242-
if (_omegaSorting && _mirrorInput) {
242+
243+
if (_omegaSorting && _includeReversedInput) {
244+
// ABC -> ABC CBA ABC CBA
245+
// AB$ -> AB$ $BA AB$ BA$
243246
output.resize(_input.size()*4);
244247
for (size_t i{0}; i < _input.size(); ++i) {
245248
output[_input.size()*0+i] = _input[i];
246249
output[_input.size()*2-i-1] = _input[i];
247250
output[_input.size()*2+i] = _input[i];
248251
output[_input.size()*4-i-1] = _input[i];
249252
}
250-
} else if (_mirrorInput) {
253+
} else if (_includeReversedInput) {
254+
// ABC -> ABC CBA
255+
// AB$ -> AB$ $BA
251256
output.resize(_input.size()*2);
252257
for (size_t i{0}; i < _input.size(); ++i) {
253258
output[_input.size()*0+i] = _input[i];
254259
output[_input.size()*2-i-1] = _input[i];
255260
}
256261
} else if (_omegaSorting) {
262+
// ABC -> ABC ABC
263+
// AB$ -> AB$ AB$
257264
output.resize(_input.size()*2);
258265
for (size_t i{0}; i < _input.size(); ++i) {
259266
output[_input.size()*0+i] = _input[i];
260267
output[_input.size()*1+i] = _input[i];
261268
}
262269
} else {
270+
// ABC -> ABC
271+
// AB$ -> AB$
263272
output.resize(_input.size());
264273
for (size_t i{0}; i < _input.size(); ++i) {
265274
output[i] = _input[i];
@@ -300,13 +309,19 @@ auto createSequences(Sequences auto const& _input, bool reverse=false) -> std::t
300309
return {totalSize, inputText, inputSizes};
301310
}
302311

303-
auto createSequencesAndReverse(Sequences auto const& _input) -> std::tuple<size_t, std::vector<uint8_t>, std::vector<size_t>> {
312+
auto createSequences(Sequences auto const& _input, bool _addReversed, bool _useDelimiters) -> std::tuple<size_t, std::vector<uint8_t>, std::vector<size_t>> {
304313
// compute total numbers of bytes of the text including delimiters "$"
305314
size_t totalSize{};
306315
for (auto const& l : _input) {
307-
totalSize += l.size()+1;
316+
totalSize += l.size();
317+
if (_useDelimiters) {
318+
totalSize += 1;
319+
}
320+
}
321+
322+
if (_addReversed) {
323+
totalSize = totalSize*2; // including reverse text
308324
}
309-
totalSize = totalSize*2; // including reverse text
310325

311326
// our concatenated sequences with delimiters
312327
auto inputText = std::vector<uint8_t>{};
@@ -320,21 +335,28 @@ auto createSequencesAndReverse(Sequences auto const& _input) -> std::tuple<size_
320335
for (auto const& l : _input) {
321336
inputText.insert(inputText.end(), begin(l), end(l));
322337

323-
// fill with delimiters/zeros
324-
inputText.resize(inputText.size() + 1);
338+
// insert size of input text
339+
inputSizes.emplace_back(l.size());
325340

326-
inputSizes.emplace_back(l.size()+1);
341+
if (_useDelimiters) {
342+
inputText.resize(inputText.size()+ 1);
343+
inputSizes.back() += 1;
344+
}
327345
}
328346

329-
// add reversed text
330-
for (auto const& l : std::views::reverse(_input)) {
331-
auto l2 = std::views::reverse(l);
332-
inputText.insert(inputText.end(), begin(l2), end(l2));
347+
if (_addReversed) {
348+
// add reversed text
349+
for (auto const& l : std::views::reverse(_input)) {
350+
auto l2 = std::views::reverse(l);
351+
inputText.insert(inputText.end(), begin(l2), end(l2));
333352

334-
// fill with delimiters/zeros
335-
inputText.resize(inputText.size() + 1);
353+
inputSizes.emplace_back(l.size());
336354

337-
inputSizes.emplace_back(l.size()+1);
355+
if (_useDelimiters) {
356+
inputText.resize(inputText.size()+ 1);
357+
inputSizes.back() += 1;
358+
}
359+
}
338360
}
339361

340362
return {totalSize, inputText, inputSizes};

0 commit comments

Comments
 (0)