Skip to content

Commit 4e7087c

Browse files
stroomdev66stroomdev66
committed
#4093 Fix value type for stored values
1 parent d3c5a98 commit 4e7087c

File tree

11 files changed

+273
-230
lines changed

11 files changed

+273
-230
lines changed

Diff for: ‎stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexingFilter.java

+39-84
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,11 @@
3636
import stroom.pipeline.shared.data.PipelineElementType.Category;
3737
import stroom.pipeline.state.MetaHolder;
3838
import stroom.query.common.v2.IndexFieldCache;
39-
import stroom.query.language.functions.ValString;
39+
import stroom.query.language.functions.Val;
4040
import stroom.search.extraction.FieldValue;
41+
import stroom.search.extraction.IndexFieldUtil;
4142
import stroom.svg.shared.SvgImage;
4243
import stroom.util.CharBuffer;
43-
import stroom.util.date.DateUtil;
4444
import stroom.util.shared.Severity;
4545

4646
import jakarta.inject.Inject;
@@ -245,97 +245,52 @@ private void processDocument() {
245245

246246
private void processIndexContent(final IndexField indexField, final String value) {
247247
try {
248-
if (currentEventTime == null &&
249-
FieldType.DATE.equals(indexField.getFldType()) &&
250-
indexField.getFldName().equals(index.getTimeField())) {
251-
try {
252-
// Set the current event time if this is a recognised event time field.
253-
currentEventTime = DateUtil.parseUnknownString(value);
254-
} catch (final RuntimeException e) {
255-
LOGGER.trace(e.getMessage(), e);
248+
final Val val = convertValue(indexField, value);
249+
if (val != null) {
250+
if (currentEventTime == null &&
251+
FieldType.DATE.equals(indexField.getFldType()) &&
252+
indexField.getFldName().equals(index.getTimeField())) {
253+
try {
254+
// Set the current event time if this is a recognised event time field.
255+
currentEventTime = val.toLong();
256+
} catch (final RuntimeException e) {
257+
LOGGER.trace(e.getMessage(), e);
258+
}
256259
}
257-
}
258260

261+
// Add the current field to the document if it is not null.
262+
final FieldValue fieldValue = new FieldValue(indexField, val);
263+
264+
// Output some debug.
265+
if (LOGGER.isDebugEnabled()) {
266+
debugBuffer.append("processIndexContent() - Adding to index indexName=");
267+
debugBuffer.append(indexRef.getName());
268+
debugBuffer.append(" name=");
269+
debugBuffer.append(indexField.getFldName());
270+
debugBuffer.append(" value=");
271+
debugBuffer.append(value);
272+
273+
final String debug = debugBuffer.toString();
274+
debugBuffer.clear();
275+
276+
LOGGER.debug(debug);
277+
}
259278

260-
// Field field = null;
261-
//
262-
// if (IndexFieldType.INTEGER_FIELD.equals(indexField.getFieldType())) {
263-
// FieldValue fieldValue = new FieldValue(indexField, ValString.create(value));
264-
//
265-
// try {
266-
// final int val = Integer.parseInt(value);
267-
// field = FieldFactory.createInt(indexField, val);
268-
// } catch (final Exception e) {
269-
// LOGGER.trace(e.getMessage(), e);
270-
// }
271-
// } else if (IndexFieldType.LONG_FIELD.equals(indexField.getFieldType())) {
272-
// try {
273-
// final long val = Long.parseLong(value);
274-
// field = FieldFactory.create(indexField, val);
275-
// } catch (final Exception e) {
276-
// LOGGER.trace(e.getMessage(), e);
277-
// }
278-
// } else if (IndexFieldType.FLOAT_FIELD.equals(indexField.getFieldType())) {
279-
// try {
280-
// final float val = Float.parseFloat(value);
281-
// field = FieldFactory.createFloat(indexField, val);
282-
// } catch (final Exception e) {
283-
// LOGGER.trace(e.getMessage(), e);
284-
// }
285-
// } else if (IndexFieldType.DOUBLE_FIELD.equals(indexField.getFieldType())) {
286-
// try {
287-
// final double val = Double.parseDouble(value);
288-
// field = FieldFactory.createDouble(indexField, val);
289-
// } catch (final Exception e) {
290-
// LOGGER.trace(e.getMessage(), e);
291-
// }
292-
// } else if (IndexFieldType.DATE_FIELD.equals(indexField.getFieldType())) {
293-
// try {
294-
// final long val = DateUtil.parseUnknownString(value);
295-
//
296-
// // Set the current event time if this is a recognised event time field.
297-
// if (currentEventTime == null && indexField.getFieldName().equals(index.getTimeField())) {
298-
// currentEventTime = val;
299-
// }
300-
//
301-
// field = FieldFactory.create(indexField, val);
302-
// } catch (final RuntimeException e) {
303-
// LOGGER.trace(e.getMessage(), e);
304-
// }
305-
// } else if (indexField.getFieldType().isNumeric()) {
306-
// try {
307-
// final long val = Long.parseLong(value);
308-
// field = FieldFactory.create(indexField, val);
309-
// } catch (final Exception e) {
310-
// LOGGER.trace(e.getMessage(), e);
311-
// }
312-
// } else {
313-
// field = FieldFactory.create(indexField, value);
314-
// }
315-
316-
// Add the current field to the document if it is not null.
317-
final FieldValue fieldValue = new FieldValue(indexField, ValString.create(value));
318-
319-
// Output some debug.
320-
if (LOGGER.isDebugEnabled()) {
321-
debugBuffer.append("processIndexContent() - Adding to index indexName=");
322-
debugBuffer.append(indexRef.getName());
323-
debugBuffer.append(" name=");
324-
debugBuffer.append(indexField.getFldName());
325-
debugBuffer.append(" value=");
326-
debugBuffer.append(value);
327-
328-
final String debug = debugBuffer.toString();
329-
debugBuffer.clear();
330-
331-
LOGGER.debug(debug);
279+
document.add(fieldValue);
332280
}
333281

334-
document.add(fieldValue);
282+
} catch (final RuntimeException e) {
283+
log(Severity.ERROR, e.getMessage(), e);
284+
}
285+
}
335286

287+
private Val convertValue(final IndexField indexField, final String value) {
288+
try {
289+
return IndexFieldUtil.convertValue(indexField, value);
336290
} catch (final RuntimeException e) {
337291
log(Severity.ERROR, e.getMessage(), e);
338292
}
293+
return null;
339294
}
340295

341296
@PipelineProperty(description = "The index to send records to.", displayPriority = 1)

Diff for: ‎stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/LuceneShardSearcher.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,20 @@
11
package stroom.index.impl;
22

3+
import stroom.datasource.api.v2.IndexField;
34
import stroom.index.shared.IndexShard;
45
import stroom.query.language.functions.ValuesConsumer;
56
import stroom.query.language.functions.ref.ErrorConsumer;
67
import stroom.task.api.TaskContext;
78

9+
import java.util.Set;
810
import java.util.concurrent.atomic.LongAdder;
911

1012
public interface LuceneShardSearcher {
1113

1214
void searchShard(final TaskContext taskContext,
1315
final IndexShard indexShard,
14-
final String[] storedFieldNames,
16+
final IndexField[] storedFields,
17+
final Set<String> fieldsToLoad,
1518
final LongAdder hitCount,
1619
final int shardNumber,
1720
final int shardTotal,

Diff for: ‎stroom-index/stroom-index-lucene553/src/main/java/stroom/index/lucene553/FieldFactory.java

+42
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,21 @@
1919
import stroom.datasource.api.v2.IndexField;
2020
import stroom.index.shared.LuceneIndexField;
2121
import stroom.query.language.functions.Val;
22+
import stroom.query.language.functions.ValBoolean;
23+
import stroom.query.language.functions.ValDate;
24+
import stroom.query.language.functions.ValDouble;
25+
import stroom.query.language.functions.ValFloat;
26+
import stroom.query.language.functions.ValInteger;
27+
import stroom.query.language.functions.ValLong;
28+
import stroom.query.language.functions.ValString;
2229
import stroom.search.extraction.FieldValue;
2330

2431
import org.apache.lucene553.document.DoubleField;
2532
import org.apache.lucene553.document.Field;
2633
import org.apache.lucene553.document.FloatField;
2734
import org.apache.lucene553.document.IntField;
2835
import org.apache.lucene553.document.LongField;
36+
import org.apache.lucene553.index.IndexableField;
2937
import org.slf4j.Logger;
3038
import org.slf4j.LoggerFactory;
3139

@@ -108,4 +116,38 @@ public static Field create(final FieldValue fieldValue) {
108116

109117
return field;
110118
}
119+
120+
public static Val convertValue(final IndexField indexField, final IndexableField indexableField) {
121+
switch (indexField.getFldType()) {
122+
case LONG, ID -> {
123+
final long val = indexableField.numericValue().longValue();
124+
return ValLong.create(val);
125+
}
126+
case BOOLEAN -> {
127+
// TODO : We are indexing boolean as String, not sure this is right.
128+
final boolean val = Boolean.parseBoolean(indexableField.stringValue());
129+
return ValBoolean.create(val);
130+
}
131+
case INTEGER -> {
132+
final int val = indexableField.numericValue().intValue();
133+
return ValInteger.create(val);
134+
}
135+
case FLOAT -> {
136+
final float val = indexableField.numericValue().floatValue();
137+
return ValFloat.create(val);
138+
}
139+
case DOUBLE -> {
140+
final double val = indexableField.numericValue().doubleValue();
141+
return ValDouble.create(val);
142+
}
143+
case DATE -> {
144+
final long val = indexableField.numericValue().longValue();
145+
return ValDate.create(val);
146+
}
147+
case TEXT -> {
148+
return ValString.create(indexableField.stringValue());
149+
}
150+
}
151+
return ValString.create(indexableField.stringValue());
152+
}
111153
}

Diff for: ‎stroom-index/stroom-index-lucene553/src/main/java/stroom/index/lucene553/Lucene553ShardSearcher.java

+26-17
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package stroom.index.lucene553;
1818

19+
import stroom.datasource.api.v2.IndexField;
1920
import stroom.dictionary.api.WordListProvider;
2021
import stroom.docref.DocRef;
2122
import stroom.expression.api.DateTimeSettings;
@@ -31,7 +32,6 @@
3132
import stroom.query.common.v2.SearchProgressLog;
3233
import stroom.query.common.v2.SearchProgressLog.SearchPhase;
3334
import stroom.query.language.functions.Val;
34-
import stroom.query.language.functions.ValString;
3535
import stroom.query.language.functions.ValuesConsumer;
3636
import stroom.query.language.functions.ref.ErrorConsumer;
3737
import stroom.search.impl.SearchException;
@@ -55,6 +55,7 @@
5555
import org.apache.lucene553.search.SearcherManager;
5656

5757
import java.io.IOException;
58+
import java.util.Set;
5859
import java.util.concurrent.CompletableFuture;
5960
import java.util.concurrent.Executor;
6061
import java.util.concurrent.atomic.LongAdder;
@@ -113,7 +114,8 @@ class Lucene553ShardSearcher implements LuceneShardSearcher {
113114
@Override
114115
public void searchShard(final TaskContext taskContext,
115116
final IndexShard indexShard,
116-
final String[] storedFieldNames,
117+
final IndexField[] storedFields,
118+
final Set<String> fieldsToLoad,
117119
final LongAdder hitCount,
118120
final int shardNumber,
119121
final int shardTotal,
@@ -134,7 +136,8 @@ public void searchShard(final TaskContext taskContext,
134136
// Start searching.
135137
searchShard(
136138
taskContext,
137-
storedFieldNames,
139+
storedFields,
140+
fieldsToLoad,
138141
hitCount,
139142
indexShardSearcher,
140143
valuesConsumer,
@@ -165,7 +168,8 @@ private IndexWriter getWriter(final Long indexShardId) {
165168
}
166169

167170
private void searchShard(final TaskContext parentContext,
168-
final String[] storedFieldNames,
171+
final IndexField[] storedFields,
172+
final Set<String> fieldsToLoad,
169173
final LongAdder hitCount,
170174
final IndexShardSearcher indexShardSearcher,
171175
final ValuesConsumer valuesConsumer,
@@ -244,7 +248,13 @@ private void searchShard(final TaskContext parentContext,
244248
// If we have a doc id then retrieve the stored data for it.
245249
SearchProgressLog.increment(queryKey,
246250
SearchPhase.INDEX_SHARD_SEARCH_TASK_HANDLER_DOC_ID_STORE_TAKE);
247-
getStoredData(storedFieldNames, valuesConsumer, searcher, docId, errorConsumer);
251+
getStoredData(
252+
storedFields,
253+
fieldsToLoad,
254+
valuesConsumer,
255+
searcher,
256+
docId,
257+
errorConsumer);
248258
} catch (final RuntimeException e) {
249259
error(errorConsumer, e);
250260
}
@@ -267,31 +277,30 @@ private void searchShard(final TaskContext parentContext,
267277
* only want to get stream and event ids, in these cases no values are
268278
* retrieved, only stream and event ids.
269279
*/
270-
private void getStoredData(final String[] storedFieldNames,
280+
private void getStoredData(final IndexField[] storedFields,
281+
final Set<String> fieldsToLoad,
271282
final ValuesConsumer valuesConsumer,
272283
final IndexSearcher searcher,
273284
final int docId,
274285
final ErrorConsumer errorConsumer) {
275286
try {
276287
SearchProgressLog.increment(queryKey, SearchPhase.INDEX_SHARD_SEARCH_TASK_HANDLER_GET_STORED_DATA);
277-
final Val[] values = new Val[storedFieldNames.length];
278-
final Document document = searcher.doc(docId);
288+
final Val[] values = new Val[storedFields.length];
289+
final Document document = searcher.getIndexReader().document(docId, fieldsToLoad);
279290

280-
for (int i = 0; i < storedFieldNames.length; i++) {
281-
final String storedField = storedFieldNames[i];
291+
for (int i = 0; i < storedFields.length; i++) {
292+
final IndexField storedField = storedFields[i];
282293

283294
// If the field is null then it isn't stored.
284295
if (storedField != null) {
285-
final IndexableField indexableField = document.getField(storedField);
296+
final IndexableField indexableField = document.getField(storedField.getFldName());
286297

287298
// If the field is not in fact stored then it will be null here.
288299
if (indexableField != null) {
289-
final String value = indexableField.stringValue();
290-
if (value != null) {
291-
final String trimmed = value.trim();
292-
if (trimmed.length() > 0) {
293-
values[i] = ValString.create(trimmed);
294-
}
300+
try {
301+
values[i] = FieldFactory.convertValue(storedField, indexableField);
302+
} catch (final RuntimeException e) {
303+
error(errorConsumer, e);
295304
}
296305
}
297306
}

0 commit comments

Comments
 (0)