Skip to content

Commit b9573b3

Browse files
parkertimminsparkertimmins
authored
Add template_id to patterned-text type (#131401)
For patterned-text mapper foo, add a sub-field called foo.template_id. This is the an 8-byte hash of the template doc_value column. Unlike the template, template_id is accessible and can be used for querying, aggregations, etc. The template_id is stored as a KeywordField and can use any features of the KeyworkFieldType. template_id has doc_values, and is not stored or indexed. It uses doc value skippers, which should be quite fast given that the index will be sorted on template.
1 parent 631cbac commit b9573b3

File tree

7 files changed

+420
-41
lines changed

7 files changed

+420
-41
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 56 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,8 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
209209
private final IndexAnalyzers indexAnalyzers;
210210
private final ScriptCompiler scriptCompiler;
211211
private final IndexVersion indexCreatedVersion;
212-
private final boolean useDocValuesSkipper;
212+
private final boolean enableDocValuesSkipper;
213+
private final boolean forceDocValuesSkipper;
213214
private final SourceKeepMode indexSourceKeepMode;
214215

215216
public Builder(final String name, final MappingParserContext mappingParserContext) {
@@ -222,6 +223,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex
222223
mappingParserContext.getIndexSettings().getMode(),
223224
mappingParserContext.getIndexSettings().getIndexSortConfig(),
224225
USE_DOC_VALUES_SKIPPER.get(mappingParserContext.getSettings()),
226+
false,
225227
mappingParserContext.getIndexSettings().sourceKeepMode()
226228
);
227229
}
@@ -243,6 +245,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex
243245
IndexMode.STANDARD,
244246
null,
245247
false,
248+
false,
246249
sourceKeepMode
247250
);
248251
}
@@ -255,7 +258,8 @@ private Builder(
255258
IndexVersion indexCreatedVersion,
256259
IndexMode indexMode,
257260
IndexSortConfig indexSortConfig,
258-
boolean useDocValuesSkipper,
261+
boolean enableDocValuesSkipper,
262+
boolean forceDocValuesSkipper,
259263
SourceKeepMode indexSourceKeepMode
260264
) {
261265
super(name);
@@ -293,14 +297,36 @@ private Builder(
293297
});
294298
this.indexSortConfig = indexSortConfig;
295299
this.indexMode = indexMode;
296-
this.useDocValuesSkipper = useDocValuesSkipper;
300+
this.enableDocValuesSkipper = enableDocValuesSkipper;
301+
this.forceDocValuesSkipper = forceDocValuesSkipper;
297302
this.indexSourceKeepMode = indexSourceKeepMode;
298303
}
299304

300305
public Builder(String name, IndexVersion indexCreatedVersion) {
301306
this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion, SourceKeepMode.NONE);
302307
}
303308

309+
public static Builder buildWithDocValuesSkipper(
310+
String name,
311+
IndexMode indexMode,
312+
IndexVersion indexCreatedVersion,
313+
boolean enableDocValuesSkipper
314+
) {
315+
return new Builder(
316+
name,
317+
null,
318+
ScriptCompiler.NONE,
319+
Integer.MAX_VALUE,
320+
indexCreatedVersion,
321+
indexMode,
322+
// Sort config is used to decide if DocValueSkippers can be used. Since skippers are forced, a sort config is not needed.
323+
null,
324+
enableDocValuesSkipper,
325+
true,
326+
SourceKeepMode.NONE
327+
);
328+
}
329+
304330
public Builder ignoreAbove(int ignoreAbove) {
305331
this.ignoreAbove.setValue(ignoreAbove);
306332
return this;
@@ -422,7 +448,9 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType
422448
@Override
423449
public KeywordFieldMapper build(MapperBuilderContext context) {
424450
FieldType fieldtype = resolveFieldType(
425-
useDocValuesSkipper,
451+
enableDocValuesSkipper,
452+
forceDocValuesSkipper,
453+
hasDocValues,
426454
indexCreatedVersion,
427455
indexSortConfig,
428456
indexMode,
@@ -460,24 +488,30 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
460488
buildFieldType(context, fieldtype),
461489
builderParams(this, context),
462490
context.isSourceSynthetic(),
463-
useDocValuesSkipper,
464491
this,
465492
offsetsFieldName,
466493
indexSourceKeepMode
467494
);
468495
}
469496

470-
private FieldType resolveFieldType(
471-
final boolean useDocValuesSkipper,
497+
private static FieldType resolveFieldType(
498+
final boolean enableDocValuesSkipper,
499+
final boolean forceDocValuesSkipper,
500+
final Parameter<Boolean> hasDocValues,
472501
final IndexVersion indexCreatedVersion,
473502
final IndexSortConfig indexSortConfig,
474503
final IndexMode indexMode,
475504
final String fullFieldName
476505
) {
477-
if (useDocValuesSkipper
478-
&& indexCreatedVersion.onOrAfter(IndexVersions.HOSTNAME_DOC_VALUES_SPARSE_INDEX)
479-
&& shouldUseDocValuesSkipper(hasDocValues.getValue(), indexSortConfig, indexMode, fullFieldName)) {
480-
return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES);
506+
if (enableDocValuesSkipper) {
507+
if (forceDocValuesSkipper) {
508+
assert hasDocValues.getValue();
509+
return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES);
510+
}
511+
if (indexCreatedVersion.onOrAfter(IndexVersions.HOSTNAME_DOC_VALUES_SPARSE_INDEX)
512+
&& shouldUseDocValuesSkipper(hasDocValues.getValue(), indexSortConfig, indexMode, fullFieldName)) {
513+
return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES);
514+
}
481515
}
482516
return new FieldType(Defaults.FIELD_TYPE);
483517
}
@@ -1088,7 +1122,8 @@ public String originalName() {
10881122
private final int ignoreAboveDefault;
10891123
private final IndexMode indexMode;
10901124
private final IndexSortConfig indexSortConfig;
1091-
private final boolean useDocValuesSkipper;
1125+
private final boolean enableDocValuesSkipper;
1126+
private final boolean forceDocValuesSkipper;
10921127
private final String offsetsFieldName;
10931128
private final SourceKeepMode indexSourceKeepMode;
10941129
private final String originalName;
@@ -1099,7 +1134,6 @@ private KeywordFieldMapper(
10991134
KeywordFieldType mappedFieldType,
11001135
BuilderParams builderParams,
11011136
boolean isSyntheticSource,
1102-
boolean useDocValuesSkipper,
11031137
Builder builder,
11041138
String offsetsFieldName,
11051139
SourceKeepMode indexSourceKeepMode
@@ -1120,7 +1154,8 @@ private KeywordFieldMapper(
11201154
this.ignoreAboveDefault = builder.ignoreAboveDefault;
11211155
this.indexMode = builder.indexMode;
11221156
this.indexSortConfig = builder.indexSortConfig;
1123-
this.useDocValuesSkipper = useDocValuesSkipper;
1157+
this.enableDocValuesSkipper = builder.enableDocValuesSkipper;
1158+
this.forceDocValuesSkipper = builder.forceDocValuesSkipper;
11241159
this.offsetsFieldName = offsetsFieldName;
11251160
this.indexSourceKeepMode = indexSourceKeepMode;
11261161
this.originalName = mappedFieldType.originalName();
@@ -1219,7 +1254,7 @@ private boolean indexValue(DocumentParserContext context, XContentString value)
12191254
throw new IllegalArgumentException(msg);
12201255
}
12211256

1222-
Field field = new KeywordField(fieldType().name(), binaryValue, fieldType);
1257+
Field field = buildKeywordField(binaryValue);
12231258
context.doc().add(field);
12241259

12251260
if (fieldType().hasDocValues() == false && fieldType.omitNorms()) {
@@ -1276,11 +1311,16 @@ public FieldMapper.Builder getMergeBuilder() {
12761311
indexCreatedVersion,
12771312
indexMode,
12781313
indexSortConfig,
1279-
useDocValuesSkipper,
1314+
enableDocValuesSkipper,
1315+
forceDocValuesSkipper,
12801316
indexSourceKeepMode
12811317
).dimension(fieldType().isDimension()).init(this);
12821318
}
12831319

1320+
public Field buildKeywordField(BytesRef binaryValue) {
1321+
return new KeywordField(fieldType().name(), binaryValue, fieldType);
1322+
}
1323+
12841324
@Override
12851325
public void doValidate(MappingLookup lookup) {
12861326
if (fieldType().isDimension() && null != lookup.nestedLookup().getNestedParent(fullPath())) {

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java‎

Lines changed: 63 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,28 @@
1313
import org.apache.lucene.index.IndexOptions;
1414
import org.apache.lucene.util.BytesRef;
1515
import org.elasticsearch.common.util.FeatureFlag;
16+
import org.elasticsearch.index.IndexSettings;
1617
import org.elasticsearch.index.IndexVersion;
1718
import org.elasticsearch.index.analysis.IndexAnalyzers;
1819
import org.elasticsearch.index.analysis.NamedAnalyzer;
1920
import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
2021
import org.elasticsearch.index.mapper.DocumentParserContext;
2122
import org.elasticsearch.index.mapper.FieldMapper;
23+
import org.elasticsearch.index.mapper.KeywordFieldMapper;
24+
import org.elasticsearch.index.mapper.Mapper;
2225
import org.elasticsearch.index.mapper.MapperBuilderContext;
26+
import org.elasticsearch.index.mapper.MappingParserContext;
2327
import org.elasticsearch.index.mapper.TextParams;
2428
import org.elasticsearch.index.mapper.TextSearchInfo;
2529

2630
import java.io.IOException;
31+
import java.util.ArrayList;
32+
import java.util.Iterator;
33+
import java.util.List;
2734
import java.util.Map;
2835

36+
import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER;
37+
2938
/**
3039
* A {@link FieldMapper} that assigns every document the same value.
3140
*/
@@ -50,20 +59,38 @@ public static class Defaults {
5059
public static class Builder extends FieldMapper.Builder {
5160

5261
private final IndexVersion indexCreatedVersion;
53-
62+
private final IndexSettings indexSettings;
5463
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
55-
5664
private final TextParams.Analyzers analyzers;
65+
private final boolean enableDocValuesSkipper;
66+
67+
public Builder(String name, MappingParserContext context) {
68+
this(
69+
name,
70+
context.indexVersionCreated(),
71+
context.getIndexSettings(),
72+
context.getIndexAnalyzers(),
73+
USE_DOC_VALUES_SKIPPER.get(context.getSettings())
74+
);
75+
}
5776

58-
public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers) {
77+
public Builder(
78+
String name,
79+
IndexVersion indexCreatedVersion,
80+
IndexSettings indexSettings,
81+
IndexAnalyzers indexAnalyzers,
82+
boolean enableDocValuesSkipper
83+
) {
5984
super(name);
6085
this.indexCreatedVersion = indexCreatedVersion;
86+
this.indexSettings = indexSettings;
6187
this.analyzers = new TextParams.Analyzers(
6288
indexAnalyzers,
6389
m -> ((PatternedTextFieldMapper) m).indexAnalyzer,
6490
m -> ((PatternedTextFieldMapper) m).positionIncrementGap,
6591
indexCreatedVersion
6692
);
93+
this.enableDocValuesSkipper = enableDocValuesSkipper;
6794
}
6895

6996
@Override
@@ -87,23 +114,35 @@ private PatternedTextFieldType buildFieldType(MapperBuilderContext context) {
87114

88115
@Override
89116
public PatternedTextFieldMapper build(MapperBuilderContext context) {
90-
return new PatternedTextFieldMapper(leafName(), buildFieldType(context), builderParams(this, context), this);
117+
PatternedTextFieldType patternedTextFieldType = buildFieldType(context);
118+
BuilderParams builderParams = builderParams(this, context);
119+
var templateIdMapper = KeywordFieldMapper.Builder.buildWithDocValuesSkipper(
120+
patternedTextFieldType.templateIdFieldName(),
121+
indexSettings.getMode(),
122+
indexCreatedVersion,
123+
enableDocValuesSkipper
124+
).build(context);
125+
return new PatternedTextFieldMapper(leafName(), patternedTextFieldType, builderParams, this, templateIdMapper);
91126
}
92127
}
93128

94-
public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers()));
129+
public static final TypeParser PARSER = new TypeParser(Builder::new);
95130

96131
private final IndexVersion indexCreatedVersion;
97132
private final IndexAnalyzers indexAnalyzers;
133+
private final IndexSettings indexSettings;
98134
private final NamedAnalyzer indexAnalyzer;
135+
private final boolean enableDocValuesSkipper;
99136
private final int positionIncrementGap;
100137
private final FieldType fieldType;
138+
private final KeywordFieldMapper templateIdMapper;
101139

102140
private PatternedTextFieldMapper(
103141
String simpleName,
104142
PatternedTextFieldType mappedFieldPatternedTextFieldType,
105143
BuilderParams builderParams,
106-
Builder builder
144+
Builder builder,
145+
KeywordFieldMapper templateIdMapper
107146
) {
108147
super(simpleName, mappedFieldPatternedTextFieldType, builderParams);
109148
assert mappedFieldPatternedTextFieldType.getTextSearchInfo().isTokenized();
@@ -112,7 +151,10 @@ private PatternedTextFieldMapper(
112151
this.indexCreatedVersion = builder.indexCreatedVersion;
113152
this.indexAnalyzers = builder.analyzers.indexAnalyzers;
114153
this.indexAnalyzer = builder.analyzers.getIndexAnalyzer();
154+
this.indexSettings = builder.indexSettings;
155+
this.enableDocValuesSkipper = builder.enableDocValuesSkipper;
115156
this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue();
157+
this.templateIdMapper = templateIdMapper;
116158
}
117159

118160
@Override
@@ -122,7 +164,18 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {
122164

123165
@Override
124166
public FieldMapper.Builder getMergeBuilder() {
125-
return new Builder(leafName(), indexCreatedVersion, indexAnalyzers).init(this);
167+
return new Builder(leafName(), indexCreatedVersion, indexSettings, indexAnalyzers, enableDocValuesSkipper).init(this);
168+
}
169+
170+
@Override
171+
public Iterator<Mapper> iterator() {
172+
List<Mapper> mappers = new ArrayList<>();
173+
Iterator<Mapper> m = super.iterator();
174+
while (m.hasNext()) {
175+
mappers.add(m.next());
176+
}
177+
mappers.add(templateIdMapper);
178+
return mappers.iterator();
126179
}
127180

128181
@Override
@@ -146,6 +199,9 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
146199
// Add template doc_values
147200
context.doc().add(new SortedSetDocValuesField(fieldType().templateFieldName(), new BytesRef(parts.template())));
148201

202+
// Add template_id doc_values
203+
context.doc().add(templateIdMapper.buildKeywordField(new BytesRef(parts.templateId())));
204+
149205
// Add args doc_values
150206
if (parts.args().isEmpty() == false) {
151207
String remainingArgs = PatternedTextValueProcessor.encodeRemainingArgs(parts);

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
public class PatternedTextFieldType extends StringFieldType {
5656

5757
private static final String TEMPLATE_SUFFIX = ".template";
58+
private static final String TEMPLATE_ID_SUFFIX = ".template_id";
5859
private static final String ARGS_SUFFIX = ".args";
5960

6061
public static final String CONTENT_TYPE = "patterned_text";
@@ -145,7 +146,7 @@ public Query fuzzyQuery(
145146

146147
@Override
147148
public Query existsQuery(SearchExecutionContext context) {
148-
return new FieldExistsQuery(templateFieldName());
149+
return new FieldExistsQuery(templateIdFieldName());
149150
}
150151

151152
@Override
@@ -263,6 +264,10 @@ String templateFieldName() {
263264
return name() + TEMPLATE_SUFFIX;
264265
}
265266

267+
String templateIdFieldName() {
268+
return name() + TEMPLATE_ID_SUFFIX;
269+
}
270+
266271
String argsFieldName() {
267272
return name() + ARGS_SUFFIX;
268273
}

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextValueProcessor.java‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@
77

88
package org.elasticsearch.xpack.logsdb.patternedtext;
99

10+
import org.elasticsearch.common.Strings;
11+
import org.elasticsearch.common.hash.MurmurHash3;
12+
import org.elasticsearch.common.util.ByteUtils;
13+
14+
import java.nio.charset.StandardCharsets;
1015
import java.util.ArrayList;
1116
import java.util.Arrays;
1217
import java.util.List;
@@ -16,7 +21,20 @@ public class PatternedTextValueProcessor {
1621
private static final String DELIMITER = "[\\s\\[\\]]";
1722
private static final String SPACE = " ";
1823

19-
record Parts(String template, List<String> args) {}
24+
record Parts(String template, String templateId, List<String> args) {
25+
Parts(String template, List<String> args) {
26+
this(template, PatternedTextValueProcessor.templateId(template), args);
27+
}
28+
}
29+
30+
static String templateId(String template) {
31+
byte[] bytes = template.getBytes(StandardCharsets.UTF_8);
32+
MurmurHash3.Hash128 hash = new MurmurHash3.Hash128();
33+
MurmurHash3.hash128(bytes, 0, bytes.length, 0, hash);
34+
byte[] hashBytes = new byte[8];
35+
ByteUtils.writeLongLE(hash.h1, hashBytes, 0);
36+
return Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(hashBytes);
37+
}
2038

2139
static Parts split(String text) {
2240
StringBuilder template = new StringBuilder();

0 commit comments

Comments
 (0)