Skip to content

Commit 65ceef1

Browse files
MangernMangern
authored
Merge pull request #35788 from vespa-engine/magnus/lsp-ranking-foreach
Magnus/lsp ranking foreach
2 parents b7b8491 + 65516c5 commit 65ceef1

21 files changed

Lines changed: 311 additions & 94 deletions

File tree

‎integration/schema-language-server/clients/vscode/README.md.in‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# VSCode extension for Schema Language Server
1+
# Extension for Vespa Schema Language Server
22
Language support for the Vespa Schema language using LSP.
33

44
The extension acts as a client for the Schema Language Server, providing tools for developing Vespa Schema files.
@@ -33,4 +33,6 @@ The extension also requires [Vespa CLI](https://docs.vespa.ai/en/clients/vespa-c
3333
## XML support
3434
This extension bundles with an extension to the [LemMinX XML Language server](https://github.com/eclipse/lemminx).
3535
This is to provide additional support when editing the services.xml file in Vespa applications.
36-
For the best possible experience, install the [VSCode XML extension](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-xml) as well.
36+
For the best possible experience, install the appropriate extension for your editor:
37+
- [VS Code](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-xml)
38+
- [Open VSX](https://open-vsx.org/extension/redhat/vscode-xml)

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/SchemaDiagnostic.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@ public static enum DiagnosticCode {
1515
ACCESS_UNIMPORTED_FIELD,
1616
DOCUMENTLESS_SCHEMA,
1717
DOCUMENT_REFERENCE_ATTRIBUTE,
18-
IMPORT_FIELD_ATTRIBUTE,
1918
EXPLICITLY_INHERIT_DOCUMENT,
19+
FOREACH_INVALID,
20+
IMPORT_FIELD_ATTRIBUTE,
2021
INHERITS_STRUCT_FIELD_REDECLARED,
2122
ANNOTATION_REFERENCE_OUTSIDE_ANNOTATION,
2223
DEPRECATED_ARRAY_SYNTAX,

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/index/Symbol.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ public enum SymbolType {
149149
DOCUMENT_SUMMARY,
150150
FIELD,
151151
FIELDSET,
152+
FOREACH,
152153
FUNCTION,
153154
LABEL,
154155
LAMBDA_FUNCTION,

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/hover/SchemaHover.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import org.eclipse.lsp4j.MarkupKind;
1515
import org.eclipse.lsp4j.Range;
1616

17-
import ai.vespa.schemals.SchemaLanguageServer;
1817
import ai.vespa.schemals.common.FileUtils;
1918
import ai.vespa.schemals.context.EventPositionContext;
2019
import ai.vespa.schemals.index.Symbol;
@@ -164,20 +163,23 @@ private static Hover getFieldHover(Node node, EventPositionContext context) {
164163
return new Hover(new MarkupContent(MarkupKind.MARKDOWN, "```\n" + hoverText + "\n```"));
165164
}
166165

167-
private static Hover getSymbolHover(Node node, EventPositionContext context) {
166+
private static Hover getSymbolHover(Node node, Path documentationPath, EventPositionContext context) {
168167
switch(node.getSymbol().getType()) {
169168
case STRUCT:
170169
return getStructHover(node, context);
171170
case FIELD:
172171
return getFieldHover(node, context);
172+
case FOREACH:
173+
return getFileHoverInformation(
174+
documentationPath.resolve("rankExpression"),
175+
"foreach(dimension,variable,feature,condition,operation)",
176+
node.getRange()).orElse(null);
177+
case LABEL:
178+
return new Hover(new MarkupContent(MarkupKind.PLAINTEXT, "label"));
173179
default:
174180
break;
175181
}
176182

177-
if (node.getSymbol().getType() == SymbolType.LABEL) {
178-
return new Hover(new MarkupContent(MarkupKind.PLAINTEXT, "label"));
179-
}
180-
181183
if (node.getSymbol().getStatus() == SymbolStatus.BUILTIN_REFERENCE) {
182184
return new Hover(new MarkupContent(MarkupKind.PLAINTEXT, "builtin"));
183185
}
@@ -299,7 +301,7 @@ public static Hover getHover(EventPositionContext context, Path documentationPat
299301

300302
if (node != null && node.isSchemaNode()) {
301303
SchemaNode schemaNode = node.getSchemaNode();
302-
Hover symbolHover = getSymbolHover(schemaNode, context);
304+
Hover symbolHover = getSymbolHover(schemaNode, documentationPath, context);
303305

304306
if (symbolHover == null) return null;
305307

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokenConfig.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,7 @@ class SchemaSemanticTokenConfig {
313313
put(SymbolType.DOCUMENT_SUMMARY, SemanticTokenTypes.Variable);
314314
put(SymbolType.FIELD, SemanticTokenTypes.Variable);
315315
put(SymbolType.FIELDSET, SemanticTokenTypes.Variable);
316+
put(SymbolType.FOREACH, SemanticTokenTypes.Macro);
316317
put(SymbolType.FUNCTION, SemanticTokenTypes.Function);
317318
put(SymbolType.LABEL, SemanticTokenTypes.Variable);
318319
put(SymbolType.LAMBDA_FUNCTION, SemanticTokenTypes.Keyword);

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ private static List<SemanticTokenMarker> findSemanticMarkersForSymbol(SchemaNode
235235
ret.add(tokenMarker);
236236
}
237237

238-
} else if (symbol.getStatus() == SymbolStatus.BUILTIN_REFERENCE) {
238+
} else if (symbol.getStatus() == SymbolStatus.BUILTIN_REFERENCE || symbol.getType() == SymbolType.FOREACH) {
239239
SymbolType type = symbol.getType();
240240
Integer tokenType = identifierTypeMap.get(type);
241241

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocument.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -114,9 +114,9 @@ public void updateFileContent(String content) {
114114
this.CST = parsingResult.CST().get();
115115
lexer.setCST(CST);
116116

117-
//logger.info("======== CST for file: " + fileURI + " ========");
118-
119-
//CSTUtils.printTree(logger, CST);
117+
// logger.info("======== CST for file: " + fileURI + " ========");
118+
119+
// CSTUtils.printTree(logger, CST);
120120
}
121121

122122
//schemaIndex.dumpIndex();

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifySymbolDefinition.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import ai.vespa.schemals.parser.ast.structFieldDefinition;
3939
import ai.vespa.schemals.parser.ast.tensorTypeElm;
4040
import ai.vespa.schemals.parser.rankingexpression.ast.LCURLY;
41+
import ai.vespa.schemals.parser.rankingexpression.ast.feature;
4142
import ai.vespa.schemals.parser.rankingexpression.ast.lambdaFunction;
4243
import ai.vespa.schemals.parser.rankingexpression.ast.tensorType;
4344
import ai.vespa.schemals.parser.rankingexpression.ast.tensorTypeDimension;
@@ -284,6 +285,11 @@ private void identifyDefinitionInRankExpression(SchemaNode node, List<Diagnostic
284285
return;
285286
}
286287

288+
if (node.getText().equals("foreach") && parent.isASTInstance(feature.class)) {
289+
handleForeachDefinitions(node, diagnostics);
290+
return;
291+
}
292+
287293
if (!grandParent.isASTInstance(lambdaFunction.class) || grandParent.size() < 1) {
288294
return;
289295
}
@@ -358,6 +364,32 @@ private void handleTensorTypeDefinitions(SchemaNode identifierNode, SchemaNode t
358364
context.schemaIndex().insertSymbolDefinition(identifierNode.getSymbol());
359365
}
360366

367+
private void handleForeachDefinitions(SchemaNode identifierNode, List<Diagnostic> diagnostics) {
368+
Optional<Symbol> scope = CSTUtils.findScope(identifierNode);
369+
identifierNode.setSymbol(SymbolType.FOREACH, context.fileURI(), scope.get(), "foreach_" + identifierNode.hashCode());
370+
identifierNode.setSymbolStatus(SymbolStatus.DEFINITION);
371+
context.schemaIndex().insertSymbolDefinition(identifierNode.getSymbol());
372+
373+
Node argsNode = identifierNode.getNextSibling();
374+
if (argsNode == null) return;
375+
376+
argsNode = argsNode.getNextSibling();
377+
378+
if (argsNode == null) return;
379+
if (argsNode.size() < 3) return;
380+
381+
Node iteratorExpressionNode = argsNode.get(2);
382+
if (iteratorExpressionNode == null) return;
383+
384+
Node iteratorIdentifier = iteratorExpressionNode.findFirstLeaf().getParent();
385+
if (!iteratorIdentifier.isASTInstance(ai.vespa.schemals.parser.rankingexpression.ast.identifierStr.class))
386+
return;
387+
388+
iteratorIdentifier.setSymbol(SymbolType.PARAMETER, context.fileURI(), identifierNode.getSymbol());
389+
iteratorIdentifier.getSymbol().setStatus(SymbolStatus.DEFINITION);
390+
context.schemaIndex().insertSymbolDefinition(iteratorIdentifier.getSymbol());
391+
}
392+
361393
private static final Set<String> reservedFunctionNames = ReservedFunctionNames.getReservedNames();
362394
// TODO: Maybe add distance and bm25 to the list?
363395
private void verifySymbolFunctionName(SchemaNode node, List<Diagnostic> diagnostics) {

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/resolvers/RankExpression/BuiltInFunctions.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -347,24 +347,26 @@ public class BuiltInFunctions {
347347
put("rawScore", new GenericFunction("rawScore", new FunctionSignature(new FieldArgument())));
348348
put("itemRawScore", new GenericFunction("itemRawScore", new FunctionSignature(new LabelArgument())));
349349

350+
351+
// See RankExpressionSymbolResolver.resolveForeach for more details
350352
put("foreach", new GenericFunction("foreach", List.of(
351353
new FunctionSignature(List.of(
352-
new KeywordArgument("fields", "dimension"),
353-
new StringArgument("variable"),
354+
new KeywordArgument("fields", "fields"),
355+
new StringArgument("variable", false),
354356
new ExpressionArgument("feature"),
355357
new StringArgument("condition"),
356358
new EnumArgument("operation", List.of("sum", "product", "average", "min", "max", "count"))
357359
)),
358360
new FunctionSignature(List.of(
359-
new KeywordArgument("terms", "dimension"),
360-
new StringArgument("variable"),
361+
new KeywordArgument("terms", "terms"),
362+
new StringArgument("variable", false),
361363
new ExpressionArgument("feature"),
362364
new StringArgument("condition"),
363365
new EnumArgument("operation", List.of("sum", "product", "average", "min", "max", "count"))
364366
)),
365367
new FunctionSignature(List.of(
366-
new KeywordArgument("attributes", "dimension"),
367-
new StringArgument("variable"),
368+
new KeywordArgument("attributes", "attributes"),
369+
new StringArgument("variable", false),
368370
new ExpressionArgument("feature"),
369371
new StringArgument("condition"),
370372
new EnumArgument("operation", List.of("sum", "product", "average", "min", "max", "count"))
@@ -395,7 +397,7 @@ public class BuiltInFunctions {
395397
add("nativeProximity");
396398
add("randomNormal");
397399
add("randomNormalStable");
398-
add("rankingExpression"); // TODO: deprecated (?)
400+
add("rankingExpression");
399401

400402
// TODO: these are only allowed in global-phase
401403
add("normalize_linear");

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/resolvers/RankExpression/GenericFunction.java‎

Lines changed: 43 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -100,54 +100,61 @@ public List<Diagnostic> handleArgumentList(ParseContext context, RankNode node,
100100

101101
diagnostics.addAll(signature.get().handleArgumentList(context, node.getChildren()));
102102

103-
Set<String> signatureProps = signature.get().getProperties();
103+
Optional<SpecificFunction> specificFunction = instantiate(context, signature.get(), node, ignoreProperty, diagnostics);
104+
specificFunction.ifPresent(instantiation -> node.setFunctionSignature(instantiation));
104105

105-
if (property.isEmpty() && (signatureProps.contains("") || signatureProps.size() == 0)) {
106+
return diagnostics;
107+
}
108+
109+
public Optional<SpecificFunction> instantiate(ParseContext context, FunctionSignature signature, RankNode node, boolean ignoreProperty, List<Diagnostic> diagnostics) {
110+
Set<String> signatureProps = signature.getProperties();
111+
112+
Optional<SchemaNode> propertyNode = ignoreProperty ? Optional.empty() : node.getProperty();
113+
Optional<String> propertyString = Optional.empty();
114+
if (propertyNode.isPresent()) {
115+
propertyString = Optional.of(propertyNode.get().getText());
116+
}
117+
118+
if (propertyString.isEmpty() && (signatureProps.contains("") || signatureProps.size() == 0)) {
106119
// This is valid
107-
node.setFunctionSignature(new SpecificFunction(this, signature.get()));
108-
return diagnostics;
120+
return Optional.of(new SpecificFunction(this, signature));
109121
}
110122

111-
if (signature.get().anyPropertyAllowed()) {
112-
if (property.isEmpty()) {
113-
node.setFunctionSignature(new SpecificFunction(this, signature.get()));
114-
} else {
115-
node.setFunctionSignature(new SpecificFunction(this, signature.get(), Optional.of(property.get().getText())));
116-
}
117-
return diagnostics;
123+
if (signature.anyPropertyAllowed()) {
124+
return Optional.of(new SpecificFunction(this, signature, propertyString));
118125
}
119126

120-
String availableProps = (signatureProps.size() == 0) ? "No one" : String.join(", ", signatureProps);
121-
if (!property.isPresent()) {
127+
String availableProps = (signatureProps.size() == 0) ? "No one" : String.join(", ", signatureProps.stream().filter(prop -> !prop.isEmpty()).toList());
128+
if (!propertyString.isPresent()) {
122129
String message = "The function '" + node.getSchemaNode().getText() + "' must be used with a property. Available properties are: " + availableProps;
123130
diagnostics.add(new SchemaDiagnostic.Builder()
124131
.setRange(node.getRange())
125132
.setMessage(message)
126133
.setSeverity(DiagnosticSeverity.Error)
127134
.build());
128-
return diagnostics;
135+
return Optional.empty();
129136
}
130137

131-
if (!properties.contains(property.get().getText())) {
132-
String message = "Invalid property '" + property.get().getText() + "'. Available properties are: " + availableProps;
138+
if (!properties.contains(propertyString.get())) {
139+
String message = "Invalid property '" + propertyString.get() + "'. Available properties are: " + availableProps;
133140
diagnostics.add(new SchemaDiagnostic.Builder()
134-
.setRange(property.get().getRange())
141+
.setRange(propertyNode.get().getRange())
135142
.setMessage(message)
136143
.setSeverity(DiagnosticSeverity.Error)
137144
.build());
138-
return diagnostics;
145+
return Optional.empty();
139146
}
140147

141-
if (!signature.get().getProperties().contains(property.get().getText())) {
148+
if (!signatureProps.contains(propertyString.get())) {
142149
String message = "This property is not available with with this signature. Available properties are: " + availableProps;
143150
diagnostics.add(new SchemaDiagnostic.Builder()
144-
.setRange(property.get().getRange())
151+
.setRange(propertyNode.get().getRange())
145152
.setMessage(message)
146153
.setSeverity(DiagnosticSeverity.Warning)
147154
.build());
148155
}
149156

150-
Node symbolNode = property.get();
157+
Node symbolNode = propertyNode.get();
151158
while (!symbolNode.isASTInstance(identifierStr.class) && symbolNode.size() > 0) {
152159
symbolNode = symbolNode.get(0);
153160
}
@@ -157,25 +164,10 @@ public List<Diagnostic> handleArgumentList(ParseContext context, RankNode node,
157164
.setStatus(SymbolStatus.BUILTIN_REFERENCE);
158165
}
159166

160-
node.setFunctionSignature(new SpecificFunction(this, signature.get(), propertyString));
161-
162-
return diagnostics;
167+
return Optional.of(new SpecificFunction(this, signature, propertyString));
163168
}
164169

165-
private static boolean propertyInSet(Optional<String> string, Set<String> propertySet) {
166-
if (string.isEmpty() && (
167-
propertySet.size() == 0 ||
168-
propertySet.contains("")
169-
)) {
170-
return true;
171-
}
172-
173-
if (string.isEmpty()) return false;
174-
175-
return propertySet.contains(string.get());
176-
}
177-
178-
private Optional<FunctionSignature> findFunctionSignature(List<RankNode> arguments, Optional<String> property) {
170+
public Optional<FunctionSignature> findFunctionSignature(List<RankNode> arguments, Optional<String> property) {
179171

180172
List<FunctionSignature> bestMatches = new ArrayList<>();
181173
int maxScore = 0;
@@ -209,4 +201,17 @@ private Optional<FunctionSignature> findFunctionSignature(List<RankNode> argumen
209201

210202
return Optional.empty();
211203
}
204+
205+
private static boolean propertyInSet(Optional<String> string, Set<String> propertySet) {
206+
if (string.isEmpty() && (
207+
propertySet.size() == 0 ||
208+
propertySet.contains("")
209+
)) {
210+
return true;
211+
}
212+
213+
if (string.isEmpty()) return false;
214+
215+
return propertySet.contains(string.get());
216+
}
212217
}

0 commit comments

Comments
 (0)