From cda0b6fd1c87f48ef1c64af7fc6c6c81e1e8ee64 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Wed, 26 Aug 2026 21:33:17 -0400 Subject: [PATCH 001/136] feat(hogql): add native parser and AST compiler --- core/trino-hogql-compiler/pom.xml | 56 ++++++ .../trino/hogql/compiler/HogQlCompiler.java | 50 +++++ .../trino/hogql/compiler/TrinoAstFactory.java | 133 ++++++++++++ .../hogql/compiler/TestHogQlCompiler.java | 113 +++++++++++ core/trino-hogql-parser/pom.xml | 55 +++++ .../io/trino/hogql/parser/antlr/HogQlBase.g4 | 101 ++++++++++ .../io/trino/hogql/parser/HogQlParser.java | 189 ++++++++++++++++++ .../hogql/parser/HogQlParsingException.java | 55 +++++ .../trino/hogql/parser/tree/HogQlQuery.java | 140 +++++++++++++ .../trino/hogql/parser/TestHogQlParser.java | 43 ++++ pom.xml | 14 ++ 11 files changed, 949 insertions(+) create mode 100644 core/trino-hogql-compiler/pom.xml create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java create mode 100644 core/trino-hogql-parser/pom.xml create mode 100644 core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/antlr/HogQlBase.g4 create mode 100644 core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java create mode 100644 core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParsingException.java create mode 100644 core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java create mode 100644 core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java diff --git a/core/trino-hogql-compiler/pom.xml b/core/trino-hogql-compiler/pom.xml new file mode 100644 index 000000000000..692be279716b --- /dev/null +++ b/core/trino-hogql-compiler/pom.xml @@ -0,0 +1,56 @@ + + + 4.0.0 + + + io.trino + trino-root + 484-SNAPSHOT + ../../pom.xml + + + trino-hogql-compiler + ${project.artifactId} + Trino - HogQL compiler + + + + io.trino + trino-hogql-parser + + + + io.trino + trino-parser + + + + org.antlr + antlr4-runtime + + + + org.assertj + assertj-core + test + + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.junit.jupiter + junit-jupiter-engine + test + + + + org.junit.jupiter + junit-jupiter-params + test + + + diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java new file mode 100644 index 000000000000..0aa3c6772abc --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.parser.HogQlParser; +import io.trino.hogql.parser.HogQlParsingException; +import io.trino.sql.parser.ParsingException; +import io.trino.sql.tree.Statement; + +import static java.util.Objects.requireNonNull; + +public final class HogQlCompiler +{ + private final HogQlParser parser; + + public HogQlCompiler() + { + this(new HogQlParser()); + } + + HogQlCompiler(HogQlParser parser) + { + this.parser = requireNonNull(parser, "parser is null"); + } + + public Statement compile(String hogql) + { + try { + return TrinoAstFactory.createStatement(parser.parseStatement(hogql)); + } + catch (HogQlParsingException exception) { + throw new ParsingException( + exception.getErrorMessage(), + null, + exception.getLineNumber(), + exception.getColumnNumber()); + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java new file mode 100644 index 000000000000..6fb864e418d2 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -0,0 +1,133 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import io.trino.sql.tree.AllColumns; +import io.trino.sql.tree.BooleanLiteral; +import io.trino.sql.tree.DereferenceExpression; +import io.trino.sql.tree.Expression; +import io.trino.sql.tree.LongLiteral; +import io.trino.sql.tree.NodeLocation; +import io.trino.sql.tree.NullLiteral; +import io.trino.sql.tree.QualifiedName; +import io.trino.sql.tree.Query; +import io.trino.sql.tree.QuerySpecification; +import io.trino.sql.tree.Select; +import io.trino.sql.tree.SelectItem; +import io.trino.sql.tree.SingleColumn; +import io.trino.sql.tree.Statement; +import io.trino.sql.tree.StringLiteral; +import io.trino.sql.tree.Table; + +import java.util.List; +import java.util.Optional; + +final class TrinoAstFactory +{ + private TrinoAstFactory() {} + + public static Statement createStatement(HogQlQuery query) + { + NodeLocation location = location(query.span()); + QuerySpecification querySpecification = new QuerySpecification( + location, + new Select(location, false, query.projections().stream() + .map(TrinoAstFactory::createSelectItem) + .toList()), + query.from().map(TrinoAstFactory::createTable), + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + return new Query( + location, + List.of(), + List.of(), + Optional.empty(), + querySpecification, + Optional.empty(), + Optional.empty(), + Optional.empty()); + } + + private static SelectItem createSelectItem(Projection projection) + { + return switch (projection) { + case Star star -> new AllColumns(location(star.span())); + case ExpressionProjection expression -> new SingleColumn( + location(expression.span()), + createExpression(expression.expression()), + Optional.empty()); + }; + } + + private static Expression createExpression(HogQlQuery.Expression expression) + { + return switch (expression) { + case ColumnReference reference -> createColumnReference(reference); + case Literal literal -> createLiteral(literal); + }; + } + + private static Expression createLiteral(Literal literal) + { + NodeLocation location = location(literal.span()); + return switch (literal.kind()) { + case BOOLEAN -> new BooleanLiteral(location, literal.value()); + case INTEGER -> new LongLiteral(location, literal.value()); + case NULL -> new NullLiteral(location); + case STRING -> new StringLiteral(location, literal.value()); + }; + } + + private static Expression createColumnReference(ColumnReference reference) + { + List parts = reference.parts(); + Expression expression = createIdentifier(parts.getFirst()); + for (Identifier part : parts.subList(1, parts.size())) { + expression = new DereferenceExpression(location(reference.span()), expression, createIdentifier(part)); + } + return expression; + } + + private static Table createTable(TableReference table) + { + List parts = table.parts().stream() + .map(TrinoAstFactory::createIdentifier) + .toList(); + return new Table(location(table.span()), QualifiedName.of(parts)); + } + + private static io.trino.sql.tree.Identifier createIdentifier(Identifier identifier) + { + return new io.trino.sql.tree.Identifier(location(identifier.span()), identifier.value(), identifier.delimited()); + } + + private static NodeLocation location(SourceSpan span) + { + return new NodeLocation(span.line(), span.column()); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java new file mode 100644 index 000000000000..e506cbfa5841 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -0,0 +1,113 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.sql.SqlFormatter; +import io.trino.sql.parser.ParsingException; +import io.trino.sql.parser.SqlParser; +import io.trino.sql.tree.AllColumns; +import io.trino.sql.tree.Identifier; +import io.trino.sql.tree.Node; +import io.trino.sql.tree.NodeLocation; +import io.trino.sql.tree.Query; +import io.trino.sql.tree.QuerySpecification; +import io.trino.sql.tree.SingleColumn; +import io.trino.sql.tree.Statement; +import io.trino.sql.tree.Table; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayDeque; +import java.util.Deque; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlCompiler +{ + private final HogQlCompiler compiler = new HogQlCompiler(); + private final SqlParser sqlParser = new SqlParser(); + + @ParameterizedTest + @ValueSource(strings = { + "SELECT 1", + "select 'event'", + "SELECT TRUE, FALSE, NULL", + "SELECT * FROM events", + "SELECT event, properties FROM ducklake.default.events", + "SELECT * FROM \"MiXeD\".\"Event Table\"", + }) + public void testLowersToEquivalentStockTrinoAst(String hogql) + { + Statement statement = compiler.compile(hogql); + + assertThat(statement).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @Test + public void testPreservesQuotedIdentifiersAndSourceLocations() + { + Statement statement = compiler.compile( + """ + SELECT + `Event Name`, + * + FROM + `Analytics`.`Event Table` + """); + + Query query = (Query) statement; + QuerySpecification querySpecification = (QuerySpecification) query.getQueryBody(); + SingleColumn column = (SingleColumn) querySpecification.getSelect().getSelectItems().getFirst(); + Identifier columnIdentifier = (Identifier) column.getExpression(); + AllColumns allColumns = (AllColumns) querySpecification.getSelect().getSelectItems().get(1); + Table table = (Table) querySpecification.getFrom().orElseThrow(); + + assertThat(query.getLocation()).contains(new NodeLocation(1, 1)); + assertThat(column.getLocation()).contains(new NodeLocation(2, 5)); + assertThat(columnIdentifier.getValue()).isEqualTo("Event Name"); + assertThat(columnIdentifier.isDelimited()).isTrue(); + assertThat(allColumns.getLocation()).contains(new NodeLocation(3, 5)); + assertThat(table.getLocation()).contains(new NodeLocation(5, 5)); + assertThat(table.getName().getOriginalParts()) + .extracting(Identifier::getValue) + .containsExactly("Analytics", "Event Table"); + assertThat(table.getName().getOriginalParts()) + .allMatch(Identifier::isDelimited); + } + + @Test + public void testReturnsOnlyStockTrinoTreeNodes() + { + Statement statement = compiler.compile("SELECT event, * FROM ducklake.default.events"); + Deque pending = new ArrayDeque<>(); + pending.add(statement); + + while (!pending.isEmpty()) { + Node node = pending.removeFirst(); + assertThat(node.getClass().getPackageName()).isEqualTo("io.trino.sql.tree"); + pending.addAll(node.getChildren()); + } + } + + @Test + public void testReportsParsingFailureAsTrinoSyntaxError() + { + assertThatThrownBy(() -> compiler.compile("SELECT 1\nWHERE TRUE")) + .isInstanceOf(ParsingException.class) + .hasMessageStartingWith("line 2:1:"); + } +} diff --git a/core/trino-hogql-parser/pom.xml b/core/trino-hogql-parser/pom.xml new file mode 100644 index 000000000000..958723e2cff7 --- /dev/null +++ b/core/trino-hogql-parser/pom.xml @@ -0,0 +1,55 @@ + + + 4.0.0 + + + io.trino + trino-root + 484-SNAPSHOT + ../../pom.xml + + + trino-hogql-parser + ${project.artifactId} + Trino - HogQL parser + + + + org.antlr + antlr4-runtime + + + + org.assertj + assertj-core + test + + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.junit.jupiter + junit-jupiter-engine + test + + + + org.junit.jupiter + junit-jupiter-params + test + + + + + + + org.antlr + antlr4-maven-plugin + + + + diff --git a/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/antlr/HogQlBase.g4 b/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/antlr/HogQlBase.g4 new file mode 100644 index 000000000000..43f3658f43c7 --- /dev/null +++ b/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/antlr/HogQlBase.g4 @@ -0,0 +1,101 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +grammar HogQlBase; + +options { caseInsensitive = true; } + +singleStatement + : query SEMICOLON? EOF + ; + +query + : SELECT projection (COMMA projection)* (FROM qualifiedName)? + ; + +projection + : ASTERISK + | expression + ; + +expression + : literal + | qualifiedName + ; + +literal + : INTEGER_VALUE + | STRING + | TRUE + | FALSE + | NULL + ; + +qualifiedName + : identifier (DOT identifier)* + ; + +identifier + : IDENTIFIER + | QUOTED_IDENTIFIER + | BACKQUOTED_IDENTIFIER + ; + +SELECT: 'SELECT'; +FROM: 'FROM'; +TRUE: 'TRUE'; +FALSE: 'FALSE'; +NULL: 'NULL'; + +ASTERISK: '*'; +COMMA: ','; +DOT: '.'; +SEMICOLON: ';'; + +STRING + : '\'' (~['\\] | '\'\'')* '\'' + ; + +INTEGER_VALUE + : '0' + | [1-9] DIGIT* + ; + +IDENTIFIER + : (LETTER | '_') (LETTER | DIGIT | '_' | '$')* + ; + +QUOTED_IDENTIFIER + : '"' (~'"' | '""')+ '"' + ; + +BACKQUOTED_IDENTIFIER + : '`' (~'`' | '``')+ '`' + ; + +fragment DIGIT + : [0-9] + ; + +fragment LETTER + : [A-Z] + ; + +WHITESPACE + : [ \r\n\t]+ -> channel(HIDDEN) + ; + +UNRECOGNIZED + : . + ; diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java new file mode 100644 index 000000000000..6c9375871425 --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -0,0 +1,189 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import io.trino.hogql.parser.antlr.HogQlBaseBaseVisitor; +import io.trino.hogql.parser.antlr.HogQlBaseLexer; +import io.trino.hogql.parser.antlr.HogQlBaseParser; +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.LiteralKind; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import org.antlr.v4.runtime.ANTLRErrorListener; +import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.atn.PredictionMode; +import org.antlr.v4.runtime.misc.ParseCancellationException; + +import java.util.List; +import java.util.Optional; + +import static io.trino.hogql.parser.tree.HogQlQuery.LiteralKind.BOOLEAN; +import static io.trino.hogql.parser.tree.HogQlQuery.LiteralKind.INTEGER; +import static io.trino.hogql.parser.tree.HogQlQuery.LiteralKind.NULL; +import static io.trino.hogql.parser.tree.HogQlQuery.LiteralKind.STRING; +import static java.util.Objects.requireNonNull; + +public final class HogQlParser +{ + private static final ANTLRErrorListener ERROR_LISTENER = new BaseErrorListener() + { + @Override + public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, String message, RecognitionException cause) + { + throw new HogQlParsingException(message, cause, line, charPositionInLine + 1); + } + }; + + public HogQlQuery parseStatement(String hogql) + { + requireNonNull(hogql, "hogql is null"); + + try { + HogQlBaseLexer lexer = new HogQlBaseLexer(CharStreams.fromString(hogql)); + CommonTokenStream tokenStream = new CommonTokenStream(lexer); + HogQlBaseParser parser = new HogQlBaseParser(tokenStream); + + lexer.removeErrorListeners(); + lexer.addErrorListener(ERROR_LISTENER); + parser.removeErrorListeners(); + + HogQlBaseParser.SingleStatementContext tree; + try { + parser.getInterpreter().setPredictionMode(PredictionMode.SLL); + parser.setErrorHandler(new BailErrorStrategy()); + tree = parser.singleStatement(); + } + catch (ParseCancellationException _) { + parser.reset(); + parser.getInterpreter().setPredictionMode(PredictionMode.LL); + parser.setErrorHandler(new DefaultErrorStrategy()); + parser.addErrorListener(ERROR_LISTENER); + tree = parser.singleStatement(); + } + + return new AstBuilder().build(tree.query()); + } + catch (StackOverflowError _) { + throw new HogQlParsingException("statement is too large", null, 1, 1); + } + } + + private static final class AstBuilder + extends HogQlBaseBaseVisitor + { + public HogQlQuery build(HogQlBaseParser.QueryContext context) + { + List projections = context.projection().stream() + .map(this::buildProjection) + .toList(); + Optional from = Optional.ofNullable(context.qualifiedName()) + .map(this::buildTableReference); + return new HogQlQuery(projections, from, sourceSpan(context)); + } + + private Projection buildProjection(HogQlBaseParser.ProjectionContext context) + { + if (context.ASTERISK() != null) { + return new Star(sourceSpan(context)); + } + return new ExpressionProjection(buildExpression(context.expression())); + } + + private Expression buildExpression(HogQlBaseParser.ExpressionContext context) + { + if (context.literal() != null) { + return buildLiteral(context.literal()); + } + return new ColumnReference(buildIdentifiers(context.qualifiedName()), sourceSpan(context)); + } + + private Literal buildLiteral(HogQlBaseParser.LiteralContext context) + { + String value = context.getText(); + LiteralKind kind; + if (context.INTEGER_VALUE() != null) { + kind = INTEGER; + } + else if (context.STRING() != null) { + kind = STRING; + value = decodeString(value); + } + else if (context.NULL() != null) { + kind = NULL; + } + else { + kind = BOOLEAN; + } + return new Literal(kind, value, sourceSpan(context)); + } + + private TableReference buildTableReference(HogQlBaseParser.QualifiedNameContext context) + { + return new TableReference(buildIdentifiers(context), sourceSpan(context)); + } + + private List buildIdentifiers(HogQlBaseParser.QualifiedNameContext context) + { + return context.identifier().stream() + .map(this::buildIdentifier) + .toList(); + } + + private Identifier buildIdentifier(HogQlBaseParser.IdentifierContext context) + { + String value = context.getText(); + boolean delimited = context.IDENTIFIER() == null; + if (context.QUOTED_IDENTIFIER() != null) { + value = unquoteAndUnescape(value, '"'); + } + else if (context.BACKQUOTED_IDENTIFIER() != null) { + value = unquoteAndUnescape(value, '`'); + } + return new Identifier(value, delimited, sourceSpan(context)); + } + + private static String decodeString(String value) + { + return value.substring(1, value.length() - 1).replace("''", "'"); + } + + private static String unquoteAndUnescape(String value, char quote) + { + String quoted = value.substring(1, value.length() - 1); + return quoted.replace(String.valueOf(quote).repeat(2), String.valueOf(quote)); + } + + private static SourceSpan sourceSpan(ParserRuleContext context) + { + Token start = context.getStart(); + Token stop = context.getStop(); + return new SourceSpan(start.getStartIndex(), stop.getStopIndex() + 1, start.getLine(), start.getCharPositionInLine() + 1); + } + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParsingException.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParsingException.java new file mode 100644 index 000000000000..7fc12141df41 --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParsingException.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +public final class HogQlParsingException + extends RuntimeException +{ + private final int line; + private final int column; + + public HogQlParsingException(String message, Throwable cause, int line, int column) + { + super(message, cause); + if (line < 1) { + throw new IllegalArgumentException("line must be positive"); + } + if (column < 1) { + throw new IllegalArgumentException("column must be positive"); + } + this.line = line; + this.column = column; + } + + public int getLineNumber() + { + return line; + } + + public int getColumnNumber() + { + return column; + } + + public String getErrorMessage() + { + return super.getMessage(); + } + + @Override + public String getMessage() + { + return "line %s:%s: %s".formatted(line, column, getErrorMessage()); + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java new file mode 100644 index 000000000000..fa161a966697 --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -0,0 +1,140 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser.tree; + +import java.util.List; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public record HogQlQuery(List projections, Optional from, SourceSpan span) +{ + public HogQlQuery + { + projections = List.copyOf(requireNonNull(projections, "projections is null")); + from = requireNonNull(from, "from is null"); + span = requireNonNull(span, "span is null"); + } + + public sealed interface Projection + permits ExpressionProjection, Star + { + SourceSpan span(); + } + + public record Star(SourceSpan span) + implements Projection + { + public Star + { + span = requireNonNull(span, "span is null"); + } + } + + public record ExpressionProjection(Expression expression) + implements Projection + { + public ExpressionProjection + { + expression = requireNonNull(expression, "expression is null"); + } + + @Override + public SourceSpan span() + { + return expression.span(); + } + } + + public sealed interface Expression + permits ColumnReference, Literal + { + SourceSpan span(); + } + + public record ColumnReference(List parts, SourceSpan span) + implements Expression + { + public ColumnReference + { + parts = List.copyOf(requireNonNull(parts, "parts is null")); + if (parts.isEmpty()) { + throw new IllegalArgumentException("parts is empty"); + } + span = requireNonNull(span, "span is null"); + } + } + + public record Literal(LiteralKind kind, String value, SourceSpan span) + implements Expression + { + public Literal + { + kind = requireNonNull(kind, "kind is null"); + value = requireNonNull(value, "value is null"); + span = requireNonNull(span, "span is null"); + } + } + + public enum LiteralKind + { + BOOLEAN, + INTEGER, + NULL, + STRING, + } + + public record TableReference(List parts, SourceSpan span) + { + public TableReference + { + parts = List.copyOf(requireNonNull(parts, "parts is null")); + if (parts.isEmpty()) { + throw new IllegalArgumentException("parts is empty"); + } + span = requireNonNull(span, "span is null"); + } + } + + public record Identifier(String value, boolean delimited, SourceSpan span) + { + public Identifier + { + value = requireNonNull(value, "value is null"); + if (value.isEmpty()) { + throw new IllegalArgumentException("value is empty"); + } + span = requireNonNull(span, "span is null"); + } + } + + public record SourceSpan(int startOffset, int endOffset, int line, int column) + { + public SourceSpan + { + if (startOffset < 0) { + throw new IllegalArgumentException("startOffset is negative"); + } + if (endOffset < startOffset) { + throw new IllegalArgumentException("endOffset is before startOffset"); + } + if (line < 1) { + throw new IllegalArgumentException("line must be positive"); + } + if (column < 1) { + throw new IllegalArgumentException("column must be positive"); + } + } + } +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java new file mode 100644 index 000000000000..47badb42b002 --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlParser +{ + private final HogQlParser parser = new HogQlParser(); + + @ParameterizedTest + @ValueSource(strings = { + "SELECT", + "SELECT 1 WHERE TRUE", + "SELECT count(*)", + "SELECT * FROM first JOIN second", + "DROP TABLE events", + "SELECT 1; SELECT 2", + "SELECT 'unsupported\\nescape'", + "SELECT * FROM", + "SELECT 01", + }) + public void testRejectsSyntaxOutsideM0(String hogql) + { + assertThatThrownBy(() -> parser.parseStatement(hogql)) + .isInstanceOf(HogQlParsingException.class) + .hasMessageStartingWith("line "); + } +} diff --git a/pom.xml b/pom.xml index 82d64d8b4849..3c3ef659b022 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,8 @@ client/trino-client client/trino-jdbc core/trino-grammar + core/trino-hogql-compiler + core/trino-hogql-parser core/trino-main core/trino-parser core/trino-server @@ -1121,6 +1123,18 @@ ${project.version} + + io.trino + trino-hogql-compiler + ${project.version} + + + + io.trino + trino-hogql-parser + ${project.version} + + io.trino trino-hudi From 455a5a920816925407647217c434a8a7f3c88137 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Wed, 26 Aug 2026 21:33:42 -0400 Subject: [PATCH 002/136] feat(hogql): add coordinator submission seam --- core/trino-hogql-compiler/README.md | 123 +++++++++++++++ core/trino-main/pom.xml | 5 + .../io/trino/dispatcher/DispatchManager.java | 14 +- .../dispatcher/HogQlStatementResource.java | 55 +++++++ .../dispatcher/QueuedStatementResource.java | 26 +++- .../io/trino/execution/QueryLanguage.java | 20 +++ .../io/trino/execution/QueryPreparer.java | 27 +++- .../main/java/io/trino/hogql/HogQlConfig.java | 35 +++++ .../io/trino/server/CoordinatorModule.java | 8 + .../TestHogQlStatementResource.java | 142 ++++++++++++++++++ .../execution/BenchmarkQueryPreparer.java | 65 ++++++++ .../io/trino/execution/TestQueryPreparer.java | 11 ++ .../java/io/trino/hogql/TestHogQlConfig.java | 40 +++++ 13 files changed, 560 insertions(+), 11 deletions(-) create mode 100644 core/trino-hogql-compiler/README.md create mode 100644 core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java create mode 100644 core/trino-main/src/main/java/io/trino/execution/QueryLanguage.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlConfig.java create mode 100644 core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java create mode 100644 core/trino-main/src/test/java/io/trino/execution/BenchmarkQueryPreparer.java create mode 100644 core/trino-main/src/test/java/io/trino/hogql/TestHogQlConfig.java diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md new file mode 100644 index 000000000000..8093ea9dabbf --- /dev/null +++ b/core/trino-hogql-compiler/README.md @@ -0,0 +1,123 @@ +# Native HogQL compiler + +This module compiles HogQL directly to Trino's public SQL AST. It deliberately +does not render an intermediate SQL string or add HogQL nodes to Trino's +analyzer and planner. + +The implementation is split into two modules: + +- `trino-hogql-parser` owns the HogQL grammar and a private, source-located + parser AST. It has no dependency on Trino's SQL tree. +- `trino-hogql-compiler` is the only translation boundary. It converts every + parser node into an `io.trino.sql.tree` node. + +The coordinator submits the resulting statement through the standard Trino +analysis, planning, scheduling, paging, and cancellation paths. The original +HogQL text remains the query text used by limits, history, and resource-group +selection. + +## M0 contract + +M0 accepts a deliberately small grammar: + +```text +SELECT [, ...] [FROM ] +``` + +Projections can be `*`, qualified column references, integers, strings, +booleans, or `NULL`. Identifiers can be unquoted, double quoted, or backquoted. +The parser rejects unsupported clauses and multiple statements instead of +silently interpreting them as SQL. + +Set `hogql.enabled=true` on the coordinator to register `POST /v1/hogql`. The +request body is raw HogQL text and the response uses Trino's existing statement +protocol. Follow-up result and cancellation URIs remain under `/v1/statement`. +The endpoint is absent when the property is false, which is the default. + +Development is based on the fork's `ducklake-connector` integration branch. At +the start of this milestone its head was `posthog-484-ducklake.9`, commit +`3065d56e4d5e256962c8297b4299909db8301c0a`. + +Charts `main` still provides the deployment compatibility baseline: +`posthog-484-ducklake.6` at commit +`3bb054e5b80bc6ad740b26f6748b7b3f5548ff32`, published as +`ghcr.io/posthog/trino:484-ducklake.6`. Its linux/arm64 digest is +`sha256:85d2b37dfb7c966a1bfe7b0470efaa6a0116c86cb6a05d14a6bdc8c02f7a6787`. + +## Fork maintenance + +The `ducklake-connector` branch's common ancestor with `trinodb/trino` is +`67f588f0c81b21b425e1a43b05d70f9cf8798d6c`. The first nine PostHog commits +produce the Charts `main` `.6` baseline: + +1. `a002cc56980` adds the DuckLake connector and PostHog image workflow. +2. `4cd0573a57f` adds column-name mapping. +3. `500cdf40aa3` fixes Parquet millisecond timestamp reads. +4. `c03df89358a` reads fully visible partial files. +5. `6ef914e4149` splits large data files. +6. `09690b4fc8b` pools metadata connections and supports password files. +7. `e6c3fbd9d83` adds the PostgreSQL catalog store. +8. `0bab82c7520` counts rows from catalog metadata. +9. `3bb054e5b80` reads Parquet footers using the recorded size. + +The rolling integration branch continues through `.7`-`.9` with the DuckLake +write path, Trino views and schema evolution, partitioned-table reads, field-ID +based reads, and missing-file-column handling. Merge commits and CI maintenance +also live on this line; inspect the exact range from the upstream common +ancestor rather than assuming the release-tag history is linear. + +Keep native HogQL changes as a separate ordered stack above those commits: +parser/compiler modules, coordinator submission seam, metadata provider, +optional functions, then packaging and tests. During an upstream update, fetch +`trinodb/trino`, identify the new common ancestor, rebase the DuckLake series, +then rebase the HogQL series onto `ducklake-connector`. Resolve AST API changes +in `TrinoAstFactory` and submission conflicts in `QueuedStatementResource`, +`DispatchManager`, and `QueryPreparer`; do not introduce HogQL branches into +later execution stages. +Run the unchanged SQL/coordinator suites, the complete DuckLake suite, and the +HogQL suite before creating a release tag. + +`.github/workflows/docker-publish-posthog.yml` publishes linux/arm64 images. +Pushing `posthog-` builds the complete distribution and publishes +`ghcr.io/posthog/trino:`; a manual workflow dispatch accepts the same +image-tag without the `posthog-` prefix. The workflow intentionally skips tests +and checks, so its source commit must already have passed the suites above. + +## Validation + +On a fresh checkout, install their reactor dependencies without running tests: + +```shell +./mvnw -pl :trino-hogql-parser,:trino-hogql-compiler,:trino-main -am install \ + -DskipTests +``` + +Then run the parser, compiler, and coordinator tests together: + +```shell +./mvnw -pl :trino-hogql-parser,:trino-hogql-compiler,:trino-main test \ + -Dtest=TestHogQlParser,TestHogQlCompiler,TestQueryPreparer,TestHogQlConfig,TestHogQlStatementResource \ + -Dsurefire.failIfNoSpecifiedTests=false +``` + +The compiler tests compare the shared SQL subset with Trino's SQL parser, +format and reparse the generated tree, verify source locations, and assert that +no private HogQL AST node crosses the compiler boundary. + +Measure the standard SQL preparation path with and without the compiler +available using `BenchmarkQueryPreparer`. On JDK 25, enable annotation +processing when generating the JMH benchmark index: + +```shell +./mvnw -pl :trino-main clean test-compile exec:exec \ + -Dmaven.compiler.proc=full \ + -Dexec.executable=java \ + -Dexec.classpathScope=test \ + -Dexec.args='-cp %classpath io.trino.execution.BenchmarkQueryPreparer' +``` + +On 2026-08-26, one linux/arm64 Temurin 25 fork measured standard SQL +preparation at `3.792 ± 0.096 us/op` with the HogQL compiler available and +`3.708 ± 0.066 us/op` without it. The confidence intervals overlap; this M0 +sample does not show a distinguishable unused-path regression. Retain the +benchmark for repeated measurements on release hardware. diff --git a/core/trino-main/pom.xml b/core/trino-main/pom.xml index eae766945833..5118c2a894d5 100644 --- a/core/trino-main/pom.xml +++ b/core/trino-main/pom.xml @@ -245,6 +245,11 @@ trino-geospatial-toolkit + + io.trino + trino-hogql-compiler + + io.trino trino-matching diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/DispatchManager.java b/core/trino-main/src/main/java/io/trino/dispatcher/DispatchManager.java index 08de0825365e..58bf61588a8c 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/DispatchManager.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/DispatchManager.java @@ -27,6 +27,7 @@ import io.trino.event.QueryMonitor; import io.trino.execution.QueryIdGenerator; import io.trino.execution.QueryInfo; +import io.trino.execution.QueryLanguage; import io.trino.execution.QueryManagerConfig; import io.trino.execution.QueryManagerStats; import io.trino.execution.QueryPreparer; @@ -59,6 +60,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static io.trino.execution.QueryLanguage.SQL; import static io.trino.execution.QueryState.FINISHING; import static io.trino.execution.QueryState.QUEUED; import static io.trino.execution.QueryState.RUNNING; @@ -174,11 +176,17 @@ public QueryId createQueryId() } public ListenableFuture createQuery(QueryId queryId, Span querySpan, Slug slug, SessionContext sessionContext, String query) + { + return createQuery(queryId, querySpan, slug, sessionContext, query, SQL); + } + + public ListenableFuture createQuery(QueryId queryId, Span querySpan, Slug slug, SessionContext sessionContext, String query, QueryLanguage queryLanguage) { requireNonNull(queryId, "queryId is null"); requireNonNull(querySpan, "querySpan is null"); requireNonNull(sessionContext, "sessionContext is null"); requireNonNull(query, "query is null"); + requireNonNull(queryLanguage, "queryLanguage is null"); checkArgument(!query.isEmpty(), "query must not be empty string"); checkArgument(!queryTracker.hasQuery(queryId), "query %s already exists", queryId); @@ -192,7 +200,7 @@ public ListenableFuture createQuery(QueryId queryId, Span querySpan, Slug .setParent(Context.current().with(querySpan)) .startSpan(); try (var _ = scopedSpan(span)) { - createQueryInternal(queryId, querySpan, slug, sessionContext, query, resourceGroupManager); + createQueryInternal(queryId, querySpan, slug, sessionContext, query, queryLanguage, resourceGroupManager); } finally { queryCreationFuture.set(null); @@ -205,7 +213,7 @@ public ListenableFuture createQuery(QueryId queryId, Span querySpan, Slug * Creates and registers a dispatch query with the query tracker. This method will never fail to register a query with the query * tracker. If an error occurs while creating a dispatch query, a failed dispatch will be created and registered. */ - private void createQueryInternal(QueryId queryId, Span querySpan, Slug slug, SessionContext sessionContext, String query, ResourceGroupManager resourceGroupManager) + private void createQueryInternal(QueryId queryId, Span querySpan, Slug slug, SessionContext sessionContext, String query, QueryLanguage queryLanguage, ResourceGroupManager resourceGroupManager) { Session session = null; PreparedQuery preparedQuery = null; @@ -223,7 +231,7 @@ private void createQueryInternal(QueryId queryId, Span querySpan, Slug slug, accessControl.checkCanExecuteQuery(sessionContext.getIdentity(), queryId); // prepare query - preparedQuery = queryPreparer.prepareQuery(session, query); + preparedQuery = queryPreparer.prepareQuery(session, query, queryLanguage); // select resource group Optional queryType = getQueryType(preparedQuery.getStatement()).map(Enum::name); diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java new file mode 100644 index 000000000000..bf15f36e5422 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.dispatcher; + +import com.google.inject.Inject; +import io.trino.server.ExternalUriInfo; +import io.trino.server.security.ResourceSecurity; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.ws.rs.BeanParam; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.HttpHeaders; +import jakarta.ws.rs.core.Response; + +import static io.trino.execution.QueryLanguage.HOGQL; +import static io.trino.server.security.ResourceSecurity.AccessType.AUTHENTICATED_USER; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static java.util.Objects.requireNonNull; + +@Path("/v1/hogql") +public class HogQlStatementResource +{ + private final QueuedStatementResource queuedStatementResource; + + @Inject + public HogQlStatementResource(QueuedStatementResource queuedStatementResource) + { + this.queuedStatementResource = requireNonNull(queuedStatementResource, "queuedStatementResource is null"); + } + + @ResourceSecurity(AUTHENTICATED_USER) + @POST + @Produces(APPLICATION_JSON) + public Response postStatement( + String statement, + @Context HttpServletRequest servletRequest, + @Context HttpHeaders httpHeaders, + @BeanParam ExternalUriInfo externalUriInfo) + { + return queuedStatementResource.postStatement(statement, HOGQL, servletRequest, httpHeaders, externalUriInfo); + } +} diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java b/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java index eff1bfadcead..7a4cedc0c8b0 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java @@ -29,6 +29,7 @@ import io.trino.client.QueryResults; import io.trino.client.StatementStats; import io.trino.execution.ExecutionFailureInfo; +import io.trino.execution.QueryLanguage; import io.trino.execution.QueryManagerConfig; import io.trino.execution.QueryState; import io.trino.server.ExternalUriInfo; @@ -89,6 +90,7 @@ import static io.trino.dispatcher.QueuedStatementResource.SubmissionState.ABANDONED; import static io.trino.dispatcher.QueuedStatementResource.SubmissionState.NOT_SUBMITTED; import static io.trino.dispatcher.QueuedStatementResource.SubmissionState.SUBMITTED; +import static io.trino.execution.QueryLanguage.SQL; import static io.trino.execution.QueryState.FAILED; import static io.trino.execution.QueryState.QUEUED; import static io.trino.server.ServletSecurityUtils.authenticatedIdentity; @@ -174,16 +176,26 @@ public Response postStatement( @Context HttpServletRequest servletRequest, @Context HttpHeaders httpHeaders, @BeanParam ExternalUriInfo externalUriInfo) + { + return postStatement(statement, SQL, servletRequest, httpHeaders, externalUriInfo); + } + + Response postStatement( + String statement, + QueryLanguage queryLanguage, + HttpServletRequest servletRequest, + HttpHeaders httpHeaders, + ExternalUriInfo externalUriInfo) { if (isNullOrEmpty(statement)) { - throw new BadRequestException("SQL statement is empty"); + throw new BadRequestException(queryLanguage == SQL ? "SQL statement is empty" : "HogQL query is empty"); } - Query query = registerQuery(statement, servletRequest, httpHeaders); + Query query = registerQuery(statement, queryLanguage, servletRequest, httpHeaders); return createQueryResultsResponse(query.getQueryResults(query.getLastToken(), externalUriInfo), query.sessionContext.getQueryDataEncoding()); } - private Query registerQuery(String statement, HttpServletRequest servletRequest, HttpHeaders httpHeaders) + private Query registerQuery(String statement, QueryLanguage queryLanguage, HttpServletRequest servletRequest, HttpHeaders httpHeaders) { Optional remoteAddress = Optional.ofNullable(servletRequest.getRemoteAddr()); Optional identity = authenticatedIdentity(servletRequest); @@ -194,7 +206,7 @@ private Query registerQuery(String statement, HttpServletRequest servletRequest, MultivaluedMap headers = httpHeaders.getRequestHeaders(); SessionContext sessionContext = sessionContextFactory.createSessionContext(headers, remoteAddress, identity); - Query query = new Query(statement, sessionContext, dispatchManager, queryInfoUrlFactory, tracer); + Query query = new Query(statement, queryLanguage, sessionContext, dispatchManager, queryInfoUrlFactory, tracer); queryManager.registerQuery(query); // let authentication filter know that identity lifecycle has been handed off @@ -315,6 +327,7 @@ enum SubmissionState private static final class Query { private final String query; + private final QueryLanguage queryLanguage; private final SessionContext sessionContext; private final DispatchManager dispatchManager; private final QueryId queryId; @@ -327,9 +340,10 @@ private static final class Query private final AtomicReference submissionGate = new AtomicReference<>(NOT_SUBMITTED); private final SettableFuture creationFuture = SettableFuture.create(); - public Query(String query, SessionContext sessionContext, DispatchManager dispatchManager, QueryInfoUrlFactory queryInfoUrlFactory, Tracer tracer) + public Query(String query, QueryLanguage queryLanguage, SessionContext sessionContext, DispatchManager dispatchManager, QueryInfoUrlFactory queryInfoUrlFactory, Tracer tracer) { this.query = requireNonNull(query, "query is null"); + this.queryLanguage = requireNonNull(queryLanguage, "queryLanguage is null"); this.sessionContext = requireNonNull(sessionContext, "sessionContext is null"); this.dispatchManager = requireNonNull(dispatchManager, "dispatchManager is null"); this.queryId = dispatchManager.createQueryId(); @@ -385,7 +399,7 @@ private void submitIfNeeded() { if (submissionGate.compareAndSet(NOT_SUBMITTED, SUBMITTED)) { querySpan.addEvent("submit"); - creationFuture.setFuture(dispatchManager.createQuery(queryId, querySpan, slug, sessionContext, query)); + creationFuture.setFuture(dispatchManager.createQuery(queryId, querySpan, slug, sessionContext, query, queryLanguage)); } } diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryLanguage.java b/core/trino-main/src/main/java/io/trino/execution/QueryLanguage.java new file mode 100644 index 000000000000..245ea30a749d --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/execution/QueryLanguage.java @@ -0,0 +1,20 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +public enum QueryLanguage +{ + SQL, + HOGQL, +} diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java index dc093feb4a01..14a7cece996d 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java @@ -16,6 +16,7 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Inject; import io.trino.Session; +import io.trino.hogql.compiler.HogQlCompiler; import io.trino.spi.TrinoException; import io.trino.spi.resourcegroups.QueryType; import io.trino.sql.parser.ParsingException; @@ -41,17 +42,39 @@ public class QueryPreparer { private final SqlParser sqlParser; + private final Optional hogQlCompiler; - @Inject public QueryPreparer(SqlParser sqlParser) + { + this(sqlParser, Optional.empty()); + } + + @Inject + public QueryPreparer(SqlParser sqlParser, HogQlCompiler hogQlCompiler) + { + this(sqlParser, Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null"))); + } + + private QueryPreparer(SqlParser sqlParser, Optional hogQlCompiler) { this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + this.hogQlCompiler = requireNonNull(hogQlCompiler, "hogQlCompiler is null"); } public PreparedQuery prepareQuery(Session session, String query) throws ParsingException, TrinoException { - Statement wrappedStatement = sqlParser.createStatement(query); + return prepareQuery(session, query, QueryLanguage.SQL); + } + + public PreparedQuery prepareQuery(Session session, String query, QueryLanguage queryLanguage) + throws ParsingException, TrinoException + { + requireNonNull(queryLanguage, "queryLanguage is null"); + Statement wrappedStatement = switch (queryLanguage) { + case SQL -> sqlParser.createStatement(query); + case HOGQL -> hogQlCompiler.orElseThrow(() -> new IllegalStateException("HogQL compiler is not configured")).compile(query); + }; return prepareQuery(session, wrappedStatement); } diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlConfig.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlConfig.java new file mode 100644 index 000000000000..525350a90d02 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlConfig.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; + +public class HogQlConfig +{ + private boolean enabled; + + public boolean isEnabled() + { + return enabled; + } + + @Config("hogql.enabled") + @ConfigDescription("Enable the native HogQL query submission endpoint") + public HogQlConfig setEnabled(boolean enabled) + { + this.enabled = enabled; + return this; + } +} diff --git a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java index c6e492a8f409..5527eba49799 100644 --- a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java +++ b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java @@ -41,6 +41,7 @@ import io.trino.dispatcher.DispatchManager; import io.trino.dispatcher.DispatchQueryFactory; import io.trino.dispatcher.FailedDispatchQueryFactory; +import io.trino.dispatcher.HogQlStatementResource; import io.trino.dispatcher.LocalDispatchQueryFactory; import io.trino.dispatcher.QueuedStatementResource; import io.trino.event.QueryMonitor; @@ -97,6 +98,8 @@ import io.trino.execution.scheduler.policy.AllAtOnceExecutionPolicy; import io.trino.execution.scheduler.policy.ExecutionPolicy; import io.trino.execution.scheduler.policy.PhasedExecutionPolicy; +import io.trino.hogql.HogQlConfig; +import io.trino.hogql.compiler.HogQlCompiler; import io.trino.memory.ClusterMemoryManager; import io.trino.memory.ForMemoryManager; import io.trino.memory.LeastWastedEffortTaskLowMemoryKiller; @@ -175,6 +178,11 @@ protected void setup(Binder binder) jsonCodecBinder(binder).bindJsonCodec(TaskInfo.class); jaxrsBinder(binder).bind(QueuedStatementResource.class); jaxrsBinder(binder).bind(ExecutingStatementResource.class); + configBinder(binder).bindConfig(HogQlConfig.class); + binder.bind(HogQlCompiler.class).in(Scopes.SINGLETON); + if (buildConfigObject(HogQlConfig.class).isEnabled()) { + jaxrsBinder(binder).bind(HogQlStatementResource.class); + } binder.bind(StatementHttpExecutionMBean.class).in(Scopes.SINGLETON); newExporter(binder).export(StatementHttpExecutionMBean.class).withGeneratedName(); binder.bind(QueryInfoUrlFactory.class).in(Scopes.SINGLETON); diff --git a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java new file mode 100644 index 000000000000..c2d6872b68e0 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java @@ -0,0 +1,142 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.dispatcher; + +import com.google.common.collect.ImmutableList; +import io.airlift.http.client.HeaderName; +import io.airlift.http.client.HttpClient; +import io.airlift.http.client.StatusResponseHandler.StatusResponse; +import io.airlift.http.client.jetty.JettyHttpClient; +import io.airlift.json.JsonCodec; +import io.airlift.json.JsonCodecFactory; +import io.airlift.json.JsonMapperProvider; +import io.trino.client.QueryDataJacksonModule; +import io.trino.client.QueryResults; +import io.trino.client.ResultRowsDecoder; +import io.trino.plugin.tpch.TpchPlugin; +import io.trino.server.testing.TestingTrinoServer; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static io.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; +import static io.airlift.http.client.Request.Builder.prepareGet; +import static io.airlift.http.client.Request.Builder.preparePost; +import static io.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; +import static io.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; +import static io.airlift.testing.Closeables.closeAll; +import static io.trino.client.ProtocolHeaders.TRINO_HEADERS; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +class TestHogQlStatementResource +{ + private static final HeaderName REQUEST_USER_HEADER = HeaderName.of(TRINO_HEADERS.requestUser()); + private static final JsonCodec QUERY_RESULTS_CODEC = new JsonCodecFactory(new JsonMapperProvider() + .withModules(Set.of(new QueryDataJacksonModule())) + .get()) + .jsonCodec(QueryResults.class); + + @Test + public void testEndpointIsDisabledByDefault() + throws Exception + { + HttpClient client = new JettyHttpClient(); + TestingTrinoServer server = TestingTrinoServer.create(); + try { + StatusResponse response = client.execute( + preparePost() + .setUri(server.resolve("/v1/hogql")) + .setHeader(REQUEST_USER_HEADER, "user") + .setBodyGenerator(createStaticBodyGenerator("SELECT 1", UTF_8)) + .build(), + createStatusResponseHandler()); + + assertThat(response.getStatusCode()).isEqualTo(404); + } + finally { + closeAll(server, client); + } + } + + @Test + public void testEnabledEndpointUsesStatementProtocol() + throws Exception + { + HttpClient client = new JettyHttpClient(); + TestingTrinoServer server = TestingTrinoServer.builder() + .setProperties(Map.of("hogql.enabled", "true")) + .build(); + try { + server.installPlugin(new TpchPlugin()); + server.createCatalog("tpch", "tpch"); + + List hogqlResults = runToCompletion(client, server, "/v1/hogql", "SELECT 1"); + assertThat(hogqlResults.getFirst().getNextUri().getPath()).startsWith("/v1/statement/queued/"); + assertThat(rows(hogqlResults)).containsExactly(ImmutableList.of(1)); + + List tableScanResults = runToCompletion(client, server, "/v1/hogql", "SELECT nationkey FROM tpch.tiny.nation"); + assertThat(rows(tableScanResults)).hasSize(25); + + List sqlResults = runToCompletion(client, server, "/v1/statement", "SELECT 2"); + assertThat(rows(sqlResults)).containsExactly(ImmutableList.of(2)); + + List invalidResults = runToCompletion(client, server, "/v1/hogql", "SELECT FROM"); + assertThat(invalidResults.getLast().getError().getErrorName()).isEqualTo("SYNTAX_ERROR"); + } + finally { + closeAll(server, client); + } + } + + private static List runToCompletion(HttpClient client, TestingTrinoServer server, String path, String query) + { + ImmutableList.Builder results = ImmutableList.builder(); + QueryResults current = client.execute( + preparePost() + .setUri(server.resolve(path)) + .setHeader(REQUEST_USER_HEADER, "user") + .setBodyGenerator(createStaticBodyGenerator(query, UTF_8)) + .build(), + createJsonResponseHandler(QUERY_RESULTS_CODEC)); + results.add(current); + + while (current.getNextUri() != null) { + current = client.execute( + prepareGet() + .setUri(current.getNextUri()) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createJsonResponseHandler(QUERY_RESULTS_CODEC)); + results.add(current); + } + return results.build(); + } + + private static List> rows(List results) + throws Exception + { + ImmutableList.Builder> rows = ImmutableList.builder(); + try (ResultRowsDecoder decoder = new ResultRowsDecoder()) { + for (QueryResults result : results) { + if (result.getData() != null) { + rows.addAll(decoder.toRows(result)); + } + } + } + return rows.build(); + } +} diff --git a/core/trino-main/src/test/java/io/trino/execution/BenchmarkQueryPreparer.java b/core/trino-main/src/test/java/io/trino/execution/BenchmarkQueryPreparer.java new file mode 100644 index 000000000000..f84f03b4e85e --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/execution/BenchmarkQueryPreparer.java @@ -0,0 +1,65 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.sql.parser.SqlParser; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.RunnerException; + +import java.util.concurrent.TimeUnit; + +import static io.trino.SessionTestUtils.TEST_SESSION; +import static io.trino.jmh.Benchmarks.benchmark; + +@State(Scope.Thread) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) +public class BenchmarkQueryPreparer +{ + private static final String QUERY = "SELECT event, properties FROM ducklake.default.events"; + + private final SqlParser sqlParser = new SqlParser(); + private final QueryPreparer sqlOnlyQueryPreparer = new QueryPreparer(sqlParser); + private final QueryPreparer hogQlCapableQueryPreparer = new QueryPreparer(sqlParser, new HogQlCompiler()); + + @Benchmark + public QueryPreparer.PreparedQuery prepareStandardSqlWithoutHogQl() + { + return sqlOnlyQueryPreparer.prepareQuery(TEST_SESSION, QUERY); + } + + @Benchmark + public QueryPreparer.PreparedQuery prepareStandardSqlWithHogQlAvailable() + { + return hogQlCapableQueryPreparer.prepareQuery(TEST_SESSION, QUERY); + } + + static void main() + throws RunnerException + { + benchmark(BenchmarkQueryPreparer.class).run(); + } +} diff --git a/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java b/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java index a45c0673617f..4269d4281a69 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java @@ -15,13 +15,16 @@ import io.trino.Session; import io.trino.execution.QueryPreparer.PreparedQuery; +import io.trino.hogql.compiler.HogQlCompiler; import io.trino.sql.parser.ParsingException; import io.trino.sql.parser.SqlParser; import io.trino.sql.tree.AllColumns; import io.trino.sql.tree.QualifiedName; +import io.trino.sql.tree.Query; import org.junit.jupiter.api.Test; import static io.trino.SessionTestUtils.TEST_SESSION; +import static io.trino.execution.QueryLanguage.HOGQL; import static io.trino.spi.StandardErrorCode.INVALID_PARAMETER_USAGE; import static io.trino.spi.StandardErrorCode.NOT_FOUND; import static io.trino.sql.QueryUtil.selectList; @@ -36,6 +39,7 @@ public class TestQueryPreparer { private static final SqlParser SQL_PARSER = new SqlParser(); private static final QueryPreparer QUERY_PREPARER = new QueryPreparer(SQL_PARSER); + private static final QueryPreparer HOGQL_QUERY_PREPARER = new QueryPreparer(SQL_PARSER, new HogQlCompiler()); @Test public void testSelectStatement() @@ -44,6 +48,13 @@ public void testSelectStatement() assertThat(preparedQuery.getStatement()).isEqualTo(simpleQuery(selectList(new AllColumns()), table(QualifiedName.of("foo")))); } + @Test + public void testHogQlStatement() + { + PreparedQuery preparedQuery = HOGQL_QUERY_PREPARER.prepareQuery(TEST_SESSION, "SELECT 1", HOGQL); + assertThat(preparedQuery.getStatement()).isInstanceOf(Query.class); + } + @Test public void testExecuteStatement() { diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlConfig.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlConfig.java new file mode 100644 index 000000000000..bdbb8e2bf1ef --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlConfig.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestHogQlConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(HogQlConfig.class) + .setEnabled(false)); + } + + @Test + public void testExplicitPropertyMappings() + { + assertFullMapping( + Map.of("hogql.enabled", "true"), + new HogQlConfig().setEnabled(true)); + } +} From cca1e82694b5de5a4a2cbc868b975769ff601da1 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Wed, 26 Aug 2026 21:54:01 -0400 Subject: [PATCH 003/136] feat(hogql): pin canonical language contract --- core/trino-hogql-parser/pom.xml | 10 + .../hogql/parser/canonical/HogQLLexer.g4 | 450 +++++++++++++++++ .../hogql/parser/canonical/HogQLParser.g4 | 458 ++++++++++++++++++ .../hogql/parser/HogQlLanguageContract.java | 94 ++++ .../hogql/parser/HogQlLanguageVersion.java | 52 ++ .../io/trino/hogql/parser/HogQlParser.java | 11 + .../META-INF/LICENSE-posthog-hogql.txt | 19 + .../1.0.0/grammar/HogQLLexer.common.g4 | 357 ++++++++++++++ .../language/1.0.0/grammar/HogQLLexer.java.g4 | 81 ++++ .../language/1.0.0/grammar/HogQLParser.g4 | 445 +++++++++++++++++ .../hogql/parser/language/1.0.0/language.json | 25 + .../parser/language/1.0.0/source-lock.json | 6 + .../parser/TestHogQlLanguageContract.java | 109 +++++ 13 files changed, 2117 insertions(+) create mode 100644 core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/canonical/HogQLLexer.g4 create mode 100644 core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/canonical/HogQLParser.g4 create mode 100644 core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageContract.java create mode 100644 core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageVersion.java create mode 100644 core/trino-hogql-parser/src/main/resources/META-INF/LICENSE-posthog-hogql.txt create mode 100644 core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLLexer.common.g4 create mode 100644 core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLLexer.java.g4 create mode 100644 core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLParser.g4 create mode 100644 core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/language.json create mode 100644 core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/source-lock.json create mode 100644 core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlLanguageContract.java diff --git a/core/trino-hogql-parser/pom.xml b/core/trino-hogql-parser/pom.xml index 958723e2cff7..0e9b42d6bdcd 100644 --- a/core/trino-hogql-parser/pom.xml +++ b/core/trino-hogql-parser/pom.xml @@ -14,6 +14,16 @@ Trino - HogQL parser + + com.fasterxml.jackson.core + jackson-annotations + + + + io.airlift + json + + org.antlr antlr4-runtime diff --git a/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/canonical/HogQLLexer.g4 b/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/canonical/HogQLLexer.g4 new file mode 100644 index 000000000000..1dd9302c0341 --- /dev/null +++ b/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/canonical/HogQLLexer.g4 @@ -0,0 +1,450 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +lexer grammar HogQLLexer; + +@members { + +private static boolean isAsciiAlpha(int character) { + return character >= 0 && character < 128 && + ((character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z')); +} + +private static boolean isAsciiAlphanumeric(int character) { + return isAsciiAlpha(character) || (character >= '0' && character <= '9'); +} + +private static boolean isAsciiWhitespace(int character) { + return character == ' ' || character == '\t' || character == '\n' || + character == '\u000B' || character == '\f' || character == '\r'; +} + +private int skipWhitespaceAndComments(int index) { + while (true) { + int character = _input.LA(index); + if (isAsciiWhitespace(character)) { + index++; + continue; + } + + if (character == '/' && _input.LA(index + 1) == '/') { + index += 2; + } + else if (character == '-' && _input.LA(index + 1) == '-') { + index += 2; + } + else if (character == '#') { + index++; + } + else { + return index; + } + + while (true) { + character = _input.LA(index); + if (character <= 0 || character == '\n' || character == '\r') { + break; + } + index++; + } + } +} + +private boolean isOpeningTag() { + int firstCharacter = _input.LA(1); + if (!isAsciiAlpha(firstCharacter) && firstCharacter != '_') { + return false; + } + + int index = 2; + while (true) { + int character = _input.LA(index); + if (isAsciiAlphanumeric(character) || character == '_' || character == '-') { + index++; + } + else { + break; + } + } + + int character = _input.LA(index); + if (character == '>' || character == '/') { + return true; + } + + if (isAsciiWhitespace(character)) { + index = skipWhitespaceAndComments(index + 1); + character = _input.LA(index); + return isAsciiAlphanumeric(character) || character == '_' || character == '>' || character == '/'; + } + + return false; +} + +} + +// NB! We cat either HogQLLexter.cpp.g4 or HogQLLexter.python.g4 when generating the grammar. + +// NOTE: don't forget to add new keywords to the parser rule "keyword"! + +// Keywords + +ALL: A L L; +AND: A N D; +ANTI: A N T I; +ANY: A N Y; +ARRAY: A R R A Y; +AS: A S; +ASCENDING: A S C | A S C E N D I N G; +ASOF: A S O F; +BETWEEN: B E T W E E N; +BOTH: B O T H; +BY: B Y; +CASE: C A S E; +CAST: C A S T; +CATCH: C A T C H; +COHORT: C O H O R T; +COLLATE: C O L L A T E; +COLUMNS: C O L U M N S; +CROSS: C R O S S; +CUBE: C U B E; +CURRENT: C U R R E N T; +DATE: D A T E; +DAY: D A Y; +DESC: D E S C; +DESCENDING: D E S C E N D I N G; +DISTINCT: D I S T I N C T; +ELSE: E L S E; +END: E N D; +EXCEPT: E X C E P T; +EXCLUDE: E X C L U D E; +EXTRACT: E X T R A C T; +FINAL: F I N A L; +FILL: F I L L; +FILTER: F I L T E R; +FINALLY: F I N A L L Y; +FIRST: F I R S T; +FN: F N; +FOLLOWING: F O L L O W I N G; +FOR: F O R; +FROM: F R O M; +FULL: F U L L; +FUN: F U N; +GROUP: G R O U P; +GROUPING: G R O U P I N G; +HAVING: H A V I N G; +HOUR: H O U R; +ID: I D; +IF: I F; +ILIKE: I L I K E; +IGNORE: I G N O R E; +INCLUDE: I N C L U D E; +IN: I N; +INF: I N F | I N F I N I T Y; +INNER: I N N E R; +INTERSECT: I N T E R S E C T; +INTERPOLATE: I N T E R P O L A T E; +INTERVAL: I N T E R V A L; +IS: I S; +JOIN: J O I N; +KEY: K E Y; +LAMBDA: L A M B D A; +LAST: L A S T; +LEADING: L E A D I N G; +LEFT: L E F T; +LET: L E T; +LIKE: L I K E; +LIMIT: L I M I T; +MATERIALIZED: M A T E R I A L I Z E D; +MINUTE: M I N U T E; +MONTH: M O N T H; +NAME: N A M E; +NATURAL: N A T U R A L; +NAN_SQL: N A N; // conflicts with macro NAN +NOT: N O T; +NULL_SQL: N U L L; // conflicts with macro NULL +NULLS: N U L L S; +OFFSET: O F F S E T; +ON: O N; +OR: O R; +ORDER: O R D E R; +OUTER: O U T E R; +OVER: O V E R; +PARTITION: P A R T I T I O N; +PIVOT: P I V O T; +POSITIONAL: P O S I T I O N A L; +PRECEDING: P R E C E D I N G; +PREWHERE: P R E W H E R E; +QUALIFY: Q U A L I F Y; +QUARTER: Q U A R T E R; +RANGE: R A N G E; +RECURSIVE: R E C U R S I V E; +REPLACE: R E P L A C E; +RETURN: R E T U R N; +RIGHT: R I G H T; +ROLLUP: R O L L U P; +ROW: R O W; +ROWS: R O W S; +SAMPLE: S A M P L E; +SECOND: S E C O N D; +SELECT: S E L E C T; +SEMI: S E M I; +SETS: S E T S; +SETTINGS: S E T T I N G S; +STEP: S T E P; +SUBSTRING: S U B S T R I N G; +THEN: T H E N; +THROW: T H R O W; +TIES: T I E S; +TIMESTAMP: T I M E S T A M P; +TIME: T I M E; +LOCAL: L O C A L; +ZONE: Z O N E; +TO: T O; +TOP: T O P; +TOTALS: T O T A L S; +TRAILING: T R A I L I N G; +TRIM: T R I M; +TRUNCATE: T R U N C A T E; +TRY: T R Y; +TRY_CAST: T R Y '_' C A S T; +UNBOUNDED: U N B O U N D E D; +UNION: U N I O N; +UNPIVOT: U N P I V O T; +USING: U S I N G; +VALUES: V A L U E S; +WEEK: W E E K; +WHEN: W H E N; +WHERE: W H E R E; +WHILE: W H I L E; +WINDOW: W I N D O W; +WITH: W I T H; +WITHIN: W I T H I N; +YEAR: Y E A R | Y Y Y Y; + +// Tokens + +// copied from clickhouse_driver/util/escape.py +ESCAPE_CHAR_COMMON + : BACKSLASH B + | BACKSLASH F + | BACKSLASH R + | BACKSLASH N + | BACKSLASH T + | BACKSLASH '0' + | BACKSLASH A + | BACKSLASH V + | BACKSLASH BACKSLASH + | BACKSLASH X HEX_DIGIT HEX_DIGIT; + +IDENTIFIER + : (LETTER | UNDERSCORE | DOLLAR) (LETTER | UNDERSCORE | DEC_DIGIT | DOLLAR)* + ; +QUOTED_IDENTIFIER + : BACKQUOTE ( ~([\\`]) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_SINGLE | (BACKQUOTE BACKQUOTE) )* BACKQUOTE + | QUOTE_DOUBLE ( ~([\\"]) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_DOUBLE | (QUOTE_DOUBLE QUOTE_DOUBLE) )* QUOTE_DOUBLE + ; +FLOATING_LITERAL + // Hex-float exponent: strict C99 `p`/`P` only — `e`/`E` stays a hex digit, so `0x1e5` is 485, not a float. + : HEXADECIMAL_LITERAL DOT HEX_DIGIT* P (PLUS | DASH)? DEC_DIGIT+ + | HEXADECIMAL_LITERAL P (PLUS | DASH)? DEC_DIGIT+ + | DECIMAL_LITERAL DOT DEC_DIGIT* E (PLUS | DASH)? DEC_DIGIT+ + | DOT DECIMAL_LITERAL E (PLUS | DASH)? DEC_DIGIT+ + | DECIMAL_LITERAL E (PLUS | DASH)? DEC_DIGIT+ + ; +// Binary literals (`0b1010`). Declared first so it wins the length-tie against MALFORMED_BINARY_LITERAL. +BINARY_LITERAL: '0' B BIN_DIGIT+; +OCTAL_LITERAL: '0' OCT_DIGIT+; +DECIMAL_LITERAL: DEC_DIGIT+; +HEXADECIMAL_LITERAL: '0' X HEX_DIGIT+; +// Postgres-16 `0o` octal — unsupported; lexed as a real token so the visitor can reject it clearly. +OCTAL_PREFIX_LITERAL: '0' [oO] DEC_DIGIT+; +// Malformed binary (`0b22`) BINARY_LITERAL didn't consume — caught so it can't re-tokenise as `0` + IDENTIFIER. +MALFORMED_BINARY_LITERAL: '0' [bB] DEC_DIGIT+; + +// It's important that quote-symbol is a single character. +STRING_LITERAL: QUOTE_SINGLE ( ~([\\']) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_SINGLE | (QUOTE_SINGLE QUOTE_SINGLE) )* QUOTE_SINGLE; + + +// Alphabet and allowed symbols + +fragment A: [aA]; +fragment B: [bB]; +fragment C: [cC]; +fragment D: [dD]; +fragment E: [eE]; +fragment F: [fF]; +fragment G: [gG]; +fragment H: [hH]; +fragment I: [iI]; +fragment J: [jJ]; +fragment K: [kK]; +fragment L: [lL]; +fragment M: [mM]; +fragment N: [nN]; +fragment O: [oO]; +fragment P: [pP]; +fragment Q: [qQ]; +fragment R: [rR]; +fragment S: [sS]; +fragment T: [tT]; +fragment U: [uU]; +fragment V: [vV]; +fragment W: [wW]; +fragment X: [xX]; +fragment Y: [yY]; +fragment Z: [zZ]; + +fragment LETTER: [a-zA-Z]; +fragment BIN_DIGIT: [01]; +fragment OCT_DIGIT: [0-7]; +fragment DEC_DIGIT: [0-9]; +fragment HEX_DIGIT: [0-9a-fA-F]; + +ARROW: '->'; +ASTERISK: '*'; +BACKQUOTE: '`'; +BACKSLASH: '\\'; +DOUBLECOLON: '::'; +COLONEQUALS: ':='; +COLON: ':'; +COMMA: ','; +CONCAT: '||'; +DASH: '-'; +DOLLAR: '$'; +DOT: '.'; +EQ_DOUBLE: '=='; +EQ_SINGLE: '='; +GT_EQ: '>='; +GT: '>'; +HASH: '#'; +IREGEX_SINGLE: '~*'; +IREGEX_DOUBLE: '=~*'; +LBRACE: '{' -> pushMode(DEFAULT_MODE); +LBRACKET: '['; +LPAREN: '('; +NULL_SAFE_EQ: '<=>'; +LT_EQ: '<='; +TAG_LT_SLASH: ' type(LT_SLASH), pushMode(HOGQLX_TAG_CLOSE); +TAG_LT_OPEN: '<' {isOpeningTag()}? -> type(LT), pushMode(HOGQLX_TAG_OPEN); +LT: '<'; +LT_SLASH: ''; +NOT_IREGEX: '!~*'; +NOT_REGEX: '!~'; +NULL_PROPERTY: '?.'; +NULLISH: '??'; +PERCENT: '%'; +PLUS: '+'; +QUERY: '?'; +QUOTE_DOUBLE: '"'; +QUOTE_SINGLE_TEMPLATE: 'f\'' -> pushMode(IN_TEMPLATE_STRING); // start of regular f'' template strings +QUOTE_SINGLE_TEMPLATE_FULL: 'F\'' -> pushMode(IN_FULL_TEMPLATE_STRING); // magic F' symbol used to parse "full text" templates +QUOTE_SINGLE: '\''; +REGEX_SINGLE: '~'; +REGEX_DOUBLE: '=~'; +RBRACE: '}' -> popMode; +RBRACKET: ']'; +RPAREN: ')'; +SEMICOLON: ';'; +SLASH: '/'; +SLASH_GT: '/>'; +UNDERSCORE: '_'; + +// Comments and whitespace +MULTI_LINE_COMMENT: '/*' .*? '*/' -> skip; +SINGLE_LINE_COMMENT: ('--' | '//') ~('\n'|'\r')* ('\n' | '\r' | EOF) -> skip; +// MySQL-style `#` comments. `#` is excluded so positional references (`#1`) keep +// working — a `#` comment whose text starts with a digit is the one MySQL-ism this rejects. +HASH_COMMENT: '#' (~[0-9\n\r] ~[\n\r]*)? ('\n' | '\r' | EOF) -> skip; +// whitespace is hidden and not skipped so that it's preserved in ANTLR errors like "no viable alternative" +// The class is the full Unicode `White_Space` set, not just ASCII: a +// NO-BREAK SPACE or other Unicode space (often pasted in from rich +// editors or docs) is genuine whitespace and must keep separating +// tokens. Recognising it here keeps such programs valid — otherwise it +// would fall through to UNEXPECTED_CHARACTER below and fail the whole +// parse. U+FEFF (BOM) is included too, so a file saved with a +// byte-order mark still parses. +WHITESPACE: [ \t\r\n\u000B\u000C\u0085\u00A0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF] -> channel(HIDDEN); + +// Catch-all for any character no rule above matched. Without this the +// lexer raises a recoverable token-recognition error and DROPS the +// character — so stray input (a JavaScript `!`, `&&`, …) silently +// vanishes and the surrounding text parses as a different, valid-looking +// program. Emitting an explicit token instead means the parser has no +// rule for it and fails loudly with a SyntaxError. Listed last so it +// only ever fires as a true fallback (maximal munch keeps `!=`, `!~`, +// multi-character operators, comments, etc. intact). +UNEXPECTED_CHARACTER: . ; + +// ───────── f' TEMPLATE STRING MODE ───────── +mode IN_TEMPLATE_STRING; +STRING_TEXT: ((~([\\'{])) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_SINGLE | (BACKSLASH LBRACE) | (QUOTE_SINGLE QUOTE_SINGLE))+; +STRING_ESCAPE_TRIGGER: LBRACE -> pushMode(DEFAULT_MODE); +STRING_QUOTE_SINGLE: QUOTE_SINGLE -> type(QUOTE_SINGLE), popMode; + +// ───────── F' FULL TEMPLATE STRING MODE ───────── +// a magic F' takes us to "full template strings" mode, where we don't need to escape single quotes and parse until EOF +// this can't be used within a normal columnExpr, but has to be parsed for separately +mode IN_FULL_TEMPLATE_STRING; +FULL_STRING_TEXT: ((~([{])) | ESCAPE_CHAR_COMMON | (BACKSLASH LBRACE))+; +FULL_STRING_ESCAPE_TRIGGER: LBRACE -> pushMode(DEFAULT_MODE); + +// ───────── HOGQLX TAG MODE for opening/self-closing tags ───────── +mode HOGQLX_TAG_OPEN; + +TAG_SELF_CLOSE_GT : '/>' -> type(SLASH_GT), popMode; // +TAG_OPEN_GT : '>' -> type(GT), popMode, pushMode(HOGQLX_TEXT); // + +// Skip comments between attributes — without these, the recoverable lexer error drops the delimiters and re-tokenises the body as phantom attributes. +TAG_MULTI_LINE_COMMENT : '/*' .*? '*/' -> skip; +TAG_SINGLE_LINE_COMMENT : ('--' | '//') ~('\n'|'\r')* ('\n' | '\r' | EOF) -> skip; + +// minimal token set; map everything back to the default token types +TAG_IDENT : [a-zA-Z_][a-zA-Z0-9_-]* -> type(IDENTIFIER); +TAG_EQ : '=' -> type(EQ_SINGLE); +TAG_STRING : STRING_LITERAL -> type(STRING_LITERAL); +TAG_WS : [ \t\r\n]+ -> channel(HIDDEN); +TAG_LBRACE : '{' -> type(LBRACE), pushMode(DEFAULT_MODE); +// Catch-all for unmatched bytes (e.g. `#`, `&`, `@`) so the parser fails loudly instead of silently re-tokenising the surrounding text. +TAG_UNEXPECTED : . -> type(UNEXPECTED_CHARACTER); + + +// ───────── HOGQLX TAG MODE for closing tags ───────── +mode HOGQLX_TAG_CLOSE; + +TAGC_GT : '>' -> type(GT), popMode; // *** no TEXT push *** +TAGC_MULTI_LINE_COMMENT : '/*' .*? '*/' -> skip; +TAGC_SINGLE_LINE_COMMENT : ('--' | '//') ~('\n'|'\r')* ('\n' | '\r' | EOF) -> skip; +TAGC_IDENT : [a-zA-Z_][a-zA-Z0-9_-]* -> type(IDENTIFIER); +TAGC_WS : [ \t\r\n]+ -> channel(HIDDEN); +TAGC_UNEXPECTED : . -> type(UNEXPECTED_CHARACTER); + + +// ───────── HOGQLX TEXT MODE ───────── +mode HOGQLX_TEXT; + +HOGQLX_TEXT_TEXT + : ~[<{]+ ; // everything except “{” or “<” + +HOGQLX_TEXT_LBRACE + : '{' -> type(LBRACE), pushMode(DEFAULT_MODE); + +HOGQLX_TEXT_LT_SLASH + : ' type(LT_SLASH), popMode, pushMode(HOGQLX_TAG_CLOSE); + +HOGQLX_TEXT_LT + : '<' -> type(LT), pushMode(HOGQLX_TAG_OPEN); + +HOGQLX_TEXT_WS + : [ \t\r\n]+ -> channel(HIDDEN); diff --git a/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/canonical/HogQLParser.g4 b/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/canonical/HogQLParser.g4 new file mode 100644 index 000000000000..a91e0a2d0f78 --- /dev/null +++ b/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/canonical/HogQLParser.g4 @@ -0,0 +1,458 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +parser grammar HogQLParser; + +options { + tokenVocab = HogQLLexer; +} + + +program: declaration* EOF; + +declaration: varDecl | statement ; + +expression: columnExpr; + +varDecl: LET identifier ( COLONEQUALS expression )? ; +identifierList: nestedIdentifier (COMMA nestedIdentifier)* COMMA?; + +statement : returnStmt + | throwStmt + | tryCatchStmt + | ifStmt + | whileStmt + | forInStmt + | forStmt + | funcStmt + | block + | exprStmt + | emptyStmt + ; + +returnStmt : RETURN expression? SEMICOLON?; +throwStmt : THROW expression SEMICOLON?; +catchBlock : CATCH (LPAREN catchVar=identifier (COLON catchType=identifier)? RPAREN)? catchStmt=block; +tryCatchStmt : TRY tryStmt=block catchBlock* (FINALLY finallyStmt=block)?; +ifStmt : IF LPAREN expression RPAREN statement ( ELSE statement )? ; +whileStmt : WHILE LPAREN expression RPAREN statement SEMICOLON?; +forStmt : FOR LPAREN + (initializerVarDeclr=varDecl | initializerVarAssignment=varAssignment | initializerExpression=expression)? SEMICOLON + condition=expression? SEMICOLON + (incrementVarDeclr=varDecl | incrementVarAssignment=varAssignment | incrementExpression=expression)? + RPAREN statement SEMICOLON?; +forInStmt : FOR LPAREN LET identifier (COMMA identifier)? IN expression RPAREN statement SEMICOLON?; +funcStmt : (FN | FUN) identifier LPAREN identifierList? RPAREN block; +varAssignment : expression COLONEQUALS expression ; +// Assignment folded in as an optional suffix: one expression-leading alternative in +// `statement` means `declaration*` parses without unbounded lookahead. `varAssignment` is forStmt-only. +exprStmt : expression (COLONEQUALS expression)? SEMICOLON?; +emptyStmt : SEMICOLON ; +block : LBRACE declaration* RBRACE ; + +kvPair: expression ':' expression ; +kvPairList: kvPair (COMMA kvPair)* COMMA?; + + +// SELECT statement +select: (selectSetStmt | selectStmt | hogqlxTagElement) SEMICOLON? EOF; + +selectStmtWithParens: selectStmt | withClause LPAREN selectSetStmt RPAREN | LPAREN selectSetStmt RPAREN | placeholder; + +subsequentSelectSetClause: (EXCEPT ALL (BY NAME)? | EXCEPT (BY NAME)? | UNION ALL (BY NAME)? | UNION DISTINCT (BY NAME)? | UNION (BY NAME)? | INTERSECT ALL (BY NAME)? | INTERSECT DISTINCT (BY NAME)? | INTERSECT (BY NAME)?) selectStmtWithParens; +selectSetStmt: selectStmtWithParens (subsequentSelectSetClause)* orderByClause? limitAndOffsetClauseOptional?; +limitAndOffsetClauseOptional + : LIMIT columnExpr PERCENT? (COMMA columnExpr)? (WITH TIES)? + | LIMIT columnExpr PERCENT? (WITH TIES)? OFFSET columnExpr + | OFFSET columnExpr + ; + +selectStmt: + with=withClause? + SELECT DISTINCT? topClause? + columns=selectColumnExprListBeforeFrom + from=fromClause? + arrayJoinClause? + prewhereClause? + where=whereClause? + (USING? sampleClause)? + groupByClause? (WITH (CUBE | ROLLUP))? (WITH TOTALS)? + havingClause? + qualifyClause? + (USING sampleClause)? + windowClause? + orderByClause? + limitByClause? + (limitAndOffsetClause | offsetOnlyClause)? + settingsClause? + ; + +withClause: WITH RECURSIVE? withExprList; +topClause: TOP DECIMAL_LITERAL (WITH TIES)?; +fromClause: FROM joinExpr; +arrayJoinClause: (LEFT | INNER)? ARRAY JOIN columnExprList; +windowClause: WINDOW identifier AS LPAREN windowExpr RPAREN (COMMA identifier AS LPAREN windowExpr RPAREN)*; +prewhereClause: PREWHERE columnExpr; +whereClause: WHERE columnExpr; +groupByClause: GROUP BY ( + ALL + | (CUBE | ROLLUP) LPAREN columnExprList RPAREN + | GROUPING SETS LPAREN groupingSetList RPAREN + | columnExprList + ); +groupingSetList: groupingSet (COMMA groupingSet)*; +groupingSet: LPAREN columnExprList? RPAREN; +havingClause: HAVING columnExpr; +qualifyClause: QUALIFY columnExpr; +orderByClause: ORDER BY orderExprList interpolateClause?; +interpolateClause: INTERPOLATE (LPAREN interpolateExpr (COMMA interpolateExpr)* RPAREN)?; +projectionOrderByClause: ORDER BY columnExprList; +limitByClause: LIMIT limitExpr BY columnExprList; +limitAndOffsetClause + : LIMIT columnExpr PERCENT? (COMMA columnExpr)? (WITH TIES)? // compact OFFSET-optional form + | LIMIT columnExpr PERCENT? (WITH TIES)? OFFSET columnExpr // verbose OFFSET-included form with WITH TIES + ; +offsetOnlyClause: OFFSET columnExpr; +settingsClause: SETTINGS settingExprList; + +valuesClause: VALUES valuesRow (COMMA valuesRow)*; +valuesRow: LPAREN columnExpr (COMMA columnExpr)* RPAREN; + +joinExpr + : joinExpr NATURAL? joinOp? JOIN joinExpr joinConstraintClause? # JoinExprOp + | joinExpr POSITIONAL JOIN joinExpr joinConstraintClause? # JoinExprPositional + | joinExpr joinOpCross joinExpr # JoinExprCrossOp + | joinExpr PIVOT LPAREN columnExprList pivotColumnList (GROUP BY columnExprList)? RPAREN # JoinExprPivot + | joinExpr UNPIVOT (INCLUDE NULLS)? LPAREN unpivotColumnList RPAREN # JoinExprUnpivot + | tableExpr FINAL? sampleClause? # JoinExprTable + | LPAREN joinExpr RPAREN # JoinExprParens + ; +joinOp + : ((ALL | ANY | ASOF)? INNER | INNER (ALL | ANY | ASOF)? | (ALL | ANY | ASOF) | ANTI | SEMI | ASOF (ANTI | SEMI)) # JoinOpInner + | ( (SEMI | ALL | ANTI | ANY | ASOF)? (LEFT | RIGHT) OUTER? + | (LEFT | RIGHT) OUTER? (SEMI | ALL | ANTI | ANY | ASOF)? + | ASOF (ANTI | SEMI) (LEFT | RIGHT) OUTER? + ) # JoinOpLeftRight + | ((ALL | ANY | ASOF)? FULL OUTER? | FULL OUTER? (ALL | ANY | ASOF)?) # JoinOpFull + ; +joinOpCross + : CROSS JOIN + | COMMA + ; +joinConstraintClause + : ON columnExprList + | USING LPAREN columnExprList RPAREN + | USING columnExprList + ; + +sampleClause: SAMPLE ratioExpr PERCENT? (OFFSET ratioExpr)? (LPAREN identifier RPAREN)?; +limitExpr: columnExpr ((COMMA | OFFSET) columnExpr)?; +orderExprList: orderExpr (COMMA orderExpr)*; +orderExpr: columnExpr (ASCENDING | DESCENDING | DESC)? (NULLS (FIRST | LAST))? (COLLATE STRING_LITERAL)? withFillClause?; +withFillClause: WITH FILL (FROM columnExpr)? (TO columnExpr)? (STEP columnExpr)?; +interpolateExpr: columnExpr (AS columnExpr)?; +ratioExpr: placeholder | numberLiteral (SLASH numberLiteral)?; +settingExprList: settingExpr (COMMA settingExpr)*; +settingExpr: identifier EQ_SINGLE literal; + +windowExpr: winPartitionByClause? winOrderByClause? winFrameClause?; +winPartitionByClause: PARTITION BY columnExprList; +winOrderByClause: ORDER BY orderExprList; +withinGroupClause: WITHIN GROUP LPAREN orderByClause RPAREN; +winFrameClause: (ROWS | RANGE) winFrameExtend; +winFrameExtend + : winFrameBound # frameStart + | BETWEEN winFrameBound AND winFrameBound # frameBetween + ; +winFrameBound: (CURRENT ROW | UNBOUNDED PRECEDING | UNBOUNDED FOLLOWING | columnExpr PRECEDING | columnExpr FOLLOWING); +//rangeClause: RANGE LPAREN (MIN identifier MAX identifier | MAX identifier MIN identifier) RPAREN; + +// Columns +expr: columnExpr EOF; +columnTypeExpr + : columnTypeExpr LBRACKET DECIMAL_LITERAL? RBRACKET # ColumnTypeExprArray // INTEGER[], VARCHAR[3] + | identifier LPAREN identifier columnTypeExpr (COMMA identifier columnTypeExpr)* COMMA? RPAREN # ColumnTypeExprNested // Nested + | identifier LPAREN enumValue (COMMA enumValue)* COMMA? RPAREN # ColumnTypeExprEnum // Enum + | identifier LPAREN columnTypeExpr (COMMA columnTypeExpr)* COMMA? RPAREN # ColumnTypeExprComplex // Array, Tuple + | identifier LPAREN columnExprList? RPAREN # ColumnTypeExprParam // FixedString(N) + | identifier identifier+ # ColumnTypeExprCompound // TIME WITH TIME ZONE + | identifier # ColumnTypeExprSimple // UInt64 + ; +// Restricted type expr for :: casts — no parenthesized variants to avoid ambiguity with function calls +columnTypeCastExpr + : columnTypeCastIdentifier WITH LOCAL? TIME ZONE # ColumnTypeCastExprWithTimeZone + | columnTypeCastIdentifier # ColumnTypeCastExprSimple + ; +columnTypeCastIdentifier + : IDENTIFIER + | QUOTED_IDENTIFIER + | interval + | keywordForTypeCast + ; +keywordForTypeCast + : DATE + | TIME + | TIMESTAMP + | INTERVAL + ; +columnExprList: columnExpr (COMMA columnExpr)* COMMA?; +selectColumnExprListBeforeFrom + : selectColumnExpr (COMMA selectColumnExpr)* COMMA # SelectColumnExprListBeforeFromTrailingComma + | selectColumnExprList # SelectColumnExprListBeforeFromPlain + ; +selectColumnExprList: selectColumnExpr (COMMA selectColumnExpr)* COMMA?; +selectColumnExpr + : identifier COLON columnExpr # ColumnExprAliasBefore + | FROM implicitAlias # ColumnExprInvalidFromImplicitAlias + | columnExpr # ColumnExprSelectValue + | columnExpr implicitAlias # ColumnExprAliasImplicit + ; +// Two precedence layers. `columnExpr` is the outer boolean/ternary/alias tier (loosest +// binding). `columnExprValue` holds everything tighter — arithmetic, comparisons, NOT, +// BETWEEN, and the primary/leaf productions. BETWEEN lives in the value tier at the comparison +// level, so its tested expression and both bounds bind tighter than AND/OR/NOT (matching +// ClickHouse and SQL); this is what fixes `a BETWEEN low AND high AND rest` grouping as +// `(a BETWEEN low AND high) AND rest` rather than letting the bounds swallow the AND chain. +// Splitting into two rules is the only way to express this in ANTLR4: the interior operand of +// a left-recursive alternative always parses at precedence 0, so a flat rule cannot stop the +// bounds from consuming AND (the reason for ilezhankin's original TODO). NOT stays in +// `columnExprValue` (looser than every value operator, tighter than AND/OR) so it keeps sitting +// *after* `ColumnExprFunction` in the same rule — ANTLR then still prefers the function form for +// `not(args)` (a `not` function call) over the `NOT (args)` operator, matching the old grammar. +columnExpr + : columnExpr AND columnExpr # ColumnExprAnd + | columnExpr OR columnExpr # ColumnExprOr + | columnExpr QUERY columnExpr COLON columnExpr # ColumnExprTernaryOp + | columnExpr AS (identifier | STRING_LITERAL) # ColumnExprAlias + | columnExprValue # ColumnExprValuePassthrough + ; + +columnExprValue + : CASE caseExpr=columnExpr? (WHEN whenExpr=columnExpr THEN thenExpr=columnExpr)+ (ELSE elseExpr=columnExpr)? END # ColumnExprCase + | CAST LPAREN columnExpr AS columnTypeExpr RPAREN # ColumnExprCast + | TRY_CAST LPAREN columnExpr AS columnTypeExpr RPAREN # ColumnExprTryCast + | DATE STRING_LITERAL # ColumnExprDate +// | EXTRACT LPAREN interval FROM columnExpr RPAREN # ColumnExprExtract // Interferes with a function call + | INTERVAL columnExpr interval # ColumnExprInterval + | INTERVAL STRING_LITERAL # ColumnExprIntervalString + | SUBSTRING LPAREN columnExpr FROM columnExpr (FOR columnExpr)? RPAREN # ColumnExprSubstring + | TIMESTAMP STRING_LITERAL # ColumnExprTimestamp + | TRIM LPAREN (BOTH | LEADING | TRAILING) string FROM columnExpr RPAREN # ColumnExprTrim + | COLUMNS LPAREN STRING_LITERAL RPAREN # ColumnExprColumnsRegex + | COLUMNS LPAREN columnExprList RPAREN # ColumnExprColumnsList + | (COLUMNS LPAREN ASTERISK EXCLUDE LPAREN identifierList RPAREN REPLACE LPAREN columnsReplaceList RPAREN RPAREN + | LPAREN ASTERISK EXCLUDE LPAREN identifierList RPAREN REPLACE LPAREN columnsReplaceList RPAREN RPAREN + ) # ColumnExprColumnsExcludeReplace + | COLUMNS LPAREN ASTERISK EXCLUDE LPAREN identifierList RPAREN RPAREN # ColumnExprColumnsExclude + | (COLUMNS LPAREN ASTERISK REPLACE LPAREN columnsReplaceList RPAREN RPAREN + | LPAREN ASTERISK REPLACE LPAREN columnsReplaceList RPAREN RPAREN + ) # ColumnExprColumnsReplace + | COLUMNS LPAREN ASTERISK RPAREN # ColumnExprColumnsAll + | COLUMNS LPAREN identifier DOT ASTERISK EXCLUDE LPAREN identifierList RPAREN REPLACE LPAREN columnsReplaceList RPAREN RPAREN # ColumnExprColumnsQualifiedExcludeReplace + | COLUMNS LPAREN identifier DOT ASTERISK EXCLUDE LPAREN identifierList RPAREN RPAREN # ColumnExprColumnsQualifiedExclude + | COLUMNS LPAREN identifier DOT ASTERISK REPLACE LPAREN columnsReplaceList RPAREN RPAREN # ColumnExprColumnsQualifiedReplace + | COLUMNS LPAREN identifier DOT ASTERISK RPAREN # ColumnExprColumnsQualifiedAll + | ASTERISK COLUMNS LPAREN STRING_LITERAL RPAREN # ColumnExprSpreadColumnsRegex + | ASTERISK COLUMNS LPAREN columnExprList RPAREN # ColumnExprSpreadColumnsList + | identifier LPAREN columnExprs=columnExprList? RPAREN withinGroupClause # ColumnExprFunctionWithinGroup + | identifier (LPAREN columnExprs=columnExprList? RPAREN) (LPAREN DISTINCT? columnArgList=columnExprList? RPAREN)? (FILTER LPAREN WHERE filterExpr=columnExpr RPAREN)? OVER LPAREN windowExpr RPAREN # ColumnExprWinFunction + | identifier (LPAREN columnExprs=columnExprList? RPAREN) (LPAREN DISTINCT? columnArgList=columnExprList? RPAREN)? (FILTER LPAREN WHERE filterExpr=columnExpr RPAREN)? OVER identifier # ColumnExprWinFunctionTarget + | identifier (LPAREN columnExprs=columnExprList? RPAREN)? LPAREN DISTINCT? columnArgList=columnExprList? (ORDER BY orderExprList)? RPAREN (FILTER LPAREN WHERE filterExpr=columnExpr RPAREN)? # ColumnExprFunction + | columnExprValue LPAREN selectSetStmt RPAREN # ColumnExprCallSelect + | columnExprValue LPAREN columnExprList? RPAREN # ColumnExprCall + | hogqlxTagElement # ColumnExprTagElement + | templateString # ColumnExprTemplateString + | literal # ColumnExprLiteral + + // FIXME(ilezhankin): this part looks very ugly, maybe there is another way to express it + | columnExprValue LBRACKET columnExpr RBRACKET # ColumnExprArrayAccess + | columnExprValue LBRACKET columnExpr? COLON columnExpr? RBRACKET # ColumnExprArraySlice + | columnExprValue DOT DECIMAL_LITERAL # ColumnExprTupleAccess + | columnExprValue DOT identifier # ColumnExprPropertyAccess + | columnExprValue NULL_PROPERTY LBRACKET columnExpr RBRACKET # ColumnExprNullArrayAccess + | columnExprValue NULL_PROPERTY DECIMAL_LITERAL # ColumnExprNullTupleAccess + | columnExprValue NULL_PROPERTY identifier # ColumnExprNullPropertyAccess + | columnExprValue DOUBLECOLON columnTypeCastExpr # ColumnExprTypeCast + | DASH columnExprValue # ColumnExprNegate + | left=columnExprValue ( operator=ASTERISK // * + | operator=SLASH // / + | operator=PERCENT // % + ) right=columnExprValue # ColumnExprPrecedence1 + | left=columnExprValue ( operator=PLUS // + + | operator=DASH // - + | operator=CONCAT // || + ) right=columnExprValue # ColumnExprPrecedence2 + | left=columnExprValue ( operator=EQ_DOUBLE // = + | operator=EQ_SINGLE // == + | operator=NOT_EQ // != + | operator=LT_EQ // <= + | operator=LT // < + | operator=GT_EQ // >= + | operator=GT // > + | operator=NOT? IN COHORT? // in, not in; in cohort; not in cohort + | operator=NOT? (LIKE | ILIKE) // like, not like, ilike, not ilike + | operator=REGEX_SINGLE // ~ + | operator=REGEX_DOUBLE // =~ + | operator=NOT_REGEX // !~ + | operator=IREGEX_SINGLE // ~* + | operator=IREGEX_DOUBLE // =~* + | operator=NOT_IREGEX // !~* + ) right=columnExprValue # ColumnExprPrecedence3 + | columnExprValue IGNORE NULLS # ColumnExprIgnoreNulls + | columnExprValue IS NOT? NULL_SQL # ColumnExprIsNull + | columnExprValue IS NOT? DISTINCT FROM columnExprValue # ColumnExprIsDistinctFrom + | columnExprValue NULL_SAFE_EQ columnExprValue # ColumnExprNullSafeEq // MySQL `a <=> b` ≡ `a IS NOT DISTINCT FROM b` + | columnExprValue NULLISH columnExprValue # ColumnExprNullish + | columnExprValue NOT? BETWEEN columnExprValue AND columnExprValue # ColumnExprBetween + | NOT columnExprValue # ColumnExprNot + | (tableIdentifier DOT)? ASTERISK (EXCLUDE LPAREN identifierList RPAREN)? # ColumnExprAsterisk // single-column only + | LAMBDA identifier (COMMA identifier)* COMMA? COLON columnExpr # ColumnExprColonLambda + | LPAREN selectSetStmt RPAREN # ColumnExprSubquery // single-column only + | LPAREN columnExpr RPAREN # ColumnExprParens // single-column only + | LPAREN columnExprList RPAREN # ColumnExprTuple + | ARRAY? LBRACKET columnExprList? RBRACKET # ColumnExprArray + | LBRACE (kvPairList)? RBRACE # ColumnExprDict + | columnLambdaExpr # ColumnExprLambda + | identifier COLONEQUALS columnExpr # ColumnExprNamedArg + | HASH DECIMAL_LITERAL # ColumnExprPositional + | columnIdentifier # ColumnExprIdentifier + ; + +columnLambdaExpr: + ( LPAREN identifier (COMMA identifier)* COMMA? RPAREN + | identifier (COMMA identifier)* COMMA? + | LPAREN RPAREN + ) + ARROW (columnExpr | block) # ArrowLambda + | LAMBDA identifier (COMMA identifier)* COMMA? COLON columnExpr # ColonLambda + ; + +columnsReplaceList: columnsReplaceItem (COMMA columnsReplaceItem)*; +columnsReplaceItem: columnExpr AS identifier; + +hogqlxChildElement + : hogqlxTagElement + | hogqlxText + | LBRACE columnExpr RBRACE; + +hogqlxText : HOGQLX_TEXT_TEXT ; + +hogqlxTagElement + : LT identifier hogqlxTagAttribute* SLASH_GT # HogqlxTagElementClosed + | LT identifier hogqlxTagAttribute* GT hogqlxChildElement* LT_SLASH identifier GT # HogqlxTagElementNested + ; +hogqlxTagAttribute + : identifier EQ_SINGLE string + | identifier EQ_SINGLE LBRACE columnExpr RBRACE + | identifier + ; + +withExprList: withExpr (COMMA withExpr)* COMMA?; +withExpr + : identifier withExprColumnNameList? (USING KEY withExprColumnNameList)? AS (NOT? MATERIALIZED)? LPAREN selectSetStmt RPAREN # WithExprSubquery + // NOTE: asterisk and subquery goes before |columnExpr| so that we can mark them as multi-column expressions. + | columnExpr AS identifier # WithExprColumn + ; + +withExprColumnNameList: LPAREN identifier (COMMA identifier)* RPAREN; + + +// This is slightly different in HogQL compared to ClickHouse SQL +// HogQL allows unlimited ("*") nestedIdentifier-s "properties.b.a.a.w.a.s". +// We parse and convert "databaseIdentifier.tableIdentifier.columnIdentifier.nestedIdentifier.*" +// to just one ast.Field(chain=['a','b','columnIdentifier','on','and','on']). +columnIdentifier: placeholder | ((tableIdentifier DOT)? nestedIdentifier); +nestedIdentifier: identifier (DOT identifier)*; +tableExpr + : tableIdentifier # TableExprIdentifier + | tableFunctionExpr # TableExprFunction + | LPAREN selectSetStmt RPAREN # TableExprSubquery + | LPAREN valuesClause RPAREN # TableExprValues + | tableExpr PIVOT LPAREN columnExprList pivotColumnList (GROUP BY columnExprList)? RPAREN # TableExprPivot + | tableExpr UNPIVOT (INCLUDE NULLS)? LPAREN unpivotColumnList RPAREN # TableExprUnpivot + | tableExpr (alias | AS identifier) columnAliases? # TableExprAlias + | hogqlxTagElement # TableExprTag + | placeholder # TableExprPlaceholder + ; + +pivotColumnList: FOR pivotColumn+; +pivotColumn: columnExprTupleOrSingle IN LPAREN columnExprList RPAREN; +unpivotColumnList: unpivotColumn (COMMA unpivotColumn)* COMMA?; +unpivotColumn: columnExprTupleOrSingle FOR columnExprTupleOrSingle IN LPAREN columnExprList RPAREN (columnExprTupleOrSingle IN LPAREN columnExprList RPAREN)*; +columnExprTupleOrSingle: LPAREN columnExprList RPAREN | columnExpr; +columnAliases: LPAREN identifier (COMMA identifier)* RPAREN; +tableFunctionExpr: identifier LPAREN tableArgList? RPAREN; +tableIdentifier: (databaseIdentifier DOT)? nestedIdentifier; +tableArgList: columnExpr (COMMA columnExpr)* COMMA?; + +// Databases + +databaseIdentifier: identifier; + +// Basics + +floatingLiteral + : FLOATING_LITERAL + | DOT (DECIMAL_LITERAL | OCTAL_LITERAL) + | DECIMAL_LITERAL DOT (DECIMAL_LITERAL | OCTAL_LITERAL)? // can't move this to the lexer or it will break nested tuple access: t.1.2 + ; +numberLiteral: (PLUS | DASH)? (floatingLiteral | BINARY_LITERAL | OCTAL_LITERAL | OCTAL_PREFIX_LITERAL | DECIMAL_LITERAL | HEXADECIMAL_LITERAL | INF | NAN_SQL); +literal + : numberLiteral + | STRING_LITERAL + | NULL_SQL + ; +interval: SECOND | MINUTE | HOUR | DAY | WEEK | MONTH | QUARTER | YEAR; +keyword + // except NULL_SQL, INF, NAN_SQL + : ALL | AND | ANTI | ANY | ARRAY | AS | ASCENDING | ASOF | BETWEEN | BOTH | BY | CASE + | CAST | COHORT | COLLATE | COLUMNS | CROSS | CUBE | CURRENT | DATE | DESC | DESCENDING + | DISTINCT | ELSE | END | EXCLUDE | EXTRACT | FILL | FILTER | FINAL | FIRST + | FOR | FOLLOWING | FROM | FULL | GROUP | HAVING | ID | INTERPOLATE | IS + | GROUPING | IF | IGNORE | ILIKE | INCLUDE | IN | INNER | INTERVAL | JOIN | KEY + | LAMBDA | LAST | LEADING | LEFT | LIKE | LIMIT + | LOCAL | NAME | NATURAL | NOT | NULLS | OFFSET | ON | OR | ORDER | OUTER | OVER | PARTITION + | PIVOT | POSITIONAL | PRECEDING | PREWHERE | QUALIFY | RANGE | RECURSIVE | REPLACE | RETURN | RIGHT | ROLLUP | ROW + | ROWS | SAMPLE | SELECT | SEMI | SETS | SETTINGS | STEP | SUBSTRING + | THEN | TIES | TIME | TIMESTAMP | TOTALS | TRAILING | TRIM | TRUNCATE | TRY_CAST | TO | TOP + | UNBOUNDED | UNION | UNPIVOT | USING | VALUES | WHEN | WHERE | WINDOW | WITH + | ZONE + ; +keywordForAlias + : DATE | FIRST | ID | KEY + ; +keywordForImplicitAlias + : ASCENDING + | COHORT + | DATE + | DESCENDING + | FINAL + | ID + | RETURN + | TOP + | TOTALS + ; +alias: IDENTIFIER | QUOTED_IDENTIFIER | keywordForAlias; // |interval| can't be an alias, otherwise 'INTERVAL 1 SOMETHING' becomes ambiguous. +implicitAlias: IDENTIFIER | QUOTED_IDENTIFIER | keywordForImplicitAlias; +identifier: IDENTIFIER | QUOTED_IDENTIFIER | interval | keyword; +enumValue: string EQ_SINGLE numberLiteral; +placeholder: LBRACE columnExpr RBRACE; + +string: STRING_LITERAL | templateString; +templateString : QUOTE_SINGLE_TEMPLATE stringContents* QUOTE_SINGLE ; +stringContents : STRING_ESCAPE_TRIGGER columnExpr RBRACE | STRING_TEXT; + +// These are magic "full template strings", which are used to parse "full text field" templates without the surrounding SQL. +// We will need to add F' to the start of the string to change the lexer's mode. +fullTemplateString: QUOTE_SINGLE_TEMPLATE_FULL stringContentsFull* EOF ; +stringContentsFull : FULL_STRING_ESCAPE_TRIGGER columnExpr RBRACE | FULL_STRING_TEXT; diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageContract.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageContract.java new file mode 100644 index 000000000000..d560f9aaf6f2 --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageContract.java @@ -0,0 +1,94 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.airlift.json.JsonCodec; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; + +public record HogQlLanguageContract( + int schemaVersion, + HogQlLanguageVersion languageVersion, + String antlrVersion, + String canonicalParser, + Map entryPoints, + List files, + String grammarSha256) +{ + private static final String CURRENT_CONTRACT_RESOURCE = "/io/trino/hogql/parser/language/1.0.0/language.json"; + private static final JsonCodec CONTRACT_CODEC = jsonCodec(HogQlLanguageContract.class); + private static final HogQlLanguageContract CURRENT = loadCurrent(); + + @JsonCreator + public HogQlLanguageContract( + @JsonProperty("schemaVersion") int schemaVersion, + @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, + @JsonProperty("antlrVersion") String antlrVersion, + @JsonProperty("canonicalParser") String canonicalParser, + @JsonProperty("entryPoints") Map entryPoints, + @JsonProperty("files") List files, + @JsonProperty("grammarSha256") String grammarSha256) + { + if (schemaVersion != 1) { + throw new IllegalArgumentException("unsupported HogQL language contract schema: " + schemaVersion); + } + this.schemaVersion = schemaVersion; + this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + this.antlrVersion = requireNonNull(antlrVersion, "antlrVersion is null"); + this.canonicalParser = requireNonNull(canonicalParser, "canonicalParser is null"); + this.entryPoints = Map.copyOf(requireNonNull(entryPoints, "entryPoints is null")); + this.files = List.copyOf(requireNonNull(files, "files is null")); + this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); + } + + public static HogQlLanguageContract current() + { + return CURRENT; + } + + private static HogQlLanguageContract loadCurrent() + { + try (InputStream input = HogQlLanguageContract.class.getResourceAsStream(CURRENT_CONTRACT_RESOURCE)) { + if (input == null) { + throw new IllegalStateException("HogQL language contract resource is missing: " + CURRENT_CONTRACT_RESOURCE); + } + return CONTRACT_CODEC.fromJson(new String(input.readAllBytes(), StandardCharsets.UTF_8)); + } + catch (IOException e) { + throw new UncheckedIOException("failed to read HogQL language contract", e); + } + } + + public record GrammarFile(String path, String sha256) + { + @JsonCreator + public GrammarFile( + @JsonProperty("path") String path, + @JsonProperty("sha256") String sha256) + { + this.path = requireNonNull(path, "path is null"); + this.sha256 = requireNonNull(sha256, "sha256 is null"); + } + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageVersion.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageVersion.java new file mode 100644 index 000000000000..a6262fffe821 --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageVersion.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public record HogQlLanguageVersion(int major, int minor, int patch) +{ + private static final Pattern VERSION_PATTERN = Pattern.compile("(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)"); + + public HogQlLanguageVersion + { + if (major < 0 || minor < 0 || patch < 0) { + throw new IllegalArgumentException("language version components must be non-negative"); + } + } + + @JsonCreator + public static HogQlLanguageVersion valueOf(String value) + { + Matcher matcher = VERSION_PATTERN.matcher(value); + if (!matcher.matches()) { + throw new IllegalArgumentException("invalid HogQL language version: " + value); + } + return new HogQlLanguageVersion( + Integer.parseInt(matcher.group(1)), + Integer.parseInt(matcher.group(2)), + Integer.parseInt(matcher.group(3))); + } + + @Override + @JsonValue + public String toString() + { + return "%s.%s.%s".formatted(major, minor, patch); + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 6c9375871425..47b8f5d2e0bd 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -51,6 +51,8 @@ public final class HogQlParser { + private static final HogQlLanguageVersion CURRENT_LANGUAGE_VERSION = HogQlLanguageContract.current().languageVersion(); + private static final ANTLRErrorListener ERROR_LISTENER = new BaseErrorListener() { @Override @@ -61,8 +63,17 @@ public void syntaxError(Recognizer recognizer, Object offendingSymbol, int }; public HogQlQuery parseStatement(String hogql) + { + return parseStatement(hogql, CURRENT_LANGUAGE_VERSION); + } + + public HogQlQuery parseStatement(String hogql, HogQlLanguageVersion languageVersion) { requireNonNull(hogql, "hogql is null"); + requireNonNull(languageVersion, "languageVersion is null"); + if (!languageVersion.equals(CURRENT_LANGUAGE_VERSION)) { + throw new IllegalArgumentException("unsupported HogQL language version: " + languageVersion); + } try { HogQlBaseLexer lexer = new HogQlBaseLexer(CharStreams.fromString(hogql)); diff --git a/core/trino-hogql-parser/src/main/resources/META-INF/LICENSE-posthog-hogql.txt b/core/trino-hogql-parser/src/main/resources/META-INF/LICENSE-posthog-hogql.txt new file mode 100644 index 000000000000..46ad1d9ce1ee --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/META-INF/LICENSE-posthog-hogql.txt @@ -0,0 +1,19 @@ +Copyright (c) 2020-2026 PostHog Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLLexer.common.g4 b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLLexer.common.g4 new file mode 100644 index 000000000000..6525c654f2a2 --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLLexer.common.g4 @@ -0,0 +1,357 @@ +lexer grammar HogQLLexer; + +// NB! We cat either HogQLLexter.cpp.g4 or HogQLLexter.python.g4 when generating the grammar. + +// NOTE: don't forget to add new keywords to the parser rule "keyword"! + +// Keywords + +ALL: A L L; +AND: A N D; +ANTI: A N T I; +ANY: A N Y; +ARRAY: A R R A Y; +AS: A S; +ASCENDING: A S C | A S C E N D I N G; +ASOF: A S O F; +BETWEEN: B E T W E E N; +BOTH: B O T H; +BY: B Y; +CASE: C A S E; +CAST: C A S T; +CATCH: C A T C H; +COHORT: C O H O R T; +COLLATE: C O L L A T E; +COLUMNS: C O L U M N S; +CROSS: C R O S S; +CUBE: C U B E; +CURRENT: C U R R E N T; +DATE: D A T E; +DAY: D A Y; +DESC: D E S C; +DESCENDING: D E S C E N D I N G; +DISTINCT: D I S T I N C T; +ELSE: E L S E; +END: E N D; +EXCEPT: E X C E P T; +EXCLUDE: E X C L U D E; +EXTRACT: E X T R A C T; +FINAL: F I N A L; +FILL: F I L L; +FILTER: F I L T E R; +FINALLY: F I N A L L Y; +FIRST: F I R S T; +FN: F N; +FOLLOWING: F O L L O W I N G; +FOR: F O R; +FROM: F R O M; +FULL: F U L L; +FUN: F U N; +GROUP: G R O U P; +GROUPING: G R O U P I N G; +HAVING: H A V I N G; +HOUR: H O U R; +ID: I D; +IF: I F; +ILIKE: I L I K E; +IGNORE: I G N O R E; +INCLUDE: I N C L U D E; +IN: I N; +INF: I N F | I N F I N I T Y; +INNER: I N N E R; +INTERSECT: I N T E R S E C T; +INTERPOLATE: I N T E R P O L A T E; +INTERVAL: I N T E R V A L; +IS: I S; +JOIN: J O I N; +KEY: K E Y; +LAMBDA: L A M B D A; +LAST: L A S T; +LEADING: L E A D I N G; +LEFT: L E F T; +LET: L E T; +LIKE: L I K E; +LIMIT: L I M I T; +MATERIALIZED: M A T E R I A L I Z E D; +MINUTE: M I N U T E; +MONTH: M O N T H; +NAME: N A M E; +NATURAL: N A T U R A L; +NAN_SQL: N A N; // conflicts with macro NAN +NOT: N O T; +NULL_SQL: N U L L; // conflicts with macro NULL +NULLS: N U L L S; +OFFSET: O F F S E T; +ON: O N; +OR: O R; +ORDER: O R D E R; +OUTER: O U T E R; +OVER: O V E R; +PARTITION: P A R T I T I O N; +PIVOT: P I V O T; +POSITIONAL: P O S I T I O N A L; +PRECEDING: P R E C E D I N G; +PREWHERE: P R E W H E R E; +QUALIFY: Q U A L I F Y; +QUARTER: Q U A R T E R; +RANGE: R A N G E; +RECURSIVE: R E C U R S I V E; +REPLACE: R E P L A C E; +RETURN: R E T U R N; +RIGHT: R I G H T; +ROLLUP: R O L L U P; +ROW: R O W; +ROWS: R O W S; +SAMPLE: S A M P L E; +SECOND: S E C O N D; +SELECT: S E L E C T; +SEMI: S E M I; +SETS: S E T S; +SETTINGS: S E T T I N G S; +STEP: S T E P; +SUBSTRING: S U B S T R I N G; +THEN: T H E N; +THROW: T H R O W; +TIES: T I E S; +TIMESTAMP: T I M E S T A M P; +TIME: T I M E; +LOCAL: L O C A L; +ZONE: Z O N E; +TO: T O; +TOP: T O P; +TOTALS: T O T A L S; +TRAILING: T R A I L I N G; +TRIM: T R I M; +TRUNCATE: T R U N C A T E; +TRY: T R Y; +TRY_CAST: T R Y '_' C A S T; +UNBOUNDED: U N B O U N D E D; +UNION: U N I O N; +UNPIVOT: U N P I V O T; +USING: U S I N G; +VALUES: V A L U E S; +WEEK: W E E K; +WHEN: W H E N; +WHERE: W H E R E; +WHILE: W H I L E; +WINDOW: W I N D O W; +WITH: W I T H; +WITHIN: W I T H I N; +YEAR: Y E A R | Y Y Y Y; + +// Tokens + +// copied from clickhouse_driver/util/escape.py +ESCAPE_CHAR_COMMON + : BACKSLASH B + | BACKSLASH F + | BACKSLASH R + | BACKSLASH N + | BACKSLASH T + | BACKSLASH '0' + | BACKSLASH A + | BACKSLASH V + | BACKSLASH BACKSLASH + | BACKSLASH X HEX_DIGIT HEX_DIGIT; + +IDENTIFIER + : (LETTER | UNDERSCORE | DOLLAR) (LETTER | UNDERSCORE | DEC_DIGIT | DOLLAR)* + ; +QUOTED_IDENTIFIER + : BACKQUOTE ( ~([\\`]) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_SINGLE | (BACKQUOTE BACKQUOTE) )* BACKQUOTE + | QUOTE_DOUBLE ( ~([\\"]) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_DOUBLE | (QUOTE_DOUBLE QUOTE_DOUBLE) )* QUOTE_DOUBLE + ; +FLOATING_LITERAL + // Hex-float exponent: strict C99 `p`/`P` only — `e`/`E` stays a hex digit, so `0x1e5` is 485, not a float. + : HEXADECIMAL_LITERAL DOT HEX_DIGIT* P (PLUS | DASH)? DEC_DIGIT+ + | HEXADECIMAL_LITERAL P (PLUS | DASH)? DEC_DIGIT+ + | DECIMAL_LITERAL DOT DEC_DIGIT* E (PLUS | DASH)? DEC_DIGIT+ + | DOT DECIMAL_LITERAL E (PLUS | DASH)? DEC_DIGIT+ + | DECIMAL_LITERAL E (PLUS | DASH)? DEC_DIGIT+ + ; +// Binary literals (`0b1010`). Declared first so it wins the length-tie against MALFORMED_BINARY_LITERAL. +BINARY_LITERAL: '0' B BIN_DIGIT+; +OCTAL_LITERAL: '0' OCT_DIGIT+; +DECIMAL_LITERAL: DEC_DIGIT+; +HEXADECIMAL_LITERAL: '0' X HEX_DIGIT+; +// Postgres-16 `0o` octal — unsupported; lexed as a real token so the visitor can reject it clearly. +OCTAL_PREFIX_LITERAL: '0' [oO] DEC_DIGIT+; +// Malformed binary (`0b22`) BINARY_LITERAL didn't consume — caught so it can't re-tokenise as `0` + IDENTIFIER. +MALFORMED_BINARY_LITERAL: '0' [bB] DEC_DIGIT+; + +// It's important that quote-symbol is a single character. +STRING_LITERAL: QUOTE_SINGLE ( ~([\\']) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_SINGLE | (QUOTE_SINGLE QUOTE_SINGLE) )* QUOTE_SINGLE; + + +// Alphabet and allowed symbols + +fragment A: [aA]; +fragment B: [bB]; +fragment C: [cC]; +fragment D: [dD]; +fragment E: [eE]; +fragment F: [fF]; +fragment G: [gG]; +fragment H: [hH]; +fragment I: [iI]; +fragment J: [jJ]; +fragment K: [kK]; +fragment L: [lL]; +fragment M: [mM]; +fragment N: [nN]; +fragment O: [oO]; +fragment P: [pP]; +fragment Q: [qQ]; +fragment R: [rR]; +fragment S: [sS]; +fragment T: [tT]; +fragment U: [uU]; +fragment V: [vV]; +fragment W: [wW]; +fragment X: [xX]; +fragment Y: [yY]; +fragment Z: [zZ]; + +fragment LETTER: [a-zA-Z]; +fragment BIN_DIGIT: [01]; +fragment OCT_DIGIT: [0-7]; +fragment DEC_DIGIT: [0-9]; +fragment HEX_DIGIT: [0-9a-fA-F]; + +ARROW: '->'; +ASTERISK: '*'; +BACKQUOTE: '`'; +BACKSLASH: '\\'; +DOUBLECOLON: '::'; +COLONEQUALS: ':='; +COLON: ':'; +COMMA: ','; +CONCAT: '||'; +DASH: '-'; +DOLLAR: '$'; +DOT: '.'; +EQ_DOUBLE: '=='; +EQ_SINGLE: '='; +GT_EQ: '>='; +GT: '>'; +HASH: '#'; +IREGEX_SINGLE: '~*'; +IREGEX_DOUBLE: '=~*'; +LBRACE: '{' -> pushMode(DEFAULT_MODE); +LBRACKET: '['; +LPAREN: '('; +NULL_SAFE_EQ: '<=>'; +LT_EQ: '<='; +TAG_LT_SLASH: ' type(LT_SLASH), pushMode(HOGQLX_TAG_CLOSE); +TAG_LT_OPEN: '<' {isOpeningTag()}? -> type(LT), pushMode(HOGQLX_TAG_OPEN); +LT: '<'; +LT_SLASH: ''; +NOT_IREGEX: '!~*'; +NOT_REGEX: '!~'; +NULL_PROPERTY: '?.'; +NULLISH: '??'; +PERCENT: '%'; +PLUS: '+'; +QUERY: '?'; +QUOTE_DOUBLE: '"'; +QUOTE_SINGLE_TEMPLATE: 'f\'' -> pushMode(IN_TEMPLATE_STRING); // start of regular f'' template strings +QUOTE_SINGLE_TEMPLATE_FULL: 'F\'' -> pushMode(IN_FULL_TEMPLATE_STRING); // magic F' symbol used to parse "full text" templates +QUOTE_SINGLE: '\''; +REGEX_SINGLE: '~'; +REGEX_DOUBLE: '=~'; +RBRACE: '}' -> popMode; +RBRACKET: ']'; +RPAREN: ')'; +SEMICOLON: ';'; +SLASH: '/'; +SLASH_GT: '/>'; +UNDERSCORE: '_'; + +// Comments and whitespace +MULTI_LINE_COMMENT: '/*' .*? '*/' -> skip; +SINGLE_LINE_COMMENT: ('--' | '//') ~('\n'|'\r')* ('\n' | '\r' | EOF) -> skip; +// MySQL-style `#` comments. `#` is excluded so positional references (`#1`) keep +// working — a `#` comment whose text starts with a digit is the one MySQL-ism this rejects. +HASH_COMMENT: '#' (~[0-9\n\r] ~[\n\r]*)? ('\n' | '\r' | EOF) -> skip; +// whitespace is hidden and not skipped so that it's preserved in ANTLR errors like "no viable alternative" +// The class is the full Unicode `White_Space` set, not just ASCII: a +// NO-BREAK SPACE or other Unicode space (often pasted in from rich +// editors or docs) is genuine whitespace and must keep separating +// tokens. Recognising it here keeps such programs valid — otherwise it +// would fall through to UNEXPECTED_CHARACTER below and fail the whole +// parse. U+FEFF (BOM) is included too, so a file saved with a +// byte-order mark still parses. +WHITESPACE: [ \t\r\n\u000B\u000C\u0085\u00A0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF] -> channel(HIDDEN); + +// Catch-all for any character no rule above matched. Without this the +// lexer raises a recoverable token-recognition error and DROPS the +// character — so stray input (a JavaScript `!`, `&&`, …) silently +// vanishes and the surrounding text parses as a different, valid-looking +// program. Emitting an explicit token instead means the parser has no +// rule for it and fails loudly with a SyntaxError. Listed last so it +// only ever fires as a true fallback (maximal munch keeps `!=`, `!~`, +// multi-character operators, comments, etc. intact). +UNEXPECTED_CHARACTER: . ; + +// ───────── f' TEMPLATE STRING MODE ───────── +mode IN_TEMPLATE_STRING; +STRING_TEXT: ((~([\\'{])) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_SINGLE | (BACKSLASH LBRACE) | (QUOTE_SINGLE QUOTE_SINGLE))+; +STRING_ESCAPE_TRIGGER: LBRACE -> pushMode(DEFAULT_MODE); +STRING_QUOTE_SINGLE: QUOTE_SINGLE -> type(QUOTE_SINGLE), popMode; + +// ───────── F' FULL TEMPLATE STRING MODE ───────── +// a magic F' takes us to "full template strings" mode, where we don't need to escape single quotes and parse until EOF +// this can't be used within a normal columnExpr, but has to be parsed for separately +mode IN_FULL_TEMPLATE_STRING; +FULL_STRING_TEXT: ((~([{])) | ESCAPE_CHAR_COMMON | (BACKSLASH LBRACE))+; +FULL_STRING_ESCAPE_TRIGGER: LBRACE -> pushMode(DEFAULT_MODE); + +// ───────── HOGQLX TAG MODE for opening/self-closing tags ───────── +mode HOGQLX_TAG_OPEN; + +TAG_SELF_CLOSE_GT : '/>' -> type(SLASH_GT), popMode; // +TAG_OPEN_GT : '>' -> type(GT), popMode, pushMode(HOGQLX_TEXT); // + +// Skip comments between attributes — without these, the recoverable lexer error drops the delimiters and re-tokenises the body as phantom attributes. +TAG_MULTI_LINE_COMMENT : '/*' .*? '*/' -> skip; +TAG_SINGLE_LINE_COMMENT : ('--' | '//') ~('\n'|'\r')* ('\n' | '\r' | EOF) -> skip; + +// minimal token set; map everything back to the default token types +TAG_IDENT : [a-zA-Z_][a-zA-Z0-9_-]* -> type(IDENTIFIER); +TAG_EQ : '=' -> type(EQ_SINGLE); +TAG_STRING : STRING_LITERAL -> type(STRING_LITERAL); +TAG_WS : [ \t\r\n]+ -> channel(HIDDEN); +TAG_LBRACE : '{' -> type(LBRACE), pushMode(DEFAULT_MODE); +// Catch-all for unmatched bytes (e.g. `#`, `&`, `@`) so the parser fails loudly instead of silently re-tokenising the surrounding text. +TAG_UNEXPECTED : . -> type(UNEXPECTED_CHARACTER); + + +// ───────── HOGQLX TAG MODE for closing tags ───────── +mode HOGQLX_TAG_CLOSE; + +TAGC_GT : '>' -> type(GT), popMode; // *** no TEXT push *** +TAGC_MULTI_LINE_COMMENT : '/*' .*? '*/' -> skip; +TAGC_SINGLE_LINE_COMMENT : ('--' | '//') ~('\n'|'\r')* ('\n' | '\r' | EOF) -> skip; +TAGC_IDENT : [a-zA-Z_][a-zA-Z0-9_-]* -> type(IDENTIFIER); +TAGC_WS : [ \t\r\n]+ -> channel(HIDDEN); +TAGC_UNEXPECTED : . -> type(UNEXPECTED_CHARACTER); + + +// ───────── HOGQLX TEXT MODE ───────── +mode HOGQLX_TEXT; + +HOGQLX_TEXT_TEXT + : ~[<{]+ ; // everything except “{” or “<” + +HOGQLX_TEXT_LBRACE + : '{' -> type(LBRACE), pushMode(DEFAULT_MODE); + +HOGQLX_TEXT_LT_SLASH + : ' type(LT_SLASH), popMode, pushMode(HOGQLX_TAG_CLOSE); + +HOGQLX_TEXT_LT + : '<' -> type(LT), pushMode(HOGQLX_TAG_OPEN); + +HOGQLX_TEXT_WS + : [ \t\r\n]+ -> channel(HIDDEN); diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLLexer.java.g4 b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLLexer.java.g4 new file mode 100644 index 000000000000..7afce8bfce2a --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLLexer.java.g4 @@ -0,0 +1,81 @@ +lexer grammar HogQLLexer; + +@members { + +private static boolean isAsciiAlpha(int character) { + return character >= 0 && character < 128 && + ((character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z')); +} + +private static boolean isAsciiAlphanumeric(int character) { + return isAsciiAlpha(character) || (character >= '0' && character <= '9'); +} + +private static boolean isAsciiWhitespace(int character) { + return character == ' ' || character == '\t' || character == '\n' || + character == '\u000B' || character == '\f' || character == '\r'; +} + +private int skipWhitespaceAndComments(int index) { + while (true) { + int character = _input.LA(index); + if (isAsciiWhitespace(character)) { + index++; + continue; + } + + if (character == '/' && _input.LA(index + 1) == '/') { + index += 2; + } + else if (character == '-' && _input.LA(index + 1) == '-') { + index += 2; + } + else if (character == '#') { + index++; + } + else { + return index; + } + + while (true) { + character = _input.LA(index); + if (character <= 0 || character == '\n' || character == '\r') { + break; + } + index++; + } + } +} + +private boolean isOpeningTag() { + int firstCharacter = _input.LA(1); + if (!isAsciiAlpha(firstCharacter) && firstCharacter != '_') { + return false; + } + + int index = 2; + while (true) { + int character = _input.LA(index); + if (isAsciiAlphanumeric(character) || character == '_' || character == '-') { + index++; + } + else { + break; + } + } + + int character = _input.LA(index); + if (character == '>' || character == '/') { + return true; + } + + if (isAsciiWhitespace(character)) { + index = skipWhitespaceAndComments(index + 1); + character = _input.LA(index); + return isAsciiAlphanumeric(character) || character == '_' || character == '>' || character == '/'; + } + + return false; +} + +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLParser.g4 b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLParser.g4 new file mode 100644 index 000000000000..6fe244c0a1f3 --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLParser.g4 @@ -0,0 +1,445 @@ +parser grammar HogQLParser; + +options { + tokenVocab = HogQLLexer; +} + + +program: declaration* EOF; + +declaration: varDecl | statement ; + +expression: columnExpr; + +varDecl: LET identifier ( COLONEQUALS expression )? ; +identifierList: nestedIdentifier (COMMA nestedIdentifier)* COMMA?; + +statement : returnStmt + | throwStmt + | tryCatchStmt + | ifStmt + | whileStmt + | forInStmt + | forStmt + | funcStmt + | block + | exprStmt + | emptyStmt + ; + +returnStmt : RETURN expression? SEMICOLON?; +throwStmt : THROW expression SEMICOLON?; +catchBlock : CATCH (LPAREN catchVar=identifier (COLON catchType=identifier)? RPAREN)? catchStmt=block; +tryCatchStmt : TRY tryStmt=block catchBlock* (FINALLY finallyStmt=block)?; +ifStmt : IF LPAREN expression RPAREN statement ( ELSE statement )? ; +whileStmt : WHILE LPAREN expression RPAREN statement SEMICOLON?; +forStmt : FOR LPAREN + (initializerVarDeclr=varDecl | initializerVarAssignment=varAssignment | initializerExpression=expression)? SEMICOLON + condition=expression? SEMICOLON + (incrementVarDeclr=varDecl | incrementVarAssignment=varAssignment | incrementExpression=expression)? + RPAREN statement SEMICOLON?; +forInStmt : FOR LPAREN LET identifier (COMMA identifier)? IN expression RPAREN statement SEMICOLON?; +funcStmt : (FN | FUN) identifier LPAREN identifierList? RPAREN block; +varAssignment : expression COLONEQUALS expression ; +// Assignment folded in as an optional suffix: one expression-leading alternative in +// `statement` means `declaration*` parses without unbounded lookahead. `varAssignment` is forStmt-only. +exprStmt : expression (COLONEQUALS expression)? SEMICOLON?; +emptyStmt : SEMICOLON ; +block : LBRACE declaration* RBRACE ; + +kvPair: expression ':' expression ; +kvPairList: kvPair (COMMA kvPair)* COMMA?; + + +// SELECT statement +select: (selectSetStmt | selectStmt | hogqlxTagElement) SEMICOLON? EOF; + +selectStmtWithParens: selectStmt | withClause LPAREN selectSetStmt RPAREN | LPAREN selectSetStmt RPAREN | placeholder; + +subsequentSelectSetClause: (EXCEPT ALL (BY NAME)? | EXCEPT (BY NAME)? | UNION ALL (BY NAME)? | UNION DISTINCT (BY NAME)? | UNION (BY NAME)? | INTERSECT ALL (BY NAME)? | INTERSECT DISTINCT (BY NAME)? | INTERSECT (BY NAME)?) selectStmtWithParens; +selectSetStmt: selectStmtWithParens (subsequentSelectSetClause)* orderByClause? limitAndOffsetClauseOptional?; +limitAndOffsetClauseOptional + : LIMIT columnExpr PERCENT? (COMMA columnExpr)? (WITH TIES)? + | LIMIT columnExpr PERCENT? (WITH TIES)? OFFSET columnExpr + | OFFSET columnExpr + ; + +selectStmt: + with=withClause? + SELECT DISTINCT? topClause? + columns=selectColumnExprListBeforeFrom + from=fromClause? + arrayJoinClause? + prewhereClause? + where=whereClause? + (USING? sampleClause)? + groupByClause? (WITH (CUBE | ROLLUP))? (WITH TOTALS)? + havingClause? + qualifyClause? + (USING sampleClause)? + windowClause? + orderByClause? + limitByClause? + (limitAndOffsetClause | offsetOnlyClause)? + settingsClause? + ; + +withClause: WITH RECURSIVE? withExprList; +topClause: TOP DECIMAL_LITERAL (WITH TIES)?; +fromClause: FROM joinExpr; +arrayJoinClause: (LEFT | INNER)? ARRAY JOIN columnExprList; +windowClause: WINDOW identifier AS LPAREN windowExpr RPAREN (COMMA identifier AS LPAREN windowExpr RPAREN)*; +prewhereClause: PREWHERE columnExpr; +whereClause: WHERE columnExpr; +groupByClause: GROUP BY ( + ALL + | (CUBE | ROLLUP) LPAREN columnExprList RPAREN + | GROUPING SETS LPAREN groupingSetList RPAREN + | columnExprList + ); +groupingSetList: groupingSet (COMMA groupingSet)*; +groupingSet: LPAREN columnExprList? RPAREN; +havingClause: HAVING columnExpr; +qualifyClause: QUALIFY columnExpr; +orderByClause: ORDER BY orderExprList interpolateClause?; +interpolateClause: INTERPOLATE (LPAREN interpolateExpr (COMMA interpolateExpr)* RPAREN)?; +projectionOrderByClause: ORDER BY columnExprList; +limitByClause: LIMIT limitExpr BY columnExprList; +limitAndOffsetClause + : LIMIT columnExpr PERCENT? (COMMA columnExpr)? (WITH TIES)? // compact OFFSET-optional form + | LIMIT columnExpr PERCENT? (WITH TIES)? OFFSET columnExpr // verbose OFFSET-included form with WITH TIES + ; +offsetOnlyClause: OFFSET columnExpr; +settingsClause: SETTINGS settingExprList; + +valuesClause: VALUES valuesRow (COMMA valuesRow)*; +valuesRow: LPAREN columnExpr (COMMA columnExpr)* RPAREN; + +joinExpr + : joinExpr NATURAL? joinOp? JOIN joinExpr joinConstraintClause? # JoinExprOp + | joinExpr POSITIONAL JOIN joinExpr joinConstraintClause? # JoinExprPositional + | joinExpr joinOpCross joinExpr # JoinExprCrossOp + | joinExpr PIVOT LPAREN columnExprList pivotColumnList (GROUP BY columnExprList)? RPAREN # JoinExprPivot + | joinExpr UNPIVOT (INCLUDE NULLS)? LPAREN unpivotColumnList RPAREN # JoinExprUnpivot + | tableExpr FINAL? sampleClause? # JoinExprTable + | LPAREN joinExpr RPAREN # JoinExprParens + ; +joinOp + : ((ALL | ANY | ASOF)? INNER | INNER (ALL | ANY | ASOF)? | (ALL | ANY | ASOF) | ANTI | SEMI | ASOF (ANTI | SEMI)) # JoinOpInner + | ( (SEMI | ALL | ANTI | ANY | ASOF)? (LEFT | RIGHT) OUTER? + | (LEFT | RIGHT) OUTER? (SEMI | ALL | ANTI | ANY | ASOF)? + | ASOF (ANTI | SEMI) (LEFT | RIGHT) OUTER? + ) # JoinOpLeftRight + | ((ALL | ANY | ASOF)? FULL OUTER? | FULL OUTER? (ALL | ANY | ASOF)?) # JoinOpFull + ; +joinOpCross + : CROSS JOIN + | COMMA + ; +joinConstraintClause + : ON columnExprList + | USING LPAREN columnExprList RPAREN + | USING columnExprList + ; + +sampleClause: SAMPLE ratioExpr PERCENT? (OFFSET ratioExpr)? (LPAREN identifier RPAREN)?; +limitExpr: columnExpr ((COMMA | OFFSET) columnExpr)?; +orderExprList: orderExpr (COMMA orderExpr)*; +orderExpr: columnExpr (ASCENDING | DESCENDING | DESC)? (NULLS (FIRST | LAST))? (COLLATE STRING_LITERAL)? withFillClause?; +withFillClause: WITH FILL (FROM columnExpr)? (TO columnExpr)? (STEP columnExpr)?; +interpolateExpr: columnExpr (AS columnExpr)?; +ratioExpr: placeholder | numberLiteral (SLASH numberLiteral)?; +settingExprList: settingExpr (COMMA settingExpr)*; +settingExpr: identifier EQ_SINGLE literal; + +windowExpr: winPartitionByClause? winOrderByClause? winFrameClause?; +winPartitionByClause: PARTITION BY columnExprList; +winOrderByClause: ORDER BY orderExprList; +withinGroupClause: WITHIN GROUP LPAREN orderByClause RPAREN; +winFrameClause: (ROWS | RANGE) winFrameExtend; +winFrameExtend + : winFrameBound # frameStart + | BETWEEN winFrameBound AND winFrameBound # frameBetween + ; +winFrameBound: (CURRENT ROW | UNBOUNDED PRECEDING | UNBOUNDED FOLLOWING | columnExpr PRECEDING | columnExpr FOLLOWING); +//rangeClause: RANGE LPAREN (MIN identifier MAX identifier | MAX identifier MIN identifier) RPAREN; + +// Columns +expr: columnExpr EOF; +columnTypeExpr + : columnTypeExpr LBRACKET DECIMAL_LITERAL? RBRACKET # ColumnTypeExprArray // INTEGER[], VARCHAR[3] + | identifier LPAREN identifier columnTypeExpr (COMMA identifier columnTypeExpr)* COMMA? RPAREN # ColumnTypeExprNested // Nested + | identifier LPAREN enumValue (COMMA enumValue)* COMMA? RPAREN # ColumnTypeExprEnum // Enum + | identifier LPAREN columnTypeExpr (COMMA columnTypeExpr)* COMMA? RPAREN # ColumnTypeExprComplex // Array, Tuple + | identifier LPAREN columnExprList? RPAREN # ColumnTypeExprParam // FixedString(N) + | identifier identifier+ # ColumnTypeExprCompound // TIME WITH TIME ZONE + | identifier # ColumnTypeExprSimple // UInt64 + ; +// Restricted type expr for :: casts — no parenthesized variants to avoid ambiguity with function calls +columnTypeCastExpr + : columnTypeCastIdentifier WITH LOCAL? TIME ZONE # ColumnTypeCastExprWithTimeZone + | columnTypeCastIdentifier # ColumnTypeCastExprSimple + ; +columnTypeCastIdentifier + : IDENTIFIER + | QUOTED_IDENTIFIER + | interval + | keywordForTypeCast + ; +keywordForTypeCast + : DATE + | TIME + | TIMESTAMP + | INTERVAL + ; +columnExprList: columnExpr (COMMA columnExpr)* COMMA?; +selectColumnExprListBeforeFrom + : selectColumnExpr (COMMA selectColumnExpr)* COMMA # SelectColumnExprListBeforeFromTrailingComma + | selectColumnExprList # SelectColumnExprListBeforeFromPlain + ; +selectColumnExprList: selectColumnExpr (COMMA selectColumnExpr)* COMMA?; +selectColumnExpr + : identifier COLON columnExpr # ColumnExprAliasBefore + | FROM implicitAlias # ColumnExprInvalidFromImplicitAlias + | columnExpr # ColumnExprSelectValue + | columnExpr implicitAlias # ColumnExprAliasImplicit + ; +// Two precedence layers. `columnExpr` is the outer boolean/ternary/alias tier (loosest +// binding). `columnExprValue` holds everything tighter — arithmetic, comparisons, NOT, +// BETWEEN, and the primary/leaf productions. BETWEEN lives in the value tier at the comparison +// level, so its tested expression and both bounds bind tighter than AND/OR/NOT (matching +// ClickHouse and SQL); this is what fixes `a BETWEEN low AND high AND rest` grouping as +// `(a BETWEEN low AND high) AND rest` rather than letting the bounds swallow the AND chain. +// Splitting into two rules is the only way to express this in ANTLR4: the interior operand of +// a left-recursive alternative always parses at precedence 0, so a flat rule cannot stop the +// bounds from consuming AND (the reason for ilezhankin's original TODO). NOT stays in +// `columnExprValue` (looser than every value operator, tighter than AND/OR) so it keeps sitting +// *after* `ColumnExprFunction` in the same rule — ANTLR then still prefers the function form for +// `not(args)` (a `not` function call) over the `NOT (args)` operator, matching the old grammar. +columnExpr + : columnExpr AND columnExpr # ColumnExprAnd + | columnExpr OR columnExpr # ColumnExprOr + | columnExpr QUERY columnExpr COLON columnExpr # ColumnExprTernaryOp + | columnExpr AS (identifier | STRING_LITERAL) # ColumnExprAlias + | columnExprValue # ColumnExprValuePassthrough + ; + +columnExprValue + : CASE caseExpr=columnExpr? (WHEN whenExpr=columnExpr THEN thenExpr=columnExpr)+ (ELSE elseExpr=columnExpr)? END # ColumnExprCase + | CAST LPAREN columnExpr AS columnTypeExpr RPAREN # ColumnExprCast + | TRY_CAST LPAREN columnExpr AS columnTypeExpr RPAREN # ColumnExprTryCast + | DATE STRING_LITERAL # ColumnExprDate +// | EXTRACT LPAREN interval FROM columnExpr RPAREN # ColumnExprExtract // Interferes with a function call + | INTERVAL columnExpr interval # ColumnExprInterval + | INTERVAL STRING_LITERAL # ColumnExprIntervalString + | SUBSTRING LPAREN columnExpr FROM columnExpr (FOR columnExpr)? RPAREN # ColumnExprSubstring + | TIMESTAMP STRING_LITERAL # ColumnExprTimestamp + | TRIM LPAREN (BOTH | LEADING | TRAILING) string FROM columnExpr RPAREN # ColumnExprTrim + | COLUMNS LPAREN STRING_LITERAL RPAREN # ColumnExprColumnsRegex + | COLUMNS LPAREN columnExprList RPAREN # ColumnExprColumnsList + | (COLUMNS LPAREN ASTERISK EXCLUDE LPAREN identifierList RPAREN REPLACE LPAREN columnsReplaceList RPAREN RPAREN + | LPAREN ASTERISK EXCLUDE LPAREN identifierList RPAREN REPLACE LPAREN columnsReplaceList RPAREN RPAREN + ) # ColumnExprColumnsExcludeReplace + | COLUMNS LPAREN ASTERISK EXCLUDE LPAREN identifierList RPAREN RPAREN # ColumnExprColumnsExclude + | (COLUMNS LPAREN ASTERISK REPLACE LPAREN columnsReplaceList RPAREN RPAREN + | LPAREN ASTERISK REPLACE LPAREN columnsReplaceList RPAREN RPAREN + ) # ColumnExprColumnsReplace + | COLUMNS LPAREN ASTERISK RPAREN # ColumnExprColumnsAll + | COLUMNS LPAREN identifier DOT ASTERISK EXCLUDE LPAREN identifierList RPAREN REPLACE LPAREN columnsReplaceList RPAREN RPAREN # ColumnExprColumnsQualifiedExcludeReplace + | COLUMNS LPAREN identifier DOT ASTERISK EXCLUDE LPAREN identifierList RPAREN RPAREN # ColumnExprColumnsQualifiedExclude + | COLUMNS LPAREN identifier DOT ASTERISK REPLACE LPAREN columnsReplaceList RPAREN RPAREN # ColumnExprColumnsQualifiedReplace + | COLUMNS LPAREN identifier DOT ASTERISK RPAREN # ColumnExprColumnsQualifiedAll + | ASTERISK COLUMNS LPAREN STRING_LITERAL RPAREN # ColumnExprSpreadColumnsRegex + | ASTERISK COLUMNS LPAREN columnExprList RPAREN # ColumnExprSpreadColumnsList + | identifier LPAREN columnExprs=columnExprList? RPAREN withinGroupClause # ColumnExprFunctionWithinGroup + | identifier (LPAREN columnExprs=columnExprList? RPAREN) (LPAREN DISTINCT? columnArgList=columnExprList? RPAREN)? (FILTER LPAREN WHERE filterExpr=columnExpr RPAREN)? OVER LPAREN windowExpr RPAREN # ColumnExprWinFunction + | identifier (LPAREN columnExprs=columnExprList? RPAREN) (LPAREN DISTINCT? columnArgList=columnExprList? RPAREN)? (FILTER LPAREN WHERE filterExpr=columnExpr RPAREN)? OVER identifier # ColumnExprWinFunctionTarget + | identifier (LPAREN columnExprs=columnExprList? RPAREN)? LPAREN DISTINCT? columnArgList=columnExprList? (ORDER BY orderExprList)? RPAREN (FILTER LPAREN WHERE filterExpr=columnExpr RPAREN)? # ColumnExprFunction + | columnExprValue LPAREN selectSetStmt RPAREN # ColumnExprCallSelect + | columnExprValue LPAREN columnExprList? RPAREN # ColumnExprCall + | hogqlxTagElement # ColumnExprTagElement + | templateString # ColumnExprTemplateString + | literal # ColumnExprLiteral + + // FIXME(ilezhankin): this part looks very ugly, maybe there is another way to express it + | columnExprValue LBRACKET columnExpr RBRACKET # ColumnExprArrayAccess + | columnExprValue LBRACKET columnExpr? COLON columnExpr? RBRACKET # ColumnExprArraySlice + | columnExprValue DOT DECIMAL_LITERAL # ColumnExprTupleAccess + | columnExprValue DOT identifier # ColumnExprPropertyAccess + | columnExprValue NULL_PROPERTY LBRACKET columnExpr RBRACKET # ColumnExprNullArrayAccess + | columnExprValue NULL_PROPERTY DECIMAL_LITERAL # ColumnExprNullTupleAccess + | columnExprValue NULL_PROPERTY identifier # ColumnExprNullPropertyAccess + | columnExprValue DOUBLECOLON columnTypeCastExpr # ColumnExprTypeCast + | DASH columnExprValue # ColumnExprNegate + | left=columnExprValue ( operator=ASTERISK // * + | operator=SLASH // / + | operator=PERCENT // % + ) right=columnExprValue # ColumnExprPrecedence1 + | left=columnExprValue ( operator=PLUS // + + | operator=DASH // - + | operator=CONCAT // || + ) right=columnExprValue # ColumnExprPrecedence2 + | left=columnExprValue ( operator=EQ_DOUBLE // = + | operator=EQ_SINGLE // == + | operator=NOT_EQ // != + | operator=LT_EQ // <= + | operator=LT // < + | operator=GT_EQ // >= + | operator=GT // > + | operator=NOT? IN COHORT? // in, not in; in cohort; not in cohort + | operator=NOT? (LIKE | ILIKE) // like, not like, ilike, not ilike + | operator=REGEX_SINGLE // ~ + | operator=REGEX_DOUBLE // =~ + | operator=NOT_REGEX // !~ + | operator=IREGEX_SINGLE // ~* + | operator=IREGEX_DOUBLE // =~* + | operator=NOT_IREGEX // !~* + ) right=columnExprValue # ColumnExprPrecedence3 + | columnExprValue IGNORE NULLS # ColumnExprIgnoreNulls + | columnExprValue IS NOT? NULL_SQL # ColumnExprIsNull + | columnExprValue IS NOT? DISTINCT FROM columnExprValue # ColumnExprIsDistinctFrom + | columnExprValue NULL_SAFE_EQ columnExprValue # ColumnExprNullSafeEq // MySQL `a <=> b` ≡ `a IS NOT DISTINCT FROM b` + | columnExprValue NULLISH columnExprValue # ColumnExprNullish + | columnExprValue NOT? BETWEEN columnExprValue AND columnExprValue # ColumnExprBetween + | NOT columnExprValue # ColumnExprNot + | (tableIdentifier DOT)? ASTERISK (EXCLUDE LPAREN identifierList RPAREN)? # ColumnExprAsterisk // single-column only + | LAMBDA identifier (COMMA identifier)* COMMA? COLON columnExpr # ColumnExprColonLambda + | LPAREN selectSetStmt RPAREN # ColumnExprSubquery // single-column only + | LPAREN columnExpr RPAREN # ColumnExprParens // single-column only + | LPAREN columnExprList RPAREN # ColumnExprTuple + | ARRAY? LBRACKET columnExprList? RBRACKET # ColumnExprArray + | LBRACE (kvPairList)? RBRACE # ColumnExprDict + | columnLambdaExpr # ColumnExprLambda + | identifier COLONEQUALS columnExpr # ColumnExprNamedArg + | HASH DECIMAL_LITERAL # ColumnExprPositional + | columnIdentifier # ColumnExprIdentifier + ; + +columnLambdaExpr: + ( LPAREN identifier (COMMA identifier)* COMMA? RPAREN + | identifier (COMMA identifier)* COMMA? + | LPAREN RPAREN + ) + ARROW (columnExpr | block) # ArrowLambda + | LAMBDA identifier (COMMA identifier)* COMMA? COLON columnExpr # ColonLambda + ; + +columnsReplaceList: columnsReplaceItem (COMMA columnsReplaceItem)*; +columnsReplaceItem: columnExpr AS identifier; + +hogqlxChildElement + : hogqlxTagElement + | hogqlxText + | LBRACE columnExpr RBRACE; + +hogqlxText : HOGQLX_TEXT_TEXT ; + +hogqlxTagElement + : LT identifier hogqlxTagAttribute* SLASH_GT # HogqlxTagElementClosed + | LT identifier hogqlxTagAttribute* GT hogqlxChildElement* LT_SLASH identifier GT # HogqlxTagElementNested + ; +hogqlxTagAttribute + : identifier EQ_SINGLE string + | identifier EQ_SINGLE LBRACE columnExpr RBRACE + | identifier + ; + +withExprList: withExpr (COMMA withExpr)* COMMA?; +withExpr + : identifier withExprColumnNameList? (USING KEY withExprColumnNameList)? AS (NOT? MATERIALIZED)? LPAREN selectSetStmt RPAREN # WithExprSubquery + // NOTE: asterisk and subquery goes before |columnExpr| so that we can mark them as multi-column expressions. + | columnExpr AS identifier # WithExprColumn + ; + +withExprColumnNameList: LPAREN identifier (COMMA identifier)* RPAREN; + + +// This is slightly different in HogQL compared to ClickHouse SQL +// HogQL allows unlimited ("*") nestedIdentifier-s "properties.b.a.a.w.a.s". +// We parse and convert "databaseIdentifier.tableIdentifier.columnIdentifier.nestedIdentifier.*" +// to just one ast.Field(chain=['a','b','columnIdentifier','on','and','on']). +columnIdentifier: placeholder | ((tableIdentifier DOT)? nestedIdentifier); +nestedIdentifier: identifier (DOT identifier)*; +tableExpr + : tableIdentifier # TableExprIdentifier + | tableFunctionExpr # TableExprFunction + | LPAREN selectSetStmt RPAREN # TableExprSubquery + | LPAREN valuesClause RPAREN # TableExprValues + | tableExpr PIVOT LPAREN columnExprList pivotColumnList (GROUP BY columnExprList)? RPAREN # TableExprPivot + | tableExpr UNPIVOT (INCLUDE NULLS)? LPAREN unpivotColumnList RPAREN # TableExprUnpivot + | tableExpr (alias | AS identifier) columnAliases? # TableExprAlias + | hogqlxTagElement # TableExprTag + | placeholder # TableExprPlaceholder + ; + +pivotColumnList: FOR pivotColumn+; +pivotColumn: columnExprTupleOrSingle IN LPAREN columnExprList RPAREN; +unpivotColumnList: unpivotColumn (COMMA unpivotColumn)* COMMA?; +unpivotColumn: columnExprTupleOrSingle FOR columnExprTupleOrSingle IN LPAREN columnExprList RPAREN (columnExprTupleOrSingle IN LPAREN columnExprList RPAREN)*; +columnExprTupleOrSingle: LPAREN columnExprList RPAREN | columnExpr; +columnAliases: LPAREN identifier (COMMA identifier)* RPAREN; +tableFunctionExpr: identifier LPAREN tableArgList? RPAREN; +tableIdentifier: (databaseIdentifier DOT)? nestedIdentifier; +tableArgList: columnExpr (COMMA columnExpr)* COMMA?; + +// Databases + +databaseIdentifier: identifier; + +// Basics + +floatingLiteral + : FLOATING_LITERAL + | DOT (DECIMAL_LITERAL | OCTAL_LITERAL) + | DECIMAL_LITERAL DOT (DECIMAL_LITERAL | OCTAL_LITERAL)? // can't move this to the lexer or it will break nested tuple access: t.1.2 + ; +numberLiteral: (PLUS | DASH)? (floatingLiteral | BINARY_LITERAL | OCTAL_LITERAL | OCTAL_PREFIX_LITERAL | DECIMAL_LITERAL | HEXADECIMAL_LITERAL | INF | NAN_SQL); +literal + : numberLiteral + | STRING_LITERAL + | NULL_SQL + ; +interval: SECOND | MINUTE | HOUR | DAY | WEEK | MONTH | QUARTER | YEAR; +keyword + // except NULL_SQL, INF, NAN_SQL + : ALL | AND | ANTI | ANY | ARRAY | AS | ASCENDING | ASOF | BETWEEN | BOTH | BY | CASE + | CAST | COHORT | COLLATE | COLUMNS | CROSS | CUBE | CURRENT | DATE | DESC | DESCENDING + | DISTINCT | ELSE | END | EXCLUDE | EXTRACT | FILL | FILTER | FINAL | FIRST + | FOR | FOLLOWING | FROM | FULL | GROUP | HAVING | ID | INTERPOLATE | IS + | GROUPING | IF | IGNORE | ILIKE | INCLUDE | IN | INNER | INTERVAL | JOIN | KEY + | LAMBDA | LAST | LEADING | LEFT | LIKE | LIMIT + | LOCAL | NAME | NATURAL | NOT | NULLS | OFFSET | ON | OR | ORDER | OUTER | OVER | PARTITION + | PIVOT | POSITIONAL | PRECEDING | PREWHERE | QUALIFY | RANGE | RECURSIVE | REPLACE | RETURN | RIGHT | ROLLUP | ROW + | ROWS | SAMPLE | SELECT | SEMI | SETS | SETTINGS | STEP | SUBSTRING + | THEN | TIES | TIME | TIMESTAMP | TOTALS | TRAILING | TRIM | TRUNCATE | TRY_CAST | TO | TOP + | UNBOUNDED | UNION | UNPIVOT | USING | VALUES | WHEN | WHERE | WINDOW | WITH + | ZONE + ; +keywordForAlias + : DATE | FIRST | ID | KEY + ; +keywordForImplicitAlias + : ASCENDING + | COHORT + | DATE + | DESCENDING + | FINAL + | ID + | RETURN + | TOP + | TOTALS + ; +alias: IDENTIFIER | QUOTED_IDENTIFIER | keywordForAlias; // |interval| can't be an alias, otherwise 'INTERVAL 1 SOMETHING' becomes ambiguous. +implicitAlias: IDENTIFIER | QUOTED_IDENTIFIER | keywordForImplicitAlias; +identifier: IDENTIFIER | QUOTED_IDENTIFIER | interval | keyword; +enumValue: string EQ_SINGLE numberLiteral; +placeholder: LBRACE columnExpr RBRACE; + +string: STRING_LITERAL | templateString; +templateString : QUOTE_SINGLE_TEMPLATE stringContents* QUOTE_SINGLE ; +stringContents : STRING_ESCAPE_TRIGGER columnExpr RBRACE | STRING_TEXT; + +// These are magic "full template strings", which are used to parse "full text field" templates without the surrounding SQL. +// We will need to add F' to the start of the string to change the lexer's mode. +fullTemplateString: QUOTE_SINGLE_TEMPLATE_FULL stringContentsFull* EOF ; +stringContentsFull : FULL_STRING_ESCAPE_TRIGGER columnExpr RBRACE | FULL_STRING_TEXT; diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/language.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/language.json new file mode 100644 index 000000000000..163384afcafa --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/language.json @@ -0,0 +1,25 @@ +{ + "antlrVersion": "4.13.2", + "canonicalParser": "cpp-antlr", + "entryPoints": { + "expression": "expr", + "query": "select" + }, + "files": [ + { + "path": "HogQLLexer.common.g4", + "sha256": "f5a92ca0c19267ee8ded72fe4f5ca05939aea4bb4e1d6ee3b6bf095622adf268" + }, + { + "path": "HogQLLexer.java.g4", + "sha256": "04f799086dfa92aacf7caa30c2e1bf75032118c61fe7eb1d80b8dddba7ad3afc" + }, + { + "path": "HogQLParser.g4", + "sha256": "4cc3cf6f6f8d2584df7cae12a31d546c6bffef3e0e85d7101e7749d6bd0f5430" + } + ], + "grammarSha256": "c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242", + "languageVersion": "1.0.0", + "schemaVersion": 1 +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/source-lock.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/source-lock.json new file mode 100644 index 000000000000..183a9d3dbae4 --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/source-lock.json @@ -0,0 +1,6 @@ +{ + "descriptorSha256": "dfb983a1b449cbf0e09481ac4fbbceb8e12f62184f361971f51768b0f6575b25", + "schemaVersion": 1, + "sourceRepository": "https://github.com/PostHog/posthog", + "sourceRevision": "02508a8eafa30ca2b1ae561c921a8e64d4332dd6" +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlLanguageContract.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlLanguageContract.java new file mode 100644 index 000000000000..089f66d4457e --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlLanguageContract.java @@ -0,0 +1,109 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.fail; + +public class TestHogQlLanguageContract +{ + private static final String LANGUAGE_RESOURCE_ROOT = "/io/trino/hogql/parser/language/1.0.0/"; + + @Test + public void testLoadsCurrentLanguageContract() + { + HogQlLanguageContract contract = HogQlLanguageContract.current(); + + assertThat(contract.schemaVersion()).isEqualTo(1); + assertThat(contract.languageVersion()).isEqualTo(new HogQlLanguageVersion(1, 0, 0)); + assertThat(contract.antlrVersion()).isEqualTo("4.13.2"); + assertThat(contract.canonicalParser()).isEqualTo("cpp-antlr"); + assertThat(contract.entryPoints()).containsEntry("query", "select"); + assertThat(contract.grammarSha256()).isEqualTo("c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242"); + } + + @ParameterizedTest + @ValueSource(strings = {"", "1", "1.0", "01.0.0", "1.0.0-beta", "-1.0.0"}) + public void testRejectsInvalidLanguageVersions(String value) + { + assertThatThrownBy(() -> HogQlLanguageVersion.valueOf(value)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("invalid HogQL language version"); + } + + @Test + public void testRejectsUnsupportedLanguageVersion() + { + assertThatThrownBy(() -> new HogQlParser().parseStatement("SELECT 1", new HogQlLanguageVersion(1, 1, 0))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("unsupported HogQL language version: 1.1.0"); + } + + @Test + public void testGrammarFilesMatchLanguageDescriptor() + throws IOException + { + HogQlLanguageContract contract = HogQlLanguageContract.current(); + MessageDigest aggregate = sha256Digest(); + + for (HogQlLanguageContract.GrammarFile file : contract.files()) { + byte[] content = readResource(LANGUAGE_RESOURCE_ROOT + "grammar/" + file.path()); + assertThat(sha256(content)).isEqualTo(file.sha256()); + aggregate.update(file.path().getBytes(StandardCharsets.UTF_8)); + aggregate.update((byte) 0); + aggregate.update(content); + aggregate.update((byte) 0); + } + + assertThat(HexFormat.of().formatHex(aggregate.digest())).isEqualTo(contract.grammarSha256()); + } + + private static byte[] readResource(String path) + throws IOException + { + try (InputStream input = TestHogQlLanguageContract.class.getResourceAsStream(path)) { + if (input == null) { + fail("missing test resource: " + path); + } + return input.readAllBytes(); + } + } + + private static String sha256(byte[] content) + { + return HexFormat.of().formatHex(sha256Digest().digest(content)); + } + + private static MessageDigest sha256Digest() + { + try { + return MessageDigest.getInstance("SHA-256"); + } + catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } +} From c923e6ace93de3e3e24286d5fcf8a985ab2a2396 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Wed, 26 Aug 2026 22:17:09 -0400 Subject: [PATCH 004/136] feat(hogql): parse canonical expressions --- .../trino/hogql/compiler/TrinoAstFactory.java | 54 ++- .../hogql/compiler/TestHogQlCompiler.java | 6 +- .../io/trino/hogql/parser/HogQlParser.java | 439 +++++++++++++++--- .../trino/hogql/parser/tree/HogQlQuery.java | 86 +++- .../trino/hogql/parser/TestHogQlParser.java | 33 +- 5 files changed, 546 insertions(+), 72 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 6fb864e418d2..8e52e7de7d8c 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -14,21 +14,30 @@ package io.trino.hogql.compiler; import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.Identifier; import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.Star; import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; import io.trino.sql.tree.AllColumns; +import io.trino.sql.tree.ArithmeticBinaryExpression; +import io.trino.sql.tree.ArithmeticUnaryExpression; import io.trino.sql.tree.BooleanLiteral; +import io.trino.sql.tree.ComparisonPredicate; import io.trino.sql.tree.DereferenceExpression; import io.trino.sql.tree.Expression; +import io.trino.sql.tree.LogicalExpression; import io.trino.sql.tree.LongLiteral; import io.trino.sql.tree.NodeLocation; +import io.trino.sql.tree.NotExpression; import io.trino.sql.tree.NullLiteral; +import io.trino.sql.tree.Predicated; import io.trino.sql.tree.QualifiedName; import io.trino.sql.tree.Query; import io.trino.sql.tree.QuerySpecification; @@ -55,7 +64,7 @@ public static Statement createStatement(HogQlQuery query) .map(TrinoAstFactory::createSelectItem) .toList()), query.from().map(TrinoAstFactory::createTable), - Optional.empty(), + query.where().map(TrinoAstFactory::createExpression), Optional.empty(), Optional.empty(), List.of(), @@ -80,18 +89,57 @@ private static SelectItem createSelectItem(Projection projection) case ExpressionProjection expression -> new SingleColumn( location(expression.span()), createExpression(expression.expression()), - Optional.empty()); + expression.alias().map(TrinoAstFactory::createIdentifier)); }; } private static Expression createExpression(HogQlQuery.Expression expression) { return switch (expression) { + case BinaryExpression binary -> createBinaryExpression(binary); case ColumnReference reference -> createColumnReference(reference); + case FunctionCall function -> new io.trino.sql.tree.FunctionCall( + location(function.span()), + QualifiedName.of(List.of(createIdentifier(function.name()))), + function.arguments().stream() + .map(TrinoAstFactory::createExpression) + .toList()); case Literal literal -> createLiteral(literal); + case UnaryExpression unary -> switch (unary.operator()) { + case NEGATE -> ArithmeticUnaryExpression.negative(location(unary.span()), createExpression(unary.operand())); + case NOT -> new NotExpression(location(unary.span()), createExpression(unary.operand())); + case POSITIVE -> ArithmeticUnaryExpression.positive(location(unary.span()), createExpression(unary.operand())); + }; + }; + } + + private static Expression createBinaryExpression(BinaryExpression binary) + { + NodeLocation location = location(binary.span()); + Expression left = createExpression(binary.left()); + Expression right = createExpression(binary.right()); + return switch (binary.operator()) { + case ADD -> new ArithmeticBinaryExpression(location, ArithmeticBinaryExpression.Operator.ADD, left, right); + case SUBTRACT -> new ArithmeticBinaryExpression(location, ArithmeticBinaryExpression.Operator.SUBTRACT, left, right); + case MULTIPLY -> new ArithmeticBinaryExpression(location, ArithmeticBinaryExpression.Operator.MULTIPLY, left, right); + case DIVIDE -> new ArithmeticBinaryExpression(location, ArithmeticBinaryExpression.Operator.DIVIDE, left, right); + case MODULO -> new ArithmeticBinaryExpression(location, ArithmeticBinaryExpression.Operator.MODULO, left, right); + case AND -> new LogicalExpression(location, LogicalExpression.Operator.AND, List.of(left, right)); + case OR -> new LogicalExpression(location, LogicalExpression.Operator.OR, List.of(left, right)); + case EQUAL -> comparison(location, ComparisonPredicate.Operator.EQUAL, left, right); + case NOT_EQUAL -> comparison(location, ComparisonPredicate.Operator.NOT_EQUAL, left, right); + case LESS_THAN -> comparison(location, ComparisonPredicate.Operator.LESS_THAN, left, right); + case LESS_THAN_OR_EQUAL -> comparison(location, ComparisonPredicate.Operator.LESS_THAN_OR_EQUAL, left, right); + case GREATER_THAN -> comparison(location, ComparisonPredicate.Operator.GREATER_THAN, left, right); + case GREATER_THAN_OR_EQUAL -> comparison(location, ComparisonPredicate.Operator.GREATER_THAN_OR_EQUAL, left, right); }; } + private static Expression comparison(NodeLocation location, ComparisonPredicate.Operator operator, Expression left, Expression right) + { + return new Predicated(location, left, new ComparisonPredicate(location, operator, right)); + } + private static Expression createLiteral(Literal literal) { NodeLocation location = location(literal.span()); @@ -128,6 +176,6 @@ private static io.trino.sql.tree.Identifier createIdentifier(Identifier identifi private static NodeLocation location(SourceSpan span) { - return new NodeLocation(span.line(), span.column()); + return new NodeLocation(span.startLine(), span.startColumn()); } } diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index e506cbfa5841..eb804ca57c26 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -48,6 +48,10 @@ public class TestHogQlCompiler "SELECT * FROM events", "SELECT event, properties FROM ducklake.default.events", "SELECT * FROM \"MiXeD\".\"Event Table\"", + "SELECT event + 1 * 2 FROM events WHERE event >= 3 AND NOT false", + "SELECT lower(event) AS lowered FROM events", + "SELECT 01", + "SELECT -1, +2", }) public void testLowersToEquivalentStockTrinoAst(String hogql) { @@ -106,7 +110,7 @@ public void testReturnsOnlyStockTrinoTreeNodes() @Test public void testReportsParsingFailureAsTrinoSyntaxError() { - assertThatThrownBy(() -> compiler.compile("SELECT 1\nWHERE TRUE")) + assertThatThrownBy(() -> compiler.compile("SELECT 1\nGROUP BY 1")) .isInstanceOf(ParsingException.class) .hasMessageStartingWith("line 2:1:"); } diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 47b8f5d2e0bd..a1481083393a 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -13,20 +13,22 @@ */ package io.trino.hogql.parser; -import io.trino.hogql.parser.antlr.HogQlBaseBaseVisitor; -import io.trino.hogql.parser.antlr.HogQlBaseLexer; -import io.trino.hogql.parser.antlr.HogQlBaseParser; +import io.trino.hogql.parser.canonical.HogQLLexer; +import io.trino.hogql.parser.canonical.HogQLParser; import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; import io.trino.hogql.parser.tree.HogQlQuery.Expression; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.Identifier; import io.trino.hogql.parser.tree.HogQlQuery.Literal; -import io.trino.hogql.parser.tree.HogQlQuery.LiteralKind; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.Star; import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; import org.antlr.v4.runtime.ANTLRErrorListener; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.BaseErrorListener; @@ -40,13 +42,30 @@ import org.antlr.v4.runtime.atn.PredictionMode; import org.antlr.v4.runtime.misc.ParseCancellationException; +import java.util.ArrayList; import java.util.List; import java.util.Optional; -import static io.trino.hogql.parser.tree.HogQlQuery.LiteralKind.BOOLEAN; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.ADD; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.AND; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.DIVIDE; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.EQUAL; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.GREATER_THAN; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.GREATER_THAN_OR_EQUAL; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.LESS_THAN; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.LESS_THAN_OR_EQUAL; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.MODULO; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.MULTIPLY; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.NOT_EQUAL; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.OR; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.SUBTRACT; import static io.trino.hogql.parser.tree.HogQlQuery.LiteralKind.INTEGER; import static io.trino.hogql.parser.tree.HogQlQuery.LiteralKind.NULL; import static io.trino.hogql.parser.tree.HogQlQuery.LiteralKind.STRING; +import static io.trino.hogql.parser.tree.HogQlQuery.UnaryOperator.NEGATE; +import static io.trino.hogql.parser.tree.HogQlQuery.UnaryOperator.NOT; +import static io.trino.hogql.parser.tree.HogQlQuery.UnaryOperator.POSITIVE; +import static java.lang.Character.digit; import static java.util.Objects.requireNonNull; public final class HogQlParser @@ -76,29 +95,29 @@ public HogQlQuery parseStatement(String hogql, HogQlLanguageVersion languageVers } try { - HogQlBaseLexer lexer = new HogQlBaseLexer(CharStreams.fromString(hogql)); + HogQLLexer lexer = new HogQLLexer(CharStreams.fromString(hogql)); CommonTokenStream tokenStream = new CommonTokenStream(lexer); - HogQlBaseParser parser = new HogQlBaseParser(tokenStream); + HogQLParser parser = new HogQLParser(tokenStream); lexer.removeErrorListeners(); lexer.addErrorListener(ERROR_LISTENER); parser.removeErrorListeners(); - HogQlBaseParser.SingleStatementContext tree; + HogQLParser.SelectContext tree; try { parser.getInterpreter().setPredictionMode(PredictionMode.SLL); parser.setErrorHandler(new BailErrorStrategy()); - tree = parser.singleStatement(); + tree = parser.select(); } catch (ParseCancellationException _) { parser.reset(); parser.getInterpreter().setPredictionMode(PredictionMode.LL); parser.setErrorHandler(new DefaultErrorStrategy()); parser.addErrorListener(ERROR_LISTENER); - tree = parser.singleStatement(); + tree = parser.select(); } - return new AstBuilder().build(tree.query()); + return new AstBuilder(hogql).build(tree); } catch (StackOverflowError _) { throw new HogQlParsingException("statement is too large", null, 1, 1); @@ -106,95 +125,403 @@ public HogQlQuery parseStatement(String hogql, HogQlLanguageVersion languageVers } private static final class AstBuilder - extends HogQlBaseBaseVisitor { - public HogQlQuery build(HogQlBaseParser.QueryContext context) + private final SourcePositions sourcePositions; + + private AstBuilder(String source) + { + sourcePositions = new SourcePositions(source); + } + + public HogQlQuery build(HogQLParser.SelectContext context) { - List projections = context.projection().stream() + HogQLParser.SelectStmtContext select = extractPlainSelect(context); + rejectUnsupportedClauses(select); + + List projections = selectColumns(select.selectColumnExprListBeforeFrom()).stream() .map(this::buildProjection) .toList(); - Optional from = Optional.ofNullable(context.qualifiedName()) + Optional from = Optional.ofNullable(select.fromClause()) + .map(HogQLParser.FromClauseContext::joinExpr) .map(this::buildTableReference); - return new HogQlQuery(projections, from, sourceSpan(context)); + Optional where = Optional.ofNullable(select.whereClause()) + .map(HogQLParser.WhereClauseContext::columnExpr) + .map(this::buildExpression); + return new HogQlQuery(projections, from, where, sourceSpan(select)); } - private Projection buildProjection(HogQlBaseParser.ProjectionContext context) + private List selectColumns(HogQLParser.SelectColumnExprListBeforeFromContext context) { - if (context.ASTERISK() != null) { - return new Star(sourceSpan(context)); + if (context instanceof HogQLParser.SelectColumnExprListBeforeFromTrailingCommaContext trailingComma) { + return trailingComma.selectColumnExpr(); } - return new ExpressionProjection(buildExpression(context.expression())); + if (context instanceof HogQLParser.SelectColumnExprListBeforeFromPlainContext plain) { + return plain.selectColumnExprList().selectColumnExpr(); + } + throw unsupported(context, "select list"); } - private Expression buildExpression(HogQlBaseParser.ExpressionContext context) + private HogQLParser.SelectStmtContext extractPlainSelect(HogQLParser.SelectContext context) { - if (context.literal() != null) { - return buildLiteral(context.literal()); + if (context.selectStmt() != null) { + return context.selectStmt(); + } + HogQLParser.SelectSetStmtContext set = context.selectSetStmt(); + if (set == null || !set.subsequentSelectSetClause().isEmpty() || set.orderByClause() != null || set.limitAndOffsetClauseOptional() != null) { + throw unsupported(context, "set query"); + } + HogQLParser.SelectStmtWithParensContext wrapped = set.selectStmtWithParens(); + if (wrapped.selectStmt() == null) { + throw unsupported(wrapped, "parenthesized or placeholder query"); } - return new ColumnReference(buildIdentifiers(context.qualifiedName()), sourceSpan(context)); + return wrapped.selectStmt(); } - private Literal buildLiteral(HogQlBaseParser.LiteralContext context) + private void rejectUnsupportedClauses(HogQLParser.SelectStmtContext context) { - String value = context.getText(); - LiteralKind kind; - if (context.INTEGER_VALUE() != null) { - kind = INTEGER; + List clauses = new ArrayList<>(); + clauses.add(context.withClause()); + clauses.add(context.topClause()); + clauses.add(context.arrayJoinClause()); + clauses.add(context.prewhereClause()); + clauses.addAll(context.sampleClause()); + clauses.add(context.groupByClause()); + clauses.add(context.havingClause()); + clauses.add(context.qualifyClause()); + clauses.add(context.windowClause()); + clauses.add(context.orderByClause()); + clauses.add(context.limitByClause()); + clauses.add(context.limitAndOffsetClause()); + clauses.add(context.offsetOnlyClause()); + clauses.add(context.settingsClause()); + Optional firstClause = clauses.stream() + .filter(requireNonNullClause -> requireNonNullClause != null) + .min((left, right) -> Integer.compare(left.getStart().getStartIndex(), right.getStart().getStartIndex())); + if (firstClause.isPresent()) { + throw unsupported(firstClause.orElseThrow(), "query clause"); } - else if (context.STRING() != null) { - kind = STRING; - value = decodeString(value); + if (context.DISTINCT() != null) { + throw unsupported(context, "distinct query"); } - else if (context.NULL() != null) { - kind = NULL; + } + + private Projection buildProjection(HogQLParser.SelectColumnExprContext context) + { + HogQLParser.ColumnExprContext expression; + Optional alias = Optional.empty(); + if (context instanceof HogQLParser.ColumnExprAliasBeforeContext aliased) { + expression = aliased.columnExpr(); + alias = Optional.of(buildIdentifier(aliased.identifier())); + } + else if (context instanceof HogQLParser.ColumnExprAliasImplicitContext aliased) { + expression = aliased.columnExpr(); + alias = Optional.of(buildIdentifier(aliased.implicitAlias().getText(), aliased.implicitAlias())); + } + else if (context instanceof HogQLParser.ColumnExprSelectValueContext selected) { + expression = selected.columnExpr(); + if (expression instanceof HogQLParser.ColumnExprAliasContext aliased) { + expression = aliased.columnExpr(); + if (aliased.identifier() != null) { + alias = Optional.of(buildIdentifier(aliased.identifier())); + } + else { + alias = Optional.of(new Identifier(decodeQuoted(aliased.STRING_LITERAL().getText()), true, sourceSpan(aliased))); + } + } } else { - kind = BOOLEAN; + throw unsupported(context, "projection"); + } + + if (expression instanceof HogQLParser.ColumnExprValuePassthroughContext passthrough && passthrough.columnExprValue() instanceof HogQLParser.ColumnExprAsteriskContext asterisk) { + if (alias.isPresent() || asterisk.tableIdentifier() != null || asterisk.EXCLUDE() != null) { + throw unsupported(asterisk, "qualified or transformed star"); + } + return new Star(sourceSpan(context)); } - return new Literal(kind, value, sourceSpan(context)); + return new ExpressionProjection(buildExpression(expression), alias); } - private TableReference buildTableReference(HogQlBaseParser.QualifiedNameContext context) + private Expression buildExpression(HogQLParser.ColumnExprContext context) { - return new TableReference(buildIdentifiers(context), sourceSpan(context)); + if (context instanceof HogQLParser.ColumnExprValuePassthroughContext passthrough) { + return buildExpression(passthrough.columnExprValue()); + } + if (context instanceof HogQLParser.ColumnExprAndContext binary) { + return binary(AND, binary.columnExpr(0), binary.columnExpr(1), binary); + } + if (context instanceof HogQLParser.ColumnExprOrContext binary) { + return binary(OR, binary.columnExpr(0), binary.columnExpr(1), binary); + } + throw unsupported(context, "expression " + context.getClass().getSimpleName()); } - private List buildIdentifiers(HogQlBaseParser.QualifiedNameContext context) + private Expression buildExpression(HogQLParser.ColumnExprValueContext context) + { + if (context instanceof HogQLParser.ColumnExprLiteralContext literal) { + return buildLiteral(literal.literal()); + } + if (context instanceof HogQLParser.ColumnExprIdentifierContext identifier) { + return buildColumnReference(identifier.columnIdentifier()); + } + if (context instanceof HogQLParser.ColumnExprParensContext parens) { + return buildExpression(parens.columnExpr()); + } + if (context instanceof HogQLParser.ColumnExprNegateContext negate) { + return new UnaryExpression(NEGATE, buildExpression(negate.columnExprValue()), sourceSpan(negate)); + } + if (context instanceof HogQLParser.ColumnExprNotContext not) { + return new UnaryExpression(NOT, buildExpression(not.columnExprValue()), sourceSpan(not)); + } + if (context instanceof HogQLParser.ColumnExprPrecedence1Context binary) { + BinaryOperator operator = switch (binary.operator.getType()) { + case HogQLParser.ASTERISK -> MULTIPLY; + case HogQLParser.SLASH -> DIVIDE; + case HogQLParser.PERCENT -> MODULO; + default -> throw unsupported(binary, "multiplicative operator"); + }; + return binary(operator, binary.left, binary.right, binary); + } + if (context instanceof HogQLParser.ColumnExprPrecedence2Context binary) { + BinaryOperator operator = switch (binary.operator.getType()) { + case HogQLParser.PLUS -> ADD; + case HogQLParser.DASH -> SUBTRACT; + default -> throw unsupported(binary, "additive operator"); + }; + return binary(operator, binary.left, binary.right, binary); + } + if (context instanceof HogQLParser.ColumnExprPrecedence3Context binary) { + BinaryOperator operator = switch (binary.operator.getType()) { + case HogQLParser.EQ_DOUBLE, HogQLParser.EQ_SINGLE -> EQUAL; + case HogQLParser.NOT_EQ -> NOT_EQUAL; + case HogQLParser.LT -> LESS_THAN; + case HogQLParser.LT_EQ -> LESS_THAN_OR_EQUAL; + case HogQLParser.GT -> GREATER_THAN; + case HogQLParser.GT_EQ -> GREATER_THAN_OR_EQUAL; + default -> throw unsupported(binary, "comparison operator"); + }; + return binary(operator, binary.left, binary.right, binary); + } + if (context instanceof HogQLParser.ColumnExprFunctionContext function) { + return buildFunction(function); + } + throw unsupported(context, "expression " + context.getClass().getSimpleName()); + } + + private BinaryExpression binary(BinaryOperator operator, HogQLParser.ColumnExprContext left, HogQLParser.ColumnExprContext right, ParserRuleContext context) + { + return new BinaryExpression(operator, buildExpression(left), buildExpression(right), sourceSpan(context)); + } + + private BinaryExpression binary(BinaryOperator operator, HogQLParser.ColumnExprValueContext left, HogQLParser.ColumnExprValueContext right, ParserRuleContext context) + { + return new BinaryExpression(operator, buildExpression(left), buildExpression(right), sourceSpan(context)); + } + + private FunctionCall buildFunction(HogQLParser.ColumnExprFunctionContext context) + { + if (context.columnExprs != null || context.DISTINCT() != null || context.orderExprList() != null || context.filterExpr != null) { + throw unsupported(context, "parametric, distinct, ordered, or filtered function"); + } + List arguments = context.columnArgList == null ? List.of() : context.columnArgList.columnExpr().stream() + .map(this::buildExpression) + .toList(); + return new FunctionCall(buildIdentifier(context.identifier()), arguments, sourceSpan(context)); + } + + private Expression buildLiteral(HogQLParser.LiteralContext context) + { + if (context.NULL_SQL() != null) { + return new Literal(NULL, "null", sourceSpan(context)); + } + if (context.STRING_LITERAL() != null) { + return new Literal(STRING, decodeQuoted(context.getText()), sourceSpan(context)); + } + + String value = context.numberLiteral().getText(); + if (!value.matches("[+-]?[0-9]+")) { + throw unsupported(context, "numeric literal"); + } + if (value.startsWith("-")) { + return new Literal(INTEGER, "-" + normalizeInteger(value.substring(1)), sourceSpan(context)); + } + if (value.startsWith("+")) { + Literal magnitude = new Literal(INTEGER, normalizeInteger(value.substring(1)), sourceSpan(context)); + return new UnaryExpression(POSITIVE, magnitude, sourceSpan(context)); + } + return new Literal(INTEGER, normalizeInteger(value), sourceSpan(context)); + } + + private Expression buildColumnReference(HogQLParser.ColumnIdentifierContext context) + { + if (context.placeholder() != null) { + throw unsupported(context, "placeholder"); + } + List parts = new ArrayList<>(); + if (context.tableIdentifier() != null) { + parts.addAll(buildIdentifiers(context.tableIdentifier())); + } + parts.addAll(buildIdentifiers(context.nestedIdentifier())); + if (parts.size() == 1 && !parts.getFirst().delimited()) { + if (parts.getFirst().value().equalsIgnoreCase("true")) { + return new Literal(HogQlQuery.LiteralKind.BOOLEAN, "true", sourceSpan(context)); + } + if (parts.getFirst().value().equalsIgnoreCase("false")) { + return new Literal(HogQlQuery.LiteralKind.BOOLEAN, "false", sourceSpan(context)); + } + } + return new ColumnReference(parts, sourceSpan(context)); + } + + private TableReference buildTableReference(HogQLParser.JoinExprContext context) + { + if (!(context instanceof HogQLParser.JoinExprTableContext table) || table.FINAL() != null || table.sampleClause() != null || !(table.tableExpr() instanceof HogQLParser.TableExprIdentifierContext identifier)) { + throw unsupported(context, "table expression or join"); + } + return new TableReference(buildIdentifiers(identifier.tableIdentifier()), sourceSpan(context)); + } + + private List buildIdentifiers(HogQLParser.TableIdentifierContext context) + { + List parts = new ArrayList<>(); + if (context.databaseIdentifier() != null) { + parts.add(buildIdentifier(context.databaseIdentifier().identifier())); + } + parts.addAll(buildIdentifiers(context.nestedIdentifier())); + return List.copyOf(parts); + } + + private List buildIdentifiers(HogQLParser.NestedIdentifierContext context) { return context.identifier().stream() .map(this::buildIdentifier) .toList(); } - private Identifier buildIdentifier(HogQlBaseParser.IdentifierContext context) + private Identifier buildIdentifier(HogQLParser.IdentifierContext context) + { + return buildIdentifier(context.getText(), context); + } + + private Identifier buildIdentifier(String text, ParserRuleContext context) + { + boolean delimited = text.startsWith("`") || text.startsWith("\""); + return new Identifier(delimited ? decodeQuoted(text) : text, delimited, sourceSpan(context)); + } + + private SourceSpan sourceSpan(ParserRuleContext context) + { + Token start = context.getStart(); + Token stop = context.getStop(); + int startOffset = start.getStartIndex(); + int endOffset = stop.getStopIndex() + 1; + return new SourceSpan( + startOffset, + endOffset, + sourcePositions.line(startOffset), + sourcePositions.column(startOffset), + sourcePositions.line(endOffset), + sourcePositions.column(endOffset)); + } + + private HogQlParsingException unsupported(ParserRuleContext context, String feature) + { + SourceSpan span = sourceSpan(context); + return new HogQlParsingException("HogQL feature is not lowered yet: " + feature, null, span.startLine(), span.startColumn()); + } + + private static String normalizeInteger(String value) { - String value = context.getText(); - boolean delimited = context.IDENTIFIER() == null; - if (context.QUOTED_IDENTIFIER() != null) { - value = unquoteAndUnescape(value, '"'); + int firstNonZero = 0; + while (firstNonZero < value.length() - 1 && value.charAt(firstNonZero) == '0') { + firstNonZero++; } - else if (context.BACKQUOTED_IDENTIFIER() != null) { - value = unquoteAndUnescape(value, '`'); + return value.substring(firstNonZero); + } + + private static String decodeQuoted(String text) + { + char quote = text.charAt(0); + StringBuilder decoded = new StringBuilder(text.length() - 2); + for (int index = 1; index < text.length() - 1; index++) { + char character = text.charAt(index); + if (character == quote && text.charAt(index + 1) == quote) { + decoded.append(quote); + index++; + } + else if (character != '\\') { + decoded.append(character); + } + else { + char escaped = text.charAt(++index); + if (escaped == 'x') { + decoded.append((char) ((digit(text.charAt(++index), 16) << 4) | digit(text.charAt(++index), 16))); + } + else { + decoded.append(switch (escaped) { + case '0' -> '\0'; + case 'a' -> '\u0007'; + case 'b' -> '\b'; + case 'f' -> '\f'; + case 'n' -> '\n'; + case 'r' -> '\r'; + case 't' -> '\t'; + case 'v' -> '\u000B'; + default -> escaped; + }); + } + } } - return new Identifier(value, delimited, sourceSpan(context)); + return decoded.toString(); } + } - private static String decodeString(String value) + private static final class SourcePositions + { + private final int[] lines; + private final int[] columns; + + private SourcePositions(String source) { - return value.substring(1, value.length() - 1).replace("''", "'"); + int length = source.codePointCount(0, source.length()); + lines = new int[length + 1]; + columns = new int[length + 1]; + lines[0] = 1; + columns[0] = 1; + + int line = 1; + int column = 1; + int codePointOffset = 0; + boolean previousWasCarriageReturn = false; + for (int charOffset = 0; charOffset < source.length(); ) { + int codePoint = source.codePointAt(charOffset); + charOffset += Character.charCount(codePoint); + codePointOffset++; + if (codePoint == '\n' && previousWasCarriageReturn) { + previousWasCarriageReturn = false; + } + else if (codePoint == '\n' || codePoint == '\r') { + line++; + column = 1; + previousWasCarriageReturn = codePoint == '\r'; + } + else { + column++; + previousWasCarriageReturn = false; + } + lines[codePointOffset] = line; + columns[codePointOffset] = column; + } } - private static String unquoteAndUnescape(String value, char quote) + public int line(int offset) { - String quoted = value.substring(1, value.length() - 1); - return quoted.replace(String.valueOf(quote).repeat(2), String.valueOf(quote)); + return lines[offset]; } - private static SourceSpan sourceSpan(ParserRuleContext context) + public int column(int offset) { - Token start = context.getStart(); - Token stop = context.getStop(); - return new SourceSpan(start.getStartIndex(), stop.getStopIndex() + 1, start.getLine(), start.getCharPositionInLine() + 1); + return columns[offset]; } } } diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index fa161a966697..89b8b51d3043 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -18,12 +18,13 @@ import static java.util.Objects.requireNonNull; -public record HogQlQuery(List projections, Optional from, SourceSpan span) +public record HogQlQuery(List projections, Optional from, Optional where, SourceSpan span) { public HogQlQuery { projections = List.copyOf(requireNonNull(projections, "projections is null")); from = requireNonNull(from, "from is null"); + where = requireNonNull(where, "where is null"); span = requireNonNull(span, "span is null"); } @@ -42,12 +43,13 @@ public record Star(SourceSpan span) } } - public record ExpressionProjection(Expression expression) + public record ExpressionProjection(Expression expression, Optional alias) implements Projection { public ExpressionProjection { expression = requireNonNull(expression, "expression is null"); + alias = requireNonNull(alias, "alias is null"); } @Override @@ -58,11 +60,73 @@ public SourceSpan span() } public sealed interface Expression - permits ColumnReference, Literal + permits BinaryExpression, + ColumnReference, + FunctionCall, + Literal, + UnaryExpression { SourceSpan span(); } + public record UnaryExpression(UnaryOperator operator, Expression operand, SourceSpan span) + implements Expression + { + public UnaryExpression + { + operator = requireNonNull(operator, "operator is null"); + operand = requireNonNull(operand, "operand is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record BinaryExpression(BinaryOperator operator, Expression left, Expression right, SourceSpan span) + implements Expression + { + public BinaryExpression + { + operator = requireNonNull(operator, "operator is null"); + left = requireNonNull(left, "left is null"); + right = requireNonNull(right, "right is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record FunctionCall(Identifier name, List arguments, SourceSpan span) + implements Expression + { + public FunctionCall + { + name = requireNonNull(name, "name is null"); + arguments = List.copyOf(requireNonNull(arguments, "arguments is null")); + span = requireNonNull(span, "span is null"); + } + } + + public enum UnaryOperator + { + NEGATE, + NOT, + POSITIVE, + } + + public enum BinaryOperator + { + ADD, + AND, + DIVIDE, + EQUAL, + GREATER_THAN, + GREATER_THAN_OR_EQUAL, + LESS_THAN, + LESS_THAN_OR_EQUAL, + MODULO, + MULTIPLY, + NOT_EQUAL, + OR, + SUBTRACT, + } + public record ColumnReference(List parts, SourceSpan span) implements Expression { @@ -119,7 +183,7 @@ public record Identifier(String value, boolean delimited, SourceSpan span) } } - public record SourceSpan(int startOffset, int endOffset, int line, int column) + public record SourceSpan(int startOffset, int endOffset, int startLine, int startColumn, int endLine, int endColumn) { public SourceSpan { @@ -129,11 +193,17 @@ public record SourceSpan(int startOffset, int endOffset, int line, int column) if (endOffset < startOffset) { throw new IllegalArgumentException("endOffset is before startOffset"); } - if (line < 1) { - throw new IllegalArgumentException("line must be positive"); + if (startLine < 1) { + throw new IllegalArgumentException("startLine must be positive"); + } + if (startColumn < 1) { + throw new IllegalArgumentException("startColumn must be positive"); + } + if (endLine < startLine) { + throw new IllegalArgumentException("endLine is before startLine"); } - if (column < 1) { - throw new IllegalArgumentException("column must be positive"); + if (endColumn < 1) { + throw new IllegalArgumentException("endColumn must be positive"); } } } diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 47badb42b002..054ef6d2e776 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -13,9 +13,14 @@ */ package io.trino.hogql.parser; +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TestHogQlParser @@ -25,19 +30,39 @@ public class TestHogQlParser @ParameterizedTest @ValueSource(strings = { "SELECT", - "SELECT 1 WHERE TRUE", "SELECT count(*)", "SELECT * FROM first JOIN second", "DROP TABLE events", "SELECT 1; SELECT 2", - "SELECT 'unsupported\\nescape'", "SELECT * FROM", - "SELECT 01", + "SELECT {value}", + "SELECT 1 GROUP BY 1", }) - public void testRejectsSyntaxOutsideM0(String hogql) + public void testRejectsSyntaxWithoutAnAstMapping(String hogql) { assertThatThrownBy(() -> parser.parseStatement(hogql)) .isInstanceOf(HogQlParsingException.class) .hasMessageStartingWith("line "); } + + @Test + public void testUsesCanonicalGrammarForExpressionsAndWhereClause() + { + HogQlQuery query = parser.parseStatement("SELECT event + 1 * 2 FROM events WHERE event >= 3 AND NOT false"); + + assertThat(query.projections()).hasSize(1); + assertThat(query.from()).isPresent(); + assertThat(query.where()).isPresent(); + } + + @Test + public void testSourceSpansUseUnicodeCodePointOffsetsAndEndPositions() + { + HogQlQuery query = parser.parseStatement("SELECT '😀',\n event"); + ExpressionProjection projection = (ExpressionProjection) query.projections().get(1); + ColumnReference reference = (ColumnReference) projection.expression(); + + assertThat(query.span()).isEqualTo(new HogQlQuery.SourceSpan(0, 18, 1, 1, 2, 7)); + assertThat(reference.span()).isEqualTo(new HogQlQuery.SourceSpan(13, 18, 2, 2, 2, 7)); + } } From 5301a22ed1e2a3b4f3180545908e1f21b843ff19 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Wed, 26 Aug 2026 22:21:24 -0400 Subject: [PATCH 005/136] feat(hogql): vendor compatibility artifact --- .../hogql/parser/HogQlLanguageContract.java | 52 + .../1.0.0/corpus/expr_select_cases.json | 210 ++ .../1.0.0/corpus/expr_select_cpp_oracle.jsonl | 28 + .../language/1.0.0/grammar-features.json | 2771 +++++++++++++++++ .../hogql/parser/language/1.0.0/language.json | 15 + .../parser/language/1.0.0/source-lock.json | 4 +- .../parser/TestHogQlLanguageContract.java | 20 + 7 files changed, 3098 insertions(+), 2 deletions(-) create mode 100644 core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/corpus/expr_select_cases.json create mode 100644 core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/corpus/expr_select_cpp_oracle.jsonl create mode 100644 core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-features.json diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageContract.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageContract.java index d560f9aaf6f2..c653198c3b58 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageContract.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageContract.java @@ -32,8 +32,10 @@ public record HogQlLanguageContract( HogQlLanguageVersion languageVersion, String antlrVersion, String canonicalParser, + List corpora, Map entryPoints, List files, + GrammarFeatureManifest grammarFeatureManifest, String grammarSha256) { private static final String CURRENT_CONTRACT_RESOURCE = "/io/trino/hogql/parser/language/1.0.0/language.json"; @@ -46,8 +48,10 @@ public HogQlLanguageContract( @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, @JsonProperty("antlrVersion") String antlrVersion, @JsonProperty("canonicalParser") String canonicalParser, + @JsonProperty("corpora") List corpora, @JsonProperty("entryPoints") Map entryPoints, @JsonProperty("files") List files, + @JsonProperty("grammarFeatureManifest") GrammarFeatureManifest grammarFeatureManifest, @JsonProperty("grammarSha256") String grammarSha256) { if (schemaVersion != 1) { @@ -57,8 +61,10 @@ public HogQlLanguageContract( this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); this.antlrVersion = requireNonNull(antlrVersion, "antlrVersion is null"); this.canonicalParser = requireNonNull(canonicalParser, "canonicalParser is null"); + this.corpora = List.copyOf(requireNonNull(corpora, "corpora is null")); this.entryPoints = Map.copyOf(requireNonNull(entryPoints, "entryPoints is null")); this.files = List.copyOf(requireNonNull(files, "files is null")); + this.grammarFeatureManifest = requireNonNull(grammarFeatureManifest, "grammarFeatureManifest is null"); this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); } @@ -91,4 +97,50 @@ public GrammarFile( this.sha256 = requireNonNull(sha256, "sha256 is null"); } } + + public record GrammarFeatureManifest(String path, int schemaVersion, String sha256) + { + @JsonCreator + public GrammarFeatureManifest( + @JsonProperty("path") String path, + @JsonProperty("schemaVersion") int schemaVersion, + @JsonProperty("sha256") String sha256) + { + this.path = requireNonNull(path, "path is null"); + if (schemaVersion != 1) { + throw new IllegalArgumentException("unsupported HogQL grammar feature schema: " + schemaVersion); + } + this.schemaVersion = schemaVersion; + this.sha256 = requireNonNull(sha256, "sha256 is null"); + } + } + + public record Corpus( + String manifestPath, + String manifestSha256, + String oraclePath, + String oracleSha256, + int schemaVersion, + String slice) + { + @JsonCreator + public Corpus( + @JsonProperty("manifestPath") String manifestPath, + @JsonProperty("manifestSha256") String manifestSha256, + @JsonProperty("oraclePath") String oraclePath, + @JsonProperty("oracleSha256") String oracleSha256, + @JsonProperty("schemaVersion") int schemaVersion, + @JsonProperty("slice") String slice) + { + this.manifestPath = requireNonNull(manifestPath, "manifestPath is null"); + this.manifestSha256 = requireNonNull(manifestSha256, "manifestSha256 is null"); + this.oraclePath = requireNonNull(oraclePath, "oraclePath is null"); + this.oracleSha256 = requireNonNull(oracleSha256, "oracleSha256 is null"); + if (schemaVersion != 1) { + throw new IllegalArgumentException("unsupported HogQL corpus schema: " + schemaVersion); + } + this.schemaVersion = schemaVersion; + this.slice = requireNonNull(slice, "slice is null"); + } + } } diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/corpus/expr_select_cases.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/corpus/expr_select_cases.json new file mode 100644 index 000000000000..9b2540ca6161 --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/corpus/expr_select_cases.json @@ -0,0 +1,210 @@ +{ + "cases": [ + { + "accepted": true, + "category": "arithmetic-expression", + "entryPoint": "expression", + "id": "expr.accept.arithmetic", + "source": "-value * 2 + 1" + }, + { + "accepted": true, + "category": "array-expression", + "entryPoint": "expression", + "id": "expr.accept.array", + "source": "[1, 2, 3]" + }, + { + "accepted": true, + "category": "between-predicate", + "entryPoint": "expression", + "id": "expr.accept.between", + "source": "value BETWEEN 1 AND 10" + }, + { + "accepted": true, + "category": "boolean-expression", + "entryPoint": "expression", + "id": "expr.accept.boolean", + "source": "NOT enabled OR value >= 10" + }, + { + "accepted": true, + "category": "case-expression", + "entryPoint": "expression", + "id": "expr.accept.case", + "source": "CASE WHEN enabled THEN 1 ELSE 0 END" + }, + { + "accepted": true, + "category": "cast-expression", + "entryPoint": "expression", + "id": "expr.accept.cast", + "source": "CAST(value AS INT)" + }, + { + "accepted": true, + "category": "field-reference", + "entryPoint": "expression", + "id": "expr.accept.dotted-field", + "source": "properties.plan" + }, + { + "accepted": true, + "category": "function-call", + "entryPoint": "expression", + "id": "expr.accept.function", + "source": "coalesce(value, 0)" + }, + { + "accepted": true, + "category": "in-predicate", + "entryPoint": "expression", + "id": "expr.accept.in-list", + "source": "value IN (1, 2, 3)" + }, + { + "accepted": true, + "category": "literal-expression", + "entryPoint": "expression", + "id": "expr.accept.literal-boolean", + "source": "true" + }, + { + "accepted": true, + "category": "literal-expression", + "entryPoint": "expression", + "id": "expr.accept.literal-null", + "source": "null" + }, + { + "accepted": true, + "category": "literal-expression", + "entryPoint": "expression", + "id": "expr.accept.literal-number", + "source": "42" + }, + { + "accepted": true, + "category": "literal-expression", + "entryPoint": "expression", + "id": "expr.accept.literal-string", + "source": "'example'" + }, + { + "accepted": true, + "category": "null-predicate", + "entryPoint": "expression", + "id": "expr.accept.null-predicate", + "source": "value IS NULL" + }, + { + "accepted": true, + "category": "tuple-expression", + "entryPoint": "expression", + "id": "expr.accept.tuple", + "source": "(1, 'example')" + }, + { + "accepted": false, + "category": "invalid-expression", + "entryPoint": "expression", + "errorCategory": "incomplete-input", + "id": "expr.reject.incomplete-binary", + "source": "1 +" + }, + { + "accepted": false, + "category": "invalid-expression", + "entryPoint": "expression", + "errorCategory": "unclosed-delimiter", + "id": "expr.reject.unclosed-tuple", + "source": "(1, 2" + }, + { + "accepted": false, + "category": "invalid-expression", + "entryPoint": "expression", + "errorCategory": "unexpected-character", + "id": "expr.reject.unexpected-character", + "source": "value @ 1" + }, + { + "accepted": true, + "category": "select-aggregation", + "entryPoint": "query", + "id": "query.accept.aggregate", + "source": "SELECT event, count() AS total FROM events GROUP BY event HAVING total > 1" + }, + { + "accepted": true, + "category": "select-query", + "entryPoint": "query", + "id": "query.accept.constant", + "source": "SELECT 1" + }, + { + "accepted": true, + "category": "select-query", + "entryPoint": "query", + "id": "query.accept.multiline", + "source": "SELECT event,\n value\nFROM events" + }, + { + "accepted": true, + "category": "select-ordering", + "entryPoint": "query", + "id": "query.accept.order-limit-offset", + "source": "SELECT event FROM events ORDER BY event DESC LIMIT 10 OFFSET 2" + }, + { + "accepted": true, + "category": "select-query", + "entryPoint": "query", + "id": "query.accept.projection-from", + "source": "SELECT event, value + 1 AS next_value FROM events" + }, + { + "accepted": true, + "category": "select-filter", + "entryPoint": "query", + "id": "query.accept.where", + "source": "SELECT event FROM events WHERE event = '$pageview'" + }, + { + "accepted": false, + "category": "invalid-query", + "entryPoint": "query", + "errorCategory": "missing-projection", + "id": "query.reject.missing-projection", + "source": "SELECT" + }, + { + "accepted": false, + "category": "invalid-query", + "entryPoint": "query", + "errorCategory": "missing-source", + "id": "query.reject.missing-source", + "source": "SELECT 1 FROM" + }, + { + "accepted": false, + "category": "invalid-query", + "entryPoint": "query", + "errorCategory": "missing-predicate", + "id": "query.reject.missing-where-expression", + "source": "SELECT 1 WHERE" + }, + { + "accepted": false, + "category": "invalid-query", + "entryPoint": "query", + "errorCategory": "incomplete-input", + "id": "query.reject.trailing-comparison", + "source": "SELECT 1 FROM events WHERE value =" + } + ], + "provenance": "synthetic-public-safe", + "schemaVersion": 1, + "slice": "expression-and-plain-select" +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/corpus/expr_select_cpp_oracle.jsonl b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/corpus/expr_select_cpp_oracle.jsonl new file mode 100644 index 000000000000..1be8c076a4b6 --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/corpus/expr_select_cpp_oracle.jsonl @@ -0,0 +1,28 @@ +{"accepted":true,"ast":{"end":{"column":14,"line":1,"offset":14},"left":{"end":{"column":10,"line":1,"offset":10},"left":{"end":{"column":6,"line":1,"offset":6},"left":{"node":"Constant","value":0},"node":"ArithmeticOperation","op":"-","right":{"chain":["value"],"end":{"column":6,"line":1,"offset":6},"node":"Field","start":{"column":1,"line":1,"offset":1}},"start":{"column":0,"line":1,"offset":0}},"node":"ArithmeticOperation","op":"*","right":{"end":{"column":10,"line":1,"offset":10},"node":"Constant","start":{"column":9,"line":1,"offset":9},"value":2},"start":{"column":0,"line":1,"offset":0}},"node":"ArithmeticOperation","op":"+","right":{"end":{"column":14,"line":1,"offset":14},"node":"Constant","start":{"column":13,"line":1,"offset":13},"value":1},"start":{"column":0,"line":1,"offset":0}},"category":"arithmetic-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.arithmetic","languageVersion":"1.0.0","schemaVersion":1,"source":"-value * 2 + 1","span":{"end":{"column":14,"line":1,"offset":14},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":9,"line":1,"offset":9},"exprs":[{"end":{"column":2,"line":1,"offset":2},"node":"Constant","start":{"column":1,"line":1,"offset":1},"value":1},{"end":{"column":5,"line":1,"offset":5},"node":"Constant","start":{"column":4,"line":1,"offset":4},"value":2},{"end":{"column":8,"line":1,"offset":8},"node":"Constant","start":{"column":7,"line":1,"offset":7},"value":3}],"node":"Array","start":{"column":0,"line":1,"offset":0}},"category":"array-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.array","languageVersion":"1.0.0","schemaVersion":1,"source":"[1, 2, 3]","span":{"end":{"column":9,"line":1,"offset":9},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":22,"line":1,"offset":22},"expr":{"chain":["value"],"end":{"column":5,"line":1,"offset":5},"node":"Field","start":{"column":0,"line":1,"offset":0}},"high":{"end":{"column":22,"line":1,"offset":22},"node":"Constant","start":{"column":20,"line":1,"offset":20},"value":10},"low":{"end":{"column":15,"line":1,"offset":15},"node":"Constant","start":{"column":14,"line":1,"offset":14},"value":1},"negated":false,"node":"BetweenExpr","start":{"column":0,"line":1,"offset":0}},"category":"between-predicate","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.between","languageVersion":"1.0.0","schemaVersion":1,"source":"value BETWEEN 1 AND 10","span":{"end":{"column":22,"line":1,"offset":22},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":26,"line":1,"offset":26},"exprs":[{"end":{"column":11,"line":1,"offset":11},"expr":{"chain":["enabled"],"end":{"column":11,"line":1,"offset":11},"node":"Field","start":{"column":4,"line":1,"offset":4}},"node":"Not","start":{"column":0,"line":1,"offset":0}},{"end":{"column":26,"line":1,"offset":26},"left":{"chain":["value"],"end":{"column":20,"line":1,"offset":20},"node":"Field","start":{"column":15,"line":1,"offset":15}},"node":"CompareOperation","op":">=","right":{"end":{"column":26,"line":1,"offset":26},"node":"Constant","start":{"column":24,"line":1,"offset":24},"value":10},"start":{"column":15,"line":1,"offset":15}}],"node":"Or","start":{"column":0,"line":1,"offset":0}},"category":"boolean-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.boolean","languageVersion":"1.0.0","schemaVersion":1,"source":"NOT enabled OR value >= 10","span":{"end":{"column":26,"line":1,"offset":26},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"args":[{"chain":["enabled"],"end":{"column":17,"line":1,"offset":17},"node":"Field","start":{"column":10,"line":1,"offset":10}},{"end":{"column":24,"line":1,"offset":24},"node":"Constant","start":{"column":23,"line":1,"offset":23},"value":1},{"end":{"column":31,"line":1,"offset":31},"node":"Constant","start":{"column":30,"line":1,"offset":30},"value":0}],"end":{"column":35,"line":1,"offset":35},"name":"if","node":"Call","start":{"column":0,"line":1,"offset":0}},"category":"case-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.case","languageVersion":"1.0.0","schemaVersion":1,"source":"CASE WHEN enabled THEN 1 ELSE 0 END","span":{"end":{"column":35,"line":1,"offset":35},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":18,"line":1,"offset":18},"expr":{"chain":["value"],"end":{"column":10,"line":1,"offset":10},"node":"Field","start":{"column":5,"line":1,"offset":5}},"node":"TypeCast","start":{"column":0,"line":1,"offset":0},"type_name":"int"},"category":"cast-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.cast","languageVersion":"1.0.0","schemaVersion":1,"source":"CAST(value AS INT)","span":{"end":{"column":18,"line":1,"offset":18},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"chain":["properties","plan"],"end":{"column":15,"line":1,"offset":15},"node":"Field","start":{"column":0,"line":1,"offset":0}},"category":"field-reference","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.dotted-field","languageVersion":"1.0.0","schemaVersion":1,"source":"properties.plan","span":{"end":{"column":15,"line":1,"offset":15},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"args":[{"chain":["value"],"end":{"column":14,"line":1,"offset":14},"node":"Field","start":{"column":9,"line":1,"offset":9}},{"end":{"column":17,"line":1,"offset":17},"node":"Constant","start":{"column":16,"line":1,"offset":16},"value":0}],"distinct":false,"end":{"column":18,"line":1,"offset":18},"filter_expr":null,"name":"coalesce","node":"Call","order_by":null,"params":null,"start":{"column":0,"line":1,"offset":0}},"category":"function-call","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.function","languageVersion":"1.0.0","schemaVersion":1,"source":"coalesce(value, 0)","span":{"end":{"column":18,"line":1,"offset":18},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":18,"line":1,"offset":18},"left":{"chain":["value"],"end":{"column":5,"line":1,"offset":5},"node":"Field","start":{"column":0,"line":1,"offset":0}},"node":"CompareOperation","op":"in","right":{"end":{"column":18,"line":1,"offset":18},"exprs":[{"end":{"column":11,"line":1,"offset":11},"node":"Constant","start":{"column":10,"line":1,"offset":10},"value":1},{"end":{"column":14,"line":1,"offset":14},"node":"Constant","start":{"column":13,"line":1,"offset":13},"value":2},{"end":{"column":17,"line":1,"offset":17},"node":"Constant","start":{"column":16,"line":1,"offset":16},"value":3}],"node":"Tuple","start":{"column":9,"line":1,"offset":9}},"start":{"column":0,"line":1,"offset":0}},"category":"in-predicate","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.in-list","languageVersion":"1.0.0","schemaVersion":1,"source":"value IN (1, 2, 3)","span":{"end":{"column":18,"line":1,"offset":18},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":4,"line":1,"offset":4},"node":"Constant","start":{"column":0,"line":1,"offset":0},"value":true},"category":"literal-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.literal-boolean","languageVersion":"1.0.0","schemaVersion":1,"source":"true","span":{"end":{"column":4,"line":1,"offset":4},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":4,"line":1,"offset":4},"node":"Constant","start":{"column":0,"line":1,"offset":0},"value":null},"category":"literal-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.literal-null","languageVersion":"1.0.0","schemaVersion":1,"source":"null","span":{"end":{"column":4,"line":1,"offset":4},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":2,"line":1,"offset":2},"node":"Constant","start":{"column":0,"line":1,"offset":0},"value":42},"category":"literal-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.literal-number","languageVersion":"1.0.0","schemaVersion":1,"source":"42","span":{"end":{"column":2,"line":1,"offset":2},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":9,"line":1,"offset":9},"node":"Constant","start":{"column":0,"line":1,"offset":0},"value":"example"},"category":"literal-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.literal-string","languageVersion":"1.0.0","schemaVersion":1,"source":"'example'","span":{"end":{"column":9,"line":1,"offset":9},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":13,"line":1,"offset":13},"is_null_comparison_style":true,"left":{"chain":["value"],"end":{"column":5,"line":1,"offset":5},"node":"Field","start":{"column":0,"line":1,"offset":0}},"node":"CompareOperation","op":"==","right":{"node":"Constant","value":null},"start":{"column":0,"line":1,"offset":0}},"category":"null-predicate","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.null-predicate","languageVersion":"1.0.0","schemaVersion":1,"source":"value IS NULL","span":{"end":{"column":13,"line":1,"offset":13},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":14,"line":1,"offset":14},"exprs":[{"end":{"column":2,"line":1,"offset":2},"node":"Constant","start":{"column":1,"line":1,"offset":1},"value":1},{"end":{"column":13,"line":1,"offset":13},"node":"Constant","start":{"column":4,"line":1,"offset":4},"value":"example"}],"node":"Tuple","start":{"column":0,"line":1,"offset":0}},"category":"tuple-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.tuple","languageVersion":"1.0.0","schemaVersion":1,"source":"(1, 'example')","span":{"end":{"column":14,"line":1,"offset":14},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":false,"category":"invalid-expression","entryPoint":"expression","errorCategory":"incomplete-input","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.reject.incomplete-binary","languageVersion":"1.0.0","schemaVersion":1,"source":"1 +","span":{"end":{"column":3,"line":1,"offset":3},"start":{"column":3,"line":1,"offset":3}}} +{"accepted":false,"category":"invalid-expression","entryPoint":"expression","errorCategory":"unclosed-delimiter","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.reject.unclosed-tuple","languageVersion":"1.0.0","schemaVersion":1,"source":"(1, 2","span":{"end":{"column":5,"line":1,"offset":5},"start":{"column":5,"line":1,"offset":5}}} +{"accepted":false,"category":"invalid-expression","entryPoint":"expression","errorCategory":"unexpected-character","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.reject.unexpected-character","languageVersion":"1.0.0","schemaVersion":1,"source":"value @ 1","span":{"end":{"column":9,"line":1,"offset":9},"start":{"column":6,"line":1,"offset":6}}} +{"accepted":true,"ast":{"ctes":null,"distinct":null,"end":{"column":74,"line":1,"offset":74},"group_by":[{"chain":["event"],"end":{"column":57,"line":1,"offset":57},"node":"Field","start":{"column":52,"line":1,"offset":52}}],"having":{"end":{"column":74,"line":1,"offset":74},"left":{"chain":["total"],"end":{"column":70,"line":1,"offset":70},"node":"Field","start":{"column":65,"line":1,"offset":65}},"node":"CompareOperation","op":">","right":{"end":{"column":74,"line":1,"offset":74},"node":"Constant","start":{"column":73,"line":1,"offset":73},"value":1},"start":{"column":65,"line":1,"offset":65}},"node":"SelectQuery","prewhere":null,"qualify":null,"select":[{"chain":["event"],"end":{"column":12,"line":1,"offset":12},"node":"Field","start":{"column":7,"line":1,"offset":7}},{"alias":"total","end":{"column":30,"line":1,"offset":30},"expr":{"args":[],"distinct":false,"end":{"column":21,"line":1,"offset":21},"filter_expr":null,"name":"count","node":"Call","order_by":null,"params":null,"start":{"column":14,"line":1,"offset":14}},"node":"Alias","start":{"column":14,"line":1,"offset":14}}],"select_from":{"alias":null,"end":{"column":42,"line":1,"offset":42},"next_join":null,"node":"JoinExpr","sample":null,"start":{"column":36,"line":1,"offset":36},"table":{"chain":["events"],"end":{"column":42,"line":1,"offset":42},"node":"Field","start":{"column":36,"line":1,"offset":36}},"table_final":null},"start":{"column":0,"line":1,"offset":0},"where":null},"category":"select-aggregation","entryPoint":"query","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.accept.aggregate","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT event, count() AS total FROM events GROUP BY event HAVING total > 1","span":{"end":{"column":74,"line":1,"offset":74},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"ctes":null,"distinct":null,"end":{"column":8,"line":1,"offset":8},"group_by":null,"having":null,"node":"SelectQuery","prewhere":null,"qualify":null,"select":[{"end":{"column":8,"line":1,"offset":8},"node":"Constant","start":{"column":7,"line":1,"offset":7},"value":1}],"select_from":null,"start":{"column":0,"line":1,"offset":0},"where":null},"category":"select-query","entryPoint":"query","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.accept.constant","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT 1","span":{"end":{"column":8,"line":1,"offset":8},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"ctes":null,"distinct":null,"end":{"column":11,"line":3,"offset":35},"group_by":null,"having":null,"node":"SelectQuery","prewhere":null,"qualify":null,"select":[{"chain":["event"],"end":{"column":12,"line":1,"offset":12},"node":"Field","start":{"column":7,"line":1,"offset":7}},{"chain":["value"],"end":{"column":9,"line":2,"offset":23},"node":"Field","start":{"column":4,"line":2,"offset":18}}],"select_from":{"alias":null,"end":{"column":11,"line":3,"offset":35},"next_join":null,"node":"JoinExpr","sample":null,"start":{"column":5,"line":3,"offset":29},"table":{"chain":["events"],"end":{"column":11,"line":3,"offset":35},"node":"Field","start":{"column":5,"line":3,"offset":29}},"table_final":null},"start":{"column":0,"line":1,"offset":0},"where":null},"category":"select-query","entryPoint":"query","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.accept.multiline","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT event,\n value\nFROM events","span":{"end":{"column":11,"line":3,"offset":35},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"ctes":null,"distinct":null,"end":{"column":53,"line":1,"offset":53},"group_by":null,"having":null,"limit":{"end":{"column":53,"line":1,"offset":53},"node":"Constant","start":{"column":51,"line":1,"offset":51},"value":10},"node":"SelectQuery","offset":{"end":{"column":62,"line":1,"offset":62},"node":"Constant","start":{"column":61,"line":1,"offset":61},"value":2},"order_by":[{"end":{"column":44,"line":1,"offset":44},"expr":{"chain":["event"],"end":{"column":39,"line":1,"offset":39},"node":"Field","start":{"column":34,"line":1,"offset":34}},"node":"OrderExpr","order":"DESC","start":{"column":34,"line":1,"offset":34}}],"prewhere":null,"qualify":null,"select":[{"chain":["event"],"end":{"column":12,"line":1,"offset":12},"node":"Field","start":{"column":7,"line":1,"offset":7}}],"select_from":{"alias":null,"end":{"column":24,"line":1,"offset":24},"next_join":null,"node":"JoinExpr","sample":null,"start":{"column":18,"line":1,"offset":18},"table":{"chain":["events"],"end":{"column":24,"line":1,"offset":24},"node":"Field","start":{"column":18,"line":1,"offset":18}},"table_final":null},"start":{"column":0,"line":1,"offset":0},"where":null},"category":"select-ordering","entryPoint":"query","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.accept.order-limit-offset","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT event FROM events ORDER BY event DESC LIMIT 10 OFFSET 2","span":{"end":{"column":53,"line":1,"offset":53},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"ctes":null,"distinct":null,"end":{"column":49,"line":1,"offset":49},"group_by":null,"having":null,"node":"SelectQuery","prewhere":null,"qualify":null,"select":[{"chain":["event"],"end":{"column":12,"line":1,"offset":12},"node":"Field","start":{"column":7,"line":1,"offset":7}},{"alias":"next_value","end":{"column":37,"line":1,"offset":37},"expr":{"end":{"column":23,"line":1,"offset":23},"left":{"chain":["value"],"end":{"column":19,"line":1,"offset":19},"node":"Field","start":{"column":14,"line":1,"offset":14}},"node":"ArithmeticOperation","op":"+","right":{"end":{"column":23,"line":1,"offset":23},"node":"Constant","start":{"column":22,"line":1,"offset":22},"value":1},"start":{"column":14,"line":1,"offset":14}},"node":"Alias","start":{"column":14,"line":1,"offset":14}}],"select_from":{"alias":null,"end":{"column":49,"line":1,"offset":49},"next_join":null,"node":"JoinExpr","sample":null,"start":{"column":43,"line":1,"offset":43},"table":{"chain":["events"],"end":{"column":49,"line":1,"offset":49},"node":"Field","start":{"column":43,"line":1,"offset":43}},"table_final":null},"start":{"column":0,"line":1,"offset":0},"where":null},"category":"select-query","entryPoint":"query","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.accept.projection-from","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT event, value + 1 AS next_value FROM events","span":{"end":{"column":49,"line":1,"offset":49},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"ctes":null,"distinct":null,"end":{"column":50,"line":1,"offset":50},"group_by":null,"having":null,"node":"SelectQuery","prewhere":null,"qualify":null,"select":[{"chain":["event"],"end":{"column":12,"line":1,"offset":12},"node":"Field","start":{"column":7,"line":1,"offset":7}}],"select_from":{"alias":null,"end":{"column":24,"line":1,"offset":24},"next_join":null,"node":"JoinExpr","sample":null,"start":{"column":18,"line":1,"offset":18},"table":{"chain":["events"],"end":{"column":24,"line":1,"offset":24},"node":"Field","start":{"column":18,"line":1,"offset":18}},"table_final":null},"start":{"column":0,"line":1,"offset":0},"where":{"end":{"column":50,"line":1,"offset":50},"left":{"chain":["event"],"end":{"column":36,"line":1,"offset":36},"node":"Field","start":{"column":31,"line":1,"offset":31}},"node":"CompareOperation","op":"==","right":{"end":{"column":50,"line":1,"offset":50},"node":"Constant","start":{"column":39,"line":1,"offset":39},"value":"$pageview"},"start":{"column":31,"line":1,"offset":31}}},"category":"select-filter","entryPoint":"query","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.accept.where","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT event FROM events WHERE event = '$pageview'","span":{"end":{"column":50,"line":1,"offset":50},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":false,"category":"invalid-query","entryPoint":"query","errorCategory":"missing-projection","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.reject.missing-projection","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT","span":{"end":{"column":6,"line":1,"offset":6},"start":{"column":6,"line":1,"offset":6}}} +{"accepted":false,"category":"invalid-query","entryPoint":"query","errorCategory":"missing-source","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.reject.missing-source","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT 1 FROM","span":{"end":{"column":13,"line":1,"offset":13},"start":{"column":13,"line":1,"offset":13}}} +{"accepted":false,"category":"invalid-query","entryPoint":"query","errorCategory":"missing-predicate","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.reject.missing-where-expression","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT 1 WHERE","span":{"end":{"column":14,"line":1,"offset":14},"start":{"column":14,"line":1,"offset":14}}} +{"accepted":false,"category":"invalid-query","entryPoint":"query","errorCategory":"incomplete-input","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.reject.trailing-comparison","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT 1 FROM events WHERE value =","span":{"end":{"column":34,"line":1,"offset":34},"start":{"column":34,"line":1,"offset":34}}} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-features.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-features.json new file mode 100644 index 000000000000..39bd8b5ed995 --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-features.json @@ -0,0 +1,2771 @@ +{ + "features": [ + { + "id": "alternative:arrayJoinClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "arrayJoinClause" + }, + { + "id": "alternative:block", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "block" + }, + { + "id": "alternative:catchBlock", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "catchBlock" + }, + { + "id": "alternative:columnAliases", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "columnAliases" + }, + { + "id": "alternative:columnExpr:ColumnExprAnd", + "kind": "parserAlternative", + "label": "ColumnExprAnd", + "queryReachable": true, + "rule": "columnExpr" + }, + { + "id": "alternative:columnExprList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "columnExprList" + }, + { + "id": "alternative:columnExprValue:ColumnExprCase", + "kind": "parserAlternative", + "label": "ColumnExprCase", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnLambdaExpr:ArrowLambda", + "kind": "parserAlternative", + "label": "ArrowLambda", + "queryReachable": true, + "rule": "columnLambdaExpr" + }, + { + "id": "alternative:columnLambdaExpr:ColonLambda", + "kind": "parserAlternative", + "label": "ColonLambda", + "queryReachable": true, + "rule": "columnLambdaExpr" + }, + { + "id": "alternative:columnTypeCastExpr:ColumnTypeCastExprSimple", + "kind": "parserAlternative", + "label": "ColumnTypeCastExprSimple", + "queryReachable": true, + "rule": "columnTypeCastExpr" + }, + { + "id": "alternative:columnTypeCastExpr:ColumnTypeCastExprWithTimeZone", + "kind": "parserAlternative", + "label": "ColumnTypeCastExprWithTimeZone", + "queryReachable": true, + "rule": "columnTypeCastExpr" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprArray", + "kind": "parserAlternative", + "label": "ColumnTypeExprArray", + "queryReachable": true, + "rule": "columnTypeExpr" + }, + { + "id": "alternative:columnsReplaceItem", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "columnsReplaceItem" + }, + { + "id": "alternative:columnsReplaceList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "columnsReplaceList" + }, + { + "id": "alternative:databaseIdentifier", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "databaseIdentifier" + }, + { + "id": "alternative:emptyStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "emptyStmt" + }, + { + "id": "alternative:enumValue", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "enumValue" + }, + { + "id": "alternative:expr", + "kind": "parserAlternative", + "queryReachable": false, + "rule": "expr" + }, + { + "id": "alternative:exprStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "exprStmt" + }, + { + "id": "alternative:expression", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "expression" + }, + { + "id": "alternative:forInStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "forInStmt" + }, + { + "id": "alternative:forStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "forStmt" + }, + { + "id": "alternative:fromClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "fromClause" + }, + { + "id": "alternative:fullTemplateString", + "kind": "parserAlternative", + "queryReachable": false, + "rule": "fullTemplateString" + }, + { + "id": "alternative:funcStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "funcStmt" + }, + { + "id": "alternative:groupByClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "groupByClause" + }, + { + "id": "alternative:groupingSet", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "groupingSet" + }, + { + "id": "alternative:groupingSetList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "groupingSetList" + }, + { + "id": "alternative:havingClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "havingClause" + }, + { + "id": "alternative:hogqlxTagElement:HogqlxTagElementClosed", + "kind": "parserAlternative", + "label": "HogqlxTagElementClosed", + "queryReachable": true, + "rule": "hogqlxTagElement" + }, + { + "id": "alternative:hogqlxTagElement:HogqlxTagElementNested", + "kind": "parserAlternative", + "label": "HogqlxTagElementNested", + "queryReachable": true, + "rule": "hogqlxTagElement" + }, + { + "id": "alternative:hogqlxText", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "hogqlxText" + }, + { + "id": "alternative:identifierList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "identifierList" + }, + { + "id": "alternative:ifStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "ifStmt" + }, + { + "id": "alternative:interpolateClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "interpolateClause" + }, + { + "id": "alternative:interpolateExpr", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "interpolateExpr" + }, + { + "id": "alternative:interval", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "interval" + }, + { + "id": "alternative:joinExpr:JoinExprOp", + "kind": "parserAlternative", + "label": "JoinExprOp", + "queryReachable": true, + "rule": "joinExpr" + }, + { + "id": "alternative:joinOp:JoinOpFull", + "kind": "parserAlternative", + "label": "JoinOpFull", + "queryReachable": true, + "rule": "joinOp" + }, + { + "id": "alternative:joinOp:JoinOpInner", + "kind": "parserAlternative", + "label": "JoinOpInner", + "queryReachable": true, + "rule": "joinOp" + }, + { + "id": "alternative:joinOp:JoinOpLeftRight", + "kind": "parserAlternative", + "label": "JoinOpLeftRight", + "queryReachable": true, + "rule": "joinOp" + }, + { + "id": "alternative:keyword", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "keyword" + }, + { + "id": "alternative:keywordForAlias", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "keywordForAlias" + }, + { + "id": "alternative:keywordForImplicitAlias", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "keywordForImplicitAlias" + }, + { + "id": "alternative:keywordForTypeCast", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "keywordForTypeCast" + }, + { + "id": "alternative:kvPair", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "kvPair" + }, + { + "id": "alternative:kvPairList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "kvPairList" + }, + { + "id": "alternative:limitByClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "limitByClause" + }, + { + "id": "alternative:limitExpr", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "limitExpr" + }, + { + "id": "alternative:nestedIdentifier", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "nestedIdentifier" + }, + { + "id": "alternative:numberLiteral", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "numberLiteral" + }, + { + "id": "alternative:offsetOnlyClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "offsetOnlyClause" + }, + { + "id": "alternative:orderByClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "orderByClause" + }, + { + "id": "alternative:orderExpr", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "orderExpr" + }, + { + "id": "alternative:orderExprList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "orderExprList" + }, + { + "id": "alternative:pivotColumn", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "pivotColumn" + }, + { + "id": "alternative:pivotColumnList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "pivotColumnList" + }, + { + "id": "alternative:placeholder", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "placeholder" + }, + { + "id": "alternative:prewhereClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "prewhereClause" + }, + { + "id": "alternative:program", + "kind": "parserAlternative", + "queryReachable": false, + "rule": "program" + }, + { + "id": "alternative:projectionOrderByClause", + "kind": "parserAlternative", + "queryReachable": false, + "rule": "projectionOrderByClause" + }, + { + "id": "alternative:qualifyClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "qualifyClause" + }, + { + "id": "alternative:returnStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "returnStmt" + }, + { + "id": "alternative:sampleClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "sampleClause" + }, + { + "id": "alternative:select", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "select" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprAliasBefore", + "kind": "parserAlternative", + "label": "ColumnExprAliasBefore", + "queryReachable": true, + "rule": "selectColumnExpr" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprAliasImplicit", + "kind": "parserAlternative", + "label": "ColumnExprAliasImplicit", + "queryReachable": true, + "rule": "selectColumnExpr" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprInvalidFromImplicitAlias", + "kind": "parserAlternative", + "label": "ColumnExprInvalidFromImplicitAlias", + "queryReachable": true, + "rule": "selectColumnExpr" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprSelectValue", + "kind": "parserAlternative", + "label": "ColumnExprSelectValue", + "queryReachable": true, + "rule": "selectColumnExpr" + }, + { + "id": "alternative:selectColumnExprList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "selectColumnExprList" + }, + { + "id": "alternative:selectColumnExprListBeforeFrom:SelectColumnExprListBeforeFromPlain", + "kind": "parserAlternative", + "label": "SelectColumnExprListBeforeFromPlain", + "queryReachable": true, + "rule": "selectColumnExprListBeforeFrom" + }, + { + "id": "alternative:selectColumnExprListBeforeFrom:SelectColumnExprListBeforeFromTrailingComma", + "kind": "parserAlternative", + "label": "SelectColumnExprListBeforeFromTrailingComma", + "queryReachable": true, + "rule": "selectColumnExprListBeforeFrom" + }, + { + "id": "alternative:selectSetStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "selectSetStmt" + }, + { + "id": "alternative:selectStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "selectStmt" + }, + { + "id": "alternative:settingExpr", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "settingExpr" + }, + { + "id": "alternative:settingExprList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "settingExprList" + }, + { + "id": "alternative:settingsClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "settingsClause" + }, + { + "id": "alternative:subsequentSelectSetClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "subsequentSelectSetClause" + }, + { + "id": "alternative:tableArgList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "tableArgList" + }, + { + "id": "alternative:tableExpr:TableExprIdentifier", + "kind": "parserAlternative", + "label": "TableExprIdentifier", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableFunctionExpr", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "tableFunctionExpr" + }, + { + "id": "alternative:tableIdentifier", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "tableIdentifier" + }, + { + "id": "alternative:templateString", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "templateString" + }, + { + "id": "alternative:throwStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "throwStmt" + }, + { + "id": "alternative:topClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "topClause" + }, + { + "id": "alternative:tryCatchStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "tryCatchStmt" + }, + { + "id": "alternative:unpivotColumn", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "unpivotColumn" + }, + { + "id": "alternative:unpivotColumnList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "unpivotColumnList" + }, + { + "id": "alternative:valuesClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "valuesClause" + }, + { + "id": "alternative:valuesRow", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "valuesRow" + }, + { + "id": "alternative:varAssignment", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "varAssignment" + }, + { + "id": "alternative:varDecl", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "varDecl" + }, + { + "id": "alternative:whereClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "whereClause" + }, + { + "id": "alternative:whileStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "whileStmt" + }, + { + "id": "alternative:winFrameBound", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "winFrameBound" + }, + { + "id": "alternative:winFrameClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "winFrameClause" + }, + { + "id": "alternative:winFrameExtend:frameBetween", + "kind": "parserAlternative", + "label": "frameBetween", + "queryReachable": true, + "rule": "winFrameExtend" + }, + { + "id": "alternative:winFrameExtend:frameStart", + "kind": "parserAlternative", + "label": "frameStart", + "queryReachable": true, + "rule": "winFrameExtend" + }, + { + "id": "alternative:winOrderByClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "winOrderByClause" + }, + { + "id": "alternative:winPartitionByClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "winPartitionByClause" + }, + { + "id": "alternative:windowClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "windowClause" + }, + { + "id": "alternative:windowExpr", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "windowExpr" + }, + { + "id": "alternative:withClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "withClause" + }, + { + "id": "alternative:withExpr:WithExprColumn", + "kind": "parserAlternative", + "label": "WithExprColumn", + "queryReachable": true, + "rule": "withExpr" + }, + { + "id": "alternative:withExpr:WithExprSubquery", + "kind": "parserAlternative", + "label": "WithExprSubquery", + "queryReachable": true, + "rule": "withExpr" + }, + { + "id": "alternative:withExprColumnNameList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "withExprColumnNameList" + }, + { + "id": "alternative:withExprList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "withExprList" + }, + { + "id": "alternative:withFillClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "withFillClause" + }, + { + "id": "alternative:withinGroupClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "withinGroupClause" + }, + { + "id": "rule:alias", + "kind": "parserRule", + "name": "alias", + "queryReachable": true + }, + { + "id": "rule:arrayJoinClause", + "kind": "parserRule", + "name": "arrayJoinClause", + "queryReachable": true + }, + { + "id": "rule:block", + "kind": "parserRule", + "name": "block", + "queryReachable": true + }, + { + "id": "rule:catchBlock", + "kind": "parserRule", + "name": "catchBlock", + "queryReachable": true + }, + { + "id": "rule:columnAliases", + "kind": "parserRule", + "name": "columnAliases", + "queryReachable": true + }, + { + "id": "rule:columnExpr", + "kind": "parserRule", + "name": "columnExpr", + "queryReachable": true + }, + { + "id": "rule:columnExprList", + "kind": "parserRule", + "name": "columnExprList", + "queryReachable": true + }, + { + "id": "rule:columnExprTupleOrSingle", + "kind": "parserRule", + "name": "columnExprTupleOrSingle", + "queryReachable": true + }, + { + "id": "rule:columnExprValue", + "kind": "parserRule", + "name": "columnExprValue", + "queryReachable": true + }, + { + "id": "rule:columnIdentifier", + "kind": "parserRule", + "name": "columnIdentifier", + "queryReachable": true + }, + { + "id": "rule:columnLambdaExpr", + "kind": "parserRule", + "name": "columnLambdaExpr", + "queryReachable": true + }, + { + "id": "rule:columnTypeCastExpr", + "kind": "parserRule", + "name": "columnTypeCastExpr", + "queryReachable": true + }, + { + "id": "rule:columnTypeCastIdentifier", + "kind": "parserRule", + "name": "columnTypeCastIdentifier", + "queryReachable": true + }, + { + "id": "rule:columnTypeExpr", + "kind": "parserRule", + "name": "columnTypeExpr", + "queryReachable": true + }, + { + "id": "rule:columnsReplaceItem", + "kind": "parserRule", + "name": "columnsReplaceItem", + "queryReachable": true + }, + { + "id": "rule:columnsReplaceList", + "kind": "parserRule", + "name": "columnsReplaceList", + "queryReachable": true + }, + { + "id": "rule:databaseIdentifier", + "kind": "parserRule", + "name": "databaseIdentifier", + "queryReachable": true + }, + { + "id": "rule:declaration", + "kind": "parserRule", + "name": "declaration", + "queryReachable": true + }, + { + "id": "rule:emptyStmt", + "kind": "parserRule", + "name": "emptyStmt", + "queryReachable": true + }, + { + "id": "rule:enumValue", + "kind": "parserRule", + "name": "enumValue", + "queryReachable": true + }, + { + "id": "rule:expr", + "kind": "parserRule", + "name": "expr", + "queryReachable": false + }, + { + "id": "rule:exprStmt", + "kind": "parserRule", + "name": "exprStmt", + "queryReachable": true + }, + { + "id": "rule:expression", + "kind": "parserRule", + "name": "expression", + "queryReachable": true + }, + { + "id": "rule:floatingLiteral", + "kind": "parserRule", + "name": "floatingLiteral", + "queryReachable": true + }, + { + "id": "rule:forInStmt", + "kind": "parserRule", + "name": "forInStmt", + "queryReachable": true + }, + { + "id": "rule:forStmt", + "kind": "parserRule", + "name": "forStmt", + "queryReachable": true + }, + { + "id": "rule:fromClause", + "kind": "parserRule", + "name": "fromClause", + "queryReachable": true + }, + { + "id": "rule:fullTemplateString", + "kind": "parserRule", + "name": "fullTemplateString", + "queryReachable": false + }, + { + "id": "rule:funcStmt", + "kind": "parserRule", + "name": "funcStmt", + "queryReachable": true + }, + { + "id": "rule:groupByClause", + "kind": "parserRule", + "name": "groupByClause", + "queryReachable": true + }, + { + "id": "rule:groupingSet", + "kind": "parserRule", + "name": "groupingSet", + "queryReachable": true + }, + { + "id": "rule:groupingSetList", + "kind": "parserRule", + "name": "groupingSetList", + "queryReachable": true + }, + { + "id": "rule:havingClause", + "kind": "parserRule", + "name": "havingClause", + "queryReachable": true + }, + { + "id": "rule:hogqlxChildElement", + "kind": "parserRule", + "name": "hogqlxChildElement", + "queryReachable": true + }, + { + "id": "rule:hogqlxTagAttribute", + "kind": "parserRule", + "name": "hogqlxTagAttribute", + "queryReachable": true + }, + { + "id": "rule:hogqlxTagElement", + "kind": "parserRule", + "name": "hogqlxTagElement", + "queryReachable": true + }, + { + "id": "rule:hogqlxText", + "kind": "parserRule", + "name": "hogqlxText", + "queryReachable": true + }, + { + "id": "rule:identifier", + "kind": "parserRule", + "name": "identifier", + "queryReachable": true + }, + { + "id": "rule:identifierList", + "kind": "parserRule", + "name": "identifierList", + "queryReachable": true + }, + { + "id": "rule:ifStmt", + "kind": "parserRule", + "name": "ifStmt", + "queryReachable": true + }, + { + "id": "rule:implicitAlias", + "kind": "parserRule", + "name": "implicitAlias", + "queryReachable": true + }, + { + "id": "rule:interpolateClause", + "kind": "parserRule", + "name": "interpolateClause", + "queryReachable": true + }, + { + "id": "rule:interpolateExpr", + "kind": "parserRule", + "name": "interpolateExpr", + "queryReachable": true + }, + { + "id": "rule:interval", + "kind": "parserRule", + "name": "interval", + "queryReachable": true + }, + { + "id": "rule:joinConstraintClause", + "kind": "parserRule", + "name": "joinConstraintClause", + "queryReachable": true + }, + { + "id": "rule:joinExpr", + "kind": "parserRule", + "name": "joinExpr", + "queryReachable": true + }, + { + "id": "rule:joinOp", + "kind": "parserRule", + "name": "joinOp", + "queryReachable": true + }, + { + "id": "rule:joinOpCross", + "kind": "parserRule", + "name": "joinOpCross", + "queryReachable": true + }, + { + "id": "rule:keyword", + "kind": "parserRule", + "name": "keyword", + "queryReachable": true + }, + { + "id": "rule:keywordForAlias", + "kind": "parserRule", + "name": "keywordForAlias", + "queryReachable": true + }, + { + "id": "rule:keywordForImplicitAlias", + "kind": "parserRule", + "name": "keywordForImplicitAlias", + "queryReachable": true + }, + { + "id": "rule:keywordForTypeCast", + "kind": "parserRule", + "name": "keywordForTypeCast", + "queryReachable": true + }, + { + "id": "rule:kvPair", + "kind": "parserRule", + "name": "kvPair", + "queryReachable": true + }, + { + "id": "rule:kvPairList", + "kind": "parserRule", + "name": "kvPairList", + "queryReachable": true + }, + { + "id": "rule:limitAndOffsetClause", + "kind": "parserRule", + "name": "limitAndOffsetClause", + "queryReachable": true + }, + { + "id": "rule:limitAndOffsetClauseOptional", + "kind": "parserRule", + "name": "limitAndOffsetClauseOptional", + "queryReachable": true + }, + { + "id": "rule:limitByClause", + "kind": "parserRule", + "name": "limitByClause", + "queryReachable": true + }, + { + "id": "rule:limitExpr", + "kind": "parserRule", + "name": "limitExpr", + "queryReachable": true + }, + { + "id": "rule:literal", + "kind": "parserRule", + "name": "literal", + "queryReachable": true + }, + { + "id": "rule:nestedIdentifier", + "kind": "parserRule", + "name": "nestedIdentifier", + "queryReachable": true + }, + { + "id": "rule:numberLiteral", + "kind": "parserRule", + "name": "numberLiteral", + "queryReachable": true + }, + { + "id": "rule:offsetOnlyClause", + "kind": "parserRule", + "name": "offsetOnlyClause", + "queryReachable": true + }, + { + "id": "rule:orderByClause", + "kind": "parserRule", + "name": "orderByClause", + "queryReachable": true + }, + { + "id": "rule:orderExpr", + "kind": "parserRule", + "name": "orderExpr", + "queryReachable": true + }, + { + "id": "rule:orderExprList", + "kind": "parserRule", + "name": "orderExprList", + "queryReachable": true + }, + { + "id": "rule:pivotColumn", + "kind": "parserRule", + "name": "pivotColumn", + "queryReachable": true + }, + { + "id": "rule:pivotColumnList", + "kind": "parserRule", + "name": "pivotColumnList", + "queryReachable": true + }, + { + "id": "rule:placeholder", + "kind": "parserRule", + "name": "placeholder", + "queryReachable": true + }, + { + "id": "rule:prewhereClause", + "kind": "parserRule", + "name": "prewhereClause", + "queryReachable": true + }, + { + "id": "rule:program", + "kind": "parserRule", + "name": "program", + "queryReachable": false + }, + { + "id": "rule:projectionOrderByClause", + "kind": "parserRule", + "name": "projectionOrderByClause", + "queryReachable": false + }, + { + "id": "rule:qualifyClause", + "kind": "parserRule", + "name": "qualifyClause", + "queryReachable": true + }, + { + "id": "rule:ratioExpr", + "kind": "parserRule", + "name": "ratioExpr", + "queryReachable": true + }, + { + "id": "rule:returnStmt", + "kind": "parserRule", + "name": "returnStmt", + "queryReachable": true + }, + { + "id": "rule:sampleClause", + "kind": "parserRule", + "name": "sampleClause", + "queryReachable": true + }, + { + "id": "rule:select", + "kind": "parserRule", + "name": "select", + "queryReachable": true + }, + { + "id": "rule:selectColumnExpr", + "kind": "parserRule", + "name": "selectColumnExpr", + "queryReachable": true + }, + { + "id": "rule:selectColumnExprList", + "kind": "parserRule", + "name": "selectColumnExprList", + "queryReachable": true + }, + { + "id": "rule:selectColumnExprListBeforeFrom", + "kind": "parserRule", + "name": "selectColumnExprListBeforeFrom", + "queryReachable": true + }, + { + "id": "rule:selectSetStmt", + "kind": "parserRule", + "name": "selectSetStmt", + "queryReachable": true + }, + { + "id": "rule:selectStmt", + "kind": "parserRule", + "name": "selectStmt", + "queryReachable": true + }, + { + "id": "rule:selectStmtWithParens", + "kind": "parserRule", + "name": "selectStmtWithParens", + "queryReachable": true + }, + { + "id": "rule:settingExpr", + "kind": "parserRule", + "name": "settingExpr", + "queryReachable": true + }, + { + "id": "rule:settingExprList", + "kind": "parserRule", + "name": "settingExprList", + "queryReachable": true + }, + { + "id": "rule:settingsClause", + "kind": "parserRule", + "name": "settingsClause", + "queryReachable": true + }, + { + "id": "rule:statement", + "kind": "parserRule", + "name": "statement", + "queryReachable": true + }, + { + "id": "rule:string", + "kind": "parserRule", + "name": "string", + "queryReachable": true + }, + { + "id": "rule:stringContents", + "kind": "parserRule", + "name": "stringContents", + "queryReachable": true + }, + { + "id": "rule:stringContentsFull", + "kind": "parserRule", + "name": "stringContentsFull", + "queryReachable": false + }, + { + "id": "rule:subsequentSelectSetClause", + "kind": "parserRule", + "name": "subsequentSelectSetClause", + "queryReachable": true + }, + { + "id": "rule:tableArgList", + "kind": "parserRule", + "name": "tableArgList", + "queryReachable": true + }, + { + "id": "rule:tableExpr", + "kind": "parserRule", + "name": "tableExpr", + "queryReachable": true + }, + { + "id": "rule:tableFunctionExpr", + "kind": "parserRule", + "name": "tableFunctionExpr", + "queryReachable": true + }, + { + "id": "rule:tableIdentifier", + "kind": "parserRule", + "name": "tableIdentifier", + "queryReachable": true + }, + { + "id": "rule:templateString", + "kind": "parserRule", + "name": "templateString", + "queryReachable": true + }, + { + "id": "rule:throwStmt", + "kind": "parserRule", + "name": "throwStmt", + "queryReachable": true + }, + { + "id": "rule:topClause", + "kind": "parserRule", + "name": "topClause", + "queryReachable": true + }, + { + "id": "rule:tryCatchStmt", + "kind": "parserRule", + "name": "tryCatchStmt", + "queryReachable": true + }, + { + "id": "rule:unpivotColumn", + "kind": "parserRule", + "name": "unpivotColumn", + "queryReachable": true + }, + { + "id": "rule:unpivotColumnList", + "kind": "parserRule", + "name": "unpivotColumnList", + "queryReachable": true + }, + { + "id": "rule:valuesClause", + "kind": "parserRule", + "name": "valuesClause", + "queryReachable": true + }, + { + "id": "rule:valuesRow", + "kind": "parserRule", + "name": "valuesRow", + "queryReachable": true + }, + { + "id": "rule:varAssignment", + "kind": "parserRule", + "name": "varAssignment", + "queryReachable": true + }, + { + "id": "rule:varDecl", + "kind": "parserRule", + "name": "varDecl", + "queryReachable": true + }, + { + "id": "rule:whereClause", + "kind": "parserRule", + "name": "whereClause", + "queryReachable": true + }, + { + "id": "rule:whileStmt", + "kind": "parserRule", + "name": "whileStmt", + "queryReachable": true + }, + { + "id": "rule:winFrameBound", + "kind": "parserRule", + "name": "winFrameBound", + "queryReachable": true + }, + { + "id": "rule:winFrameClause", + "kind": "parserRule", + "name": "winFrameClause", + "queryReachable": true + }, + { + "id": "rule:winFrameExtend", + "kind": "parserRule", + "name": "winFrameExtend", + "queryReachable": true + }, + { + "id": "rule:winOrderByClause", + "kind": "parserRule", + "name": "winOrderByClause", + "queryReachable": true + }, + { + "id": "rule:winPartitionByClause", + "kind": "parserRule", + "name": "winPartitionByClause", + "queryReachable": true + }, + { + "id": "rule:windowClause", + "kind": "parserRule", + "name": "windowClause", + "queryReachable": true + }, + { + "id": "rule:windowExpr", + "kind": "parserRule", + "name": "windowExpr", + "queryReachable": true + }, + { + "id": "rule:withClause", + "kind": "parserRule", + "name": "withClause", + "queryReachable": true + }, + { + "id": "rule:withExpr", + "kind": "parserRule", + "name": "withExpr", + "queryReachable": true + }, + { + "id": "rule:withExprColumnNameList", + "kind": "parserRule", + "name": "withExprColumnNameList", + "queryReachable": true + }, + { + "id": "rule:withExprList", + "kind": "parserRule", + "name": "withExprList", + "queryReachable": true + }, + { + "id": "rule:withFillClause", + "kind": "parserRule", + "name": "withFillClause", + "queryReachable": true + }, + { + "id": "rule:withinGroupClause", + "kind": "parserRule", + "name": "withinGroupClause", + "queryReachable": true + }, + { + "id": "token:ALL", + "kind": "token", + "name": "ALL", + "queryReachable": true + }, + { + "id": "token:AND", + "kind": "token", + "name": "AND", + "queryReachable": true + }, + { + "id": "token:ANTI", + "kind": "token", + "name": "ANTI", + "queryReachable": true + }, + { + "id": "token:ANY", + "kind": "token", + "name": "ANY", + "queryReachable": true + }, + { + "id": "token:ARRAY", + "kind": "token", + "name": "ARRAY", + "queryReachable": true + }, + { + "id": "token:ARROW", + "kind": "token", + "name": "ARROW", + "queryReachable": true + }, + { + "id": "token:AS", + "kind": "token", + "name": "AS", + "queryReachable": true + }, + { + "id": "token:ASCENDING", + "kind": "token", + "name": "ASCENDING", + "queryReachable": true + }, + { + "id": "token:ASOF", + "kind": "token", + "name": "ASOF", + "queryReachable": true + }, + { + "id": "token:ASTERISK", + "kind": "token", + "name": "ASTERISK", + "queryReachable": true + }, + { + "id": "token:BACKQUOTE", + "kind": "token", + "name": "BACKQUOTE", + "queryReachable": false + }, + { + "id": "token:BACKSLASH", + "kind": "token", + "name": "BACKSLASH", + "queryReachable": false + }, + { + "id": "token:BETWEEN", + "kind": "token", + "name": "BETWEEN", + "queryReachable": true + }, + { + "id": "token:BINARY_LITERAL", + "kind": "token", + "name": "BINARY_LITERAL", + "queryReachable": true + }, + { + "id": "token:BOTH", + "kind": "token", + "name": "BOTH", + "queryReachable": true + }, + { + "id": "token:BY", + "kind": "token", + "name": "BY", + "queryReachable": true + }, + { + "id": "token:CASE", + "kind": "token", + "name": "CASE", + "queryReachable": true + }, + { + "id": "token:CAST", + "kind": "token", + "name": "CAST", + "queryReachable": true + }, + { + "id": "token:CATCH", + "kind": "token", + "name": "CATCH", + "queryReachable": true + }, + { + "id": "token:COHORT", + "kind": "token", + "name": "COHORT", + "queryReachable": true + }, + { + "id": "token:COLLATE", + "kind": "token", + "name": "COLLATE", + "queryReachable": true + }, + { + "id": "token:COLON", + "kind": "token", + "name": "COLON", + "queryReachable": true + }, + { + "id": "token:COLONEQUALS", + "kind": "token", + "name": "COLONEQUALS", + "queryReachable": true + }, + { + "id": "token:COLUMNS", + "kind": "token", + "name": "COLUMNS", + "queryReachable": true + }, + { + "id": "token:COMMA", + "kind": "token", + "name": "COMMA", + "queryReachable": true + }, + { + "id": "token:CONCAT", + "kind": "token", + "name": "CONCAT", + "queryReachable": true + }, + { + "id": "token:CROSS", + "kind": "token", + "name": "CROSS", + "queryReachable": true + }, + { + "id": "token:CUBE", + "kind": "token", + "name": "CUBE", + "queryReachable": true + }, + { + "id": "token:CURRENT", + "kind": "token", + "name": "CURRENT", + "queryReachable": true + }, + { + "id": "token:DASH", + "kind": "token", + "name": "DASH", + "queryReachable": true + }, + { + "id": "token:DATE", + "kind": "token", + "name": "DATE", + "queryReachable": true + }, + { + "id": "token:DAY", + "kind": "token", + "name": "DAY", + "queryReachable": true + }, + { + "id": "token:DECIMAL_LITERAL", + "kind": "token", + "name": "DECIMAL_LITERAL", + "queryReachable": true + }, + { + "id": "token:DESC", + "kind": "token", + "name": "DESC", + "queryReachable": true + }, + { + "id": "token:DESCENDING", + "kind": "token", + "name": "DESCENDING", + "queryReachable": true + }, + { + "id": "token:DISTINCT", + "kind": "token", + "name": "DISTINCT", + "queryReachable": true + }, + { + "id": "token:DOLLAR", + "kind": "token", + "name": "DOLLAR", + "queryReachable": false + }, + { + "id": "token:DOT", + "kind": "token", + "name": "DOT", + "queryReachable": true + }, + { + "id": "token:DOUBLECOLON", + "kind": "token", + "name": "DOUBLECOLON", + "queryReachable": true + }, + { + "id": "token:ELSE", + "kind": "token", + "name": "ELSE", + "queryReachable": true + }, + { + "id": "token:END", + "kind": "token", + "name": "END", + "queryReachable": true + }, + { + "id": "token:EQ_DOUBLE", + "kind": "token", + "name": "EQ_DOUBLE", + "queryReachable": true + }, + { + "id": "token:EQ_SINGLE", + "kind": "token", + "name": "EQ_SINGLE", + "queryReachable": true + }, + { + "id": "token:ESCAPE_CHAR_COMMON", + "kind": "token", + "name": "ESCAPE_CHAR_COMMON", + "queryReachable": false + }, + { + "id": "token:EXCEPT", + "kind": "token", + "name": "EXCEPT", + "queryReachable": true + }, + { + "id": "token:EXCLUDE", + "kind": "token", + "name": "EXCLUDE", + "queryReachable": true + }, + { + "id": "token:EXTRACT", + "kind": "token", + "name": "EXTRACT", + "queryReachable": true + }, + { + "id": "token:FILL", + "kind": "token", + "name": "FILL", + "queryReachable": true + }, + { + "id": "token:FILTER", + "kind": "token", + "name": "FILTER", + "queryReachable": true + }, + { + "id": "token:FINAL", + "kind": "token", + "name": "FINAL", + "queryReachable": true + }, + { + "id": "token:FINALLY", + "kind": "token", + "name": "FINALLY", + "queryReachable": true + }, + { + "id": "token:FIRST", + "kind": "token", + "name": "FIRST", + "queryReachable": true + }, + { + "id": "token:FLOATING_LITERAL", + "kind": "token", + "name": "FLOATING_LITERAL", + "queryReachable": true + }, + { + "id": "token:FN", + "kind": "token", + "name": "FN", + "queryReachable": true + }, + { + "id": "token:FOLLOWING", + "kind": "token", + "name": "FOLLOWING", + "queryReachable": true + }, + { + "id": "token:FOR", + "kind": "token", + "name": "FOR", + "queryReachable": true + }, + { + "id": "token:FROM", + "kind": "token", + "name": "FROM", + "queryReachable": true + }, + { + "id": "token:FULL", + "kind": "token", + "name": "FULL", + "queryReachable": true + }, + { + "id": "token:FULL_STRING_ESCAPE_TRIGGER", + "kind": "token", + "name": "FULL_STRING_ESCAPE_TRIGGER", + "queryReachable": false + }, + { + "id": "token:FULL_STRING_TEXT", + "kind": "token", + "name": "FULL_STRING_TEXT", + "queryReachable": false + }, + { + "id": "token:FUN", + "kind": "token", + "name": "FUN", + "queryReachable": true + }, + { + "id": "token:GROUP", + "kind": "token", + "name": "GROUP", + "queryReachable": true + }, + { + "id": "token:GROUPING", + "kind": "token", + "name": "GROUPING", + "queryReachable": true + }, + { + "id": "token:GT", + "kind": "token", + "name": "GT", + "queryReachable": true + }, + { + "id": "token:GT_EQ", + "kind": "token", + "name": "GT_EQ", + "queryReachable": true + }, + { + "id": "token:HASH", + "kind": "token", + "name": "HASH", + "queryReachable": true + }, + { + "id": "token:HASH_COMMENT", + "kind": "token", + "name": "HASH_COMMENT", + "queryReachable": false + }, + { + "id": "token:HAVING", + "kind": "token", + "name": "HAVING", + "queryReachable": true + }, + { + "id": "token:HEXADECIMAL_LITERAL", + "kind": "token", + "name": "HEXADECIMAL_LITERAL", + "queryReachable": true + }, + { + "id": "token:HOGQLX_TEXT_TEXT", + "kind": "token", + "name": "HOGQLX_TEXT_TEXT", + "queryReachable": true + }, + { + "id": "token:HOGQLX_TEXT_WS", + "kind": "token", + "name": "HOGQLX_TEXT_WS", + "queryReachable": false + }, + { + "id": "token:HOUR", + "kind": "token", + "name": "HOUR", + "queryReachable": true + }, + { + "id": "token:ID", + "kind": "token", + "name": "ID", + "queryReachable": true + }, + { + "id": "token:IDENTIFIER", + "kind": "token", + "name": "IDENTIFIER", + "queryReachable": true + }, + { + "id": "token:IF", + "kind": "token", + "name": "IF", + "queryReachable": true + }, + { + "id": "token:IGNORE", + "kind": "token", + "name": "IGNORE", + "queryReachable": true + }, + { + "id": "token:ILIKE", + "kind": "token", + "name": "ILIKE", + "queryReachable": true + }, + { + "id": "token:IN", + "kind": "token", + "name": "IN", + "queryReachable": true + }, + { + "id": "token:INCLUDE", + "kind": "token", + "name": "INCLUDE", + "queryReachable": true + }, + { + "id": "token:INF", + "kind": "token", + "name": "INF", + "queryReachable": true + }, + { + "id": "token:INNER", + "kind": "token", + "name": "INNER", + "queryReachable": true + }, + { + "id": "token:INTERPOLATE", + "kind": "token", + "name": "INTERPOLATE", + "queryReachable": true + }, + { + "id": "token:INTERSECT", + "kind": "token", + "name": "INTERSECT", + "queryReachable": true + }, + { + "id": "token:INTERVAL", + "kind": "token", + "name": "INTERVAL", + "queryReachable": true + }, + { + "id": "token:IREGEX_DOUBLE", + "kind": "token", + "name": "IREGEX_DOUBLE", + "queryReachable": true + }, + { + "id": "token:IREGEX_SINGLE", + "kind": "token", + "name": "IREGEX_SINGLE", + "queryReachable": true + }, + { + "id": "token:IS", + "kind": "token", + "name": "IS", + "queryReachable": true + }, + { + "id": "token:JOIN", + "kind": "token", + "name": "JOIN", + "queryReachable": true + }, + { + "id": "token:KEY", + "kind": "token", + "name": "KEY", + "queryReachable": true + }, + { + "id": "token:LAMBDA", + "kind": "token", + "name": "LAMBDA", + "queryReachable": true + }, + { + "id": "token:LAST", + "kind": "token", + "name": "LAST", + "queryReachable": true + }, + { + "id": "token:LBRACE", + "kind": "token", + "name": "LBRACE", + "queryReachable": true + }, + { + "id": "token:LBRACKET", + "kind": "token", + "name": "LBRACKET", + "queryReachable": true + }, + { + "id": "token:LEADING", + "kind": "token", + "name": "LEADING", + "queryReachable": true + }, + { + "id": "token:LEFT", + "kind": "token", + "name": "LEFT", + "queryReachable": true + }, + { + "id": "token:LET", + "kind": "token", + "name": "LET", + "queryReachable": true + }, + { + "id": "token:LIKE", + "kind": "token", + "name": "LIKE", + "queryReachable": true + }, + { + "id": "token:LIMIT", + "kind": "token", + "name": "LIMIT", + "queryReachable": true + }, + { + "id": "token:LOCAL", + "kind": "token", + "name": "LOCAL", + "queryReachable": true + }, + { + "id": "token:LPAREN", + "kind": "token", + "name": "LPAREN", + "queryReachable": true + }, + { + "id": "token:LT", + "kind": "token", + "name": "LT", + "queryReachable": true + }, + { + "id": "token:LT_EQ", + "kind": "token", + "name": "LT_EQ", + "queryReachable": true + }, + { + "id": "token:LT_SLASH", + "kind": "token", + "name": "LT_SLASH", + "queryReachable": true + }, + { + "id": "token:MALFORMED_BINARY_LITERAL", + "kind": "token", + "name": "MALFORMED_BINARY_LITERAL", + "queryReachable": false + }, + { + "id": "token:MATERIALIZED", + "kind": "token", + "name": "MATERIALIZED", + "queryReachable": true + }, + { + "id": "token:MINUTE", + "kind": "token", + "name": "MINUTE", + "queryReachable": true + }, + { + "id": "token:MONTH", + "kind": "token", + "name": "MONTH", + "queryReachable": true + }, + { + "id": "token:MULTI_LINE_COMMENT", + "kind": "token", + "name": "MULTI_LINE_COMMENT", + "queryReachable": false + }, + { + "id": "token:NAME", + "kind": "token", + "name": "NAME", + "queryReachable": true + }, + { + "id": "token:NAN_SQL", + "kind": "token", + "name": "NAN_SQL", + "queryReachable": true + }, + { + "id": "token:NATURAL", + "kind": "token", + "name": "NATURAL", + "queryReachable": true + }, + { + "id": "token:NOT", + "kind": "token", + "name": "NOT", + "queryReachable": true + }, + { + "id": "token:NOT_EQ", + "kind": "token", + "name": "NOT_EQ", + "queryReachable": true + }, + { + "id": "token:NOT_IREGEX", + "kind": "token", + "name": "NOT_IREGEX", + "queryReachable": true + }, + { + "id": "token:NOT_REGEX", + "kind": "token", + "name": "NOT_REGEX", + "queryReachable": true + }, + { + "id": "token:NULLISH", + "kind": "token", + "name": "NULLISH", + "queryReachable": true + }, + { + "id": "token:NULLS", + "kind": "token", + "name": "NULLS", + "queryReachable": true + }, + { + "id": "token:NULL_PROPERTY", + "kind": "token", + "name": "NULL_PROPERTY", + "queryReachable": true + }, + { + "id": "token:NULL_SAFE_EQ", + "kind": "token", + "name": "NULL_SAFE_EQ", + "queryReachable": true + }, + { + "id": "token:NULL_SQL", + "kind": "token", + "name": "NULL_SQL", + "queryReachable": true + }, + { + "id": "token:OCTAL_LITERAL", + "kind": "token", + "name": "OCTAL_LITERAL", + "queryReachable": true + }, + { + "id": "token:OCTAL_PREFIX_LITERAL", + "kind": "token", + "name": "OCTAL_PREFIX_LITERAL", + "queryReachable": true + }, + { + "id": "token:OFFSET", + "kind": "token", + "name": "OFFSET", + "queryReachable": true + }, + { + "id": "token:ON", + "kind": "token", + "name": "ON", + "queryReachable": true + }, + { + "id": "token:OR", + "kind": "token", + "name": "OR", + "queryReachable": true + }, + { + "id": "token:ORDER", + "kind": "token", + "name": "ORDER", + "queryReachable": true + }, + { + "id": "token:OUTER", + "kind": "token", + "name": "OUTER", + "queryReachable": true + }, + { + "id": "token:OVER", + "kind": "token", + "name": "OVER", + "queryReachable": true + }, + { + "id": "token:PARTITION", + "kind": "token", + "name": "PARTITION", + "queryReachable": true + }, + { + "id": "token:PERCENT", + "kind": "token", + "name": "PERCENT", + "queryReachable": true + }, + { + "id": "token:PIVOT", + "kind": "token", + "name": "PIVOT", + "queryReachable": true + }, + { + "id": "token:PLUS", + "kind": "token", + "name": "PLUS", + "queryReachable": true + }, + { + "id": "token:POSITIONAL", + "kind": "token", + "name": "POSITIONAL", + "queryReachable": true + }, + { + "id": "token:PRECEDING", + "kind": "token", + "name": "PRECEDING", + "queryReachable": true + }, + { + "id": "token:PREWHERE", + "kind": "token", + "name": "PREWHERE", + "queryReachable": true + }, + { + "id": "token:QUALIFY", + "kind": "token", + "name": "QUALIFY", + "queryReachable": true + }, + { + "id": "token:QUARTER", + "kind": "token", + "name": "QUARTER", + "queryReachable": true + }, + { + "id": "token:QUERY", + "kind": "token", + "name": "QUERY", + "queryReachable": true + }, + { + "id": "token:QUOTED_IDENTIFIER", + "kind": "token", + "name": "QUOTED_IDENTIFIER", + "queryReachable": true + }, + { + "id": "token:QUOTE_DOUBLE", + "kind": "token", + "name": "QUOTE_DOUBLE", + "queryReachable": false + }, + { + "id": "token:QUOTE_SINGLE", + "kind": "token", + "name": "QUOTE_SINGLE", + "queryReachable": true + }, + { + "id": "token:QUOTE_SINGLE_TEMPLATE", + "kind": "token", + "name": "QUOTE_SINGLE_TEMPLATE", + "queryReachable": true + }, + { + "id": "token:QUOTE_SINGLE_TEMPLATE_FULL", + "kind": "token", + "name": "QUOTE_SINGLE_TEMPLATE_FULL", + "queryReachable": false + }, + { + "id": "token:RANGE", + "kind": "token", + "name": "RANGE", + "queryReachable": true + }, + { + "id": "token:RBRACE", + "kind": "token", + "name": "RBRACE", + "queryReachable": true + }, + { + "id": "token:RBRACKET", + "kind": "token", + "name": "RBRACKET", + "queryReachable": true + }, + { + "id": "token:RECURSIVE", + "kind": "token", + "name": "RECURSIVE", + "queryReachable": true + }, + { + "id": "token:REGEX_DOUBLE", + "kind": "token", + "name": "REGEX_DOUBLE", + "queryReachable": true + }, + { + "id": "token:REGEX_SINGLE", + "kind": "token", + "name": "REGEX_SINGLE", + "queryReachable": true + }, + { + "id": "token:REPLACE", + "kind": "token", + "name": "REPLACE", + "queryReachable": true + }, + { + "id": "token:RETURN", + "kind": "token", + "name": "RETURN", + "queryReachable": true + }, + { + "id": "token:RIGHT", + "kind": "token", + "name": "RIGHT", + "queryReachable": true + }, + { + "id": "token:ROLLUP", + "kind": "token", + "name": "ROLLUP", + "queryReachable": true + }, + { + "id": "token:ROW", + "kind": "token", + "name": "ROW", + "queryReachable": true + }, + { + "id": "token:ROWS", + "kind": "token", + "name": "ROWS", + "queryReachable": true + }, + { + "id": "token:RPAREN", + "kind": "token", + "name": "RPAREN", + "queryReachable": true + }, + { + "id": "token:SAMPLE", + "kind": "token", + "name": "SAMPLE", + "queryReachable": true + }, + { + "id": "token:SECOND", + "kind": "token", + "name": "SECOND", + "queryReachable": true + }, + { + "id": "token:SELECT", + "kind": "token", + "name": "SELECT", + "queryReachable": true + }, + { + "id": "token:SEMI", + "kind": "token", + "name": "SEMI", + "queryReachable": true + }, + { + "id": "token:SEMICOLON", + "kind": "token", + "name": "SEMICOLON", + "queryReachable": true + }, + { + "id": "token:SETS", + "kind": "token", + "name": "SETS", + "queryReachable": true + }, + { + "id": "token:SETTINGS", + "kind": "token", + "name": "SETTINGS", + "queryReachable": true + }, + { + "id": "token:SINGLE_LINE_COMMENT", + "kind": "token", + "name": "SINGLE_LINE_COMMENT", + "queryReachable": false + }, + { + "id": "token:SLASH", + "kind": "token", + "name": "SLASH", + "queryReachable": true + }, + { + "id": "token:SLASH_GT", + "kind": "token", + "name": "SLASH_GT", + "queryReachable": true + }, + { + "id": "token:STEP", + "kind": "token", + "name": "STEP", + "queryReachable": true + }, + { + "id": "token:STRING_ESCAPE_TRIGGER", + "kind": "token", + "name": "STRING_ESCAPE_TRIGGER", + "queryReachable": true + }, + { + "id": "token:STRING_LITERAL", + "kind": "token", + "name": "STRING_LITERAL", + "queryReachable": true + }, + { + "id": "token:STRING_TEXT", + "kind": "token", + "name": "STRING_TEXT", + "queryReachable": true + }, + { + "id": "token:SUBSTRING", + "kind": "token", + "name": "SUBSTRING", + "queryReachable": true + }, + { + "id": "token:TAGC_MULTI_LINE_COMMENT", + "kind": "token", + "name": "TAGC_MULTI_LINE_COMMENT", + "queryReachable": false + }, + { + "id": "token:TAGC_SINGLE_LINE_COMMENT", + "kind": "token", + "name": "TAGC_SINGLE_LINE_COMMENT", + "queryReachable": false + }, + { + "id": "token:TAGC_WS", + "kind": "token", + "name": "TAGC_WS", + "queryReachable": false + }, + { + "id": "token:TAG_MULTI_LINE_COMMENT", + "kind": "token", + "name": "TAG_MULTI_LINE_COMMENT", + "queryReachable": false + }, + { + "id": "token:TAG_SINGLE_LINE_COMMENT", + "kind": "token", + "name": "TAG_SINGLE_LINE_COMMENT", + "queryReachable": false + }, + { + "id": "token:TAG_WS", + "kind": "token", + "name": "TAG_WS", + "queryReachable": false + }, + { + "id": "token:THEN", + "kind": "token", + "name": "THEN", + "queryReachable": true + }, + { + "id": "token:THROW", + "kind": "token", + "name": "THROW", + "queryReachable": true + }, + { + "id": "token:TIES", + "kind": "token", + "name": "TIES", + "queryReachable": true + }, + { + "id": "token:TIME", + "kind": "token", + "name": "TIME", + "queryReachable": true + }, + { + "id": "token:TIMESTAMP", + "kind": "token", + "name": "TIMESTAMP", + "queryReachable": true + }, + { + "id": "token:TO", + "kind": "token", + "name": "TO", + "queryReachable": true + }, + { + "id": "token:TOP", + "kind": "token", + "name": "TOP", + "queryReachable": true + }, + { + "id": "token:TOTALS", + "kind": "token", + "name": "TOTALS", + "queryReachable": true + }, + { + "id": "token:TRAILING", + "kind": "token", + "name": "TRAILING", + "queryReachable": true + }, + { + "id": "token:TRIM", + "kind": "token", + "name": "TRIM", + "queryReachable": true + }, + { + "id": "token:TRUNCATE", + "kind": "token", + "name": "TRUNCATE", + "queryReachable": true + }, + { + "id": "token:TRY", + "kind": "token", + "name": "TRY", + "queryReachable": true + }, + { + "id": "token:TRY_CAST", + "kind": "token", + "name": "TRY_CAST", + "queryReachable": true + }, + { + "id": "token:UNBOUNDED", + "kind": "token", + "name": "UNBOUNDED", + "queryReachable": true + }, + { + "id": "token:UNDERSCORE", + "kind": "token", + "name": "UNDERSCORE", + "queryReachable": false + }, + { + "id": "token:UNEXPECTED_CHARACTER", + "kind": "token", + "name": "UNEXPECTED_CHARACTER", + "queryReachable": false + }, + { + "id": "token:UNION", + "kind": "token", + "name": "UNION", + "queryReachable": true + }, + { + "id": "token:UNPIVOT", + "kind": "token", + "name": "UNPIVOT", + "queryReachable": true + }, + { + "id": "token:USING", + "kind": "token", + "name": "USING", + "queryReachable": true + }, + { + "id": "token:VALUES", + "kind": "token", + "name": "VALUES", + "queryReachable": true + }, + { + "id": "token:WEEK", + "kind": "token", + "name": "WEEK", + "queryReachable": true + }, + { + "id": "token:WHEN", + "kind": "token", + "name": "WHEN", + "queryReachable": true + }, + { + "id": "token:WHERE", + "kind": "token", + "name": "WHERE", + "queryReachable": true + }, + { + "id": "token:WHILE", + "kind": "token", + "name": "WHILE", + "queryReachable": true + }, + { + "id": "token:WHITESPACE", + "kind": "token", + "name": "WHITESPACE", + "queryReachable": false + }, + { + "id": "token:WINDOW", + "kind": "token", + "name": "WINDOW", + "queryReachable": true + }, + { + "id": "token:WITH", + "kind": "token", + "name": "WITH", + "queryReachable": true + }, + { + "id": "token:WITHIN", + "kind": "token", + "name": "WITHIN", + "queryReachable": true + }, + { + "id": "token:YEAR", + "kind": "token", + "name": "YEAR", + "queryReachable": true + }, + { + "id": "token:ZONE", + "kind": "token", + "name": "ZONE", + "queryReachable": true + } + ], + "grammarSha256": "c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242", + "languageVersion": "1.0.0", + "queryEntryPoint": "select", + "schemaVersion": 1, + "valid": false, + "validationErrors": [ + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "alias" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "columnExprTupleOrSingle" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "columnIdentifier" + }, + { + "alternativeCount": 4, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "columnTypeCastIdentifier" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "declaration" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "floatingLiteral" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "hogqlxChildElement" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "hogqlxTagAttribute" + }, + { + "alternativeCount": 4, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "identifier" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "implicitAlias" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "joinConstraintClause" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "joinOpCross" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "limitAndOffsetClause" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "limitAndOffsetClauseOptional" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "literal" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "ratioExpr" + }, + { + "alternativeCount": 4, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "selectStmtWithParens" + }, + { + "alternativeCount": 11, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "statement" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "string" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "stringContents" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": false, + "rule": "stringContentsFull" + } + ] +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/language.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/language.json index 163384afcafa..cae2a2f6c4ca 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/language.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/language.json @@ -1,6 +1,16 @@ { "antlrVersion": "4.13.2", "canonicalParser": "cpp-antlr", + "corpora": [ + { + "manifestPath": "corpus/expr_select_cases.json", + "manifestSha256": "100c08ef64c9a02b956f5bbaca265c3b47993acf26e000e1812f110ae5112a3d", + "oraclePath": "corpus/expr_select_cpp_oracle.jsonl", + "oracleSha256": "c45fa533a9fb9cb80b7a4009fc468b6e1a76ef1ac347220a0feb5a91453c500b", + "schemaVersion": 1, + "slice": "expression-and-plain-select" + } + ], "entryPoints": { "expression": "expr", "query": "select" @@ -19,6 +29,11 @@ "sha256": "4cc3cf6f6f8d2584df7cae12a31d546c6bffef3e0e85d7101e7749d6bd0f5430" } ], + "grammarFeatureManifest": { + "path": "grammar-features.json", + "schemaVersion": 1, + "sha256": "b116128c3972a3c0ac3988f9d7ed9b16f4d0f58514e788c41604fae0b97ecbae" + }, "grammarSha256": "c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242", "languageVersion": "1.0.0", "schemaVersion": 1 diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/source-lock.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/source-lock.json index 183a9d3dbae4..86a14069dc97 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/source-lock.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/source-lock.json @@ -1,6 +1,6 @@ { - "descriptorSha256": "dfb983a1b449cbf0e09481ac4fbbceb8e12f62184f361971f51768b0f6575b25", + "descriptorSha256": "42250da46fd4dda2b8889007c4aa5b7443b674c800066a716fc413a9f0f8889d", "schemaVersion": 1, "sourceRepository": "https://github.com/PostHog/posthog", - "sourceRevision": "02508a8eafa30ca2b1ae561c921a8e64d4332dd6" + "sourceRevision": "47a7b39826348448f82ee5bdf5a399cbc0a4c4b2" } diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlLanguageContract.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlLanguageContract.java index 089f66d4457e..758e5b33a95f 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlLanguageContract.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlLanguageContract.java @@ -42,6 +42,9 @@ public void testLoadsCurrentLanguageContract() assertThat(contract.antlrVersion()).isEqualTo("4.13.2"); assertThat(contract.canonicalParser()).isEqualTo("cpp-antlr"); assertThat(contract.entryPoints()).containsEntry("query", "select"); + assertThat(contract.grammarFeatureManifest().schemaVersion()).isEqualTo(1); + assertThat(contract.corpora()).singleElement() + .satisfies(corpus -> assertThat(corpus.slice()).isEqualTo("expression-and-plain-select")); assertThat(contract.grammarSha256()).isEqualTo("c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242"); } @@ -81,6 +84,23 @@ public void testGrammarFilesMatchLanguageDescriptor() assertThat(HexFormat.of().formatHex(aggregate.digest())).isEqualTo(contract.grammarSha256()); } + @Test + public void testPublishedManifestsMatchLanguageDescriptor() + throws IOException + { + HogQlLanguageContract contract = HogQlLanguageContract.current(); + HogQlLanguageContract.GrammarFeatureManifest featureManifest = contract.grammarFeatureManifest(); + assertThat(sha256(readResource(LANGUAGE_RESOURCE_ROOT + featureManifest.path()))) + .isEqualTo(featureManifest.sha256()); + + for (HogQlLanguageContract.Corpus corpus : contract.corpora()) { + assertThat(sha256(readResource(LANGUAGE_RESOURCE_ROOT + corpus.manifestPath()))) + .isEqualTo(corpus.manifestSha256()); + assertThat(sha256(readResource(LANGUAGE_RESOURCE_ROOT + corpus.oraclePath()))) + .isEqualTo(corpus.oracleSha256()); + } + } + private static byte[] readResource(String path) throws IOException { From 677fcd9c653ef97d4a5f3154eaa1482592d89c6a Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Wed, 26 Aug 2026 22:44:51 -0400 Subject: [PATCH 006/136] feat(hogql): add exhaustive compatibility manifest --- .../parser/HogQlCompatibilityManifest.java | 291 + .../io/trino/hogql/parser/HogQlParser.java | 3 + .../language/1.0.0/grammar-features.json | 581 ++ .../hogql/parser/language/1.0.0/language.json | 2 +- .../parser/language/1.0.0/source-lock.json | 4 +- .../1.0.0/trino-compatibility-overrides.json | 314 + .../language/1.0.0/trino-compatibility.json | 5837 +++++++++++++++++ .../parser/TestHogQlCompatibilityCorpus.java | 207 + .../TestHogQlCompatibilityManifest.java | 110 + 9 files changed, 7346 insertions(+), 3 deletions(-) create mode 100644 core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlCompatibilityManifest.java create mode 100644 core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json create mode 100644 core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json create mode 100644 core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java create mode 100644 core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityManifest.java diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlCompatibilityManifest.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlCompatibilityManifest.java new file mode 100644 index 000000000000..f1a6f5e37e6b --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlCompatibilityManifest.java @@ -0,0 +1,291 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; +import io.airlift.json.JsonCodec; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; + +public record HogQlCompatibilityManifest( + int schemaVersion, + HogQlLanguageVersion languageVersion, + String grammarSha256, + String grammarFeatureManifestSha256, + List unresolvedGrammarAlternativeRules, + List features) +{ + private static final String LANGUAGE_RESOURCE_ROOT = "/io/trino/hogql/parser/language/1.0.0/"; + private static final String CURRENT_MANIFEST_RESOURCE = LANGUAGE_RESOURCE_ROOT + "trino-compatibility.json"; + private static final JsonCodec MANIFEST_CODEC = jsonCodec(HogQlCompatibilityManifest.class); + private static final JsonCodec GRAMMAR_FEATURE_MANIFEST_CODEC = jsonCodec(PublishedGrammarFeatureManifest.class); + private static final HogQlCompatibilityManifest CURRENT = loadCurrent(); + + @JsonCreator + public HogQlCompatibilityManifest( + @JsonProperty("schemaVersion") int schemaVersion, + @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, + @JsonProperty("grammarSha256") String grammarSha256, + @JsonProperty("grammarFeatureManifestSha256") String grammarFeatureManifestSha256, + @JsonProperty("unresolvedGrammarAlternativeRules") List unresolvedGrammarAlternativeRules, + @JsonProperty("features") List features) + { + if (schemaVersion != 1) { + throw new IllegalArgumentException("unsupported HogQL compatibility manifest schema: " + schemaVersion); + } + this.schemaVersion = schemaVersion; + this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); + this.grammarFeatureManifestSha256 = requireNonNull(grammarFeatureManifestSha256, "grammarFeatureManifestSha256 is null"); + this.unresolvedGrammarAlternativeRules = List.copyOf(requireNonNull(unresolvedGrammarAlternativeRules, "unresolvedGrammarAlternativeRules is null")); + this.features = List.copyOf(requireNonNull(features, "features is null")); + } + + public static HogQlCompatibilityManifest current() + { + return CURRENT; + } + + private static HogQlCompatibilityManifest loadCurrent() + { + HogQlCompatibilityManifest manifest = MANIFEST_CODEC.fromJson(readResource(CURRENT_MANIFEST_RESOURCE)); + HogQlLanguageContract languageContract = HogQlLanguageContract.current(); + if (!manifest.languageVersion().equals(languageContract.languageVersion())) { + throw new IllegalStateException("HogQL compatibility manifest language version does not match the language contract"); + } + if (!manifest.grammarSha256().equals(languageContract.grammarSha256())) { + throw new IllegalStateException("HogQL compatibility manifest grammar hash does not match the language contract"); + } + if (!manifest.grammarFeatureManifestSha256().equals(languageContract.grammarFeatureManifest().sha256())) { + throw new IllegalStateException("HogQL compatibility manifest feature hash does not match the language contract"); + } + + String featureManifestResource = LANGUAGE_RESOURCE_ROOT + languageContract.grammarFeatureManifest().path(); + byte[] featureManifestBytes = readResourceBytes(featureManifestResource); + if (!sha256(featureManifestBytes).equals(manifest.grammarFeatureManifestSha256())) { + throw new IllegalStateException("published HogQL grammar feature manifest checksum does not match the compatibility manifest"); + } + PublishedGrammarFeatureManifest published = GRAMMAR_FEATURE_MANIFEST_CODEC.fromJson(new String(featureManifestBytes, StandardCharsets.UTF_8)); + validateFeatures(manifest, published); + return manifest; + } + + private static void validateFeatures(HogQlCompatibilityManifest manifest, PublishedGrammarFeatureManifest published) + { + if (!published.languageVersion().equals(manifest.languageVersion()) || !published.grammarSha256().equals(manifest.grammarSha256())) { + throw new IllegalStateException("published HogQL grammar features do not match the compatibility manifest language"); + } + + Map publishedById = new HashMap<>(); + for (PublishedFeature feature : published.features()) { + if (publishedById.put(feature.id(), feature) != null) { + throw new IllegalStateException("duplicate published HogQL grammar feature: " + feature.id()); + } + } + + Map compatibilityById = new HashMap<>(); + for (Feature feature : manifest.features()) { + if (compatibilityById.put(feature.id(), feature) != null) { + throw new IllegalStateException("duplicate HogQL compatibility feature: " + feature.id()); + } + PublishedFeature source = publishedById.get(feature.id()); + if (source == null) { + throw new IllegalStateException("unknown HogQL compatibility feature: " + feature.id()); + } + if (!feature.kind().equals(source.kind()) || feature.queryReachable() != source.queryReachable()) { + throw new IllegalStateException("HogQL compatibility feature does not match its published definition: " + feature.id()); + } + if (feature.queryReachable() == (feature.status() == Status.NOT_QUERY_LANGUAGE)) { + throw new IllegalStateException("invalid HogQL query-language status for feature: " + feature.id()); + } + } + if (!compatibilityById.keySet().equals(publishedById.keySet())) { + throw new IllegalStateException("HogQL compatibility manifest does not account for every published grammar feature"); + } + if (!manifest.unresolvedGrammarAlternativeRules().equals(published.validationErrors())) { + throw new IllegalStateException("HogQL compatibility manifest does not acknowledge every unresolved grammar alternative rule"); + } + } + + private static String readResource(String path) + { + return new String(readResourceBytes(path), StandardCharsets.UTF_8); + } + + private static byte[] readResourceBytes(String path) + { + try (InputStream input = HogQlCompatibilityManifest.class.getResourceAsStream(path)) { + if (input == null) { + throw new IllegalStateException("HogQL compatibility resource is missing: " + path); + } + return input.readAllBytes(); + } + catch (IOException e) { + throw new UncheckedIOException("failed to read HogQL compatibility resource", e); + } + } + + private static String sha256(byte[] content) + { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content)); + } + catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } + + public record Feature( + String id, + String kind, + boolean queryReachable, + Status status, + String parseHandler, + String loweringHandler, + List testCaseIds) + { + @JsonCreator + public Feature( + @JsonProperty("id") String id, + @JsonProperty("kind") String kind, + @JsonProperty("queryReachable") boolean queryReachable, + @JsonProperty("status") Status status, + @JsonProperty("parseHandler") String parseHandler, + @JsonProperty("loweringHandler") String loweringHandler, + @JsonProperty("testCaseIds") List testCaseIds) + { + this.id = requireNonNull(id, "id is null"); + this.kind = requireNonNull(kind, "kind is null"); + this.queryReachable = queryReachable; + this.status = requireNonNull(status, "status is null"); + this.parseHandler = requireNonNull(parseHandler, "parseHandler is null"); + this.loweringHandler = requireNonNull(loweringHandler, "loweringHandler is null"); + this.testCaseIds = List.copyOf(requireNonNull(testCaseIds, "testCaseIds is null")); + if (id.isBlank() || kind.isBlank() || parseHandler.isBlank() || loweringHandler.isBlank() || testCaseIds.isEmpty() || testCaseIds.stream().anyMatch(String::isBlank)) { + throw new IllegalArgumentException("HogQL compatibility feature fields must be non-empty: " + id); + } + if (status == Status.SUPPORTED && (parseHandler.equals("UnsupportedFeature") || loweringHandler.equals("UnsupportedFeature"))) { + throw new IllegalArgumentException("supported HogQL feature must have parse and lowering handlers: " + id); + } + if (status == Status.EXPLICIT_CURRENT_ERROR && !loweringHandler.equals("UnsupportedFeature")) { + throw new IllegalArgumentException("unsupported HogQL feature must use the explicit error handler: " + id); + } + if (status == Status.NOT_QUERY_LANGUAGE && (!parseHandler.equals("NotQueryLanguage") || !loweringHandler.equals("NotQueryLanguage"))) { + throw new IllegalArgumentException("non-query HogQL feature must not have query handlers: " + id); + } + } + } + + public enum Status + { + SUPPORTED("supported"), + EXPLICIT_CURRENT_ERROR("explicitCurrentError"), + NOT_QUERY_LANGUAGE("notQueryLanguage"); + + private final String value; + + Status(String value) + { + this.value = value; + } + + @JsonCreator + public static Status fromJson(String value) + { + for (Status status : values()) { + if (status.value.equals(value)) { + return status; + } + } + throw new IllegalArgumentException("unknown HogQL compatibility status: " + value); + } + + @JsonValue + public String toJson() + { + return value; + } + } + + public record UnresolvedGrammarAlternativeRule( + int alternativeCount, + String code, + boolean queryReachable, + String rule) + { + @JsonCreator + public UnresolvedGrammarAlternativeRule( + @JsonProperty("alternativeCount") int alternativeCount, + @JsonProperty("code") String code, + @JsonProperty("queryReachable") boolean queryReachable, + @JsonProperty("rule") String rule) + { + if (alternativeCount < 2) { + throw new IllegalArgumentException("unresolved grammar rule must have multiple alternatives: " + rule); + } + this.alternativeCount = alternativeCount; + this.code = requireNonNull(code, "code is null"); + this.queryReachable = queryReachable; + this.rule = requireNonNull(rule, "rule is null"); + } + } + + public record PublishedGrammarFeatureManifest( + HogQlLanguageVersion languageVersion, + String grammarSha256, + List features, + List validationErrors) + { + @JsonCreator + public PublishedGrammarFeatureManifest( + @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, + @JsonProperty("grammarSha256") String grammarSha256, + @JsonProperty("features") List features, + @JsonProperty("validationErrors") List validationErrors) + { + this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); + this.features = List.copyOf(requireNonNull(features, "features is null")); + this.validationErrors = List.copyOf(requireNonNull(validationErrors, "validationErrors is null")); + } + } + + public record PublishedFeature(String id, String kind, boolean queryReachable) + { + @JsonCreator + public PublishedFeature( + @JsonProperty("id") String id, + @JsonProperty("kind") String kind, + @JsonProperty("queryReachable") boolean queryReachable) + { + this.id = requireNonNull(id, "id is null"); + this.kind = requireNonNull(kind, "kind is null"); + this.queryReachable = queryReachable; + } + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index a1481083393a..e53d0ba36ebe 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -291,6 +291,9 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) return binary(operator, binary.left, binary.right, binary); } if (context instanceof HogQLParser.ColumnExprPrecedence3Context binary) { + if (binary.operator == null) { + throw unsupported(binary, "comparison operator"); + } BinaryOperator operator = switch (binary.operator.getType()) { case HogQLParser.EQ_DOUBLE, HogQLParser.EQ_SINGLE -> EQUAL; case HogQLParser.NOT_EQ -> NOT_EQUAL; diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-features.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-features.json index 39bd8b5ed995..11180dae076e 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-features.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-features.json @@ -24,6 +24,13 @@ "queryReachable": true, "rule": "columnAliases" }, + { + "id": "alternative:columnExpr:ColumnExprAlias", + "kind": "parserAlternative", + "label": "ColumnExprAlias", + "queryReachable": true, + "rule": "columnExpr" + }, { "id": "alternative:columnExpr:ColumnExprAnd", "kind": "parserAlternative", @@ -31,12 +38,82 @@ "queryReachable": true, "rule": "columnExpr" }, + { + "id": "alternative:columnExpr:ColumnExprOr", + "kind": "parserAlternative", + "label": "ColumnExprOr", + "queryReachable": true, + "rule": "columnExpr" + }, + { + "id": "alternative:columnExpr:ColumnExprTernaryOp", + "kind": "parserAlternative", + "label": "ColumnExprTernaryOp", + "queryReachable": true, + "rule": "columnExpr" + }, + { + "id": "alternative:columnExpr:ColumnExprValuePassthrough", + "kind": "parserAlternative", + "label": "ColumnExprValuePassthrough", + "queryReachable": true, + "rule": "columnExpr" + }, { "id": "alternative:columnExprList", "kind": "parserAlternative", "queryReachable": true, "rule": "columnExprList" }, + { + "id": "alternative:columnExprValue:ColumnExprArray", + "kind": "parserAlternative", + "label": "ColumnExprArray", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprArrayAccess", + "kind": "parserAlternative", + "label": "ColumnExprArrayAccess", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprArraySlice", + "kind": "parserAlternative", + "label": "ColumnExprArraySlice", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprAsterisk", + "kind": "parserAlternative", + "label": "ColumnExprAsterisk", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprBetween", + "kind": "parserAlternative", + "label": "ColumnExprBetween", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprCall", + "kind": "parserAlternative", + "label": "ColumnExprCall", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprCallSelect", + "kind": "parserAlternative", + "label": "ColumnExprCallSelect", + "queryReachable": true, + "rule": "columnExprValue" + }, { "id": "alternative:columnExprValue:ColumnExprCase", "kind": "parserAlternative", @@ -44,6 +121,370 @@ "queryReachable": true, "rule": "columnExprValue" }, + { + "id": "alternative:columnExprValue:ColumnExprCast", + "kind": "parserAlternative", + "label": "ColumnExprCast", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColonLambda", + "kind": "parserAlternative", + "label": "ColumnExprColonLambda", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsAll", + "kind": "parserAlternative", + "label": "ColumnExprColumnsAll", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsExclude", + "kind": "parserAlternative", + "label": "ColumnExprColumnsExclude", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsExcludeReplace", + "kind": "parserAlternative", + "label": "ColumnExprColumnsExcludeReplace", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsList", + "kind": "parserAlternative", + "label": "ColumnExprColumnsList", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedAll", + "kind": "parserAlternative", + "label": "ColumnExprColumnsQualifiedAll", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedExclude", + "kind": "parserAlternative", + "label": "ColumnExprColumnsQualifiedExclude", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedExcludeReplace", + "kind": "parserAlternative", + "label": "ColumnExprColumnsQualifiedExcludeReplace", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedReplace", + "kind": "parserAlternative", + "label": "ColumnExprColumnsQualifiedReplace", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsRegex", + "kind": "parserAlternative", + "label": "ColumnExprColumnsRegex", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsReplace", + "kind": "parserAlternative", + "label": "ColumnExprColumnsReplace", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprDate", + "kind": "parserAlternative", + "label": "ColumnExprDate", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprDict", + "kind": "parserAlternative", + "label": "ColumnExprDict", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprFunction", + "kind": "parserAlternative", + "label": "ColumnExprFunction", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprFunctionWithinGroup", + "kind": "parserAlternative", + "label": "ColumnExprFunctionWithinGroup", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprIdentifier", + "kind": "parserAlternative", + "label": "ColumnExprIdentifier", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprIgnoreNulls", + "kind": "parserAlternative", + "label": "ColumnExprIgnoreNulls", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprInterval", + "kind": "parserAlternative", + "label": "ColumnExprInterval", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprIntervalString", + "kind": "parserAlternative", + "label": "ColumnExprIntervalString", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprIsDistinctFrom", + "kind": "parserAlternative", + "label": "ColumnExprIsDistinctFrom", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprIsNull", + "kind": "parserAlternative", + "label": "ColumnExprIsNull", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprLambda", + "kind": "parserAlternative", + "label": "ColumnExprLambda", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprLiteral", + "kind": "parserAlternative", + "label": "ColumnExprLiteral", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNamedArg", + "kind": "parserAlternative", + "label": "ColumnExprNamedArg", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNegate", + "kind": "parserAlternative", + "label": "ColumnExprNegate", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNot", + "kind": "parserAlternative", + "label": "ColumnExprNot", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullArrayAccess", + "kind": "parserAlternative", + "label": "ColumnExprNullArrayAccess", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullPropertyAccess", + "kind": "parserAlternative", + "label": "ColumnExprNullPropertyAccess", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullSafeEq", + "kind": "parserAlternative", + "label": "ColumnExprNullSafeEq", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullTupleAccess", + "kind": "parserAlternative", + "label": "ColumnExprNullTupleAccess", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullish", + "kind": "parserAlternative", + "label": "ColumnExprNullish", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprParens", + "kind": "parserAlternative", + "label": "ColumnExprParens", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprPositional", + "kind": "parserAlternative", + "label": "ColumnExprPositional", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence1", + "kind": "parserAlternative", + "label": "ColumnExprPrecedence1", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence2", + "kind": "parserAlternative", + "label": "ColumnExprPrecedence2", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence3", + "kind": "parserAlternative", + "label": "ColumnExprPrecedence3", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprPropertyAccess", + "kind": "parserAlternative", + "label": "ColumnExprPropertyAccess", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprSpreadColumnsList", + "kind": "parserAlternative", + "label": "ColumnExprSpreadColumnsList", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprSpreadColumnsRegex", + "kind": "parserAlternative", + "label": "ColumnExprSpreadColumnsRegex", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprSubquery", + "kind": "parserAlternative", + "label": "ColumnExprSubquery", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprSubstring", + "kind": "parserAlternative", + "label": "ColumnExprSubstring", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTagElement", + "kind": "parserAlternative", + "label": "ColumnExprTagElement", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTemplateString", + "kind": "parserAlternative", + "label": "ColumnExprTemplateString", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTimestamp", + "kind": "parserAlternative", + "label": "ColumnExprTimestamp", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTrim", + "kind": "parserAlternative", + "label": "ColumnExprTrim", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTryCast", + "kind": "parserAlternative", + "label": "ColumnExprTryCast", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTuple", + "kind": "parserAlternative", + "label": "ColumnExprTuple", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTupleAccess", + "kind": "parserAlternative", + "label": "ColumnExprTupleAccess", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTypeCast", + "kind": "parserAlternative", + "label": "ColumnExprTypeCast", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunction", + "kind": "parserAlternative", + "label": "ColumnExprWinFunction", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunctionTarget", + "kind": "parserAlternative", + "label": "ColumnExprWinFunctionTarget", + "queryReachable": true, + "rule": "columnExprValue" + }, { "id": "alternative:columnLambdaExpr:ArrowLambda", "kind": "parserAlternative", @@ -79,6 +520,48 @@ "queryReachable": true, "rule": "columnTypeExpr" }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprComplex", + "kind": "parserAlternative", + "label": "ColumnTypeExprComplex", + "queryReachable": true, + "rule": "columnTypeExpr" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprCompound", + "kind": "parserAlternative", + "label": "ColumnTypeExprCompound", + "queryReachable": true, + "rule": "columnTypeExpr" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprEnum", + "kind": "parserAlternative", + "label": "ColumnTypeExprEnum", + "queryReachable": true, + "rule": "columnTypeExpr" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprNested", + "kind": "parserAlternative", + "label": "ColumnTypeExprNested", + "queryReachable": true, + "rule": "columnTypeExpr" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprParam", + "kind": "parserAlternative", + "label": "ColumnTypeExprParam", + "queryReachable": true, + "rule": "columnTypeExpr" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprSimple", + "kind": "parserAlternative", + "label": "ColumnTypeExprSimple", + "queryReachable": true, + "rule": "columnTypeExpr" + }, { "id": "alternative:columnsReplaceItem", "kind": "parserAlternative", @@ -231,6 +714,13 @@ "queryReachable": true, "rule": "interval" }, + { + "id": "alternative:joinExpr:JoinExprCrossOp", + "kind": "parserAlternative", + "label": "JoinExprCrossOp", + "queryReachable": true, + "rule": "joinExpr" + }, { "id": "alternative:joinExpr:JoinExprOp", "kind": "parserAlternative", @@ -238,6 +728,41 @@ "queryReachable": true, "rule": "joinExpr" }, + { + "id": "alternative:joinExpr:JoinExprParens", + "kind": "parserAlternative", + "label": "JoinExprParens", + "queryReachable": true, + "rule": "joinExpr" + }, + { + "id": "alternative:joinExpr:JoinExprPivot", + "kind": "parserAlternative", + "label": "JoinExprPivot", + "queryReachable": true, + "rule": "joinExpr" + }, + { + "id": "alternative:joinExpr:JoinExprPositional", + "kind": "parserAlternative", + "label": "JoinExprPositional", + "queryReachable": true, + "rule": "joinExpr" + }, + { + "id": "alternative:joinExpr:JoinExprTable", + "kind": "parserAlternative", + "label": "JoinExprTable", + "queryReachable": true, + "rule": "joinExpr" + }, + { + "id": "alternative:joinExpr:JoinExprUnpivot", + "kind": "parserAlternative", + "label": "JoinExprUnpivot", + "queryReachable": true, + "rule": "joinExpr" + }, { "id": "alternative:joinOp:JoinOpFull", "kind": "parserAlternative", @@ -493,6 +1018,20 @@ "queryReachable": true, "rule": "tableArgList" }, + { + "id": "alternative:tableExpr:TableExprAlias", + "kind": "parserAlternative", + "label": "TableExprAlias", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableExpr:TableExprFunction", + "kind": "parserAlternative", + "label": "TableExprFunction", + "queryReachable": true, + "rule": "tableExpr" + }, { "id": "alternative:tableExpr:TableExprIdentifier", "kind": "parserAlternative", @@ -500,6 +1039,48 @@ "queryReachable": true, "rule": "tableExpr" }, + { + "id": "alternative:tableExpr:TableExprPivot", + "kind": "parserAlternative", + "label": "TableExprPivot", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableExpr:TableExprPlaceholder", + "kind": "parserAlternative", + "label": "TableExprPlaceholder", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableExpr:TableExprSubquery", + "kind": "parserAlternative", + "label": "TableExprSubquery", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableExpr:TableExprTag", + "kind": "parserAlternative", + "label": "TableExprTag", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableExpr:TableExprUnpivot", + "kind": "parserAlternative", + "label": "TableExprUnpivot", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableExpr:TableExprValues", + "kind": "parserAlternative", + "label": "TableExprValues", + "queryReachable": true, + "rule": "tableExpr" + }, { "id": "alternative:tableFunctionExpr", "kind": "parserAlternative", diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/language.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/language.json index cae2a2f6c4ca..eac6113fbd90 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/language.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/language.json @@ -32,7 +32,7 @@ "grammarFeatureManifest": { "path": "grammar-features.json", "schemaVersion": 1, - "sha256": "b116128c3972a3c0ac3988f9d7ed9b16f4d0f58514e788c41604fae0b97ecbae" + "sha256": "fbf7236643b43d929e9ef1c8fd01a04b746736dca2f965cc2e78ad62f1a98b10" }, "grammarSha256": "c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242", "languageVersion": "1.0.0", diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/source-lock.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/source-lock.json index 86a14069dc97..f223a0d77807 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/source-lock.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/source-lock.json @@ -1,6 +1,6 @@ { - "descriptorSha256": "42250da46fd4dda2b8889007c4aa5b7443b674c800066a716fc413a9f0f8889d", + "descriptorSha256": "4abfb6ced4fcdeb77aac3e2b5694913ad123a2a7cbc8d8f18a9e586b038c9de3", "schemaVersion": 1, "sourceRepository": "https://github.com/PostHog/posthog", - "sourceRevision": "47a7b39826348448f82ee5bdf5a399cbc0a4c4b2" + "sourceRevision": "f55d075d51ca2cc756ef2000699ac37dffa2c25c" } diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json new file mode 100644 index 000000000000..b6ad21fefacd --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json @@ -0,0 +1,314 @@ +{ + "corpusCases": [ + { + "id": "expr.accept.arithmetic", + "status": "supported" + }, + { + "id": "expr.accept.array", + "status": "explicitCurrentError" + }, + { + "id": "expr.accept.between", + "status": "explicitCurrentError" + }, + { + "id": "expr.accept.boolean", + "status": "supported" + }, + { + "id": "expr.accept.case", + "status": "explicitCurrentError" + }, + { + "id": "expr.accept.cast", + "status": "explicitCurrentError" + }, + { + "id": "expr.accept.dotted-field", + "status": "supported" + }, + { + "id": "expr.accept.function", + "status": "supported" + }, + { + "id": "expr.accept.in-list", + "status": "explicitCurrentError" + }, + { + "id": "expr.accept.literal-boolean", + "status": "supported" + }, + { + "id": "expr.accept.literal-null", + "status": "supported" + }, + { + "id": "expr.accept.literal-number", + "status": "supported" + }, + { + "id": "expr.accept.literal-string", + "status": "supported" + }, + { + "id": "expr.accept.null-predicate", + "status": "explicitCurrentError" + }, + { + "id": "expr.accept.tuple", + "status": "explicitCurrentError" + }, + { + "id": "expr.reject.incomplete-binary", + "status": "canonicalRejection" + }, + { + "id": "expr.reject.unclosed-tuple", + "status": "canonicalRejection" + }, + { + "id": "expr.reject.unexpected-character", + "status": "canonicalRejection" + }, + { + "id": "query.accept.aggregate", + "status": "explicitCurrentError" + }, + { + "id": "query.accept.constant", + "status": "supported" + }, + { + "id": "query.accept.multiline", + "status": "supported" + }, + { + "id": "query.accept.order-limit-offset", + "status": "explicitCurrentError" + }, + { + "id": "query.accept.projection-from", + "status": "supported" + }, + { + "id": "query.accept.where", + "status": "supported" + }, + { + "id": "query.reject.missing-projection", + "status": "canonicalRejection" + }, + { + "id": "query.reject.missing-source", + "status": "canonicalRejection" + }, + { + "id": "query.reject.missing-where-expression", + "status": "canonicalRejection" + }, + { + "id": "query.reject.trailing-comparison", + "status": "canonicalRejection" + } + ], + "features": [ + { + "id": "alternative:columnExpr:ColumnExprAnd", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause", + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExpr:ColumnExprOr", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.boolean" + ] + }, + { + "id": "alternative:columnExpr:ColumnExprValuePassthrough", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNegate", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNot", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause", + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprParens", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.boolean" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence1", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause", + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:databaseIdentifier", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:fromClause", + "loweringHandler": "TrinoAstFactory#createTable", + "parseHandler": "AstBuilder#buildTableReference", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:nestedIdentifier", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:selectColumnExpr:ColumnExprSelectValue", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#buildProjection", + "testCaseIds": [ + "TestHogQlCompiler#testReturnsOnlyStockTrinoTreeNodes" + ] + }, + { + "id": "alternative:selectColumnExprList", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:selectColumnExprListBeforeFrom:SelectColumnExprListBeforeFromPlain", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:tableExpr:TableExprIdentifier", + "loweringHandler": "TrinoAstFactory#createTable", + "parseHandler": "AstBuilder#buildTableReference", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:tableIdentifier", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:whereClause", + "loweringHandler": "TrinoAstFactory#createStatement", + "parseHandler": "AstBuilder#build", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause" + ] + }, + { + "id": "rule:databaseIdentifier", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:fromClause", + "loweringHandler": "TrinoAstFactory#createTable", + "parseHandler": "AstBuilder#buildTableReference", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:identifier", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:nestedIdentifier", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:selectColumnExprList", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:selectColumnExprListBeforeFrom", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:tableIdentifier", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:whereClause", + "loweringHandler": "TrinoAstFactory#createStatement", + "parseHandler": "AstBuilder#build", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause" + ] + } + ], + "schemaVersion": 1 +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json new file mode 100644 index 000000000000..7f6080774d71 --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json @@ -0,0 +1,5837 @@ +{ + "features": [ + { + "id": "alternative:arrayJoinClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:block", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:catchBlock", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnAliases", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExpr:ColumnExprAlias", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExpr:ColumnExprAnd", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause", + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExpr:ColumnExprOr", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.boolean" + ] + }, + { + "id": "alternative:columnExpr:ColumnExprTernaryOp", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExpr:ColumnExprValuePassthrough", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprArray", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprArrayAccess", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprArraySlice", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprAsterisk", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprBetween", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprCall", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprCallSelect", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprCase", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprCast", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColonLambda", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsAll", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsExclude", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsExcludeReplace", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedAll", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedExclude", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedExcludeReplace", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedReplace", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsRegex", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsReplace", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprDate", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprDict", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprFunction", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprFunctionWithinGroup", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIdentifier", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIgnoreNulls", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprInterval", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIntervalString", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIsDistinctFrom", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIsNull", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprLambda", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprLiteral", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNamedArg", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNegate", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNot", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause", + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNullArrayAccess", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNullPropertyAccess", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNullSafeEq", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNullTupleAccess", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNullish", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprParens", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.boolean" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprPositional", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence1", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause", + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence2", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence3", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprPropertyAccess", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprSpreadColumnsList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprSpreadColumnsRegex", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprSubquery", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprSubstring", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTagElement", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTemplateString", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTimestamp", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTrim", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTryCast", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTuple", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTupleAccess", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTypeCast", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunction", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunctionTarget", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnLambdaExpr:ArrowLambda", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnLambdaExpr:ColonLambda", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeCastExpr:ColumnTypeCastExprSimple", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeCastExpr:ColumnTypeCastExprWithTimeZone", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprArray", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprComplex", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprCompound", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprEnum", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprNested", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprParam", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprSimple", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnsReplaceItem", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnsReplaceList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:databaseIdentifier", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:emptyStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:enumValue", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:expr", + "kind": "parserAlternative", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "alternative:exprStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:expression", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:forInStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:forStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:fromClause", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createTable", + "parseHandler": "AstBuilder#buildTableReference", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:fullTemplateString", + "kind": "parserAlternative", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "alternative:funcStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:groupByClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:groupingSet", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:groupingSetList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:havingClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:hogqlxTagElement:HogqlxTagElementClosed", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:hogqlxTagElement:HogqlxTagElementNested", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:hogqlxText", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:identifierList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:ifStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:interpolateClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:interpolateExpr", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:interval", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:joinExpr:JoinExprCrossOp", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:joinExpr:JoinExprOp", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:joinExpr:JoinExprParens", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:joinExpr:JoinExprPivot", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:joinExpr:JoinExprPositional", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:joinExpr:JoinExprTable", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:joinExpr:JoinExprUnpivot", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:joinOp:JoinOpFull", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:joinOp:JoinOpInner", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:joinOp:JoinOpLeftRight", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:keyword", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:keywordForAlias", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:keywordForImplicitAlias", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:keywordForTypeCast", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:kvPair", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:kvPairList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitByClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitExpr", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:nestedIdentifier", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:numberLiteral", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:offsetOnlyClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:orderByClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:orderExpr", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:orderExprList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:pivotColumn", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:pivotColumnList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:placeholder", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:prewhereClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:program", + "kind": "parserAlternative", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "alternative:projectionOrderByClause", + "kind": "parserAlternative", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "alternative:qualifyClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:returnStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:sampleClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:select", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectColumnExpr:ColumnExprAliasBefore", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectColumnExpr:ColumnExprAliasImplicit", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectColumnExpr:ColumnExprInvalidFromImplicitAlias", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectColumnExpr:ColumnExprSelectValue", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#buildProjection", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testReturnsOnlyStockTrinoTreeNodes" + ] + }, + { + "id": "alternative:selectColumnExprList", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:selectColumnExprListBeforeFrom:SelectColumnExprListBeforeFromPlain", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:selectColumnExprListBeforeFrom:SelectColumnExprListBeforeFromTrailingComma", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectSetStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:settingExpr", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:settingExprList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:settingsClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:subsequentSelectSetClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableArgList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableExpr:TableExprAlias", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableExpr:TableExprFunction", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableExpr:TableExprIdentifier", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createTable", + "parseHandler": "AstBuilder#buildTableReference", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:tableExpr:TableExprPivot", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableExpr:TableExprPlaceholder", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableExpr:TableExprSubquery", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableExpr:TableExprTag", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableExpr:TableExprUnpivot", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableExpr:TableExprValues", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableFunctionExpr", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableIdentifier", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:templateString", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:throwStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:topClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tryCatchStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:unpivotColumn", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:unpivotColumnList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:valuesClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:valuesRow", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:varAssignment", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:varDecl", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:whereClause", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createStatement", + "parseHandler": "AstBuilder#build", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause" + ] + }, + { + "id": "alternative:whileStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:winFrameBound", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:winFrameClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:winFrameExtend:frameBetween", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:winFrameExtend:frameStart", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:winOrderByClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:winPartitionByClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:windowClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:windowExpr", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:withClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:withExpr:WithExprColumn", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:withExpr:WithExprSubquery", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:withExprColumnNameList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:withExprList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:withFillClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:withinGroupClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:alias", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:arrayJoinClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:block", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:catchBlock", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnAliases", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnExprList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnExprTupleOrSingle", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnExprValue", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnIdentifier", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnLambdaExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnTypeCastExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnTypeCastIdentifier", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnTypeExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnsReplaceItem", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnsReplaceList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:databaseIdentifier", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:declaration", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:emptyStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:enumValue", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:expr", + "kind": "parserRule", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "rule:exprStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:expression", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:floatingLiteral", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:forInStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:forStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:fromClause", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createTable", + "parseHandler": "AstBuilder#buildTableReference", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:fullTemplateString", + "kind": "parserRule", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "rule:funcStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:groupByClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:groupingSet", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:groupingSetList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:havingClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:hogqlxChildElement", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:hogqlxTagAttribute", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:hogqlxTagElement", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:hogqlxText", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:identifier", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:identifierList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:ifStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:implicitAlias", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:interpolateClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:interpolateExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:interval", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:joinConstraintClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:joinExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:joinOp", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:joinOpCross", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:keyword", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:keywordForAlias", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:keywordForImplicitAlias", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:keywordForTypeCast", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:kvPair", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:kvPairList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:limitAndOffsetClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:limitAndOffsetClauseOptional", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:limitByClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:limitExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:literal", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:nestedIdentifier", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:numberLiteral", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:offsetOnlyClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:orderByClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:orderExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:orderExprList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:pivotColumn", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:pivotColumnList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:placeholder", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:prewhereClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:program", + "kind": "parserRule", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "rule:projectionOrderByClause", + "kind": "parserRule", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "rule:qualifyClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:ratioExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:returnStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:sampleClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:select", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:selectColumnExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:selectColumnExprList", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:selectColumnExprListBeforeFrom", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:selectSetStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:selectStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:selectStmtWithParens", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:settingExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:settingExprList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:settingsClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:statement", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:string", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:stringContents", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:stringContentsFull", + "kind": "parserRule", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "rule:subsequentSelectSetClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:tableArgList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:tableExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:tableFunctionExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:tableIdentifier", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:templateString", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:throwStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:topClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:tryCatchStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:unpivotColumn", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:unpivotColumnList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:valuesClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:valuesRow", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:varAssignment", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:varDecl", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:whereClause", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createStatement", + "parseHandler": "AstBuilder#build", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause" + ] + }, + { + "id": "rule:whileStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:winFrameBound", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:winFrameClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:winFrameExtend", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:winOrderByClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:winPartitionByClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:windowClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:windowExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:withClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:withExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:withExprColumnNameList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:withExprList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:withFillClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:withinGroupClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ALL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:AND", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ANTI", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ANY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ARRAY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ARROW", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:AS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ASCENDING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ASOF", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ASTERISK", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:BACKQUOTE", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:BACKSLASH", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:BETWEEN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:BINARY_LITERAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:BOTH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:BY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:CASE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:CAST", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:CATCH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:COHORT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:COLLATE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:COLON", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:COLONEQUALS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:COLUMNS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:COMMA", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:CONCAT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:CROSS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:CUBE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:CURRENT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DASH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DATE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DAY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DECIMAL_LITERAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DESC", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DESCENDING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DISTINCT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DOLLAR", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:DOT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DOUBLECOLON", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ELSE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:END", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:EQ_DOUBLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:EQ_SINGLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ESCAPE_CHAR_COMMON", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:EXCEPT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:EXCLUDE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:EXTRACT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FILL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FILTER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FINAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FINALLY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FIRST", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FLOATING_LITERAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FOLLOWING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FOR", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FROM", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FULL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FULL_STRING_ESCAPE_TRIGGER", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:FULL_STRING_TEXT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:FUN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:GROUP", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:GROUPING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:GT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:GT_EQ", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:HASH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:HASH_COMMENT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:HAVING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:HEXADECIMAL_LITERAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:HOGQLX_TEXT_TEXT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:HOGQLX_TEXT_WS", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:HOUR", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ID", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:IDENTIFIER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:IF", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:IGNORE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ILIKE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:IN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:INCLUDE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:INF", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:INNER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:INTERPOLATE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:INTERSECT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:INTERVAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:IREGEX_DOUBLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:IREGEX_SINGLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:IS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:JOIN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:KEY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LAMBDA", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LAST", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LBRACE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LBRACKET", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LEADING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LEFT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LET", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LIKE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LIMIT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LOCAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LPAREN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LT_EQ", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LT_SLASH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:MALFORMED_BINARY_LITERAL", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:MATERIALIZED", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:MINUTE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:MONTH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:MULTI_LINE_COMMENT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:NAME", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NAN_SQL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NATURAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NOT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NOT_EQ", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NOT_IREGEX", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NOT_REGEX", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NULLISH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NULLS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NULL_PROPERTY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NULL_SAFE_EQ", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NULL_SQL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:OCTAL_LITERAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:OCTAL_PREFIX_LITERAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:OFFSET", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ON", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:OR", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ORDER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:OUTER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:OVER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:PARTITION", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:PERCENT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:PIVOT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:PLUS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:POSITIONAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:PRECEDING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:PREWHERE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:QUALIFY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:QUARTER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:QUERY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:QUOTED_IDENTIFIER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:QUOTE_DOUBLE", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:QUOTE_SINGLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:QUOTE_SINGLE_TEMPLATE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:QUOTE_SINGLE_TEMPLATE_FULL", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:RANGE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:RBRACE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:RBRACKET", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:RECURSIVE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:REGEX_DOUBLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:REGEX_SINGLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:REPLACE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:RETURN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:RIGHT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ROLLUP", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ROW", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ROWS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:RPAREN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SAMPLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SECOND", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SELECT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SEMI", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SEMICOLON", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SETS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SETTINGS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SINGLE_LINE_COMMENT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:SLASH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SLASH_GT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:STEP", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:STRING_ESCAPE_TRIGGER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:STRING_LITERAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:STRING_TEXT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SUBSTRING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TAGC_MULTI_LINE_COMMENT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:TAGC_SINGLE_LINE_COMMENT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:TAGC_WS", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:TAG_MULTI_LINE_COMMENT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:TAG_SINGLE_LINE_COMMENT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:TAG_WS", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:THEN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:THROW", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TIES", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TIME", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TIMESTAMP", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TO", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TOP", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TOTALS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TRAILING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TRIM", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TRUNCATE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TRY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TRY_CAST", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:UNBOUNDED", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:UNDERSCORE", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:UNEXPECTED_CHARACTER", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:UNION", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:UNPIVOT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:USING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:VALUES", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:WEEK", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:WHEN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:WHERE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:WHILE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:WHITESPACE", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:WINDOW", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:WITH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:WITHIN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:YEAR", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ZONE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + } + ], + "grammarFeatureManifestSha256": "fbf7236643b43d929e9ef1c8fd01a04b746736dca2f965cc2e78ad62f1a98b10", + "grammarSha256": "c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242", + "languageVersion": "1.0.0", + "schemaVersion": 1, + "unresolvedGrammarAlternativeRules": [ + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "alias" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "columnExprTupleOrSingle" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "columnIdentifier" + }, + { + "alternativeCount": 4, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "columnTypeCastIdentifier" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "declaration" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "floatingLiteral" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "hogqlxChildElement" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "hogqlxTagAttribute" + }, + { + "alternativeCount": 4, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "identifier" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "implicitAlias" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "joinConstraintClause" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "joinOpCross" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "limitAndOffsetClause" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "limitAndOffsetClauseOptional" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "literal" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "ratioExpr" + }, + { + "alternativeCount": 4, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "selectStmtWithParens" + }, + { + "alternativeCount": 11, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "statement" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "string" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "stringContents" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": false, + "rule": "stringContentsFull" + } + ] +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java new file mode 100644 index 000000000000..176948afb20d --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java @@ -0,0 +1,207 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.airlift.json.JsonCodec; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.fail; +import static org.junit.jupiter.api.DynamicTest.dynamicTest; + +public class TestHogQlCompatibilityCorpus +{ + private static final String RESOURCE_ROOT = "/io/trino/hogql/parser/language/1.0.0/"; + private static final JsonCodec CORPUS_MANIFEST_CODEC = jsonCodec(CorpusManifest.class); + private static final JsonCodec ORACLE_CASE_CODEC = jsonCodec(OracleCase.class); + private static final JsonCodec OVERRIDES_CODEC = jsonCodec(CompatibilityOverrides.class); + + private final HogQlParser parser = new HogQlParser(); + + @Test + public void testOracleMetadataMatchesThePublishedCorpus() + throws IOException + { + CorpusManifest manifest = loadCorpusManifest(); + Map oracleById = loadOracleCases().stream() + .collect(toMap(OracleCase::id, oracleCase -> oracleCase)); + + assertThat(oracleById).hasSize(manifest.cases().size()); + for (CorpusCase corpusCase : manifest.cases()) { + OracleCase oracleCase = oracleById.remove(corpusCase.id()); + assertThat(oracleCase).isNotNull(); + assertThat(oracleCase.source()).isEqualTo(corpusCase.source()); + assertThat(oracleCase.entryPoint()).isEqualTo(corpusCase.entryPoint()); + assertThat(oracleCase.accepted()).isEqualTo(corpusCase.accepted()); + assertThat(oracleCase.languageVersion()).isEqualTo(HogQlLanguageContract.current().languageVersion()); + assertThat(oracleCase.grammarSha256()).isEqualTo(HogQlLanguageContract.current().grammarSha256()); + } + assertThat(oracleById).isEmpty(); + } + + @TestFactory + public Stream testTrinoDispositionForEveryCorpusCase() + throws IOException + { + CorpusManifest manifest = loadCorpusManifest(); + Map dispositionById = loadCompatibilityOverrides().corpusCases().stream() + .collect(toMap(CorpusDisposition::id, disposition -> disposition)); + + assertThat(dispositionById.keySet()) + .containsExactlyInAnyOrderElementsOf(manifest.cases().stream().map(CorpusCase::id).toList()); + return manifest.cases().stream() + .map(corpusCase -> dynamicTest(corpusCase.id(), () -> assertDisposition(corpusCase, dispositionById.get(corpusCase.id())))); + } + + private void assertDisposition(CorpusCase corpusCase, CorpusDisposition disposition) + { + String query = switch (corpusCase.entryPoint()) { + case "expression" -> "SELECT " + corpusCase.source(); + case "query" -> corpusCase.source(); + default -> throw new IllegalArgumentException("unknown corpus entry point: " + corpusCase.entryPoint()); + }; + switch (disposition.status()) { + case "supported" -> assertThatCode(() -> parser.parseStatement(query)).doesNotThrowAnyException(); + case "explicitCurrentError" -> assertThatThrownBy(() -> parser.parseStatement(query)) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("HogQL feature is not lowered yet"); + case "canonicalRejection" -> assertThatThrownBy(() -> parser.parseStatement(query)) + .isInstanceOf(HogQlParsingException.class) + .hasMessageStartingWith("line "); + default -> throw new IllegalArgumentException("unknown corpus disposition: " + disposition.status()); + } + } + + private static CorpusManifest loadCorpusManifest() + throws IOException + { + return CORPUS_MANIFEST_CODEC.fromJson(readResource("corpus/expr_select_cases.json")); + } + + private static List loadOracleCases() + throws IOException + { + return Arrays.stream(readResource("corpus/expr_select_cpp_oracle.jsonl").split("\\R")) + .filter(line -> !line.isBlank()) + .map(ORACLE_CASE_CODEC::fromJson) + .toList(); + } + + private static CompatibilityOverrides loadCompatibilityOverrides() + throws IOException + { + return OVERRIDES_CODEC.fromJson(readResource("trino-compatibility-overrides.json")); + } + + private static String readResource(String relativePath) + throws IOException + { + String path = RESOURCE_ROOT + relativePath; + try (InputStream input = TestHogQlCompatibilityCorpus.class.getResourceAsStream(path)) { + if (input == null) { + fail("missing test resource: " + path); + } + return new String(input.readAllBytes(), StandardCharsets.UTF_8); + } + } + + public record CorpusManifest(List cases) + { + @JsonCreator + public CorpusManifest(@JsonProperty("cases") List cases) + { + this.cases = List.copyOf(requireNonNull(cases, "cases is null")); + } + } + + public record CorpusCase(String id, String source, String entryPoint, boolean accepted) + { + @JsonCreator + public CorpusCase( + @JsonProperty("id") String id, + @JsonProperty("source") String source, + @JsonProperty("entryPoint") String entryPoint, + @JsonProperty("accepted") boolean accepted) + { + this.id = requireNonNull(id, "id is null"); + this.source = requireNonNull(source, "source is null"); + this.entryPoint = requireNonNull(entryPoint, "entryPoint is null"); + this.accepted = accepted; + } + } + + public record OracleCase( + String id, + String source, + String entryPoint, + boolean accepted, + HogQlLanguageVersion languageVersion, + String grammarSha256) + { + @JsonCreator + public OracleCase( + @JsonProperty("id") String id, + @JsonProperty("source") String source, + @JsonProperty("entryPoint") String entryPoint, + @JsonProperty("accepted") boolean accepted, + @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, + @JsonProperty("grammarSha256") String grammarSha256) + { + this.id = requireNonNull(id, "id is null"); + this.source = requireNonNull(source, "source is null"); + this.entryPoint = requireNonNull(entryPoint, "entryPoint is null"); + this.accepted = accepted; + this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); + } + } + + public record CompatibilityOverrides(List corpusCases) + { + @JsonCreator + public CompatibilityOverrides(@JsonProperty("corpusCases") List corpusCases) + { + this.corpusCases = List.copyOf(requireNonNull(corpusCases, "corpusCases is null")); + } + } + + public record CorpusDisposition(String id, String status) + { + @JsonCreator + public CorpusDisposition( + @JsonProperty("id") String id, + @JsonProperty("status") String status) + { + this.id = requireNonNull(id, "id is null"); + this.status = requireNonNull(status, "status is null"); + } + } +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityManifest.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityManifest.java new file mode 100644 index 000000000000..7f68ace2a76d --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityManifest.java @@ -0,0 +1,110 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.airlift.json.JsonCodec; +import io.trino.hogql.parser.HogQlCompatibilityManifest.Status; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +public class TestHogQlCompatibilityManifest +{ + private static final String OVERRIDES_RESOURCE = "/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json"; + private static final JsonCodec OVERRIDES_CODEC = jsonCodec(CompatibilityOverrides.class); + + @Test + public void testAccountsForEveryPublishedGrammarFeature() + { + HogQlCompatibilityManifest manifest = HogQlCompatibilityManifest.current(); + + assertThat(manifest.schemaVersion()).isEqualTo(1); + assertThat(manifest.languageVersion()).isEqualTo(HogQlLanguageContract.current().languageVersion()); + assertThat(manifest.features()).hasSize(518); + assertThat(manifest.unresolvedGrammarAlternativeRules()).hasSize(21); + assertThat(manifest.features()) + .allSatisfy(feature -> assertThat(feature.testCaseIds()).isNotEmpty()); + } + + @Test + public void testUsesOnlyExplicitDispositionsForUnsupportedFeatures() + { + HogQlCompatibilityManifest manifest = HogQlCompatibilityManifest.current(); + + assertThat(manifest.features()) + .filteredOn(feature -> !feature.queryReachable()) + .allSatisfy(feature -> assertThat(feature.status()).isEqualTo(Status.NOT_QUERY_LANGUAGE)); + assertThat(manifest.features()) + .filteredOn(feature -> feature.status() == Status.EXPLICIT_CURRENT_ERROR) + .allSatisfy(feature -> assertThat(feature.loweringHandler()).isEqualTo("UnsupportedFeature")); + } + + @Test + public void testSupportedFeaturesMatchTheSparseOverrides() + throws IOException + { + CompatibilityOverrides overrides; + try (InputStream input = TestHogQlCompatibilityManifest.class.getResourceAsStream(OVERRIDES_RESOURCE)) { + if (input == null) { + fail("missing test resource: " + OVERRIDES_RESOURCE); + } + overrides = OVERRIDES_CODEC.fromJson(new String(input.readAllBytes(), StandardCharsets.UTF_8)); + } + + Map overridesById = overrides.features().stream() + .collect(toMap(CompatibilityOverride::id, feature -> feature)); + assertThat(HogQlCompatibilityManifest.current().features()) + .filteredOn(feature -> feature.status() == Status.SUPPORTED) + .allSatisfy(feature -> assertThat(overridesById.remove(feature.id())) + .isEqualTo(new CompatibilityOverride(feature.id(), feature.parseHandler(), feature.loweringHandler(), feature.testCaseIds()))); + assertThat(overridesById).isEmpty(); + } + + public record CompatibilityOverrides(List features) + { + @JsonCreator + public CompatibilityOverrides(@JsonProperty("features") List features) + { + this.features = List.copyOf(requireNonNull(features, "features is null")); + } + } + + public record CompatibilityOverride(String id, String parseHandler, String loweringHandler, List testCaseIds) + { + @JsonCreator + public CompatibilityOverride( + @JsonProperty("id") String id, + @JsonProperty("parseHandler") String parseHandler, + @JsonProperty("loweringHandler") String loweringHandler, + @JsonProperty("testCaseIds") List testCaseIds) + { + this.id = requireNonNull(id, "id is null"); + this.parseHandler = requireNonNull(parseHandler, "parseHandler is null"); + this.loweringHandler = requireNonNull(loweringHandler, "loweringHandler is null"); + this.testCaseIds = List.copyOf(requireNonNull(testCaseIds, "testCaseIds is null")); + } + } +} From fb5078225c328fc4864c9a1c8024f81d142b1adb Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 07:48:01 -0400 Subject: [PATCH 007/136] feat(hogql): lower collection and predicate expressions --- .../trino/hogql/compiler/TrinoAstFactory.java | 57 +++++++++++++++ .../hogql/compiler/TestHogQlCompiler.java | 41 +++++++++++ .../io/trino/hogql/parser/HogQlParser.java | 61 +++++++++++++++- .../trino/hogql/parser/tree/HogQlQuery.java | 69 ++++++++++++++++++- .../1.0.0/trino-compatibility-overrides.json | 64 +++++++++++++++-- .../language/1.0.0/trino-compatibility.json | 54 ++++++++------- 6 files changed, 314 insertions(+), 32 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 8e52e7de7d8c..0863ea59f4fd 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -14,24 +14,34 @@ package io.trino.hogql.compiler; import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.Star; import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; import io.trino.sql.tree.AllColumns; import io.trino.sql.tree.ArithmeticBinaryExpression; import io.trino.sql.tree.ArithmeticUnaryExpression; +import io.trino.sql.tree.Array; +import io.trino.sql.tree.BetweenPredicate; import io.trino.sql.tree.BooleanLiteral; import io.trino.sql.tree.ComparisonPredicate; import io.trino.sql.tree.DereferenceExpression; import io.trino.sql.tree.Expression; +import io.trino.sql.tree.InListExpression; +import io.trino.sql.tree.InPredicate; +import io.trino.sql.tree.IsNullPredicate; import io.trino.sql.tree.LogicalExpression; import io.trino.sql.tree.LongLiteral; import io.trino.sql.tree.NodeLocation; @@ -41,6 +51,7 @@ import io.trino.sql.tree.QualifiedName; import io.trino.sql.tree.Query; import io.trino.sql.tree.QuerySpecification; +import io.trino.sql.tree.Row; import io.trino.sql.tree.Select; import io.trino.sql.tree.SelectItem; import io.trino.sql.tree.SingleColumn; @@ -96,6 +107,12 @@ private static SelectItem createSelectItem(Projection projection) private static Expression createExpression(HogQlQuery.Expression expression) { return switch (expression) { + case ArrayExpression array -> new Array( + location(array.span()), + array.values().stream() + .map(TrinoAstFactory::createExpression) + .toList()); + case BetweenExpression between -> createBetweenExpression(between); case BinaryExpression binary -> createBinaryExpression(binary); case ColumnReference reference -> createColumnReference(reference); case FunctionCall function -> new io.trino.sql.tree.FunctionCall( @@ -104,7 +121,17 @@ private static Expression createExpression(HogQlQuery.Expression expression) function.arguments().stream() .map(TrinoAstFactory::createExpression) .toList()); + case InExpression in -> createInExpression(in); + case IsNullExpression isNull -> new Predicated( + location(isNull.predicateSpan()), + createExpression(isNull.value()), + new IsNullPredicate(location(isNull.predicateSpan()), isNull.negated())); case Literal literal -> createLiteral(literal); + case TupleExpression tuple -> new Row( + location(tuple.span()), + tuple.values().stream() + .map(value -> new Row.Field(location(value.span()), Optional.empty(), createExpression(value))) + .toList()); case UnaryExpression unary -> switch (unary.operator()) { case NEGATE -> ArithmeticUnaryExpression.negative(location(unary.span()), createExpression(unary.operand())); case NOT -> new NotExpression(location(unary.span()), createExpression(unary.operand())); @@ -113,6 +140,36 @@ private static Expression createExpression(HogQlQuery.Expression expression) }; } + private static Expression createBetweenExpression(BetweenExpression between) + { + NodeLocation location = location(between.predicateSpan()); + return new Predicated( + location, + createExpression(between.value()), + new BetweenPredicate( + location, + between.negated(), + Optional.empty(), + createExpression(between.min()), + createExpression(between.max()))); + } + + private static Expression createInExpression(InExpression in) + { + NodeLocation location = location(in.predicateSpan()); + return new Predicated( + location, + createExpression(in.value()), + new InPredicate( + location, + in.negated(), + new InListExpression( + location, + in.values().stream() + .map(TrinoAstFactory::createExpression) + .toList()))); + } + private static Expression createBinaryExpression(BinaryExpression binary) { NodeLocation location = location(binary.span()); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index eb804ca57c26..cebf0a11ae5a 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -20,6 +20,7 @@ import io.trino.sql.tree.Identifier; import io.trino.sql.tree.Node; import io.trino.sql.tree.NodeLocation; +import io.trino.sql.tree.Predicated; import io.trino.sql.tree.Query; import io.trino.sql.tree.QuerySpecification; import io.trino.sql.tree.SingleColumn; @@ -27,6 +28,7 @@ import io.trino.sql.tree.Table; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; import java.util.ArrayDeque; @@ -61,6 +63,45 @@ public void testLowersToEquivalentStockTrinoAst(String hogql) assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); } + @ParameterizedTest + @CsvSource(delimiter = '|', textBlock = """ + SELECT [1, 2, 3] | SELECT ARRAY[1, 2, 3] + SELECT ARRAY[] | SELECT ARRAY[] + SELECT (1, 'example') | SELECT (1, 'example') + SELECT (1,) | SELECT ROW(1) + SELECT value BETWEEN 1 AND 10 | SELECT value BETWEEN 1 AND 10 + SELECT value NOT BETWEEN 1 AND 10| SELECT value NOT BETWEEN 1 AND 10 + SELECT value IS NULL | SELECT value IS NULL + SELECT value IS NOT NULL | SELECT value IS NOT NULL + SELECT value IN (1, 2, 3) | SELECT value IN (1, 2, 3) + SELECT value IN (1) | SELECT value IN (1) + SELECT value NOT IN (1, 2, 3) | SELECT value NOT IN (1, 2, 3) + """) + public void testLowersCollectionAndPredicateExpressions(String hogql, String trinoSql) + { + assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(trinoSql)); + } + + @ParameterizedTest + @ValueSource(strings = { + "SELECT value BETWEEN 1 AND 10", + "SELECT value NOT BETWEEN 1 AND 10", + "SELECT value IS NULL", + "SELECT value IS NOT NULL", + "SELECT value IN (1, 2, 3)", + "SELECT value NOT IN (1, 2, 3)", + }) + public void testPreservesPredicateSourceLocations(String hogql) + { + Query query = (Query) compiler.compile(hogql); + QuerySpecification querySpecification = (QuerySpecification) query.getQueryBody(); + SingleColumn column = (SingleColumn) querySpecification.getSelect().getSelectItems().getFirst(); + Predicated predicated = (Predicated) column.getExpression(); + + assertThat(predicated.getLocation()).contains(new NodeLocation(1, 14)); + assertThat(predicated.getPredicate().getLocation()).contains(new NodeLocation(1, 14)); + } + @Test public void testPreservesQuotedIdentifiersAndSourceLocations() { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index e53d0ba36ebe..79085745602b 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -16,6 +16,8 @@ import io.trino.hogql.parser.canonical.HogQLLexer; import io.trino.hogql.parser.canonical.HogQLParser; import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; @@ -23,11 +25,14 @@ import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.Star; import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; import org.antlr.v4.runtime.ANTLRErrorListener; import org.antlr.v4.runtime.BailErrorStrategy; @@ -267,6 +272,13 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) if (context instanceof HogQLParser.ColumnExprParensContext parens) { return buildExpression(parens.columnExpr()); } + if (context instanceof HogQLParser.ColumnExprArrayContext array) { + List values = array.columnExprList() == null ? List.of() : buildExpressions(array.columnExprList()); + return new ArrayExpression(values, sourceSpan(array)); + } + if (context instanceof HogQLParser.ColumnExprTupleContext tuple) { + return new TupleExpression(buildExpressions(tuple.columnExprList()), sourceSpan(tuple)); + } if (context instanceof HogQLParser.ColumnExprNegateContext negate) { return new UnaryExpression(NEGATE, buildExpression(negate.columnExprValue()), sourceSpan(negate)); } @@ -291,6 +303,14 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) return binary(operator, binary.left, binary.right, binary); } if (context instanceof HogQLParser.ColumnExprPrecedence3Context binary) { + if (binary.IN() != null && binary.COHORT() == null) { + return new InExpression( + buildExpression(binary.left), + buildInValues(binary.right), + binary.NOT() != null, + sourceSpan(binary.NOT() == null ? binary.IN().getSymbol() : binary.NOT().getSymbol(), binary.getStop()), + sourceSpan(binary)); + } if (binary.operator == null) { throw unsupported(binary, "comparison operator"); } @@ -305,12 +325,46 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) }; return binary(operator, binary.left, binary.right, binary); } + if (context instanceof HogQLParser.ColumnExprIsNullContext isNull) { + return new IsNullExpression( + buildExpression(isNull.columnExprValue()), + isNull.NOT() != null, + sourceSpan(isNull.IS().getSymbol(), isNull.getStop()), + sourceSpan(isNull)); + } + if (context instanceof HogQLParser.ColumnExprBetweenContext between) { + return new BetweenExpression( + buildExpression(between.columnExprValue(0)), + buildExpression(between.columnExprValue(1)), + buildExpression(between.columnExprValue(2)), + between.NOT() != null, + sourceSpan(between.NOT() == null ? between.BETWEEN().getSymbol() : between.NOT().getSymbol(), between.getStop()), + sourceSpan(between)); + } if (context instanceof HogQLParser.ColumnExprFunctionContext function) { return buildFunction(function); } throw unsupported(context, "expression " + context.getClass().getSimpleName()); } + private List buildExpressions(HogQLParser.ColumnExprListContext context) + { + return context.columnExpr().stream() + .map(this::buildExpression) + .toList(); + } + + private List buildInValues(HogQLParser.ColumnExprValueContext context) + { + if (context instanceof HogQLParser.ColumnExprTupleContext tuple) { + return buildExpressions(tuple.columnExprList()); + } + if (context instanceof HogQLParser.ColumnExprParensContext parens) { + return List.of(buildExpression(parens.columnExpr())); + } + throw unsupported(context, "IN value list"); + } + private BinaryExpression binary(BinaryOperator operator, HogQLParser.ColumnExprContext left, HogQLParser.ColumnExprContext right, ParserRuleContext context) { return new BinaryExpression(operator, buildExpression(left), buildExpression(right), sourceSpan(context)); @@ -414,8 +468,11 @@ private Identifier buildIdentifier(String text, ParserRuleContext context) private SourceSpan sourceSpan(ParserRuleContext context) { - Token start = context.getStart(); - Token stop = context.getStop(); + return sourceSpan(context.getStart(), context.getStop()); + } + + private SourceSpan sourceSpan(Token start, Token stop) + { int startOffset = start.getStartIndex(); int endOffset = stop.getStopIndex() + 1; return new SourceSpan( diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index 89b8b51d3043..a8a5309d0c46 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -60,15 +60,82 @@ public SourceSpan span() } public sealed interface Expression - permits BinaryExpression, + permits ArrayExpression, + BetweenExpression, + BinaryExpression, ColumnReference, FunctionCall, + InExpression, + IsNullExpression, Literal, + TupleExpression, UnaryExpression { SourceSpan span(); } + public record ArrayExpression(List values, SourceSpan span) + implements Expression + { + public ArrayExpression + { + values = List.copyOf(requireNonNull(values, "values is null")); + span = requireNonNull(span, "span is null"); + } + } + + public record TupleExpression(List values, SourceSpan span) + implements Expression + { + public TupleExpression + { + values = List.copyOf(requireNonNull(values, "values is null")); + if (values.isEmpty()) { + throw new IllegalArgumentException("values is empty"); + } + span = requireNonNull(span, "span is null"); + } + } + + public record BetweenExpression(Expression value, Expression min, Expression max, boolean negated, SourceSpan predicateSpan, SourceSpan span) + implements Expression + { + public BetweenExpression + { + value = requireNonNull(value, "value is null"); + min = requireNonNull(min, "min is null"); + max = requireNonNull(max, "max is null"); + predicateSpan = requireNonNull(predicateSpan, "predicateSpan is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record InExpression(Expression value, List values, boolean negated, SourceSpan predicateSpan, SourceSpan span) + implements Expression + { + public InExpression + { + value = requireNonNull(value, "value is null"); + values = List.copyOf(requireNonNull(values, "values is null")); + if (values.isEmpty()) { + throw new IllegalArgumentException("values is empty"); + } + predicateSpan = requireNonNull(predicateSpan, "predicateSpan is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record IsNullExpression(Expression value, boolean negated, SourceSpan predicateSpan, SourceSpan span) + implements Expression + { + public IsNullExpression + { + value = requireNonNull(value, "value is null"); + predicateSpan = requireNonNull(predicateSpan, "predicateSpan is null"); + span = requireNonNull(span, "span is null"); + } + } + public record UnaryExpression(UnaryOperator operator, Expression operand, SourceSpan span) implements Expression { diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json index b6ad21fefacd..7a928c2c50ba 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json @@ -6,11 +6,11 @@ }, { "id": "expr.accept.array", - "status": "explicitCurrentError" + "status": "supported" }, { "id": "expr.accept.between", - "status": "explicitCurrentError" + "status": "supported" }, { "id": "expr.accept.boolean", @@ -34,7 +34,7 @@ }, { "id": "expr.accept.in-list", - "status": "explicitCurrentError" + "status": "supported" }, { "id": "expr.accept.literal-boolean", @@ -54,11 +54,11 @@ }, { "id": "expr.accept.null-predicate", - "status": "explicitCurrentError" + "status": "supported" }, { "id": "expr.accept.tuple", - "status": "explicitCurrentError" + "status": "supported" }, { "id": "expr.reject.incomplete-binary", @@ -139,6 +139,42 @@ "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" ] }, + { + "id": "alternative:columnExprList", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpressions", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.array", + "TestHogQlCompatibilityCorpus#expr.accept.tuple" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprArray", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.array", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprBetween", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.between", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIsNull", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.null-predicate", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, { "id": "alternative:columnExprValue:ColumnExprNegate", "loweringHandler": "TrinoAstFactory#createExpression", @@ -173,6 +209,15 @@ "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" ] }, + { + "id": "alternative:columnExprValue:ColumnExprTuple", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.tuple", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, { "id": "alternative:databaseIdentifier", "loweringHandler": "TrinoAstFactory#createIdentifier", @@ -245,6 +290,15 @@ "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause" ] }, + { + "id": "rule:columnExprList", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpressions", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.array", + "TestHogQlCompatibilityCorpus#expr.accept.tuple" + ] + }, { "id": "rule:databaseIdentifier", "loweringHandler": "TrinoAstFactory#createIdentifier", diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json index 7f6080774d71..2fa756af98e7 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json @@ -103,23 +103,25 @@ { "id": "alternative:columnExprList", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpressions", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompatibilityCorpus#expr.accept.array", + "TestHogQlCompatibilityCorpus#expr.accept.tuple" ] }, { "id": "alternative:columnExprValue:ColumnExprArray", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompatibilityCorpus#expr.accept.array", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" ] }, { @@ -158,12 +160,13 @@ { "id": "alternative:columnExprValue:ColumnExprBetween", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompatibilityCorpus#expr.accept.between", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" ] }, { @@ -433,12 +436,13 @@ { "id": "alternative:columnExprValue:ColumnExprIsNull", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompatibilityCorpus#expr.accept.null-predicate", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" ] }, { @@ -721,12 +725,13 @@ { "id": "alternative:columnExprValue:ColumnExprTuple", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompatibilityCorpus#expr.accept.tuple", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" ] }, { @@ -2184,12 +2189,13 @@ { "id": "rule:columnExprList", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpressions", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompatibilityCorpus#expr.accept.array", + "TestHogQlCompatibilityCorpus#expr.accept.tuple" ] }, { From dedcde812444c21491fc7b5616f70d635ad9a733 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 07:48:24 -0400 Subject: [PATCH 008/136] feat(hogql): lower case and cast expressions --- .../trino/hogql/compiler/TrinoAstFactory.java | 28 +++++++++++++ .../hogql/compiler/TestHogQlCompiler.java | 12 ++++++ .../io/trino/hogql/parser/HogQlParser.java | 42 +++++++++++++++++++ .../trino/hogql/parser/tree/HogQlQuery.java | 38 +++++++++++++++++ .../1.0.0/trino-compatibility-overrides.json | 39 ++++++++++++++++- .../language/1.0.0/trino-compatibility.json | 35 +++++++++------- 6 files changed, 176 insertions(+), 18 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 0863ea59f4fd..5b0655aba513 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -17,6 +17,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; @@ -36,9 +38,11 @@ import io.trino.sql.tree.Array; import io.trino.sql.tree.BetweenPredicate; import io.trino.sql.tree.BooleanLiteral; +import io.trino.sql.tree.Cast; import io.trino.sql.tree.ComparisonPredicate; import io.trino.sql.tree.DereferenceExpression; import io.trino.sql.tree.Expression; +import io.trino.sql.tree.GenericDataType; import io.trino.sql.tree.InListExpression; import io.trino.sql.tree.InPredicate; import io.trino.sql.tree.IsNullPredicate; @@ -52,12 +56,15 @@ import io.trino.sql.tree.Query; import io.trino.sql.tree.QuerySpecification; import io.trino.sql.tree.Row; +import io.trino.sql.tree.SearchedCaseExpression; import io.trino.sql.tree.Select; import io.trino.sql.tree.SelectItem; +import io.trino.sql.tree.SimpleCaseExpression; import io.trino.sql.tree.SingleColumn; import io.trino.sql.tree.Statement; import io.trino.sql.tree.StringLiteral; import io.trino.sql.tree.Table; +import io.trino.sql.tree.WhenClause; import java.util.List; import java.util.Optional; @@ -114,6 +121,12 @@ private static Expression createExpression(HogQlQuery.Expression expression) .toList()); case BetweenExpression between -> createBetweenExpression(between); case BinaryExpression binary -> createBinaryExpression(binary); + case CaseExpression caseExpression -> createCaseExpression(caseExpression); + case CastExpression cast -> new Cast( + location(cast.span()), + createExpression(cast.value()), + new GenericDataType(location(cast.type().span()), createIdentifier(cast.type()), List.of()), + cast.safe()); case ColumnReference reference -> createColumnReference(reference); case FunctionCall function -> new io.trino.sql.tree.FunctionCall( location(function.span()), @@ -140,6 +153,21 @@ private static Expression createExpression(HogQlQuery.Expression expression) }; } + private static Expression createCaseExpression(CaseExpression caseExpression) + { + NodeLocation location = location(caseExpression.span()); + List whenClauses = caseExpression.whenClauses().stream() + .map(when -> new WhenClause( + location(when.span()), + createExpression(when.operand()), + createExpression(when.result()))) + .toList(); + Optional defaultValue = caseExpression.defaultValue().map(TrinoAstFactory::createExpression); + return caseExpression.operand() + .map(operand -> new SimpleCaseExpression(location, createExpression(operand), whenClauses, defaultValue)) + .orElseGet(() -> new SearchedCaseExpression(location, whenClauses, defaultValue)); + } + private static Expression createBetweenExpression(BetweenExpression between) { NodeLocation location = location(between.predicateSpan()); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index cebf0a11ae5a..cc574429bc18 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -82,6 +82,18 @@ public void testLowersCollectionAndPredicateExpressions(String hogql, String tri assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(trinoSql)); } + @ParameterizedTest + @ValueSource(strings = { + "SELECT CASE WHEN enabled THEN 1 ELSE 0 END", + "SELECT CASE value WHEN 1 THEN 'one' WHEN 2 THEN 'two' ELSE 'other' END", + "SELECT CAST(value AS INT)", + "SELECT TRY_CAST(value AS VARCHAR)", + }) + public void testLowersCaseAndCastExpressions(String hogql) + { + assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(hogql)); + } + @ParameterizedTest @ValueSource(strings = { "SELECT value BETWEEN 1 AND 10", diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 79085745602b..c75fd90fa1c2 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -20,6 +20,9 @@ import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseWhen; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; import io.trino.hogql.parser.tree.HogQlQuery.Expression; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; @@ -266,6 +269,15 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) if (context instanceof HogQLParser.ColumnExprLiteralContext literal) { return buildLiteral(literal.literal()); } + if (context instanceof HogQLParser.ColumnExprCaseContext caseExpression) { + return buildCaseExpression(caseExpression); + } + if (context instanceof HogQLParser.ColumnExprCastContext cast) { + return new CastExpression(buildExpression(cast.columnExpr()), buildSimpleType(cast.columnTypeExpr()), false, sourceSpan(cast)); + } + if (context instanceof HogQLParser.ColumnExprTryCastContext cast) { + return new CastExpression(buildExpression(cast.columnExpr()), buildSimpleType(cast.columnTypeExpr()), true, sourceSpan(cast)); + } if (context instanceof HogQLParser.ColumnExprIdentifierContext identifier) { return buildColumnReference(identifier.columnIdentifier()); } @@ -347,6 +359,36 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) throw unsupported(context, "expression " + context.getClass().getSimpleName()); } + private CaseExpression buildCaseExpression(HogQLParser.ColumnExprCaseContext context) + { + List expressions = context.columnExpr(); + int index = 0; + Optional operand = Optional.empty(); + if (context.caseExpr != null) { + operand = Optional.of(buildExpression(expressions.get(index++))); + } + + List whenClauses = new ArrayList<>(); + for (int whenIndex = 0; whenIndex < context.WHEN().size(); whenIndex++) { + HogQLParser.ColumnExprContext when = expressions.get(index++); + HogQLParser.ColumnExprContext result = expressions.get(index++); + whenClauses.add(new CaseWhen( + buildExpression(when), + buildExpression(result), + sourceSpan(context.WHEN(whenIndex).getSymbol(), result.getStop()))); + } + Optional defaultValue = context.ELSE() == null ? Optional.empty() : Optional.of(buildExpression(expressions.get(index))); + return new CaseExpression(operand, whenClauses, defaultValue, sourceSpan(context)); + } + + private Identifier buildSimpleType(HogQLParser.ColumnTypeExprContext context) + { + if (context instanceof HogQLParser.ColumnTypeExprSimpleContext simple) { + return buildIdentifier(simple.identifier()); + } + throw unsupported(context, "complex cast type"); + } + private List buildExpressions(HogQLParser.ColumnExprListContext context) { return context.columnExpr().stream() diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index a8a5309d0c46..1d03947d2827 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -63,6 +63,8 @@ public sealed interface Expression permits ArrayExpression, BetweenExpression, BinaryExpression, + CaseExpression, + CastExpression, ColumnReference, FunctionCall, InExpression, @@ -74,6 +76,42 @@ public sealed interface Expression SourceSpan span(); } + public record CaseExpression(Optional operand, List whenClauses, Optional defaultValue, SourceSpan span) + implements Expression + { + public CaseExpression + { + operand = requireNonNull(operand, "operand is null"); + whenClauses = List.copyOf(requireNonNull(whenClauses, "whenClauses is null")); + if (whenClauses.isEmpty()) { + throw new IllegalArgumentException("whenClauses is empty"); + } + defaultValue = requireNonNull(defaultValue, "defaultValue is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record CaseWhen(Expression operand, Expression result, SourceSpan span) + { + public CaseWhen + { + operand = requireNonNull(operand, "operand is null"); + result = requireNonNull(result, "result is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record CastExpression(Expression value, Identifier type, boolean safe, SourceSpan span) + implements Expression + { + public CastExpression + { + value = requireNonNull(value, "value is null"); + type = requireNonNull(type, "type is null"); + span = requireNonNull(span, "span is null"); + } + } + public record ArrayExpression(List values, SourceSpan span) implements Expression { diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json index 7a928c2c50ba..9ca9de75db89 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json @@ -18,11 +18,11 @@ }, { "id": "expr.accept.case", - "status": "explicitCurrentError" + "status": "supported" }, { "id": "expr.accept.cast", - "status": "explicitCurrentError" + "status": "supported" }, { "id": "expr.accept.dotted-field", @@ -166,6 +166,24 @@ "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" ] }, + { + "id": "alternative:columnExprValue:ColumnExprCase", + "loweringHandler": "TrinoAstFactory#createCaseExpression", + "parseHandler": "AstBuilder#buildCaseExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.case", + "TestHogQlCompiler#testLowersCaseAndCastExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprCast", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.cast", + "TestHogQlCompiler#testLowersCaseAndCastExpressions" + ] + }, { "id": "alternative:columnExprValue:ColumnExprIsNull", "loweringHandler": "TrinoAstFactory#createExpression", @@ -209,6 +227,14 @@ "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" ] }, + { + "id": "alternative:columnExprValue:ColumnExprTryCast", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCaseAndCastExpressions" + ] + }, { "id": "alternative:columnExprValue:ColumnExprTuple", "loweringHandler": "TrinoAstFactory#createExpression", @@ -218,6 +244,15 @@ "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" ] }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprSimple", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildSimpleType", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.cast", + "TestHogQlCompiler#testLowersCaseAndCastExpressions" + ] + }, { "id": "alternative:databaseIdentifier", "loweringHandler": "TrinoAstFactory#createIdentifier", diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json index 2fa756af98e7..0130cf2876c5 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json @@ -194,23 +194,25 @@ { "id": "alternative:columnExprValue:ColumnExprCase", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createCaseExpression", + "parseHandler": "AstBuilder#buildCaseExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompatibilityCorpus#expr.accept.case", + "TestHogQlCompiler#testLowersCaseAndCastExpressions" ] }, { "id": "alternative:columnExprValue:ColumnExprCast", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompatibilityCorpus#expr.accept.cast", + "TestHogQlCompiler#testLowersCaseAndCastExpressions" ] }, { @@ -714,12 +716,12 @@ { "id": "alternative:columnExprValue:ColumnExprTryCast", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersCaseAndCastExpressions" ] }, { @@ -891,12 +893,13 @@ { "id": "alternative:columnTypeExpr:ColumnTypeExprSimple", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildSimpleType", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompatibilityCorpus#expr.accept.cast", + "TestHogQlCompiler#testLowersCaseAndCastExpressions" ] }, { From 446cbc7f2bd1e721ba4625047ef6085f76f9d7de Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 08:47:55 -0400 Subject: [PATCH 009/136] feat(hogql): identify unlabeled grammar alternatives --- core/trino-hogql-parser/pom.xml | 6 + .../parser/HogQlCompatibilityManifest.java | 56 +- .../HogQlGrammarAlternativeIdentities.java | 165 ++++ .../1.0.0/grammar-alternative-identities.json | 393 +++++++++ .../1.0.0/trino-compatibility-overrides.json | 32 + .../language/1.0.0/trino-compatibility.json | 832 ++++++++++++++++-- .../TestHogQlCompatibilityManifest.java | 4 +- ...TestHogQlGrammarAlternativeIdentities.java | 106 +++ pom.xml | 6 + 9 files changed, 1528 insertions(+), 72 deletions(-) create mode 100644 core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlGrammarAlternativeIdentities.java create mode 100644 core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-alternative-identities.json create mode 100644 core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlGrammarAlternativeIdentities.java diff --git a/core/trino-hogql-parser/pom.xml b/core/trino-hogql-parser/pom.xml index 0e9b42d6bdcd..496956ef0ec8 100644 --- a/core/trino-hogql-parser/pom.xml +++ b/core/trino-hogql-parser/pom.xml @@ -29,6 +29,12 @@ antlr4-runtime + + org.antlr + antlr4 + test + + org.assertj assertj-core diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlCompatibilityManifest.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlCompatibilityManifest.java index f1a6f5e37e6b..cd7cacd28601 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlCompatibilityManifest.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlCompatibilityManifest.java @@ -36,8 +36,9 @@ public record HogQlCompatibilityManifest( int schemaVersion, HogQlLanguageVersion languageVersion, String grammarSha256, + String grammarAlternativeIdentityManifestSha256, String grammarFeatureManifestSha256, - List unresolvedGrammarAlternativeRules, + List sourceUnlabeledAlternativeRules, List features) { private static final String LANGUAGE_RESOURCE_ROOT = "/io/trino/hogql/parser/language/1.0.0/"; @@ -51,8 +52,9 @@ public HogQlCompatibilityManifest( @JsonProperty("schemaVersion") int schemaVersion, @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, @JsonProperty("grammarSha256") String grammarSha256, + @JsonProperty("grammarAlternativeIdentityManifestSha256") String grammarAlternativeIdentityManifestSha256, @JsonProperty("grammarFeatureManifestSha256") String grammarFeatureManifestSha256, - @JsonProperty("unresolvedGrammarAlternativeRules") List unresolvedGrammarAlternativeRules, + @JsonProperty("sourceUnlabeledAlternativeRules") List sourceUnlabeledAlternativeRules, @JsonProperty("features") List features) { if (schemaVersion != 1) { @@ -61,8 +63,9 @@ public HogQlCompatibilityManifest( this.schemaVersion = schemaVersion; this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); + this.grammarAlternativeIdentityManifestSha256 = requireNonNull(grammarAlternativeIdentityManifestSha256, "grammarAlternativeIdentityManifestSha256 is null"); this.grammarFeatureManifestSha256 = requireNonNull(grammarFeatureManifestSha256, "grammarFeatureManifestSha256 is null"); - this.unresolvedGrammarAlternativeRules = List.copyOf(requireNonNull(unresolvedGrammarAlternativeRules, "unresolvedGrammarAlternativeRules is null")); + this.sourceUnlabeledAlternativeRules = List.copyOf(requireNonNull(sourceUnlabeledAlternativeRules, "sourceUnlabeledAlternativeRules is null")); this.features = List.copyOf(requireNonNull(features, "features is null")); } @@ -84,6 +87,10 @@ private static HogQlCompatibilityManifest loadCurrent() if (!manifest.grammarFeatureManifestSha256().equals(languageContract.grammarFeatureManifest().sha256())) { throw new IllegalStateException("HogQL compatibility manifest feature hash does not match the language contract"); } + HogQlGrammarAlternativeIdentities alternativeIdentities = HogQlGrammarAlternativeIdentities.current(); + if (!manifest.grammarAlternativeIdentityManifestSha256().equals(HogQlGrammarAlternativeIdentities.currentSha256())) { + throw new IllegalStateException("HogQL compatibility manifest alternative identity hash does not match the published sidecar"); + } String featureManifestResource = LANGUAGE_RESOURCE_ROOT + languageContract.grammarFeatureManifest().path(); byte[] featureManifestBytes = readResourceBytes(featureManifestResource); @@ -91,11 +98,14 @@ private static HogQlCompatibilityManifest loadCurrent() throw new IllegalStateException("published HogQL grammar feature manifest checksum does not match the compatibility manifest"); } PublishedGrammarFeatureManifest published = GRAMMAR_FEATURE_MANIFEST_CODEC.fromJson(new String(featureManifestBytes, StandardCharsets.UTF_8)); - validateFeatures(manifest, published); + validateFeatures(manifest, published, alternativeIdentities); return manifest; } - private static void validateFeatures(HogQlCompatibilityManifest manifest, PublishedGrammarFeatureManifest published) + private static void validateFeatures( + HogQlCompatibilityManifest manifest, + PublishedGrammarFeatureManifest published, + HogQlGrammarAlternativeIdentities alternativeIdentities) { if (!published.languageVersion().equals(manifest.languageVersion()) || !published.grammarSha256().equals(manifest.grammarSha256())) { throw new IllegalStateException("published HogQL grammar features do not match the compatibility manifest language"); @@ -108,6 +118,28 @@ private static void validateFeatures(HogQlCompatibilityManifest manifest, Publis } } + Map sourceUnlabeledByRule = new HashMap<>(); + for (SourceUnlabeledAlternativeRule sourceUnlabeled : published.validationErrors()) { + if (sourceUnlabeledByRule.put(sourceUnlabeled.rule(), sourceUnlabeled) != null) { + throw new IllegalStateException("duplicate source-unlabeled HogQL grammar alternative rule: " + sourceUnlabeled.rule()); + } + } + for (HogQlGrammarAlternativeIdentities.AlternativeRule rule : alternativeIdentities.rules()) { + SourceUnlabeledAlternativeRule sourceUnlabeled = sourceUnlabeledByRule.remove(rule.rule()); + if (sourceUnlabeled == null || sourceUnlabeled.alternativeCount() != rule.alternatives().size() || sourceUnlabeled.queryReachable() != rule.queryReachable()) { + throw new IllegalStateException("HogQL grammar alternative identities do not match the published rule: " + rule.rule()); + } + for (HogQlGrammarAlternativeIdentities.Alternative alternative : rule.alternatives()) { + PublishedFeature feature = new PublishedFeature(alternative.id(), "parserAlternative", rule.queryReachable()); + if (publishedById.put(feature.id(), feature) != null) { + throw new IllegalStateException("duplicate published HogQL grammar feature: " + feature.id()); + } + } + } + if (!sourceUnlabeledByRule.isEmpty()) { + throw new IllegalStateException("HogQL grammar alternative identities do not cover every source-unlabeled rule"); + } + Map compatibilityById = new HashMap<>(); for (Feature feature : manifest.features()) { if (compatibilityById.put(feature.id(), feature) != null) { @@ -127,8 +159,8 @@ private static void validateFeatures(HogQlCompatibilityManifest manifest, Publis if (!compatibilityById.keySet().equals(publishedById.keySet())) { throw new IllegalStateException("HogQL compatibility manifest does not account for every published grammar feature"); } - if (!manifest.unresolvedGrammarAlternativeRules().equals(published.validationErrors())) { - throw new IllegalStateException("HogQL compatibility manifest does not acknowledge every unresolved grammar alternative rule"); + if (!manifest.sourceUnlabeledAlternativeRules().equals(published.validationErrors())) { + throw new IllegalStateException("HogQL compatibility manifest does not acknowledge every source-unlabeled grammar alternative rule"); } } @@ -232,21 +264,21 @@ public String toJson() } } - public record UnresolvedGrammarAlternativeRule( + public record SourceUnlabeledAlternativeRule( int alternativeCount, String code, boolean queryReachable, String rule) { @JsonCreator - public UnresolvedGrammarAlternativeRule( + public SourceUnlabeledAlternativeRule( @JsonProperty("alternativeCount") int alternativeCount, @JsonProperty("code") String code, @JsonProperty("queryReachable") boolean queryReachable, @JsonProperty("rule") String rule) { if (alternativeCount < 2) { - throw new IllegalArgumentException("unresolved grammar rule must have multiple alternatives: " + rule); + throw new IllegalArgumentException("source-unlabeled grammar rule must have multiple alternatives: " + rule); } this.alternativeCount = alternativeCount; this.code = requireNonNull(code, "code is null"); @@ -259,14 +291,14 @@ public record PublishedGrammarFeatureManifest( HogQlLanguageVersion languageVersion, String grammarSha256, List features, - List validationErrors) + List validationErrors) { @JsonCreator public PublishedGrammarFeatureManifest( @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, @JsonProperty("grammarSha256") String grammarSha256, @JsonProperty("features") List features, - @JsonProperty("validationErrors") List validationErrors) + @JsonProperty("validationErrors") List validationErrors) { this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlGrammarAlternativeIdentities.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlGrammarAlternativeIdentities.java new file mode 100644 index 000000000000..a3b5dd26117e --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlGrammarAlternativeIdentities.java @@ -0,0 +1,165 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.airlift.json.JsonCodec; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.List; +import java.util.Set; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; + +public record HogQlGrammarAlternativeIdentities( + int schemaVersion, + HogQlLanguageVersion languageVersion, + String grammarSha256, + List rules) +{ + private static final String CURRENT_RESOURCE = "/io/trino/hogql/parser/language/1.0.0/grammar-alternative-identities.json"; + private static final JsonCodec CODEC = jsonCodec(HogQlGrammarAlternativeIdentities.class); + private static final LoadedIdentities CURRENT = loadCurrent(); + + @JsonCreator + public HogQlGrammarAlternativeIdentities( + @JsonProperty("schemaVersion") int schemaVersion, + @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, + @JsonProperty("grammarSha256") String grammarSha256, + @JsonProperty("rules") List rules) + { + if (schemaVersion != 1) { + throw new IllegalArgumentException("unsupported HogQL grammar alternative identity schema: " + schemaVersion); + } + this.schemaVersion = schemaVersion; + this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); + this.rules = List.copyOf(requireNonNull(rules, "rules is null")); + + Set ruleNames = new HashSet<>(); + Set featureIds = new HashSet<>(); + for (AlternativeRule rule : this.rules) { + if (!ruleNames.add(rule.rule())) { + throw new IllegalArgumentException("duplicate HogQL grammar alternative identity rule: " + rule.rule()); + } + for (Alternative alternative : rule.alternatives()) { + if (!featureIds.add(alternative.id())) { + throw new IllegalArgumentException("duplicate HogQL grammar alternative identity: " + alternative.id()); + } + } + } + } + + public static HogQlGrammarAlternativeIdentities current() + { + return CURRENT.identities(); + } + + public static String currentSha256() + { + return CURRENT.sha256(); + } + + private static LoadedIdentities loadCurrent() + { + byte[] content = readResourceBytes(); + HogQlGrammarAlternativeIdentities identities = CODEC.fromJson(new String(content, StandardCharsets.UTF_8)); + HogQlLanguageContract languageContract = HogQlLanguageContract.current(); + if (!identities.languageVersion().equals(languageContract.languageVersion()) || !identities.grammarSha256().equals(languageContract.grammarSha256())) { + throw new IllegalStateException("HogQL grammar alternative identities do not match the language contract"); + } + return new LoadedIdentities(identities, sha256(content)); + } + + private static byte[] readResourceBytes() + { + try (InputStream input = HogQlGrammarAlternativeIdentities.class.getResourceAsStream(CURRENT_RESOURCE)) { + if (input == null) { + throw new IllegalStateException("HogQL grammar alternative identities are missing: " + CURRENT_RESOURCE); + } + return input.readAllBytes(); + } + catch (IOException e) { + throw new UncheckedIOException("failed to read HogQL grammar alternative identities", e); + } + } + + private static String sha256(byte[] content) + { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content)); + } + catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } + + private record LoadedIdentities(HogQlGrammarAlternativeIdentities identities, String sha256) {} + + public record AlternativeRule(String rule, boolean queryReachable, List alternatives) + { + @JsonCreator + public AlternativeRule( + @JsonProperty("rule") String rule, + @JsonProperty("queryReachable") boolean queryReachable, + @JsonProperty("alternatives") List alternatives) + { + this.rule = requireNonNull(rule, "rule is null"); + this.queryReachable = queryReachable; + this.alternatives = List.copyOf(requireNonNull(alternatives, "alternatives is null")); + if (rule.isBlank() || this.alternatives.size() < 2) { + throw new IllegalArgumentException("HogQL grammar alternative identity rule must have multiple alternatives: " + rule); + } + + Set fingerprints = new HashSet<>(); + for (Alternative alternative : this.alternatives) { + String prefix = "alternative:" + rule + ":"; + if (!alternative.id().startsWith(prefix)) { + throw new IllegalArgumentException("HogQL grammar alternative identity does not match its rule: " + alternative.id()); + } + String semanticId = alternative.id().substring(prefix.length()); + if (semanticId.isBlank() || semanticId.matches("(?:alt(?:ernative)?[-:]?)?\\d+")) { + throw new IllegalArgumentException("HogQL grammar alternative identity must not be positional: " + alternative.id()); + } + if (!fingerprints.add(alternative.structuralFingerprint())) { + throw new IllegalArgumentException("duplicate structural fingerprint for HogQL grammar rule: " + rule); + } + } + } + } + + public record Alternative(String id, String structuralFingerprint) + { + @JsonCreator + public Alternative( + @JsonProperty("id") String id, + @JsonProperty("structuralFingerprint") String structuralFingerprint) + { + this.id = requireNonNull(id, "id is null"); + this.structuralFingerprint = requireNonNull(structuralFingerprint, "structuralFingerprint is null"); + if (id.isBlank() || !structuralFingerprint.matches("[0-9a-f]{64}")) { + throw new IllegalArgumentException("invalid HogQL grammar alternative identity: " + id); + } + } + } +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-alternative-identities.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-alternative-identities.json new file mode 100644 index 000000000000..04296fa8f4bc --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-alternative-identities.json @@ -0,0 +1,393 @@ +{ + "grammarSha256": "c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242", + "languageVersion": "1.0.0", + "rules": [ + { + "alternatives": [ + { + "id": "alternative:alias:keyword", + "structuralFingerprint": "165f32605f10e147a97ee2d4f2f93bceeb5f6d6eb98f2ef8727355cb59c05bc3" + }, + { + "id": "alternative:alias:quoted-identifier", + "structuralFingerprint": "0abeb8c8fdc827ac36ae2e683162ce80f95e5072fadbcfccd45390ecc8ad2f67" + }, + { + "id": "alternative:alias:unquoted-identifier", + "structuralFingerprint": "73733fbd5549f93da457208e6512bd7d993c6bcaa6c383faac1b225b157ffce2" + } + ], + "queryReachable": true, + "rule": "alias" + }, + { + "alternatives": [ + { + "id": "alternative:columnExprTupleOrSingle:scalar", + "structuralFingerprint": "802b98aaa0c9404525d83dacb80290db7fdff49d34dba43e0e8c6518536197e6" + }, + { + "id": "alternative:columnExprTupleOrSingle:tuple", + "structuralFingerprint": "2b49f26a20cf05b31e6f7071f13a014eca01e5711bad4fcf3825b08d49e63464" + } + ], + "queryReachable": true, + "rule": "columnExprTupleOrSingle" + }, + { + "alternatives": [ + { + "id": "alternative:columnIdentifier:field-path", + "structuralFingerprint": "a4d34630e93d0f240a41eb30229aec9c6363cb7de0b70ebd303df105b5c1a6e4" + }, + { + "id": "alternative:columnIdentifier:placeholder", + "structuralFingerprint": "ac7187ca5c2d10611ac749649461be8e95b68c896f26c5e03b426f4d45a26531" + } + ], + "queryReachable": true, + "rule": "columnIdentifier" + }, + { + "alternatives": [ + { + "id": "alternative:columnTypeCastIdentifier:interval-keyword", + "structuralFingerprint": "30dfdd38947f7e4f75b790b875db86a7f630c63352fbd145fe885fa1a4fe6a1c" + }, + { + "id": "alternative:columnTypeCastIdentifier:quoted-identifier", + "structuralFingerprint": "0abeb8c8fdc827ac36ae2e683162ce80f95e5072fadbcfccd45390ecc8ad2f67" + }, + { + "id": "alternative:columnTypeCastIdentifier:type-keyword", + "structuralFingerprint": "5d58081b9087f3da068f1ad6b457461b5d8deaae59f21cd53b85fcd1bd1f1331" + }, + { + "id": "alternative:columnTypeCastIdentifier:unquoted-identifier", + "structuralFingerprint": "73733fbd5549f93da457208e6512bd7d993c6bcaa6c383faac1b225b157ffce2" + } + ], + "queryReachable": true, + "rule": "columnTypeCastIdentifier" + }, + { + "alternatives": [ + { + "id": "alternative:declaration:statement", + "structuralFingerprint": "5f99fed7cff39aef459246e59fb6f13a5413ddeea7adad8864398797455986f9" + }, + { + "id": "alternative:declaration:variable", + "structuralFingerprint": "4e7fc741b05bde1862641b814dccef54f6927a80c6d9e349ef7684664d44fd82" + } + ], + "queryReachable": true, + "rule": "declaration" + }, + { + "alternatives": [ + { + "id": "alternative:floatingLiteral:floating-token", + "structuralFingerprint": "c88b9f8fead5722931e73444ea92d55372027c411f4f89618d6e0418310ab819" + }, + { + "id": "alternative:floatingLiteral:leading-decimal-point", + "structuralFingerprint": "5b339918b24e7eb9571fa5c007b3b65a6f4c6d3b9c0562e21dcccdd3fd64cb84" + }, + { + "id": "alternative:floatingLiteral:trailing-decimal-point", + "structuralFingerprint": "304631df88730bc6643859c047b4d5a3414e318bd46efc9830a3b292e1312786" + } + ], + "queryReachable": true, + "rule": "floatingLiteral" + }, + { + "alternatives": [ + { + "id": "alternative:hogqlxChildElement:expression", + "structuralFingerprint": "92f247a454640b4807cf6c5f79cf4869c396255034642c950ed2ff7b256c4894" + }, + { + "id": "alternative:hogqlxChildElement:tag", + "structuralFingerprint": "638198a63464a5654b4202c71ae03a9d8abb39b3f6727aeb46165c1635d4410b" + }, + { + "id": "alternative:hogqlxChildElement:text", + "structuralFingerprint": "1acc0716f32f7f1346f847725d21990fd6861790bdbd91c30054c4a35c912d57" + } + ], + "queryReachable": true, + "rule": "hogqlxChildElement" + }, + { + "alternatives": [ + { + "id": "alternative:hogqlxTagAttribute:boolean", + "structuralFingerprint": "f9ec2f5892a5ee11ecee8b7898937fc268f9b2b758ea9a2b4a295f95b0d8e18e" + }, + { + "id": "alternative:hogqlxTagAttribute:expression", + "structuralFingerprint": "7c8aa67035024acc0a0829432ea042be18da3d8ee08fe8000fb60020e9aacbf1" + }, + { + "id": "alternative:hogqlxTagAttribute:string", + "structuralFingerprint": "88dfc10a32800ba48ebe8400d56499520c4ad6601dcc8e7befad72eed782b50e" + } + ], + "queryReachable": true, + "rule": "hogqlxTagAttribute" + }, + { + "alternatives": [ + { + "id": "alternative:identifier:interval-keyword", + "structuralFingerprint": "30dfdd38947f7e4f75b790b875db86a7f630c63352fbd145fe885fa1a4fe6a1c" + }, + { + "id": "alternative:identifier:keyword", + "structuralFingerprint": "bdc4cccbde824cbcccd1a6102bd60f67ab401a5cc6058526f5ac571269ef6030" + }, + { + "id": "alternative:identifier:quoted", + "structuralFingerprint": "0abeb8c8fdc827ac36ae2e683162ce80f95e5072fadbcfccd45390ecc8ad2f67" + }, + { + "id": "alternative:identifier:unquoted", + "structuralFingerprint": "73733fbd5549f93da457208e6512bd7d993c6bcaa6c383faac1b225b157ffce2" + } + ], + "queryReachable": true, + "rule": "identifier" + }, + { + "alternatives": [ + { + "id": "alternative:implicitAlias:keyword", + "structuralFingerprint": "c7b9f7299ab2aa1875c1ee89b24483887dec9de9834dd720cf174b9e104e9f25" + }, + { + "id": "alternative:implicitAlias:quoted-identifier", + "structuralFingerprint": "0abeb8c8fdc827ac36ae2e683162ce80f95e5072fadbcfccd45390ecc8ad2f67" + }, + { + "id": "alternative:implicitAlias:unquoted-identifier", + "structuralFingerprint": "73733fbd5549f93da457208e6512bd7d993c6bcaa6c383faac1b225b157ffce2" + } + ], + "queryReachable": true, + "rule": "implicitAlias" + }, + { + "alternatives": [ + { + "id": "alternative:joinConstraintClause:on", + "structuralFingerprint": "07ff66fad5bfb8968fb35cc93e06ee8bd49028e1854a50b7a40ecaf87a8a97e4" + }, + { + "id": "alternative:joinConstraintClause:using", + "structuralFingerprint": "2754567ba69ffb7298adf0d55cda11bd4b16ec0f9ed66692882b6f958e008327" + }, + { + "id": "alternative:joinConstraintClause:using-parenthesized", + "structuralFingerprint": "c53b790c3e3f8f17d08669249304555a30541a4b37868d86043a794e5eff94e9" + } + ], + "queryReachable": true, + "rule": "joinConstraintClause" + }, + { + "alternatives": [ + { + "id": "alternative:joinOpCross:comma", + "structuralFingerprint": "bcf9583b8a06a904da0c0ea03e289809ead51e2cda7d4084edc1ce18c4f2e025" + }, + { + "id": "alternative:joinOpCross:cross-join", + "structuralFingerprint": "21eefbaad9b940960d7c860ca5bac0ab771e7f71f33b5dc9b9fb9250b4e7be95" + } + ], + "queryReachable": true, + "rule": "joinOpCross" + }, + { + "alternatives": [ + { + "id": "alternative:limitAndOffsetClause:compact", + "structuralFingerprint": "8766cba8afea27bbf1418b00176a7981db472ab89a25675f1a415461a19d9fea" + }, + { + "id": "alternative:limitAndOffsetClause:with-offset", + "structuralFingerprint": "b08e7c7ac1035886c8d50c94cac6b85ed8d5b8a169512b4d3601b87209495c49" + } + ], + "queryReachable": true, + "rule": "limitAndOffsetClause" + }, + { + "alternatives": [ + { + "id": "alternative:limitAndOffsetClauseOptional:limit", + "structuralFingerprint": "8766cba8afea27bbf1418b00176a7981db472ab89a25675f1a415461a19d9fea" + }, + { + "id": "alternative:limitAndOffsetClauseOptional:limit-with-offset", + "structuralFingerprint": "b08e7c7ac1035886c8d50c94cac6b85ed8d5b8a169512b4d3601b87209495c49" + }, + { + "id": "alternative:limitAndOffsetClauseOptional:offset-only", + "structuralFingerprint": "3d9f0279272167d83cd5f876e7cf17e262cb676205fb7bd28f6e570596f0e383" + } + ], + "queryReachable": true, + "rule": "limitAndOffsetClauseOptional" + }, + { + "alternatives": [ + { + "id": "alternative:literal:null", + "structuralFingerprint": "7da720bab69a8562e12edf2c66b42136fbcc198035a585ca5a893d200e5b677a" + }, + { + "id": "alternative:literal:number", + "structuralFingerprint": "56c96aa898d8b78e683c7c140904730921bb5c67201b2be9dc07a804535aba3a" + }, + { + "id": "alternative:literal:string", + "structuralFingerprint": "aab18d5f0771f8fd97a25ffd2112e667e894acf92a430b02dcd8be9522140e85" + } + ], + "queryReachable": true, + "rule": "literal" + }, + { + "alternatives": [ + { + "id": "alternative:ratioExpr:numeric", + "structuralFingerprint": "77d0012bf666d0471f0c5fd47f2f749b3654961ab020288c8b5e524d218bc0a0" + }, + { + "id": "alternative:ratioExpr:placeholder", + "structuralFingerprint": "ac7187ca5c2d10611ac749649461be8e95b68c896f26c5e03b426f4d45a26531" + } + ], + "queryReachable": true, + "rule": "ratioExpr" + }, + { + "alternatives": [ + { + "id": "alternative:selectStmtWithParens:direct", + "structuralFingerprint": "d5e2b3fcc7a599513aa9c07f0a080bc96138881bda01b9885f451cdc4f47c9f1" + }, + { + "id": "alternative:selectStmtWithParens:parenthesized", + "structuralFingerprint": "a70746a6394bdd78e9a2bf2f3b95085009b3c95808b9e2d491c1507129fa1766" + }, + { + "id": "alternative:selectStmtWithParens:placeholder", + "structuralFingerprint": "ac7187ca5c2d10611ac749649461be8e95b68c896f26c5e03b426f4d45a26531" + }, + { + "id": "alternative:selectStmtWithParens:with-parenthesized", + "structuralFingerprint": "e459d96cb925b44a2cc38e03a5dffa2719f59eada2bf788ed797a41540efbfc4" + } + ], + "queryReachable": true, + "rule": "selectStmtWithParens" + }, + { + "alternatives": [ + { + "id": "alternative:statement:block", + "structuralFingerprint": "5acb58cd596f9ea97bc696bddd4f1a5113266af83e22c8f076e96eb5a25fd9bb" + }, + { + "id": "alternative:statement:empty", + "structuralFingerprint": "1d7ce4f5e96efc68d3a3d0388001437e449425f07fbdb3a9bbf02289e6055489" + }, + { + "id": "alternative:statement:expression", + "structuralFingerprint": "792d983175ddbb24989b35af49d0c512d46945699df2423920d79d6a84749f6f" + }, + { + "id": "alternative:statement:for", + "structuralFingerprint": "e70251237e3fd8baf257e1e5e3d675d79bbbbb6ed301885637730065d8f15a58" + }, + { + "id": "alternative:statement:for-in", + "structuralFingerprint": "8d49fe14f87bfe8baf981431e57d79f2c92516fdba0f2cdef472969ab5735d35" + }, + { + "id": "alternative:statement:function", + "structuralFingerprint": "d319855a6ecd0c15fbef3322531043be6aec21545e25f2a4e32392a3e61f0453" + }, + { + "id": "alternative:statement:if", + "structuralFingerprint": "912215ae9664d6d31e8d5102cabc877b5ebbfe6adac5e3261bff90f20475947c" + }, + { + "id": "alternative:statement:return", + "structuralFingerprint": "24012c3b96984fd79cafc9d9a956d947e7ad6fdb21c1f3a2cbc968f80c141cfa" + }, + { + "id": "alternative:statement:throw", + "structuralFingerprint": "85bf0d409a8909ef496fb3219b6d2e38ae9c3c6906953855c9326fcd00f1985a" + }, + { + "id": "alternative:statement:try-catch", + "structuralFingerprint": "fa0c783e64388c6f9a5b32852170481d7b837dfea973e3e37cdc131b7f912f80" + }, + { + "id": "alternative:statement:while", + "structuralFingerprint": "4ce361ee94a08639a64c93c45a3b6bdef28723e71ab7b4c5aaba60698b571477" + } + ], + "queryReachable": true, + "rule": "statement" + }, + { + "alternatives": [ + { + "id": "alternative:string:literal", + "structuralFingerprint": "aab18d5f0771f8fd97a25ffd2112e667e894acf92a430b02dcd8be9522140e85" + }, + { + "id": "alternative:string:template", + "structuralFingerprint": "a16ebcac7c3b3a8da8a3d035c74039354e90500773df3c4c2067bd96d5d43f9b" + } + ], + "queryReachable": true, + "rule": "string" + }, + { + "alternatives": [ + { + "id": "alternative:stringContents:interpolation", + "structuralFingerprint": "c162361fa3590d43840dae422ae3f80c406018732b902d01b5188d263b9dea29" + }, + { + "id": "alternative:stringContents:text", + "structuralFingerprint": "18cf885a4e28d578222b6878f728ac5f9065790527e565818e2a87a761b6bfdf" + } + ], + "queryReachable": true, + "rule": "stringContents" + }, + { + "alternatives": [ + { + "id": "alternative:stringContentsFull:interpolation", + "structuralFingerprint": "df68917607042cf2b91ee6c6f141fe1fcb4d578d2bb356001f5caec3c898bf2f" + }, + { + "id": "alternative:stringContentsFull:text", + "structuralFingerprint": "1abe294c80a61ee18740150362bdafcbb0b7a344d69860589932df63cb1ccab8" + } + ], + "queryReachable": false, + "rule": "stringContentsFull" + } + ], + "schemaVersion": 1 +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json index 9ca9de75db89..45763aa53f72 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json @@ -269,6 +269,38 @@ "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" ] }, + { + "id": "alternative:identifier:interval-keyword", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:identifier:keyword", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:identifier:quoted", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:identifier:unquoted", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, { "id": "alternative:nestedIdentifier", "loweringHandler": "TrinoAstFactory#createIdentifier", diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json index 0130cf2876c5..d485d90d73f5 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json @@ -1,5 +1,38 @@ { "features": [ + { + "id": "alternative:alias:keyword", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:alias:quoted-identifier", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:alias:unquoted-identifier", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, { "id": "alternative:arrayJoinClause", "kind": "parserAlternative", @@ -112,6 +145,28 @@ "TestHogQlCompatibilityCorpus#expr.accept.tuple" ] }, + { + "id": "alternative:columnExprTupleOrSingle:scalar", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprTupleOrSingle:tuple", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, { "id": "alternative:columnExprValue:ColumnExprArray", "kind": "parserAlternative", @@ -780,6 +835,28 @@ "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" ] }, + { + "id": "alternative:columnIdentifier:field-path", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnIdentifier:placeholder", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, { "id": "alternative:columnLambdaExpr:ArrowLambda", "kind": "parserAlternative", @@ -824,6 +901,50 @@ "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" ] }, + { + "id": "alternative:columnTypeCastIdentifier:interval-keyword", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeCastIdentifier:quoted-identifier", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeCastIdentifier:type-keyword", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeCastIdentifier:unquoted-identifier", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, { "id": "alternative:columnTypeExpr:ColumnTypeExprArray", "kind": "parserAlternative", @@ -935,6 +1056,28 @@ "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" ] }, + { + "id": "alternative:declaration:statement", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:declaration:variable", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, { "id": "alternative:emptyStmt", "kind": "parserAlternative", @@ -990,6 +1133,39 @@ "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" ] }, + { + "id": "alternative:floatingLiteral:floating-token", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:floatingLiteral:leading-decimal-point", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:floatingLiteral:trailing-decimal-point", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, { "id": "alternative:forInStmt", "kind": "parserAlternative", @@ -1090,7 +1266,7 @@ ] }, { - "id": "alternative:hogqlxTagElement:HogqlxTagElementClosed", + "id": "alternative:hogqlxChildElement:expression", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1101,7 +1277,7 @@ ] }, { - "id": "alternative:hogqlxTagElement:HogqlxTagElementNested", + "id": "alternative:hogqlxChildElement:tag", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1112,7 +1288,7 @@ ] }, { - "id": "alternative:hogqlxText", + "id": "alternative:hogqlxChildElement:text", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1123,7 +1299,7 @@ ] }, { - "id": "alternative:identifierList", + "id": "alternative:hogqlxTagAttribute:boolean", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1134,7 +1310,7 @@ ] }, { - "id": "alternative:ifStmt", + "id": "alternative:hogqlxTagAttribute:expression", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1145,7 +1321,7 @@ ] }, { - "id": "alternative:interpolateClause", + "id": "alternative:hogqlxTagAttribute:string", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1156,7 +1332,7 @@ ] }, { - "id": "alternative:interpolateExpr", + "id": "alternative:hogqlxTagElement:HogqlxTagElementClosed", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1167,7 +1343,7 @@ ] }, { - "id": "alternative:interval", + "id": "alternative:hogqlxTagElement:HogqlxTagElementNested", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1178,7 +1354,7 @@ ] }, { - "id": "alternative:joinExpr:JoinExprCrossOp", + "id": "alternative:hogqlxText", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1189,51 +1365,51 @@ ] }, { - "id": "alternative:joinExpr:JoinExprOp", + "id": "alternative:identifier:interval-keyword", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" ] }, { - "id": "alternative:joinExpr:JoinExprParens", + "id": "alternative:identifier:keyword", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" ] }, { - "id": "alternative:joinExpr:JoinExprPivot", + "id": "alternative:identifier:quoted", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" ] }, { - "id": "alternative:joinExpr:JoinExprPositional", + "id": "alternative:identifier:unquoted", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" ] }, { - "id": "alternative:joinExpr:JoinExprTable", + "id": "alternative:identifierList", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1244,7 +1420,7 @@ ] }, { - "id": "alternative:joinExpr:JoinExprUnpivot", + "id": "alternative:ifStmt", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1255,7 +1431,7 @@ ] }, { - "id": "alternative:joinOp:JoinOpFull", + "id": "alternative:implicitAlias:keyword", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1266,7 +1442,7 @@ ] }, { - "id": "alternative:joinOp:JoinOpInner", + "id": "alternative:implicitAlias:quoted-identifier", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1277,7 +1453,7 @@ ] }, { - "id": "alternative:joinOp:JoinOpLeftRight", + "id": "alternative:implicitAlias:unquoted-identifier", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1288,7 +1464,7 @@ ] }, { - "id": "alternative:keyword", + "id": "alternative:interpolateClause", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1299,7 +1475,7 @@ ] }, { - "id": "alternative:keywordForAlias", + "id": "alternative:interpolateExpr", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1310,7 +1486,7 @@ ] }, { - "id": "alternative:keywordForImplicitAlias", + "id": "alternative:interval", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1321,7 +1497,7 @@ ] }, { - "id": "alternative:keywordForTypeCast", + "id": "alternative:joinConstraintClause:on", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1332,7 +1508,7 @@ ] }, { - "id": "alternative:kvPair", + "id": "alternative:joinConstraintClause:using", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1343,7 +1519,7 @@ ] }, { - "id": "alternative:kvPairList", + "id": "alternative:joinConstraintClause:using-parenthesized", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1354,7 +1530,7 @@ ] }, { - "id": "alternative:limitByClause", + "id": "alternative:joinExpr:JoinExprCrossOp", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1365,7 +1541,7 @@ ] }, { - "id": "alternative:limitExpr", + "id": "alternative:joinExpr:JoinExprOp", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1376,18 +1552,18 @@ ] }, { - "id": "alternative:nestedIdentifier", + "id": "alternative:joinExpr:JoinExprParens", "kind": "parserAlternative", - "loweringHandler": "TrinoAstFactory#createIdentifier", - "parseHandler": "AstBuilder#buildIdentifiers", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", "queryReachable": true, - "status": "supported", + "status": "explicitCurrentError", "testCaseIds": [ - "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" ] }, { - "id": "alternative:numberLiteral", + "id": "alternative:joinExpr:JoinExprPivot", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1398,7 +1574,7 @@ ] }, { - "id": "alternative:offsetOnlyClause", + "id": "alternative:joinExpr:JoinExprPositional", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1409,7 +1585,7 @@ ] }, { - "id": "alternative:orderByClause", + "id": "alternative:joinExpr:JoinExprTable", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1420,7 +1596,7 @@ ] }, { - "id": "alternative:orderExpr", + "id": "alternative:joinExpr:JoinExprUnpivot", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1431,7 +1607,7 @@ ] }, { - "id": "alternative:orderExprList", + "id": "alternative:joinOp:JoinOpFull", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1442,7 +1618,7 @@ ] }, { - "id": "alternative:pivotColumn", + "id": "alternative:joinOp:JoinOpInner", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1453,7 +1629,7 @@ ] }, { - "id": "alternative:pivotColumnList", + "id": "alternative:joinOp:JoinOpLeftRight", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1464,7 +1640,7 @@ ] }, { - "id": "alternative:placeholder", + "id": "alternative:joinOpCross:comma", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1475,7 +1651,7 @@ ] }, { - "id": "alternative:prewhereClause", + "id": "alternative:joinOpCross:cross-join", "kind": "parserAlternative", "loweringHandler": "UnsupportedFeature", "parseHandler": "CanonicalAntlrParser", @@ -1486,7 +1662,293 @@ ] }, { - "id": "alternative:program", + "id": "alternative:keyword", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:keywordForAlias", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:keywordForImplicitAlias", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:keywordForTypeCast", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:kvPair", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:kvPairList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitAndOffsetClause:compact", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitAndOffsetClause:with-offset", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitAndOffsetClauseOptional:limit", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitAndOffsetClauseOptional:limit-with-offset", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitAndOffsetClauseOptional:offset-only", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitByClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitExpr", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:literal:null", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:literal:number", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:literal:string", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:nestedIdentifier", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:numberLiteral", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:offsetOnlyClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:orderByClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:orderExpr", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:orderExprList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:pivotColumn", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:pivotColumnList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:placeholder", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:prewhereClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:program", "kind": "parserAlternative", "loweringHandler": "NotQueryLanguage", "parseHandler": "NotQueryLanguage", @@ -1518,6 +1980,28 @@ "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" ] }, + { + "id": "alternative:ratioExpr:numeric", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:ratioExpr:placeholder", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, { "id": "alternative:returnStmt", "kind": "parserAlternative", @@ -1650,6 +2134,50 @@ "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" ] }, + { + "id": "alternative:selectStmtWithParens:direct", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectStmtWithParens:parenthesized", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectStmtWithParens:placeholder", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectStmtWithParens:with-parenthesized", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, { "id": "alternative:settingExpr", "kind": "parserAlternative", @@ -1683,6 +2211,193 @@ "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" ] }, + { + "id": "alternative:statement:block", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:empty", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:expression", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:for", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:for-in", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:function", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:if", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:return", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:throw", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:try-catch", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:while", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:string:literal", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:string:template", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:stringContents:interpolation", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:stringContents:text", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:stringContentsFull:interpolation", + "kind": "parserAlternative", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "alternative:stringContentsFull:text", + "kind": "parserAlternative", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, { "id": "alternative:subsequentSelectSetClause", "kind": "parserAlternative", @@ -5711,11 +6426,12 @@ ] } ], + "grammarAlternativeIdentityManifestSha256": "55e33d04906b058bdbab00507b402766d386e4f771b60e4f5106fef63e18383d", "grammarFeatureManifestSha256": "fbf7236643b43d929e9ef1c8fd01a04b746736dca2f965cc2e78ad62f1a98b10", "grammarSha256": "c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242", "languageVersion": "1.0.0", "schemaVersion": 1, - "unresolvedGrammarAlternativeRules": [ + "sourceUnlabeledAlternativeRules": [ { "alternativeCount": 3, "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityManifest.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityManifest.java index 7f68ace2a76d..c2f25df777be 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityManifest.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityManifest.java @@ -43,8 +43,8 @@ public void testAccountsForEveryPublishedGrammarFeature() assertThat(manifest.schemaVersion()).isEqualTo(1); assertThat(manifest.languageVersion()).isEqualTo(HogQlLanguageContract.current().languageVersion()); - assertThat(manifest.features()).hasSize(518); - assertThat(manifest.unresolvedGrammarAlternativeRules()).hasSize(21); + assertThat(manifest.features()).hasSize(583); + assertThat(manifest.sourceUnlabeledAlternativeRules()).hasSize(21); assertThat(manifest.features()) .allSatisfy(feature -> assertThat(feature.testCaseIds()).isNotEmpty()); } diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlGrammarAlternativeIdentities.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlGrammarAlternativeIdentities.java new file mode 100644 index 000000000000..888b9bee0ef1 --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlGrammarAlternativeIdentities.java @@ -0,0 +1,106 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import io.trino.hogql.parser.HogQlGrammarAlternativeIdentities.AlternativeRule; +import org.antlr.v4.Tool; +import org.antlr.v4.tool.Grammar; +import org.antlr.v4.tool.LexerGrammar; +import org.antlr.v4.tool.Rule; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlGrammarAlternativeIdentities +{ + private static final String GRAMMAR_RESOURCE_ROOT = "/io/trino/hogql/parser/language/1.0.0/grammar/"; + private static final Map> CANONICAL_FINGERPRINTS = loadCanonicalFingerprints(); + + @ParameterizedTest(name = "{0}") + @MethodSource("alternativeRules") + public void testSemanticIdentitiesMatchCanonicalAlternativeStructure(String ruleName, AlternativeRule expected) + { + assertThat(CANONICAL_FINGERPRINTS.get(ruleName)) + .containsExactlyInAnyOrderElementsOf(expected.alternatives().stream() + .map(HogQlGrammarAlternativeIdentities.Alternative::structuralFingerprint) + .toList()); + } + + private static Stream alternativeRules() + { + return HogQlGrammarAlternativeIdentities.current().rules().stream() + .map(rule -> Arguments.of(rule.rule(), rule)); + } + + private static Map> loadCanonicalFingerprints() + { + try { + assertThat(Tool.VERSION).isEqualTo(HogQlLanguageContract.current().antlrVersion()); + String javaLexer = readResource("HogQLLexer.java.g4"); + String commonLexer = readResource("HogQLLexer.common.g4"); + String completeLexer = javaLexer + commonLexer.substring(commonLexer.indexOf('\n') + 1); + LexerGrammar lexer = new LexerGrammar(completeLexer); + Grammar parser = new Grammar(readResource("HogQLParser.g4"), lexer); + + Map> fingerprints = new HashMap<>(); + for (AlternativeRule expected : HogQlGrammarAlternativeIdentities.current().rules()) { + Rule rule = parser.getRule(expected.rule()); + Set alternatives = new HashSet<>(); + for (int alternative = 1; alternative <= rule.numberOfAlts; alternative++) { + alternatives.add(sha256(rule.alt[alternative].ast.toStringTree())); + } + fingerprints.put(expected.rule(), Set.copyOf(alternatives)); + } + return Map.copyOf(fingerprints); + } + catch (Exception e) { + throw new AssertionError("failed to inspect canonical HogQL grammar", e); + } + } + + private static String readResource(String name) + throws IOException + { + try (InputStream input = TestHogQlGrammarAlternativeIdentities.class.getResourceAsStream(GRAMMAR_RESOURCE_ROOT + name)) { + if (input == null) { + throw new IOException("missing canonical HogQL grammar resource: " + name); + } + return new String(input.readAllBytes(), StandardCharsets.UTF_8); + } + } + + private static String sha256(String value) + { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); + } + catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } +} diff --git a/pom.xml b/pom.xml index 3c3ef659b022..5511ddb43c17 100644 --- a/pom.xml +++ b/pom.xml @@ -1804,6 +1804,12 @@ + + org.antlr + antlr4 + ${dep.antlr.version} + + org.antlr antlr4-runtime From 161ad0a9e890d47b4ca906e1954537f3bd93594c Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 08:50:16 -0400 Subject: [PATCH 010/136] feat(hogql): preserve the complete query syntax tree --- core/trino-hogql-compiler/README.md | 14 ++ .../io/trino/hogql/parser/HogQlParser.java | 133 +++++++++++++++--- .../hogql/parser/tree/HogQlSyntaxTree.java | 69 +++++++++ .../parser/TestHogQlCompatibilityCorpus.java | 76 ++++++++++ .../trino/hogql/parser/TestHogQlParser.java | 41 ++++++ 5 files changed, 313 insertions(+), 20 deletions(-) create mode 100644 core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlSyntaxTree.java diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 8093ea9dabbf..0e10794d8b7a 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -29,6 +29,20 @@ booleans, or `NULL`. Identifiers can be unquoted, double quoted, or backquoted. The parser rejects unsupported clauses and multiple statements instead of silently interpreting them as SQL. +## M1 syntax contract + +`HogQlParser.parseSyntax` accepts the complete canonical read-only query entry +point and returns an immutable, source-located tree containing grammar rules, +ANTLR alternative labels when present, and tokens. The tree contains no ANTLR +or Trino SQL objects. It classifies declarative HogQLX separately from read-only +queries. + +This syntax contract is intentionally independent from executable lowering. +`HogQlParser.parseStatement` and `HogQlCompiler.compile` continue to return an +explicit error for valid constructs that do not yet have a stock Trino AST +mapping. Adding syntax support therefore cannot silently claim semantic or +execution parity. + Set `hogql.enabled=true` on the coordinator to register `POST /v1/hogql`. The request body is raw HogQL text and the response uses Trino's existing statement protocol. Follow-up result and cancellation URIs remain under `/v1/statement`. diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index c75fd90fa1c2..b9274af638b0 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -37,6 +37,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlSyntaxTree; import org.antlr.v4.runtime.ANTLRErrorListener; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.BaseErrorListener; @@ -49,6 +50,8 @@ import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.atn.PredictionMode; import org.antlr.v4.runtime.misc.ParseCancellationException; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.TerminalNode; import java.util.ArrayList; import java.util.List; @@ -95,6 +98,33 @@ public HogQlQuery parseStatement(String hogql) } public HogQlQuery parseStatement(String hogql, HogQlLanguageVersion languageVersion) + { + try { + ParsedQuery parsed = parseQuery(hogql, languageVersion); + return new AstBuilder(parsed.source()).build(parsed.tree()); + } + catch (StackOverflowError _) { + throw new HogQlParsingException("statement is too large", null, 1, 1); + } + } + + public HogQlSyntaxTree parseSyntax(String hogql) + { + return parseSyntax(hogql, CURRENT_LANGUAGE_VERSION); + } + + public HogQlSyntaxTree parseSyntax(String hogql, HogQlLanguageVersion languageVersion) + { + try { + ParsedQuery parsed = parseQuery(hogql, languageVersion); + return new SyntaxTreeBuilder(parsed.source()).build(parsed.tree()); + } + catch (StackOverflowError _) { + throw new HogQlParsingException("statement is too large", null, 1, 1); + } + } + + private static ParsedQuery parseQuery(String hogql, HogQlLanguageVersion languageVersion) { requireNonNull(hogql, "hogql is null"); requireNonNull(languageVersion, "languageVersion is null"); @@ -102,33 +132,96 @@ public HogQlQuery parseStatement(String hogql, HogQlLanguageVersion languageVers throw new IllegalArgumentException("unsupported HogQL language version: " + languageVersion); } + HogQLLexer lexer = new HogQLLexer(CharStreams.fromString(hogql)); + CommonTokenStream tokenStream = new CommonTokenStream(lexer); + HogQLParser parser = new HogQLParser(tokenStream); + + lexer.removeErrorListeners(); + lexer.addErrorListener(ERROR_LISTENER); + parser.removeErrorListeners(); + + HogQLParser.SelectContext tree; try { - HogQLLexer lexer = new HogQLLexer(CharStreams.fromString(hogql)); - CommonTokenStream tokenStream = new CommonTokenStream(lexer); - HogQLParser parser = new HogQLParser(tokenStream); + parser.getInterpreter().setPredictionMode(PredictionMode.SLL); + parser.setErrorHandler(new BailErrorStrategy()); + tree = parser.select(); + } + catch (ParseCancellationException _) { + parser.reset(); + parser.getInterpreter().setPredictionMode(PredictionMode.LL); + parser.setErrorHandler(new DefaultErrorStrategy()); + parser.addErrorListener(ERROR_LISTENER); + tree = parser.select(); + } + return new ParsedQuery(hogql, tree); + } + + private record ParsedQuery(String source, HogQLParser.SelectContext tree) {} + + private static final class SyntaxTreeBuilder + { + private final SourcePositions sourcePositions; + + private SyntaxTreeBuilder(String source) + { + sourcePositions = new SourcePositions(source); + } - lexer.removeErrorListeners(); - lexer.addErrorListener(ERROR_LISTENER); - parser.removeErrorListeners(); + public HogQlSyntaxTree build(HogQLParser.SelectContext context) + { + HogQlSyntaxTree.LanguageClass languageClass = context.hogqlxTagElement() == null ? + HogQlSyntaxTree.LanguageClass.READ_ONLY_QUERY : + HogQlSyntaxTree.LanguageClass.HOGQLX; + return new HogQlSyntaxTree(languageClass, buildNode(context)); + } - HogQLParser.SelectContext tree; - try { - parser.getInterpreter().setPredictionMode(PredictionMode.SLL); - parser.setErrorHandler(new BailErrorStrategy()); - tree = parser.select(); + private HogQlSyntaxTree.Node buildNode(ParserRuleContext context) + { + String rule = HogQLParser.ruleNames[context.getRuleIndex()]; + String contextName = context.getClass().getSimpleName(); + String baseContextName = Character.toUpperCase(rule.charAt(0)) + rule.substring(1) + "Context"; + Optional alternative = contextName.equals(baseContextName) ? + Optional.empty() : + Optional.of(contextName.substring(0, contextName.length() - "Context".length())); + + List children = new ArrayList<>(); + for (int index = 0; index < context.getChildCount(); index++) { + ParseTree child = context.getChild(index); + if (child instanceof ParserRuleContext ruleChild) { + children.add(buildNode(ruleChild)); + } + else if (child instanceof TerminalNode terminal && terminal.getSymbol().getType() != Token.EOF) { + children.add(buildToken(terminal.getSymbol())); + } } - catch (ParseCancellationException _) { - parser.reset(); - parser.getInterpreter().setPredictionMode(PredictionMode.LL); - parser.setErrorHandler(new DefaultErrorStrategy()); - parser.addErrorListener(ERROR_LISTENER); - tree = parser.select(); + return new HogQlSyntaxTree.Node(rule, alternative, children, sourceSpan(context)); + } + + private HogQlSyntaxTree.Token buildToken(Token token) + { + String type = HogQLLexer.VOCABULARY.getSymbolicName(token.getType()); + if (type == null) { + type = HogQLLexer.VOCABULARY.getDisplayName(token.getType()); } + return new HogQlSyntaxTree.Token(type, token.getText(), sourceSpan(token, token)); + } - return new AstBuilder(hogql).build(tree); + private SourceSpan sourceSpan(ParserRuleContext context) + { + return sourceSpan(context.getStart(), context.getStop()); } - catch (StackOverflowError _) { - throw new HogQlParsingException("statement is too large", null, 1, 1); + + private SourceSpan sourceSpan(Token start, Token stop) + { + int startOffset = Math.max(0, start.getStartIndex()); + int endOffset = Math.max(startOffset, stop.getStopIndex() + 1); + return new SourceSpan( + startOffset, + endOffset, + sourcePositions.line(startOffset), + sourcePositions.column(startOffset), + sourcePositions.line(endOffset), + sourcePositions.column(endOffset)); } } diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlSyntaxTree.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlSyntaxTree.java new file mode 100644 index 000000000000..1f53b43e4a1d --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlSyntaxTree.java @@ -0,0 +1,69 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser.tree; + +import java.util.List; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public record HogQlSyntaxTree(LanguageClass languageClass, Node root) +{ + public HogQlSyntaxTree + { + languageClass = requireNonNull(languageClass, "languageClass is null"); + root = requireNonNull(root, "root is null"); + } + + public enum LanguageClass + { + READ_ONLY_QUERY, + HOGQLX, + } + + public sealed interface Element + permits Node, Token + { + HogQlQuery.SourceSpan span(); + } + + public record Node(String rule, Optional alternative, List children, HogQlQuery.SourceSpan span) + implements Element + { + public Node + { + rule = requireNonNull(rule, "rule is null"); + alternative = requireNonNull(alternative, "alternative is null"); + children = List.copyOf(requireNonNull(children, "children is null")); + span = requireNonNull(span, "span is null"); + if (rule.isBlank() || alternative.stream().anyMatch(String::isBlank)) { + throw new IllegalArgumentException("syntax node identity is empty"); + } + } + } + + public record Token(String type, String text, HogQlQuery.SourceSpan span) + implements Element + { + public Token + { + type = requireNonNull(type, "type is null"); + text = requireNonNull(text, "text is null"); + span = requireNonNull(span, "span is null"); + if (type.isBlank()) { + throw new IllegalArgumentException("token type is empty"); + } + } + } +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java index 176948afb20d..d7824fa15ef4 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java @@ -16,6 +16,10 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import io.airlift.json.JsonCodec; +import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.hogql.parser.tree.HogQlSyntaxTree; +import io.trino.hogql.parser.tree.HogQlSyntaxTree.Element; +import io.trino.hogql.parser.tree.HogQlSyntaxTree.Node; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestFactory; @@ -81,6 +85,14 @@ public Stream testTrinoDispositionForEveryCorpusCase() .map(corpusCase -> dynamicTest(corpusCase.id(), () -> assertDisposition(corpusCase, dispositionById.get(corpusCase.id())))); } + @TestFactory + public Stream testSyntaxTreeDispositionForEveryCorpusCase() + throws IOException + { + return loadCorpusManifest().cases().stream() + .map(corpusCase -> dynamicTest(corpusCase.id(), () -> assertSyntaxDisposition(corpusCase))); + } + private void assertDisposition(CorpusCase corpusCase, CorpusDisposition disposition) { String query = switch (corpusCase.entryPoint()) { @@ -100,6 +112,70 @@ private void assertDisposition(CorpusCase corpusCase, CorpusDisposition disposit } } + private void assertSyntaxDisposition(CorpusCase corpusCase) + { + String query = switch (corpusCase.entryPoint()) { + case "expression" -> "SELECT " + corpusCase.source(); + case "query" -> corpusCase.source(); + default -> throw new IllegalArgumentException("unknown corpus entry point: " + corpusCase.entryPoint()); + }; + if (!corpusCase.accepted()) { + assertThatThrownBy(() -> parser.parseSyntax(query)) + .isInstanceOf(HogQlParsingException.class) + .hasMessageStartingWith("line "); + return; + } + + HogQlSyntaxTree syntaxTree = parser.parseSyntax(query); + int sourceLength = query.codePointCount(0, query.length()); + assertThat(syntaxTree.root().span()).isEqualTo(new SourceSpan(0, sourceLength, 1, 1, lineCount(query), finalColumn(query))); + assertContainedSpans(syntaxTree.root(), sourceLength); + } + + private static void assertContainedSpans(Element element, int sourceLength) + { + SourceSpan span = element.span(); + assertThat(span.startOffset()).isBetween(0, sourceLength); + assertThat(span.endOffset()).isBetween(span.startOffset(), sourceLength); + if (element instanceof Node node) { + for (Element child : node.children()) { + assertThat(child.span().startOffset()).isGreaterThanOrEqualTo(span.startOffset()); + assertThat(child.span().endOffset()).isLessThanOrEqualTo(span.endOffset()); + assertContainedSpans(child, sourceLength); + } + } + } + + private static int lineCount(String source) + { + int lines = 1; + boolean previousWasCarriageReturn = false; + for (int offset = 0; offset < source.length(); ) { + int codePoint = source.codePointAt(offset); + offset += Character.charCount(codePoint); + if (codePoint == '\r') { + lines++; + previousWasCarriageReturn = true; + } + else if (codePoint == '\n') { + if (!previousWasCarriageReturn) { + lines++; + } + previousWasCarriageReturn = false; + } + else { + previousWasCarriageReturn = false; + } + } + return lines; + } + + private static int finalColumn(String source) + { + int lastNewline = Math.max(source.lastIndexOf('\n'), source.lastIndexOf('\r')); + return source.codePointCount(lastNewline + 1, source.length()) + 1; + } + private static CorpusManifest loadCorpusManifest() throws IOException { diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 054ef6d2e776..5e545a2fd8be 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -16,10 +16,15 @@ import io.trino.hogql.parser.tree.HogQlQuery; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlSyntaxTree; +import io.trino.hogql.parser.tree.HogQlSyntaxTree.Element; +import io.trino.hogql.parser.tree.HogQlSyntaxTree.Node; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import java.util.stream.Stream; + import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -65,4 +70,40 @@ public void testSourceSpansUseUnicodeCodePointOffsetsAndEndPositions() assertThat(query.span()).isEqualTo(new HogQlQuery.SourceSpan(0, 18, 1, 1, 2, 7)); assertThat(reference.span()).isEqualTo(new HogQlQuery.SourceSpan(13, 18, 2, 2, 2, 7)); } + + @Test + public void testSyntaxTreeRepresentsReadOnlyGrammarBeforeLowering() + { + String hogql = "WITH x AS (SELECT 1) SELECT * FROM x UNION ALL SELECT event FROM events ORDER BY event LIMIT 10 OFFSET 2;"; + + HogQlSyntaxTree syntaxTree = parser.parseSyntax(hogql); + + assertThat(syntaxTree.languageClass()).isEqualTo(HogQlSyntaxTree.LanguageClass.READ_ONLY_QUERY); + assertThat(syntaxTree.root().span()).isEqualTo(new HogQlQuery.SourceSpan(0, hogql.length(), 1, 1, 1, hogql.length() + 1)); + assertThat(nodes(syntaxTree.root())) + .extracting(Node::rule) + .contains("withClause", "selectSetStmt", "subsequentSelectSetClause", "orderByClause", "limitAndOffsetClauseOptional"); + assertThat(nodes(syntaxTree.root())) + .extracting(Node::alternative) + .contains(java.util.Optional.of("WithExprSubquery"), java.util.Optional.of("ColumnExprIdentifier")); + } + + @Test + public void testSyntaxTreeClassifiesHogQlXAsNonQuerySyntax() + { + HogQlSyntaxTree syntaxTree = parser.parseSyntax(""); + + assertThat(syntaxTree.languageClass()).isEqualTo(HogQlSyntaxTree.LanguageClass.HOGQLX); + assertThat(nodes(syntaxTree.root())) + .extracting(Node::rule) + .contains("hogqlxTagElement"); + } + + private static Stream nodes(Element element) + { + if (!(element instanceof Node node)) { + return Stream.empty(); + } + return Stream.concat(Stream.of(node), node.children().stream().flatMap(TestHogQlParser::nodes)); + } } From 74430778c964f600ab5241578e3d05e422ecc02c Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 08:54:14 -0400 Subject: [PATCH 011/136] feat(hogql): bound parser resource usage --- .../io/trino/hogql/parser/HogQlParser.java | 137 +++++++++++++++++- .../trino/hogql/parser/HogQlParserLimits.java | 37 +++++ .../hogql/parser/TestHogQlParserLimits.java | 62 ++++++++ 3 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParserLimits.java create mode 100644 core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserLimits.java diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index b9274af638b0..a7b3636e72ee 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -41,6 +41,7 @@ import org.antlr.v4.runtime.ANTLRErrorListener; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.DefaultErrorStrategy; @@ -48,9 +49,13 @@ import org.antlr.v4.runtime.RecognitionException; import org.antlr.v4.runtime.Recognizer; import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenFactory; +import org.antlr.v4.runtime.TokenSource; import org.antlr.v4.runtime.atn.PredictionMode; import org.antlr.v4.runtime.misc.ParseCancellationException; +import org.antlr.v4.runtime.tree.ErrorNode; import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.ParseTreeListener; import org.antlr.v4.runtime.tree.TerminalNode; import java.util.ArrayList; @@ -83,6 +88,8 @@ public final class HogQlParser { private static final HogQlLanguageVersion CURRENT_LANGUAGE_VERSION = HogQlLanguageContract.current().languageVersion(); + private final HogQlParserLimits limits; + private static final ANTLRErrorListener ERROR_LISTENER = new BaseErrorListener() { @Override @@ -92,6 +99,16 @@ public void syntaxError(Recognizer recognizer, Object offendingSymbol, int } }; + public HogQlParser() + { + this(HogQlParserLimits.defaults()); + } + + HogQlParser(HogQlParserLimits limits) + { + this.limits = requireNonNull(limits, "limits is null"); + } + public HogQlQuery parseStatement(String hogql) { return parseStatement(hogql, CURRENT_LANGUAGE_VERSION); @@ -124,7 +141,7 @@ public HogQlSyntaxTree parseSyntax(String hogql, HogQlLanguageVersion languageVe } } - private static ParsedQuery parseQuery(String hogql, HogQlLanguageVersion languageVersion) + private ParsedQuery parseQuery(String hogql, HogQlLanguageVersion languageVersion) { requireNonNull(hogql, "hogql is null"); requireNonNull(languageVersion, "languageVersion is null"); @@ -133,7 +150,7 @@ private static ParsedQuery parseQuery(String hogql, HogQlLanguageVersion languag } HogQLLexer lexer = new HogQLLexer(CharStreams.fromString(hogql)); - CommonTokenStream tokenStream = new CommonTokenStream(lexer); + CommonTokenStream tokenStream = new CommonTokenStream(new BoundedTokenSource(lexer, limits.maxTokens())); HogQLParser parser = new HogQLParser(tokenStream); lexer.removeErrorListeners(); @@ -144,6 +161,7 @@ private static ParsedQuery parseQuery(String hogql, HogQlLanguageVersion languag try { parser.getInterpreter().setPredictionMode(PredictionMode.SLL); parser.setErrorHandler(new BailErrorStrategy()); + parser.addParseListener(new ParseBudgetListener(limits)); tree = parser.select(); } catch (ParseCancellationException _) { @@ -151,6 +169,8 @@ private static ParsedQuery parseQuery(String hogql, HogQlLanguageVersion languag parser.getInterpreter().setPredictionMode(PredictionMode.LL); parser.setErrorHandler(new DefaultErrorStrategy()); parser.addErrorListener(ERROR_LISTENER); + parser.removeParseListeners(); + parser.addParseListener(new ParseBudgetListener(limits)); tree = parser.select(); } return new ParsedQuery(hogql, tree); @@ -158,6 +178,119 @@ private static ParsedQuery parseQuery(String hogql, HogQlLanguageVersion languag private record ParsedQuery(String source, HogQLParser.SelectContext tree) {} + private static final class BoundedTokenSource + implements TokenSource + { + private final TokenSource delegate; + private final int maxTokens; + private int tokenCount; + + private BoundedTokenSource(TokenSource delegate, int maxTokens) + { + this.delegate = requireNonNull(delegate, "delegate is null"); + this.maxTokens = maxTokens; + } + + @Override + public Token nextToken() + { + Token token = delegate.nextToken(); + if (token.getType() != Token.EOF && ++tokenCount > maxTokens) { + throw new HogQlParsingException("token limit exceeded", null, token.getLine(), token.getCharPositionInLine() + 1); + } + return token; + } + + @Override + public int getLine() + { + return delegate.getLine(); + } + + @Override + public int getCharPositionInLine() + { + return delegate.getCharPositionInLine(); + } + + @Override + public CharStream getInputStream() + { + return delegate.getInputStream(); + } + + @Override + public String getSourceName() + { + return delegate.getSourceName(); + } + + @Override + public void setTokenFactory(TokenFactory factory) + { + delegate.setTokenFactory(factory); + } + + @Override + public TokenFactory getTokenFactory() + { + return delegate.getTokenFactory(); + } + } + + private static final class ParseBudgetListener + implements ParseTreeListener + { + private final HogQlParserLimits limits; + private int depth; + private int nodes; + + private ParseBudgetListener(HogQlParserLimits limits) + { + this.limits = requireNonNull(limits, "limits is null"); + } + + @Override + public void visitTerminal(TerminalNode node) + { + addNode(node.getSymbol()); + } + + @Override + public void visitErrorNode(ErrorNode node) + { + addNode(node.getSymbol()); + } + + @Override + public void enterEveryRule(ParserRuleContext context) + { + depth++; + if (depth > limits.maxParseDepth()) { + throw limitExceeded("parse depth limit exceeded", context.getStart()); + } + addNode(context.getStart()); + } + + @Override + public void exitEveryRule(ParserRuleContext context) + { + depth--; + } + + private void addNode(Token token) + { + if (++nodes > limits.maxParseTreeNodes()) { + throw limitExceeded("parse tree node limit exceeded", token); + } + } + + private static HogQlParsingException limitExceeded(String message, Token token) + { + return new HogQlParsingException(message, null, token.getLine(), token.getCharPositionInLine() + 1); + } + } + private static final class SyntaxTreeBuilder { private final SourcePositions sourcePositions; diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParserLimits.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParserLimits.java new file mode 100644 index 000000000000..5aca7dd88308 --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParserLimits.java @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +public record HogQlParserLimits(int maxTokens, int maxParseDepth, int maxParseTreeNodes) +{ + private static final HogQlParserLimits DEFAULTS = new HogQlParserLimits(100_000, 256, 500_000); + + public HogQlParserLimits + { + if (maxTokens < 1) { + throw new IllegalArgumentException("maxTokens must be positive"); + } + if (maxParseDepth < 1) { + throw new IllegalArgumentException("maxParseDepth must be positive"); + } + if (maxParseTreeNodes < 1) { + throw new IllegalArgumentException("maxParseTreeNodes must be positive"); + } + } + + public static HogQlParserLimits defaults() + { + return DEFAULTS; + } +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserLimits.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserLimits.java new file mode 100644 index 000000000000..a93bf654a92f --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserLimits.java @@ -0,0 +1,62 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlParserLimits +{ + @Test + public void testRejectsTokenBomb() + { + HogQlParser parser = new HogQlParser(new HogQlParserLimits(5, 100, 1_000)); + + assertThatThrownBy(() -> parser.parseSyntax("SELECT one, two, three")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("token limit exceeded"); + } + + @Test + public void testRejectsDeeplyNestedExpression() + { + HogQlParser parser = new HogQlParser(new HogQlParserLimits(1_000, 32, 10_000)); + String query = "SELECT " + "(".repeat(64) + "1" + ")".repeat(64); + + assertThatThrownBy(() -> parser.parseSyntax(query)) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("parse depth limit exceeded"); + } + + @Test + public void testRejectsParseTreeNodeBomb() + { + HogQlParser parser = new HogQlParser(new HogQlParserLimits(1_000, 100, 25)); + + assertThatThrownBy(() -> parser.parseSyntax("SELECT one, two, three, four")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("parse tree node limit exceeded"); + } + + @Test + public void testAcceptsInputWithinInjectedLimits() + { + HogQlParser parser = new HogQlParser(new HogQlParserLimits(100, 100, 1_000)); + + assertThatCode(() -> parser.parseSyntax("SELECT event FROM events WHERE event = 'signup'")) + .doesNotThrowAnyException(); + } +} From cb2564a059c4b630a5b6ab542bf1e851e7f06270 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 08:55:30 -0400 Subject: [PATCH 012/136] feat(hogql): classify non-query syntax --- .../io/trino/hogql/parser/HogQlParser.java | 26 ++++++++++++++++--- .../hogql/parser/tree/HogQlSyntaxTree.java | 1 + .../trino/hogql/parser/TestHogQlParser.java | 11 ++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index a7b3636e72ee..ccebb5a4bea9 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -302,12 +302,32 @@ private SyntaxTreeBuilder(String source) public HogQlSyntaxTree build(HogQLParser.SelectContext context) { - HogQlSyntaxTree.LanguageClass languageClass = context.hogqlxTagElement() == null ? - HogQlSyntaxTree.LanguageClass.READ_ONLY_QUERY : - HogQlSyntaxTree.LanguageClass.HOGQLX; + HogQlSyntaxTree.LanguageClass languageClass; + if (context.hogqlxTagElement() != null) { + languageClass = HogQlSyntaxTree.LanguageClass.HOGQLX; + } + else if (containsRule(context, HogQLParser.RULE_block)) { + languageClass = HogQlSyntaxTree.LanguageClass.PROCEDURAL; + } + else { + languageClass = HogQlSyntaxTree.LanguageClass.READ_ONLY_QUERY; + } return new HogQlSyntaxTree(languageClass, buildNode(context)); } + private static boolean containsRule(ParserRuleContext context, int ruleIndex) + { + if (context.getRuleIndex() == ruleIndex) { + return true; + } + for (int index = 0; index < context.getChildCount(); index++) { + if (context.getChild(index) instanceof ParserRuleContext child && containsRule(child, ruleIndex)) { + return true; + } + } + return false; + } + private HogQlSyntaxTree.Node buildNode(ParserRuleContext context) { String rule = HogQLParser.ruleNames[context.getRuleIndex()]; diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlSyntaxTree.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlSyntaxTree.java index 1f53b43e4a1d..74cdf2516667 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlSyntaxTree.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlSyntaxTree.java @@ -30,6 +30,7 @@ public enum LanguageClass { READ_ONLY_QUERY, HOGQLX, + PROCEDURAL, } public sealed interface Element diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 5e545a2fd8be..40e77d66405e 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -99,6 +99,17 @@ public void testSyntaxTreeClassifiesHogQlXAsNonQuerySyntax() .contains("hogqlxTagElement"); } + @Test + public void testSyntaxTreeClassifiesBlockLambdaAsProcedural() + { + HogQlSyntaxTree syntaxTree = parser.parseSyntax("SELECT value -> { RETURN value }"); + + assertThat(syntaxTree.languageClass()).isEqualTo(HogQlSyntaxTree.LanguageClass.PROCEDURAL); + assertThat(nodes(syntaxTree.root())) + .extracting(Node::rule) + .contains("block", "returnStmt"); + } + private static Stream nodes(Element element) { if (!(element instanceof Node node)) { From 76cfc1101f3649eb33be01acbeb9b19d9cf5c4c9 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 08:59:04 -0400 Subject: [PATCH 013/136] feat(hogql): publish the syntax AST manifest --- .../hogql/parser/HogQlSyntaxAstManifest.java | 273 ++ .../language/1.0.0/trino-syntax-ast.json | 2343 +++++++++++++++++ .../parser/TestHogQlSyntaxAstManifest.java | 124 + 3 files changed, 2740 insertions(+) create mode 100644 core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlSyntaxAstManifest.java create mode 100644 core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-syntax-ast.json create mode 100644 core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlSyntaxAstManifest.java diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlSyntaxAstManifest.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlSyntaxAstManifest.java new file mode 100644 index 000000000000..1ecc3e7e4ffd --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlSyntaxAstManifest.java @@ -0,0 +1,273 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; +import io.airlift.json.JsonCodec; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; + +public record HogQlSyntaxAstManifest( + int schemaVersion, + HogQlLanguageVersion languageVersion, + String grammarSha256, + String grammarFeatureManifestSha256, + String grammarAlternativeIdentityManifestSha256, + SyntaxNodeKind syntaxTreeKind, + SourceSpanGuarantee sourceSpanGuarantee, + List features) +{ + private static final String LANGUAGE_RESOURCE_ROOT = "/io/trino/hogql/parser/language/1.0.0/"; + private static final String CURRENT_RESOURCE = LANGUAGE_RESOURCE_ROOT + "trino-syntax-ast.json"; + private static final JsonCodec MANIFEST_CODEC = jsonCodec(HogQlSyntaxAstManifest.class); + private static final JsonCodec GRAMMAR_FEATURE_MANIFEST_CODEC = jsonCodec(HogQlCompatibilityManifest.PublishedGrammarFeatureManifest.class); + private static final HogQlSyntaxAstManifest CURRENT = loadCurrent(); + + @JsonCreator + public HogQlSyntaxAstManifest( + @JsonProperty("schemaVersion") int schemaVersion, + @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, + @JsonProperty("grammarSha256") String grammarSha256, + @JsonProperty("grammarFeatureManifestSha256") String grammarFeatureManifestSha256, + @JsonProperty("grammarAlternativeIdentityManifestSha256") String grammarAlternativeIdentityManifestSha256, + @JsonProperty("syntaxTreeKind") SyntaxNodeKind syntaxTreeKind, + @JsonProperty("sourceSpanGuarantee") SourceSpanGuarantee sourceSpanGuarantee, + @JsonProperty("features") List features) + { + if (schemaVersion != 1) { + throw new IllegalArgumentException("unsupported HogQL syntax AST manifest schema: " + schemaVersion); + } + this.schemaVersion = schemaVersion; + this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); + this.grammarFeatureManifestSha256 = requireNonNull(grammarFeatureManifestSha256, "grammarFeatureManifestSha256 is null"); + this.grammarAlternativeIdentityManifestSha256 = requireNonNull(grammarAlternativeIdentityManifestSha256, "grammarAlternativeIdentityManifestSha256 is null"); + this.syntaxTreeKind = requireNonNull(syntaxTreeKind, "syntaxTreeKind is null"); + this.sourceSpanGuarantee = requireNonNull(sourceSpanGuarantee, "sourceSpanGuarantee is null"); + this.features = List.copyOf(requireNonNull(features, "features is null")); + if (syntaxTreeKind != SyntaxNodeKind.TREE) { + throw new IllegalArgumentException("HogQL syntax AST root must use the tree node kind"); + } + } + + public static HogQlSyntaxAstManifest current() + { + return CURRENT; + } + + private static HogQlSyntaxAstManifest loadCurrent() + { + HogQlSyntaxAstManifest manifest = MANIFEST_CODEC.fromJson(readResource(CURRENT_RESOURCE)); + HogQlLanguageContract languageContract = HogQlLanguageContract.current(); + if (!manifest.languageVersion().equals(languageContract.languageVersion())) { + throw new IllegalStateException("HogQL syntax AST manifest language version does not match the language contract"); + } + if (!manifest.grammarSha256().equals(languageContract.grammarSha256())) { + throw new IllegalStateException("HogQL syntax AST manifest grammar hash does not match the language contract"); + } + if (!manifest.grammarFeatureManifestSha256().equals(languageContract.grammarFeatureManifest().sha256())) { + throw new IllegalStateException("HogQL syntax AST manifest feature hash does not match the language contract"); + } + if (!manifest.grammarAlternativeIdentityManifestSha256().equals(HogQlGrammarAlternativeIdentities.currentSha256())) { + throw new IllegalStateException("HogQL syntax AST manifest alternative identity hash does not match the published sidecar"); + } + + String featureManifestResource = LANGUAGE_RESOURCE_ROOT + languageContract.grammarFeatureManifest().path(); + byte[] featureManifestBytes = readResourceBytes(featureManifestResource); + if (!sha256(featureManifestBytes).equals(manifest.grammarFeatureManifestSha256())) { + throw new IllegalStateException("published HogQL grammar feature manifest checksum does not match the syntax AST manifest"); + } + HogQlCompatibilityManifest.PublishedGrammarFeatureManifest published = GRAMMAR_FEATURE_MANIFEST_CODEC.fromJson(new String(featureManifestBytes, StandardCharsets.UTF_8)); + validateFeatureCoverage(manifest.features(), published, HogQlGrammarAlternativeIdentities.current()); + return manifest; + } + + static void validateFeatureCoverage( + List features, + HogQlCompatibilityManifest.PublishedGrammarFeatureManifest published, + HogQlGrammarAlternativeIdentities alternativeIdentities) + { + HogQlLanguageContract languageContract = HogQlLanguageContract.current(); + if (!published.languageVersion().equals(languageContract.languageVersion()) || !published.grammarSha256().equals(languageContract.grammarSha256())) { + throw new IllegalStateException("published HogQL grammar features do not match the language contract"); + } + + Map expectedById = new HashMap<>(); + for (HogQlCompatibilityManifest.PublishedFeature feature : published.features()) { + SyntaxNodeKind nodeKind = switch (feature.kind()) { + case "token" -> SyntaxNodeKind.TOKEN; + case "parserRule", "parserAlternative" -> SyntaxNodeKind.RULE; + default -> throw new IllegalStateException("unknown published HogQL grammar feature kind: " + feature.kind()); + }; + if (expectedById.put(feature.id(), nodeKind) != null) { + throw new IllegalStateException("duplicate published HogQL grammar feature: " + feature.id()); + } + } + + Map sourceUnlabeledByRule = new HashMap<>(); + for (HogQlCompatibilityManifest.SourceUnlabeledAlternativeRule rule : published.validationErrors()) { + if (sourceUnlabeledByRule.put(rule.rule(), rule) != null) { + throw new IllegalStateException("duplicate source-unlabeled HogQL grammar alternative rule: " + rule.rule()); + } + } + for (HogQlGrammarAlternativeIdentities.AlternativeRule rule : alternativeIdentities.rules()) { + HogQlCompatibilityManifest.SourceUnlabeledAlternativeRule sourceRule = sourceUnlabeledByRule.remove(rule.rule()); + if (sourceRule == null || sourceRule.alternativeCount() != rule.alternatives().size() || sourceRule.queryReachable() != rule.queryReachable()) { + throw new IllegalStateException("HogQL grammar alternative identities do not match the published rule: " + rule.rule()); + } + for (HogQlGrammarAlternativeIdentities.Alternative alternative : rule.alternatives()) { + if (expectedById.put(alternative.id(), SyntaxNodeKind.RULE) != null) { + throw new IllegalStateException("duplicate published HogQL grammar feature: " + alternative.id()); + } + } + } + if (!sourceUnlabeledByRule.isEmpty()) { + throw new IllegalStateException("HogQL grammar alternative identities do not cover every source-unlabeled rule"); + } + + Map manifestById = new HashMap<>(); + for (Feature feature : features) { + if (manifestById.put(feature.id(), feature) != null) { + throw new IllegalStateException("duplicate HogQL syntax AST feature: " + feature.id()); + } + SyntaxNodeKind expected = expectedById.get(feature.id()); + if (expected == null) { + throw new IllegalStateException("unknown HogQL syntax AST feature: " + feature.id()); + } + if (feature.syntaxNodeKind() != expected) { + throw new IllegalStateException("HogQL syntax AST feature has the wrong node kind: " + feature.id()); + } + } + if (!manifestById.keySet().equals(expectedById.keySet())) { + throw new IllegalStateException("HogQL syntax AST manifest does not account for every grammar feature"); + } + } + + private static String readResource(String path) + { + return new String(readResourceBytes(path), StandardCharsets.UTF_8); + } + + private static byte[] readResourceBytes(String path) + { + try (InputStream input = HogQlSyntaxAstManifest.class.getResourceAsStream(path)) { + if (input == null) { + throw new IllegalStateException("HogQL syntax AST resource is missing: " + path); + } + return input.readAllBytes(); + } + catch (IOException e) { + throw new UncheckedIOException("failed to read HogQL syntax AST resource", e); + } + } + + private static String sha256(byte[] content) + { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content)); + } + catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } + + public record Feature(String id, SyntaxNodeKind syntaxNodeKind) + { + @JsonCreator + public Feature( + @JsonProperty("id") String id, + @JsonProperty("syntaxNodeKind") SyntaxNodeKind syntaxNodeKind) + { + this.id = requireNonNull(id, "id is null"); + this.syntaxNodeKind = requireNonNull(syntaxNodeKind, "syntaxNodeKind is null"); + if (id.isBlank() || syntaxNodeKind == SyntaxNodeKind.TREE) { + throw new IllegalArgumentException("invalid HogQL syntax AST feature: " + id); + } + } + } + + public enum SyntaxNodeKind + { + TREE("tree"), + RULE("rule"), + TOKEN("token"); + + private final String value; + + SyntaxNodeKind(String value) + { + this.value = value; + } + + @JsonCreator + public static SyntaxNodeKind fromJson(String value) + { + for (SyntaxNodeKind kind : values()) { + if (kind.value.equals(value)) { + return kind; + } + } + throw new IllegalArgumentException("unknown HogQL syntax node kind: " + value); + } + + @JsonValue + public String toJson() + { + return value; + } + } + + public enum SourceSpanGuarantee + { + CODE_POINT_OFFSETS_END_EXCLUSIVE_ONE_BASED_LINE_COLUMNS("codePointOffsetsEndExclusiveOneBasedLineColumns"); + + private final String value; + + SourceSpanGuarantee(String value) + { + this.value = value; + } + + @JsonCreator + public static SourceSpanGuarantee fromJson(String value) + { + for (SourceSpanGuarantee guarantee : values()) { + if (guarantee.value.equals(value)) { + return guarantee; + } + } + throw new IllegalArgumentException("unknown HogQL source span guarantee: " + value); + } + + @JsonValue + public String toJson() + { + return value; + } + } +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-syntax-ast.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-syntax-ast.json new file mode 100644 index 000000000000..ef67c6be9f79 --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-syntax-ast.json @@ -0,0 +1,2343 @@ +{ + "schemaVersion": 1, + "languageVersion": "1.0.0", + "grammarSha256": "c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242", + "grammarFeatureManifestSha256": "fbf7236643b43d929e9ef1c8fd01a04b746736dca2f965cc2e78ad62f1a98b10", + "grammarAlternativeIdentityManifestSha256": "55e33d04906b058bdbab00507b402766d386e4f771b60e4f5106fef63e18383d", + "syntaxTreeKind": "tree", + "sourceSpanGuarantee": "codePointOffsetsEndExclusiveOneBasedLineColumns", + "features": [ + { + "id": "alternative:alias:keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:alias:quoted-identifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:alias:unquoted-identifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:arrayJoinClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:block", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:catchBlock", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnAliases", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExpr:ColumnExprAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExpr:ColumnExprAnd", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExpr:ColumnExprOr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExpr:ColumnExprTernaryOp", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExpr:ColumnExprValuePassthrough", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprTupleOrSingle:scalar", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprTupleOrSingle:tuple", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprArray", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprArrayAccess", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprArraySlice", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprAsterisk", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprBetween", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprCall", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprCallSelect", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprCase", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprCast", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColonLambda", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsAll", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsExclude", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsExcludeReplace", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedAll", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedExclude", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedExcludeReplace", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedReplace", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsRegex", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsReplace", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprDate", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprDict", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprFunction", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprFunctionWithinGroup", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprIgnoreNulls", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprInterval", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprIntervalString", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprIsDistinctFrom", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprIsNull", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprLambda", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprLiteral", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNamedArg", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNegate", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNot", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullArrayAccess", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullPropertyAccess", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullSafeEq", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullTupleAccess", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullish", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprParens", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprPositional", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence1", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence2", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence3", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprPropertyAccess", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprSpreadColumnsList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprSpreadColumnsRegex", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprSubquery", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprSubstring", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTagElement", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTemplateString", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTimestamp", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTrim", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTryCast", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTuple", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTupleAccess", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTypeCast", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunction", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunctionTarget", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnIdentifier:field-path", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnIdentifier:placeholder", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnLambdaExpr:ArrowLambda", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnLambdaExpr:ColonLambda", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeCastExpr:ColumnTypeCastExprSimple", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeCastExpr:ColumnTypeCastExprWithTimeZone", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeCastIdentifier:interval-keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeCastIdentifier:quoted-identifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeCastIdentifier:type-keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeCastIdentifier:unquoted-identifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprArray", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprComplex", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprCompound", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprEnum", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprNested", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprParam", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprSimple", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnsReplaceItem", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnsReplaceList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:databaseIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:declaration:statement", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:declaration:variable", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:emptyStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:enumValue", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:expr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:exprStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:expression", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:floatingLiteral:floating-token", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:floatingLiteral:leading-decimal-point", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:floatingLiteral:trailing-decimal-point", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:forInStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:forStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:fromClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:fullTemplateString", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:funcStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:groupByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:groupingSet", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:groupingSetList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:havingClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxChildElement:expression", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxChildElement:tag", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxChildElement:text", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxTagAttribute:boolean", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxTagAttribute:expression", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxTagAttribute:string", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxTagElement:HogqlxTagElementClosed", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxTagElement:HogqlxTagElementNested", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxText", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:identifier:interval-keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:identifier:keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:identifier:quoted", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:identifier:unquoted", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:identifierList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:ifStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:implicitAlias:keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:implicitAlias:quoted-identifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:implicitAlias:unquoted-identifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:interpolateClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:interpolateExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:interval", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinConstraintClause:on", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinConstraintClause:using", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinConstraintClause:using-parenthesized", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinExpr:JoinExprCrossOp", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinExpr:JoinExprOp", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinExpr:JoinExprParens", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinExpr:JoinExprPivot", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinExpr:JoinExprPositional", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinExpr:JoinExprTable", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinExpr:JoinExprUnpivot", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinOp:JoinOpFull", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinOp:JoinOpInner", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinOp:JoinOpLeftRight", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinOpCross:comma", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinOpCross:cross-join", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:keywordForAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:keywordForImplicitAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:keywordForTypeCast", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:kvPair", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:kvPairList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:limitAndOffsetClause:compact", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:limitAndOffsetClause:with-offset", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:limitAndOffsetClauseOptional:limit", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:limitAndOffsetClauseOptional:limit-with-offset", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:limitAndOffsetClauseOptional:offset-only", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:limitByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:limitExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:literal:null", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:literal:number", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:literal:string", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:nestedIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:numberLiteral", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:offsetOnlyClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:orderByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:orderExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:orderExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:pivotColumn", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:pivotColumnList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:placeholder", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:prewhereClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:program", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:projectionOrderByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:qualifyClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:ratioExpr:numeric", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:ratioExpr:placeholder", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:returnStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:sampleClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:select", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprAliasBefore", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprAliasImplicit", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprInvalidFromImplicitAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprSelectValue", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectColumnExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectColumnExprListBeforeFrom:SelectColumnExprListBeforeFromPlain", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectColumnExprListBeforeFrom:SelectColumnExprListBeforeFromTrailingComma", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectSetStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectStmtWithParens:direct", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectStmtWithParens:parenthesized", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectStmtWithParens:placeholder", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectStmtWithParens:with-parenthesized", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:settingExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:settingExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:settingsClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:block", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:empty", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:expression", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:for", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:for-in", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:function", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:if", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:return", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:throw", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:try-catch", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:while", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:string:literal", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:string:template", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:stringContents:interpolation", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:stringContents:text", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:stringContentsFull:interpolation", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:stringContentsFull:text", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:subsequentSelectSetClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableArgList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprFunction", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprPivot", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprPlaceholder", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprSubquery", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprTag", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprUnpivot", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprValues", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableFunctionExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:templateString", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:throwStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:topClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tryCatchStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:unpivotColumn", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:unpivotColumnList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:valuesClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:valuesRow", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:varAssignment", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:varDecl", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:whereClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:whileStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:winFrameBound", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:winFrameClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:winFrameExtend:frameBetween", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:winFrameExtend:frameStart", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:winOrderByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:winPartitionByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:windowClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:windowExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:withClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:withExpr:WithExprColumn", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:withExpr:WithExprSubquery", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:withExprColumnNameList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:withExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:withFillClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:withinGroupClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:alias", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:arrayJoinClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:block", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:catchBlock", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnAliases", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnExprTupleOrSingle", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnExprValue", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnLambdaExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnTypeCastExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnTypeCastIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnTypeExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnsReplaceItem", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnsReplaceList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:databaseIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:declaration", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:emptyStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:enumValue", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:expr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:exprStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:expression", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:floatingLiteral", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:forInStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:forStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:fromClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:fullTemplateString", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:funcStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:groupByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:groupingSet", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:groupingSetList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:havingClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:hogqlxChildElement", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:hogqlxTagAttribute", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:hogqlxTagElement", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:hogqlxText", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:identifier", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:identifierList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:ifStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:implicitAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:interpolateClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:interpolateExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:interval", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:joinConstraintClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:joinExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:joinOp", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:joinOpCross", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:keywordForAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:keywordForImplicitAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:keywordForTypeCast", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:kvPair", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:kvPairList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:limitAndOffsetClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:limitAndOffsetClauseOptional", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:limitByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:limitExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:literal", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:nestedIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:numberLiteral", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:offsetOnlyClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:orderByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:orderExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:orderExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:pivotColumn", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:pivotColumnList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:placeholder", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:prewhereClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:program", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:projectionOrderByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:qualifyClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:ratioExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:returnStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:sampleClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:select", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:selectColumnExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:selectColumnExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:selectColumnExprListBeforeFrom", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:selectSetStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:selectStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:selectStmtWithParens", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:settingExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:settingExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:settingsClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:statement", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:string", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:stringContents", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:stringContentsFull", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:subsequentSelectSetClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:tableArgList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:tableExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:tableFunctionExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:tableIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:templateString", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:throwStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:topClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:tryCatchStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:unpivotColumn", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:unpivotColumnList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:valuesClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:valuesRow", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:varAssignment", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:varDecl", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:whereClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:whileStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:winFrameBound", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:winFrameClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:winFrameExtend", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:winOrderByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:winPartitionByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:windowClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:windowExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:withClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:withExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:withExprColumnNameList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:withExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:withFillClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:withinGroupClause", + "syntaxNodeKind": "rule" + }, + { + "id": "token:ALL", + "syntaxNodeKind": "token" + }, + { + "id": "token:AND", + "syntaxNodeKind": "token" + }, + { + "id": "token:ANTI", + "syntaxNodeKind": "token" + }, + { + "id": "token:ANY", + "syntaxNodeKind": "token" + }, + { + "id": "token:ARRAY", + "syntaxNodeKind": "token" + }, + { + "id": "token:ARROW", + "syntaxNodeKind": "token" + }, + { + "id": "token:AS", + "syntaxNodeKind": "token" + }, + { + "id": "token:ASCENDING", + "syntaxNodeKind": "token" + }, + { + "id": "token:ASOF", + "syntaxNodeKind": "token" + }, + { + "id": "token:ASTERISK", + "syntaxNodeKind": "token" + }, + { + "id": "token:BACKQUOTE", + "syntaxNodeKind": "token" + }, + { + "id": "token:BACKSLASH", + "syntaxNodeKind": "token" + }, + { + "id": "token:BETWEEN", + "syntaxNodeKind": "token" + }, + { + "id": "token:BINARY_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:BOTH", + "syntaxNodeKind": "token" + }, + { + "id": "token:BY", + "syntaxNodeKind": "token" + }, + { + "id": "token:CASE", + "syntaxNodeKind": "token" + }, + { + "id": "token:CAST", + "syntaxNodeKind": "token" + }, + { + "id": "token:CATCH", + "syntaxNodeKind": "token" + }, + { + "id": "token:COHORT", + "syntaxNodeKind": "token" + }, + { + "id": "token:COLLATE", + "syntaxNodeKind": "token" + }, + { + "id": "token:COLON", + "syntaxNodeKind": "token" + }, + { + "id": "token:COLONEQUALS", + "syntaxNodeKind": "token" + }, + { + "id": "token:COLUMNS", + "syntaxNodeKind": "token" + }, + { + "id": "token:COMMA", + "syntaxNodeKind": "token" + }, + { + "id": "token:CONCAT", + "syntaxNodeKind": "token" + }, + { + "id": "token:CROSS", + "syntaxNodeKind": "token" + }, + { + "id": "token:CUBE", + "syntaxNodeKind": "token" + }, + { + "id": "token:CURRENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:DASH", + "syntaxNodeKind": "token" + }, + { + "id": "token:DATE", + "syntaxNodeKind": "token" + }, + { + "id": "token:DAY", + "syntaxNodeKind": "token" + }, + { + "id": "token:DECIMAL_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:DESC", + "syntaxNodeKind": "token" + }, + { + "id": "token:DESCENDING", + "syntaxNodeKind": "token" + }, + { + "id": "token:DISTINCT", + "syntaxNodeKind": "token" + }, + { + "id": "token:DOLLAR", + "syntaxNodeKind": "token" + }, + { + "id": "token:DOT", + "syntaxNodeKind": "token" + }, + { + "id": "token:DOUBLECOLON", + "syntaxNodeKind": "token" + }, + { + "id": "token:ELSE", + "syntaxNodeKind": "token" + }, + { + "id": "token:END", + "syntaxNodeKind": "token" + }, + { + "id": "token:EQ_DOUBLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:EQ_SINGLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:ESCAPE_CHAR_COMMON", + "syntaxNodeKind": "token" + }, + { + "id": "token:EXCEPT", + "syntaxNodeKind": "token" + }, + { + "id": "token:EXCLUDE", + "syntaxNodeKind": "token" + }, + { + "id": "token:EXTRACT", + "syntaxNodeKind": "token" + }, + { + "id": "token:FILL", + "syntaxNodeKind": "token" + }, + { + "id": "token:FILTER", + "syntaxNodeKind": "token" + }, + { + "id": "token:FINAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:FINALLY", + "syntaxNodeKind": "token" + }, + { + "id": "token:FIRST", + "syntaxNodeKind": "token" + }, + { + "id": "token:FLOATING_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:FN", + "syntaxNodeKind": "token" + }, + { + "id": "token:FOLLOWING", + "syntaxNodeKind": "token" + }, + { + "id": "token:FOR", + "syntaxNodeKind": "token" + }, + { + "id": "token:FROM", + "syntaxNodeKind": "token" + }, + { + "id": "token:FULL", + "syntaxNodeKind": "token" + }, + { + "id": "token:FULL_STRING_ESCAPE_TRIGGER", + "syntaxNodeKind": "token" + }, + { + "id": "token:FULL_STRING_TEXT", + "syntaxNodeKind": "token" + }, + { + "id": "token:FUN", + "syntaxNodeKind": "token" + }, + { + "id": "token:GROUP", + "syntaxNodeKind": "token" + }, + { + "id": "token:GROUPING", + "syntaxNodeKind": "token" + }, + { + "id": "token:GT", + "syntaxNodeKind": "token" + }, + { + "id": "token:GT_EQ", + "syntaxNodeKind": "token" + }, + { + "id": "token:HASH", + "syntaxNodeKind": "token" + }, + { + "id": "token:HASH_COMMENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:HAVING", + "syntaxNodeKind": "token" + }, + { + "id": "token:HEXADECIMAL_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:HOGQLX_TEXT_TEXT", + "syntaxNodeKind": "token" + }, + { + "id": "token:HOGQLX_TEXT_WS", + "syntaxNodeKind": "token" + }, + { + "id": "token:HOUR", + "syntaxNodeKind": "token" + }, + { + "id": "token:ID", + "syntaxNodeKind": "token" + }, + { + "id": "token:IDENTIFIER", + "syntaxNodeKind": "token" + }, + { + "id": "token:IF", + "syntaxNodeKind": "token" + }, + { + "id": "token:IGNORE", + "syntaxNodeKind": "token" + }, + { + "id": "token:ILIKE", + "syntaxNodeKind": "token" + }, + { + "id": "token:IN", + "syntaxNodeKind": "token" + }, + { + "id": "token:INCLUDE", + "syntaxNodeKind": "token" + }, + { + "id": "token:INF", + "syntaxNodeKind": "token" + }, + { + "id": "token:INNER", + "syntaxNodeKind": "token" + }, + { + "id": "token:INTERPOLATE", + "syntaxNodeKind": "token" + }, + { + "id": "token:INTERSECT", + "syntaxNodeKind": "token" + }, + { + "id": "token:INTERVAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:IREGEX_DOUBLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:IREGEX_SINGLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:IS", + "syntaxNodeKind": "token" + }, + { + "id": "token:JOIN", + "syntaxNodeKind": "token" + }, + { + "id": "token:KEY", + "syntaxNodeKind": "token" + }, + { + "id": "token:LAMBDA", + "syntaxNodeKind": "token" + }, + { + "id": "token:LAST", + "syntaxNodeKind": "token" + }, + { + "id": "token:LBRACE", + "syntaxNodeKind": "token" + }, + { + "id": "token:LBRACKET", + "syntaxNodeKind": "token" + }, + { + "id": "token:LEADING", + "syntaxNodeKind": "token" + }, + { + "id": "token:LEFT", + "syntaxNodeKind": "token" + }, + { + "id": "token:LET", + "syntaxNodeKind": "token" + }, + { + "id": "token:LIKE", + "syntaxNodeKind": "token" + }, + { + "id": "token:LIMIT", + "syntaxNodeKind": "token" + }, + { + "id": "token:LOCAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:LPAREN", + "syntaxNodeKind": "token" + }, + { + "id": "token:LT", + "syntaxNodeKind": "token" + }, + { + "id": "token:LT_EQ", + "syntaxNodeKind": "token" + }, + { + "id": "token:LT_SLASH", + "syntaxNodeKind": "token" + }, + { + "id": "token:MALFORMED_BINARY_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:MATERIALIZED", + "syntaxNodeKind": "token" + }, + { + "id": "token:MINUTE", + "syntaxNodeKind": "token" + }, + { + "id": "token:MONTH", + "syntaxNodeKind": "token" + }, + { + "id": "token:MULTI_LINE_COMMENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:NAME", + "syntaxNodeKind": "token" + }, + { + "id": "token:NAN_SQL", + "syntaxNodeKind": "token" + }, + { + "id": "token:NATURAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:NOT", + "syntaxNodeKind": "token" + }, + { + "id": "token:NOT_EQ", + "syntaxNodeKind": "token" + }, + { + "id": "token:NOT_IREGEX", + "syntaxNodeKind": "token" + }, + { + "id": "token:NOT_REGEX", + "syntaxNodeKind": "token" + }, + { + "id": "token:NULLISH", + "syntaxNodeKind": "token" + }, + { + "id": "token:NULLS", + "syntaxNodeKind": "token" + }, + { + "id": "token:NULL_PROPERTY", + "syntaxNodeKind": "token" + }, + { + "id": "token:NULL_SAFE_EQ", + "syntaxNodeKind": "token" + }, + { + "id": "token:NULL_SQL", + "syntaxNodeKind": "token" + }, + { + "id": "token:OCTAL_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:OCTAL_PREFIX_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:OFFSET", + "syntaxNodeKind": "token" + }, + { + "id": "token:ON", + "syntaxNodeKind": "token" + }, + { + "id": "token:OR", + "syntaxNodeKind": "token" + }, + { + "id": "token:ORDER", + "syntaxNodeKind": "token" + }, + { + "id": "token:OUTER", + "syntaxNodeKind": "token" + }, + { + "id": "token:OVER", + "syntaxNodeKind": "token" + }, + { + "id": "token:PARTITION", + "syntaxNodeKind": "token" + }, + { + "id": "token:PERCENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:PIVOT", + "syntaxNodeKind": "token" + }, + { + "id": "token:PLUS", + "syntaxNodeKind": "token" + }, + { + "id": "token:POSITIONAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:PRECEDING", + "syntaxNodeKind": "token" + }, + { + "id": "token:PREWHERE", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUALIFY", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUARTER", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUERY", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUOTED_IDENTIFIER", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUOTE_DOUBLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUOTE_SINGLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUOTE_SINGLE_TEMPLATE", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUOTE_SINGLE_TEMPLATE_FULL", + "syntaxNodeKind": "token" + }, + { + "id": "token:RANGE", + "syntaxNodeKind": "token" + }, + { + "id": "token:RBRACE", + "syntaxNodeKind": "token" + }, + { + "id": "token:RBRACKET", + "syntaxNodeKind": "token" + }, + { + "id": "token:RECURSIVE", + "syntaxNodeKind": "token" + }, + { + "id": "token:REGEX_DOUBLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:REGEX_SINGLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:REPLACE", + "syntaxNodeKind": "token" + }, + { + "id": "token:RETURN", + "syntaxNodeKind": "token" + }, + { + "id": "token:RIGHT", + "syntaxNodeKind": "token" + }, + { + "id": "token:ROLLUP", + "syntaxNodeKind": "token" + }, + { + "id": "token:ROW", + "syntaxNodeKind": "token" + }, + { + "id": "token:ROWS", + "syntaxNodeKind": "token" + }, + { + "id": "token:RPAREN", + "syntaxNodeKind": "token" + }, + { + "id": "token:SAMPLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:SECOND", + "syntaxNodeKind": "token" + }, + { + "id": "token:SELECT", + "syntaxNodeKind": "token" + }, + { + "id": "token:SEMI", + "syntaxNodeKind": "token" + }, + { + "id": "token:SEMICOLON", + "syntaxNodeKind": "token" + }, + { + "id": "token:SETS", + "syntaxNodeKind": "token" + }, + { + "id": "token:SETTINGS", + "syntaxNodeKind": "token" + }, + { + "id": "token:SINGLE_LINE_COMMENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:SLASH", + "syntaxNodeKind": "token" + }, + { + "id": "token:SLASH_GT", + "syntaxNodeKind": "token" + }, + { + "id": "token:STEP", + "syntaxNodeKind": "token" + }, + { + "id": "token:STRING_ESCAPE_TRIGGER", + "syntaxNodeKind": "token" + }, + { + "id": "token:STRING_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:STRING_TEXT", + "syntaxNodeKind": "token" + }, + { + "id": "token:SUBSTRING", + "syntaxNodeKind": "token" + }, + { + "id": "token:TAGC_MULTI_LINE_COMMENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:TAGC_SINGLE_LINE_COMMENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:TAGC_WS", + "syntaxNodeKind": "token" + }, + { + "id": "token:TAG_MULTI_LINE_COMMENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:TAG_SINGLE_LINE_COMMENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:TAG_WS", + "syntaxNodeKind": "token" + }, + { + "id": "token:THEN", + "syntaxNodeKind": "token" + }, + { + "id": "token:THROW", + "syntaxNodeKind": "token" + }, + { + "id": "token:TIES", + "syntaxNodeKind": "token" + }, + { + "id": "token:TIME", + "syntaxNodeKind": "token" + }, + { + "id": "token:TIMESTAMP", + "syntaxNodeKind": "token" + }, + { + "id": "token:TO", + "syntaxNodeKind": "token" + }, + { + "id": "token:TOP", + "syntaxNodeKind": "token" + }, + { + "id": "token:TOTALS", + "syntaxNodeKind": "token" + }, + { + "id": "token:TRAILING", + "syntaxNodeKind": "token" + }, + { + "id": "token:TRIM", + "syntaxNodeKind": "token" + }, + { + "id": "token:TRUNCATE", + "syntaxNodeKind": "token" + }, + { + "id": "token:TRY", + "syntaxNodeKind": "token" + }, + { + "id": "token:TRY_CAST", + "syntaxNodeKind": "token" + }, + { + "id": "token:UNBOUNDED", + "syntaxNodeKind": "token" + }, + { + "id": "token:UNDERSCORE", + "syntaxNodeKind": "token" + }, + { + "id": "token:UNEXPECTED_CHARACTER", + "syntaxNodeKind": "token" + }, + { + "id": "token:UNION", + "syntaxNodeKind": "token" + }, + { + "id": "token:UNPIVOT", + "syntaxNodeKind": "token" + }, + { + "id": "token:USING", + "syntaxNodeKind": "token" + }, + { + "id": "token:VALUES", + "syntaxNodeKind": "token" + }, + { + "id": "token:WEEK", + "syntaxNodeKind": "token" + }, + { + "id": "token:WHEN", + "syntaxNodeKind": "token" + }, + { + "id": "token:WHERE", + "syntaxNodeKind": "token" + }, + { + "id": "token:WHILE", + "syntaxNodeKind": "token" + }, + { + "id": "token:WHITESPACE", + "syntaxNodeKind": "token" + }, + { + "id": "token:WINDOW", + "syntaxNodeKind": "token" + }, + { + "id": "token:WITH", + "syntaxNodeKind": "token" + }, + { + "id": "token:WITHIN", + "syntaxNodeKind": "token" + }, + { + "id": "token:YEAR", + "syntaxNodeKind": "token" + }, + { + "id": "token:ZONE", + "syntaxNodeKind": "token" + } + ] +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlSyntaxAstManifest.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlSyntaxAstManifest.java new file mode 100644 index 000000000000..6c9838214ef1 --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlSyntaxAstManifest.java @@ -0,0 +1,124 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import io.airlift.json.JsonCodec; +import io.trino.hogql.parser.HogQlCompatibilityManifest.PublishedGrammarFeatureManifest; +import io.trino.hogql.parser.HogQlSyntaxAstManifest.Feature; +import io.trino.hogql.parser.HogQlSyntaxAstManifest.SourceSpanGuarantee; +import io.trino.hogql.parser.HogQlSyntaxAstManifest.SyntaxNodeKind; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlSyntaxAstManifest +{ + private static final String LANGUAGE_RESOURCE_ROOT = "/io/trino/hogql/parser/language/1.0.0/"; + private static final String GRAMMAR_FEATURE_RESOURCE = LANGUAGE_RESOURCE_ROOT + "grammar-features.json"; + private static final String ALTERNATIVE_IDENTITIES_RESOURCE = LANGUAGE_RESOURCE_ROOT + "grammar-alternative-identities.json"; + private static final JsonCodec GRAMMAR_FEATURE_MANIFEST_CODEC = jsonCodec(PublishedGrammarFeatureManifest.class); + + @Test + public void testBindsTheCompletePrivateSyntaxAstContract() + { + HogQlSyntaxAstManifest manifest = HogQlSyntaxAstManifest.current(); + + assertThat(manifest.schemaVersion()).isEqualTo(1); + assertThat(manifest.languageVersion()).isEqualTo(HogQlLanguageContract.current().languageVersion()); + assertThat(manifest.grammarSha256()).isEqualTo(HogQlLanguageContract.current().grammarSha256()); + assertThat(manifest.grammarFeatureManifestSha256()).isEqualTo(sha256(readResourceBytes(GRAMMAR_FEATURE_RESOURCE))); + assertThat(manifest.grammarAlternativeIdentityManifestSha256()).isEqualTo(sha256(readResourceBytes(ALTERNATIVE_IDENTITIES_RESOURCE))); + assertThat(manifest.syntaxTreeKind()).isEqualTo(SyntaxNodeKind.TREE); + assertThat(manifest.sourceSpanGuarantee()).isEqualTo(SourceSpanGuarantee.CODE_POINT_OFFSETS_END_EXCLUSIVE_ONE_BASED_LINE_COLUMNS); + assertThat(manifest.features()).hasSize(583); + assertThat(manifest.features()).extracting(Feature::id).doesNotHaveDuplicates(); + assertThat(manifest.features()).filteredOn(feature -> feature.syntaxNodeKind() == SyntaxNodeKind.TOKEN).hasSize(207); + assertThat(manifest.features()).filteredOn(feature -> feature.syntaxNodeKind() == SyntaxNodeKind.RULE).hasSize(376); + assertThat(manifest.features()) + .allSatisfy(feature -> assertThat(feature.syntaxNodeKind() == SyntaxNodeKind.TOKEN) + .isEqualTo(feature.id().startsWith("token:"))); + } + + @Test + public void testRejectsIncompleteOrInvalidFeatureCoverage() + { + PublishedGrammarFeatureManifest published = loadPublishedGrammarFeatures(); + HogQlGrammarAlternativeIdentities alternativeIdentities = HogQlGrammarAlternativeIdentities.current(); + List current = HogQlSyntaxAstManifest.current().features(); + + assertThatThrownBy(() -> HogQlSyntaxAstManifest.validateFeatureCoverage(current.subList(1, current.size()), published, alternativeIdentities)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("HogQL syntax AST manifest does not account for every grammar feature"); + + List duplicate = new ArrayList<>(current); + duplicate.add(current.getFirst()); + assertThatThrownBy(() -> HogQlSyntaxAstManifest.validateFeatureCoverage(duplicate, published, alternativeIdentities)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("duplicate HogQL syntax AST feature: " + current.getFirst().id()); + + List unknown = new ArrayList<>(current); + unknown.add(new Feature("token:NOT_A_CANONICAL_HOGQL_TOKEN", SyntaxNodeKind.TOKEN)); + assertThatThrownBy(() -> HogQlSyntaxAstManifest.validateFeatureCoverage(unknown, published, alternativeIdentities)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("unknown HogQL syntax AST feature: token:NOT_A_CANONICAL_HOGQL_TOKEN"); + + List wrongNodeKind = new ArrayList<>(current); + Feature original = wrongNodeKind.getFirst(); + SyntaxNodeKind replacementKind = original.syntaxNodeKind() == SyntaxNodeKind.TOKEN ? SyntaxNodeKind.RULE : SyntaxNodeKind.TOKEN; + wrongNodeKind.set(0, new Feature(original.id(), replacementKind)); + assertThatThrownBy(() -> HogQlSyntaxAstManifest.validateFeatureCoverage(wrongNodeKind, published, alternativeIdentities)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("HogQL syntax AST feature has the wrong node kind: " + original.id()); + } + + private static PublishedGrammarFeatureManifest loadPublishedGrammarFeatures() + { + return GRAMMAR_FEATURE_MANIFEST_CODEC.fromJson(new String(readResourceBytes(GRAMMAR_FEATURE_RESOURCE), StandardCharsets.UTF_8)); + } + + private static byte[] readResourceBytes(String path) + { + try (InputStream input = TestHogQlSyntaxAstManifest.class.getResourceAsStream(path)) { + if (input == null) { + throw new IllegalStateException("missing test resource: " + path); + } + return input.readAllBytes(); + } + catch (IOException e) { + throw new UncheckedIOException("failed to read test resource: " + path, e); + } + } + + private static String sha256(byte[] content) + { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content)); + } + catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } +} From 4f75e8aaaca0f23c8073c54a8a1ca57ae9fe56d6 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 09:02:40 -0400 Subject: [PATCH 014/136] feat(hogql): expose the canonical expression parser --- .../io/trino/hogql/parser/HogQlParser.java | 68 ++++++++++++++++--- .../parser/TestHogQlCompatibilityCorpus.java | 23 ++++--- .../trino/hogql/parser/TestHogQlParser.java | 12 ++++ 3 files changed, 84 insertions(+), 19 deletions(-) diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index ccebb5a4bea9..965f0ccefc38 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -117,8 +117,8 @@ public HogQlQuery parseStatement(String hogql) public HogQlQuery parseStatement(String hogql, HogQlLanguageVersion languageVersion) { try { - ParsedQuery parsed = parseQuery(hogql, languageVersion); - return new AstBuilder(parsed.source()).build(parsed.tree()); + ParsedSyntax parsed = parse(hogql, languageVersion, EntryPoint.QUERY); + return new AstBuilder(parsed.source()).build((HogQLParser.SelectContext) parsed.tree()); } catch (StackOverflowError _) { throw new HogQlParsingException("statement is too large", null, 1, 1); @@ -133,15 +133,31 @@ public HogQlSyntaxTree parseSyntax(String hogql) public HogQlSyntaxTree parseSyntax(String hogql, HogQlLanguageVersion languageVersion) { try { - ParsedQuery parsed = parseQuery(hogql, languageVersion); - return new SyntaxTreeBuilder(parsed.source()).build(parsed.tree()); + ParsedSyntax parsed = parse(hogql, languageVersion, EntryPoint.QUERY); + return new SyntaxTreeBuilder(parsed.source()).build((HogQLParser.SelectContext) parsed.tree()); } catch (StackOverflowError _) { throw new HogQlParsingException("statement is too large", null, 1, 1); } } - private ParsedQuery parseQuery(String hogql, HogQlLanguageVersion languageVersion) + public HogQlSyntaxTree parseExpressionSyntax(String hogql) + { + return parseExpressionSyntax(hogql, CURRENT_LANGUAGE_VERSION); + } + + public HogQlSyntaxTree parseExpressionSyntax(String hogql, HogQlLanguageVersion languageVersion) + { + try { + ParsedSyntax parsed = parse(hogql, languageVersion, EntryPoint.EXPRESSION); + return new SyntaxTreeBuilder(parsed.source()).build(parsed.tree()); + } + catch (StackOverflowError _) { + throw new HogQlParsingException("expression is too large", null, 1, 1); + } + } + + private ParsedSyntax parse(String hogql, HogQlLanguageVersion languageVersion, EntryPoint entryPoint) { requireNonNull(hogql, "hogql is null"); requireNonNull(languageVersion, "languageVersion is null"); @@ -157,12 +173,12 @@ private ParsedQuery parseQuery(String hogql, HogQlLanguageVersion languageVersio lexer.addErrorListener(ERROR_LISTENER); parser.removeErrorListeners(); - HogQLParser.SelectContext tree; + ParserRuleContext tree; try { parser.getInterpreter().setPredictionMode(PredictionMode.SLL); parser.setErrorHandler(new BailErrorStrategy()); parser.addParseListener(new ParseBudgetListener(limits)); - tree = parser.select(); + tree = entryPoint.parse(parser); } catch (ParseCancellationException _) { parser.reset(); @@ -171,12 +187,36 @@ private ParsedQuery parseQuery(String hogql, HogQlLanguageVersion languageVersio parser.addErrorListener(ERROR_LISTENER); parser.removeParseListeners(); parser.addParseListener(new ParseBudgetListener(limits)); - tree = parser.select(); + tree = entryPoint.parse(parser); + } + if (parser.getCurrentToken().getType() != Token.EOF) { + Token trailing = parser.getCurrentToken(); + throw new HogQlParsingException("unexpected trailing input", null, trailing.getLine(), trailing.getCharPositionInLine() + 1); } - return new ParsedQuery(hogql, tree); + return new ParsedSyntax(hogql, tree); } - private record ParsedQuery(String source, HogQLParser.SelectContext tree) {} + private record ParsedSyntax(String source, ParserRuleContext tree) {} + + private enum EntryPoint + { + QUERY { + @Override + public ParserRuleContext parse(HogQLParser parser) + { + return parser.select(); + } + }, + EXPRESSION { + @Override + public ParserRuleContext parse(HogQLParser parser) + { + return parser.expression(); + } + }; + + public abstract ParserRuleContext parse(HogQLParser parser); + } private static final class BoundedTokenSource implements TokenSource @@ -315,6 +355,14 @@ else if (containsRule(context, HogQLParser.RULE_block)) { return new HogQlSyntaxTree(languageClass, buildNode(context)); } + public HogQlSyntaxTree build(ParserRuleContext context) + { + HogQlSyntaxTree.LanguageClass languageClass = containsRule(context, HogQLParser.RULE_block) ? + HogQlSyntaxTree.LanguageClass.PROCEDURAL : + HogQlSyntaxTree.LanguageClass.READ_ONLY_QUERY; + return new HogQlSyntaxTree(languageClass, buildNode(context)); + } + private static boolean containsRule(ParserRuleContext context, int ruleIndex) { if (context.getRuleIndex() == ruleIndex) { diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java index d7824fa15ef4..b76df5193bac 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java @@ -114,24 +114,29 @@ private void assertDisposition(CorpusCase corpusCase, CorpusDisposition disposit private void assertSyntaxDisposition(CorpusCase corpusCase) { - String query = switch (corpusCase.entryPoint()) { - case "expression" -> "SELECT " + corpusCase.source(); - case "query" -> corpusCase.source(); - default -> throw new IllegalArgumentException("unknown corpus entry point: " + corpusCase.entryPoint()); - }; if (!corpusCase.accepted()) { - assertThatThrownBy(() -> parser.parseSyntax(query)) + assertThatThrownBy(() -> parseSyntax(corpusCase)) .isInstanceOf(HogQlParsingException.class) .hasMessageStartingWith("line "); return; } - HogQlSyntaxTree syntaxTree = parser.parseSyntax(query); - int sourceLength = query.codePointCount(0, query.length()); - assertThat(syntaxTree.root().span()).isEqualTo(new SourceSpan(0, sourceLength, 1, 1, lineCount(query), finalColumn(query))); + HogQlSyntaxTree syntaxTree = parseSyntax(corpusCase); + String source = corpusCase.source(); + int sourceLength = source.codePointCount(0, source.length()); + assertThat(syntaxTree.root().span()).isEqualTo(new SourceSpan(0, sourceLength, 1, 1, lineCount(source), finalColumn(source))); assertContainedSpans(syntaxTree.root(), sourceLength); } + private HogQlSyntaxTree parseSyntax(CorpusCase corpusCase) + { + return switch (corpusCase.entryPoint()) { + case "expression" -> parser.parseExpressionSyntax(corpusCase.source()); + case "query" -> parser.parseSyntax(corpusCase.source()); + default -> throw new IllegalArgumentException("unknown corpus entry point: " + corpusCase.entryPoint()); + }; + } + private static void assertContainedSpans(Element element, int sourceLength) { SourceSpan span = element.span(); diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 40e77d66405e..d15bb6444842 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -110,6 +110,18 @@ public void testSyntaxTreeClassifiesBlockLambdaAsProcedural() .contains("block", "returnStmt"); } + @Test + public void testParsesCanonicalExpressionEntryPointWithoutAQueryWrapper() + { + HogQlSyntaxTree syntaxTree = parser.parseExpressionSyntax("value + 1"); + + assertThat(syntaxTree.root().rule()).isEqualTo("expression"); + assertThat(syntaxTree.root().span()).isEqualTo(new HogQlQuery.SourceSpan(0, 9, 1, 1, 1, 10)); + assertThatThrownBy(() -> parser.parseExpressionSyntax("value + 1 trailing")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("unexpected trailing input"); + } + private static Stream nodes(Element element) { if (!(element instanceof Node node)) { From ef5b6a12c3e19f8ec29b34bac49affddcd68d2b5 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 09:07:37 -0400 Subject: [PATCH 015/136] chore(hogql): compare canonical parser diagnostics --- .../parser/TestHogQlCompatibilityCorpus.java | 70 +++++++++++++++++-- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java index b76df5193bac..9d6811bab4f9 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java @@ -67,6 +67,15 @@ public void testOracleMetadataMatchesThePublishedCorpus() assertThat(oracleCase.accepted()).isEqualTo(corpusCase.accepted()); assertThat(oracleCase.languageVersion()).isEqualTo(HogQlLanguageContract.current().languageVersion()); assertThat(oracleCase.grammarSha256()).isEqualTo(HogQlLanguageContract.current().grammarSha256()); + assertThat(oracleCase.span()).isNotNull(); + if (oracleCase.accepted()) { + assertThat(oracleCase.ast()).isNotNull(); + assertThat(oracleCase.errorCategory()).isNull(); + } + else { + assertThat(oracleCase.ast()).isNull(); + assertThat(oracleCase.errorCategory()).isNotBlank(); + } } assertThat(oracleById).isEmpty(); } @@ -89,8 +98,10 @@ public Stream testTrinoDispositionForEveryCorpusCase() public Stream testSyntaxTreeDispositionForEveryCorpusCase() throws IOException { + Map oracleById = loadOracleCases().stream() + .collect(toMap(OracleCase::id, oracleCase -> oracleCase)); return loadCorpusManifest().cases().stream() - .map(corpusCase -> dynamicTest(corpusCase.id(), () -> assertSyntaxDisposition(corpusCase))); + .map(corpusCase -> dynamicTest(corpusCase.id(), () -> assertSyntaxDisposition(corpusCase, oracleById.get(corpusCase.id())))); } private void assertDisposition(CorpusCase corpusCase, CorpusDisposition disposition) @@ -112,22 +123,48 @@ private void assertDisposition(CorpusCase corpusCase, CorpusDisposition disposit } } - private void assertSyntaxDisposition(CorpusCase corpusCase) + private void assertSyntaxDisposition(CorpusCase corpusCase, OracleCase oracleCase) { if (!corpusCase.accepted()) { assertThatThrownBy(() -> parseSyntax(corpusCase)) .isInstanceOf(HogQlParsingException.class) - .hasMessageStartingWith("line "); + .satisfies(error -> { + HogQlParsingException parsingException = (HogQlParsingException) error; + assertThat(parsingException.getLineNumber()).isEqualTo(oracleCase.span().start().line()); + if (oracleCase.errorCategory().equals("unexpected-character")) { + assertThat(parsingException.getColumnNumber()).isEqualTo(oracleCase.span().start().column() + 1); + } + else { + assertThat(parsingException.getColumnNumber()).isBetween(1, oracleCase.span().start().column() + 1); + } + }); return; } HogQlSyntaxTree syntaxTree = parseSyntax(corpusCase); String source = corpusCase.source(); int sourceLength = source.codePointCount(0, source.length()); + SourceSpan oracleSpan = toSourceSpan(oracleCase.span()); + assertThat(syntaxTree.root().span().startOffset()).isEqualTo(oracleSpan.startOffset()); + assertThat(syntaxTree.root().span().endOffset()).isGreaterThanOrEqualTo(oracleSpan.endOffset()); + if (corpusCase.entryPoint().equals("expression")) { + assertThat(syntaxTree.root().span()).isEqualTo(oracleSpan); + } assertThat(syntaxTree.root().span()).isEqualTo(new SourceSpan(0, sourceLength, 1, 1, lineCount(source), finalColumn(source))); assertContainedSpans(syntaxTree.root(), sourceLength); } + private static SourceSpan toSourceSpan(OracleSpan span) + { + return new SourceSpan( + span.start().offset(), + span.end().offset(), + span.start().line(), + span.start().column() + 1, + span.end().line(), + span.end().column() + 1); + } + private HogQlSyntaxTree parseSyntax(CorpusCase corpusCase) { return switch (corpusCase.entryPoint()) { @@ -245,7 +282,10 @@ public record OracleCase( String entryPoint, boolean accepted, HogQlLanguageVersion languageVersion, - String grammarSha256) + String grammarSha256, + Map ast, + OracleSpan span, + String errorCategory) { @JsonCreator public OracleCase( @@ -254,7 +294,10 @@ public OracleCase( @JsonProperty("entryPoint") String entryPoint, @JsonProperty("accepted") boolean accepted, @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, - @JsonProperty("grammarSha256") String grammarSha256) + @JsonProperty("grammarSha256") String grammarSha256, + @JsonProperty("ast") Map ast, + @JsonProperty("span") OracleSpan span, + @JsonProperty("errorCategory") String errorCategory) { this.id = requireNonNull(id, "id is null"); this.source = requireNonNull(source, "source is null"); @@ -262,9 +305,26 @@ public OracleCase( this.accepted = accepted; this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); + this.ast = ast; + this.span = requireNonNull(span, "span is null"); + this.errorCategory = errorCategory; + } + } + + public record OracleSpan(OraclePoint start, OraclePoint end) + { + @JsonCreator + public OracleSpan( + @JsonProperty("start") OraclePoint start, + @JsonProperty("end") OraclePoint end) + { + this.start = requireNonNull(start, "start is null"); + this.end = requireNonNull(end, "end is null"); } } + public record OraclePoint(int line, int column, int offset) {} + public record CompatibilityOverrides(List corpusCases) { @JsonCreator From 84d95db5b7c62e37c8c60cc696ce16a17e3c0774 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 09:08:02 -0400 Subject: [PATCH 016/136] chore(hogql): add deterministic parser robustness corpus --- .../parser/TestHogQlParserRobustness.java | 325 ++++++++++++++++++ .../robustness/malformed-regressions.jsonl | 8 + 2 files changed, 333 insertions(+) create mode 100644 core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserRobustness.java create mode 100644 core/trino-hogql-parser/src/test/resources/io/trino/hogql/parser/language/1.0.0/robustness/malformed-regressions.jsonl diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserRobustness.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserRobustness.java new file mode 100644 index 000000000000..23d626e3b914 --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserRobustness.java @@ -0,0 +1,325 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.airlift.json.JsonCodec; +import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.hogql.parser.tree.HogQlSyntaxTree; +import io.trino.hogql.parser.tree.HogQlSyntaxTree.Element; +import io.trino.hogql.parser.tree.HogQlSyntaxTree.Node; +import io.trino.hogql.parser.tree.HogQlSyntaxTree.Token; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +public class TestHogQlParserRobustness +{ + private static final long RANDOM_SEED = 0x484C0FFEEL; + private static final int GENERATED_ROUNDS = 64; + private static final int MUTATIONS_PER_INPUT = 2; + private static final String MALFORMED_RESOURCE = "/io/trino/hogql/parser/language/1.0.0/robustness/malformed-regressions.jsonl"; + private static final JsonCodec MALFORMED_REGRESSION_CODEC = jsonCodec(MalformedRegression.class); + + private static final List ATOMS = List.of( + "event", + "value", + "enabled", + "properties.plan", + "0", + "17", + "true", + "NULL", + "'example'", + "'😀'"); + private static final List INSERTIONS = List.of("(", ")", "]", "'", "@", ","); + private static final List APPENDAGES = List.of(" +", " )", " '", " FROM", " WHERE", " /*"); + + private final HogQlParser parser = new HogQlParser(); + + @Test + public void testGeneratedAndMutatedInputsHaveOnlyDocumentedOutcomes() + { + Random random = new Random(RANDOM_SEED); + List generated = IntStream.range(0, GENERATED_ROUNDS) + .boxed() + .flatMap(index -> generatedInputs(random, index)) + .toList(); + + assertThat(generated).hasSize(GENERATED_ROUNDS * 2); + for (GeneratedInput input : generated) { + Optional outcome = parseSafely(input.source(), input.id()); + assertThat(outcome) + .as("generated input %s must be valid: %s", input.id(), display(input.source())) + .isPresent(); + outcome.ifPresent(tree -> assertValidTree(input.source(), tree)); + + for (int mutation = 0; mutation < MUTATIONS_PER_INPUT; mutation++) { + String mutated = mutate(random, input.source()); + parseSafely(mutated, input.id() + "-mutation-" + mutation) + .ifPresent(tree -> assertValidTree(mutated, tree)); + } + } + } + + @Test + public void testMinimizedMalformedRegressionSeedsAreRejectedCleanly() + throws IOException + { + List regressions = loadMalformedRegressions(); + + assertThat(regressions) + .extracting(MalformedRegression::id) + .containsExactly( + "missing-projection", + "trailing-binary-operator", + "unclosed-tuple", + "unexpected-character", + "missing-source", + "missing-predicate", + "unclosed-string", + "second-statement"); + for (MalformedRegression regression : regressions) { + assertThat(regression.source().codePointCount(0, regression.source().length())) + .as("regression %s remains minimized", regression.id()) + .isLessThanOrEqualTo(32); + assertThat(parseSafely(regression.source(), regression.id())) + .as("malformed regression must be rejected: %s", regression.id()) + .isEmpty(); + } + } + + private Optional parseSafely(String source, String id) + { + try { + return Optional.of(parser.parseSyntax(source)); + } + catch (HogQlParsingException _) { + return Optional.empty(); + } + catch (RuntimeException | Error unexpected) { + throw new AssertionError("unexpected parser failure for " + id + ": " + display(source), unexpected); + } + } + + private static Stream generatedInputs(Random random, int index) + { + String expression = expression(random, 3); + return Stream.of( + new GeneratedInput("expression-" + index, "SELECT " + expression), + new GeneratedInput("query-" + index, query(random, expression))); + } + + private static String expression(Random random, int depth) + { + if (depth == 0) { + return ATOMS.get(random.nextInt(ATOMS.size())); + } + return switch (random.nextInt(8)) { + case 0 -> "(" + expression(random, depth - 1) + ")"; + case 1 -> "NOT (" + expression(random, depth - 1) + ")"; + case 2 -> "-(" + expression(random, depth - 1) + ")"; + case 3 -> "(" + expression(random, depth - 1) + " + " + expression(random, depth - 1) + ")"; + case 4 -> "(" + expression(random, depth - 1) + " = " + expression(random, depth - 1) + ")"; + case 5 -> "coalesce(" + expression(random, depth - 1) + ", " + expression(random, depth - 1) + ")"; + case 6 -> "[" + expression(random, depth - 1) + ", " + expression(random, depth - 1) + "]"; + default -> "CASE WHEN " + predicate(random, depth - 1) + " THEN " + expression(random, depth - 1) + " ELSE 0 END"; + }; + } + + private static String predicate(Random random, int depth) + { + if (depth == 0) { + return switch (random.nextInt(4)) { + case 0 -> "event = '$pageview'"; + case 1 -> "value >= 0"; + case 2 -> "enabled"; + default -> "properties.plan IS NOT NULL"; + }; + } + return switch (random.nextInt(4)) { + case 0 -> "NOT (" + predicate(random, depth - 1) + ")"; + case 1 -> "(" + predicate(random, depth - 1) + " AND " + predicate(random, depth - 1) + ")"; + case 2 -> "(" + predicate(random, depth - 1) + " OR " + predicate(random, depth - 1) + ")"; + default -> "(" + expression(random, depth - 1) + " = " + expression(random, depth - 1) + ")"; + }; + } + + private static String query(Random random, String projection) + { + return switch (random.nextInt(6)) { + case 0 -> "SELECT " + projection; + case 1 -> "SELECT " + projection + " AS result FROM events"; + case 2 -> "SELECT " + projection + ", " + expression(random, 2) + " FROM events WHERE " + predicate(random, 2); + case 3 -> "SELECT\n " + projection + "\r\nFROM events\nWHERE " + predicate(random, 2); + case 4 -> "WITH sample AS (SELECT " + projection + " AS result) SELECT result FROM sample"; + default -> "SELECT " + projection + " FROM events ORDER BY event LIMIT " + (random.nextInt(20) + 1); + }; + } + + private static String mutate(Random random, String source) + { + int codePointLength = source.codePointCount(0, source.length()); + return switch (random.nextInt(6)) { + case 0 -> source.substring(0, source.offsetByCodePoints(0, random.nextInt(codePointLength + 1))); + case 1 -> deleteCodePoint(source, random.nextInt(codePointLength)); + case 2 -> insertAtCodePoint(source, random.nextInt(codePointLength + 1), INSERTIONS.get(random.nextInt(INSERTIONS.size()))); + case 3 -> replaceCodePoint(source, random.nextInt(codePointLength), ')'); + case 4 -> duplicateCodePoint(source, random.nextInt(codePointLength)); + default -> source + APPENDAGES.get(random.nextInt(APPENDAGES.size())); + }; + } + + private static String deleteCodePoint(String source, int codePointOffset) + { + int start = source.offsetByCodePoints(0, codePointOffset); + int end = source.offsetByCodePoints(start, 1); + return source.substring(0, start) + source.substring(end); + } + + private static String insertAtCodePoint(String source, int codePointOffset, String insertion) + { + int offset = source.offsetByCodePoints(0, codePointOffset); + return source.substring(0, offset) + insertion + source.substring(offset); + } + + private static String replaceCodePoint(String source, int codePointOffset, int replacement) + { + int start = source.offsetByCodePoints(0, codePointOffset); + int end = source.offsetByCodePoints(start, 1); + return source.substring(0, start) + Character.toString(replacement) + source.substring(end); + } + + private static String duplicateCodePoint(String source, int codePointOffset) + { + int start = source.offsetByCodePoints(0, codePointOffset); + int end = source.offsetByCodePoints(start, 1); + return source.substring(0, end) + source.substring(start); + } + + private static void assertValidTree(String source, HogQlSyntaxTree tree) + { + int sourceLength = source.codePointCount(0, source.length()); + Deque pending = new ArrayDeque<>(); + pending.add(new PendingElement(tree.root(), Optional.empty())); + + while (!pending.isEmpty()) { + PendingElement current = pending.removeFirst(); + Element element = current.element(); + SourceSpan span = element.span(); + assertThat(span.startOffset()).isBetween(0, sourceLength); + assertThat(span.endOffset()).isBetween(span.startOffset(), sourceLength); + assertPosition(source, span.startOffset(), span.startLine(), span.startColumn()); + assertPosition(source, span.endOffset(), span.endLine(), span.endColumn()); + current.parentSpan().ifPresent(parent -> { + assertThat(span.startOffset()).isGreaterThanOrEqualTo(parent.startOffset()); + assertThat(span.endOffset()).isLessThanOrEqualTo(parent.endOffset()); + }); + + if (element instanceof Node node) { + node.children().forEach(child -> pending.addLast(new PendingElement(child, Optional.of(span)))); + } + else if (element instanceof Token token) { + int start = source.offsetByCodePoints(0, span.startOffset()); + int end = source.offsetByCodePoints(0, span.endOffset()); + assertThat(source.substring(start, end)).isEqualTo(token.text()); + } + } + } + + private static void assertPosition(String source, int targetOffset, int expectedLine, int expectedColumn) + { + Position position = positionAt(source, targetOffset); + assertThat(expectedLine).isEqualTo(position.line()); + assertThat(expectedColumn).isEqualTo(position.column()); + } + + private static Position positionAt(String source, int targetOffset) + { + int line = 1; + int column = 1; + int codePointOffset = 0; + boolean previousWasCarriageReturn = false; + for (int charOffset = 0; codePointOffset < targetOffset; codePointOffset++) { + int codePoint = source.codePointAt(charOffset); + charOffset += Character.charCount(codePoint); + if (codePoint == '\n' && previousWasCarriageReturn) { + previousWasCarriageReturn = false; + } + else if (codePoint == '\n' || codePoint == '\r') { + line++; + column = 1; + previousWasCarriageReturn = codePoint == '\r'; + } + else { + column++; + previousWasCarriageReturn = false; + } + } + return new Position(line, column); + } + + private static List loadMalformedRegressions() + throws IOException + { + try (InputStream input = TestHogQlParserRobustness.class.getResourceAsStream(MALFORMED_RESOURCE)) { + if (input == null) { + fail("missing test resource: " + MALFORMED_RESOURCE); + } + return Arrays.stream(new String(input.readAllBytes(), StandardCharsets.UTF_8).split("\\R")) + .filter(line -> !line.isBlank()) + .map(MALFORMED_REGRESSION_CODEC::fromJson) + .toList(); + } + } + + private static String display(String source) + { + return source.replace("\r", "\\r").replace("\n", "\\n"); + } + + private record GeneratedInput(String id, String source) {} + + private record PendingElement(Element element, Optional parentSpan) {} + + private record Position(int line, int column) {} + + public record MalformedRegression(String id, String source) + { + @JsonCreator + public MalformedRegression( + @JsonProperty("id") String id, + @JsonProperty("source") String source) + { + this.id = requireNonNull(id, "id is null"); + this.source = requireNonNull(source, "source is null"); + } + } +} diff --git a/core/trino-hogql-parser/src/test/resources/io/trino/hogql/parser/language/1.0.0/robustness/malformed-regressions.jsonl b/core/trino-hogql-parser/src/test/resources/io/trino/hogql/parser/language/1.0.0/robustness/malformed-regressions.jsonl new file mode 100644 index 000000000000..c5c7814622a9 --- /dev/null +++ b/core/trino-hogql-parser/src/test/resources/io/trino/hogql/parser/language/1.0.0/robustness/malformed-regressions.jsonl @@ -0,0 +1,8 @@ +{"id":"missing-projection","source":"SELECT"} +{"id":"trailing-binary-operator","source":"SELECT 1 +"} +{"id":"unclosed-tuple","source":"SELECT (1, 2"} +{"id":"unexpected-character","source":"SELECT value @ 1"} +{"id":"missing-source","source":"SELECT 1 FROM"} +{"id":"missing-predicate","source":"SELECT 1 WHERE"} +{"id":"unclosed-string","source":"SELECT 'example"} +{"id":"second-statement","source":"SELECT 1; SELECT 2"} From 2e86b89701897b82eef674b2bf2c3316c171d564 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 09:47:28 -0400 Subject: [PATCH 017/136] feat(hogql): add compiler diagnostics --- core/trino-hogql-compiler/pom.xml | 4 +- .../trino/hogql/compiler/HogQlCompiler.java | 14 ++++-- .../trino/hogql/compiler/HogQlErrorCode.java | 49 +++++++++++++++++++ .../hogql/compiler/TestHogQlCompiler.java | 17 ++++--- 4 files changed, 71 insertions(+), 13 deletions(-) create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlErrorCode.java diff --git a/core/trino-hogql-compiler/pom.xml b/core/trino-hogql-compiler/pom.xml index 692be279716b..08aae74b5290 100644 --- a/core/trino-hogql-compiler/pom.xml +++ b/core/trino-hogql-compiler/pom.xml @@ -25,8 +25,8 @@ - org.antlr - antlr4-runtime + io.trino + trino-spi diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index 0aa3c6772abc..c5d73a578839 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -15,9 +15,13 @@ import io.trino.hogql.parser.HogQlParser; import io.trino.hogql.parser.HogQlParsingException; -import io.trino.sql.parser.ParsingException; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; import io.trino.sql.tree.Statement; +import java.util.Optional; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_SYNTAX_ERROR; import static java.util.Objects.requireNonNull; public final class HogQlCompiler @@ -40,11 +44,11 @@ public Statement compile(String hogql) return TrinoAstFactory.createStatement(parser.parseStatement(hogql)); } catch (HogQlParsingException exception) { - throw new ParsingException( + throw new TrinoException( + HOGQL_SYNTAX_ERROR, + Optional.of(new Location(exception.getLineNumber(), exception.getColumnNumber())), exception.getErrorMessage(), - null, - exception.getLineNumber(), - exception.getColumnNumber()); + exception); } } } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlErrorCode.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlErrorCode.java new file mode 100644 index 000000000000..34e42896d086 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlErrorCode.java @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.spi.ErrorCode; +import io.trino.spi.ErrorCodeSupplier; +import io.trino.spi.ErrorType; + +import static io.trino.spi.ErrorType.INTERNAL_ERROR; +import static io.trino.spi.ErrorType.USER_ERROR; + +public enum HogQlErrorCode + implements ErrorCodeSupplier +{ + HOGQL_SYNTAX_ERROR(0, USER_ERROR), + HOGQL_BINDING_ERROR(1, USER_ERROR), + HOGQL_RESOLUTION_ERROR(2, USER_ERROR), + HOGQL_TYPE_ERROR(3, USER_ERROR), + HOGQL_UNSUPPORTED_FEATURE(4, USER_ERROR), + HOGQL_CATALOG_NOT_READY(5, USER_ERROR), + HOGQL_CATALOG_GENERATION_MISMATCH(6, USER_ERROR), + HOGQL_COMPILER_LIMIT_EXCEEDED(7, USER_ERROR), + HOGQL_COMPILER_INTERNAL_ERROR(8, INTERNAL_ERROR), + ; + + private final ErrorCode errorCode; + + HogQlErrorCode(int code, ErrorType type) + { + errorCode = new ErrorCode(code + 0x0521_0000, name(), type); + } + + @Override + public ErrorCode toErrorCode() + { + return errorCode; + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index cc574429bc18..231389a11a6b 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -13,8 +13,9 @@ */ package io.trino.hogql.compiler; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; import io.trino.sql.SqlFormatter; -import io.trino.sql.parser.ParsingException; import io.trino.sql.parser.SqlParser; import io.trino.sql.tree.AllColumns; import io.trino.sql.tree.Identifier; @@ -35,7 +36,7 @@ import java.util.Deque; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowableOfType; public class TestHogQlCompiler { @@ -161,10 +162,14 @@ public void testReturnsOnlyStockTrinoTreeNodes() } @Test - public void testReportsParsingFailureAsTrinoSyntaxError() + public void testReportsSourceLocatedHogQlSyntaxError() { - assertThatThrownBy(() -> compiler.compile("SELECT 1\nGROUP BY 1")) - .isInstanceOf(ParsingException.class) - .hasMessageStartingWith("line 2:1:"); + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile("SELECT 1\nGROUP BY 1")); + + assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_SYNTAX_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(2, 1)); + assertThat(exception).hasMessageStartingWith("line 2:1:"); } } From 13771ba9bf80ce49cecccbf09149311685f131c6 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 09:47:54 -0400 Subject: [PATCH 018/136] feat(hogql): define the typed request envelope --- .../hogql/compiler/HogQlCompileEnvelope.java | 111 ++++++++++++ .../trino/hogql/compiler/HogQlTypedValue.java | 147 ++++++++++++++++ .../compiler/TestHogQlCompileEnvelope.java | 162 ++++++++++++++++++ 3 files changed, 420 insertions(+) create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompileEnvelope.java create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlTypedValue.java create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompileEnvelope.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompileEnvelope.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompileEnvelope.java new file mode 100644 index 000000000000..0a73142dbe61 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompileEnvelope.java @@ -0,0 +1,111 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.hogql.parser.HogQlLanguageVersion; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public record HogQlCompileEnvelope( + String query, + int protocolVersion, + HogQlLanguageVersion languageVersion, + Map parameters, + Map variables, + Map filters, + Map modifiers, + Optional catalogGeneration) +{ + public static final int PROTOCOL_VERSION = 1; + + private static final Set SEMANTIC_FIELDS = Set.of("parameters", "variables", "filters", "modifiers"); + + public HogQlCompileEnvelope + { + query = requireNonNull(query, "query is null"); + if (query.isBlank()) { + throw new IllegalArgumentException("query is empty"); + } + if (protocolVersion != PROTOCOL_VERSION) { + throw new IllegalArgumentException("unsupported HogQL protocol version"); + } + languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + if (!languageVersion.equals(HogQlLanguageContract.current().languageVersion())) { + throw new IllegalArgumentException("unsupported HogQL language version"); + } + parameters = immutableValues("parameters", parameters); + variables = immutableValues("variables", variables); + filters = immutableValues("filters", filters); + modifiers = immutableValues("modifiers", modifiers); + catalogGeneration = requireNonNull(catalogGeneration, "catalogGeneration is null"); + if (catalogGeneration.stream().anyMatch(String::isBlank)) { + throw new IllegalArgumentException("catalog generation is empty"); + } + } + + public static HogQlCompileEnvelope fromSemanticFields( + String query, + int protocolVersion, + HogQlLanguageVersion languageVersion, + Map> semanticFields, + Optional catalogGeneration) + { + requireNonNull(semanticFields, "semanticFields is null"); + if (!SEMANTIC_FIELDS.containsAll(semanticFields.keySet())) { + throw new IllegalArgumentException("unknown HogQL semantic field"); + } + return new HogQlCompileEnvelope( + query, + protocolVersion, + languageVersion, + semanticFields.getOrDefault("parameters", Map.of()), + semanticFields.getOrDefault("variables", Map.of()), + semanticFields.getOrDefault("filters", Map.of()), + semanticFields.getOrDefault("modifiers", Map.of()), + catalogGeneration); + } + + private static Map immutableValues(String field, Map values) + { + requireNonNull(values, field + " is null"); + for (Map.Entry entry : values.entrySet()) { + if (entry.getKey() == null || entry.getKey().isBlank()) { + throw new IllegalArgumentException(field + " contains an empty name"); + } + if (entry.getValue() == null) { + throw new IllegalArgumentException(field + " contains a missing typed value"); + } + } + return Map.copyOf(values); + } + + @Override + public String toString() + { + return "HogQlCompileEnvelope[protocolVersion=%s, languageVersion=%s, query=, parameters=%s, variables=%s, filters=%s, modifiers=%s, catalogGenerationPresent=%s]" + .formatted( + protocolVersion, + languageVersion, + parameters.size(), + variables.size(), + filters.size(), + modifiers.size(), + catalogGeneration.isPresent()); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlTypedValue.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlTypedValue.java new file mode 100644 index 000000000000..f69cace05536 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlTypedValue.java @@ -0,0 +1,147 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; + +import static java.util.Objects.requireNonNull; + +public record HogQlTypedValue(String type, Value value) +{ + public HogQlTypedValue + { + type = requireNonNull(type, "type is null"); + value = requireNonNull(value, "value is null"); + if (type.isBlank()) { + throw new IllegalArgumentException("type is empty"); + } + } + + @Override + public String toString() + { + return "HogQlTypedValue[type=%s, value=]".formatted(type); + } + + public sealed interface Value + permits ArrayValue, + BooleanValue, + NullValue, + NumberValue, + ObjectValue, + StringValue {} + + public enum NullValue + implements Value + { + NULL; + + @Override + public String toString() + { + return ""; + } + } + + public record BooleanValue(boolean value) + implements Value + { + @Override + public String toString() + { + return ""; + } + } + + public record NumberValue(String value) + implements Value + { + public NumberValue + { + value = requireNonNull(value, "number value is null"); + try { + new BigDecimal(value); + } + catch (NumberFormatException _) { + throw new IllegalArgumentException("number value is invalid"); + } + } + + @Override + public String toString() + { + return ""; + } + } + + public record StringValue(String value) + implements Value + { + public StringValue + { + value = requireNonNull(value, "string value is null"); + } + + @Override + public String toString() + { + return ""; + } + } + + public record ArrayValue(List value) + implements Value + { + public ArrayValue + { + requireNonNull(value, "array value is null"); + if (value.stream().anyMatch(element -> element == null)) { + throw new IllegalArgumentException("array value contains a missing element"); + } + value = List.copyOf(value); + } + + @Override + public String toString() + { + return ""; + } + } + + public record ObjectValue(Map value) + implements Value + { + public ObjectValue + { + requireNonNull(value, "object value is null"); + for (Map.Entry entry : value.entrySet()) { + if (entry.getKey() == null || entry.getKey().isBlank()) { + throw new IllegalArgumentException("object value contains an empty field name"); + } + if (entry.getValue() == null) { + throw new IllegalArgumentException("object value contains a missing field value"); + } + } + value = Map.copyOf(value); + } + + @Override + public String toString() + { + return ""; + } + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompileEnvelope.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompileEnvelope.java new file mode 100644 index 000000000000..7699d7028d33 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompileEnvelope.java @@ -0,0 +1,162 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.HogQlTypedValue.ArrayValue; +import io.trino.hogql.compiler.HogQlTypedValue.NumberValue; +import io.trino.hogql.compiler.HogQlTypedValue.ObjectValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.hogql.compiler.HogQlTypedValue.Value; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.hogql.parser.HogQlLanguageVersion; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +public class TestHogQlCompileEnvelope +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageContract.current().languageVersion(); + + @ParameterizedTest + @ValueSource(ints = {-1, 0, 2, Integer.MAX_VALUE}) + public void testRejectsUnknownProtocolVersions(int protocolVersion) + { + assertThatThrownBy(() -> envelope(protocolVersion, LANGUAGE_VERSION, Map.of(), Map.of(), Map.of(), Map.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("unsupported HogQL protocol version"); + } + + @ParameterizedTest + @ValueSource(strings = {"0.0.0", "1.0.1", "2.0.0"}) + public void testRejectsUnknownLanguageVersions(String languageVersion) + { + assertThatThrownBy(() -> envelope(HogQlCompileEnvelope.PROTOCOL_VERSION, HogQlLanguageVersion.valueOf(languageVersion), Map.of(), Map.of(), Map.of(), Map.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("unsupported HogQL language version"); + } + + @ParameterizedTest + @MethodSource("ambiguousTypedValues") + public void testRejectsAmbiguousTypedValues(Supplier invalidValue, String sensitiveValue) + { + assertThatThrownBy(invalidValue::get) + .isInstanceOfAny(IllegalArgumentException.class, NullPointerException.class) + .message() + .doesNotContain(sensitiveValue); + } + + @ParameterizedTest + @ValueSource(strings = {"actions", "cohorts", "savedQueries"}) + public void testRejectsUnknownSemanticFields(String semanticField) + { + Map> fields = Map.of( + semanticField, Map.of("input", new HogQlTypedValue("varchar", new StringValue("sensitive-value")))); + + assertThatThrownBy(() -> HogQlCompileEnvelope.fromSemanticFields( + "SELECT 1", + HogQlCompileEnvelope.PROTOCOL_VERSION, + LANGUAGE_VERSION, + fields, + Optional.empty())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("unknown HogQL semantic field"); + } + + @ParameterizedTest + @MethodSource("sensitiveValues") + public void testTypedValuesAreRedacted(Value value, String sensitiveValue) + { + HogQlTypedValue typedValue = new HogQlTypedValue("varchar", value); + + assertThat(value.toString()).doesNotContain(sensitiveValue); + assertThat(typedValue.toString()).doesNotContain(sensitiveValue); + } + + @ParameterizedTest + @ValueSource(strings = {"parameters", "variables", "filters", "modifiers"}) + public void testEnvelopeRenderingRedactsEveryValueCollection(String semanticField) + { + String sensitiveValue = "sensitive-value-" + semanticField; + Map> fields = Map.of( + semanticField, Map.of("input", new HogQlTypedValue("varchar", new StringValue(sensitiveValue)))); + HogQlCompileEnvelope envelope = HogQlCompileEnvelope.fromSemanticFields( + "SELECT 'sensitive-query'", + HogQlCompileEnvelope.PROTOCOL_VERSION, + LANGUAGE_VERSION, + fields, + Optional.of("sensitive-generation")); + + assertThat(envelope.toString()) + .doesNotContain(sensitiveValue, "sensitive-query", "sensitive-generation") + .contains("query="); + } + + private static Stream ambiguousTypedValues() + { + String sensitiveValue = "sensitive-value"; + List arrayWithMissingValue = new ArrayList<>(); + arrayWithMissingValue.add(new StringValue(sensitiveValue)); + arrayWithMissingValue.add(null); + Map objectWithMissingValue = new HashMap<>(); + objectWithMissingValue.put("input", new StringValue(sensitiveValue)); + objectWithMissingValue.put("missing", null); + return Stream.of( + arguments((Supplier) () -> new HogQlTypedValue(null, new StringValue(sensitiveValue)), sensitiveValue), + arguments((Supplier) () -> new HogQlTypedValue("", new StringValue(sensitiveValue)), sensitiveValue), + arguments((Supplier) () -> new HogQlTypedValue("varchar", null), sensitiveValue), + arguments((Supplier) () -> new ArrayValue(arrayWithMissingValue), sensitiveValue), + arguments((Supplier) () -> new ObjectValue(objectWithMissingValue), sensitiveValue)); + } + + private static Stream sensitiveValues() + { + return Stream.of( + arguments(new StringValue("sensitive-string"), "sensitive-string"), + arguments(new NumberValue("98765432101234567890"), "98765432101234567890"), + arguments(new ArrayValue(List.of(new StringValue("sensitive-array"))), "sensitive-array"), + arguments(new ObjectValue(Map.of("sensitive-key", new StringValue("sensitive-object"))), "sensitive-object")); + } + + private static HogQlCompileEnvelope envelope( + int protocolVersion, + HogQlLanguageVersion languageVersion, + Map parameters, + Map variables, + Map filters, + Map modifiers) + { + return new HogQlCompileEnvelope( + "SELECT 1", + protocolVersion, + languageVersion, + parameters, + variables, + filters, + modifiers, + Optional.empty()); + } +} From 436b4b6e0fedfdd2fcfa88c918e2fe7262412d85 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 09:48:08 -0400 Subject: [PATCH 019/136] feat(hogql): carry typed query submissions --- .../io/trino/dispatcher/DispatchManager.java | 20 +++++----- .../dispatcher/HogQlStatementResource.java | 4 +- .../dispatcher/QueuedStatementResource.java | 31 ++++++++-------- .../io/trino/execution/QueryLanguage.java | 2 +- .../io/trino/execution/QueryPreparer.java | 14 ++++--- .../io/trino/execution/QuerySubmission.java | 37 +++++++++++++++++++ .../io/trino/execution/TestQueryPreparer.java | 16 +++++++- 7 files changed, 87 insertions(+), 37 deletions(-) create mode 100644 core/trino-main/src/main/java/io/trino/execution/QuerySubmission.java diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/DispatchManager.java b/core/trino-main/src/main/java/io/trino/dispatcher/DispatchManager.java index 58bf61588a8c..be3ee7f938ac 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/DispatchManager.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/DispatchManager.java @@ -27,11 +27,11 @@ import io.trino.event.QueryMonitor; import io.trino.execution.QueryIdGenerator; import io.trino.execution.QueryInfo; -import io.trino.execution.QueryLanguage; import io.trino.execution.QueryManagerConfig; import io.trino.execution.QueryManagerStats; import io.trino.execution.QueryPreparer; import io.trino.execution.QueryPreparer.PreparedQuery; +import io.trino.execution.QuerySubmission; import io.trino.execution.QueryTracker; import io.trino.execution.resourcegroups.ResourceGroupManager; import io.trino.metadata.SessionPropertyManager; @@ -60,11 +60,11 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.execution.QueryLanguage.SQL; import static io.trino.execution.QueryState.FINISHING; import static io.trino.execution.QueryState.QUEUED; import static io.trino.execution.QueryState.RUNNING; import static io.trino.execution.QueryState.WAITING_FOR_RESOURCES; +import static io.trino.execution.QuerySubmission.trino; import static io.trino.spi.StandardErrorCode.QUERY_TEXT_TOO_LARGE; import static io.trino.tracing.ScopedSpan.scopedSpan; import static io.trino.util.Failures.toFailure; @@ -177,17 +177,16 @@ public QueryId createQueryId() public ListenableFuture createQuery(QueryId queryId, Span querySpan, Slug slug, SessionContext sessionContext, String query) { - return createQuery(queryId, querySpan, slug, sessionContext, query, SQL); + return createQuery(queryId, querySpan, slug, sessionContext, trino(query)); } - public ListenableFuture createQuery(QueryId queryId, Span querySpan, Slug slug, SessionContext sessionContext, String query, QueryLanguage queryLanguage) + public ListenableFuture createQuery(QueryId queryId, Span querySpan, Slug slug, SessionContext sessionContext, QuerySubmission submission) { requireNonNull(queryId, "queryId is null"); requireNonNull(querySpan, "querySpan is null"); requireNonNull(sessionContext, "sessionContext is null"); - requireNonNull(query, "query is null"); - requireNonNull(queryLanguage, "queryLanguage is null"); - checkArgument(!query.isEmpty(), "query must not be empty string"); + requireNonNull(submission, "submission is null"); + checkArgument(!submission.originalText().isEmpty(), "query must not be empty string"); checkArgument(!queryTracker.hasQuery(queryId), "query %s already exists", queryId); // It is important to return a future implementation which ignores cancellation request. @@ -200,7 +199,7 @@ public ListenableFuture createQuery(QueryId queryId, Span querySpan, Slug .setParent(Context.current().with(querySpan)) .startSpan(); try (var _ = scopedSpan(span)) { - createQueryInternal(queryId, querySpan, slug, sessionContext, query, queryLanguage, resourceGroupManager); + createQueryInternal(queryId, querySpan, slug, sessionContext, submission, resourceGroupManager); } finally { queryCreationFuture.set(null); @@ -213,8 +212,9 @@ public ListenableFuture createQuery(QueryId queryId, Span querySpan, Slug * Creates and registers a dispatch query with the query tracker. This method will never fail to register a query with the query * tracker. If an error occurs while creating a dispatch query, a failed dispatch will be created and registered. */ - private void createQueryInternal(QueryId queryId, Span querySpan, Slug slug, SessionContext sessionContext, String query, QueryLanguage queryLanguage, ResourceGroupManager resourceGroupManager) + private void createQueryInternal(QueryId queryId, Span querySpan, Slug slug, SessionContext sessionContext, QuerySubmission submission, ResourceGroupManager resourceGroupManager) { + String query = submission.originalText(); Session session = null; PreparedQuery preparedQuery = null; try { @@ -231,7 +231,7 @@ private void createQueryInternal(QueryId queryId, Span querySpan, Slug slug, accessControl.checkCanExecuteQuery(sessionContext.getIdentity(), queryId); // prepare query - preparedQuery = queryPreparer.prepareQuery(session, query, queryLanguage); + preparedQuery = queryPreparer.prepareQuery(session, submission); // select resource group Optional queryType = getQueryType(preparedQuery.getStatement()).map(Enum::name); diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java index bf15f36e5422..e17e0939c0ec 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java @@ -25,7 +25,7 @@ import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.Response; -import static io.trino.execution.QueryLanguage.HOGQL; +import static io.trino.execution.QuerySubmission.hogQl; import static io.trino.server.security.ResourceSecurity.AccessType.AUTHENTICATED_USER; import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; import static java.util.Objects.requireNonNull; @@ -50,6 +50,6 @@ public Response postStatement( @Context HttpHeaders httpHeaders, @BeanParam ExternalUriInfo externalUriInfo) { - return queuedStatementResource.postStatement(statement, HOGQL, servletRequest, httpHeaders, externalUriInfo); + return queuedStatementResource.postStatement(hogQl(statement), servletRequest, httpHeaders, externalUriInfo); } } diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java b/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java index 7a4cedc0c8b0..975e78ff6ad6 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java @@ -29,9 +29,9 @@ import io.trino.client.QueryResults; import io.trino.client.StatementStats; import io.trino.execution.ExecutionFailureInfo; -import io.trino.execution.QueryLanguage; import io.trino.execution.QueryManagerConfig; import io.trino.execution.QueryState; +import io.trino.execution.QuerySubmission; import io.trino.server.ExternalUriInfo; import io.trino.server.GoneException; import io.trino.server.HttpRequestSessionContextFactory; @@ -90,9 +90,10 @@ import static io.trino.dispatcher.QueuedStatementResource.SubmissionState.ABANDONED; import static io.trino.dispatcher.QueuedStatementResource.SubmissionState.NOT_SUBMITTED; import static io.trino.dispatcher.QueuedStatementResource.SubmissionState.SUBMITTED; -import static io.trino.execution.QueryLanguage.SQL; +import static io.trino.execution.QueryLanguage.TRINO; import static io.trino.execution.QueryState.FAILED; import static io.trino.execution.QueryState.QUEUED; +import static io.trino.execution.QuerySubmission.trino; import static io.trino.server.ServletSecurityUtils.authenticatedIdentity; import static io.trino.server.ServletSecurityUtils.clearAuthenticatedIdentity; import static io.trino.server.protocol.QueryInfoUrlFactory.getQueryInfoUri; @@ -177,25 +178,25 @@ public Response postStatement( @Context HttpHeaders httpHeaders, @BeanParam ExternalUriInfo externalUriInfo) { - return postStatement(statement, SQL, servletRequest, httpHeaders, externalUriInfo); + return postStatement(trino(statement), servletRequest, httpHeaders, externalUriInfo); } Response postStatement( - String statement, - QueryLanguage queryLanguage, + QuerySubmission submission, HttpServletRequest servletRequest, HttpHeaders httpHeaders, ExternalUriInfo externalUriInfo) { - if (isNullOrEmpty(statement)) { - throw new BadRequestException(queryLanguage == SQL ? "SQL statement is empty" : "HogQL query is empty"); + requireNonNull(submission, "submission is null"); + if (isNullOrEmpty(submission.originalText())) { + throw new BadRequestException(submission.language() == TRINO ? "SQL statement is empty" : "HogQL query is empty"); } - Query query = registerQuery(statement, queryLanguage, servletRequest, httpHeaders); + Query query = registerQuery(submission, servletRequest, httpHeaders); return createQueryResultsResponse(query.getQueryResults(query.getLastToken(), externalUriInfo), query.sessionContext.getQueryDataEncoding()); } - private Query registerQuery(String statement, QueryLanguage queryLanguage, HttpServletRequest servletRequest, HttpHeaders httpHeaders) + private Query registerQuery(QuerySubmission submission, HttpServletRequest servletRequest, HttpHeaders httpHeaders) { Optional remoteAddress = Optional.ofNullable(servletRequest.getRemoteAddr()); Optional identity = authenticatedIdentity(servletRequest); @@ -206,7 +207,7 @@ private Query registerQuery(String statement, QueryLanguage queryLanguage, HttpS MultivaluedMap headers = httpHeaders.getRequestHeaders(); SessionContext sessionContext = sessionContextFactory.createSessionContext(headers, remoteAddress, identity); - Query query = new Query(statement, queryLanguage, sessionContext, dispatchManager, queryInfoUrlFactory, tracer); + Query query = new Query(submission, sessionContext, dispatchManager, queryInfoUrlFactory, tracer); queryManager.registerQuery(query); // let authentication filter know that identity lifecycle has been handed off @@ -326,8 +327,7 @@ enum SubmissionState private static final class Query { - private final String query; - private final QueryLanguage queryLanguage; + private final QuerySubmission submission; private final SessionContext sessionContext; private final DispatchManager dispatchManager; private final QueryId queryId; @@ -340,10 +340,9 @@ private static final class Query private final AtomicReference submissionGate = new AtomicReference<>(NOT_SUBMITTED); private final SettableFuture creationFuture = SettableFuture.create(); - public Query(String query, QueryLanguage queryLanguage, SessionContext sessionContext, DispatchManager dispatchManager, QueryInfoUrlFactory queryInfoUrlFactory, Tracer tracer) + public Query(QuerySubmission submission, SessionContext sessionContext, DispatchManager dispatchManager, QueryInfoUrlFactory queryInfoUrlFactory, Tracer tracer) { - this.query = requireNonNull(query, "query is null"); - this.queryLanguage = requireNonNull(queryLanguage, "queryLanguage is null"); + this.submission = requireNonNull(submission, "submission is null"); this.sessionContext = requireNonNull(sessionContext, "sessionContext is null"); this.dispatchManager = requireNonNull(dispatchManager, "dispatchManager is null"); this.queryId = dispatchManager.createQueryId(); @@ -399,7 +398,7 @@ private void submitIfNeeded() { if (submissionGate.compareAndSet(NOT_SUBMITTED, SUBMITTED)) { querySpan.addEvent("submit"); - creationFuture.setFuture(dispatchManager.createQuery(queryId, querySpan, slug, sessionContext, query, queryLanguage)); + creationFuture.setFuture(dispatchManager.createQuery(queryId, querySpan, slug, sessionContext, submission)); } } diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryLanguage.java b/core/trino-main/src/main/java/io/trino/execution/QueryLanguage.java index 245ea30a749d..2d56e69400d2 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryLanguage.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryLanguage.java @@ -15,6 +15,6 @@ public enum QueryLanguage { - SQL, + TRINO, HOGQL, } diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java index 14a7cece996d..3e5c8cff0136 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java @@ -64,16 +64,18 @@ private QueryPreparer(SqlParser sqlParser, Optional hogQlCompiler public PreparedQuery prepareQuery(Session session, String query) throws ParsingException, TrinoException { - return prepareQuery(session, query, QueryLanguage.SQL); + return prepareQuery(session, QuerySubmission.trino(query)); } - public PreparedQuery prepareQuery(Session session, String query, QueryLanguage queryLanguage) + public PreparedQuery prepareQuery(Session session, QuerySubmission submission) throws ParsingException, TrinoException { - requireNonNull(queryLanguage, "queryLanguage is null"); - Statement wrappedStatement = switch (queryLanguage) { - case SQL -> sqlParser.createStatement(query); - case HOGQL -> hogQlCompiler.orElseThrow(() -> new IllegalStateException("HogQL compiler is not configured")).compile(query); + requireNonNull(submission, "submission is null"); + Statement wrappedStatement = switch (submission.language()) { + case TRINO -> sqlParser.createStatement(submission.originalText()); + case HOGQL -> hogQlCompiler + .orElseThrow(() -> new TrinoException(NOT_SUPPORTED, "HogQL query submission is disabled")) + .compile(submission.originalText()); }; return prepareQuery(session, wrappedStatement); } diff --git a/core/trino-main/src/main/java/io/trino/execution/QuerySubmission.java b/core/trino-main/src/main/java/io/trino/execution/QuerySubmission.java new file mode 100644 index 000000000000..97d41c619bbb --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/execution/QuerySubmission.java @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +import static io.trino.execution.QueryLanguage.HOGQL; +import static io.trino.execution.QueryLanguage.TRINO; +import static java.util.Objects.requireNonNull; + +public record QuerySubmission(QueryLanguage language, String originalText) +{ + public QuerySubmission + { + requireNonNull(language, "language is null"); + requireNonNull(originalText, "originalText is null"); + } + + public static QuerySubmission trino(String originalText) + { + return new QuerySubmission(TRINO, originalText); + } + + public static QuerySubmission hogQl(String originalText) + { + return new QuerySubmission(HOGQL, originalText); + } +} diff --git a/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java b/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java index 4269d4281a69..2e669b3a89c9 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java @@ -24,9 +24,11 @@ import org.junit.jupiter.api.Test; import static io.trino.SessionTestUtils.TEST_SESSION; -import static io.trino.execution.QueryLanguage.HOGQL; +import static io.trino.execution.QuerySubmission.hogQl; +import static io.trino.execution.QuerySubmission.trino; import static io.trino.spi.StandardErrorCode.INVALID_PARAMETER_USAGE; import static io.trino.spi.StandardErrorCode.NOT_FOUND; +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; import static io.trino.sql.QueryUtil.selectList; import static io.trino.sql.QueryUtil.simpleQuery; import static io.trino.sql.QueryUtil.table; @@ -45,16 +47,26 @@ public class TestQueryPreparer public void testSelectStatement() { PreparedQuery preparedQuery = QUERY_PREPARER.prepareQuery(TEST_SESSION, "SELECT * FROM foo"); + PreparedQuery submittedQuery = QUERY_PREPARER.prepareQuery(TEST_SESSION, trino("SELECT * FROM foo")); assertThat(preparedQuery.getStatement()).isEqualTo(simpleQuery(selectList(new AllColumns()), table(QualifiedName.of("foo")))); + assertThat(submittedQuery.getStatement()).isEqualTo(preparedQuery.getStatement()); } @Test public void testHogQlStatement() { - PreparedQuery preparedQuery = HOGQL_QUERY_PREPARER.prepareQuery(TEST_SESSION, "SELECT 1", HOGQL); + PreparedQuery preparedQuery = HOGQL_QUERY_PREPARER.prepareQuery(TEST_SESSION, hogQl("SELECT 1")); assertThat(preparedQuery.getStatement()).isInstanceOf(Query.class); } + @Test + public void testHogQlSubmissionFailsClosedWithoutCompiler() + { + assertTrinoExceptionThrownBy(() -> QUERY_PREPARER.prepareQuery(TEST_SESSION, hogQl("SELECT 1"))) + .hasErrorCode(NOT_SUPPORTED) + .hasMessage("HogQL query submission is disabled"); + } + @Test public void testExecuteStatement() { From 677be76f013dc18523cf0a5bdacd13a8c9642a0b Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 09:49:24 -0400 Subject: [PATCH 020/136] feat(hogql): define semantic catalog snapshots --- .../HogQlSemanticCatalogException.java | 41 +++ .../catalog/HogQlSemanticCatalogSnapshot.java | 281 ++++++++++++++++++ .../HogQlSemanticCatalogSnapshotCache.java | 24 ++ .../HogQlSemanticCatalogSnapshotProvider.java | 80 +++++ .../TestHogQlSemanticCatalogSnapshot.java | 198 ++++++++++++ ...tHogQlSemanticCatalogSnapshotProvider.java | 121 ++++++++ 6 files changed, 745 insertions(+) create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogException.java create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotCache.java create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotProvider.java create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogException.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogException.java new file mode 100644 index 000000000000..a12a655e4e5b --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogException.java @@ -0,0 +1,41 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import static java.util.Objects.requireNonNull; + +public final class HogQlSemanticCatalogException + extends RuntimeException +{ + private final Failure failure; + + public HogQlSemanticCatalogException(Failure failure, String message) + { + super(message); + this.failure = requireNonNull(failure, "failure is null"); + } + + public Failure failure() + { + return failure; + } + + public enum Failure + { + UNAVAILABLE, + CATALOG_MISMATCH, + LANGUAGE_VERSION_MISMATCH, + GENERATION_MISMATCH, + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java new file mode 100644 index 000000000000..d97f25ae51ad --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -0,0 +1,281 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.parser.HogQlLanguageVersion; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Pattern; + +import static java.util.Objects.requireNonNull; + +public record HogQlSemanticCatalogSnapshot( + int schemaVersion, + HogQlLanguageVersion languageVersion, + PhysicalIdentifier catalog, + long generation, + List logicalTables) +{ + public HogQlSemanticCatalogSnapshot + { + if (schemaVersion != 1) { + throw new IllegalArgumentException("unsupported HogQL semantic catalog schema: " + schemaVersion); + } + if (generation <= 0) { + throw new IllegalArgumentException("HogQL semantic catalog generation must be positive"); + } + languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + catalog = requireNonNull(catalog, "catalog is null"); + logicalTables = List.copyOf(requireNonNull(logicalTables, "logicalTables is null")); + + Map tablesByName = indexTables(catalog, logicalTables); + validateReferences(tablesByName); + } + + public Optional logicalTable(String name) + { + String canonicalName = canonicalName(name, "logical table"); + return logicalTables.stream() + .filter(table -> canonicalName(table.name(), "logical table").equals(canonicalName)) + .findFirst(); + } + + private static Map indexTables(PhysicalIdentifier catalog, List tables) + { + Map tablesByName = new HashMap<>(); + for (LogicalTableDefinition table : tables) { + if (!table.physicalTable().catalog().equals(catalog)) { + throw new IllegalArgumentException("logical table physical reference uses another catalog: " + table.name()); + } + String canonicalName = canonicalName(table.name(), "logical table"); + if (tablesByName.put(canonicalName, table) != null) { + throw new IllegalArgumentException("duplicate logical table: " + table.name()); + } + validateMemberNames(table); + } + return Map.copyOf(tablesByName); + } + + private static void validateMemberNames(LogicalTableDefinition table) + { + Set names = new HashSet<>(); + table.fields().forEach(field -> addMemberName(table, names, field.name())); + table.properties().forEach(property -> addMemberName(table, names, property.name())); + table.relationships().forEach(relationship -> addMemberName(table, names, relationship.name())); + } + + private static void addMemberName(LogicalTableDefinition table, Set names, String name) + { + if (!names.add(canonicalName(name, "logical member"))) { + throw new IllegalArgumentException("duplicate logical member in table " + table.name() + ": " + name); + } + } + + private static void validateReferences(Map tablesByName) + { + for (LogicalTableDefinition table : tablesByName.values()) { + Set fieldNames = fieldNames(table); + for (PropertyDefinition property : table.properties()) { + if (!fieldNames.contains(canonicalName(property.sourceField(), "property source field"))) { + throw new IllegalArgumentException("property has unknown source field in table " + table.name() + ": " + property.name()); + } + } + for (RelationshipDefinition relationship : table.relationships()) { + LogicalTableDefinition target = tablesByName.get(canonicalName(relationship.targetTable(), "relationship target table")); + if (target == null) { + throw new IllegalArgumentException("relationship has unknown target table in table " + table.name() + ": " + relationship.name()); + } + Set targetFieldNames = fieldNames(target); + for (JoinKey joinKey : relationship.joinKeys()) { + if (!fieldNames.contains(canonicalName(joinKey.sourceField(), "relationship source field"))) { + throw new IllegalArgumentException("relationship has unknown source field in table " + table.name() + ": " + relationship.name()); + } + if (!targetFieldNames.contains(canonicalName(joinKey.targetField(), "relationship target field"))) { + throw new IllegalArgumentException("relationship has unknown target field in table " + table.name() + ": " + relationship.name()); + } + } + } + } + } + + private static Set fieldNames(LogicalTableDefinition table) + { + Set fieldNames = new HashSet<>(); + for (LogicalFieldDefinition field : table.fields()) { + fieldNames.add(canonicalName(field.name(), "logical field")); + } + return Set.copyOf(fieldNames); + } + + private static String canonicalName(String name, String kind) + { + String value = validateDefinitionText(name, kind); + return value.toLowerCase(Locale.ENGLISH); + } + + private static String validateDefinitionText(String value, String kind) + { + requireNonNull(value, kind + " is null"); + if (value.isBlank() || containsExecutableDelimiter(value)) { + throw new IllegalArgumentException("invalid " + kind + ": " + value); + } + return value; + } + + private static boolean containsExecutableDelimiter(String value) + { + return value.indexOf(';') >= 0 || value.indexOf('\0') >= 0 || value.indexOf('\n') >= 0 || value.indexOf('\r') >= 0 || value.contains("--") || value.contains("/*") || value.contains("*/"); + } + + public record LogicalTableDefinition( + String name, + PhysicalQualifiedName physicalTable, + List fields, + List properties, + List relationships) + { + public LogicalTableDefinition + { + name = validateDefinitionText(name, "logical table name"); + physicalTable = requireNonNull(physicalTable, "physicalTable is null"); + fields = List.copyOf(requireNonNull(fields, "fields is null")); + properties = List.copyOf(requireNonNull(properties, "properties is null")); + relationships = List.copyOf(requireNonNull(relationships, "relationships is null")); + } + } + + public record LogicalFieldDefinition( + String name, + PhysicalIdentifier physicalColumn, + String trinoTypeSignature, + LogicalType logicalType, + boolean nullable, + boolean starVisible) + { + public LogicalFieldDefinition + { + name = validateDefinitionText(name, "logical field name"); + physicalColumn = requireNonNull(physicalColumn, "physicalColumn is null"); + trinoTypeSignature = validateDefinitionText(trinoTypeSignature, "Trino type signature"); + logicalType = requireNonNull(logicalType, "logicalType is null"); + } + } + + public record PropertyDefinition( + String name, + String sourceField, + PropertyStorage storage, + LogicalType logicalType, + boolean nullable) + { + public PropertyDefinition + { + name = validateDefinitionText(name, "property name"); + sourceField = validateDefinitionText(sourceField, "property source field"); + storage = requireNonNull(storage, "storage is null"); + logicalType = requireNonNull(logicalType, "logicalType is null"); + } + } + + public record RelationshipDefinition( + String name, + String targetTable, + RelationshipCardinality cardinality, + List joinKeys) + { + public RelationshipDefinition + { + name = validateDefinitionText(name, "relationship name"); + targetTable = validateDefinitionText(targetTable, "relationship target table"); + cardinality = requireNonNull(cardinality, "cardinality is null"); + joinKeys = List.copyOf(requireNonNull(joinKeys, "joinKeys is null")); + if (joinKeys.isEmpty()) { + throw new IllegalArgumentException("relationship must have at least one join key: " + name); + } + } + } + + public record JoinKey(String sourceField, String targetField) + { + public JoinKey + { + sourceField = validateDefinitionText(sourceField, "relationship source field"); + targetField = validateDefinitionText(targetField, "relationship target field"); + } + } + + public record PhysicalQualifiedName(PhysicalIdentifier catalog, PhysicalIdentifier schema, PhysicalIdentifier table) + { + public PhysicalQualifiedName + { + catalog = requireNonNull(catalog, "catalog is null"); + schema = requireNonNull(schema, "schema is null"); + table = requireNonNull(table, "table is null"); + } + } + + public record PhysicalIdentifier(String value, boolean delimited) + { + private static final Pattern UNDELIMITED_IDENTIFIER = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*"); + + public PhysicalIdentifier + { + value = validateDefinitionText(value, "physical identifier"); + if (!delimited && !UNDELIMITED_IDENTIFIER.matcher(value).matches()) { + throw new IllegalArgumentException("invalid physical identifier: " + value); + } + if (!delimited) { + value = value.toLowerCase(Locale.ENGLISH); + } + } + } + + public enum LogicalType + { + UNKNOWN, + BOOLEAN, + INTEGER, + FLOAT, + DECIMAL, + STRING, + DATE, + TIMESTAMP, + INTERVAL, + UUID, + JSON, + ARRAY, + MAP, + ROW, + } + + public enum PropertyStorage + { + JSON_OBJECT, + MAP, + } + + public enum RelationshipCardinality + { + ONE_TO_ONE, + ONE_TO_MANY, + MANY_TO_ONE, + MANY_TO_MANY, + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotCache.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotCache.java new file mode 100644 index 000000000000..632a5afc5cb1 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotCache.java @@ -0,0 +1,24 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; + +import java.util.Optional; + +@FunctionalInterface +public interface HogQlSemanticCatalogSnapshotCache +{ + Optional currentSnapshot(PhysicalIdentifier catalog); +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotProvider.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotProvider.java new file mode 100644 index 000000000000..936c3a0213e0 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotProvider.java @@ -0,0 +1,80 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException.Failure; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.parser.HogQlLanguageVersion; + +import java.util.Optional; +import java.util.OptionalLong; + +import static java.util.Objects.requireNonNull; + +@FunctionalInterface +public interface HogQlSemanticCatalogSnapshotProvider +{ + PinnedSnapshot pin(PinRequest request); + + static HogQlSemanticCatalogSnapshotProvider fromCache(HogQlSemanticCatalogSnapshotCache cache) + { + requireNonNull(cache, "cache is null"); + return request -> { + HogQlSemanticCatalogSnapshot snapshot = cache.currentSnapshot(request.catalog()) + .orElseThrow(() -> new HogQlSemanticCatalogException(Failure.UNAVAILABLE, "HogQL semantic catalog snapshot is unavailable")); + if (!snapshot.catalog().equals(request.catalog())) { + throw new HogQlSemanticCatalogException(Failure.CATALOG_MISMATCH, "HogQL semantic catalog snapshot does not match the requested catalog"); + } + if (!snapshot.languageVersion().equals(request.languageVersion())) { + throw new HogQlSemanticCatalogException(Failure.LANGUAGE_VERSION_MISMATCH, "HogQL semantic catalog snapshot language version does not match the compiler"); + } + if (request.expectedGeneration().isPresent() && request.expectedGeneration().orElseThrow() != snapshot.generation()) { + throw new HogQlSemanticCatalogException(Failure.GENERATION_MISMATCH, "HogQL semantic catalog snapshot generation does not match the request"); + } + return new PinnedSnapshot(snapshot); + }; + } + + record PinRequest(PhysicalIdentifier catalog, HogQlLanguageVersion languageVersion, OptionalLong expectedGeneration) + { + public PinRequest + { + catalog = requireNonNull(catalog, "catalog is null"); + languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + expectedGeneration = requireNonNull(expectedGeneration, "expectedGeneration is null"); + if (expectedGeneration.isPresent() && expectedGeneration.orElseThrow() <= 0) { + throw new IllegalArgumentException("expected HogQL semantic catalog generation must be positive"); + } + } + } + + record PinnedSnapshot(HogQlSemanticCatalogSnapshot snapshot) + { + public PinnedSnapshot + { + snapshot = requireNonNull(snapshot, "snapshot is null"); + } + + public long generation() + { + return snapshot.generation(); + } + + public Optional logicalTable(String name) + { + return snapshot.logicalTable(name); + } + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java new file mode 100644 index 000000000000..400f05fdf723 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java @@ -0,0 +1,198 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.parser.HogQlLanguageVersion; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlSemanticCatalogSnapshot +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageVersion.valueOf("1.0.0"); + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("ducklake", false); + + @Test + public void testDoesNotExposePublisherMutability() + { + List fields = new ArrayList<>(List.of(field("id"))); + List properties = new ArrayList<>(List.of(property("properties", "id"))); + List joinKeys = new ArrayList<>(List.of(new JoinKey("id", "id"))); + List relationships = new ArrayList<>(List.of( + new RelationshipDefinition("person", "persons", RelationshipCardinality.MANY_TO_ONE, joinKeys))); + List tables = new ArrayList<>(List.of( + table("events", fields, properties, relationships), + table("persons"))); + HogQlSemanticCatalogSnapshot snapshot = snapshot(tables); + + fields.add(field("event")); + properties.clear(); + joinKeys.clear(); + relationships.clear(); + tables.clear(); + + assertThat(snapshot.logicalTables()).hasSize(2); + assertThat(snapshot.logicalTable("events")).get().satisfies(table -> { + assertThat(table.fields()).hasSize(1); + assertThat(table.properties()).hasSize(1); + assertThat(table.relationships()).singleElement().satisfies(relationship -> assertThat(relationship.joinKeys()).hasSize(1)); + }); + assertThatThrownBy(() -> snapshot.logicalTables().add(table("persons"))) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> snapshot.logicalTable("events").orElseThrow().fields().add(field("event"))) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> snapshot.logicalTable("events").orElseThrow().relationships().getFirst().joinKeys().clear()) + .isInstanceOf(UnsupportedOperationException.class); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("invalidSnapshots") + public void testRejectsInvalidDefinitions(String name, List tables, String message) + { + assertThatThrownBy(() -> snapshot(tables)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(message); + } + + @Test + public void testAcceptsBidirectionalLazyRelationships() + { + HogQlSemanticCatalogSnapshot snapshot = snapshot(List.of( + table("events", List.of(field("id")), List.of(), List.of(relationship("person", "persons", "id", "id"))), + table("persons", List.of(field("id")), List.of(), List.of(relationship("events", "events", "id", "id"))))); + + assertThat(snapshot.logicalTable("events")).get() + .extracting(LogicalTableDefinition::relationships) + .satisfies(relationships -> assertThat(relationships).singleElement().extracting(RelationshipDefinition::targetTable).isEqualTo("persons")); + assertThat(snapshot.logicalTable("persons")).get() + .extracting(LogicalTableDefinition::relationships) + .satisfies(relationships -> assertThat(relationships).singleElement().extracting(RelationshipDefinition::targetTable).isEqualTo("events")); + } + + private static Stream invalidSnapshots() + { + LogicalTableDefinition events = table("events"); + LogicalTableDefinition persons = table("persons"); + return Stream.of( + Arguments.of("duplicate tables", List.of(events, events), "duplicate logical table"), + Arguments.of( + "duplicate members", + List.of(table("events", List.of(field("id"), field("ID")), List.of(), List.of())), + "duplicate logical member"), + Arguments.of( + "missing property field", + List.of(table("events", List.of(field("id")), List.of(property("properties", "missing")), List.of())), + "unknown source field"), + Arguments.of( + "missing relationship table", + List.of(table("events", List.of(field("id")), List.of(), List.of(relationship("person", "persons", "id", "id")))), + "unknown target table"), + Arguments.of( + "missing relationship source field", + List.of( + table("events", List.of(field("id")), List.of(), List.of(relationship("person", "persons", "missing", "id"))), + persons), + "unknown source field"), + Arguments.of( + "missing relationship target field", + List.of( + table("events", List.of(field("id")), List.of(), List.of(relationship("person", "persons", "id", "missing"))), + persons), + "unknown target field")); + } + + @ParameterizedTest + @ValueSource(strings = { + "events; DROP TABLE events", + "events\nSELECT 1", + "events/* injected */", + "events-- injected", + }) + public void testRejectsRawExecutablePhysicalDefinitions(String definition) + { + for (boolean delimited : List.of(false, true)) { + assertThatThrownBy(() -> new PhysicalIdentifier(definition, delimited)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("physical identifier"); + } + } + + private static HogQlSemanticCatalogSnapshot snapshot(List tables) + { + return new HogQlSemanticCatalogSnapshot(1, LANGUAGE_VERSION, CATALOG, 7, tables); + } + + private static LogicalTableDefinition table(String name) + { + return table(name, List.of(field("id")), List.of(), List.of()); + } + + private static LogicalTableDefinition table( + String name, + List fields, + List properties, + List relationships) + { + return new LogicalTableDefinition( + name, + new PhysicalQualifiedName(CATALOG, new PhysicalIdentifier("default", false), new PhysicalIdentifier(name, false)), + fields, + properties, + relationships); + } + + private static LogicalFieldDefinition field(String name) + { + return new LogicalFieldDefinition( + name, + new PhysicalIdentifier(name, false), + "varchar", + LogicalType.STRING, + true, + true); + } + + private static PropertyDefinition property(String name, String sourceField) + { + return new PropertyDefinition(name, sourceField, PropertyStorage.JSON_OBJECT, LogicalType.JSON, true); + } + + private static RelationshipDefinition relationship(String name, String targetTable, String sourceField, String targetField) + { + return new RelationshipDefinition( + name, + targetTable, + RelationshipCardinality.MANY_TO_ONE, + List.of(new JoinKey(sourceField, targetField))); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java new file mode 100644 index 000000000000..fa587818b200 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java @@ -0,0 +1,121 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException.Failure; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinRequest; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageVersion; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlSemanticCatalogSnapshotProvider +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageVersion.valueOf("1.0.0"); + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("ducklake", false); + + @Test + public void testPinsOneSnapshotWithoutCrossGenerationReads() + { + AtomicInteger reads = new AtomicInteger(); + HogQlSemanticCatalogSnapshotCache cache = _ -> Optional.of(snapshot(reads.incrementAndGet())); + HogQlSemanticCatalogSnapshotProvider provider = HogQlSemanticCatalogSnapshotProvider.fromCache(cache); + + PinnedSnapshot pinned = provider.pin(new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.empty())); + + assertThat(reads).hasValue(1); + assertThat(pinned.generation()).isEqualTo(1); + assertThat(pinned.snapshot().generation()).isEqualTo(1); + assertThat(pinned.logicalTable("events")).isPresent(); + assertThat(reads).hasValue(1); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("failClosedCases") + public void testFailsClosed(String name, HogQlSemanticCatalogSnapshotCache cache, PinRequest request, Failure failure) + { + HogQlSemanticCatalogSnapshotProvider provider = HogQlSemanticCatalogSnapshotProvider.fromCache(cache); + + assertThatThrownBy(() -> provider.pin(request)) + .isInstanceOfSatisfying(HogQlSemanticCatalogException.class, exception -> assertThat(exception.failure()).isEqualTo(failure)); + } + + private static Stream failClosedCases() + { + PhysicalIdentifier otherCatalog = new PhysicalIdentifier("other", false); + return Stream.of( + Arguments.of( + "unavailable", + (HogQlSemanticCatalogSnapshotCache) _ -> Optional.empty(), + new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.empty()), + Failure.UNAVAILABLE), + Arguments.of( + "cache returned another catalog", + (HogQlSemanticCatalogSnapshotCache) _ -> Optional.of(snapshot(otherCatalog, LANGUAGE_VERSION, 1)), + new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.empty()), + Failure.CATALOG_MISMATCH), + Arguments.of( + "language mismatch", + (HogQlSemanticCatalogSnapshotCache) _ -> Optional.of(snapshot(CATALOG, HogQlLanguageVersion.valueOf("2.0.0"), 1)), + new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.empty()), + Failure.LANGUAGE_VERSION_MISMATCH), + Arguments.of( + "generation mismatch", + (HogQlSemanticCatalogSnapshotCache) _ -> Optional.of(snapshot(2)), + new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.of(1)), + Failure.GENERATION_MISMATCH)); + } + + private static HogQlSemanticCatalogSnapshot snapshot(long generation) + { + return snapshot(CATALOG, LANGUAGE_VERSION, generation); + } + + private static HogQlSemanticCatalogSnapshot snapshot(PhysicalIdentifier catalog, HogQlLanguageVersion languageVersion, long generation) + { + return new HogQlSemanticCatalogSnapshot( + 1, + languageVersion, + catalog, + generation, + List.of(new LogicalTableDefinition( + "events", + new PhysicalQualifiedName(catalog, new PhysicalIdentifier("default", false), new PhysicalIdentifier("events", false)), + List.of(new LogicalFieldDefinition( + "id", + new PhysicalIdentifier("id", false), + "varchar", + LogicalType.STRING, + false, + true)), + List.of(), + List.of()))); + } +} From c3db92ecf84e56fc0c578d777e9201cc88a91142 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 09:54:31 -0400 Subject: [PATCH 021/136] feat(hogql): classify catalog snapshot failures --- .../io/trino/hogql/compiler/HogQlErrorCode.java | 5 +++-- .../catalog/HogQlSemanticCatalogException.java | 17 +++++++++++++++-- ...estHogQlSemanticCatalogSnapshotProvider.java | 8 +++++++- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlErrorCode.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlErrorCode.java index 34e42896d086..dd1b9f2d5757 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlErrorCode.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlErrorCode.java @@ -17,6 +17,7 @@ import io.trino.spi.ErrorCodeSupplier; import io.trino.spi.ErrorType; +import static io.trino.spi.ErrorType.EXTERNAL; import static io.trino.spi.ErrorType.INTERNAL_ERROR; import static io.trino.spi.ErrorType.USER_ERROR; @@ -28,8 +29,8 @@ public enum HogQlErrorCode HOGQL_RESOLUTION_ERROR(2, USER_ERROR), HOGQL_TYPE_ERROR(3, USER_ERROR), HOGQL_UNSUPPORTED_FEATURE(4, USER_ERROR), - HOGQL_CATALOG_NOT_READY(5, USER_ERROR), - HOGQL_CATALOG_GENERATION_MISMATCH(6, USER_ERROR), + HOGQL_CATALOG_NOT_READY(5, EXTERNAL), + HOGQL_CATALOG_GENERATION_MISMATCH(6, EXTERNAL), HOGQL_COMPILER_LIMIT_EXCEEDED(7, USER_ERROR), HOGQL_COMPILER_INTERNAL_ERROR(8, INTERNAL_ERROR), ; diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogException.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogException.java index a12a655e4e5b..905fefe45ab9 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogException.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogException.java @@ -13,16 +13,21 @@ */ package io.trino.hogql.compiler.catalog; +import io.trino.hogql.compiler.HogQlErrorCode; +import io.trino.spi.TrinoException; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_CATALOG_GENERATION_MISMATCH; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_CATALOG_NOT_READY; import static java.util.Objects.requireNonNull; public final class HogQlSemanticCatalogException - extends RuntimeException + extends TrinoException { private final Failure failure; public HogQlSemanticCatalogException(Failure failure, String message) { - super(message); + super(errorCode(requireNonNull(failure, "failure is null")), message); this.failure = requireNonNull(failure, "failure is null"); } @@ -31,6 +36,14 @@ public Failure failure() return failure; } + private static HogQlErrorCode errorCode(Failure failure) + { + return switch (failure) { + case UNAVAILABLE -> HOGQL_CATALOG_NOT_READY; + case CATALOG_MISMATCH, LANGUAGE_VERSION_MISMATCH, GENERATION_MISMATCH -> HOGQL_CATALOG_GENERATION_MISMATCH; + }; + } + public enum Failure { UNAVAILABLE, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java index fa587818b200..8ec41619ff95 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java @@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; +import static io.trino.spi.ErrorType.EXTERNAL; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -64,7 +65,12 @@ public void testFailsClosed(String name, HogQlSemanticCatalogSnapshotCache cache HogQlSemanticCatalogSnapshotProvider provider = HogQlSemanticCatalogSnapshotProvider.fromCache(cache); assertThatThrownBy(() -> provider.pin(request)) - .isInstanceOfSatisfying(HogQlSemanticCatalogException.class, exception -> assertThat(exception.failure()).isEqualTo(failure)); + .isInstanceOfSatisfying(HogQlSemanticCatalogException.class, exception -> { + assertThat(exception.failure()).isEqualTo(failure); + assertThat(exception.getErrorCode().getName()).isEqualTo( + failure == Failure.UNAVAILABLE ? "HOGQL_CATALOG_NOT_READY" : "HOGQL_CATALOG_GENERATION_MISMATCH"); + assertThat(exception.getErrorCode().getType()).isEqualTo(EXTERNAL); + }); } private static Stream failClosedCases() From 1e8179f5f6b91b6d8dd426683c9b29984836f729 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 10:08:46 -0400 Subject: [PATCH 022/136] feat(hogql): lower named parameter placeholders --- .../compiler/HogQlCompilationResult.java | 29 ++++ .../trino/hogql/compiler/HogQlCompiler.java | 148 +++++++++++++++++- .../trino/hogql/compiler/TrinoAstFactory.java | 82 ++++++---- .../hogql/compiler/TestHogQlCompiler.java | 105 +++++++++++++ .../io/trino/hogql/parser/HogQlParser.java | 28 +++- .../trino/hogql/parser/tree/HogQlQuery.java | 38 ++++- .../trino/hogql/parser/TestHogQlParser.java | 21 ++- 7 files changed, 409 insertions(+), 42 deletions(-) create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java new file mode 100644 index 000000000000..e7878b099b80 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.sql.tree.Statement; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public record HogQlCompilationResult(Statement statement, List parameterNames) +{ + public HogQlCompilationResult + { + statement = requireNonNull(statement, "statement is null"); + parameterNames = List.copyOf(requireNonNull(parameterNames, "parameterNames is null")); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index c5d73a578839..c8397d1e95fb 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -15,12 +15,37 @@ import io.trino.hogql.parser.HogQlParser; import io.trino.hogql.parser.HogQlParsingException; +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; import io.trino.spi.Location; import io.trino.spi.TrinoException; import io.trino.sql.tree.Statement; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_BINDING_ERROR; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_SYNTAX_ERROR; import static java.util.Objects.requireNonNull; @@ -40,8 +65,21 @@ public HogQlCompiler() public Statement compile(String hogql) { + return compile(hogql, Map.of()).statement(); + } + + public HogQlCompilationResult compile(HogQlCompileEnvelope envelope) + { + requireNonNull(envelope, "envelope is null"); + return compile(envelope.query(), envelope.parameters()); + } + + public HogQlCompilationResult compile(String hogql, Map parameters) + { + requireNonNull(hogql, "hogql is null"); + requireNonNull(parameters, "parameters is null"); try { - return TrinoAstFactory.createStatement(parser.parseStatement(hogql)); + return compile(parser.parseStatement(hogql), parameters); } catch (HogQlParsingException exception) { throw new TrinoException( @@ -51,4 +89,112 @@ public Statement compile(String hogql) exception); } } + + private static HogQlCompilationResult compile(HogQlQuery query, Map parameters) + { + validateParameters(query, parameters); + if (query.from().orElse(null) instanceof TablePlaceholder tablePlaceholder) { + throw bindingError(tablePlaceholder.span(), "HogQL parameter placeholders are not supported in table positions: " + tablePlaceholder.placeholder().name()); + } + + List placeholders = new ArrayList<>(); + query.projections().forEach(projection -> { + if (projection instanceof HogQlQuery.ExpressionProjection expressionProjection) { + collectPlaceholders(expressionProjection.expression(), placeholders); + } + }); + query.where().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + placeholders.sort(Comparator.comparingInt(placeholder -> placeholder.span().startOffset())); + + List missing = placeholders.stream() + .map(Placeholder::name) + .filter(name -> !parameters.containsKey(name)) + .distinct() + .toList(); + if (!missing.isEmpty()) { + Placeholder firstMissing = placeholders.stream() + .filter(placeholder -> missing.contains(placeholder.name())) + .findFirst() + .orElseThrow(); + throw bindingError(firstMissing.span(), "Missing HogQL parameter bindings: " + String.join(", ", missing)); + } + + Set placeholderNames = new HashSet<>(); + placeholders.stream() + .map(Placeholder::name) + .forEach(placeholderNames::add); + List extra = parameters.keySet().stream() + .filter(name -> !placeholderNames.contains(name)) + .sorted() + .toList(); + if (!extra.isEmpty()) { + throw bindingError(query.span(), "Unused HogQL parameter bindings: " + String.join(", ", extra)); + } + + Map parameterIds = new HashMap<>(); + for (int index = 0; index < placeholders.size(); index++) { + parameterIds.put(placeholders.get(index).span(), index); + } + Statement statement = TrinoAstFactory.createStatement(query, parameterIds); + return new HogQlCompilationResult(statement, placeholders.stream() + .map(Placeholder::name) + .toList()); + } + + private static void validateParameters(HogQlQuery query, Map parameters) + { + for (Map.Entry parameter : parameters.entrySet()) { + if (parameter.getKey() == null || parameter.getKey().isBlank()) { + throw bindingError(query.span(), "HogQL parameter binding name is empty"); + } + if (parameter.getValue() == null) { + throw bindingError(query.span(), "HogQL parameter binding has no typed value: " + parameter.getKey()); + } + } + } + + private static void collectPlaceholders(Expression expression, List placeholders) + { + switch (expression) { + case ArrayExpression array -> array.values().forEach(value -> collectPlaceholders(value, placeholders)); + case BetweenExpression between -> { + collectPlaceholders(between.value(), placeholders); + collectPlaceholders(between.min(), placeholders); + collectPlaceholders(between.max(), placeholders); + } + case BinaryExpression binary -> { + collectPlaceholders(binary.left(), placeholders); + collectPlaceholders(binary.right(), placeholders); + } + case CaseExpression caseExpression -> { + caseExpression.operand().ifPresent(operand -> collectPlaceholders(operand, placeholders)); + caseExpression.whenClauses().forEach(when -> { + collectPlaceholders(when.operand(), placeholders); + collectPlaceholders(when.result(), placeholders); + }); + caseExpression.defaultValue().ifPresent(value -> collectPlaceholders(value, placeholders)); + } + case CastExpression cast -> collectPlaceholders(cast.value(), placeholders); + case ColumnReference _ -> {} + case FunctionCall function -> function.arguments().forEach(argument -> collectPlaceholders(argument, placeholders)); + case InExpression in -> { + collectPlaceholders(in.value(), placeholders); + in.values().forEach(value -> collectPlaceholders(value, placeholders)); + } + case IsNullExpression isNull -> collectPlaceholders(isNull.value(), placeholders); + case Literal _ -> {} + case Placeholder placeholder -> placeholders.add(placeholder); + case TupleExpression tuple -> tuple.values().forEach(value -> collectPlaceholders(value, placeholders)); + case UnaryExpression unary -> collectPlaceholders(unary.operand(), placeholders); + } + } + + private static TrinoException bindingError(SourceSpan span, String message) + { + return new TrinoException( + HOGQL_BINDING_ERROR, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 5b0655aba513..0f4bbc84941d 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -26,9 +26,12 @@ import io.trino.hogql.parser.tree.HogQlQuery.InExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; @@ -51,6 +54,7 @@ import io.trino.sql.tree.NodeLocation; import io.trino.sql.tree.NotExpression; import io.trino.sql.tree.NullLiteral; +import io.trino.sql.tree.Parameter; import io.trino.sql.tree.Predicated; import io.trino.sql.tree.QualifiedName; import io.trino.sql.tree.Query; @@ -67,22 +71,23 @@ import io.trino.sql.tree.WhenClause; import java.util.List; +import java.util.Map; import java.util.Optional; final class TrinoAstFactory { private TrinoAstFactory() {} - public static Statement createStatement(HogQlQuery query) + public static Statement createStatement(HogQlQuery query, Map parameterIds) { NodeLocation location = location(query.span()); QuerySpecification querySpecification = new QuerySpecification( location, new Select(location, false, query.projections().stream() - .map(TrinoAstFactory::createSelectItem) + .map(projection -> createSelectItem(projection, parameterIds)) .toList()), - query.from().map(TrinoAstFactory::createTable), - query.where().map(TrinoAstFactory::createExpression), + query.from().map(TrinoAstFactory::createRelation), + query.where().map(expression -> createExpression(expression, parameterIds)), Optional.empty(), Optional.empty(), List.of(), @@ -100,31 +105,31 @@ public static Statement createStatement(HogQlQuery query) Optional.empty()); } - private static SelectItem createSelectItem(Projection projection) + private static SelectItem createSelectItem(Projection projection, Map parameterIds) { return switch (projection) { case Star star -> new AllColumns(location(star.span())); case ExpressionProjection expression -> new SingleColumn( location(expression.span()), - createExpression(expression.expression()), + createExpression(expression.expression(), parameterIds), expression.alias().map(TrinoAstFactory::createIdentifier)); }; } - private static Expression createExpression(HogQlQuery.Expression expression) + private static Expression createExpression(HogQlQuery.Expression expression, Map parameterIds) { return switch (expression) { case ArrayExpression array -> new Array( location(array.span()), array.values().stream() - .map(TrinoAstFactory::createExpression) + .map(value -> createExpression(value, parameterIds)) .toList()); - case BetweenExpression between -> createBetweenExpression(between); - case BinaryExpression binary -> createBinaryExpression(binary); - case CaseExpression caseExpression -> createCaseExpression(caseExpression); + case BetweenExpression between -> createBetweenExpression(between, parameterIds); + case BinaryExpression binary -> createBinaryExpression(binary, parameterIds); + case CaseExpression caseExpression -> createCaseExpression(caseExpression, parameterIds); case CastExpression cast -> new Cast( location(cast.span()), - createExpression(cast.value()), + createExpression(cast.value(), parameterIds), new GenericDataType(location(cast.type().span()), createIdentifier(cast.type()), List.of()), cast.safe()); case ColumnReference reference -> createColumnReference(reference); @@ -132,77 +137,78 @@ private static Expression createExpression(HogQlQuery.Expression expression) location(function.span()), QualifiedName.of(List.of(createIdentifier(function.name()))), function.arguments().stream() - .map(TrinoAstFactory::createExpression) + .map(argument -> createExpression(argument, parameterIds)) .toList()); - case InExpression in -> createInExpression(in); + case InExpression in -> createInExpression(in, parameterIds); case IsNullExpression isNull -> new Predicated( location(isNull.predicateSpan()), - createExpression(isNull.value()), + createExpression(isNull.value(), parameterIds), new IsNullPredicate(location(isNull.predicateSpan()), isNull.negated())); case Literal literal -> createLiteral(literal); + case Placeholder placeholder -> new Parameter(location(placeholder.span()), parameterIds.get(placeholder.span())); case TupleExpression tuple -> new Row( location(tuple.span()), tuple.values().stream() - .map(value -> new Row.Field(location(value.span()), Optional.empty(), createExpression(value))) + .map(value -> new Row.Field(location(value.span()), Optional.empty(), createExpression(value, parameterIds))) .toList()); case UnaryExpression unary -> switch (unary.operator()) { - case NEGATE -> ArithmeticUnaryExpression.negative(location(unary.span()), createExpression(unary.operand())); - case NOT -> new NotExpression(location(unary.span()), createExpression(unary.operand())); - case POSITIVE -> ArithmeticUnaryExpression.positive(location(unary.span()), createExpression(unary.operand())); + case NEGATE -> ArithmeticUnaryExpression.negative(location(unary.span()), createExpression(unary.operand(), parameterIds)); + case NOT -> new NotExpression(location(unary.span()), createExpression(unary.operand(), parameterIds)); + case POSITIVE -> ArithmeticUnaryExpression.positive(location(unary.span()), createExpression(unary.operand(), parameterIds)); }; }; } - private static Expression createCaseExpression(CaseExpression caseExpression) + private static Expression createCaseExpression(CaseExpression caseExpression, Map parameterIds) { NodeLocation location = location(caseExpression.span()); List whenClauses = caseExpression.whenClauses().stream() .map(when -> new WhenClause( location(when.span()), - createExpression(when.operand()), - createExpression(when.result()))) + createExpression(when.operand(), parameterIds), + createExpression(when.result(), parameterIds))) .toList(); - Optional defaultValue = caseExpression.defaultValue().map(TrinoAstFactory::createExpression); + Optional defaultValue = caseExpression.defaultValue().map(value -> createExpression(value, parameterIds)); return caseExpression.operand() - .map(operand -> new SimpleCaseExpression(location, createExpression(operand), whenClauses, defaultValue)) + .map(operand -> new SimpleCaseExpression(location, createExpression(operand, parameterIds), whenClauses, defaultValue)) .orElseGet(() -> new SearchedCaseExpression(location, whenClauses, defaultValue)); } - private static Expression createBetweenExpression(BetweenExpression between) + private static Expression createBetweenExpression(BetweenExpression between, Map parameterIds) { NodeLocation location = location(between.predicateSpan()); return new Predicated( location, - createExpression(between.value()), + createExpression(between.value(), parameterIds), new BetweenPredicate( location, between.negated(), Optional.empty(), - createExpression(between.min()), - createExpression(between.max()))); + createExpression(between.min(), parameterIds), + createExpression(between.max(), parameterIds))); } - private static Expression createInExpression(InExpression in) + private static Expression createInExpression(InExpression in, Map parameterIds) { NodeLocation location = location(in.predicateSpan()); return new Predicated( location, - createExpression(in.value()), + createExpression(in.value(), parameterIds), new InPredicate( location, in.negated(), new InListExpression( location, in.values().stream() - .map(TrinoAstFactory::createExpression) + .map(value -> createExpression(value, parameterIds)) .toList()))); } - private static Expression createBinaryExpression(BinaryExpression binary) + private static Expression createBinaryExpression(BinaryExpression binary, Map parameterIds) { NodeLocation location = location(binary.span()); - Expression left = createExpression(binary.left()); - Expression right = createExpression(binary.right()); + Expression left = createExpression(binary.left(), parameterIds); + Expression right = createExpression(binary.right(), parameterIds); return switch (binary.operator()) { case ADD -> new ArithmeticBinaryExpression(location, ArithmeticBinaryExpression.Operator.ADD, left, right); case SUBTRACT -> new ArithmeticBinaryExpression(location, ArithmeticBinaryExpression.Operator.SUBTRACT, left, right); @@ -246,6 +252,14 @@ private static Expression createColumnReference(ColumnReference reference) return expression; } + private static io.trino.sql.tree.Relation createRelation(Relation relation) + { + return switch (relation) { + case TablePlaceholder _ -> throw new IllegalArgumentException("table placeholder was not validated"); + case TableReference table -> createTable(table); + }; + } + private static Table createTable(TableReference table) { List parts = table.parts().stream() diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index 231389a11a6b..eb05d4689954 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -13,6 +13,7 @@ */ package io.trino.hogql.compiler; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; import io.trino.spi.Location; import io.trino.spi.TrinoException; import io.trino.sql.SqlFormatter; @@ -21,6 +22,7 @@ import io.trino.sql.tree.Identifier; import io.trino.sql.tree.Node; import io.trino.sql.tree.NodeLocation; +import io.trino.sql.tree.Parameter; import io.trino.sql.tree.Predicated; import io.trino.sql.tree.Query; import io.trino.sql.tree.QuerySpecification; @@ -29,14 +31,24 @@ import io.trino.sql.tree.Table; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Comparator; import java.util.Deque; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowableOfType; +import static org.junit.jupiter.params.provider.Arguments.arguments; public class TestHogQlCompiler { @@ -147,6 +159,54 @@ public void testPreservesQuotedIdentifiersAndSourceLocations() .allMatch(Identifier::isDelimited); } + @Test + public void testLowersNamedPlaceholdersToOrderedPositionalParameters() + { + String firstValue = "sensitive-first-value"; + String laterValue = "sensitive-later-value"; + Map bindings = new LinkedHashMap<>(); + bindings.put("first", typedValue(firstValue)); + bindings.put("later", typedValue(laterValue)); + + HogQlCompilationResult result = compiler.compile("SELECT {later},\n {first} + {later}", bindings); + List parameters = parameters(result.statement()); + + assertThat(result.parameterNames()).containsExactly("later", "first", "later"); + assertThat(parameters) + .extracting(Parameter::getId) + .containsExactly(0, 1, 2); + assertThat(parameters) + .extracting(Parameter::getLocation) + .containsExactly( + Optional.of(new NodeLocation(1, 8)), + Optional.of(new NodeLocation(2, 2)), + Optional.of(new NodeLocation(2, 12))); + assertThat(SqlFormatter.formatSql(result.statement())) + .contains("?") + .doesNotContain(firstValue, laterValue, "{first}", "{later}"); + } + + @ParameterizedTest + @MethodSource("bindingErrors") + public void testReportsStableSourceLocatedBindingErrors( + String hogql, + Map bindings, + String expectedMessage, + Location expectedLocation, + String sensitiveValue) + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(hogql, bindings)); + + assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_BINDING_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(expectedLocation); + assertThat(exception) + .hasMessage(expectedMessage) + .message() + .doesNotContain(sensitiveValue); + } + @Test public void testReturnsOnlyStockTrinoTreeNodes() { @@ -172,4 +232,49 @@ public void testReportsSourceLocatedHogQlSyntaxError() assertThat(exception.getLocation()).contains(new Location(2, 1)); assertThat(exception).hasMessageStartingWith("line 2:1:"); } + + private static Stream bindingErrors() + { + return Stream.of( + arguments( + "SELECT\n {missing}", + Map.of(), + "line 2:2: Missing HogQL parameter bindings: missing", + new Location(2, 2), + "sensitive-missing-value"), + arguments( + "SELECT 1", + Map.of("extra", typedValue("sensitive-extra-value")), + "line 1:1: Unused HogQL parameter bindings: extra", + new Location(1, 1), + "sensitive-extra-value"), + arguments( + "SELECT *\nFROM {source}", + Map.of("source", typedValue("sensitive-table-value")), + "line 2:6: HogQL parameter placeholders are not supported in table positions: source", + new Location(2, 6), + "sensitive-table-value")); + } + + private static HogQlTypedValue typedValue(String value) + { + return new HogQlTypedValue("varchar", new StringValue(value)); + } + + private static List parameters(Statement statement) + { + Deque pending = new ArrayDeque<>(); + List parameters = new ArrayList<>(); + pending.add(statement); + while (!pending.isEmpty()) { + Node node = pending.removeFirst(); + if (node instanceof Parameter parameter) { + parameters.add(parameter); + } + pending.addAll(node.getChildren()); + } + return parameters.stream() + .sorted(Comparator.comparingInt(Parameter::getId)) + .toList(); + } } diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 965f0ccefc38..d40394b02c67 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -31,9 +31,12 @@ import io.trino.hogql.parser.tree.HogQlQuery.InExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; @@ -443,7 +446,7 @@ public HogQlQuery build(HogQLParser.SelectContext context) List projections = selectColumns(select.selectColumnExprListBeforeFrom()).stream() .map(this::buildProjection) .toList(); - Optional from = Optional.ofNullable(select.fromClause()) + Optional from = Optional.ofNullable(select.fromClause()) .map(HogQLParser.FromClauseContext::joinExpr) .map(this::buildTableReference); Optional where = Optional.ofNullable(select.whereClause()) @@ -748,7 +751,7 @@ private Expression buildLiteral(HogQLParser.LiteralContext context) private Expression buildColumnReference(HogQLParser.ColumnIdentifierContext context) { if (context.placeholder() != null) { - throw unsupported(context, "placeholder"); + return buildPlaceholder(context.placeholder()); } List parts = new ArrayList<>(); if (context.tableIdentifier() != null) { @@ -766,12 +769,27 @@ private Expression buildColumnReference(HogQLParser.ColumnIdentifierContext cont return new ColumnReference(parts, sourceSpan(context)); } - private TableReference buildTableReference(HogQLParser.JoinExprContext context) + private Placeholder buildPlaceholder(HogQLParser.PlaceholderContext context) { - if (!(context instanceof HogQLParser.JoinExprTableContext table) || table.FINAL() != null || table.sampleClause() != null || !(table.tableExpr() instanceof HogQLParser.TableExprIdentifierContext identifier)) { + Expression expression = buildExpression(context.columnExpr()); + if (!(expression instanceof ColumnReference reference) || reference.parts().size() != 1) { + throw unsupported(context, "non-name placeholder"); + } + return new Placeholder(reference.parts().getFirst().value(), sourceSpan(context)); + } + + private Relation buildTableReference(HogQLParser.JoinExprContext context) + { + if (!(context instanceof HogQLParser.JoinExprTableContext table) || table.FINAL() != null || table.sampleClause() != null) { throw unsupported(context, "table expression or join"); } - return new TableReference(buildIdentifiers(identifier.tableIdentifier()), sourceSpan(context)); + if (table.tableExpr() instanceof HogQLParser.TableExprIdentifierContext identifier) { + return new TableReference(buildIdentifiers(identifier.tableIdentifier()), sourceSpan(context)); + } + if (table.tableExpr() instanceof HogQLParser.TableExprPlaceholderContext placeholder) { + return new TablePlaceholder(buildPlaceholder(placeholder.placeholder())); + } + throw unsupported(context, "table expression or join"); } private List buildIdentifiers(HogQLParser.TableIdentifierContext context) diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index 1d03947d2827..89f71a1ec7c3 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -18,7 +18,7 @@ import static java.util.Objects.requireNonNull; -public record HogQlQuery(List projections, Optional from, Optional where, SourceSpan span) +public record HogQlQuery(List projections, Optional from, Optional where, SourceSpan span) { public HogQlQuery { @@ -70,6 +70,7 @@ public sealed interface Expression InExpression, IsNullExpression, Literal, + Placeholder, TupleExpression, UnaryExpression { @@ -245,6 +246,19 @@ public record ColumnReference(List parts, SourceSpan span) } } + public record Placeholder(String name, SourceSpan span) + implements Expression + { + public Placeholder + { + name = requireNonNull(name, "name is null"); + if (name.isEmpty()) { + throw new IllegalArgumentException("name is empty"); + } + span = requireNonNull(span, "span is null"); + } + } + public record Literal(LiteralKind kind, String value, SourceSpan span) implements Expression { @@ -264,7 +278,14 @@ public enum LiteralKind STRING, } + public sealed interface Relation + permits TablePlaceholder, TableReference + { + SourceSpan span(); + } + public record TableReference(List parts, SourceSpan span) + implements Relation { public TableReference { @@ -276,6 +297,21 @@ public record TableReference(List parts, SourceSpan span) } } + public record TablePlaceholder(Placeholder placeholder) + implements Relation + { + public TablePlaceholder + { + placeholder = requireNonNull(placeholder, "placeholder is null"); + } + + @Override + public SourceSpan span() + { + return placeholder.span(); + } + } + public record Identifier(String value, boolean delimited, SourceSpan span) { public Identifier diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index d15bb6444842..a22adc95fb5a 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -14,8 +14,10 @@ package io.trino.hogql.parser; import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlSyntaxTree; import io.trino.hogql.parser.tree.HogQlSyntaxTree.Element; import io.trino.hogql.parser.tree.HogQlSyntaxTree.Node; @@ -40,7 +42,7 @@ public class TestHogQlParser "DROP TABLE events", "SELECT 1; SELECT 2", "SELECT * FROM", - "SELECT {value}", + "SELECT {1 + 2}", "SELECT 1 GROUP BY 1", }) public void testRejectsSyntaxWithoutAnAstMapping(String hogql) @@ -60,6 +62,23 @@ public void testUsesCanonicalGrammarForExpressionsAndWhereClause() assertThat(query.where()).isPresent(); } + @Test + public void testPreservesNamedPlaceholderNamesAndSourceSpans() + { + HogQlQuery query = parser.parseStatement("SELECT {later},\n {first} + {later}"); + Placeholder later = (Placeholder) ((ExpressionProjection) query.projections().getFirst()).expression(); + BinaryExpression addition = (BinaryExpression) ((ExpressionProjection) query.projections().get(1)).expression(); + Placeholder first = (Placeholder) addition.left(); + Placeholder repeated = (Placeholder) addition.right(); + + assertThat(later.name()).isEqualTo("later"); + assertThat(later.span()).isEqualTo(new HogQlQuery.SourceSpan(7, 14, 1, 8, 1, 15)); + assertThat(first.name()).isEqualTo("first"); + assertThat(first.span()).isEqualTo(new HogQlQuery.SourceSpan(17, 24, 2, 2, 2, 9)); + assertThat(repeated.name()).isEqualTo("later"); + assertThat(repeated.span()).isEqualTo(new HogQlQuery.SourceSpan(27, 34, 2, 12, 2, 19)); + } + @Test public void testSourceSpansUseUnicodeCodePointOffsetsAndEndPositions() { From e8977a48979ffb4c8709e17363572e3832100d6d Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 10:09:11 -0400 Subject: [PATCH 023/136] feat(hogql): cache semantic catalog snapshots --- ...syncHogQlSemanticCatalogSnapshotCache.java | 331 ++++++++++++++++++ ...syncHogQlSemanticCatalogSnapshotCache.java | 318 +++++++++++++++++ 2 files changed, 649 insertions(+) create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlSemanticCatalogSnapshotCache.java create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlSemanticCatalogSnapshotCache.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlSemanticCatalogSnapshotCache.java new file mode 100644 index 000000000000..2a04d5b2691e --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlSemanticCatalogSnapshotCache.java @@ -0,0 +1,331 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException.Failure; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.function.LongSupplier; + +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public final class BoundedAsyncHogQlSemanticCatalogSnapshotCache + implements HogQlSemanticCatalogSnapshotCache +{ + private final Object lock = new Object(); + private final int maximumEntries; + private final long refreshAfterNanos; + private final long expireAfterNanos; + private final long failureBackoffNanos; + private final LongSupplier ticker; + private final Executor executor; + private final SnapshotLoader loader; + private final LinkedHashMap entries = new LinkedHashMap<>(16, 0.75f, true); + + public BoundedAsyncHogQlSemanticCatalogSnapshotCache( + int maximumEntries, + Duration refreshAfter, + Duration expireAfter, + Duration failureBackoff, + LongSupplier ticker, + Executor executor, + SnapshotLoader loader) + { + if (maximumEntries <= 0) { + throw new IllegalArgumentException("maximumEntries must be positive"); + } + this.maximumEntries = maximumEntries; + this.refreshAfterNanos = nonNegativeNanos(refreshAfter, "refreshAfter"); + this.expireAfterNanos = positiveNanos(expireAfter, "expireAfter"); + this.failureBackoffNanos = nonNegativeNanos(failureBackoff, "failureBackoff"); + if (refreshAfterNanos > expireAfterNanos) { + throw new IllegalArgumentException("refreshAfter must not exceed expireAfter"); + } + this.ticker = requireNonNull(ticker, "ticker is null"); + this.executor = requireNonNull(executor, "executor is null"); + this.loader = requireNonNull(loader, "loader is null"); + } + + @Override + public Optional currentSnapshot(PhysicalIdentifier catalog) + { + requireNonNull(catalog, "catalog is null"); + RefreshTask refreshTask; + Optional result; + List> evictedRefreshes = new ArrayList<>(1); + long now = ticker.getAsLong(); + synchronized (lock) { + Entry entry = entries.get(catalog); + if (entry == null) { + entry = new Entry(); + entries.put(catalog, entry); + evictEntries(evictedRefreshes); + } + + HogQlSemanticCatalogSnapshot snapshot = entry.snapshot; + long age = snapshot == null ? Long.MAX_VALUE : elapsedNanos(entry.loadedAtNanos, now); + result = snapshot != null && age < expireAfterNanos ? Optional.of(snapshot) : Optional.empty(); + refreshTask = (snapshot == null || age >= refreshAfterNanos) ? prepareRefresh(catalog, entry, now, false) : null; + } + cancelRefreshes(evictedRefreshes, "HogQL semantic catalog cache entry was evicted"); + dispatch(refreshTask); + return result; + } + + public CompletionStage prewarm(PhysicalIdentifier catalog) + { + requireNonNull(catalog, "catalog is null"); + RefreshTask refreshTask; + CompletableFuture future; + List> evictedRefreshes = new ArrayList<>(1); + synchronized (lock) { + Entry entry = entries.get(catalog); + if (entry == null) { + entry = new Entry(); + entries.put(catalog, entry); + evictEntries(evictedRefreshes); + } + refreshTask = prepareRefresh(catalog, entry, ticker.getAsLong(), true); + future = refreshTask == null ? entry.refresh : refreshTask.result(); + } + cancelRefreshes(evictedRefreshes, "HogQL semantic catalog cache entry was evicted"); + dispatch(refreshTask); + return future.minimalCompletionStage(); + } + + public void invalidate(PhysicalIdentifier catalog) + { + requireNonNull(catalog, "catalog is null"); + CompletableFuture refresh = null; + synchronized (lock) { + Entry removed = entries.remove(catalog); + if (removed != null) { + refresh = removed.refresh; + removed.refresh = null; + } + } + cancelRefresh(refresh, "HogQL semantic catalog cache entry was invalidated"); + } + + private RefreshTask prepareRefresh(PhysicalIdentifier catalog, Entry entry, long now, boolean force) + { + if (entry.refresh != null) { + return null; + } + if (!force && entry.refreshBackoffActive && elapsedNanos(entry.lastRefreshFailureAtNanos, now) < failureBackoffNanos) { + return null; + } + CompletableFuture result = new CompletableFuture<>(); + entry.refresh = result; + return new RefreshTask(catalog, entry, result); + } + + private void dispatch(RefreshTask refreshTask) + { + if (refreshTask == null) { + return; + } + try { + executor.execute(() -> load(refreshTask)); + } + catch (RuntimeException e) { + completeFailure(refreshTask, e); + } + } + + private void load(RefreshTask refreshTask) + { + CompletionStage loaded; + try { + loaded = requireNonNull(loader.load(refreshTask.catalog()), "snapshot loader returned null"); + } + catch (RuntimeException failure) { + completeFailure(refreshTask, failure); + return; + } + catch (Error failure) { + completeFailure(refreshTask, failure); + throw failure; + } + try { + loaded.whenComplete((snapshot, failure) -> { + if (failure != null) { + completeFailure(refreshTask, failure); + return; + } + if (snapshot == null) { + completeFailure(refreshTask, new NullPointerException("snapshot loader completed with null")); + return; + } + completeSuccess(refreshTask, snapshot); + }); + } + catch (RuntimeException failure) { + completeFailure(refreshTask, failure); + } + catch (Error failure) { + completeFailure(refreshTask, failure); + throw failure; + } + } + + private void completeSuccess(RefreshTask refreshTask, HogQlSemanticCatalogSnapshot snapshot) + { + Throwable rejection = validateLoadedSnapshot(refreshTask, snapshot); + if (rejection != null) { + completeFailure(refreshTask, rejection); + return; + } + + boolean published = false; + synchronized (lock) { + Entry entry = entries.get(refreshTask.catalog()); + if (entry == refreshTask.entry() && entry.refresh == refreshTask.result()) { + if (entry.snapshot != null && snapshot.generation() < entry.snapshot.generation()) { + entry.refresh = null; + entry.lastRefreshFailureAtNanos = ticker.getAsLong(); + entry.refreshBackoffActive = true; + rejection = new HogQlSemanticCatalogException( + Failure.GENERATION_MISMATCH, + format("HogQL semantic catalog generation regressed from %s to %s", entry.snapshot.generation(), snapshot.generation())); + } + else { + entry.snapshot = snapshot; + entry.loadedAtNanos = ticker.getAsLong(); + entry.refreshBackoffActive = false; + entry.refresh = null; + published = true; + } + } + } + if (published) { + refreshTask.result().complete(snapshot); + } + else if (rejection != null) { + refreshTask.result().completeExceptionally(rejection); + } + else { + refreshTask.result().completeExceptionally(new CancellationException("HogQL semantic catalog refresh no longer owns its cache entry")); + } + } + + private Throwable validateLoadedSnapshot(RefreshTask refreshTask, HogQlSemanticCatalogSnapshot snapshot) + { + if (!snapshot.catalog().equals(refreshTask.catalog())) { + return new HogQlSemanticCatalogException( + Failure.CATALOG_MISMATCH, + "HogQL semantic catalog snapshot does not match the refreshed catalog"); + } + return null; + } + + private void completeFailure(RefreshTask refreshTask, Throwable failure) + { + requireNonNull(failure, "failure is null"); + boolean owned; + synchronized (lock) { + Entry entry = entries.get(refreshTask.catalog()); + owned = entry == refreshTask.entry() && entry.refresh == refreshTask.result(); + if (owned) { + entry.refresh = null; + entry.lastRefreshFailureAtNanos = ticker.getAsLong(); + entry.refreshBackoffActive = true; + } + } + if (owned) { + refreshTask.result().completeExceptionally(failure); + } + else { + refreshTask.result().completeExceptionally(new CancellationException("HogQL semantic catalog refresh no longer owns its cache entry")); + } + } + + private void evictEntries(List> evictedRefreshes) + { + while (entries.size() > maximumEntries) { + Map.Entry eldest = entries.entrySet().iterator().next(); + entries.remove(eldest.getKey()); + if (eldest.getValue().refresh != null) { + evictedRefreshes.add(eldest.getValue().refresh); + eldest.getValue().refresh = null; + } + } + } + + private static void cancelRefreshes(List> refreshes, String message) + { + refreshes.forEach(refresh -> cancelRefresh(refresh, message)); + } + + private static void cancelRefresh(CompletableFuture refresh, String message) + { + if (refresh != null) { + refresh.completeExceptionally(new CancellationException(message)); + } + } + + private static long elapsedNanos(long start, long end) + { + return Math.max(0, end - start); + } + + private static long nonNegativeNanos(Duration duration, String name) + { + requireNonNull(duration, name + " is null"); + if (duration.isNegative()) { + throw new IllegalArgumentException(name + " must not be negative"); + } + return duration.toNanos(); + } + + private static long positiveNanos(Duration duration, String name) + { + long nanos = nonNegativeNanos(duration, name); + if (nanos == 0) { + throw new IllegalArgumentException(name + " must be positive"); + } + return nanos; + } + + @FunctionalInterface + public interface SnapshotLoader + { + CompletionStage load(PhysicalIdentifier catalog); + } + + private static final class Entry + { + private HogQlSemanticCatalogSnapshot snapshot; + private long loadedAtNanos; + private long lastRefreshFailureAtNanos; + private boolean refreshBackoffActive; + private CompletableFuture refresh; + } + + private record RefreshTask( + PhysicalIdentifier catalog, + Entry entry, + CompletableFuture result) {} +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java new file mode 100644 index 000000000000..4cc37f149c33 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java @@ -0,0 +1,318 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.BoundedAsyncHogQlSemanticCatalogSnapshotCache.SnapshotLoader; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException.Failure; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.parser.HogQlLanguageVersion; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestBoundedAsyncHogQlSemanticCatalogSnapshotCache +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageVersion.valueOf("1.0.0"); + private static final PhysicalIdentifier CATALOG = catalog("ducklake"); + private static final Duration REFRESH_AFTER = Duration.ofNanos(10); + private static final Duration EXPIRE_AFTER = Duration.ofNanos(20); + private static final Duration FAILURE_BACKOFF = Duration.ofNanos(100); + + @Test + public void testOutOfOrderRefreshCannotPublishAfterInvalidation() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + CompletableFuture oldLoad = loader.expect(CATALOG); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + + CompletionStage invalidatedRefresh = cache.prewarm(CATALOG); + cache.invalidate(CATALOG); + + CompletableFuture newLoad = loader.expect(CATALOG); + CompletionStage currentRefresh = cache.prewarm(CATALOG); + newLoad.complete(snapshot(CATALOG, 2)); + oldLoad.complete(snapshot(CATALOG, 1)); + + assertThat(currentRefresh.toCompletableFuture()).isCompletedWithValueMatching(snapshot -> snapshot.generation() == 2); + assertThat(invalidatedRefresh.toCompletableFuture()).isCompletedExceptionally(); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(2L); + } + + @Test + public void testRejectsInvalidRefreshWithoutReplacingLastKnownGood() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, snapshot(CATALOG, 2)); + + CompletableFuture olderLoad = loader.expect(CATALOG); + CompletionStage olderRefresh = cache.prewarm(CATALOG); + olderLoad.complete(snapshot(CATALOG, 1)); + + assertRefreshFailure(olderRefresh, Failure.GENERATION_MISMATCH); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(2L); + + PhysicalIdentifier otherCatalog = catalog("other"); + CompletableFuture mismatchedLoad = loader.expect(CATALOG); + CompletionStage mismatchedRefresh = cache.prewarm(CATALOG); + mismatchedLoad.complete(snapshot(otherCatalog, 3)); + + assertRefreshFailure(mismatchedRefresh, Failure.CATALOG_MISMATCH); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(2L); + } + + @Test + public void testServesStaleSnapshotWhileRefreshingAndFailsClosedAfterExpiry() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, snapshot(CATALOG, 1)); + + ticker.set(REFRESH_AFTER.toNanos() - 1); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + assertThat(loader.loadCount(CATALOG)).isEqualTo(1); + + CompletableFuture refresh = loader.expect(CATALOG); + ticker.set(REFRESH_AFTER.toNanos()); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + assertThat(loader.loadCount(CATALOG)).isEqualTo(2); + + ticker.set(EXPIRE_AFTER.toNanos()); + assertThat(cache.currentSnapshot(CATALOG)).isEmpty(); + assertThat(loader.loadCount(CATALOG)).isEqualTo(2); + + refresh.complete(snapshot(CATALOG, 2)); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(2L); + } + + @Test + public void testRefreshAgeSurvivesTickerWraparound() + { + AtomicLong ticker = new AtomicLong(Long.MAX_VALUE - 5); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, snapshot(CATALOG, 1)); + + loader.expect(CATALOG); + ticker.set(Long.MIN_VALUE + 4); + assertThat(cache.currentSnapshot(CATALOG)).isPresent(); + assertThat(loader.loadCount(CATALOG)).isEqualTo(2); + } + + @Test + public void testRefreshFailurePreservesSnapshotOnlyUntilExpiryAndBacksOff() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, snapshot(CATALOG, 1)); + + CompletableFuture failedLoad = loader.expect(CATALOG); + ticker.set(REFRESH_AFTER.toNanos()); + assertThat(cache.currentSnapshot(CATALOG)).isPresent(); + failedLoad.completeExceptionally(new IllegalStateException("metadata unavailable")); + + ticker.set(EXPIRE_AFTER.toNanos() - 1); + assertThat(cache.currentSnapshot(CATALOG)).isPresent(); + assertThat(loader.loadCount(CATALOG)).isEqualTo(2); + + ticker.set(EXPIRE_AFTER.toNanos()); + assertThat(cache.currentSnapshot(CATALOG)).isEmpty(); + assertThat(loader.loadCount(CATALOG)).isEqualTo(2); + + CompletableFuture recoveredLoad = loader.expect(CATALOG); + ticker.set(REFRESH_AFTER.plus(FAILURE_BACKOFF).toNanos()); + assertThat(cache.currentSnapshot(CATALOG)).isEmpty(); + assertThat(loader.loadCount(CATALOG)).isEqualTo(3); + recoveredLoad.complete(snapshot(CATALOG, 2)); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(2L); + } + + @Test + public void testCatalogEntriesAreIsolatedAndLeastRecentlyUsedEntryIsEvicted() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + PhysicalIdentifier alpha = catalog("alpha"); + PhysicalIdentifier beta = catalog("beta"); + PhysicalIdentifier gamma = catalog("gamma"); + + completePrewarm(cache, loader, snapshot(alpha, 1)); + completePrewarm(cache, loader, snapshot(beta, 2)); + assertThat(cache.currentSnapshot(alpha)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + completePrewarm(cache, loader, snapshot(gamma, 3)); + + assertThat(cache.currentSnapshot(alpha)).isPresent(); + assertThat(cache.currentSnapshot(gamma)).isPresent(); + loader.expect(beta); + assertThat(cache.currentSnapshot(beta)).isEmpty(); + assertThat(loader.loadCount(beta)).isEqualTo(2); + } + + @Test + public void testConcurrentRequestsShareOneRefresh() + throws Exception + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + CompletableFuture load = loader.expect(CATALOG); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + int workers = 4; + CountDownLatch ready = new CountDownLatch(workers); + CountDownLatch start = new CountDownLatch(1); + + try (ExecutorService callers = Executors.newFixedThreadPool(workers)) { + List>> requests = java.util.stream.IntStream.range(0, workers) + .mapToObj(_ -> CompletableFuture.supplyAsync(() -> { + ready.countDown(); + await(start); + return cache.prewarm(CATALOG); + }, callers)) + .toList(); + assertThat(ready.await(10, SECONDS)).isTrue(); + start.countDown(); + List> refreshes = requests.stream() + .map(CompletableFuture::join) + .toList(); + + assertThat(loader.loadCount(CATALOG)).isEqualTo(1); + assertThat(refreshes).allMatch(refresh -> !refresh.toCompletableFuture().isDone()); + + load.complete(snapshot(CATALOG, 1)); + assertThat(refreshes).allSatisfy(refresh -> assertThat(refresh.toCompletableFuture()) + .isCompletedWithValueMatching(snapshot -> snapshot.generation() == 1)); + } + } + + private static BoundedAsyncHogQlSemanticCatalogSnapshotCache cache(int maximumEntries, AtomicLong ticker, SnapshotLoader loader) + { + return new BoundedAsyncHogQlSemanticCatalogSnapshotCache( + maximumEntries, + REFRESH_AFTER, + EXPIRE_AFTER, + FAILURE_BACKOFF, + ticker::get, + Runnable::run, + loader); + } + + private static void completePrewarm( + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache, + ControlledLoader loader, + HogQlSemanticCatalogSnapshot snapshot) + { + CompletableFuture load = loader.expect(snapshot.catalog()); + CompletionStage refresh = cache.prewarm(snapshot.catalog()); + load.complete(snapshot); + assertThat(refresh.toCompletableFuture()).isCompletedWithValue(snapshot); + } + + private static void assertRefreshFailure(CompletionStage refresh, Failure failure) + { + assertThatThrownBy(refresh.toCompletableFuture()::join) + .cause() + .isInstanceOfSatisfying(HogQlSemanticCatalogException.class, exception -> assertThat(exception.failure()).isEqualTo(failure)); + } + + private static void await(CountDownLatch latch) + { + try { + assertThat(latch.await(10, SECONDS)).isTrue(); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + private static PhysicalIdentifier catalog(String name) + { + return new PhysicalIdentifier(name, false); + } + + private static HogQlSemanticCatalogSnapshot snapshot(PhysicalIdentifier catalog, long generation) + { + return new HogQlSemanticCatalogSnapshot( + 1, + LANGUAGE_VERSION, + catalog, + generation, + List.of(new LogicalTableDefinition( + "events", + new PhysicalQualifiedName(catalog, catalog("default"), catalog("events")), + List.of(new LogicalFieldDefinition( + "id", + catalog("id"), + "varchar", + LogicalType.STRING, + false, + true)), + List.of(), + List.of()))); + } + + private static final class ControlledLoader + implements SnapshotLoader + { + private final Map>> loads = new HashMap<>(); + private final Map loadCounts = new HashMap<>(); + + public CompletableFuture expect(PhysicalIdentifier catalog) + { + CompletableFuture load = new CompletableFuture<>(); + loads.computeIfAbsent(catalog, _ -> new ArrayDeque<>()).add(load); + return load; + } + + @Override + public CompletionStage load(PhysicalIdentifier catalog) + { + loadCounts.computeIfAbsent(catalog, _ -> new AtomicInteger()).incrementAndGet(); + return Optional.ofNullable(loads.get(catalog)) + .map(ArrayDeque::poll) + .orElseThrow(() -> new IllegalStateException("unexpected load for catalog " + catalog.value())); + } + + public int loadCount(PhysicalIdentifier catalog) + { + return Optional.ofNullable(loadCounts.get(catalog)) + .map(AtomicInteger::get) + .orElse(0); + } + } +} From e2a759035514360e1aad09c96a9622eafe417fd2 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 10:11:08 -0400 Subject: [PATCH 024/136] feat(hogql): accept versioned request envelopes --- .../hogql/compiler/HogQlCompileEnvelope.java | 14 +- .../compiler/TestHogQlCompileEnvelope.java | 10 +- .../io/trino/dispatcher/HogQlRequest.java | 220 ++++++++++++++++++ .../dispatcher/HogQlStatementResource.java | 15 +- .../io/trino/execution/QueryPreparer.java | 3 +- .../io/trino/execution/QuerySubmission.java | 20 +- .../TestHogQlStatementResource.java | 175 +++++++++++--- .../io/trino/execution/TestQueryPreparer.java | 26 ++- 8 files changed, 431 insertions(+), 52 deletions(-) create mode 100644 core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompileEnvelope.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompileEnvelope.java index 0a73142dbe61..c2412f112d78 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompileEnvelope.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompileEnvelope.java @@ -17,7 +17,7 @@ import io.trino.hogql.parser.HogQlLanguageVersion; import java.util.Map; -import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import static java.util.Objects.requireNonNull; @@ -30,7 +30,7 @@ public record HogQlCompileEnvelope( Map variables, Map filters, Map modifiers, - Optional catalogGeneration) + OptionalLong catalogGeneration) { public static final int PROTOCOL_VERSION = 1; @@ -54,9 +54,11 @@ public record HogQlCompileEnvelope( filters = immutableValues("filters", filters); modifiers = immutableValues("modifiers", modifiers); catalogGeneration = requireNonNull(catalogGeneration, "catalogGeneration is null"); - if (catalogGeneration.stream().anyMatch(String::isBlank)) { - throw new IllegalArgumentException("catalog generation is empty"); - } + catalogGeneration.ifPresent(generation -> { + if (generation <= 0) { + throw new IllegalArgumentException("catalog generation must be positive"); + } + }); } public static HogQlCompileEnvelope fromSemanticFields( @@ -64,7 +66,7 @@ public static HogQlCompileEnvelope fromSemanticFields( int protocolVersion, HogQlLanguageVersion languageVersion, Map> semanticFields, - Optional catalogGeneration) + OptionalLong catalogGeneration) { requireNonNull(semanticFields, "semanticFields is null"); if (!SEMANTIC_FIELDS.containsAll(semanticFields.keySet())) { diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompileEnvelope.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompileEnvelope.java index 7699d7028d33..9aafff73725d 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompileEnvelope.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompileEnvelope.java @@ -29,7 +29,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; +import java.util.OptionalLong; import java.util.function.Supplier; import java.util.stream.Stream; @@ -81,7 +81,7 @@ public void testRejectsUnknownSemanticFields(String semanticField) HogQlCompileEnvelope.PROTOCOL_VERSION, LANGUAGE_VERSION, fields, - Optional.empty())) + OptionalLong.empty())) .isInstanceOf(IllegalArgumentException.class) .hasMessage("unknown HogQL semantic field"); } @@ -108,10 +108,10 @@ public void testEnvelopeRenderingRedactsEveryValueCollection(String semanticFiel HogQlCompileEnvelope.PROTOCOL_VERSION, LANGUAGE_VERSION, fields, - Optional.of("sensitive-generation")); + OptionalLong.of(42)); assertThat(envelope.toString()) - .doesNotContain(sensitiveValue, "sensitive-query", "sensitive-generation") + .doesNotContain(sensitiveValue, "sensitive-query") .contains("query="); } @@ -157,6 +157,6 @@ private static HogQlCompileEnvelope envelope( variables, filters, modifiers, - Optional.empty()); + OptionalLong.empty()); } } diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java new file mode 100644 index 000000000000..5e943122e8c5 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java @@ -0,0 +1,220 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.dispatcher; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airlift.json.JsonCodec; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlTypedValue; +import io.trino.hogql.compiler.HogQlTypedValue.ArrayValue; +import io.trino.hogql.compiler.HogQlTypedValue.BooleanValue; +import io.trino.hogql.compiler.HogQlTypedValue.NullValue; +import io.trino.hogql.compiler.HogQlTypedValue.NumberValue; +import io.trino.hogql.compiler.HogQlTypedValue.ObjectValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.hogql.compiler.HogQlTypedValue.Value; +import io.trino.hogql.parser.HogQlLanguageVersion; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.Set; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; + +record HogQlRequest( + String query, + int protocolVersion, + HogQlLanguageVersion languageVersion, + Map parameters, + Map variables, + Map filters, + Map modifiers, + OptionalLong catalogGeneration) +{ + private static final JsonCodec JSON_CODEC = jsonCodec(JsonNode.class); + private static final Set FIELDS = Set.of( + "query", + "protocolVersion", + "languageVersion", + "parameters", + "variables", + "filters", + "modifiers", + "catalogGeneration"); + private static final Set TYPED_VALUE_FIELDS = Set.of("type", "value"); + private static final int MAX_VALUE_DEPTH = 64; + + HogQlRequest + { + query = requireNonNull(query, "query is null"); + languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + parameters = Map.copyOf(requireNonNull(parameters, "parameters is null")); + variables = Map.copyOf(requireNonNull(variables, "variables is null")); + filters = Map.copyOf(requireNonNull(filters, "filters is null")); + modifiers = Map.copyOf(requireNonNull(modifiers, "modifiers is null")); + catalogGeneration = requireNonNull(catalogGeneration, "catalogGeneration is null"); + } + + static HogQlRequest fromJson(String requestBody) + { + JsonNode root = JSON_CODEC.fromJson(requireNonNull(requestBody, "requestBody is null")); + if (root == null || !root.isObject()) { + throw new IllegalArgumentException("HogQL request must be an object"); + } + rejectUnknownFields(root, FIELDS, "unknown HogQL request field"); + + return new HogQlRequest( + requiredText(root, "query"), + requiredInt(root, "protocolVersion"), + languageVersion(root), + typedValues(root, "parameters"), + typedValues(root, "variables"), + typedValues(root, "filters"), + typedValues(root, "modifiers"), + catalogGeneration(root)); + } + + HogQlCompileEnvelope toCompileEnvelope() + { + return new HogQlCompileEnvelope( + query, + protocolVersion, + languageVersion, + parameters, + variables, + filters, + modifiers, + catalogGeneration); + } + + private static HogQlLanguageVersion languageVersion(JsonNode root) + { + try { + return HogQlLanguageVersion.valueOf(requiredText(root, "languageVersion")); + } + catch (RuntimeException _) { + throw new IllegalArgumentException("invalid HogQL language version"); + } + } + + private static OptionalLong catalogGeneration(JsonNode root) + { + JsonNode value = root.get("catalogGeneration"); + if (value == null) { + return OptionalLong.empty(); + } + if (!value.isIntegralNumber() || !value.canConvertToLong() || value.longValue() <= 0) { + throw new IllegalArgumentException("invalid HogQL catalog generation"); + } + return OptionalLong.of(value.longValue()); + } + + private static Map typedValues(JsonNode root, String field) + { + JsonNode values = root.get(field); + if (values == null) { + return Map.of(); + } + if (!values.isObject()) { + throw new IllegalArgumentException("invalid HogQL semantic field"); + } + + Map result = new LinkedHashMap<>(); + Iterator names = values.fieldNames(); + while (names.hasNext()) { + String name = names.next(); + result.put(name, typedValue(values.get(name))); + } + return Map.copyOf(result); + } + + private static HogQlTypedValue typedValue(JsonNode node) + { + if (node == null || !node.isObject()) { + throw new IllegalArgumentException("invalid HogQL typed value"); + } + rejectUnknownFields(node, TYPED_VALUE_FIELDS, "unknown HogQL typed value field"); + if (!node.has("value")) { + throw new IllegalArgumentException("missing HogQL typed value"); + } + return new HogQlTypedValue(requiredText(node, "type"), value(node.get("value"), 0)); + } + + private static Value value(JsonNode node, int depth) + { + if (depth > MAX_VALUE_DEPTH) { + throw new IllegalArgumentException("HogQL typed value is too deeply nested"); + } + if (node == null || node.isNull()) { + return NullValue.NULL; + } + if (node.isBoolean()) { + return new BooleanValue(node.booleanValue()); + } + if (node.isNumber()) { + return new NumberValue(node.asText()); + } + if (node.isTextual()) { + return new StringValue(node.textValue()); + } + if (node.isArray()) { + List values = new ArrayList<>(node.size()); + node.elements().forEachRemaining(element -> values.add(value(element, depth + 1))); + return new ArrayValue(values); + } + if (node.isObject()) { + Map values = new LinkedHashMap<>(); + Iterator names = node.fieldNames(); + while (names.hasNext()) { + String name = names.next(); + values.put(name, value(node.get(name), depth + 1)); + } + return new ObjectValue(values); + } + throw new IllegalArgumentException("invalid HogQL typed value payload"); + } + + private static String requiredText(JsonNode root, String field) + { + JsonNode value = root.get(field); + if (value == null || !value.isTextual()) { + throw new IllegalArgumentException("missing HogQL text field"); + } + return value.textValue(); + } + + private static int requiredInt(JsonNode root, String field) + { + JsonNode value = root.get(field); + if (value == null || !value.isIntegralNumber() || !value.canConvertToInt()) { + throw new IllegalArgumentException("missing HogQL integer field"); + } + return value.intValue(); + } + + private static void rejectUnknownFields(JsonNode object, Set fields, String message) + { + Iterator names = object.fieldNames(); + while (names.hasNext()) { + if (!fields.contains(names.next())) { + throw new IllegalArgumentException(message); + } + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java index e17e0939c0ec..8581c887e54b 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java @@ -14,10 +14,13 @@ package io.trino.dispatcher; import com.google.inject.Inject; +import io.trino.hogql.compiler.HogQlCompileEnvelope; import io.trino.server.ExternalUriInfo; import io.trino.server.security.ResourceSecurity; import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.BeanParam; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.Consumes; import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; @@ -43,13 +46,21 @@ public HogQlStatementResource(QueuedStatementResource queuedStatementResource) @ResourceSecurity(AUTHENTICATED_USER) @POST + @Consumes(APPLICATION_JSON) @Produces(APPLICATION_JSON) public Response postStatement( - String statement, + String requestBody, @Context HttpServletRequest servletRequest, @Context HttpHeaders httpHeaders, @BeanParam ExternalUriInfo externalUriInfo) { - return queuedStatementResource.postStatement(hogQl(statement), servletRequest, httpHeaders, externalUriInfo); + HogQlCompileEnvelope envelope; + try { + envelope = HogQlRequest.fromJson(requestBody).toCompileEnvelope(); + } + catch (RuntimeException _) { + throw new BadRequestException("Invalid HogQL request"); + } + return queuedStatementResource.postStatement(hogQl(envelope), servletRequest, httpHeaders, externalUriInfo); } } diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java index 3e5c8cff0136..56340dd91866 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java @@ -75,7 +75,8 @@ public PreparedQuery prepareQuery(Session session, QuerySubmission submission) case TRINO -> sqlParser.createStatement(submission.originalText()); case HOGQL -> hogQlCompiler .orElseThrow(() -> new TrinoException(NOT_SUPPORTED, "HogQL query submission is disabled")) - .compile(submission.originalText()); + .compile(submission.hogQlEnvelope().orElseThrow()) + .statement(); }; return prepareQuery(session, wrappedStatement); } diff --git a/core/trino-main/src/main/java/io/trino/execution/QuerySubmission.java b/core/trino-main/src/main/java/io/trino/execution/QuerySubmission.java index 97d41c619bbb..42c7776101db 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QuerySubmission.java +++ b/core/trino-main/src/main/java/io/trino/execution/QuerySubmission.java @@ -13,25 +13,37 @@ */ package io.trino.execution; +import io.trino.hogql.compiler.HogQlCompileEnvelope; + +import java.util.Optional; + import static io.trino.execution.QueryLanguage.HOGQL; import static io.trino.execution.QueryLanguage.TRINO; import static java.util.Objects.requireNonNull; -public record QuerySubmission(QueryLanguage language, String originalText) +public record QuerySubmission(QueryLanguage language, String originalText, Optional hogQlEnvelope) { public QuerySubmission { requireNonNull(language, "language is null"); requireNonNull(originalText, "originalText is null"); + hogQlEnvelope = requireNonNull(hogQlEnvelope, "hogQlEnvelope is null"); + if (language == TRINO && hogQlEnvelope.isPresent()) { + throw new IllegalArgumentException("Trino submission has a HogQL envelope"); + } + if (language == HOGQL && (hogQlEnvelope.isEmpty() || !hogQlEnvelope.orElseThrow().query().equals(originalText))) { + throw new IllegalArgumentException("HogQL submission envelope does not match its query"); + } } public static QuerySubmission trino(String originalText) { - return new QuerySubmission(TRINO, originalText); + return new QuerySubmission(TRINO, originalText, Optional.empty()); } - public static QuerySubmission hogQl(String originalText) + public static QuerySubmission hogQl(HogQlCompileEnvelope envelope) { - return new QuerySubmission(HOGQL, originalText); + requireNonNull(envelope, "envelope is null"); + return new QuerySubmission(HOGQL, envelope.query(), Optional.of(envelope)); } } diff --git a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java index c2d6872b68e0..44fff6c28343 100644 --- a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java +++ b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java @@ -16,7 +16,9 @@ import com.google.common.collect.ImmutableList; import io.airlift.http.client.HeaderName; import io.airlift.http.client.HttpClient; +import io.airlift.http.client.Request; import io.airlift.http.client.StatusResponseHandler.StatusResponse; +import io.airlift.http.client.StringResponseHandler.StringResponse; import io.airlift.http.client.jetty.JettyHttpClient; import io.airlift.json.JsonCodec; import io.airlift.json.JsonCodecFactory; @@ -24,51 +26,86 @@ import io.trino.client.QueryDataJacksonModule; import io.trino.client.QueryResults; import io.trino.client.ResultRowsDecoder; +import io.trino.hogql.parser.HogQlLanguageContract; import io.trino.plugin.tpch.TpchPlugin; import io.trino.server.testing.TestingTrinoServer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; import static io.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; import static io.airlift.http.client.Request.Builder.prepareGet; import static io.airlift.http.client.Request.Builder.preparePost; import static io.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; import static io.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; +import static io.airlift.http.client.StringResponseHandler.createStringResponseHandler; import static io.airlift.testing.Closeables.closeAll; import static io.trino.client.ProtocolHeaders.TRINO_HEADERS; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; class TestHogQlStatementResource { private static final HeaderName REQUEST_USER_HEADER = HeaderName.of(TRINO_HEADERS.requestUser()); + private static final HeaderName CONTENT_TYPE_HEADER = HeaderName.of("Content-Type"); private static final JsonCodec QUERY_RESULTS_CODEC = new JsonCodecFactory(new JsonMapperProvider() .withModules(Set.of(new QueryDataJacksonModule())) .get()) .jsonCodec(QueryResults.class); + private static final JsonCodec STRING_CODEC = JsonCodec.jsonCodec(String.class); + private static final String SECRET = "do-not-echo-this-value"; + + private static HttpClient client; + private static TestingTrinoServer server; + + @BeforeAll + static void setUp() + throws Exception + { + client = new JettyHttpClient(); + server = TestingTrinoServer.builder() + .setProperties(Map.of("hogql.enabled", "true")) + .build(); + server.installPlugin(new TpchPlugin()); + server.createCatalog("tpch", "tpch"); + } + + @AfterAll + static void tearDown() + throws Exception + { + closeAll(server, client); + } @Test public void testEndpointIsDisabledByDefault() throws Exception { - HttpClient client = new JettyHttpClient(); - TestingTrinoServer server = TestingTrinoServer.create(); + HttpClient disabledClient = new JettyHttpClient(); + TestingTrinoServer disabledServer = TestingTrinoServer.create(); try { - StatusResponse response = client.execute( + StatusResponse response = disabledClient.execute( preparePost() - .setUri(server.resolve("/v1/hogql")) + .setUri(disabledServer.resolve("/v1/hogql")) .setHeader(REQUEST_USER_HEADER, "user") - .setBodyGenerator(createStaticBodyGenerator("SELECT 1", UTF_8)) + .setHeader(CONTENT_TYPE_HEADER, APPLICATION_JSON) + .setBodyGenerator(createStaticBodyGenerator(hogQlRequest("SELECT 1"), UTF_8)) .build(), createStatusResponseHandler()); assertThat(response.getStatusCode()).isEqualTo(404); } finally { - closeAll(server, client); + closeAll(disabledServer, disabledClient); } } @@ -76,41 +113,80 @@ public void testEndpointIsDisabledByDefault() public void testEnabledEndpointUsesStatementProtocol() throws Exception { - HttpClient client = new JettyHttpClient(); - TestingTrinoServer server = TestingTrinoServer.builder() - .setProperties(Map.of("hogql.enabled", "true")) - .build(); - try { - server.installPlugin(new TpchPlugin()); - server.createCatalog("tpch", "tpch"); + List hogqlResults = runHogQlToCompletion(hogQlRequestWithTypedValues("SELECT 1")); + assertThat(hogqlResults.getFirst().getNextUri().getPath()).startsWith("/v1/statement/queued/"); + assertThat(rows(hogqlResults)).containsExactly(ImmutableList.of(1)); - List hogqlResults = runToCompletion(client, server, "/v1/hogql", "SELECT 1"); - assertThat(hogqlResults.getFirst().getNextUri().getPath()).startsWith("/v1/statement/queued/"); - assertThat(rows(hogqlResults)).containsExactly(ImmutableList.of(1)); + List tableScanResults = runHogQlToCompletion(hogQlRequest("SELECT nationkey FROM tpch.tiny.nation")); + assertThat(rows(tableScanResults)).hasSize(25); - List tableScanResults = runToCompletion(client, server, "/v1/hogql", "SELECT nationkey FROM tpch.tiny.nation"); - assertThat(rows(tableScanResults)).hasSize(25); + List sqlResults = runToCompletion("/v1/statement", "SELECT 2", false); + assertThat(rows(sqlResults)).containsExactly(ImmutableList.of(2)); + } - List sqlResults = runToCompletion(client, server, "/v1/statement", "SELECT 2"); - assertThat(rows(sqlResults)).containsExactly(ImmutableList.of(2)); + @Test + public void testEndpointRequiresJsonContentType() + { + StringResponse response = post(hogQlRequest("SELECT 1"), "text/plain"); - List invalidResults = runToCompletion(client, server, "/v1/hogql", "SELECT FROM"); - assertThat(invalidResults.getLast().getError().getErrorName()).isEqualTo("SYNTAX_ERROR"); - } - finally { - closeAll(server, client); - } + assertThat(response.getStatusCode()).isEqualTo(415); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("invalidRequests") + public void testInvalidRequestFailsClosed(String name, String request) + { + StringResponse response = post(request, APPLICATION_JSON); + + assertThat(response.getStatusCode()).isEqualTo(400); + assertThat(response.getBody()).doesNotContain(SECRET); + assertThat(response.getBody()).hasSizeLessThan(1024); } - private static List runToCompletion(HttpClient client, TestingTrinoServer server, String path, String query) + private static Stream invalidRequests() + { + String languageVersion = HogQlLanguageContract.current().languageVersion().toString(); + return Stream.of( + Arguments.of("unsupported protocol version", """ + {"query":"SELECT '%s'","protocolVersion":2,"languageVersion":"%s"} + """.formatted(SECRET, languageVersion)), + Arguments.of("unsupported language version", """ + {"query":"SELECT '%s'","protocolVersion":1,"languageVersion":"999.0.0"} + """.formatted(SECRET)), + Arguments.of("unknown top-level field", """ + {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","unknown":"%s"} + """.formatted(languageVersion, SECRET)), + Arguments.of("unknown typed-value field", """ + {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{"p":{"type":"string","value":"%s","unknown":true}}} + """.formatted(languageVersion, SECRET)), + Arguments.of("ambiguous typed value", """ + {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{"p":{"type":"string","unknown":"%s"}}} + """.formatted(languageVersion, SECRET)), + Arguments.of("nonpositive catalog generation", """ + {"query":"SELECT '%s'","protocolVersion":1,"languageVersion":"%s","catalogGeneration":0} + """.formatted(SECRET, languageVersion)), + Arguments.of("malformed JSON", """ + {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{"p":{"type":"string","value":"%s"}} + """.formatted(languageVersion, SECRET))); + } + + private static List runHogQlToCompletion(String request) + { + return runToCompletion("/v1/hogql", request, true); + } + + private static List runToCompletion(String path, String body, boolean json) { ImmutableList.Builder results = ImmutableList.builder(); + Request.Builder request = preparePost() + .setUri(server.resolve(path)) + .setHeader(REQUEST_USER_HEADER, "user") + .setBodyGenerator(createStaticBodyGenerator(body, UTF_8)); + if (json) { + request.setHeader(CONTENT_TYPE_HEADER, APPLICATION_JSON); + } QueryResults current = client.execute( - preparePost() - .setUri(server.resolve(path)) - .setHeader(REQUEST_USER_HEADER, "user") - .setBodyGenerator(createStaticBodyGenerator(query, UTF_8)) - .build(), + request.build(), createJsonResponseHandler(QUERY_RESULTS_CODEC)); results.add(current); @@ -126,6 +202,41 @@ private static List runToCompletion(HttpClient client, TestingTrin return results.build(); } + private static StringResponse post(String body, String contentType) + { + return client.execute( + preparePost() + .setUri(server.resolve("/v1/hogql")) + .setHeader(REQUEST_USER_HEADER, "user") + .setHeader(CONTENT_TYPE_HEADER, contentType) + .setBodyGenerator(createStaticBodyGenerator(body, UTF_8)) + .build(), + createStringResponseHandler()); + } + + private static String hogQlRequest(String query) + { + return """ + {"query":%s,"protocolVersion":1,"languageVersion":"%s"} + """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); + } + + private static String hogQlRequestWithTypedValues(String query) + { + return """ + { + "query": %s, + "protocolVersion": 1, + "languageVersion": "%s", + "parameters": {}, + "variables": {"array": {"type": "array", "value": [true, "value", null]}}, + "filters": {"object": {"type": "object", "value": {"nested": 2.5}}}, + "modifiers": {"missing": {"type": "nullable", "value": null}}, + "catalogGeneration": 1 + } + """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); + } + private static List> rows(List results) throws Exception { diff --git a/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java b/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java index 2e669b3a89c9..ec3103ced079 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java @@ -15,7 +15,9 @@ import io.trino.Session; import io.trino.execution.QueryPreparer.PreparedQuery; +import io.trino.hogql.compiler.HogQlCompileEnvelope; import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.parser.HogQlLanguageContract; import io.trino.sql.parser.ParsingException; import io.trino.sql.parser.SqlParser; import io.trino.sql.tree.AllColumns; @@ -23,6 +25,10 @@ import io.trino.sql.tree.Query; import org.junit.jupiter.api.Test; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; + import static io.trino.SessionTestUtils.TEST_SESSION; import static io.trino.execution.QuerySubmission.hogQl; import static io.trino.execution.QuerySubmission.trino; @@ -55,18 +61,34 @@ public void testSelectStatement() @Test public void testHogQlStatement() { - PreparedQuery preparedQuery = HOGQL_QUERY_PREPARER.prepareQuery(TEST_SESSION, hogQl("SELECT 1")); + HogQlCompileEnvelope envelope = envelope("SELECT 1"); + QuerySubmission submission = hogQl(envelope); + PreparedQuery preparedQuery = HOGQL_QUERY_PREPARER.prepareQuery(TEST_SESSION, submission); assertThat(preparedQuery.getStatement()).isInstanceOf(Query.class); + assertThat(submission.hogQlEnvelope()).containsSame(envelope); } @Test public void testHogQlSubmissionFailsClosedWithoutCompiler() { - assertTrinoExceptionThrownBy(() -> QUERY_PREPARER.prepareQuery(TEST_SESSION, hogQl("SELECT 1"))) + assertTrinoExceptionThrownBy(() -> QUERY_PREPARER.prepareQuery(TEST_SESSION, hogQl(envelope("SELECT 1")))) .hasErrorCode(NOT_SUPPORTED) .hasMessage("HogQL query submission is disabled"); } + private static HogQlCompileEnvelope envelope(String query) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.empty()); + } + @Test public void testExecuteStatement() { From 803cdcdef26c4b61371911a637125fc3d85d7782 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 10:15:44 -0400 Subject: [PATCH 025/136] feat(hogql): bound semantic request bindings --- .../main/java/io/trino/dispatcher/HogQlRequest.java | 4 ++++ .../dispatcher/TestHogQlStatementResource.java | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java index 5e943122e8c5..204e83dcf2a5 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java @@ -59,6 +59,7 @@ record HogQlRequest( "catalogGeneration"); private static final Set TYPED_VALUE_FIELDS = Set.of("type", "value"); private static final int MAX_VALUE_DEPTH = 64; + private static final int MAX_BINDINGS_PER_FIELD = 1_000; HogQlRequest { @@ -134,6 +135,9 @@ private static Map typedValues(JsonNode root, String fi if (!values.isObject()) { throw new IllegalArgumentException("invalid HogQL semantic field"); } + if (values.size() > MAX_BINDINGS_PER_FIELD) { + throw new IllegalArgumentException("HogQL semantic field has too many bindings"); + } Map result = new LinkedHashMap<>(); Iterator names = values.fieldNames(); diff --git a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java index 44fff6c28343..2e7f6ba83f92 100644 --- a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java +++ b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java @@ -51,6 +51,8 @@ import static io.trino.client.ProtocolHeaders.TRINO_HEADERS; import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.joining; +import static java.util.stream.IntStream.range; import static org.assertj.core.api.Assertions.assertThat; class TestHogQlStatementResource @@ -165,11 +167,22 @@ private static Stream invalidRequests() Arguments.of("nonpositive catalog generation", """ {"query":"SELECT '%s'","protocolVersion":1,"languageVersion":"%s","catalogGeneration":0} """.formatted(SECRET, languageVersion)), + Arguments.of("too many parameter bindings", requestWithParameterCount(languageVersion, 1_001)), Arguments.of("malformed JSON", """ {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{"p":{"type":"string","value":"%s"}} """.formatted(languageVersion, SECRET))); } + private static String requestWithParameterCount(String languageVersion, int parameterCount) + { + String parameters = range(0, parameterCount) + .mapToObj(index -> "\"p%s\":{\"type\":\"integer\",\"value\":%s}".formatted(index, index)) + .collect(joining(",")); + return """ + {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{%s}} + """.formatted(languageVersion, parameters); + } + private static List runHogQlToCompletion(String request) { return runToCompletion("/v1/hogql", request, true); From facd75424cec5ec8667461d96697d0035412be37 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 10:18:09 -0400 Subject: [PATCH 026/136] test(hogql): cover queued query lifecycle --- .../TestHogQlStatementResource.java | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java index 2e7f6ba83f92..41748f14a26d 100644 --- a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java +++ b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java @@ -42,6 +42,7 @@ import java.util.stream.Stream; import static io.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; +import static io.airlift.http.client.Request.Builder.prepareDelete; import static io.airlift.http.client.Request.Builder.prepareGet; import static io.airlift.http.client.Request.Builder.preparePost; import static io.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; @@ -126,6 +127,33 @@ public void testEnabledEndpointUsesStatementProtocol() assertThat(rows(sqlResults)).containsExactly(ImmutableList.of(2)); } + @Test + public void testEmptyResultsAndCompilerDiagnostics() + throws Exception + { + List emptyResults = runHogQlToCompletion(hogQlRequest("SELECT 1 WHERE false")); + assertThat(rows(emptyResults)).isEmpty(); + assertThat(emptyResults.getLast().getWarnings()).isEmpty(); + + List invalidResults = runHogQlToCompletion(hogQlRequest("SELECT (")); + assertThat(invalidResults.getLast().getError().getErrorName()).isEqualTo("HOGQL_SYNTAX_ERROR"); + } + + @Test + public void testQueuedQueryCanBeCancelled() + { + QueryResults queued = postQuery(hogQlRequest("SELECT nationkey FROM tpch.tiny.nation")); + + StatusResponse response = client.execute( + prepareDelete() + .setUri(queued.getNextUri()) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createStatusResponseHandler()); + + assertThat(response.getStatusCode()).isEqualTo(204); + } + @Test public void testEndpointRequiresJsonContentType() { @@ -198,9 +226,7 @@ private static List runToCompletion(String path, String body, bool if (json) { request.setHeader(CONTENT_TYPE_HEADER, APPLICATION_JSON); } - QueryResults current = client.execute( - request.build(), - createJsonResponseHandler(QUERY_RESULTS_CODEC)); + QueryResults current = client.execute(request.build(), createJsonResponseHandler(QUERY_RESULTS_CODEC)); results.add(current); while (current.getNextUri() != null) { @@ -215,6 +241,18 @@ private static List runToCompletion(String path, String body, bool return results.build(); } + private static QueryResults postQuery(String request) + { + return client.execute( + preparePost() + .setUri(server.resolve("/v1/hogql")) + .setHeader(REQUEST_USER_HEADER, "user") + .setHeader(CONTENT_TYPE_HEADER, APPLICATION_JSON) + .setBodyGenerator(createStaticBodyGenerator(request, UTF_8)) + .build(), + createJsonResponseHandler(QUERY_RESULTS_CODEC)); + } + private static StringResponse post(String body, String contentType) { return client.execute( From 2a5bf4c82647e4421379326578e16d556a609765 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 10:20:13 -0400 Subject: [PATCH 027/136] feat(hogql): decode semantic catalog manifests --- core/trino-hogql-compiler/pom.xml | 10 + ...gQlSemanticCatalogSnapshotJsonDecoder.java | 410 ++++++++++++++++++ .../HogQlSemanticCatalogSnapshotLoader.java | 70 +++ ...gQlSemanticCatalogSnapshotJsonDecoder.java | 239 ++++++++++ ...estHogQlSemanticCatalogSnapshotLoader.java | 78 ++++ 5 files changed, 807 insertions(+) create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotLoader.java create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotLoader.java diff --git a/core/trino-hogql-compiler/pom.xml b/core/trino-hogql-compiler/pom.xml index 08aae74b5290..6d86f84703c3 100644 --- a/core/trino-hogql-compiler/pom.xml +++ b/core/trino-hogql-compiler/pom.xml @@ -14,6 +14,16 @@ Trino - HogQL compiler + + com.fasterxml.jackson.core + jackson-core + + + + com.fasterxml.jackson.core + jackson-databind + + io.trino trino-hogql-parser diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java new file mode 100644 index 000000000000..7208ad6f0a71 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java @@ -0,0 +1,410 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.StreamReadFeature; +import com.fasterxml.jackson.core.exc.StreamConstraintsException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; +import io.trino.hogql.parser.HogQlLanguageVersion; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public final class HogQlSemanticCatalogSnapshotJsonDecoder +{ + public static final int PROTOCOL_VERSION = 1; + + private static final int SCHEMA_VERSION = 1; + private static final Limits DEFAULT_LIMITS = new Limits(8 * 1024 * 1024, 64, 100_000); + + private static final Set SNAPSHOT_FIELDS = Set.of( + "protocolVersion", + "schemaVersion", + "languageVersion", + "catalog", + "generation", + "logicalTables"); + private static final Set IDENTIFIER_FIELDS = Set.of("value", "delimited"); + private static final Set QUALIFIED_NAME_FIELDS = Set.of("catalog", "schema", "table"); + private static final Set TABLE_FIELDS = Set.of("name", "physicalTable", "fields", "properties", "relationships"); + private static final Set LOGICAL_FIELD_FIELDS = Set.of("name", "physicalColumn", "trinoTypeSignature", "logicalType", "nullable", "starVisible"); + private static final Set PROPERTY_FIELDS = Set.of("name", "sourceField", "storage", "logicalType", "nullable"); + private static final Set RELATIONSHIP_FIELDS = Set.of("name", "targetTable", "cardinality", "joinKeys"); + private static final Set JOIN_KEY_FIELDS = Set.of("sourceField", "targetField"); + + private final Limits limits; + private final ObjectMapper objectMapper; + + public HogQlSemanticCatalogSnapshotJsonDecoder() + { + this(DEFAULT_LIMITS); + } + + public HogQlSemanticCatalogSnapshotJsonDecoder(Limits limits) + { + this.limits = requireNonNull(limits, "limits is null"); + JsonFactory jsonFactory = JsonFactory.builder() + .streamReadConstraints(StreamReadConstraints.builder() + .maxNestingDepth(limits.maximumNestingDepth()) + .maxStringLength(limits.maximumPayloadBytes()) + .maxNumberLength(64) + .build()) + .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) + .build(); + this.objectMapper = new ObjectMapper(jsonFactory); + } + + public HogQlSemanticCatalogSnapshot decode(byte[] payload, LoadRequest request) + { + requireNonNull(request, "request is null"); + if (payload == null) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + if (payload.length > limits.maximumPayloadBytes()) { + throw failure(DecodeFailure.LIMIT_EXCEEDED); + } + + try { + ObjectNode root = parse(payload); + validateFields(root, SNAPSHOT_FIELDS); + + int protocolVersion = integer(root, "protocolVersion"); + if (protocolVersion != PROTOCOL_VERSION) { + throw failure(DecodeFailure.UNSUPPORTED_PROTOCOL); + } + int schemaVersion = integer(root, "schemaVersion"); + if (schemaVersion != SCHEMA_VERSION) { + throw failure(DecodeFailure.UNSUPPORTED_SCHEMA); + } + + HogQlLanguageVersion languageVersion = languageVersion(root); + if (!languageVersion.equals(request.languageVersion())) { + throw failure(DecodeFailure.LANGUAGE_VERSION_MISMATCH); + } + + PhysicalIdentifier catalog = physicalIdentifier(required(root, "catalog")); + if (!catalog.equals(request.catalog())) { + throw failure(DecodeFailure.CATALOG_MISMATCH); + } + + long generation = positiveGeneration(root); + if (request.expectedGeneration().isPresent() && request.expectedGeneration().orElseThrow() != generation) { + throw failure(DecodeFailure.GENERATION_MISMATCH); + } + + CollectionBudget budget = new CollectionBudget(limits.maximumCollectionEntries()); + List tables = logicalTables(required(root, "logicalTables"), budget); + return new HogQlSemanticCatalogSnapshot(schemaVersion, languageVersion, catalog, generation, tables); + } + catch (DecodeException e) { + throw e; + } + catch (StreamConstraintsException e) { + throw failure(DecodeFailure.LIMIT_EXCEEDED); + } + catch (IOException | RuntimeException e) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + } + + private ObjectNode parse(byte[] payload) + throws IOException + { + try (JsonParser parser = objectMapper.createParser(payload)) { + JsonNode root = objectMapper.readTree(parser); + if (parser.nextToken() != null) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return object(root); + } + } + + private static HogQlLanguageVersion languageVersion(ObjectNode root) + { + return HogQlLanguageVersion.valueOf(text(root, "languageVersion")); + } + + private static long positiveGeneration(ObjectNode root) + { + JsonNode node = required(root, "generation"); + if (!node.isIntegralNumber() || !node.canConvertToLong() || node.longValue() <= 0) { + throw failure(DecodeFailure.GENERATION_MISMATCH); + } + return node.longValue(); + } + + private static List logicalTables(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List tables = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode table = object(element); + validateFields(table, TABLE_FIELDS); + tables.add(new LogicalTableDefinition( + text(table, "name"), + qualifiedName(required(table, "physicalTable")), + logicalFields(required(table, "fields"), budget), + properties(required(table, "properties"), budget), + relationships(required(table, "relationships"), budget))); + } + return List.copyOf(tables); + } + + private static List logicalFields(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List fields = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode field = object(element); + validateFields(field, LOGICAL_FIELD_FIELDS); + fields.add(new LogicalFieldDefinition( + text(field, "name"), + physicalIdentifier(required(field, "physicalColumn")), + text(field, "trinoTypeSignature"), + enumValue(field, "logicalType", LogicalType.class), + bool(field, "nullable"), + bool(field, "starVisible"))); + } + return List.copyOf(fields); + } + + private static List properties(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List properties = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode property = object(element); + validateFields(property, PROPERTY_FIELDS); + properties.add(new PropertyDefinition( + text(property, "name"), + text(property, "sourceField"), + enumValue(property, "storage", PropertyStorage.class), + enumValue(property, "logicalType", LogicalType.class), + bool(property, "nullable"))); + } + return List.copyOf(properties); + } + + private static List relationships(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List relationships = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode relationship = object(element); + validateFields(relationship, RELATIONSHIP_FIELDS); + relationships.add(new RelationshipDefinition( + text(relationship, "name"), + text(relationship, "targetTable"), + enumValue(relationship, "cardinality", RelationshipCardinality.class), + joinKeys(required(relationship, "joinKeys"), budget))); + } + return List.copyOf(relationships); + } + + private static List joinKeys(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List joinKeys = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode joinKey = object(element); + validateFields(joinKey, JOIN_KEY_FIELDS); + joinKeys.add(new JoinKey(text(joinKey, "sourceField"), text(joinKey, "targetField"))); + } + return List.copyOf(joinKeys); + } + + private static PhysicalQualifiedName qualifiedName(JsonNode node) + { + ObjectNode name = object(node); + validateFields(name, QUALIFIED_NAME_FIELDS); + return new PhysicalQualifiedName( + physicalIdentifier(required(name, "catalog")), + physicalIdentifier(required(name, "schema")), + physicalIdentifier(required(name, "table"))); + } + + private static PhysicalIdentifier physicalIdentifier(JsonNode node) + { + ObjectNode identifier = object(node); + validateFields(identifier, IDENTIFIER_FIELDS); + return new PhysicalIdentifier(text(identifier, "value"), bool(identifier, "delimited")); + } + + private static ObjectNode object(JsonNode node) + { + if (!(node instanceof ObjectNode object)) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return object; + } + + private static ArrayNode array(JsonNode node, CollectionBudget budget) + { + if (!(node instanceof ArrayNode array)) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + budget.add(array.size()); + return array; + } + + private static JsonNode required(ObjectNode object, String name) + { + JsonNode node = object.get(name); + if (node == null || node.isNull()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return node; + } + + private static String text(ObjectNode object, String name) + { + JsonNode node = required(object, name); + if (!node.isTextual()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return node.textValue(); + } + + private static int integer(ObjectNode object, String name) + { + JsonNode node = required(object, name); + if (!node.isIntegralNumber() || !node.canConvertToInt()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return node.intValue(); + } + + private static boolean bool(ObjectNode object, String name) + { + JsonNode node = required(object, name); + if (!node.isBoolean()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return node.booleanValue(); + } + + private static > E enumValue(ObjectNode object, String name, Class enumType) + { + return Enum.valueOf(enumType, text(object, name)); + } + + private static void validateFields(ObjectNode object, Set allowedFields) + { + Iterator fieldNames = object.fieldNames(); + while (fieldNames.hasNext()) { + if (!allowedFields.contains(fieldNames.next())) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + } + if (object.size() != allowedFields.size()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + } + + private static DecodeException failure(DecodeFailure failure) + { + return new DecodeException(failure); + } + + public record Limits(int maximumPayloadBytes, int maximumNestingDepth, int maximumCollectionEntries) + { + public Limits + { + if (maximumPayloadBytes <= 0 || maximumNestingDepth <= 0 || maximumCollectionEntries <= 0) { + throw new IllegalArgumentException("HogQL semantic catalog decoder limits must be positive"); + } + } + } + + public static final class DecodeException + extends RuntimeException + { + private final DecodeFailure failure; + + private DecodeException(DecodeFailure failure) + { + super(requireNonNull(failure, "failure is null").message()); + this.failure = failure; + } + + public DecodeFailure failure() + { + return failure; + } + } + + public enum DecodeFailure + { + INVALID_PAYLOAD("Invalid HogQL semantic catalog payload"), + LIMIT_EXCEEDED("HogQL semantic catalog payload limit exceeded"), + UNSUPPORTED_PROTOCOL("Unsupported HogQL semantic catalog protocol"), + UNSUPPORTED_SCHEMA("Unsupported HogQL semantic catalog schema"), + LANGUAGE_VERSION_MISMATCH("HogQL semantic catalog language version mismatch"), + CATALOG_MISMATCH("HogQL semantic catalog identifier mismatch"), + GENERATION_MISMATCH("HogQL semantic catalog generation mismatch"); + + private final String message; + + DecodeFailure(String message) + { + this.message = message; + } + + private String message() + { + return message; + } + } + + private static final class CollectionBudget + { + private final int maximumEntries; + private int entries; + + private CollectionBudget(int maximumEntries) + { + this.maximumEntries = maximumEntries; + } + + private void add(int size) + { + if (size > maximumEntries - entries) { + throw failure(DecodeFailure.LIMIT_EXCEEDED); + } + entries += size; + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotLoader.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotLoader.java new file mode 100644 index 000000000000..c70ce34fe85c --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotLoader.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.parser.HogQlLanguageVersion; + +import java.util.OptionalLong; +import java.util.concurrent.CompletionStage; + +import static java.util.Objects.requireNonNull; + +@FunctionalInterface +public interface HogQlSemanticCatalogSnapshotLoader +{ + CompletionStage load(LoadRequest request); + + static HogQlSemanticCatalogSnapshotLoader fromJsonTransport( + JsonTransport transport, + HogQlSemanticCatalogSnapshotJsonDecoder decoder) + { + requireNonNull(transport, "transport is null"); + requireNonNull(decoder, "decoder is null"); + return request -> { + requireNonNull(request, "request is null"); + CompletionStage response = requireNonNull(transport.load(request), "transport returned null"); + return response.thenApply(payload -> decoder.decode(payload, request)); + }; + } + + @FunctionalInterface + interface JsonTransport + { + CompletionStage load(LoadRequest request); + } + + record LoadRequest(PhysicalIdentifier catalog, HogQlLanguageVersion languageVersion, OptionalLong expectedGeneration) + { + public LoadRequest + { + catalog = requireNonNull(catalog, "catalog is null"); + languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + expectedGeneration = requireNonNull(expectedGeneration, "expectedGeneration is null"); + if (expectedGeneration.isPresent() && expectedGeneration.orElseThrow() <= 0) { + throw new IllegalArgumentException("expected generation must be positive"); + } + } + + public static LoadRequest latest(PhysicalIdentifier catalog, HogQlLanguageVersion languageVersion) + { + return new LoadRequest(catalog, languageVersion, OptionalLong.empty()); + } + + public static LoadRequest pinned(PhysicalIdentifier catalog, HogQlLanguageVersion languageVersion, long generation) + { + return new LoadRequest(catalog, languageVersion, OptionalLong.of(generation)); + } + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java new file mode 100644 index 000000000000..472856c12a30 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java @@ -0,0 +1,239 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder.DecodeFailure; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder.Limits; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; +import io.trino.hogql.parser.HogQlLanguageVersion; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlSemanticCatalogSnapshotJsonDecoder +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageVersion.valueOf("1.0.0"); + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("ducklake", false); + private static final String SECRET = "secret-catalog-payload-value"; + + @Test + public void testDecodesCompleteSnapshot() + { + HogQlSemanticCatalogSnapshot snapshot = new HogQlSemanticCatalogSnapshotJsonDecoder() + .decode(bytes(validSnapshotJson()), LoadRequest.latest(CATALOG, LANGUAGE_VERSION)); + + assertThat(snapshot.schemaVersion()).isEqualTo(1); + assertThat(snapshot.languageVersion()).isEqualTo(LANGUAGE_VERSION); + assertThat(snapshot.catalog()).isEqualTo(CATALOG); + assertThat(snapshot.generation()).isEqualTo(7); + assertThat(snapshot.logicalTables()).singleElement().satisfies(table -> { + assertThat(table.name()).isEqualTo("events"); + assertThat(table.physicalTable().catalog()).isEqualTo(CATALOG); + assertThat(table.physicalTable().schema()).isEqualTo(new PhysicalIdentifier("Analytics", true)); + assertThat(table.fields()).singleElement().satisfies(field -> { + assertThat(field.name()).isEqualTo("properties"); + assertThat(field.trinoTypeSignature()).isEqualTo("json"); + assertThat(field.logicalType()).isEqualTo(LogicalType.JSON); + assertThat(field.nullable()).isTrue(); + assertThat(field.starVisible()).isFalse(); + }); + assertThat(table.properties()).singleElement().satisfies(property -> { + assertThat(property.storage()).isEqualTo(PropertyStorage.JSON_OBJECT); + assertThat(property.logicalType()).isEqualTo(LogicalType.STRING); + }); + assertThat(table.relationships()).singleElement().satisfies(relationship -> { + assertThat(relationship.cardinality()).isEqualTo(RelationshipCardinality.MANY_TO_ONE); + assertThat(relationship.joinKeys()).singleElement().satisfies(joinKey -> { + assertThat(joinKey.sourceField()).isEqualTo("properties"); + assertThat(joinKey.targetField()).isEqualTo("properties"); + }); + }); + }); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("compatibilityFailures") + public void testRejectsIncompatibleSnapshots(String name, String payload, LoadRequest request, DecodeFailure failure) + { + assertDecodeFailure(new HogQlSemanticCatalogSnapshotJsonDecoder(), payload, request, failure); + } + + private static Stream compatibilityFailures() + { + return Stream.of( + Arguments.of( + "protocol", + validSnapshotJson().replace("\"protocolVersion\": 1", "\"protocolVersion\": 2"), + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + DecodeFailure.UNSUPPORTED_PROTOCOL), + Arguments.of( + "schema", + validSnapshotJson().replace("\"schemaVersion\": 1", "\"schemaVersion\": 2"), + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + DecodeFailure.UNSUPPORTED_SCHEMA), + Arguments.of( + "language", + validSnapshotJson().replace("\"languageVersion\": \"1.0.0\"", "\"languageVersion\": \"2.0.0\""), + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + DecodeFailure.LANGUAGE_VERSION_MISMATCH), + Arguments.of( + "catalog", + validSnapshotJson().replace("\"value\": \"ducklake\"", "\"value\": \"other\""), + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + DecodeFailure.CATALOG_MISMATCH), + Arguments.of( + "pinned generation", + validSnapshotJson(), + LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 8), + DecodeFailure.GENERATION_MISMATCH), + Arguments.of( + "nonpositive generation", + validSnapshotJson().replace("\"generation\": 7", "\"generation\": 0"), + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + DecodeFailure.GENERATION_MISMATCH)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("strictFailures") + public void testRejectsMalformedOrExtendedPayloads(String name, String payload) + { + assertDecodeFailure( + new HogQlSemanticCatalogSnapshotJsonDecoder(), + payload, + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + DecodeFailure.INVALID_PAYLOAD); + } + + private static Stream strictFailures() + { + return Stream.of( + Arguments.of("unknown root field", validSnapshotJson().replace("\"logicalTables\":", "\"unknown\": true, \"logicalTables\":")), + Arguments.of("unknown nested field", validSnapshotJson().replace("\"starVisible\": false", "\"starVisible\": false, \"unknown\": true")), + Arguments.of("duplicate field", validSnapshotJson().replace("\"protocolVersion\": 1", "\"protocolVersion\": 1, \"protocolVersion\": 1")), + Arguments.of("missing field", validSnapshotJson().replace("\"generation\": 7,", "")), + Arguments.of("wrong field type", validSnapshotJson().replace("\"nullable\": true", "\"nullable\": \"true\"")), + Arguments.of("invalid enum", validSnapshotJson().replace("\"logicalType\": \"JSON\"", "\"logicalType\": \"" + SECRET + "\"")), + Arguments.of("trailing content", validSnapshotJson() + " true"), + Arguments.of("malformed JSON", "{\"protocolVersion\":" + SECRET)); + } + + @Test + public void testEnforcesPayloadLimit() + { + HogQlSemanticCatalogSnapshotJsonDecoder decoder = new HogQlSemanticCatalogSnapshotJsonDecoder( + new Limits(bytes(validSnapshotJson()).length - 1, 64, 100)); + + assertDecodeFailure(decoder, validSnapshotJson(), LoadRequest.latest(CATALOG, LANGUAGE_VERSION), DecodeFailure.LIMIT_EXCEEDED); + } + + @Test + public void testEnforcesDepthLimit() + { + String payload = validSnapshotJson().replace( + "\"logicalTables\":", + "\"unknown\": [[[[[true]]]]], \"logicalTables\":"); + HogQlSemanticCatalogSnapshotJsonDecoder decoder = new HogQlSemanticCatalogSnapshotJsonDecoder(new Limits(1_000_000, 4, 100)); + + assertDecodeFailure(decoder, payload, LoadRequest.latest(CATALOG, LANGUAGE_VERSION), DecodeFailure.LIMIT_EXCEEDED); + } + + @Test + public void testEnforcesCumulativeCollectionLimit() + { + HogQlSemanticCatalogSnapshotJsonDecoder decoder = new HogQlSemanticCatalogSnapshotJsonDecoder(new Limits(1_000_000, 64, 1)); + + assertDecodeFailure(decoder, validSnapshotJson(), LoadRequest.latest(CATALOG, LANGUAGE_VERSION), DecodeFailure.LIMIT_EXCEEDED); + } + + private static void assertDecodeFailure( + HogQlSemanticCatalogSnapshotJsonDecoder decoder, + String payload, + LoadRequest request, + DecodeFailure expectedFailure) + { + assertThatThrownBy(() -> decoder.decode(bytes(payload), request)) + .isInstanceOfSatisfying(HogQlSemanticCatalogSnapshotJsonDecoder.DecodeException.class, exception -> { + assertThat(exception.failure()).isEqualTo(expectedFailure); + assertThat(exception).hasMessageNotContaining(SECRET); + assertThat(exception.getMessage()).hasSizeLessThan(128); + assertThat(exception.getCause()).isNull(); + }); + } + + private static byte[] bytes(String value) + { + return value.getBytes(StandardCharsets.UTF_8); + } + + private static String validSnapshotJson() + { + return """ + { + "protocolVersion": 1, + "schemaVersion": 1, + "languageVersion": "1.0.0", + "catalog": {"value": "ducklake", "delimited": false}, + "generation": 7, + "logicalTables": [ + { + "name": "events", + "physicalTable": { + "catalog": {"value": "ducklake", "delimited": false}, + "schema": {"value": "Analytics", "delimited": true}, + "table": {"value": "events", "delimited": false} + }, + "fields": [ + { + "name": "properties", + "physicalColumn": {"value": "properties", "delimited": false}, + "trinoTypeSignature": "json", + "logicalType": "JSON", + "nullable": true, + "starVisible": false + } + ], + "properties": [ + { + "name": "browser", + "sourceField": "properties", + "storage": "JSON_OBJECT", + "logicalType": "STRING", + "nullable": true + } + ], + "relationships": [ + { + "name": "self", + "targetTable": "events", + "cardinality": "MANY_TO_ONE", + "joinKeys": [{"sourceField": "properties", "targetField": "properties"}] + } + ] + } + ] + } + """; + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotLoader.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotLoader.java new file mode 100644 index 000000000000..79cec3f77253 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotLoader.java @@ -0,0 +1,78 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.JsonTransport; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; +import io.trino.hogql.parser.HogQlLanguageVersion; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlSemanticCatalogSnapshotLoader +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageVersion.valueOf("1.0.0"); + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("ducklake", false); + + @Test + public void testPreservesLatestAndPinnedReadSemanticsThroughJsonTransport() + { + List requests = new ArrayList<>(); + JsonTransport transport = request -> { + requests.add(request); + return CompletableFuture.completedFuture(snapshotJson(request.expectedGeneration().orElse(9)).getBytes(StandardCharsets.UTF_8)); + }; + HogQlSemanticCatalogSnapshotLoader loader = HogQlSemanticCatalogSnapshotLoader.fromJsonTransport( + transport, + new HogQlSemanticCatalogSnapshotJsonDecoder()); + + HogQlSemanticCatalogSnapshot latest = loader.load(LoadRequest.latest(CATALOG, LANGUAGE_VERSION)).toCompletableFuture().join(); + HogQlSemanticCatalogSnapshot pinned = loader.load(LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 7)).toCompletableFuture().join(); + + assertThat(latest.generation()).isEqualTo(9); + assertThat(pinned.generation()).isEqualTo(7); + assertThat(requests).containsExactly( + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 7)); + } + + @Test + public void testRejectsInvalidPinnedGenerationBeforeTransport() + { + assertThatThrownBy(() -> LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 0)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageNotContaining("ducklake"); + } + + private static String snapshotJson(long generation) + { + return """ + { + "protocolVersion": 1, + "schemaVersion": 1, + "languageVersion": "1.0.0", + "catalog": {"value": "ducklake", "delimited": false}, + "generation": %s, + "logicalTables": [] + } + """.formatted(generation); + } +} From 1ed6a1d0913316570c2d0e48a10140ba6d97f577 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 10:32:38 -0400 Subject: [PATCH 028/136] feat(hogql): lower ordering and pagination --- .../trino/hogql/compiler/HogQlCompiler.java | 3 + .../trino/hogql/compiler/TrinoAstFactory.java | 36 ++++- .../hogql/compiler/TestHogQlCompiler.java | 8 ++ .../io/trino/hogql/parser/HogQlParser.java | 128 ++++++++++++++++-- .../trino/hogql/parser/tree/HogQlQuery.java | 37 ++++- .../trino/hogql/parser/TestHogQlParser.java | 16 +++ 6 files changed, 212 insertions(+), 16 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index c8397d1e95fb..df546bec97af 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -104,6 +104,9 @@ private static HogQlCompilationResult compile(HogQlQuery query, Map collectPlaceholders(expression, placeholders)); + query.orderBy().forEach(sortItem -> collectPlaceholders(sortItem.expression(), placeholders)); + query.limit().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + query.offset().ifPresent(expression -> collectPlaceholders(expression, placeholders)); placeholders.sort(Comparator.comparingInt(placeholder -> placeholder.span().startOffset())); List missing = placeholders.stream() diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 0f4bbc84941d..57bd1792ed1f 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -49,11 +49,14 @@ import io.trino.sql.tree.InListExpression; import io.trino.sql.tree.InPredicate; import io.trino.sql.tree.IsNullPredicate; +import io.trino.sql.tree.Limit; import io.trino.sql.tree.LogicalExpression; import io.trino.sql.tree.LongLiteral; import io.trino.sql.tree.NodeLocation; import io.trino.sql.tree.NotExpression; import io.trino.sql.tree.NullLiteral; +import io.trino.sql.tree.Offset; +import io.trino.sql.tree.OrderBy; import io.trino.sql.tree.Parameter; import io.trino.sql.tree.Predicated; import io.trino.sql.tree.QualifiedName; @@ -65,6 +68,8 @@ import io.trino.sql.tree.SelectItem; import io.trino.sql.tree.SimpleCaseExpression; import io.trino.sql.tree.SingleColumn; +import io.trino.sql.tree.SortItem.NullOrdering; +import io.trino.sql.tree.SortItem.Ordering; import io.trino.sql.tree.Statement; import io.trino.sql.tree.StringLiteral; import io.trino.sql.tree.Table; @@ -83,7 +88,7 @@ public static Statement createStatement(HogQlQuery query, Map createSelectItem(projection, parameterIds)) .toList()), query.from().map(TrinoAstFactory::createRelation), @@ -91,9 +96,9 @@ public static Statement createStatement(HogQlQuery query, Map new Offset(location(offset.span()), createExpression(offset, parameterIds))), + query.limit().map(limit -> new Limit(location(limit.span()), createExpression(limit, parameterIds)))); return new Query( location, List.of(), @@ -105,6 +110,29 @@ public static Statement createStatement(HogQlQuery query, Map createOrderBy(HogQlQuery query, Map parameterIds) + { + if (query.orderBy().isEmpty()) { + return Optional.empty(); + } + return Optional.of(new OrderBy( + location(query.orderBy().getFirst().span()), + query.orderBy().stream() + .map(sortItem -> new io.trino.sql.tree.SortItem( + location(sortItem.span()), + createExpression(sortItem.expression(), parameterIds), + switch (sortItem.direction()) { + case ASCENDING -> Ordering.ASCENDING; + case DESCENDING -> Ordering.DESCENDING; + }, + switch (sortItem.nullPlacement()) { + case FIRST -> NullOrdering.FIRST; + case LAST -> NullOrdering.LAST; + case UNDEFINED -> NullOrdering.UNDEFINED; + })) + .toList())); + } + private static SelectItem createSelectItem(Projection projection, Map parameterIds) { return switch (projection) { diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index eb05d4689954..262ca2a735a7 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -76,6 +76,14 @@ public void testLowersToEquivalentStockTrinoAst(String hogql) assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); } + @Test + public void testLowersDistinctOrderingAndClickHousePaginationOrder() + { + Statement statement = compiler.compile("SELECT DISTINCT event FROM events ORDER BY event DESC NULLS FIRST LIMIT 10 OFFSET 2"); + + assertThat(statement).isEqualTo(sqlParser.createStatement("SELECT DISTINCT event FROM events ORDER BY event DESC NULLS FIRST OFFSET 2 LIMIT 10")); + } + @ParameterizedTest @CsvSource(delimiter = '|', textBlock = """ SELECT [1, 2, 3] | SELECT ARRAY[1, 2, 3] diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index d40394b02c67..4940d36caa50 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -31,9 +31,12 @@ import io.trino.hogql.parser.tree.HogQlQuery.InExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; +import io.trino.hogql.parser.tree.HogQlQuery.SortItem; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.Star; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; @@ -440,7 +443,8 @@ private AstBuilder(String source) public HogQlQuery build(HogQLParser.SelectContext context) { - HogQLParser.SelectStmtContext select = extractPlainSelect(context); + SelectedQuery selectedQuery = extractPlainSelect(context); + HogQLParser.SelectStmtContext select = selectedQuery.statement(); rejectUnsupportedClauses(select); List projections = selectColumns(select.selectColumnExprListBeforeFrom()).stream() @@ -452,7 +456,20 @@ public HogQlQuery build(HogQLParser.SelectContext context) Optional where = Optional.ofNullable(select.whereClause()) .map(HogQLParser.WhereClauseContext::columnExpr) .map(this::buildExpression); - return new HogQlQuery(projections, from, where, sourceSpan(select)); + List orderBy = buildOrderBy(selectedQuery.orderBy() != null ? selectedQuery.orderBy() : select.orderByClause()); + Pagination pagination = mergePagination( + buildPagination(select.limitAndOffsetClause(), select.offsetOnlyClause()), + selectedQuery.pagination() == null ? new Pagination(Optional.empty(), Optional.empty()) : buildPagination(selectedQuery.pagination()), + select); + return new HogQlQuery( + select.DISTINCT() != null, + projections, + from, + where, + orderBy, + pagination.limit(), + pagination.offset(), + sourceSpan(select)); } private List selectColumns(HogQLParser.SelectColumnExprListBeforeFromContext context) @@ -466,22 +483,27 @@ private List selectColumns(HogQLParser.Sele throw unsupported(context, "select list"); } - private HogQLParser.SelectStmtContext extractPlainSelect(HogQLParser.SelectContext context) + private SelectedQuery extractPlainSelect(HogQLParser.SelectContext context) { if (context.selectStmt() != null) { - return context.selectStmt(); + return new SelectedQuery(context.selectStmt(), null, null); } HogQLParser.SelectSetStmtContext set = context.selectSetStmt(); - if (set == null || !set.subsequentSelectSetClause().isEmpty() || set.orderByClause() != null || set.limitAndOffsetClauseOptional() != null) { + if (set == null || !set.subsequentSelectSetClause().isEmpty()) { throw unsupported(context, "set query"); } HogQLParser.SelectStmtWithParensContext wrapped = set.selectStmtWithParens(); if (wrapped.selectStmt() == null) { throw unsupported(wrapped, "parenthesized or placeholder query"); } - return wrapped.selectStmt(); + return new SelectedQuery(wrapped.selectStmt(), set.orderByClause(), set.limitAndOffsetClauseOptional()); } + private record SelectedQuery( + HogQLParser.SelectStmtContext statement, + HogQLParser.OrderByClauseContext orderBy, + HogQLParser.LimitAndOffsetClauseOptionalContext pagination) {} + private void rejectUnsupportedClauses(HogQLParser.SelectStmtContext context) { List clauses = new ArrayList<>(); @@ -494,10 +516,7 @@ private void rejectUnsupportedClauses(HogQLParser.SelectStmtContext context) clauses.add(context.havingClause()); clauses.add(context.qualifyClause()); clauses.add(context.windowClause()); - clauses.add(context.orderByClause()); clauses.add(context.limitByClause()); - clauses.add(context.limitAndOffsetClause()); - clauses.add(context.offsetOnlyClause()); clauses.add(context.settingsClause()); Optional firstClause = clauses.stream() .filter(requireNonNullClause -> requireNonNullClause != null) @@ -505,11 +524,98 @@ private void rejectUnsupportedClauses(HogQLParser.SelectStmtContext context) if (firstClause.isPresent()) { throw unsupported(firstClause.orElseThrow(), "query clause"); } - if (context.DISTINCT() != null) { - throw unsupported(context, "distinct query"); + } + + private List buildOrderBy(HogQLParser.OrderByClauseContext context) + { + if (context == null) { + return List.of(); + } + if (context.interpolateClause() != null) { + throw unsupported(context.interpolateClause(), "ORDER BY interpolation"); + } + return context.orderExprList().orderExpr().stream() + .map(this::buildSortItem) + .toList(); + } + + private SortItem buildSortItem(HogQLParser.OrderExprContext context) + { + if (context.COLLATE() != null || context.withFillClause() != null) { + throw unsupported(context, "ORDER BY collation or fill"); + } + SortDirection direction = context.DESC() != null || context.DESCENDING() != null + ? SortDirection.DESCENDING + : SortDirection.ASCENDING; + NullPlacement nullPlacement = context.FIRST() != null + ? NullPlacement.FIRST + : context.LAST() != null ? NullPlacement.LAST : NullPlacement.UNDEFINED; + return new SortItem(buildExpression(context.columnExpr()), direction, nullPlacement, sourceSpan(context)); + } + + private Pagination buildPagination( + HogQLParser.LimitAndOffsetClauseContext limitContext, + HogQLParser.OffsetOnlyClauseContext offsetContext) + { + if (limitContext == null) { + return new Pagination(Optional.empty(), Optional.ofNullable(offsetContext) + .map(HogQLParser.OffsetOnlyClauseContext::columnExpr) + .map(this::buildPaginationExpression)); + } + if (limitContext.PERCENT() != null || limitContext.WITH() != null) { + throw unsupported(limitContext, "percent limit or ties"); + } + List expressions = limitContext.columnExpr(); + if (limitContext.COMMA() != null) { + return new Pagination( + Optional.of(buildPaginationExpression(expressions.get(1))), + Optional.of(buildPaginationExpression(expressions.getFirst()))); } + return new Pagination( + Optional.of(buildPaginationExpression(expressions.getFirst())), + expressions.size() == 2 ? Optional.of(buildPaginationExpression(expressions.get(1))) : Optional.empty()); } + private Pagination buildPagination(HogQLParser.LimitAndOffsetClauseOptionalContext context) + { + if (context.PERCENT() != null || context.WITH() != null) { + throw unsupported(context, "percent limit or ties"); + } + List expressions = context.columnExpr(); + if (context.LIMIT() == null) { + return new Pagination(Optional.empty(), Optional.of(buildPaginationExpression(expressions.getFirst()))); + } + if (context.COMMA() != null) { + return new Pagination( + Optional.of(buildPaginationExpression(expressions.get(1))), + Optional.of(buildPaginationExpression(expressions.getFirst()))); + } + return new Pagination( + Optional.of(buildPaginationExpression(expressions.getFirst())), + expressions.size() == 2 ? Optional.of(buildPaginationExpression(expressions.get(1))) : Optional.empty()); + } + + private Pagination mergePagination(Pagination inner, Pagination outer, ParserRuleContext context) + { + if (inner.limit().isPresent() && outer.limit().isPresent() || inner.offset().isPresent() && outer.offset().isPresent()) { + throw unsupported(context, "duplicate pagination"); + } + return new Pagination( + outer.limit().or(() -> inner.limit()), + outer.offset().or(() -> inner.offset())); + } + + private Expression buildPaginationExpression(HogQLParser.ColumnExprContext context) + { + Expression expression = buildExpression(context); + if (expression instanceof Literal literal && literal.kind() == INTEGER || expression instanceof Placeholder) { + return expression; + } + throw unsupported(context, "non-constant pagination"); + } + + private record Pagination(Optional limit, Optional offset) {} + private Projection buildProjection(HogQLParser.SelectColumnExprContext context) { HogQLParser.ColumnExprContext expression; diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index 89f71a1ec7c3..02616dc80ef1 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -18,16 +18,51 @@ import static java.util.Objects.requireNonNull; -public record HogQlQuery(List projections, Optional from, Optional where, SourceSpan span) +public record HogQlQuery( + boolean distinct, + List projections, + Optional from, + Optional where, + List orderBy, + Optional limit, + Optional offset, + SourceSpan span) { public HogQlQuery { projections = List.copyOf(requireNonNull(projections, "projections is null")); from = requireNonNull(from, "from is null"); where = requireNonNull(where, "where is null"); + orderBy = List.copyOf(requireNonNull(orderBy, "orderBy is null")); + limit = requireNonNull(limit, "limit is null"); + offset = requireNonNull(offset, "offset is null"); span = requireNonNull(span, "span is null"); } + public record SortItem(Expression expression, SortDirection direction, NullPlacement nullPlacement, SourceSpan span) + { + public SortItem + { + expression = requireNonNull(expression, "expression is null"); + direction = requireNonNull(direction, "direction is null"); + nullPlacement = requireNonNull(nullPlacement, "nullPlacement is null"); + span = requireNonNull(span, "span is null"); + } + } + + public enum SortDirection + { + ASCENDING, + DESCENDING, + } + + public enum NullPlacement + { + FIRST, + LAST, + UNDEFINED, + } + public sealed interface Projection permits ExpressionProjection, Star { diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index a22adc95fb5a..8b772d37d4c4 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -17,7 +17,9 @@ import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; import io.trino.hogql.parser.tree.HogQlSyntaxTree; import io.trino.hogql.parser.tree.HogQlSyntaxTree.Element; import io.trino.hogql.parser.tree.HogQlSyntaxTree.Node; @@ -79,6 +81,20 @@ public void testPreservesNamedPlaceholderNamesAndSourceSpans() assertThat(repeated.span()).isEqualTo(new HogQlQuery.SourceSpan(27, 34, 2, 12, 2, 19)); } + @Test + public void testBuildsDistinctOrderingAndPaginationAst() + { + HogQlQuery query = parser.parseStatement("SELECT DISTINCT event FROM events ORDER BY event DESC NULLS FIRST LIMIT 10 OFFSET 2"); + + assertThat(query.distinct()).isTrue(); + assertThat(query.orderBy()).singleElement().satisfies(sortItem -> { + assertThat(sortItem.direction()).isEqualTo(SortDirection.DESCENDING); + assertThat(sortItem.nullPlacement()).isEqualTo(NullPlacement.FIRST); + }); + assertThat(query.limit()).get().extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("10"); + assertThat(query.offset()).get().extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("2"); + } + @Test public void testSourceSpansUseUnicodeCodePointOffsetsAndEndPositions() { From 822e824d073d2e98b527ad47e124a002febe3759 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 10:38:19 -0400 Subject: [PATCH 029/136] feat(hogql): bind typed parameter values --- .../execution/HogQlParameterDecoder.java | 523 ++++++++++++++++++ .../io/trino/execution/QueryPreparer.java | 36 +- .../io/trino/execution/TestQueryPreparer.java | 117 +++- 3 files changed, 665 insertions(+), 11 deletions(-) create mode 100644 core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java diff --git a/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java b/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java new file mode 100644 index 000000000000..bf808ffd97d9 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java @@ -0,0 +1,523 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.BooleanNode; +import com.fasterxml.jackson.databind.node.DecimalNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.google.common.collect.ImmutableList; +import com.google.common.net.InetAddresses; +import io.trino.hogql.compiler.HogQlCompilationResult; +import io.trino.hogql.compiler.HogQlTypedValue; +import io.trino.hogql.compiler.HogQlTypedValue.ArrayValue; +import io.trino.hogql.compiler.HogQlTypedValue.BooleanValue; +import io.trino.hogql.compiler.HogQlTypedValue.NullValue; +import io.trino.hogql.compiler.HogQlTypedValue.NumberValue; +import io.trino.hogql.compiler.HogQlTypedValue.ObjectValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.hogql.compiler.HogQlTypedValue.Value; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import io.trino.sql.tree.Array; +import io.trino.sql.tree.BinaryLiteral; +import io.trino.sql.tree.BooleanLiteral; +import io.trino.sql.tree.Cast; +import io.trino.sql.tree.DataType; +import io.trino.sql.tree.DataTypeParameter; +import io.trino.sql.tree.DateTimeDataType; +import io.trino.sql.tree.DecimalLiteral; +import io.trino.sql.tree.DoubleLiteral; +import io.trino.sql.tree.Expression; +import io.trino.sql.tree.FunctionCall; +import io.trino.sql.tree.GenericDataType; +import io.trino.sql.tree.GenericLiteral; +import io.trino.sql.tree.LongLiteral; +import io.trino.sql.tree.NodeLocation; +import io.trino.sql.tree.NullLiteral; +import io.trino.sql.tree.NumericParameter; +import io.trino.sql.tree.Parameter; +import io.trino.sql.tree.QualifiedName; +import io.trino.sql.tree.Row; +import io.trino.sql.tree.RowDataType; +import io.trino.sql.tree.StringLiteral; +import io.trino.sql.tree.TypeParameter; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.OffsetTime; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_BINDING_ERROR; +import static io.trino.spi.type.CharType.MAX_LENGTH; +import static java.lang.Float.isFinite; +import static java.lang.Math.toIntExact; +import static java.util.Objects.requireNonNull; + +final class HogQlParameterDecoder +{ + private static final Set INTEGRAL_TYPES = Set.of("TINYINT", "SMALLINT", "INTEGER", "BIGINT"); + private static final Set STRING_TYPES = Set.of("VARCHAR", "CHAR", "DATE", "UUID", "IPADDRESS", "VARBINARY"); + + private final SqlParser sqlParser; + + HogQlParameterDecoder(SqlParser sqlParser) + { + this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + } + + List decode(HogQlCompilationResult result, Map bindings) + { + requireNonNull(result, "result is null"); + requireNonNull(bindings, "bindings is null"); + List parameters = ParameterExtractor.extractParameters(result.statement()); + if (parameters.size() != result.parameterNames().size()) { + throw new IllegalStateException("HogQL compiler returned inconsistent parameter metadata"); + } + + ImmutableList.Builder values = ImmutableList.builderWithExpectedSize(parameters.size()); + for (int index = 0; index < parameters.size(); index++) { + String name = result.parameterNames().get(index); + NodeLocation location = parameters.get(index).getLocation().orElseThrow(); + try { + HogQlTypedValue binding = requireNonNull(bindings.get(name), "binding is null"); + DataType type = sqlParser.createType(binding.type()); + values.add(decode(binding.value(), type, location)); + } + catch (RuntimeException _) { + throw bindingError(name, location); + } + } + return values.build(); + } + + private static Expression decode(Value value, DataType type, NodeLocation location) + { + if (value instanceof NullValue) { + validateSupportedType(type); + return new Cast(location, new NullLiteral(location), type); + } + if (type instanceof DateTimeDataType dateTimeType) { + return new Cast(location, decodeDateTime(value, dateTimeType, location), type); + } + if (type instanceof RowDataType rowType) { + return decodeRow(value, rowType, location); + } + if (!(type instanceof GenericDataType genericType)) { + throw new IllegalArgumentException("unsupported type"); + } + + String typeName = genericType.getName().getCanonicalValue(); + if (typeName.equals("JSON")) { + requireNoArguments(genericType); + return new GenericLiteral(location, "JSON", toJson(value).toString()); + } + if (typeName.equals("ARRAY")) { + return decodeArray(value, genericType, location); + } + if (typeName.equals("MAP")) { + return decodeMap(value, genericType, location); + } + + Expression scalar = switch (value) { + case BooleanValue booleanValue -> decodeBoolean(booleanValue, genericType, location); + case NumberValue numberValue -> decodeNumber(numberValue, genericType, location); + case StringValue stringValue -> decodeString(stringValue, genericType, location); + case ArrayValue _, ObjectValue _ -> throw new IllegalArgumentException("container value requires a container type"); + case NullValue _ -> throw new IllegalStateException("null handled above"); + }; + return new Cast(location, scalar, type); + } + + private static Expression decodeBoolean(BooleanValue value, GenericDataType type, NodeLocation location) + { + requireType(type, "BOOLEAN"); + return new BooleanLiteral(location, Boolean.toString(value.value())); + } + + private static Expression decodeNumber(NumberValue value, GenericDataType type, NodeLocation location) + { + String typeName = type.getName().getCanonicalValue(); + BigDecimal number = new BigDecimal(value.value()); + if (INTEGRAL_TYPES.contains(typeName)) { + requireNoArguments(type); + BigInteger integer = number.toBigIntegerExact(); + validateIntegerRange(typeName, integer); + return new LongLiteral(location, integer.toString()); + } + if (typeName.equals("REAL") || typeName.equals("DOUBLE")) { + requireNoArguments(type); + double doubleValue = number.doubleValue(); + if (!Double.isFinite(doubleValue) || (typeName.equals("REAL") && !isFinite(number.floatValue()))) { + throw new IllegalArgumentException("floating-point value is out of range"); + } + return new DoubleLiteral(location, value.value()); + } + if (typeName.equals("DECIMAL")) { + List arguments = validateDecimalType(type); + int precision = toIntExact(arguments.get(0)); + int scale = toIntExact(arguments.get(1)); + BigDecimal scaled = number.setScale(scale, RoundingMode.UNNECESSARY); + if (scaled.precision() > precision) { + throw new IllegalArgumentException("decimal value is out of range"); + } + return new DecimalLiteral(location, value.value()); + } + throw new IllegalArgumentException("number value has an incompatible type"); + } + + private static Expression decodeString(StringValue value, GenericDataType type, NodeLocation location) + { + String typeName = type.getName().getCanonicalValue(); + if (!STRING_TYPES.contains(typeName)) { + throw new IllegalArgumentException("string value has an incompatible type"); + } + + return switch (typeName) { + case "VARCHAR" -> { + validateStringLength(value.value(), type, false); + yield new StringLiteral(location, value.value()); + } + case "CHAR" -> { + validateStringLength(value.value(), type, true); + yield new StringLiteral(location, value.value()); + } + case "DATE" -> { + requireNoArguments(type); + LocalDate.parse(value.value()); + yield new GenericLiteral(location, "DATE", value.value()); + } + case "UUID" -> { + requireNoArguments(type); + UUID.fromString(value.value()); + yield new GenericLiteral(location, "UUID", value.value()); + } + case "IPADDRESS" -> { + requireNoArguments(type); + InetAddresses.forString(value.value()); + yield new GenericLiteral(location, "IPADDRESS", value.value()); + } + case "VARBINARY" -> { + requireNoArguments(type); + yield new BinaryLiteral(location, value.value()); + } + default -> throw new IllegalArgumentException("unsupported string type"); + }; + } + + private static Expression decodeDateTime(Value value, DateTimeDataType type, NodeLocation location) + { + if (!(value instanceof StringValue stringValue)) { + throw new IllegalArgumentException("date-time value must be a string"); + } + validateDateTimePrecision(type); + String normalized = stringValue.value().replace(' ', 'T'); + try { + if (type.getType() == DateTimeDataType.Type.TIMESTAMP) { + if (type.isWithTimeZone()) { + OffsetDateTime.parse(normalized); + } + else { + LocalDateTime.parse(normalized); + } + } + else if (type.isWithTimeZone()) { + OffsetTime.parse(normalized); + } + else { + java.time.LocalTime.parse(normalized); + } + } + catch (DateTimeParseException _) { + throw new IllegalArgumentException("date-time value is invalid"); + } + return new GenericLiteral(location, type.getType().name(), stringValue.value()); + } + + private static Expression decodeArray(Value value, GenericDataType type, NodeLocation location) + { + if (!(value instanceof ArrayValue arrayValue)) { + throw new IllegalArgumentException("array value is required"); + } + DataType elementType = typeArgument(type, 0, 1); + List elements = arrayValue.value().stream() + .map(element -> decode(element, elementType, location)) + .toList(); + return new Cast(location, new Array(location, elements), type); + } + + private static Expression decodeMap(Value value, GenericDataType type, NodeLocation location) + { + if (!(value instanceof ObjectValue objectValue)) { + throw new IllegalArgumentException("object value is required"); + } + DataType keyType = typeArgument(type, 0, 2); + DataType valueType = typeArgument(type, 1, 2); + if (!(keyType instanceof GenericDataType genericKeyType) || !genericKeyType.getName().getCanonicalValue().equals("VARCHAR")) { + throw new IllegalArgumentException("object keys require varchar map keys"); + } + validateStringType(genericKeyType, false); + + List> entries = objectValue.value().entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .toList(); + FunctionCall map = entries.isEmpty() + ? new FunctionCall(location, QualifiedName.of("map"), List.of()) + : new FunctionCall( + location, + QualifiedName.of("map"), + List.of( + new Array(location, entries.stream() + .map(entry -> decode(new StringValue(entry.getKey()), keyType, location)) + .toList()), + new Array(location, entries.stream() + .map(entry -> decode(entry.getValue(), valueType, location)) + .toList()))); + return new Cast(location, map, type); + } + + private static Expression decodeRow(Value value, RowDataType type, NodeLocation location) + { + List fieldValues; + if (value instanceof ObjectValue objectValue) { + if (type.getFields().stream().anyMatch(field -> field.getName().isEmpty())) { + throw new IllegalArgumentException("object values require named row fields"); + } + Set fieldNames = type.getFields().stream() + .map(field -> field.getName().orElseThrow().getValue()) + .collect(java.util.stream.Collectors.toUnmodifiableSet()); + if (!fieldNames.equals(objectValue.value().keySet())) { + throw new IllegalArgumentException("row fields do not match"); + } + fieldValues = type.getFields().stream() + .map(field -> objectValue.value().get(field.getName().orElseThrow().getValue())) + .toList(); + } + else if (value instanceof ArrayValue arrayValue && arrayValue.value().size() == type.getFields().size()) { + fieldValues = arrayValue.value(); + } + else { + throw new IllegalArgumentException("row value has an incompatible shape"); + } + + List fields = new ArrayList<>(fieldValues.size()); + for (int index = 0; index < fieldValues.size(); index++) { + fields.add(new Row.Field(location, Optional.empty(), decode(fieldValues.get(index), type.getFields().get(index).getType(), location))); + } + return new Cast(location, new Row(location, fields), type); + } + + private static JsonNode toJson(Value value) + { + return switch (value) { + case NullValue _ -> NullNode.getInstance(); + case BooleanValue booleanValue -> BooleanNode.valueOf(booleanValue.value()); + case NumberValue numberValue -> DecimalNode.valueOf(new BigDecimal(numberValue.value())); + case StringValue stringValue -> TextNode.valueOf(stringValue.value()); + case ArrayValue arrayValue -> { + ArrayNode array = JsonNodeFactory.instance.arrayNode(); + arrayValue.value().forEach(element -> array.add(toJson(element))); + yield array; + } + case ObjectValue objectValue -> { + ObjectNode object = JsonNodeFactory.instance.objectNode(); + objectValue.value().entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> object.set(entry.getKey(), toJson(entry.getValue()))); + yield object; + } + }; + } + + private static void validateSupportedType(DataType type) + { + if (type instanceof DateTimeDataType dateTimeType) { + validateDateTimePrecision(dateTimeType); + return; + } + if (type instanceof RowDataType rowType) { + rowType.getFields().forEach(field -> validateSupportedType(field.getType())); + return; + } + if (!(type instanceof GenericDataType genericType)) { + throw new IllegalArgumentException("unsupported type"); + } + String typeName = genericType.getName().getCanonicalValue(); + if (Set.of("BOOLEAN", "TINYINT", "SMALLINT", "INTEGER", "BIGINT", "REAL", "DOUBLE", "DATE", "UUID", "IPADDRESS", "VARBINARY", "JSON").contains(typeName)) { + requireNoArguments(genericType); + return; + } + if (typeName.equals("VARCHAR")) { + validateStringType(genericType, false); + return; + } + if (typeName.equals("CHAR")) { + validateStringType(genericType, true); + return; + } + if (typeName.equals("DECIMAL")) { + validateDecimalType(genericType); + return; + } + if (typeName.equals("ARRAY")) { + validateSupportedType(typeArgument(genericType, 0, 1)); + return; + } + if (typeName.equals("MAP")) { + validateSupportedType(typeArgument(genericType, 0, 2)); + validateSupportedType(typeArgument(genericType, 1, 2)); + return; + } + throw new IllegalArgumentException("unsupported type"); + } + + private static void validateIntegerRange(String type, BigInteger value) + { + BigInteger minimum; + BigInteger maximum; + switch (type) { + case "TINYINT" -> { + minimum = BigInteger.valueOf(Byte.MIN_VALUE); + maximum = BigInteger.valueOf(Byte.MAX_VALUE); + } + case "SMALLINT" -> { + minimum = BigInteger.valueOf(Short.MIN_VALUE); + maximum = BigInteger.valueOf(Short.MAX_VALUE); + } + case "INTEGER" -> { + minimum = BigInteger.valueOf(Integer.MIN_VALUE); + maximum = BigInteger.valueOf(Integer.MAX_VALUE); + } + case "BIGINT" -> { + minimum = BigInteger.valueOf(Long.MIN_VALUE); + maximum = BigInteger.valueOf(Long.MAX_VALUE); + } + default -> throw new IllegalArgumentException("unsupported integral type"); + } + if (value.compareTo(minimum) < 0 || value.compareTo(maximum) > 0) { + throw new IllegalArgumentException("integral value is out of range"); + } + } + + private static void validateStringLength(String value, GenericDataType type, boolean lengthRequired) + { + validateStringType(type, lengthRequired); + if (!type.getArguments().isEmpty()) { + long maximumLength = numericArguments(type, 1).getFirst(); + if (value.codePointCount(0, value.length()) > maximumLength) { + throw new IllegalArgumentException("string value is too long"); + } + } + } + + private static void validateStringType(GenericDataType type, boolean lengthRequired) + { + int argumentCount = type.getArguments().size(); + if ((lengthRequired && argumentCount != 1) || (!lengthRequired && argumentCount > 1)) { + throw new IllegalArgumentException("invalid string type"); + } + if (argumentCount == 1) { + long length = numericArguments(type, 1).getFirst(); + long maximumLength = type.getName().getCanonicalValue().equals("CHAR") ? MAX_LENGTH : io.trino.spi.type.VarcharType.MAX_LENGTH; + if (length < 0 || length > maximumLength) { + throw new IllegalArgumentException("invalid string length"); + } + } + } + + private static List validateDecimalType(GenericDataType type) + { + List arguments = numericArguments(type, 2); + long precision = arguments.get(0); + long scale = arguments.get(1); + if (precision < 1 || precision > 38 || scale < 0 || scale > precision) { + throw new IllegalArgumentException("invalid decimal type"); + } + return arguments; + } + + private static void validateDateTimePrecision(DateTimeDataType type) + { + type.getPrecision().ifPresent(precision -> { + if (!(precision instanceof NumericParameter numericPrecision) || numericPrecision.getParsedValue() < 0 || numericPrecision.getParsedValue() > 12) { + throw new IllegalArgumentException("invalid date-time precision"); + } + }); + } + + private static void requireType(GenericDataType type, String expected) + { + if (!type.getName().getCanonicalValue().equals(expected)) { + throw new IllegalArgumentException("value has an incompatible type"); + } + requireNoArguments(type); + } + + private static void requireNoArguments(GenericDataType type) + { + if (!type.getArguments().isEmpty()) { + throw new IllegalArgumentException("type does not accept arguments"); + } + } + + private static DataType typeArgument(GenericDataType type, int index, int expectedCount) + { + if (type.getArguments().size() != expectedCount || !(type.getArguments().get(index) instanceof TypeParameter typeParameter)) { + throw new IllegalArgumentException("invalid container type"); + } + return typeParameter.getValue(); + } + + private static List numericArguments(GenericDataType type, int expectedCount) + { + if (type.getArguments().size() != expectedCount) { + throw new IllegalArgumentException("invalid type parameters"); + } + return type.getArguments().stream() + .map(HogQlParameterDecoder::numericArgument) + .toList(); + } + + private static long numericArgument(DataTypeParameter parameter) + { + if (!(parameter instanceof NumericParameter numericParameter)) { + throw new IllegalArgumentException("numeric type parameter is required"); + } + return numericParameter.getParsedValue(); + } + + private static TrinoException bindingError(String name, NodeLocation location) + { + return new TrinoException( + HOGQL_BINDING_ERROR, + Optional.of(new Location(location.getLineNumber(), location.getColumnNumber())), + "Invalid HogQL parameter binding: " + name, + null); + } +} diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java index 56340dd91866..8e05c64ebde2 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java @@ -16,6 +16,8 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Inject; import io.trino.Session; +import io.trino.hogql.compiler.HogQlCompilationResult; +import io.trino.hogql.compiler.HogQlCompileEnvelope; import io.trino.hogql.compiler.HogQlCompiler; import io.trino.spi.TrinoException; import io.trino.spi.resourcegroups.QueryType; @@ -43,6 +45,7 @@ public class QueryPreparer { private final SqlParser sqlParser; private final Optional hogQlCompiler; + private final HogQlParameterDecoder hogQlParameterDecoder; public QueryPreparer(SqlParser sqlParser) { @@ -59,6 +62,7 @@ private QueryPreparer(SqlParser sqlParser, Optional hogQlCompiler { this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); this.hogQlCompiler = requireNonNull(hogQlCompiler, "hogQlCompiler is null"); + this.hogQlParameterDecoder = new HogQlParameterDecoder(sqlParser); } public PreparedQuery prepareQuery(Session session, String query) @@ -71,18 +75,26 @@ public PreparedQuery prepareQuery(Session session, QuerySubmission submission) throws ParsingException, TrinoException { requireNonNull(submission, "submission is null"); - Statement wrappedStatement = switch (submission.language()) { - case TRINO -> sqlParser.createStatement(submission.originalText()); - case HOGQL -> hogQlCompiler - .orElseThrow(() -> new TrinoException(NOT_SUPPORTED, "HogQL query submission is disabled")) - .compile(submission.hogQlEnvelope().orElseThrow()) - .statement(); + return switch (submission.language()) { + case TRINO -> prepareQuery(session, sqlParser.createStatement(submission.originalText())); + case HOGQL -> { + HogQlCompileEnvelope envelope = submission.hogQlEnvelope().orElseThrow(); + HogQlCompilationResult result = hogQlCompiler + .orElseThrow(() -> new TrinoException(NOT_SUPPORTED, "HogQL query submission is disabled")) + .compile(envelope); + yield prepareQuery(session, result.statement(), Optional.of(hogQlParameterDecoder.decode(result, envelope.parameters()))); + } }; - return prepareQuery(session, wrappedStatement); } public PreparedQuery prepareQuery(Session session, Statement wrappedStatement) throws ParsingException, TrinoException + { + return prepareQuery(session, wrappedStatement, Optional.empty()); + } + + private PreparedQuery prepareQuery(Session session, Statement wrappedStatement, Optional> suppliedParameters) + throws ParsingException, TrinoException { Statement statement = wrappedStatement; Optional prepareSql = Optional.empty(); @@ -103,13 +115,19 @@ else if (statement instanceof ExplainAnalyze explainAnalyzeStatement) { } } - List parameters = ImmutableList.of(); - if (wrappedStatement instanceof Execute executeStatement) { + List parameters; + if (suppliedParameters.isPresent()) { + parameters = suppliedParameters.orElseThrow(); + } + else if (wrappedStatement instanceof Execute executeStatement) { parameters = executeStatement.getParameters(); } else if (wrappedStatement instanceof ExecuteImmediate executeImmediateStatement) { parameters = executeImmediateStatement.getParameters(); } + else { + parameters = ImmutableList.of(); + } validateParameters(statement, parameters); return new PreparedQuery(statement, parameters, prepareSql); diff --git a/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java b/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java index ec3103ced079..ae170d4ee532 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java @@ -17,17 +17,36 @@ import io.trino.execution.QueryPreparer.PreparedQuery; import io.trino.hogql.compiler.HogQlCompileEnvelope; import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.HogQlErrorCode; +import io.trino.hogql.compiler.HogQlTypedValue; +import io.trino.hogql.compiler.HogQlTypedValue.ArrayValue; +import io.trino.hogql.compiler.HogQlTypedValue.BooleanValue; +import io.trino.hogql.compiler.HogQlTypedValue.NullValue; +import io.trino.hogql.compiler.HogQlTypedValue.NumberValue; +import io.trino.hogql.compiler.HogQlTypedValue.ObjectValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.sql.SqlFormatter; import io.trino.sql.parser.ParsingException; import io.trino.sql.parser.SqlParser; import io.trino.sql.tree.AllColumns; +import io.trino.sql.tree.Array; +import io.trino.sql.tree.Cast; +import io.trino.sql.tree.FunctionCall; +import io.trino.sql.tree.GenericLiteral; import io.trino.sql.tree.QualifiedName; import io.trino.sql.tree.Query; +import io.trino.sql.tree.Row; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.OptionalLong; +import java.util.stream.Stream; import static io.trino.SessionTestUtils.TEST_SESSION; import static io.trino.execution.QuerySubmission.hogQl; @@ -42,6 +61,7 @@ import static io.trino.testing.assertions.TrinoExceptionAssert.assertTrinoExceptionThrownBy; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.params.provider.Arguments.arguments; public class TestQueryPreparer { @@ -68,6 +88,75 @@ public void testHogQlStatement() assertThat(submission.hogQlEnvelope()).containsSame(envelope); } + @Test + public void testHogQlTypedParametersUseThePreparedValueBoundary() + { + Map parameters = new LinkedHashMap<>(); + parameters.put("json_value", typedValue("json", new ObjectValue(Map.of("enabled", new BooleanValue(true), "items", new ArrayValue(List.of(new NumberValue("1"), NullValue.NULL)))))); + parameters.put("row_value", typedValue("row(label varchar, count bigint)", new ObjectValue(Map.of("count", new NumberValue("2"), "label", new StringValue("synthetic-row"))))); + parameters.put("map_value", typedValue("map(varchar, bigint)", new ObjectValue(Map.of("second", NullValue.NULL, "first", new NumberValue("1"))))); + parameters.put("array_value", typedValue("array(bigint)", new ArrayValue(List.of(new NumberValue("1"), NullValue.NULL, new NumberValue("2"))))); + parameters.put("uuid_value", typedValue("uuid", new StringValue("018f6b9d-89f4-7e8a-8f5d-4c621e5d4a33"))); + parameters.put("timestamp_value", typedValue("timestamp(3)", new StringValue("2026-08-27 12:34:56.789"))); + parameters.put("date_value", typedValue("date", new StringValue("2026-08-27"))); + parameters.put("varchar_value", typedValue("varchar", new StringValue("synthetic-text"))); + parameters.put("decimal_value", typedValue("decimal(4, 2)", new NumberValue("12.30"))); + parameters.put("double_value", typedValue("double", new NumberValue("1.25"))); + parameters.put("bigint_value", typedValue("bigint", new NumberValue("42"))); + parameters.put("boolean_value", typedValue("boolean", new BooleanValue(true))); + + PreparedQuery preparedQuery = HOGQL_QUERY_PREPARER.prepareQuery( + TEST_SESSION, + hogQl(envelope( + "SELECT {boolean_value}, {bigint_value}, {double_value}, {decimal_value}, {varchar_value}, {date_value}, {timestamp_value}, {uuid_value}, {array_value}, {map_value}, {row_value}, {json_value}, {varchar_value}", + parameters))); + + assertThat(preparedQuery.getParameters()).hasSize(13); + assertThat(preparedQuery.getParameters()) + .extracting(expression -> expression.getLocation()) + .containsExactlyElementsOf(ParameterExtractor.extractParameters(preparedQuery.getStatement()).stream() + .map(parameter -> parameter.getLocation()) + .toList()); + assertThat(preparedQuery.getParameters().subList(0, 11)).allMatch(Cast.class::isInstance); + assertThat(preparedQuery.getParameters().subList(0, 11)) + .extracting(expression -> ((Cast) expression).getType()) + .containsExactly( + SQL_PARSER.createType("boolean"), + SQL_PARSER.createType("bigint"), + SQL_PARSER.createType("double"), + SQL_PARSER.createType("decimal(4, 2)"), + SQL_PARSER.createType("varchar"), + SQL_PARSER.createType("date"), + SQL_PARSER.createType("timestamp(3)"), + SQL_PARSER.createType("uuid"), + SQL_PARSER.createType("array(bigint)"), + SQL_PARSER.createType("map(varchar, bigint)"), + SQL_PARSER.createType("row(label varchar, count bigint)")); + assertThat(((Cast) preparedQuery.getParameters().get(8)).getExpression()).isInstanceOf(Array.class); + assertThat(((Cast) preparedQuery.getParameters().get(9)).getExpression()).isInstanceOf(FunctionCall.class); + assertThat(((Cast) preparedQuery.getParameters().get(10)).getExpression()).isInstanceOf(Row.class); + assertThat(preparedQuery.getParameters().get(11)) + .isInstanceOfSatisfying(GenericLiteral.class, json -> assertThat(json.getValue()).isEqualTo("{\"enabled\":true,\"items\":[1,null]}")); + assertThat(preparedQuery.getParameters().get(12)).isEqualTo(preparedQuery.getParameters().get(4)); + assertThat(SqlFormatter.formatSql(preparedQuery.getStatement())) + .doesNotContain("synthetic-text", "synthetic-row", "018f6b9d") + .contains("?"); + } + + @ParameterizedTest + @MethodSource("invalidHogQlTypedParameters") + public void testHogQlTypedParameterErrorsAreStableAndRedacted(HogQlTypedValue typedValue) + { + String sensitiveValue = "sensitive-binding-value"; + + assertTrinoExceptionThrownBy(() -> HOGQL_QUERY_PREPARER.prepareQuery( + TEST_SESSION, + hogQl(envelope("SELECT {input}", Map.of("input", typedValue))))) + .hasErrorCode(HogQlErrorCode.HOGQL_BINDING_ERROR) + .hasMessage("line 1:8: Invalid HogQL parameter binding: input") + .satisfies(exception -> assertThat(exception.getMessage()).doesNotContain(sensitiveValue)); + } + @Test public void testHogQlSubmissionFailsClosedWithoutCompiler() { @@ -77,18 +166,42 @@ public void testHogQlSubmissionFailsClosedWithoutCompiler() } private static HogQlCompileEnvelope envelope(String query) + { + return envelope(query, Map.of()); + } + + private static HogQlCompileEnvelope envelope(String query, Map parameters) { return new HogQlCompileEnvelope( query, HogQlCompileEnvelope.PROTOCOL_VERSION, HogQlLanguageContract.current().languageVersion(), - Map.of(), + parameters, Map.of(), Map.of(), Map.of(), OptionalLong.empty()); } + private static Stream invalidHogQlTypedParameters() + { + String sensitiveValue = "sensitive-binding-value"; + return Stream.of( + arguments(typedValue(sensitiveValue, new StringValue(sensitiveValue))), + arguments(typedValue("boolean", new StringValue(sensitiveValue))), + arguments(typedValue("tinyint", new NumberValue("128"))), + arguments(typedValue("decimal(3, 1)", new NumberValue("123.4"))), + arguments(typedValue("uuid", new StringValue(sensitiveValue))), + arguments(typedValue("array(bigint)", new ArrayValue(List.of(new StringValue(sensitiveValue))))), + arguments(typedValue("map(bigint, varchar)", new ObjectValue(Map.of("field", new StringValue(sensitiveValue))))), + arguments(typedValue("row(label varchar)", new ObjectValue(Map.of("unexpected", new StringValue(sensitiveValue)))))); + } + + private static HogQlTypedValue typedValue(String type, HogQlTypedValue.Value value) + { + return new HogQlTypedValue(type, value); + } + @Test public void testExecuteStatement() { From f4a8c494ed366cd8cb841061cdc7b399409dd727 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 10:47:44 -0400 Subject: [PATCH 030/136] feat(hogql): refresh semantic catalogs asynchronously --- ...gQlSemanticCatalogSnapshotJsonDecoder.java | 3 +- .../trino/hogql/ForHogQlSemanticCatalog.java | 29 +++ .../hogql/HogQlSemanticCatalogConfig.java | 166 ++++++++++++++++ .../HogQlSemanticCatalogHttpTransport.java | 180 ++++++++++++++++++ .../hogql/HogQlSemanticCatalogManager.java | 132 +++++++++++++ .../hogql/HogQlSemanticCatalogModule.java | 72 +++++++ .../io/trino/server/CoordinatorModule.java | 6 + .../hogql/TestHogQlSemanticCatalogConfig.java | 74 +++++++ ...TestHogQlSemanticCatalogHttpTransport.java | 107 +++++++++++ .../TestHogQlSemanticCatalogManager.java | 110 +++++++++++ 10 files changed, 878 insertions(+), 1 deletion(-) create mode 100644 core/trino-main/src/main/java/io/trino/hogql/ForHogQlSemanticCatalog.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogHttpTransport.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogManager.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogModule.java create mode 100644 core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java create mode 100644 core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogHttpTransport.java create mode 100644 core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java index 7208ad6f0a71..f720c71b0ec7 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java @@ -46,9 +46,10 @@ public final class HogQlSemanticCatalogSnapshotJsonDecoder { public static final int PROTOCOL_VERSION = 1; + public static final int MAXIMUM_PAYLOAD_BYTES = 8 * 1024 * 1024; private static final int SCHEMA_VERSION = 1; - private static final Limits DEFAULT_LIMITS = new Limits(8 * 1024 * 1024, 64, 100_000); + private static final Limits DEFAULT_LIMITS = new Limits(MAXIMUM_PAYLOAD_BYTES, 64, 100_000); private static final Set SNAPSHOT_FIELDS = Set.of( "protocolVersion", diff --git a/core/trino-main/src/main/java/io/trino/hogql/ForHogQlSemanticCatalog.java b/core/trino-main/src/main/java/io/trino/hogql/ForHogQlSemanticCatalog.java new file mode 100644 index 000000000000..65e39faab550 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/ForHogQlSemanticCatalog.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({FIELD, PARAMETER, METHOD}) +@BindingAnnotation +public @interface ForHogQlSemanticCatalog {} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java new file mode 100644 index 000000000000..29d342fd13d8 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java @@ -0,0 +1,166 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; +import io.airlift.units.DataSize; +import io.airlift.units.Duration; +import io.airlift.units.MaxDataSize; +import io.airlift.units.MinDataSize; +import io.airlift.units.MinDuration; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; + +import java.net.URI; + +import static io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; + +public class HogQlSemanticCatalogConfig +{ + private URI uri; + private int maximumEntries = 100; + private Duration refreshAfter = new Duration(1, MINUTES); + private Duration expireAfter = new Duration(5, MINUTES); + private Duration failureBackoff = new Duration(10, SECONDS); + private int loaderThreads = 4; + private int loaderQueueCapacity = 64; + private Duration requestTimeout = new Duration(10, SECONDS); + private DataSize maximumResponseSize = DataSize.ofBytes(MAXIMUM_PAYLOAD_BYTES); + + public URI getUri() + { + return uri; + } + + @Config("hogql.semantic-catalog.uri") + @ConfigDescription("Duckgres HogQL compatibility metadata base URI") + public HogQlSemanticCatalogConfig setUri(URI uri) + { + this.uri = uri; + return this; + } + + @Min(1) + public int getMaximumEntries() + { + return maximumEntries; + } + + @Config("hogql.semantic-catalog.maximum-entries") + public HogQlSemanticCatalogConfig setMaximumEntries(int maximumEntries) + { + this.maximumEntries = maximumEntries; + return this; + } + + @NotNull + @MinDuration("1ms") + public Duration getRefreshAfter() + { + return refreshAfter; + } + + @Config("hogql.semantic-catalog.refresh-after") + public HogQlSemanticCatalogConfig setRefreshAfter(Duration refreshAfter) + { + this.refreshAfter = refreshAfter; + return this; + } + + @NotNull + @MinDuration("1ms") + public Duration getExpireAfter() + { + return expireAfter; + } + + @Config("hogql.semantic-catalog.expire-after") + public HogQlSemanticCatalogConfig setExpireAfter(Duration expireAfter) + { + this.expireAfter = expireAfter; + return this; + } + + @NotNull + @MinDuration("0ms") + public Duration getFailureBackoff() + { + return failureBackoff; + } + + @Config("hogql.semantic-catalog.failure-backoff") + public HogQlSemanticCatalogConfig setFailureBackoff(Duration failureBackoff) + { + this.failureBackoff = failureBackoff; + return this; + } + + @Min(1) + public int getLoaderThreads() + { + return loaderThreads; + } + + @Config("hogql.semantic-catalog.loader-threads") + public HogQlSemanticCatalogConfig setLoaderThreads(int loaderThreads) + { + this.loaderThreads = loaderThreads; + return this; + } + + @Min(1) + public int getLoaderQueueCapacity() + { + return loaderQueueCapacity; + } + + @Config("hogql.semantic-catalog.loader-queue-capacity") + public HogQlSemanticCatalogConfig setLoaderQueueCapacity(int loaderQueueCapacity) + { + this.loaderQueueCapacity = loaderQueueCapacity; + return this; + } + + @NotNull + @MinDuration("1ms") + public Duration getRequestTimeout() + { + return requestTimeout; + } + + @Config("hogql.semantic-catalog.request-timeout") + public HogQlSemanticCatalogConfig setRequestTimeout(Duration requestTimeout) + { + this.requestTimeout = requestTimeout; + return this; + } + + @NotNull + @MinDataSize("1B") + @MaxDataSize("8MB") + public DataSize getMaximumResponseSize() + { + return maximumResponseSize; + } + + @Config("hogql.semantic-catalog.maximum-response-size") + public HogQlSemanticCatalogConfig setMaximumResponseSize(DataSize maximumResponseSize) + { + this.maximumResponseSize = maximumResponseSize; + return this; + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogHttpTransport.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogHttpTransport.java new file mode 100644 index 000000000000..d3b988c8006f --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogHttpTransport.java @@ -0,0 +1,180 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.common.net.MediaType; +import com.google.inject.Inject; +import io.airlift.http.client.HttpClient; +import io.airlift.http.client.Request; +import io.airlift.http.client.Response; +import io.airlift.http.client.ResponseHandler; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException.Failure; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.JsonTransport; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.concurrent.CompletionStage; + +import static com.google.common.net.MediaType.JSON_UTF_8; +import static io.airlift.concurrent.MoreFutures.toCompletableFuture; +import static io.airlift.http.client.HeaderNames.ACCEPT; +import static io.airlift.http.client.HeaderNames.CONTENT_TYPE; +import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static io.airlift.http.client.Request.Builder.prepareGet; +import static io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder.PROTOCOL_VERSION; +import static java.lang.Math.toIntExact; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +public final class HogQlSemanticCatalogHttpTransport + implements JsonTransport +{ + private static final String METADATA_PATH = "v1/hogql/compatibility/semantic-catalog"; + private static final MediaType JSON = JSON_UTF_8.withoutParameters(); + + private final URI baseUri; + private final HttpClient httpClient; + private final int maximumResponseBytes; + + @Inject + public HogQlSemanticCatalogHttpTransport( + HogQlSemanticCatalogConfig config, + @ForHogQlSemanticCatalog HttpClient httpClient) + { + this(requireNonNull(config, "config is null").getUri(), + httpClient, + toIntExact(config.getMaximumResponseSize().toBytes())); + } + + HogQlSemanticCatalogHttpTransport(URI baseUri, HttpClient httpClient) + { + this(baseUri, httpClient, HogQlSemanticCatalogSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES); + } + + private HogQlSemanticCatalogHttpTransport(URI baseUri, HttpClient httpClient, int maximumResponseBytes) + { + this.baseUri = validateBaseUri(baseUri); + this.httpClient = requireNonNull(httpClient, "httpClient is null"); + if (maximumResponseBytes <= 0 || maximumResponseBytes > HogQlSemanticCatalogSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES) { + throw new IllegalArgumentException("invalid HogQL semantic catalog response size limit"); + } + this.maximumResponseBytes = maximumResponseBytes; + } + + @Override + public CompletionStage load(LoadRequest request) + { + requireNonNull(request, "request is null"); + Request httpRequest = prepareGet() + .setUri(buildUri(baseUri, request)) + .setHeader(ACCEPT, JSON.toString()) + .build(); + return toCompletableFuture(httpClient.executeAsync(httpRequest, new MetadataResponseHandler(request, maximumResponseBytes))); + } + + static URI buildUri(URI baseUri, LoadRequest request) + { + var uriBuilder = uriBuilderFrom(validateBaseUri(baseUri)) + .appendPath(METADATA_PATH) + .addParameter("protocolVersion", Integer.toString(PROTOCOL_VERSION)) + .addParameter("languageVersion", request.languageVersion().toString()) + .addParameter("catalog", request.catalog().value()) + .addParameter("catalogDelimited", Boolean.toString(request.catalog().delimited())); + request.expectedGeneration().ifPresent(generation -> uriBuilder.addParameter("generation", Long.toString(generation))); + return uriBuilder.build(); + } + + private static URI validateBaseUri(URI baseUri) + { + requireNonNull(baseUri, "HogQL semantic catalog URI is null"); + String scheme = baseUri.getScheme(); + if (scheme == null || !(scheme.toLowerCase(ENGLISH).equals("http") || scheme.toLowerCase(ENGLISH).equals("https")) || + baseUri.getHost() == null || baseUri.getUserInfo() != null || baseUri.getQuery() != null || baseUri.getFragment() != null) { + throw new IllegalArgumentException("invalid HogQL semantic catalog URI"); + } + return baseUri; + } + + private static final class MetadataResponseHandler + implements ResponseHandler + { + private final LoadRequest loadRequest; + private final int maximumResponseBytes; + + private MetadataResponseHandler(LoadRequest loadRequest, int maximumResponseBytes) + { + this.loadRequest = loadRequest; + this.maximumResponseBytes = maximumResponseBytes; + } + + @Override + public byte[] handleException(Request request, Exception exception) + { + throw unavailable(); + } + + @Override + public byte[] handle(Request request, Response response) + { + int statusCode = response.getStatusCode(); + if (statusCode < 200 || statusCode >= 300) { + throw statusFailure(statusCode, loadRequest); + } + if (!isJson(response)) { + throw unavailable(); + } + try (InputStream input = response.getInputStream()) { + byte[] payload = input.readNBytes(maximumResponseBytes + 1); + if (payload.length > maximumResponseBytes) { + throw unavailable(); + } + return payload; + } + catch (IOException e) { + throw unavailable(); + } + } + + private static boolean isJson(Response response) + { + return response.getHeader(CONTENT_TYPE) + .map(value -> { + try { + return MediaType.parse(value).withoutParameters().equals(JSON); + } + catch (IllegalArgumentException _) { + return false; + } + }) + .orElse(false); + } + + private static HogQlSemanticCatalogException statusFailure(int statusCode, LoadRequest loadRequest) + { + if (statusCode == 409 || (statusCode == 404 && loadRequest.expectedGeneration().isPresent())) { + return new HogQlSemanticCatalogException(Failure.GENERATION_MISMATCH, "HogQL semantic catalog generation is unavailable"); + } + return unavailable(); + } + + private static HogQlSemanticCatalogException unavailable() + { + return new HogQlSemanticCatalogException(Failure.UNAVAILABLE, "HogQL semantic catalog metadata is unavailable"); + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogManager.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogManager.java new file mode 100644 index 000000000000..b62c364e8f34 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogManager.java @@ -0,0 +1,132 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.common.annotations.VisibleForTesting; +import com.google.inject.Inject; +import io.trino.hogql.compiler.catalog.BoundedAsyncHogQlSemanticCatalogSnapshotCache; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotCache; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.hogql.parser.HogQlLanguageVersion; +import jakarta.annotation.PreDestroy; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.function.Function; +import java.util.function.LongSupplier; + +import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +public final class HogQlSemanticCatalogManager +{ + private final ThreadPoolExecutor loaderExecutor; + private final HogQlSemanticCatalogSnapshotLoader loader; + private final BoundedAsyncHogQlSemanticCatalogSnapshotCache cache; + + @Inject + public HogQlSemanticCatalogManager( + HogQlSemanticCatalogConfig config, + HogQlSemanticCatalogHttpTransport transport, + HogQlSemanticCatalogSnapshotJsonDecoder decoder) + { + this(config, + HogQlLanguageContract.current().languageVersion(), + System::nanoTime, + jsonLoaderFactory(transport, decoder)); + } + + @VisibleForTesting + HogQlSemanticCatalogManager( + HogQlSemanticCatalogConfig config, + HogQlSemanticCatalogSnapshotLoader loader, + HogQlLanguageVersion languageVersion, + LongSupplier ticker) + { + this(config, languageVersion, ticker, _ -> loader); + } + + private HogQlSemanticCatalogManager( + HogQlSemanticCatalogConfig config, + HogQlLanguageVersion languageVersion, + LongSupplier ticker, + Function loaderFactory) + { + requireNonNull(config, "config is null"); + requireNonNull(languageVersion, "languageVersion is null"); + requireNonNull(ticker, "ticker is null"); + requireNonNull(loaderFactory, "loaderFactory is null"); + loaderExecutor = new ThreadPoolExecutor( + config.getLoaderThreads(), + config.getLoaderThreads(), + 0, + MILLISECONDS, + new ArrayBlockingQueue<>(config.getLoaderQueueCapacity()), + daemonThreadsNamed("hogql-semantic-catalog-loader-%s"), + new ThreadPoolExecutor.AbortPolicy()); + loader = requireNonNull(loaderFactory.apply(loaderExecutor), "loader is null"); + cache = new BoundedAsyncHogQlSemanticCatalogSnapshotCache( + config.getMaximumEntries(), + config.getRefreshAfter().toJavaTime(), + config.getExpireAfter().toJavaTime(), + config.getFailureBackoff().toJavaTime(), + ticker, + loaderExecutor, + catalog -> loader.load(LoadRequest.latest(catalog, languageVersion))); + } + + public HogQlSemanticCatalogSnapshotLoader loader() + { + return loader; + } + + private static Function jsonLoaderFactory( + HogQlSemanticCatalogHttpTransport transport, + HogQlSemanticCatalogSnapshotJsonDecoder decoder) + { + requireNonNull(transport, "transport is null"); + requireNonNull(decoder, "decoder is null"); + return executor -> request -> transport.load(request).thenApplyAsync(payload -> decoder.decode(payload, request), executor); + } + + public HogQlSemanticCatalogSnapshotCache cache() + { + return cache; + } + + public CompletionStage prewarm(PhysicalIdentifier catalog) + { + return cache.prewarm(catalog); + } + + @PreDestroy + public void shutdown() + { + loaderExecutor.shutdownNow(); + } + + @VisibleForTesting + boolean isLoaderExecutorShutdown() + { + return loaderExecutor.isShutdown(); + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogModule.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogModule.java new file mode 100644 index 000000000000..41d22b5dcd33 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogModule.java @@ -0,0 +1,72 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.Binder; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import com.google.inject.Singleton; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotCache; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider; + +import static io.airlift.http.client.HttpClientBinder.httpClientBinder; + +public class HogQlSemanticCatalogModule + extends AbstractConfigurationAwareModule +{ + @Override + protected void setup(Binder binder) + { + HogQlSemanticCatalogConfig config = buildConfigObject(HogQlSemanticCatalogConfig.class); + httpClientBinder(binder) + .bindHttpClient("hogql-semantic-catalog", ForHogQlSemanticCatalog.class) + .withConfigDefaults(httpClientConfig -> httpClientConfig + .setRequestTimeout(config.getRequestTimeout()) + .setMaxResponseContentLength(config.getMaximumResponseSize()) + .setMaxRequestsQueuedPerDestination(config.getLoaderQueueCapacity())); + binder.bind(HogQlSemanticCatalogHttpTransport.class).in(Scopes.SINGLETON); + binder.bind(HogQlSemanticCatalogManager.class).asEagerSingleton(); + } + + @Provides + @Singleton + public static HogQlSemanticCatalogSnapshotJsonDecoder provideDecoder() + { + return new HogQlSemanticCatalogSnapshotJsonDecoder(); + } + + @Provides + @Singleton + public static HogQlSemanticCatalogSnapshotLoader provideLoader(HogQlSemanticCatalogManager manager) + { + return manager.loader(); + } + + @Provides + @Singleton + public static HogQlSemanticCatalogSnapshotCache provideCache(HogQlSemanticCatalogManager manager) + { + return manager.cache(); + } + + @Provides + @Singleton + public static HogQlSemanticCatalogSnapshotProvider provideProvider(HogQlSemanticCatalogSnapshotCache cache) + { + return HogQlSemanticCatalogSnapshotProvider.fromCache(cache); + } +} diff --git a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java index 5527eba49799..12da90a8df67 100644 --- a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java +++ b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java @@ -99,6 +99,8 @@ import io.trino.execution.scheduler.policy.ExecutionPolicy; import io.trino.execution.scheduler.policy.PhasedExecutionPolicy; import io.trino.hogql.HogQlConfig; +import io.trino.hogql.HogQlSemanticCatalogConfig; +import io.trino.hogql.HogQlSemanticCatalogModule; import io.trino.hogql.compiler.HogQlCompiler; import io.trino.memory.ClusterMemoryManager; import io.trino.memory.ForMemoryManager; @@ -179,9 +181,13 @@ protected void setup(Binder binder) jaxrsBinder(binder).bind(QueuedStatementResource.class); jaxrsBinder(binder).bind(ExecutingStatementResource.class); configBinder(binder).bindConfig(HogQlConfig.class); + configBinder(binder).bindConfig(HogQlSemanticCatalogConfig.class); binder.bind(HogQlCompiler.class).in(Scopes.SINGLETON); if (buildConfigObject(HogQlConfig.class).isEnabled()) { jaxrsBinder(binder).bind(HogQlStatementResource.class); + if (buildConfigObject(HogQlSemanticCatalogConfig.class).getUri() != null) { + install(new HogQlSemanticCatalogModule()); + } } binder.bind(StatementHttpExecutionMBean.class).in(Scopes.SINGLETON); newExporter(binder).export(StatementHttpExecutionMBean.class).withGeneratedName(); diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java new file mode 100644 index 000000000000..487240c9ac3a --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.units.DataSize; +import io.airlift.units.Duration; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static io.airlift.units.DataSize.Unit.MEGABYTE; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; + +public class TestHogQlSemanticCatalogConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(HogQlSemanticCatalogConfig.class) + .setUri(null) + .setMaximumEntries(100) + .setRefreshAfter(new Duration(1, MINUTES)) + .setExpireAfter(new Duration(5, MINUTES)) + .setFailureBackoff(new Duration(10, SECONDS)) + .setLoaderThreads(4) + .setLoaderQueueCapacity(64) + .setRequestTimeout(new Duration(10, SECONDS)) + .setMaximumResponseSize(DataSize.of(8, MEGABYTE))); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = Map.of( + "hogql.semantic-catalog.uri", "https://duckgres.example/metadata", + "hogql.semantic-catalog.maximum-entries", "17", + "hogql.semantic-catalog.refresh-after", "2m", + "hogql.semantic-catalog.expire-after", "9m", + "hogql.semantic-catalog.failure-backoff", "20s", + "hogql.semantic-catalog.loader-threads", "3", + "hogql.semantic-catalog.loader-queue-capacity", "41", + "hogql.semantic-catalog.request-timeout", "7s", + "hogql.semantic-catalog.maximum-response-size", "4MB"); + + HogQlSemanticCatalogConfig expected = new HogQlSemanticCatalogConfig() + .setUri(URI.create("https://duckgres.example/metadata")) + .setMaximumEntries(17) + .setRefreshAfter(new Duration(2, MINUTES)) + .setExpireAfter(new Duration(9, MINUTES)) + .setFailureBackoff(new Duration(20, SECONDS)) + .setLoaderThreads(3) + .setLoaderQueueCapacity(41) + .setRequestTimeout(new Duration(7, SECONDS)) + .setMaximumResponseSize(DataSize.of(4, MEGABYTE)); + + assertFullMapping(properties, expected); + } +} diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogHttpTransport.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogHttpTransport.java new file mode 100644 index 000000000000..daf38445c3b7 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogHttpTransport.java @@ -0,0 +1,107 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.http.client.HttpStatus; +import io.airlift.http.client.Request; +import io.airlift.http.client.Response; +import io.airlift.http.client.testing.TestingHttpClient; +import io.airlift.http.client.testing.TestingResponse; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException.Failure; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; +import io.trino.hogql.parser.HogQlLanguageVersion; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import static com.google.common.net.MediaType.JSON_UTF_8; +import static io.airlift.http.client.HeaderNames.ACCEPT; +import static io.airlift.http.client.HttpStatus.CONFLICT; +import static io.airlift.http.client.HttpStatus.INTERNAL_SERVER_ERROR; +import static io.airlift.http.client.HttpStatus.NOT_FOUND; +import static io.airlift.http.client.HttpStatus.OK; +import static io.airlift.http.client.testing.TestingResponse.contentType; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlSemanticCatalogHttpTransport +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageVersion.valueOf("1.0.0"); + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("Sales & Growth/2026", true); + private static final URI BASE_URI = URI.create("https://duckgres.example/control-plane/"); + private static final String SECRET = "secret-metadata-response"; + + @Test + public void testConstructsEncodedLatestAndPinnedUris() + { + List requests = new ArrayList<>(); + TestingHttpClient httpClient = new TestingHttpClient(request -> { + requests.add(request); + return response(OK, "{}"); + }); + HogQlSemanticCatalogHttpTransport transport = new HogQlSemanticCatalogHttpTransport(BASE_URI, httpClient); + + assertThat(transport.load(LoadRequest.latest(CATALOG, LANGUAGE_VERSION)).toCompletableFuture().join()) + .isEqualTo("{}".getBytes(StandardCharsets.UTF_8)); + assertThat(transport.load(LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 42)).toCompletableFuture().join()) + .isEqualTo("{}".getBytes(StandardCharsets.UTF_8)); + + assertThat(requests).extracting(request -> request.getUri().toASCIIString()).containsExactly( + "https://duckgres.example/control-plane/v1/hogql/compatibility/semantic-catalog?protocolVersion=1&languageVersion=1.0.0&catalog=Sales%20%26%20Growth/2026&catalogDelimited=true", + "https://duckgres.example/control-plane/v1/hogql/compatibility/semantic-catalog?protocolVersion=1&languageVersion=1.0.0&catalog=Sales%20%26%20Growth/2026&catalogDelimited=true&generation=42"); + assertThat(requests).allSatisfy(request -> { + assertThat(request.getMethod()).isEqualTo("GET"); + assertThat(request.getHeader(ACCEPT)).isEqualTo(JSON_UTF_8.withoutParameters().toString()); + }); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("statusFailures") + public void testMapsHttpStatusWithoutResponseDisclosure(String name, HttpStatus status, LoadRequest request, Failure failure) + { + TestingHttpClient httpClient = new TestingHttpClient(_ -> response(status, SECRET)); + HogQlSemanticCatalogHttpTransport transport = new HogQlSemanticCatalogHttpTransport(BASE_URI, httpClient); + + assertThatThrownBy(() -> transport.load(request).toCompletableFuture().join()) + .cause() + .isInstanceOfSatisfying(HogQlSemanticCatalogException.class, exception -> { + assertThat(exception.failure()).isEqualTo(failure); + assertThat(exception).hasMessageNotContaining(SECRET); + assertThat(exception.getMessage()).hasSizeLessThan(128); + }); + } + + private static Stream statusFailures() + { + return Stream.of( + Arguments.of("latest missing", NOT_FOUND, LoadRequest.latest(CATALOG, LANGUAGE_VERSION), Failure.UNAVAILABLE), + Arguments.of("pinned missing", NOT_FOUND, LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 7), Failure.GENERATION_MISMATCH), + Arguments.of("generation conflict", CONFLICT, LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 7), Failure.GENERATION_MISMATCH), + Arguments.of("server failure", INTERNAL_SERVER_ERROR, LoadRequest.latest(CATALOG, LANGUAGE_VERSION), Failure.UNAVAILABLE)); + } + + private static Response response(HttpStatus status, String body) + { + return new TestingResponse(status, contentType(JSON_UTF_8), body.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java new file mode 100644 index 000000000000..46b5063a4a4a --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java @@ -0,0 +1,110 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.units.Duration; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader; +import io.trino.hogql.parser.HogQlLanguageVersion; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlSemanticCatalogManager +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageVersion.valueOf("1.0.0"); + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("ducklake", false); + + @Test + public void testColdAndExpiredReadsStayLocalWhileRefreshIsAsynchronous() + { + AtomicLong ticker = new AtomicLong(); + AtomicInteger loads = new AtomicInteger(); + CompletableFuture firstLoadStarted = new CompletableFuture<>(); + CompletableFuture secondLoadStarted = new CompletableFuture<>(); + CompletableFuture firstLoad = new CompletableFuture<>(); + CompletableFuture secondLoad = new CompletableFuture<>(); + HogQlSemanticCatalogSnapshotLoader loader = _ -> switch (loads.incrementAndGet()) { + case 1 -> { + firstLoadStarted.complete(null); + yield firstLoad; + } + case 2 -> { + secondLoadStarted.complete(null); + yield secondLoad; + } + default -> throw new IllegalStateException("unexpected semantic catalog load"); + }; + HogQlSemanticCatalogManager manager = new HogQlSemanticCatalogManager(config(), loader, LANGUAGE_VERSION, ticker::get); + try { + Optional cold = manager.cache().currentSnapshot(CATALOG); + assertThat(cold).isEmpty(); + firstLoadStarted.join(); + CompletableFuture firstRefresh = manager.prewarm(CATALOG).toCompletableFuture(); + firstLoad.complete(snapshot(1)); + assertThat(firstRefresh.join().generation()).isEqualTo(1); + assertThat(manager.cache().currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + + ticker.set(3 * SECONDS.toNanos(1)); + Optional expired = manager.cache().currentSnapshot(CATALOG); + assertThat(expired).isEmpty(); + secondLoadStarted.join(); + CompletableFuture secondRefresh = manager.prewarm(CATALOG).toCompletableFuture(); + secondLoad.complete(snapshot(2)); + assertThat(secondRefresh.join().generation()).isEqualTo(2); + assertThat(loads).hasValue(2); + } + finally { + manager.shutdown(); + } + } + + @Test + public void testShutdownTerminatesDedicatedLoaderExecutor() + { + HogQlSemanticCatalogManager manager = new HogQlSemanticCatalogManager( + config(), + _ -> CompletableFuture.completedFuture(snapshot(1)), + LANGUAGE_VERSION, + System::nanoTime); + + assertThat(manager.isLoaderExecutorShutdown()).isFalse(); + manager.shutdown(); + assertThat(manager.isLoaderExecutorShutdown()).isTrue(); + } + + private static HogQlSemanticCatalogConfig config() + { + return new HogQlSemanticCatalogConfig() + .setMaximumEntries(2) + .setRefreshAfter(new Duration(1, SECONDS)) + .setExpireAfter(new Duration(2, SECONDS)) + .setFailureBackoff(new Duration(0, SECONDS)) + .setLoaderThreads(1) + .setLoaderQueueCapacity(2); + } + + private static HogQlSemanticCatalogSnapshot snapshot(long generation) + { + return new HogQlSemanticCatalogSnapshot(1, LANGUAGE_VERSION, CATALOG, generation, List.of()); + } +} From 2cd09f418b91d55163596159dda38d6442fbd536 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 10:53:39 -0400 Subject: [PATCH 031/136] feat(hogql): lower grouping and aggregates --- .../trino/hogql/compiler/HogQlCompiler.java | 8 +- .../trino/hogql/compiler/TrinoAstFactory.java | 60 ++++++++++++--- .../hogql/compiler/TestHogQlCompiler.java | 38 ++++++++- .../io/trino/hogql/parser/HogQlParser.java | 77 ++++++++++++++++--- .../trino/hogql/parser/tree/HogQlQuery.java | 14 +++- .../1.0.0/trino-compatibility-overrides.json | 4 +- .../trino/hogql/parser/TestHogQlParser.java | 19 ++++- 7 files changed, 193 insertions(+), 27 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index df546bec97af..98929496a55a 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -104,6 +104,8 @@ private static HogQlCompilationResult compile(HogQlQuery query, Map collectPlaceholders(expression, placeholders)); + query.groupBy().forEach(expression -> collectPlaceholders(expression, placeholders)); + query.having().ifPresent(expression -> collectPlaceholders(expression, placeholders)); query.orderBy().forEach(sortItem -> collectPlaceholders(sortItem.expression(), placeholders)); query.limit().ifPresent(expression -> collectPlaceholders(expression, placeholders)); query.offset().ifPresent(expression -> collectPlaceholders(expression, placeholders)); @@ -179,7 +181,11 @@ private static void collectPlaceholders(Expression expression, List } case CastExpression cast -> collectPlaceholders(cast.value(), placeholders); case ColumnReference _ -> {} - case FunctionCall function -> function.arguments().forEach(argument -> collectPlaceholders(argument, placeholders)); + case FunctionCall function -> { + function.arguments().forEach(argument -> collectPlaceholders(argument, placeholders)); + function.orderBy().forEach(sortItem -> collectPlaceholders(sortItem.expression(), placeholders)); + function.filter().ifPresent(filter -> collectPlaceholders(filter, placeholders)); + } case InExpression in -> { collectPlaceholders(in.value(), placeholders); in.values().forEach(value -> collectPlaceholders(value, placeholders)); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 57bd1792ed1f..6b2ec7e8f9b1 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -41,11 +41,13 @@ import io.trino.sql.tree.Array; import io.trino.sql.tree.BetweenPredicate; import io.trino.sql.tree.BooleanLiteral; +import io.trino.sql.tree.CallArgument; import io.trino.sql.tree.Cast; import io.trino.sql.tree.ComparisonPredicate; import io.trino.sql.tree.DereferenceExpression; import io.trino.sql.tree.Expression; import io.trino.sql.tree.GenericDataType; +import io.trino.sql.tree.GroupBy; import io.trino.sql.tree.InListExpression; import io.trino.sql.tree.InPredicate; import io.trino.sql.tree.IsNullPredicate; @@ -67,6 +69,7 @@ import io.trino.sql.tree.Select; import io.trino.sql.tree.SelectItem; import io.trino.sql.tree.SimpleCaseExpression; +import io.trino.sql.tree.SimpleGroupBy; import io.trino.sql.tree.SingleColumn; import io.trino.sql.tree.SortItem.NullOrdering; import io.trino.sql.tree.SortItem.Ordering; @@ -93,8 +96,8 @@ public static Statement createStatement(HogQlQuery query, Map createExpression(expression, parameterIds)), - Optional.empty(), - Optional.empty(), + createGroupBy(query, parameterIds), + query.having().map(expression -> createExpression(expression, parameterIds)), List.of(), createOrderBy(query, parameterIds), query.offset().map(offset -> new Offset(location(offset.span()), createExpression(offset, parameterIds))), @@ -112,12 +115,17 @@ public static Statement createStatement(HogQlQuery query, Map createOrderBy(HogQlQuery query, Map parameterIds) { - if (query.orderBy().isEmpty()) { + return createOrderBy(query.orderBy(), parameterIds); + } + + private static Optional createOrderBy(List sortItems, Map parameterIds) + { + if (sortItems.isEmpty()) { return Optional.empty(); } return Optional.of(new OrderBy( - location(query.orderBy().getFirst().span()), - query.orderBy().stream() + location(sortItems.getFirst().span()), + sortItems.stream() .map(sortItem -> new io.trino.sql.tree.SortItem( location(sortItem.span()), createExpression(sortItem.expression(), parameterIds), @@ -133,6 +141,22 @@ private static Optional createOrderBy(HogQlQuery query, Map createGroupBy(HogQlQuery query, Map parameterIds) + { + if (query.groupBy().isEmpty()) { + return Optional.empty(); + } + NodeLocation location = location(query.groupBy().getFirst().span()); + return Optional.of(new GroupBy( + location, + false, + List.of(new SimpleGroupBy( + location, + query.groupBy().stream() + .map(expression -> createExpression(expression, parameterIds)) + .toList())))); + } + private static SelectItem createSelectItem(Projection projection, Map parameterIds) { return switch (projection) { @@ -161,12 +185,7 @@ private static Expression createExpression(HogQlQuery.Expression expression, Map new GenericDataType(location(cast.type().span()), createIdentifier(cast.type()), List.of()), cast.safe()); case ColumnReference reference -> createColumnReference(reference); - case FunctionCall function -> new io.trino.sql.tree.FunctionCall( - location(function.span()), - QualifiedName.of(List.of(createIdentifier(function.name()))), - function.arguments().stream() - .map(argument -> createExpression(argument, parameterIds)) - .toList()); + case FunctionCall function -> createFunctionCall(function, parameterIds); case InExpression in -> createInExpression(in, parameterIds); case IsNullExpression isNull -> new Predicated( location(isNull.predicateSpan()), @@ -187,6 +206,25 @@ private static Expression createExpression(HogQlQuery.Expression expression, Map }; } + private static Expression createFunctionCall(FunctionCall function, Map parameterIds) + { + return new io.trino.sql.tree.FunctionCall( + location(function.span()), + QualifiedName.of(List.of(createIdentifier(function.name()))), + Optional.empty(), + function.filter().map(filter -> createExpression(filter, parameterIds)), + createOrderBy(function.orderBy(), parameterIds), + function.distinct(), + Optional.empty(), + Optional.empty(), + function.arguments().stream() + .map(argument -> new CallArgument( + location(argument.span()), + Optional.empty(), + createExpression(argument, parameterIds))) + .toList()); + } + private static Expression createCaseExpression(CaseExpression caseExpression, Map parameterIds) { NodeLocation location = location(caseExpression.span()); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index 262ca2a735a7..67d6bb3d71ab 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -115,6 +115,42 @@ public void testLowersCaseAndCastExpressions(String hogql) assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(hogql)); } + @ParameterizedTest + @ValueSource(strings = { + "SELECT event, count(*) AS total FROM events GROUP BY event HAVING count(*) > 1", + "SELECT lower(event), count(DISTINCT person_id), sum(revenue) FROM events GROUP BY lower(event) HAVING sum(revenue) > 10", + "SELECT count(*) FILTER (WHERE enabled) FROM events", + "SELECT array_agg(event ORDER BY timestamp DESC) FROM events", + }) + public void testLowersOrdinaryGroupingHavingAndAggregateFunctions(String hogql) + { + Statement statement = compiler.compile(hogql); + + assertThat(statement).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @Test + public void testBindsPlaceholdersAcrossGroupingHavingAndAggregateModifiers() + { + String hogql = "SELECT array_agg({value} ORDER BY {sort}) FILTER (WHERE {filter}) FROM events " + + "GROUP BY {group} HAVING sum({having}) > {threshold}"; + Map bindings = Map.of( + "value", typedValue("value"), + "sort", typedValue("sort"), + "filter", typedValue("filter"), + "group", typedValue("group"), + "having", typedValue("having"), + "threshold", typedValue("threshold")); + + HogQlCompilationResult result = compiler.compile(hogql, bindings); + + assertThat(result.parameterNames()).containsExactly("value", "sort", "filter", "group", "having", "threshold"); + assertThat(parameters(result.statement())) + .extracting(Parameter::getId) + .containsExactly(0, 1, 2, 3, 4, 5); + } + @ParameterizedTest @ValueSource(strings = { "SELECT value BETWEEN 1 AND 10", @@ -234,7 +270,7 @@ public void testReportsSourceLocatedHogQlSyntaxError() { TrinoException exception = catchThrowableOfType( TrinoException.class, - () -> compiler.compile("SELECT 1\nGROUP BY 1")); + () -> compiler.compile("SELECT 1\nGROUP BY ALL")); assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_SYNTAX_ERROR.toErrorCode()); assertThat(exception.getLocation()).contains(new Location(2, 1)); diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 4940d36caa50..988ce7b64ef7 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -456,6 +456,10 @@ public HogQlQuery build(HogQLParser.SelectContext context) Optional where = Optional.ofNullable(select.whereClause()) .map(HogQLParser.WhereClauseContext::columnExpr) .map(this::buildExpression); + List groupBy = buildGroupBy(select); + Optional having = Optional.ofNullable(select.havingClause()) + .map(HogQLParser.HavingClauseContext::columnExpr) + .map(this::buildExpression); List orderBy = buildOrderBy(selectedQuery.orderBy() != null ? selectedQuery.orderBy() : select.orderByClause()); Pagination pagination = mergePagination( buildPagination(select.limitAndOffsetClause(), select.offsetOnlyClause()), @@ -466,6 +470,8 @@ public HogQlQuery build(HogQLParser.SelectContext context) projections, from, where, + groupBy, + having, orderBy, pagination.limit(), pagination.offset(), @@ -512,8 +518,6 @@ private void rejectUnsupportedClauses(HogQLParser.SelectStmtContext context) clauses.add(context.arrayJoinClause()); clauses.add(context.prewhereClause()); clauses.addAll(context.sampleClause()); - clauses.add(context.groupByClause()); - clauses.add(context.havingClause()); clauses.add(context.qualifyClause()); clauses.add(context.windowClause()); clauses.add(context.limitByClause()); @@ -534,11 +538,35 @@ private List buildOrderBy(HogQLParser.OrderByClauseContext context) if (context.interpolateClause() != null) { throw unsupported(context.interpolateClause(), "ORDER BY interpolation"); } - return context.orderExprList().orderExpr().stream() + return buildSortItems(context.orderExprList()); + } + + private List buildSortItems(HogQLParser.OrderExprListContext context) + { + return context.orderExpr().stream() .map(this::buildSortItem) .toList(); } + private List buildGroupBy(HogQLParser.SelectStmtContext select) + { + HogQLParser.GroupByClauseContext groupBy = select.groupByClause(); + if (groupBy == null) { + return List.of(); + } + if (groupBy.columnExprList() == null || + groupBy.ALL() != null || + groupBy.CUBE() != null || + groupBy.ROLLUP() != null || + groupBy.GROUPING() != null || + select.CUBE() != null || + select.ROLLUP() != null || + select.TOTALS() != null) { + throw unsupported(groupBy, "advanced grouping"); + } + return buildExpressions(groupBy.columnExprList()); + } + private SortItem buildSortItem(HogQLParser.OrderExprContext context) { if (context.COLLATE() != null || context.withFillClause() != null) { @@ -822,13 +850,44 @@ private BinaryExpression binary(BinaryOperator operator, HogQLParser.ColumnExprV private FunctionCall buildFunction(HogQLParser.ColumnExprFunctionContext context) { - if (context.columnExprs != null || context.DISTINCT() != null || context.orderExprList() != null || context.filterExpr != null) { - throw unsupported(context, "parametric, distinct, ordered, or filtered function"); + if (context.columnExprs != null) { + throw unsupported(context, "parametric function"); + } + List arguments = buildFunctionArguments(context); + List orderBy = context.orderExprList() == null ? List.of() : buildSortItems(context.orderExprList()); + Optional filter = Optional.ofNullable(context.filterExpr).map(this::buildExpression); + return new FunctionCall( + buildIdentifier(context.identifier()), + arguments, + context.DISTINCT() != null, + orderBy, + filter, + sourceSpan(context)); + } + + private List buildFunctionArguments(HogQLParser.ColumnExprFunctionContext context) + { + if (context.columnArgList == null) { + return List.of(); + } + List arguments = context.columnArgList.columnExpr(); + if (arguments.size() == 1 && isUnqualifiedStar(arguments.getFirst())) { + if (context.DISTINCT() != null) { + throw unsupported(context, "DISTINCT star function"); + } + return List.of(); } - List arguments = context.columnArgList == null ? List.of() : context.columnArgList.columnExpr().stream() - .map(this::buildExpression) - .toList(); - return new FunctionCall(buildIdentifier(context.identifier()), arguments, sourceSpan(context)); + return arguments.stream() + .map(this::buildExpression) + .toList(); + } + + private static boolean isUnqualifiedStar(HogQLParser.ColumnExprContext context) + { + return context instanceof HogQLParser.ColumnExprValuePassthroughContext passthrough && + passthrough.columnExprValue() instanceof HogQLParser.ColumnExprAsteriskContext asterisk && + asterisk.tableIdentifier() == null && + asterisk.EXCLUDE() == null; } private Expression buildLiteral(HogQLParser.LiteralContext context) diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index 02616dc80ef1..9303a537ceaa 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -23,6 +23,8 @@ public record HogQlQuery( List projections, Optional from, Optional where, + List groupBy, + Optional having, List orderBy, Optional limit, Optional offset, @@ -33,6 +35,8 @@ public record HogQlQuery( projections = List.copyOf(requireNonNull(projections, "projections is null")); from = requireNonNull(from, "from is null"); where = requireNonNull(where, "where is null"); + groupBy = List.copyOf(requireNonNull(groupBy, "groupBy is null")); + having = requireNonNull(having, "having is null"); orderBy = List.copyOf(requireNonNull(orderBy, "orderBy is null")); limit = requireNonNull(limit, "limit is null"); offset = requireNonNull(offset, "offset is null"); @@ -233,13 +237,21 @@ public record BinaryExpression(BinaryOperator operator, Expression left, Express } } - public record FunctionCall(Identifier name, List arguments, SourceSpan span) + public record FunctionCall( + Identifier name, + List arguments, + boolean distinct, + List orderBy, + Optional filter, + SourceSpan span) implements Expression { public FunctionCall { name = requireNonNull(name, "name is null"); arguments = List.copyOf(requireNonNull(arguments, "arguments is null")); + orderBy = List.copyOf(requireNonNull(orderBy, "orderBy is null")); + filter = requireNonNull(filter, "filter is null"); span = requireNonNull(span, "span is null"); } } diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json index 45763aa53f72..683c0629a07e 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json @@ -74,7 +74,7 @@ }, { "id": "query.accept.aggregate", - "status": "explicitCurrentError" + "status": "supported" }, { "id": "query.accept.constant", @@ -86,7 +86,7 @@ }, { "id": "query.accept.order-limit-offset", - "status": "explicitCurrentError" + "status": "supported" }, { "id": "query.accept.projection-from", diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 8b772d37d4c4..187656790d3f 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -17,6 +17,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; @@ -39,13 +40,14 @@ public class TestHogQlParser @ParameterizedTest @ValueSource(strings = { "SELECT", - "SELECT count(*)", "SELECT * FROM first JOIN second", + "SELECT count(*) OVER ()", + "SELECT function(1)(value)", "DROP TABLE events", "SELECT 1; SELECT 2", "SELECT * FROM", "SELECT {1 + 2}", - "SELECT 1 GROUP BY 1", + "SELECT 1 GROUP BY ALL", }) public void testRejectsSyntaxWithoutAnAstMapping(String hogql) { @@ -95,6 +97,19 @@ public void testBuildsDistinctOrderingAndPaginationAst() assertThat(query.offset()).get().extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("2"); } + @Test + public void testBuildsOrdinaryGroupingHavingAndAggregateFunctionAst() + { + HogQlQuery query = parser.parseStatement("SELECT country, count(DISTINCT person_id) FROM events GROUP BY country, lower(source) HAVING count(*) > 1"); + + assertThat(query.groupBy()).hasSize(2); + assertThat(query.having()).isPresent(); + FunctionCall count = (FunctionCall) ((ExpressionProjection) query.projections().get(1)).expression(); + assertThat(count.name().value()).isEqualTo("count"); + assertThat(count.distinct()).isTrue(); + assertThat(count.arguments()).singleElement().isInstanceOf(ColumnReference.class); + } + @Test public void testSourceSpansUseUnicodeCodePointOffsetsAndEndPositions() { From 160056b92c223f538e1432151050b92ec95c36b4 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 11:02:39 -0400 Subject: [PATCH 032/136] feat(hogql): validate temporal parameter literals --- .../execution/HogQlParameterDecoder.java | 64 +++++++++----- .../io/trino/execution/TestQueryPreparer.java | 83 ++++++++++++++++--- 2 files changed, 114 insertions(+), 33 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java b/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java index bf808ffd97d9..3d4f23f400ed 100644 --- a/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java +++ b/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java @@ -62,11 +62,6 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.math.RoundingMode; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.OffsetDateTime; -import java.time.OffsetTime; -import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -74,8 +69,18 @@ import java.util.Set; import java.util.UUID; +import static io.airlift.slice.Slices.utf8Slice; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_BINDING_ERROR; import static io.trino.spi.type.CharType.MAX_LENGTH; +import static io.trino.type.DateTimes.extractTimePrecision; +import static io.trino.type.DateTimes.extractTimestampPrecision; +import static io.trino.type.DateTimes.parseTime; +import static io.trino.type.DateTimes.parseTimeWithTimeZone; +import static io.trino.type.DateTimes.parseTimestamp; +import static io.trino.type.DateTimes.parseTimestampWithTimeZone; +import static io.trino.type.DateTimes.timeHasTimeZone; +import static io.trino.type.DateTimes.timestampHasTimeZone; +import static io.trino.util.DateTimeUtils.parseDate; import static java.lang.Float.isFinite; import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; @@ -210,12 +215,15 @@ private static Expression decodeString(StringValue value, GenericDataType type, } case "DATE" -> { requireNoArguments(type); - LocalDate.parse(value.value()); + parseDate(utf8Slice(value.value())); yield new GenericLiteral(location, "DATE", value.value()); } case "UUID" -> { requireNoArguments(type); UUID.fromString(value.value()); + if (value.value().length() != 36) { + throw new IllegalArgumentException("UUID value is not canonical"); + } yield new GenericLiteral(location, "UUID", value.value()); } case "IPADDRESS" -> { @@ -237,27 +245,34 @@ private static Expression decodeDateTime(Value value, DateTimeDataType type, Nod throw new IllegalArgumentException("date-time value must be a string"); } validateDateTimePrecision(type); - String normalized = stringValue.value().replace(' ', 'T'); - try { - if (type.getType() == DateTimeDataType.Type.TIMESTAMP) { - if (type.isWithTimeZone()) { - OffsetDateTime.parse(normalized); - } - else { - LocalDateTime.parse(normalized); - } + String literal = stringValue.value(); + if (type.getType() == DateTimeDataType.Type.TIMESTAMP) { + int precision = extractTimestampPrecision(literal); + validateLiteralPrecision(precision); + if (timestampHasTimeZone(literal) != type.isWithTimeZone()) { + throw new IllegalArgumentException("timestamp time zone does not match type"); } - else if (type.isWithTimeZone()) { - OffsetTime.parse(normalized); + if (type.isWithTimeZone()) { + parseTimestampWithTimeZone(precision, literal); } else { - java.time.LocalTime.parse(normalized); + parseTimestamp(precision, literal); } } - catch (DateTimeParseException _) { - throw new IllegalArgumentException("date-time value is invalid"); + else { + int precision = extractTimePrecision(literal); + validateLiteralPrecision(precision); + if (timeHasTimeZone(literal) != type.isWithTimeZone()) { + throw new IllegalArgumentException("time zone does not match type"); + } + if (type.isWithTimeZone()) { + parseTimeWithTimeZone(precision, literal); + } + else { + parseTime(literal); + } } - return new GenericLiteral(location, type.getType().name(), stringValue.value()); + return new GenericLiteral(location, type.getType().name(), literal); } private static Expression decodeArray(Value value, GenericDataType type, NodeLocation location) @@ -471,6 +486,13 @@ private static void validateDateTimePrecision(DateTimeDataType type) }); } + private static void validateLiteralPrecision(int precision) + { + if (precision > 12) { + throw new IllegalArgumentException("date-time literal precision is out of range"); + } + } + private static void requireType(GenericDataType type, String expected) { if (!type.getName().getCanonicalValue().equals(expected)) { diff --git a/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java b/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java index ae170d4ee532..135939343645 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java @@ -31,7 +31,9 @@ import io.trino.sql.parser.SqlParser; import io.trino.sql.tree.AllColumns; import io.trino.sql.tree.Array; +import io.trino.sql.tree.BinaryLiteral; import io.trino.sql.tree.Cast; +import io.trino.sql.tree.Expression; import io.trino.sql.tree.FunctionCall; import io.trino.sql.tree.GenericLiteral; import io.trino.sql.tree.QualifiedName; @@ -144,17 +146,47 @@ public void testHogQlTypedParametersUseThePreparedValueBoundary() } @ParameterizedTest - @MethodSource("invalidHogQlTypedParameters") - public void testHogQlTypedParameterErrorsAreStableAndRedacted(HogQlTypedValue typedValue) + @MethodSource("validHogQlLiteralParameters") + public void testHogQlLiteralParametersPreserveStockAstValues(HogQlTypedValue typedValue, Class literalType, String expectedValue) { - String sensitiveValue = "sensitive-binding-value"; + PreparedQuery preparedQuery = HOGQL_QUERY_PREPARER.prepareQuery( + TEST_SESSION, + hogQl(envelope("SELECT {input}", Map.of("input", typedValue)))); + + assertThat(preparedQuery.getParameters()).hasSize(1); + Expression parameter = preparedQuery.getParameters().getFirst(); + Expression literal; + if (typedValue.type().equalsIgnoreCase("json")) { + literal = parameter; + } + else { + assertThat(parameter).isInstanceOf(Cast.class); + Cast cast = (Cast) parameter; + assertThat(cast.getType()).isEqualTo(SQL_PARSER.createType(typedValue.type())); + literal = cast.getExpression(); + } + assertThat(literal).isInstanceOf(literalType); + String literalValue = switch (literal) { + case GenericLiteral genericLiteral -> genericLiteral.getValue(); + case BinaryLiteral binaryLiteral -> binaryLiteral.toHexString(); + default -> throw new AssertionError("unexpected literal type"); + }; + assertThat(literalValue).isEqualTo(expectedValue); + assertThat(SqlFormatter.formatSql(preparedQuery.getStatement())) + .contains("?") + .doesNotContain(expectedValue); + } + @ParameterizedTest + @MethodSource("invalidHogQlTypedParameters") + public void testHogQlTypedParameterErrorsAreStableAndRedacted(HogQlTypedValue typedValue, String redactedFragment) + { assertTrinoExceptionThrownBy(() -> HOGQL_QUERY_PREPARER.prepareQuery( TEST_SESSION, hogQl(envelope("SELECT {input}", Map.of("input", typedValue))))) .hasErrorCode(HogQlErrorCode.HOGQL_BINDING_ERROR) .hasMessage("line 1:8: Invalid HogQL parameter binding: input") - .satisfies(exception -> assertThat(exception.getMessage()).doesNotContain(sensitiveValue)); + .satisfies(exception -> assertThat(exception.getMessage()).doesNotContain(redactedFragment)); } @Test @@ -187,14 +219,41 @@ private static Stream invalidHogQlTypedParameters() { String sensitiveValue = "sensitive-binding-value"; return Stream.of( - arguments(typedValue(sensitiveValue, new StringValue(sensitiveValue))), - arguments(typedValue("boolean", new StringValue(sensitiveValue))), - arguments(typedValue("tinyint", new NumberValue("128"))), - arguments(typedValue("decimal(3, 1)", new NumberValue("123.4"))), - arguments(typedValue("uuid", new StringValue(sensitiveValue))), - arguments(typedValue("array(bigint)", new ArrayValue(List.of(new StringValue(sensitiveValue))))), - arguments(typedValue("map(bigint, varchar)", new ObjectValue(Map.of("field", new StringValue(sensitiveValue))))), - arguments(typedValue("row(label varchar)", new ObjectValue(Map.of("unexpected", new StringValue(sensitiveValue)))))); + arguments(typedValue(sensitiveValue, new StringValue(sensitiveValue)), sensitiveValue), + arguments(typedValue("boolean", new StringValue(sensitiveValue)), sensitiveValue), + arguments(typedValue("tinyint", new NumberValue("128")), "128"), + arguments(typedValue("decimal(3, 1)", new NumberValue("123.4")), "123.4"), + arguments(typedValue("uuid", new StringValue(sensitiveValue)), sensitiveValue), + arguments(typedValue("array(bigint)", new ArrayValue(List.of(new StringValue(sensitiveValue)))), sensitiveValue), + arguments(typedValue("map(bigint, varchar)", new ObjectValue(Map.of("field", new StringValue(sensitiveValue)))), sensitiveValue), + arguments(typedValue("row(label varchar)", new ObjectValue(Map.of("unexpected", new StringValue(sensitiveValue)))), sensitiveValue), + arguments(typedValue("time(13)", new StringValue("12:34:56.123456789012")), "12:34:56.123456789012"), + arguments(typedValue("time(12) with time zone", new StringValue("12:34:56.123456789012+15:00")), "+15:00"), + arguments(typedValue("time(12) with time zone", new StringValue("12:34:56.123456789012 America/Toronto")), "America/Toronto"), + arguments(typedValue("timestamp(12) with time zone", new StringValue("2026-03-08 02:30:00.123456789012 America/Toronto")), "America/Toronto"), + arguments(typedValue("timestamp(12) with time zone", new StringValue("2026-08-27 12:34:56.123456789012 Invalid/Zone")), "Invalid/Zone"), + arguments(typedValue("timestamp(12)", new StringValue("2026-08-27 12:34:56.1234567890123")), "1234567890123"), + arguments(typedValue("date", new StringValue("2026-02-29")), "2026-02-29"), + arguments(typedValue("uuid", new StringValue("1-1-1-1-1")), "1-1-1-1-1"), + arguments(typedValue("ipaddress", new StringValue("host.example.com")), "host.example.com"), + arguments(typedValue("varbinary", new StringValue("ABC")), "ABC")); + } + + private static Stream validHogQlLiteralParameters() + { + return Stream.of( + arguments(typedValue("time(0)", new StringValue("00:00")), GenericLiteral.class, "00:00"), + arguments(typedValue("time(10)", new StringValue("12:34:56.1234567890")), GenericLiteral.class, "12:34:56.1234567890"), + arguments(typedValue("time(12) with time zone", new StringValue("12:34:56.123456789012 +05:45")), GenericLiteral.class, "12:34:56.123456789012 +05:45"), + arguments(typedValue("timestamp(0)", new StringValue("2026-08-27 12:34:56")), GenericLiteral.class, "2026-08-27 12:34:56"), + arguments(typedValue("timestamp(10)", new StringValue("2026-08-27 12:34:56.1234567890")), GenericLiteral.class, "2026-08-27 12:34:56.1234567890"), + arguments(typedValue("timestamp(11) with time zone", new StringValue("2026-08-27 12:34:56.12345678901 +05:45")), GenericLiteral.class, "2026-08-27 12:34:56.12345678901 +05:45"), + arguments(typedValue("timestamp(12) with time zone", new StringValue("2026-08-27 12:34:56.123456789012 America/Toronto")), GenericLiteral.class, "2026-08-27 12:34:56.123456789012 America/Toronto"), + arguments(typedValue("date", new StringValue("2000-02-29")), GenericLiteral.class, "2000-02-29"), + arguments(typedValue("uuid", new StringValue("00000000-0000-0000-0000-000000000000")), GenericLiteral.class, "00000000-0000-0000-0000-000000000000"), + arguments(typedValue("ipaddress", new StringValue("64:ff9b::10.0.0.1")), GenericLiteral.class, "64:ff9b::10.0.0.1"), + arguments(typedValue("varbinary", new StringValue("00 ff 10")), BinaryLiteral.class, "00FF10"), + arguments(typedValue("json", new ObjectValue(Map.of("escaped", new StringValue("line\n\u2603"), "number", new NumberValue("12345678901234567890.000000000001"), "null", NullValue.NULL))), GenericLiteral.class, "{\"escaped\":\"line\\n☃\",\"null\":null,\"number\":12345678901234567890.000000000001}")); } private static HogQlTypedValue typedValue(String type, HogQlTypedValue.Value value) From 4a459a2dab17ba8d392ce8433d84e256c3f1cfe9 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 11:07:31 -0400 Subject: [PATCH 033/136] feat(hogql): observe compilation outcomes --- .../io/trino/execution/QueryPreparer.java | 42 ++++- .../io/trino/hogql/HogQlCompilationEvent.java | 105 ++++++++++++ .../trino/hogql/HogQlCompilationObserver.java | 22 +++ .../io/trino/hogql/HogQlCompilationStats.java | 149 ++++++++++++++++++ .../trino/hogql/HogQlCompilationTracker.java | 123 +++++++++++++++ .../io/trino/server/CoordinatorModule.java | 5 + .../TestHogQlQueryPreparerObservability.java | 93 +++++++++++ .../TestHogQlCompilationObservability.java | 149 ++++++++++++++++++ 8 files changed, 680 insertions(+), 8 deletions(-) create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationEvent.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationObserver.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationStats.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationTracker.java create mode 100644 core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerObservability.java create mode 100644 core/trino-main/src/test/java/io/trino/hogql/TestHogQlCompilationObservability.java diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java index 8e05c64ebde2..6c541e49e978 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java @@ -16,6 +16,9 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Inject; import io.trino.Session; +import io.trino.hogql.HogQlCompilationEvent.Dimensions; +import io.trino.hogql.HogQlCompilationObserver; +import io.trino.hogql.HogQlCompilationTracker; import io.trino.hogql.compiler.HogQlCompilationResult; import io.trino.hogql.compiler.HogQlCompileEnvelope; import io.trino.hogql.compiler.HogQlCompiler; @@ -33,6 +36,9 @@ import java.util.Optional; import static io.trino.execution.ParameterExtractor.getParameterCount; +import static io.trino.hogql.HogQlCompilationEvent.Phase.COMPILATION; +import static io.trino.hogql.HogQlCompilationEvent.Phase.PARAMETER_BINDING; +import static io.trino.hogql.HogQlCompilationObserver.NOOP; import static io.trino.spi.StandardErrorCode.INVALID_PARAMETER_USAGE; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; import static io.trino.sql.analyzer.ConstantExpressionVerifier.verifyExpressionIsConstant; @@ -46,23 +52,30 @@ public class QueryPreparer private final SqlParser sqlParser; private final Optional hogQlCompiler; private final HogQlParameterDecoder hogQlParameterDecoder; + private final HogQlCompilationObserver hogQlCompilationObserver; public QueryPreparer(SqlParser sqlParser) { - this(sqlParser, Optional.empty()); + this(sqlParser, Optional.empty(), NOOP); } - @Inject public QueryPreparer(SqlParser sqlParser, HogQlCompiler hogQlCompiler) { - this(sqlParser, Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null"))); + this(sqlParser, Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), NOOP); + } + + @Inject + public QueryPreparer(SqlParser sqlParser, HogQlCompiler hogQlCompiler, HogQlCompilationObserver hogQlCompilationObserver) + { + this(sqlParser, Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), hogQlCompilationObserver); } - private QueryPreparer(SqlParser sqlParser, Optional hogQlCompiler) + private QueryPreparer(SqlParser sqlParser, Optional hogQlCompiler, HogQlCompilationObserver hogQlCompilationObserver) { this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); this.hogQlCompiler = requireNonNull(hogQlCompiler, "hogQlCompiler is null"); this.hogQlParameterDecoder = new HogQlParameterDecoder(sqlParser); + this.hogQlCompilationObserver = requireNonNull(hogQlCompilationObserver, "hogQlCompilationObserver is null"); } public PreparedQuery prepareQuery(Session session, String query) @@ -79,10 +92,23 @@ public PreparedQuery prepareQuery(Session session, QuerySubmission submission) case TRINO -> prepareQuery(session, sqlParser.createStatement(submission.originalText())); case HOGQL -> { HogQlCompileEnvelope envelope = submission.hogQlEnvelope().orElseThrow(); - HogQlCompilationResult result = hogQlCompiler - .orElseThrow(() -> new TrinoException(NOT_SUPPORTED, "HogQL query submission is disabled")) - .compile(envelope); - yield prepareQuery(session, result.statement(), Optional.of(hogQlParameterDecoder.decode(result, envelope.parameters()))); + HogQlCompilationTracker tracker = new HogQlCompilationTracker(hogQlCompilationObserver, Dimensions.fromEnvelope(envelope)); + PreparedQuery preparedQuery; + try { + HogQlCompilationResult result = tracker.observe(COMPILATION, () -> hogQlCompiler + .orElseThrow(() -> new TrinoException(NOT_SUPPORTED, "HogQL query submission is disabled")) + .compile(envelope)); + preparedQuery = tracker.observe(PARAMETER_BINDING, () -> prepareQuery( + session, + result.statement(), + Optional.of(hogQlParameterDecoder.decode(result, envelope.parameters())))); + } + catch (RuntimeException | Error failure) { + tracker.failed(failure); + throw failure; + } + tracker.succeeded(); + yield preparedQuery; } }; } diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationEvent.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationEvent.java new file mode 100644 index 000000000000..50637da09cff --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationEvent.java @@ -0,0 +1,105 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.parser.HogQlLanguageVersion; + +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; + +import static io.trino.hogql.HogQlCompilationEvent.Outcome.SUCCESS; +import static java.util.Objects.requireNonNull; + +public record HogQlCompilationEvent( + Dimensions dimensions, + Outcome outcome, + Optional failedPhase, + long totalNanos, + Map phaseNanos) +{ + public HogQlCompilationEvent + { + dimensions = requireNonNull(dimensions, "dimensions is null"); + outcome = requireNonNull(outcome, "outcome is null"); + failedPhase = requireNonNull(failedPhase, "failedPhase is null"); + if (outcome == SUCCESS && failedPhase.isPresent()) { + throw new IllegalArgumentException("successful compilation has a failed phase"); + } + if (totalNanos < 0) { + throw new IllegalArgumentException("totalNanos is negative"); + } + phaseNanos = Map.copyOf(requireNonNull(phaseNanos, "phaseNanos is null")); + if (phaseNanos.values().stream().anyMatch(duration -> duration == null || duration < 0)) { + throw new IllegalArgumentException("phaseNanos contains an invalid duration"); + } + } + + public enum Phase + { + COMPILATION, + PARSE, + BIND, + LOWER, + PARAMETER_BINDING, + } + + public enum Outcome + { + SUCCESS, + USER_ERROR, + INTERNAL_ERROR, + EXTERNAL_ERROR, + INSUFFICIENT_RESOURCES, + } + + public record Dimensions( + int protocolVersion, + HogQlLanguageVersion languageVersion, + int parameterCount, + int variableCount, + int filterCount, + int modifierCount, + OptionalLong catalogGeneration) + { + public Dimensions + { + if (protocolVersion <= 0) { + throw new IllegalArgumentException("protocolVersion must be positive"); + } + languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + if (parameterCount < 0 || variableCount < 0 || filterCount < 0 || modifierCount < 0) { + throw new IllegalArgumentException("semantic field count is negative"); + } + catalogGeneration = requireNonNull(catalogGeneration, "catalogGeneration is null"); + if (catalogGeneration.isPresent() && catalogGeneration.orElseThrow() <= 0) { + throw new IllegalArgumentException("catalogGeneration must be positive"); + } + } + + public static Dimensions fromEnvelope(HogQlCompileEnvelope envelope) + { + requireNonNull(envelope, "envelope is null"); + return new Dimensions( + envelope.protocolVersion(), + envelope.languageVersion(), + envelope.parameters().size(), + envelope.variables().size(), + envelope.filters().size(), + envelope.modifiers().size(), + envelope.catalogGeneration()); + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationObserver.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationObserver.java new file mode 100644 index 000000000000..e1b74ed4498a --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationObserver.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +@FunctionalInterface +public interface HogQlCompilationObserver +{ + HogQlCompilationObserver NOOP = _ -> {}; + + void compilationCompleted(HogQlCompilationEvent event); +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationStats.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationStats.java new file mode 100644 index 000000000000..b2e122b34eae --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationStats.java @@ -0,0 +1,149 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.stats.CounterStat; +import io.airlift.stats.TimeStat; +import io.trino.hogql.HogQlCompilationEvent.Phase; +import org.weakref.jmx.Managed; +import org.weakref.jmx.Nested; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; + +public final class HogQlCompilationStats + implements HogQlCompilationObserver +{ + private final CounterStat completedCompilations = new CounterStat(); + private final CounterStat successfulCompilations = new CounterStat(); + private final CounterStat userErrorFailures = new CounterStat(); + private final CounterStat internalErrorFailures = new CounterStat(); + private final CounterStat externalErrorFailures = new CounterStat(); + private final CounterStat insufficientResourcesFailures = new CounterStat(); + private final TimeStat totalTime = new TimeStat(NANOSECONDS); + private final TimeStat compilationTime = new TimeStat(NANOSECONDS); + private final TimeStat parseTime = new TimeStat(NANOSECONDS); + private final TimeStat bindTime = new TimeStat(NANOSECONDS); + private final TimeStat lowerTime = new TimeStat(NANOSECONDS); + private final TimeStat parameterBindingTime = new TimeStat(NANOSECONDS); + + @Override + public void compilationCompleted(HogQlCompilationEvent event) + { + completedCompilations.update(1); + totalTime.addNanos(event.totalNanos()); + event.phaseNanos().forEach((phase, duration) -> phaseTime(phase).addNanos(duration)); + switch (event.outcome()) { + case SUCCESS -> successfulCompilations.update(1); + case USER_ERROR -> userErrorFailures.update(1); + case INTERNAL_ERROR -> internalErrorFailures.update(1); + case EXTERNAL_ERROR -> externalErrorFailures.update(1); + case INSUFFICIENT_RESOURCES -> insufficientResourcesFailures.update(1); + } + } + + private TimeStat phaseTime(Phase phase) + { + return switch (phase) { + case COMPILATION -> compilationTime; + case PARSE -> parseTime; + case BIND -> bindTime; + case LOWER -> lowerTime; + case PARAMETER_BINDING -> parameterBindingTime; + }; + } + + @Managed + @Nested + public CounterStat getCompletedCompilations() + { + return completedCompilations; + } + + @Managed + @Nested + public CounterStat getSuccessfulCompilations() + { + return successfulCompilations; + } + + @Managed + @Nested + public CounterStat getUserErrorFailures() + { + return userErrorFailures; + } + + @Managed + @Nested + public CounterStat getInternalErrorFailures() + { + return internalErrorFailures; + } + + @Managed + @Nested + public CounterStat getExternalErrorFailures() + { + return externalErrorFailures; + } + + @Managed + @Nested + public CounterStat getInsufficientResourcesFailures() + { + return insufficientResourcesFailures; + } + + @Managed + @Nested + public TimeStat getTotalTime() + { + return totalTime; + } + + @Managed + @Nested + public TimeStat getCompilationTime() + { + return compilationTime; + } + + @Managed + @Nested + public TimeStat getParseTime() + { + return parseTime; + } + + @Managed + @Nested + public TimeStat getBindTime() + { + return bindTime; + } + + @Managed + @Nested + public TimeStat getLowerTime() + { + return lowerTime; + } + + @Managed + @Nested + public TimeStat getParameterBindingTime() + { + return parameterBindingTime; + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationTracker.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationTracker.java new file mode 100644 index 000000000000..1e20e3aee31c --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationTracker.java @@ -0,0 +1,123 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.trino.hogql.HogQlCompilationEvent.Dimensions; +import io.trino.hogql.HogQlCompilationEvent.Outcome; +import io.trino.hogql.HogQlCompilationEvent.Phase; +import io.trino.spi.ErrorType; +import io.trino.spi.TrinoException; + +import java.util.EnumMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.LongSupplier; +import java.util.function.Supplier; + +import static io.trino.hogql.HogQlCompilationEvent.Outcome.EXTERNAL_ERROR; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.INSUFFICIENT_RESOURCES; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.INTERNAL_ERROR; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.SUCCESS; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.USER_ERROR; +import static java.lang.Math.max; +import static java.util.Objects.requireNonNull; + +public final class HogQlCompilationTracker +{ + private final HogQlCompilationObserver observer; + private final Dimensions dimensions; + private final LongSupplier ticker; + private final long startedNanos; + private final EnumMap phaseNanos = new EnumMap<>(Phase.class); + + private Phase failedPhase; + private boolean completed; + + public HogQlCompilationTracker(HogQlCompilationObserver observer, Dimensions dimensions) + { + this(observer, dimensions, System::nanoTime); + } + + HogQlCompilationTracker(HogQlCompilationObserver observer, Dimensions dimensions, LongSupplier ticker) + { + this.observer = requireNonNull(observer, "observer is null"); + this.dimensions = requireNonNull(dimensions, "dimensions is null"); + this.ticker = requireNonNull(ticker, "ticker is null"); + startedNanos = ticker.getAsLong(); + } + + public T observe(Phase phase, Supplier operation) + { + requireNonNull(phase, "phase is null"); + requireNonNull(operation, "operation is null"); + if (completed) { + throw new IllegalStateException("HogQL compilation observation is complete"); + } + long phaseStartedNanos = ticker.getAsLong(); + try { + return operation.get(); + } + catch (RuntimeException | Error failure) { + failedPhase = phase; + throw failure; + } + finally { + phaseNanos.merge(phase, elapsedNanos(phaseStartedNanos, ticker.getAsLong()), Long::sum); + } + } + + public void succeeded() + { + complete(SUCCESS, Optional.empty()); + } + + public void failed(Throwable failure) + { + requireNonNull(failure, "failure is null"); + complete(outcome(failure), Optional.ofNullable(failedPhase)); + } + + private void complete(Outcome outcome, Optional failedPhase) + { + if (completed) { + throw new IllegalStateException("HogQL compilation observation is complete"); + } + completed = true; + observer.compilationCompleted(new HogQlCompilationEvent( + dimensions, + outcome, + failedPhase, + elapsedNanos(startedNanos, ticker.getAsLong()), + Map.copyOf(phaseNanos))); + } + + private static Outcome outcome(Throwable failure) + { + if (!(failure instanceof TrinoException trinoException)) { + return INTERNAL_ERROR; + } + ErrorType errorType = trinoException.getErrorCode().getType(); + return switch (errorType) { + case USER_ERROR -> USER_ERROR; + case INTERNAL_ERROR -> INTERNAL_ERROR; + case EXTERNAL -> EXTERNAL_ERROR; + case INSUFFICIENT_RESOURCES -> INSUFFICIENT_RESOURCES; + }; + } + + private static long elapsedNanos(long start, long end) + { + return max(0, end - start); + } +} diff --git a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java index 12da90a8df67..c2c9c9d8b4e3 100644 --- a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java +++ b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java @@ -98,6 +98,8 @@ import io.trino.execution.scheduler.policy.AllAtOnceExecutionPolicy; import io.trino.execution.scheduler.policy.ExecutionPolicy; import io.trino.execution.scheduler.policy.PhasedExecutionPolicy; +import io.trino.hogql.HogQlCompilationObserver; +import io.trino.hogql.HogQlCompilationStats; import io.trino.hogql.HogQlConfig; import io.trino.hogql.HogQlSemanticCatalogConfig; import io.trino.hogql.HogQlSemanticCatalogModule; @@ -183,6 +185,9 @@ protected void setup(Binder binder) configBinder(binder).bindConfig(HogQlConfig.class); configBinder(binder).bindConfig(HogQlSemanticCatalogConfig.class); binder.bind(HogQlCompiler.class).in(Scopes.SINGLETON); + binder.bind(HogQlCompilationStats.class).in(Scopes.SINGLETON); + binder.bind(HogQlCompilationObserver.class).to(HogQlCompilationStats.class); + newExporter(binder).export(HogQlCompilationStats.class).withGeneratedName(); if (buildConfigObject(HogQlConfig.class).isEnabled()) { jaxrsBinder(binder).bind(HogQlStatementResource.class); if (buildConfigObject(HogQlSemanticCatalogConfig.class).getUri() != null) { diff --git a/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerObservability.java b/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerObservability.java new file mode 100644 index 000000000000..33bb59adbd15 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerObservability.java @@ -0,0 +1,93 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +import io.trino.hogql.HogQlCompilationEvent; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.HogQlTypedValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; + +import static io.trino.execution.QuerySubmission.hogQl; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.SUCCESS; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.USER_ERROR; +import static io.trino.hogql.HogQlCompilationEvent.Phase.COMPILATION; +import static io.trino.hogql.HogQlCompilationEvent.Phase.PARAMETER_BINDING; +import static io.trino.testing.TestingSession.testSessionBuilder; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlQueryPreparerObservability +{ + private static final String SECRET = "sensitive-observability-value"; + + @Test + public void testHogQlPreparationEmitsOneRedactedEventAndTrinoDoesNot() + { + List events = new ArrayList<>(); + QueryPreparer queryPreparer = new QueryPreparer(new SqlParser(), new HogQlCompiler(), events::add); + + queryPreparer.prepareQuery(testSessionBuilder().build(), hogQl(envelope("SELECT {input}", Map.of("input", typedValue())))); + queryPreparer.prepareQuery(testSessionBuilder().build(), "SELECT '" + SECRET + "'"); + + assertThat(events).singleElement().satisfies(event -> { + assertThat(event.outcome()).isEqualTo(SUCCESS); + assertThat(event.phaseNanos()).containsKeys(COMPILATION, PARAMETER_BINDING); + assertThat(event.toString()).doesNotContain(SECRET); + }); + } + + @Test + public void testCompilerFailureRecordsRedactedOutcomeAndPhase() + { + List events = new ArrayList<>(); + QueryPreparer queryPreparer = new QueryPreparer(new SqlParser(), new HogQlCompiler(), events::add); + + assertThatThrownBy(() -> queryPreparer.prepareQuery( + testSessionBuilder().build(), + hogQl(envelope("SELECT '" + SECRET, Map.of())))) + .isInstanceOf(RuntimeException.class); + + assertThat(events).singleElement().satisfies(event -> { + assertThat(event.outcome()).isEqualTo(USER_ERROR); + assertThat(event.failedPhase()).contains(COMPILATION); + assertThat(event.toString()).doesNotContain(SECRET); + }); + } + + private static HogQlCompileEnvelope envelope(String query, Map parameters) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + io.trino.hogql.parser.HogQlLanguageContract.current().languageVersion(), + parameters, + Map.of(), + Map.of(), + Map.of(), + OptionalLong.empty()); + } + + private static HogQlTypedValue typedValue() + { + return new HogQlTypedValue("varchar", new StringValue(SECRET)); + } +} diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlCompilationObservability.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlCompilationObservability.java new file mode 100644 index 000000000000..cb38dee35c79 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlCompilationObservability.java @@ -0,0 +1,149 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.trino.hogql.HogQlCompilationEvent.Dimensions; +import io.trino.hogql.HogQlCompilationEvent.Outcome; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlTypedValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.spi.TrinoException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; +import java.util.stream.Stream; + +import static io.trino.hogql.HogQlCompilationEvent.Outcome.EXTERNAL_ERROR; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.INTERNAL_ERROR; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.SUCCESS; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.USER_ERROR; +import static io.trino.hogql.HogQlCompilationEvent.Phase.BIND; +import static io.trino.hogql.HogQlCompilationEvent.Phase.LOWER; +import static io.trino.hogql.HogQlCompilationEvent.Phase.PARSE; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_CATALOG_NOT_READY; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_SYNTAX_ERROR; +import static io.trino.hogql.parser.HogQlLanguageContract.current; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchRuntimeException; + +public class TestHogQlCompilationObservability +{ + private static final String QUERY_SECRET = "literal-query-secret"; + private static final String VALUE_SECRET = "typed-value-secret"; + private static final String NAME_SECRET = "semantic-name-secret"; + + @Test + public void testEventContainsOnlyRedactedDimensionsAndPhaseTimings() + { + AtomicLong ticker = new AtomicLong(); + List events = new ArrayList<>(); + HogQlCompilationTracker tracker = new HogQlCompilationTracker(events::add, Dimensions.fromEnvelope(envelope()), ticker::get); + + assertThat(tracker.observe(PARSE, () -> { + ticker.addAndGet(11); + return "parsed"; + })).isEqualTo("parsed"); + tracker.observe(BIND, () -> { + ticker.addAndGet(13); + return null; + }); + ticker.addAndGet(17); + tracker.succeeded(); + + assertThat(events).singleElement().satisfies(event -> { + assertThat(event.dimensions()).isEqualTo(new Dimensions( + HogQlCompileEnvelope.PROTOCOL_VERSION, + current().languageVersion(), + 1, + 1, + 1, + 1, + OptionalLong.of(41))); + assertThat(event.outcome()).isEqualTo(SUCCESS); + assertThat(event.failedPhase()).isEmpty(); + assertThat(event.totalNanos()).isEqualTo(41); + assertThat(event.phaseNanos()).containsExactlyInAnyOrderEntriesOf(Map.of(PARSE, 11L, BIND, 13L)); + assertThat(event.toString()).doesNotContain(QUERY_SECRET, VALUE_SECRET, NAME_SECRET); + }); + } + + @ParameterizedTest + @MethodSource("failures") + public void testFailureOutcomeAndPhaseAreStable(Supplier failureSupplier, Outcome expectedOutcome) + { + List events = new ArrayList<>(); + HogQlCompilationTracker tracker = new HogQlCompilationTracker(events::add, Dimensions.fromEnvelope(envelope()), () -> 0); + + RuntimeException failure = catchRuntimeException(() -> tracker.observe(LOWER, () -> { + throw failureSupplier.get(); + })); + tracker.failed(failure); + + assertThat(events).singleElement().satisfies(event -> { + assertThat(event.outcome()).isEqualTo(expectedOutcome); + assertThat(event.failedPhase()).contains(LOWER); + assertThat(event.toString()).doesNotContain(QUERY_SECRET, VALUE_SECRET, NAME_SECRET); + }); + } + + @Test + public void testStatsRecordOutcomesAndPhaseDurations() + { + HogQlCompilationStats stats = new HogQlCompilationStats(); + Dimensions dimensions = Dimensions.fromEnvelope(envelope()); + + stats.compilationCompleted(new HogQlCompilationEvent(dimensions, SUCCESS, Optional.empty(), 100, Map.of(PARSE, 10L, BIND, 20L, LOWER, 30L))); + stats.compilationCompleted(new HogQlCompilationEvent(dimensions, USER_ERROR, Optional.of(PARSE), 50, Map.of(PARSE, 40L))); + + assertThat(stats.getCompletedCompilations().getTotalCount()).isEqualTo(2); + assertThat(stats.getSuccessfulCompilations().getTotalCount()).isEqualTo(1); + assertThat(stats.getUserErrorFailures().getTotalCount()).isEqualTo(1); + assertThat(stats.getInternalErrorFailures().getTotalCount()).isZero(); + assertThat(stats.getTotalTime().getAllTime().getCount()).isEqualTo(2); + assertThat(stats.getParseTime().getAllTime().getCount()).isEqualTo(2); + assertThat(stats.getBindTime().getAllTime().getCount()).isEqualTo(1); + assertThat(stats.getLowerTime().getAllTime().getCount()).isEqualTo(1); + } + + private static Stream failures() + { + return Stream.of( + Arguments.of((Supplier) () -> new TrinoException(HOGQL_SYNTAX_ERROR, "synthetic syntax failure"), USER_ERROR), + Arguments.of((Supplier) () -> new TrinoException(HOGQL_CATALOG_NOT_READY, "synthetic catalog failure"), EXTERNAL_ERROR), + Arguments.of((Supplier) () -> new IllegalStateException("synthetic internal failure"), INTERNAL_ERROR)); + } + + private static HogQlCompileEnvelope envelope() + { + HogQlTypedValue typedValue = new HogQlTypedValue("varchar", new StringValue(VALUE_SECRET)); + return new HogQlCompileEnvelope( + "SELECT '" + QUERY_SECRET + "'", + HogQlCompileEnvelope.PROTOCOL_VERSION, + current().languageVersion(), + Map.of(NAME_SECRET + "-parameter", typedValue), + Map.of(NAME_SECRET + "-variable", typedValue), + Map.of(NAME_SECRET + "-filter", typedValue), + Map.of(NAME_SECRET + "-modifier", typedValue), + OptionalLong.of(41)); + } +} From 114b7fb4401b4cac2fa53e98b217d751fd69f52d Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 11:11:07 -0400 Subject: [PATCH 034/136] feat(hogql): resolve semantic catalog tables --- .../compiler/HogQlCompilationResult.java | 4 +- .../trino/hogql/compiler/HogQlCompiler.java | 67 ++++- .../compiler/HogQlSemanticCatalogContext.java | 30 +++ .../hogql/compiler/HogQlSemanticResolver.java | 233 ++++++++++++++++++ .../compiler/TestHogQlSemanticResolution.java | 149 +++++++++++ 5 files changed, 477 insertions(+), 6 deletions(-) create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticCatalogContext.java create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java index e7878b099b80..ff2f821f63f2 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java @@ -16,14 +16,16 @@ import io.trino.sql.tree.Statement; import java.util.List; +import java.util.OptionalLong; import static java.util.Objects.requireNonNull; -public record HogQlCompilationResult(Statement statement, List parameterNames) +public record HogQlCompilationResult(Statement statement, List parameterNames, OptionalLong catalogGeneration) { public HogQlCompilationResult { statement = requireNonNull(statement, "statement is null"); parameterNames = List.copyOf(requireNonNull(parameterNames, "parameterNames is null")); + catalogGeneration = requireNonNull(catalogGeneration, "catalogGeneration is null"); } } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index 98929496a55a..8b55f762bf5b 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -13,6 +13,10 @@ */ package io.trino.hogql.compiler; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinRequest; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.hogql.parser.HogQlLanguageVersion; import io.trino.hogql.parser.HogQlParser; import io.trino.hogql.parser.HogQlParsingException; import io.trino.hogql.parser.tree.HogQlQuery; @@ -43,6 +47,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_BINDING_ERROR; @@ -69,17 +74,34 @@ public Statement compile(String hogql) } public HogQlCompilationResult compile(HogQlCompileEnvelope envelope) + { + return compile(envelope, Optional.empty()); + } + + public HogQlCompilationResult compile(HogQlCompileEnvelope envelope, Optional catalogContext) { requireNonNull(envelope, "envelope is null"); - return compile(envelope.query(), envelope.parameters()); + requireNonNull(catalogContext, "catalogContext is null"); + return compile( + parse(envelope.query()), + envelope.parameters(), + catalogContext, + envelope.languageVersion(), + envelope.catalogGeneration()); } public HogQlCompilationResult compile(String hogql, Map parameters) { requireNonNull(hogql, "hogql is null"); requireNonNull(parameters, "parameters is null"); + return compile(parse(hogql), parameters, Optional.empty(), HogQlLanguageContract.current().languageVersion(), OptionalLong.empty()); + } + + private HogQlQuery parse(String hogql) + { + requireNonNull(hogql, "hogql is null"); try { - return compile(parser.parseStatement(hogql), parameters); + return parser.parseStatement(hogql); } catch (HogQlParsingException exception) { throw new TrinoException( @@ -90,7 +112,12 @@ public HogQlCompilationResult compile(String hogql, Map } } - private static HogQlCompilationResult compile(HogQlQuery query, Map parameters) + private static HogQlCompilationResult compile( + HogQlQuery query, + Map parameters, + Optional catalogContext, + HogQlLanguageVersion languageVersion, + OptionalLong expectedCatalogGeneration) { validateParameters(query, parameters); if (query.from().orElse(null) instanceof TablePlaceholder tablePlaceholder) { @@ -140,10 +167,40 @@ private static HogQlCompilationResult compile(HogQlQuery query, Map catalogContext, + HogQlLanguageVersion languageVersion, + OptionalLong expectedCatalogGeneration) + { + if (!(query.from().orElse(null) instanceof HogQlQuery.TableReference table) || table.parts().size() != 1 || catalogContext.isEmpty()) { + return new ResolvedQuery(query, OptionalLong.empty()); + } + HogQlSemanticCatalogContext context = catalogContext.orElseThrow(); + PinnedSnapshot pinned = context.snapshotProvider().pin(new PinRequest( + context.catalog(), + requireNonNull(languageVersion, "languageVersion is null"), + expectedCatalogGeneration)); + HogQlQuery resolved = HogQlSemanticResolver.resolve(pinned, query) + .map(HogQlSemanticResolver.ResolvedQuery::query) + .orElse(query); + return new ResolvedQuery(resolved, OptionalLong.of(pinned.generation())); + } + + private record ResolvedQuery(HogQlQuery query, OptionalLong catalogGeneration) + { + private ResolvedQuery + { + query = requireNonNull(query, "query is null"); + catalogGeneration = requireNonNull(catalogGeneration, "catalogGeneration is null"); + } } private static void validateParameters(HogQlQuery query, Map parameters) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticCatalogContext.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticCatalogContext.java new file mode 100644 index 000000000000..2197b13c41a2 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticCatalogContext.java @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider; + +import static java.util.Objects.requireNonNull; + +public record HogQlSemanticCatalogContext( + PhysicalIdentifier catalog, + HogQlSemanticCatalogSnapshotProvider snapshotProvider) +{ + public HogQlSemanticCatalogContext + { + catalog = requireNonNull(catalog, "catalog is null"); + snapshotProvider = requireNonNull(snapshotProvider, "snapshotProvider is null"); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java new file mode 100644 index 000000000000..c24841913b68 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -0,0 +1,233 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseWhen; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.SortItem; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; +import static java.util.Objects.requireNonNull; + +final class HogQlSemanticResolver +{ + private final LogicalTableDefinition logicalTable; + private final Map fields; + + private HogQlSemanticResolver(LogicalTableDefinition logicalTable) + { + this.logicalTable = requireNonNull(logicalTable, "logicalTable is null"); + Map fields = new HashMap<>(); + logicalTable.fields().forEach(field -> fields.put(canonical(field.name()), field)); + this.fields = Map.copyOf(fields); + } + + public static Optional resolve(PinnedSnapshot snapshot, HogQlQuery query) + { + requireNonNull(snapshot, "snapshot is null"); + requireNonNull(query, "query is null"); + if (!(query.from().orElse(null) instanceof TableReference table) || table.parts().size() != 1) { + return Optional.empty(); + } + Optional logicalTable = snapshot.logicalTable(table.parts().getFirst().value()); + return logicalTable.map(definition -> new HogQlSemanticResolver(definition).resolveQuery(query, table)); + } + + private ResolvedQuery resolveQuery(HogQlQuery query, TableReference originalTable) + { + List projections = new ArrayList<>(); + query.projections().forEach(projection -> projections.addAll(resolveProjection(projection))); + HogQlQuery resolved = new HogQlQuery( + query.distinct(), + projections, + Optional.of(resolveTable(originalTable)), + query.where().map(this::resolveExpression), + query.groupBy().stream().map(this::resolveExpression).toList(), + query.having().map(this::resolveExpression), + resolveSortItems(query.orderBy()), + query.limit().map(this::resolveExpression), + query.offset().map(this::resolveExpression), + query.span()); + return new ResolvedQuery(resolved); + } + + private List resolveProjection(Projection projection) + { + return switch (projection) { + case Star star -> logicalTable.fields().stream() + .filter(LogicalFieldDefinition::starVisible) + .map(field -> new ExpressionProjection( + physicalColumn(field, star.span()), + Optional.of(new Identifier(field.name(), true, star.span())))) + .map(Projection.class::cast) + .toList(); + case ExpressionProjection expressionProjection -> { + Expression resolved = resolveExpression(expressionProjection.expression()); + Optional alias = expressionProjection.alias(); + if (alias.isEmpty() && expressionProjection.expression() instanceof ColumnReference reference) { + alias = Optional.of(reference.parts().getLast()); + } + yield List.of(new ExpressionProjection(resolved, alias)); + } + }; + } + + private TableReference resolveTable(TableReference table) + { + return new TableReference( + List.of( + identifier(logicalTable.physicalTable().catalog(), table), + identifier(logicalTable.physicalTable().schema(), table), + identifier(logicalTable.physicalTable().table(), table)), + table.span()); + } + + private List resolveSortItems(List sortItems) + { + return sortItems.stream() + .map(sortItem -> new SortItem(resolveExpression(sortItem.expression()), sortItem.direction(), sortItem.nullPlacement(), sortItem.span())) + .toList(); + } + + private Expression resolveExpression(Expression expression) + { + return switch (expression) { + case ArrayExpression array -> new ArrayExpression(array.values().stream().map(this::resolveExpression).toList(), array.span()); + case BetweenExpression between -> new BetweenExpression( + resolveExpression(between.value()), + resolveExpression(between.min()), + resolveExpression(between.max()), + between.negated(), + between.predicateSpan(), + between.span()); + case BinaryExpression binary -> new BinaryExpression( + binary.operator(), + resolveExpression(binary.left()), + resolveExpression(binary.right()), + binary.span()); + case CaseExpression caseExpression -> new CaseExpression( + caseExpression.operand().map(this::resolveExpression), + caseExpression.whenClauses().stream() + .map(when -> new CaseWhen(resolveExpression(when.operand()), resolveExpression(when.result()), when.span())) + .toList(), + caseExpression.defaultValue().map(this::resolveExpression), + caseExpression.span()); + case CastExpression cast -> new CastExpression(resolveExpression(cast.value()), cast.type(), cast.safe(), cast.span()); + case ColumnReference reference -> resolveColumn(reference); + case FunctionCall function -> new FunctionCall( + function.name(), + function.arguments().stream().map(this::resolveExpression).toList(), + function.distinct(), + resolveSortItems(function.orderBy()), + function.filter().map(this::resolveExpression), + function.span()); + case InExpression in -> new InExpression( + resolveExpression(in.value()), + in.values().stream().map(this::resolveExpression).toList(), + in.negated(), + in.predicateSpan(), + in.span()); + case IsNullExpression isNull -> new IsNullExpression( + resolveExpression(isNull.value()), + isNull.negated(), + isNull.predicateSpan(), + isNull.span()); + case Literal literal -> literal; + case Placeholder placeholder -> placeholder; + case TupleExpression tuple -> new TupleExpression(tuple.values().stream().map(this::resolveExpression).toList(), tuple.span()); + case UnaryExpression unary -> new UnaryExpression(unary.operator(), resolveExpression(unary.operand()), unary.span()); + }; + } + + private ColumnReference resolveColumn(ColumnReference reference) + { + List parts = reference.parts(); + if (parts.size() == 2 && !canonical(parts.getFirst().value()).equals(canonical(logicalTable.name()))) { + throw resolutionError(reference, parts.getLast().value()); + } + if (parts.size() > 2) { + throw resolutionError(reference, parts.getLast().value()); + } + String logicalName = parts.getLast().value(); + LogicalFieldDefinition field = fields.get(canonical(logicalName)); + if (field == null) { + throw resolutionError(reference, logicalName); + } + return physicalColumn(field, reference.span()); + } + + private static ColumnReference physicalColumn(LogicalFieldDefinition field, HogQlQuery.SourceSpan span) + { + return new ColumnReference(List.of(new Identifier(field.physicalColumn().value(), field.physicalColumn().delimited(), span)), span); + } + + private static Identifier identifier(PhysicalIdentifier identifier, TableReference source) + { + return new Identifier(identifier.value(), identifier.delimited(), source.span()); + } + + private static String canonical(String value) + { + return value.toLowerCase(Locale.ENGLISH); + } + + private static TrinoException resolutionError(ColumnReference reference, String name) + { + return new TrinoException( + HOGQL_RESOLUTION_ERROR, + Optional.of(new Location(reference.span().startLine(), reference.span().startColumn())), + "Unknown HogQL field: " + name, + null); + } + + record ResolvedQuery(HogQlQuery query) + { + ResolvedQuery + { + query = requireNonNull(query, "query is null"); + } + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java new file mode 100644 index 000000000000..85bd7f921601 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -0,0 +1,149 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinRequest; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; +import io.trino.sql.tree.Identifier; +import io.trino.sql.tree.Query; +import io.trino.sql.tree.QuerySpecification; +import io.trino.sql.tree.SingleColumn; +import io.trino.sql.tree.Table; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlSemanticResolution +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); + private static final HogQlSemanticCatalogSnapshot SNAPSHOT = new HogQlSemanticCatalogSnapshot( + 1, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 7, + List.of(new LogicalTableDefinition( + "events", + new PhysicalQualifiedName( + CATALOG, + new PhysicalIdentifier("Hog Data", true), + new PhysicalIdentifier("raw-events", true)), + List.of( + new LogicalFieldDefinition("event", new PhysicalIdentifier("event_name", false), "varchar", LogicalType.STRING, false, true), + new LogicalFieldDefinition("personId", new PhysicalIdentifier("Person ID", true), "varchar", LogicalType.STRING, false, true), + new LogicalFieldDefinition("hidden", new PhysicalIdentifier("hidden", false), "bigint", LogicalType.INTEGER, true, false)), + List.of(), + List.of()))); + + private final HogQlCompiler compiler = new HogQlCompiler(); + + @Test + public void testPinsAndResolvesLogicalTableFieldsAndStar() + { + AtomicReference request = new AtomicReference<>(); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, pinRequest -> { + request.set(pinRequest); + return new PinnedSnapshot(SNAPSHOT); + }); + + HogQlCompilationResult result = compiler.compile( + envelope("SELECT *, event FROM events WHERE personId = 'synthetic' GROUP BY event HAVING count(*) > 0 ORDER BY event", OptionalLong.of(7)), + Optional.of(context)); + + assertThat(request.get()).isEqualTo(new PinRequest(CATALOG, HogQlLanguageContract.current().languageVersion(), OptionalLong.of(7))); + assertThat(result.catalogGeneration()).hasValue(7); + QuerySpecification query = querySpecification(result); + assertThat(((Table) query.getFrom().orElseThrow()).getName().getOriginalParts()) + .extracting(Identifier::getValue) + .containsExactly("analytics", "Hog Data", "raw-events"); + assertThat(((Table) query.getFrom().orElseThrow()).getName().getOriginalParts()) + .extracting(Identifier::isDelimited) + .containsExactly(false, true, true); + assertThat(query.getSelect().getSelectItems()).hasSize(3); + assertThat(query.getSelect().getSelectItems()) + .allSatisfy(item -> assertThat(item).isInstanceOf(SingleColumn.class)); + assertThat(query.getSelect().getSelectItems().subList(0, 2)) + .extracting(item -> ((SingleColumn) item).getAlias().orElseThrow().getValue()) + .containsExactly("event", "personId"); + assertThat(query.getSelect().getSelectItems().subList(0, 2)) + .extracting(item -> ((Identifier) ((SingleColumn) item).getExpression()).getValue()) + .containsExactly("event_name", "Person ID"); + } + + @Test + public void testCatalogIndependentAndPhysicalQueriesDoNotFetchSemanticMetadata() + { + AtomicInteger pins = new AtomicInteger(); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + }); + + HogQlCompilationResult literal = compiler.compile(envelope("SELECT 1", OptionalLong.empty()), Optional.of(context)); + HogQlCompilationResult physical = compiler.compile(envelope("SELECT event_name FROM analytics.default.events", OptionalLong.empty()), Optional.of(context)); + + assertThat(pins).hasValue(0); + assertThat(literal.catalogGeneration()).isEmpty(); + assertThat(physical.catalogGeneration()).isEmpty(); + } + + @Test + public void testUnknownLogicalFieldFailsAtOriginalLocation() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope("SELECT missing FROM events", OptionalLong.empty()), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, 8)); + assertThat(exception).hasMessage("line 1:8: Unknown HogQL field: missing"); + } + + private static QuerySpecification querySpecification(HogQlCompilationResult result) + { + return (QuerySpecification) ((Query) result.statement()).getQueryBody(); + } + + private static HogQlCompileEnvelope envelope(String query, OptionalLong generation) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + generation); + } +} From 6bcfdc31619f94fbb848384ef04b4cc6f9a646f8 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 11:17:38 -0400 Subject: [PATCH 035/136] fix(hogql): bound request envelope resources --- .../io/trino/dispatcher/HogQlRequest.java | 12 +++- .../dispatcher/HogQlStatementResource.java | 2 +- .../TestHogQlStatementResource.java | 70 +++++++++++++------ 3 files changed, 60 insertions(+), 24 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java index 204e83dcf2a5..2e5ecf1b0352 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java @@ -14,6 +14,7 @@ package io.trino.dispatcher; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Utf8; import io.airlift.json.JsonCodec; import io.trino.hogql.compiler.HogQlCompileEnvelope; import io.trino.hogql.compiler.HogQlTypedValue; @@ -59,7 +60,9 @@ record HogQlRequest( "catalogGeneration"); private static final Set TYPED_VALUE_FIELDS = Set.of("type", "value"); private static final int MAX_VALUE_DEPTH = 64; + private static final int MAX_REQUEST_BYTES = 2 * 1024 * 1024; private static final int MAX_BINDINGS_PER_FIELD = 1_000; + private static final int MAX_TOTAL_BINDINGS = 1_000; HogQlRequest { @@ -70,11 +73,18 @@ record HogQlRequest( filters = Map.copyOf(requireNonNull(filters, "filters is null")); modifiers = Map.copyOf(requireNonNull(modifiers, "modifiers is null")); catalogGeneration = requireNonNull(catalogGeneration, "catalogGeneration is null"); + if (parameters.size() + variables.size() + filters.size() + modifiers.size() > MAX_TOTAL_BINDINGS) { + throw new IllegalArgumentException("HogQL request has too many bindings"); + } } static HogQlRequest fromJson(String requestBody) { - JsonNode root = JSON_CODEC.fromJson(requireNonNull(requestBody, "requestBody is null")); + requestBody = requireNonNull(requestBody, "requestBody is null"); + if (Utf8.encodedLength(requestBody) > MAX_REQUEST_BYTES) { + throw new IllegalArgumentException("HogQL request is too large"); + } + JsonNode root = JSON_CODEC.fromJson(requestBody); if (root == null || !root.isObject()) { throw new IllegalArgumentException("HogQL request must be an object"); } diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java index 8581c887e54b..610e8ddfb0df 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java @@ -18,8 +18,8 @@ import io.trino.server.ExternalUriInfo; import io.trino.server.security.ResourceSecurity; import jakarta.servlet.http.HttpServletRequest; -import jakarta.ws.rs.BeanParam; import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.BeanParam; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; diff --git a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java index 41748f14a26d..fe61699c478f 100644 --- a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java +++ b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java @@ -177,26 +177,35 @@ private static Stream invalidRequests() { String languageVersion = HogQlLanguageContract.current().languageVersion().toString(); return Stream.of( - Arguments.of("unsupported protocol version", """ + Arguments.of("unsupported protocol version", + """ {"query":"SELECT '%s'","protocolVersion":2,"languageVersion":"%s"} """.formatted(SECRET, languageVersion)), - Arguments.of("unsupported language version", """ + Arguments.of("unsupported language version", + """ {"query":"SELECT '%s'","protocolVersion":1,"languageVersion":"999.0.0"} """.formatted(SECRET)), - Arguments.of("unknown top-level field", """ + Arguments.of("unknown top-level field", + """ {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","unknown":"%s"} """.formatted(languageVersion, SECRET)), - Arguments.of("unknown typed-value field", """ + Arguments.of("unknown typed-value field", + """ {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{"p":{"type":"string","value":"%s","unknown":true}}} """.formatted(languageVersion, SECRET)), - Arguments.of("ambiguous typed value", """ + Arguments.of("ambiguous typed value", + """ {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{"p":{"type":"string","unknown":"%s"}}} """.formatted(languageVersion, SECRET)), - Arguments.of("nonpositive catalog generation", """ + Arguments.of("nonpositive catalog generation", + """ {"query":"SELECT '%s'","protocolVersion":1,"languageVersion":"%s","catalogGeneration":0} """.formatted(SECRET, languageVersion)), Arguments.of("too many parameter bindings", requestWithParameterCount(languageVersion, 1_001)), - Arguments.of("malformed JSON", """ + Arguments.of("too many total bindings", requestWithTotalBindingCount(languageVersion)), + Arguments.of("request exceeds byte limit", oversizedRequest(languageVersion)), + Arguments.of("malformed JSON", + """ {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{"p":{"type":"string","value":"%s"}} """.formatted(languageVersion, SECRET))); } @@ -207,8 +216,25 @@ private static String requestWithParameterCount(String languageVersion, int para .mapToObj(index -> "\"p%s\":{\"type\":\"integer\",\"value\":%s}".formatted(index, index)) .collect(joining(",")); return """ - {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{%s}} - """.formatted(languageVersion, parameters); + {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{%s}} + """.formatted(languageVersion, parameters); + } + + private static String requestWithTotalBindingCount(String languageVersion) + { + String parameters = range(0, 1_000) + .mapToObj(index -> "\"p%s\":{\"type\":\"integer\",\"value\":%s}".formatted(index, index)) + .collect(joining(",")); + return """ + {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{%s},"variables":{"extra":{"type":"integer","value":1}}} + """.formatted(languageVersion, parameters); + } + + private static String oversizedRequest(String languageVersion) + { + return """ + {"query":"SELECT '%s%s'","protocolVersion":1,"languageVersion":"%s"} + """.formatted(SECRET, "x".repeat(2 * 1024 * 1024), languageVersion); } private static List runHogQlToCompletion(String request) @@ -268,24 +294,24 @@ private static StringResponse post(String body, String contentType) private static String hogQlRequest(String query) { return """ - {"query":%s,"protocolVersion":1,"languageVersion":"%s"} - """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); + {"query":%s,"protocolVersion":1,"languageVersion":"%s"} + """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); } private static String hogQlRequestWithTypedValues(String query) { return """ - { - "query": %s, - "protocolVersion": 1, - "languageVersion": "%s", - "parameters": {}, - "variables": {"array": {"type": "array", "value": [true, "value", null]}}, - "filters": {"object": {"type": "object", "value": {"nested": 2.5}}}, - "modifiers": {"missing": {"type": "nullable", "value": null}}, - "catalogGeneration": 1 - } - """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); + { + "query": %s, + "protocolVersion": 1, + "languageVersion": "%s", + "parameters": {}, + "variables": {"array": {"type": "array", "value": [true, "value", null]}}, + "filters": {"object": {"type": "object", "value": {"nested": 2.5}}}, + "modifiers": {"missing": {"type": "nullable", "value": null}}, + "catalogGeneration": 1 + } + """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); } private static List> rows(List results) From 52a3ffeea9701940327f8cf1548ce8aa21a0ba50 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 11:17:47 -0400 Subject: [PATCH 036/136] fix(hogql): use Java IP literal validation --- .../main/java/io/trino/execution/HogQlParameterDecoder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java b/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java index 3d4f23f400ed..077af3b42fc1 100644 --- a/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java +++ b/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.collect.ImmutableList; -import com.google.common.net.InetAddresses; import io.trino.hogql.compiler.HogQlCompilationResult; import io.trino.hogql.compiler.HogQlTypedValue; import io.trino.hogql.compiler.HogQlTypedValue.ArrayValue; @@ -62,6 +61,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.math.RoundingMode; +import java.net.InetAddress; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -228,7 +228,7 @@ private static Expression decodeString(StringValue value, GenericDataType type, } case "IPADDRESS" -> { requireNoArguments(type); - InetAddresses.forString(value.value()); + InetAddress.ofLiteral(value.value()); yield new GenericLiteral(location, "IPADDRESS", value.value()); } case "VARBINARY" -> { From bb8bc395579d8f692161dd2ce38758cfbf1166f5 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 11:17:55 -0400 Subject: [PATCH 037/136] fix(hogql): declare parser dependency --- core/trino-main/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/trino-main/pom.xml b/core/trino-main/pom.xml index 5118c2a894d5..57aee720c48f 100644 --- a/core/trino-main/pom.xml +++ b/core/trino-main/pom.xml @@ -250,6 +250,11 @@ trino-hogql-compiler + + io.trino + trino-hogql-parser + + io.trino trino-matching From 6b2de4165e4ad7aeb273ef61af53cfedf7b63ac5 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 11:23:37 -0400 Subject: [PATCH 038/136] feat(hogql): resolve session semantic catalogs --- .../io/trino/execution/QueryPreparer.java | 49 +++++++- .../hogql/HogQlSemanticCatalogModule.java | 26 +++- .../io/trino/server/CoordinatorModule.java | 2 + ...TestHogQlQueryPreparerSemanticCatalog.java | 118 ++++++++++++++++++ 4 files changed, 185 insertions(+), 10 deletions(-) create mode 100644 core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerSemanticCatalog.java diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java index 6c541e49e978..2b932bdc2203 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java @@ -22,6 +22,9 @@ import io.trino.hogql.compiler.HogQlCompilationResult; import io.trino.hogql.compiler.HogQlCompileEnvelope; import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.HogQlSemanticCatalogContext; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider; import io.trino.spi.TrinoException; import io.trino.spi.resourcegroups.QueryType; import io.trino.sql.parser.ParsingException; @@ -33,7 +36,9 @@ import io.trino.sql.tree.Statement; import java.util.List; +import java.util.Locale; import java.util.Optional; +import java.util.regex.Pattern; import static io.trino.execution.ParameterExtractor.getParameterCount; import static io.trino.hogql.HogQlCompilationEvent.Phase.COMPILATION; @@ -49,33 +54,53 @@ public class QueryPreparer { + private static final Pattern UNDELIMITED_IDENTIFIER = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*"); + private final SqlParser sqlParser; private final Optional hogQlCompiler; private final HogQlParameterDecoder hogQlParameterDecoder; private final HogQlCompilationObserver hogQlCompilationObserver; + private final Optional hogQlSemanticCatalogSnapshotProvider; public QueryPreparer(SqlParser sqlParser) { - this(sqlParser, Optional.empty(), NOOP); + this(sqlParser, Optional.empty(), NOOP, Optional.empty()); } public QueryPreparer(SqlParser sqlParser, HogQlCompiler hogQlCompiler) { - this(sqlParser, Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), NOOP); + this(sqlParser, Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), NOOP, Optional.empty()); } - @Inject public QueryPreparer(SqlParser sqlParser, HogQlCompiler hogQlCompiler, HogQlCompilationObserver hogQlCompilationObserver) { - this(sqlParser, Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), hogQlCompilationObserver); + this(sqlParser, Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), hogQlCompilationObserver, Optional.empty()); + } + + @Inject + public QueryPreparer( + SqlParser sqlParser, + HogQlCompiler hogQlCompiler, + HogQlCompilationObserver hogQlCompilationObserver, + Optional hogQlSemanticCatalogSnapshotProvider) + { + this(sqlParser, + Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), + hogQlCompilationObserver, + hogQlSemanticCatalogSnapshotProvider); } - private QueryPreparer(SqlParser sqlParser, Optional hogQlCompiler, HogQlCompilationObserver hogQlCompilationObserver) + private QueryPreparer( + SqlParser sqlParser, + Optional hogQlCompiler, + HogQlCompilationObserver hogQlCompilationObserver, + Optional hogQlSemanticCatalogSnapshotProvider) { this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); this.hogQlCompiler = requireNonNull(hogQlCompiler, "hogQlCompiler is null"); this.hogQlParameterDecoder = new HogQlParameterDecoder(sqlParser); this.hogQlCompilationObserver = requireNonNull(hogQlCompilationObserver, "hogQlCompilationObserver is null"); + this.hogQlSemanticCatalogSnapshotProvider = requireNonNull(hogQlSemanticCatalogSnapshotProvider, "hogQlSemanticCatalogSnapshotProvider is null"); } public PreparedQuery prepareQuery(Session session, String query) @@ -97,7 +122,7 @@ public PreparedQuery prepareQuery(Session session, QuerySubmission submission) try { HogQlCompilationResult result = tracker.observe(COMPILATION, () -> hogQlCompiler .orElseThrow(() -> new TrinoException(NOT_SUPPORTED, "HogQL query submission is disabled")) - .compile(envelope)); + .compile(envelope, semanticCatalogContext(session))); preparedQuery = tracker.observe(PARAMETER_BINDING, () -> prepareQuery( session, result.statement(), @@ -113,6 +138,18 @@ public PreparedQuery prepareQuery(Session session, QuerySubmission submission) }; } + private Optional semanticCatalogContext(Session session) + { + if (hogQlSemanticCatalogSnapshotProvider.isEmpty() || session.getCatalog().isEmpty()) { + return Optional.empty(); + } + String catalog = session.getCatalog().orElseThrow(); + boolean delimited = !UNDELIMITED_IDENTIFIER.matcher(catalog).matches() || !catalog.equals(catalog.toLowerCase(Locale.ENGLISH)); + return Optional.of(new HogQlSemanticCatalogContext( + new PhysicalIdentifier(catalog, delimited), + hogQlSemanticCatalogSnapshotProvider.orElseThrow())); + } + public PreparedQuery prepareQuery(Session session, Statement wrappedStatement) throws ParsingException, TrinoException { diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogModule.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogModule.java index 41d22b5dcd33..c49f92c5539b 100644 --- a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogModule.java +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogModule.java @@ -14,9 +14,12 @@ package io.trino.hogql; import com.google.inject.Binder; +import com.google.inject.Inject; +import com.google.inject.Provider; import com.google.inject.Provides; import com.google.inject.Scopes; import com.google.inject.Singleton; +import com.google.inject.multibindings.OptionalBinder; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotCache; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder; @@ -40,6 +43,10 @@ protected void setup(Binder binder) .setMaxRequestsQueuedPerDestination(config.getLoaderQueueCapacity())); binder.bind(HogQlSemanticCatalogHttpTransport.class).in(Scopes.SINGLETON); binder.bind(HogQlSemanticCatalogManager.class).asEagerSingleton(); + OptionalBinder.newOptionalBinder(binder, HogQlSemanticCatalogSnapshotProvider.class) + .setBinding() + .toProvider(SnapshotProviderFactory.class) + .in(Scopes.SINGLETON); } @Provides @@ -63,10 +70,21 @@ public static HogQlSemanticCatalogSnapshotCache provideCache(HogQlSemanticCatalo return manager.cache(); } - @Provides - @Singleton - public static HogQlSemanticCatalogSnapshotProvider provideProvider(HogQlSemanticCatalogSnapshotCache cache) + public static final class SnapshotProviderFactory + implements Provider { - return HogQlSemanticCatalogSnapshotProvider.fromCache(cache); + private final HogQlSemanticCatalogSnapshotCache cache; + + @Inject + public SnapshotProviderFactory(HogQlSemanticCatalogSnapshotCache cache) + { + this.cache = cache; + } + + @Override + public HogQlSemanticCatalogSnapshotProvider get() + { + return HogQlSemanticCatalogSnapshotProvider.fromCache(cache); + } } } diff --git a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java index c2c9c9d8b4e3..b96d242e1b05 100644 --- a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java +++ b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java @@ -104,6 +104,7 @@ import io.trino.hogql.HogQlSemanticCatalogConfig; import io.trino.hogql.HogQlSemanticCatalogModule; import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider; import io.trino.memory.ClusterMemoryManager; import io.trino.memory.ForMemoryManager; import io.trino.memory.LeastWastedEffortTaskLowMemoryKiller; @@ -187,6 +188,7 @@ protected void setup(Binder binder) binder.bind(HogQlCompiler.class).in(Scopes.SINGLETON); binder.bind(HogQlCompilationStats.class).in(Scopes.SINGLETON); binder.bind(HogQlCompilationObserver.class).to(HogQlCompilationStats.class); + OptionalBinder.newOptionalBinder(binder, HogQlSemanticCatalogSnapshotProvider.class); newExporter(binder).export(HogQlCompilationStats.class).withGeneratedName(); if (buildConfigObject(HogQlConfig.class).isEnabled()) { jaxrsBinder(binder).bind(HogQlStatementResource.class); diff --git a/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerSemanticCatalog.java b/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerSemanticCatalog.java new file mode 100644 index 000000000000..32bd191418c0 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerSemanticCatalog.java @@ -0,0 +1,118 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +import io.trino.execution.QueryPreparer.PreparedQuery; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinRequest; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.sql.parser.SqlParser; +import io.trino.sql.tree.Identifier; +import io.trino.sql.tree.Query; +import io.trino.sql.tree.QuerySpecification; +import io.trino.sql.tree.Table; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static io.trino.execution.QuerySubmission.hogQl; +import static io.trino.hogql.HogQlCompilationObserver.NOOP; +import static io.trino.testing.TestingSession.testSessionBuilder; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlQueryPreparerSemanticCatalog +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); + private static final HogQlSemanticCatalogSnapshot SNAPSHOT = new HogQlSemanticCatalogSnapshot( + 1, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 7, + List.of(new LogicalTableDefinition( + "events", + new PhysicalQualifiedName(CATALOG, new PhysicalIdentifier("default", false), new PhysicalIdentifier("raw_events", false)), + List.of(new LogicalFieldDefinition("event", new PhysicalIdentifier("event_name", false), "varchar", LogicalType.STRING, false, true)), + List.of(), + List.of()))); + + @Test + public void testSessionCatalogPinsSemanticSnapshotAndResolvesLogicalTable() + { + AtomicReference pinRequest = new AtomicReference<>(); + QueryPreparer queryPreparer = new QueryPreparer( + new SqlParser(), + new HogQlCompiler(), + NOOP, + Optional.of(request -> { + pinRequest.set(request); + return new PinnedSnapshot(SNAPSHOT); + })); + + PreparedQuery preparedQuery = queryPreparer.prepareQuery( + testSessionBuilder().setCatalog("analytics").build(), + hogQl(envelope("SELECT event FROM events", OptionalLong.of(7)))); + + assertThat(pinRequest.get()).isEqualTo(new PinRequest(CATALOG, HogQlLanguageContract.current().languageVersion(), OptionalLong.of(7))); + QuerySpecification query = (QuerySpecification) ((Query) preparedQuery.getStatement()).getQueryBody(); + assertThat(((Table) query.getFrom().orElseThrow()).getName().getOriginalParts()) + .extracting(Identifier::getValue) + .containsExactly("analytics", "default", "raw_events"); + } + + @Test + public void testMissingSessionCatalogOrProviderDoesNotFetchMetadata() + { + AtomicInteger pins = new AtomicInteger(); + QueryPreparer withProvider = new QueryPreparer( + new SqlParser(), + new HogQlCompiler(), + NOOP, + Optional.of(_ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + })); + QueryPreparer withoutProvider = new QueryPreparer(new SqlParser(), new HogQlCompiler(), NOOP, Optional.empty()); + + withProvider.prepareQuery(testSessionBuilder().build(), hogQl(envelope("SELECT 1", OptionalLong.empty()))); + withoutProvider.prepareQuery(testSessionBuilder().setCatalog("analytics").build(), hogQl(envelope("SELECT 1", OptionalLong.empty()))); + + assertThat(pins).hasValue(0); + } + + private static HogQlCompileEnvelope envelope(String query, OptionalLong generation) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + generation); + } +} From da732ad0d472f168832679ebde4b10218a9c0f2b Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 11:41:50 -0400 Subject: [PATCH 039/136] feat(hogql): lower aliases and stock joins --- .../trino/hogql/compiler/HogQlCompiler.java | 54 +++- .../hogql/compiler/HogQlSemanticResolver.java | 262 +++++++++++++++--- .../trino/hogql/compiler/TrinoAstFactory.java | 30 +- .../hogql/compiler/TestHogQlCompiler.java | 60 ++++ .../compiler/TestHogQlSemanticResolution.java | 69 ++++- .../io/trino/hogql/parser/HogQlParser.java | 112 +++++++- .../trino/hogql/parser/tree/HogQlQuery.java | 70 ++++- .../1.0.0/trino-compatibility-overrides.json | 167 ++++++++++- .../language/1.0.0/trino-compatibility.json | 167 +++++------ .../trino/hogql/parser/TestHogQlParser.java | 26 ++ 10 files changed, 873 insertions(+), 144 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index 8b55f762bf5b..951dff8f1aee 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -20,6 +20,7 @@ import io.trino.hogql.parser.HogQlParser; import io.trino.hogql.parser.HogQlParsingException; import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; @@ -30,8 +31,11 @@ import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; @@ -120,9 +124,7 @@ private static HogQlCompilationResult compile( OptionalLong expectedCatalogGeneration) { validateParameters(query, parameters); - if (query.from().orElse(null) instanceof TablePlaceholder tablePlaceholder) { - throw bindingError(tablePlaceholder.span(), "HogQL parameter placeholders are not supported in table positions: " + tablePlaceholder.placeholder().name()); - } + query.from().ifPresent(HogQlCompiler::validateRelation); List placeholders = new ArrayList<>(); query.projections().forEach(projection -> { @@ -130,6 +132,7 @@ private static HogQlCompilationResult compile( collectPlaceholders(expressionProjection.expression(), placeholders); } }); + query.from().ifPresent(relation -> collectPlaceholders(relation, placeholders)); query.where().ifPresent(expression -> collectPlaceholders(expression, placeholders)); query.groupBy().forEach(expression -> collectPlaceholders(expression, placeholders)); query.having().ifPresent(expression -> collectPlaceholders(expression, placeholders)); @@ -180,7 +183,7 @@ private static ResolvedQuery resolveQuery( HogQlLanguageVersion languageVersion, OptionalLong expectedCatalogGeneration) { - if (!(query.from().orElse(null) instanceof HogQlQuery.TableReference table) || table.parts().size() != 1 || catalogContext.isEmpty()) { + if (query.from().isEmpty() || !containsSemanticCandidate(query.from().orElseThrow()) || catalogContext.isEmpty()) { return new ResolvedQuery(query, OptionalLong.empty()); } HogQlSemanticCatalogContext context = catalogContext.orElseThrow(); @@ -194,6 +197,16 @@ private static ResolvedQuery resolveQuery( return new ResolvedQuery(resolved, OptionalLong.of(pinned.generation())); } + private static boolean containsSemanticCandidate(Relation relation) + { + return switch (relation) { + case AliasedRelation alias -> containsSemanticCandidate(alias.relation()); + case JoinRelation join -> containsSemanticCandidate(join.left()) || containsSemanticCandidate(join.right()); + case TablePlaceholder _ -> false; + case HogQlQuery.TableReference table -> table.parts().size() == 1; + }; + } + private record ResolvedQuery(HogQlQuery query, OptionalLong catalogGeneration) { private ResolvedQuery @@ -255,6 +268,39 @@ private static void collectPlaceholders(Expression expression, List } } + private static void validateRelation(Relation relation) + { + switch (relation) { + case AliasedRelation alias -> validateRelation(alias.relation()); + case JoinRelation join -> { + validateRelation(join.left()); + validateRelation(join.right()); + } + case TablePlaceholder tablePlaceholder -> throw bindingError( + tablePlaceholder.span(), + "HogQL parameter placeholders are not supported in table positions: " + tablePlaceholder.placeholder().name()); + case HogQlQuery.TableReference _ -> {} + } + } + + private static void collectPlaceholders(Relation relation, List placeholders) + { + switch (relation) { + case AliasedRelation alias -> collectPlaceholders(alias.relation(), placeholders); + case JoinRelation join -> { + collectPlaceholders(join.left(), placeholders); + collectPlaceholders(join.right(), placeholders); + join.criteria().ifPresent(criteria -> { + if (criteria instanceof JoinOn on) { + collectPlaceholders(on.expression(), placeholders); + } + }); + } + case TablePlaceholder _ -> {} + case HogQlQuery.TableReference _ -> {} + } + } + private static TrinoException bindingError(SourceSpan span, String message) { return new TrinoException( diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index c24841913b68..5f731457e3f6 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -18,6 +18,7 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; @@ -31,11 +32,16 @@ import io.trino.hogql.parser.tree.HogQlQuery.Identifier; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; import io.trino.hogql.parser.tree.HogQlQuery.SortItem; import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; @@ -54,36 +60,40 @@ final class HogQlSemanticResolver { - private final LogicalTableDefinition logicalTable; - private final Map fields; + private final PinnedSnapshot snapshot; + private List bindings = List.of(); + private boolean allRelationsLogical; - private HogQlSemanticResolver(LogicalTableDefinition logicalTable) + private HogQlSemanticResolver(PinnedSnapshot snapshot) { - this.logicalTable = requireNonNull(logicalTable, "logicalTable is null"); - Map fields = new HashMap<>(); - logicalTable.fields().forEach(field -> fields.put(canonical(field.name()), field)); - this.fields = Map.copyOf(fields); + this.snapshot = requireNonNull(snapshot, "snapshot is null"); } public static Optional resolve(PinnedSnapshot snapshot, HogQlQuery query) { requireNonNull(snapshot, "snapshot is null"); requireNonNull(query, "query is null"); - if (!(query.from().orElse(null) instanceof TableReference table) || table.parts().size() != 1) { + if (query.from().isEmpty()) { return Optional.empty(); } - Optional logicalTable = snapshot.logicalTable(table.parts().getFirst().value()); - return logicalTable.map(definition -> new HogQlSemanticResolver(definition).resolveQuery(query, table)); + HogQlSemanticResolver resolver = new HogQlSemanticResolver(snapshot); + ResolvedRelation relation = resolver.resolveRelation(query.from().orElseThrow()); + if (relation.bindings().isEmpty()) { + return Optional.empty(); + } + resolver.bindings = relation.bindings(); + resolver.allRelationsLogical = relation.allLogical(); + return Optional.of(resolver.resolveQuery(query, relation.relation())); } - private ResolvedQuery resolveQuery(HogQlQuery query, TableReference originalTable) + private ResolvedQuery resolveQuery(HogQlQuery query, Relation relation) { List projections = new ArrayList<>(); query.projections().forEach(projection -> projections.addAll(resolveProjection(projection))); HogQlQuery resolved = new HogQlQuery( query.distinct(), projections, - Optional.of(resolveTable(originalTable)), + Optional.of(relation), query.where().map(this::resolveExpression), query.groupBy().stream().map(this::resolveExpression).toList(), query.having().map(this::resolveExpression), @@ -97,13 +107,19 @@ private ResolvedQuery resolveQuery(HogQlQuery query, TableReference originalTabl private List resolveProjection(Projection projection) { return switch (projection) { - case Star star -> logicalTable.fields().stream() - .filter(LogicalFieldDefinition::starVisible) - .map(field -> new ExpressionProjection( - physicalColumn(field, star.span()), - Optional.of(new Identifier(field.name(), true, star.span())))) - .map(Projection.class::cast) - .toList(); + case Star star -> { + if (!allRelationsLogical) { + yield List.of(star); + } + yield bindings.stream() + .flatMap(binding -> binding.logicalTable().fields().stream() + .filter(LogicalFieldDefinition::starVisible) + .map(field -> new ExpressionProjection( + physicalColumn(field, binding.starQualifier(bindings.size()), star.span()), + Optional.of(new Identifier(field.name(), true, star.span()))))) + .map(Projection.class::cast) + .toList(); + } case ExpressionProjection expressionProjection -> { Expression resolved = resolveExpression(expressionProjection.expression()); Optional alias = expressionProjection.alias(); @@ -115,14 +131,97 @@ private List resolveProjection(Projection projection) }; } - private TableReference resolveTable(TableReference table) + private ResolvedRelation resolveRelation(Relation relation) + { + return switch (relation) { + case AliasedRelation alias -> { + ResolvedRelation child = resolveRelation(alias.relation()); + List aliasedBindings = child.bindings().stream() + .map(binding -> binding.withAlias(alias.alias())) + .toList(); + yield new ResolvedRelation( + new AliasedRelation(child.relation(), alias.alias(), alias.span()), + aliasedBindings, + child.allLogical()); + } + case JoinRelation join -> { + ResolvedRelation left = resolveRelation(join.left()); + ResolvedRelation right = resolveRelation(join.right()); + List joinBindings = new ArrayList<>(left.bindings()); + joinBindings.addAll(right.bindings()); + List previousBindings = bindings; + boolean previousAllRelationsLogical = allRelationsLogical; + bindings = List.copyOf(joinBindings); + allRelationsLogical = left.allLogical() && right.allLogical(); + Optional criteria = join.criteria().map(value -> switch (value) { + case JoinOn on -> new JoinOn(resolveExpression(on.expression()), on.span()); + case JoinUsing using -> resolveJoinUsing(using, left.bindings(), right.bindings()); + }); + bindings = previousBindings; + allRelationsLogical = previousAllRelationsLogical; + yield new ResolvedRelation( + new JoinRelation(join.type(), left.relation(), right.relation(), criteria, join.span()), + joinBindings, + left.allLogical() && right.allLogical()); + } + case TablePlaceholder placeholder -> new ResolvedRelation(placeholder, List.of(), false); + case TableReference table -> resolveTable(table); + }; + } + + private JoinUsing resolveJoinUsing(JoinUsing using, List leftBindings, List rightBindings) + { + List columns = using.columns().stream() + .map(column -> { + List leftFields = matchingFields(leftBindings, column.value()); + List rightFields = matchingFields(rightBindings, column.value()); + if (leftFields.size() != 1 || rightFields.size() != 1) { + throw resolutionError(new ColumnReference(List.of(column), column.span()), column.value()); + } + PhysicalIdentifier left = leftFields.getFirst().physicalColumn(); + PhysicalIdentifier right = rightFields.getFirst().physicalColumn(); + if (!left.equals(right)) { + throw incompatibleUsingResolutionError(column); + } + return new Identifier(left.value(), left.delimited(), column.span()); + }) + .toList(); + return new JoinUsing(columns, using.span()); + } + + private static List matchingFields(List tableBindings, String name) + { + return tableBindings.stream() + .map(TableBinding::fields) + .map(fields -> fields.get(canonical(name))) + .filter(field -> field != null) + .toList(); + } + + private ResolvedRelation resolveTable(TableReference table) { - return new TableReference( + if (table.parts().size() != 1) { + return new ResolvedRelation(table, List.of(), false); + } + Optional logicalTable = snapshot.logicalTable(table.parts().getFirst().value()); + if (logicalTable.isEmpty()) { + return new ResolvedRelation(table, List.of(), false); + } + LogicalTableDefinition definition = logicalTable.orElseThrow(); + TableReference physicalTable = new TableReference( List.of( - identifier(logicalTable.physicalTable().catalog(), table), - identifier(logicalTable.physicalTable().schema(), table), - identifier(logicalTable.physicalTable().table(), table)), + identifier(definition.physicalTable().catalog(), table), + identifier(definition.physicalTable().schema(), table), + identifier(definition.physicalTable().table(), table)), table.span()); + return new ResolvedRelation( + physicalTable, + List.of(new TableBinding( + definition, + canonical(definition.name()), + definition.physicalTable().table(), + false)), + true); } private List resolveSortItems(List sortItems) @@ -185,23 +284,52 @@ private Expression resolveExpression(Expression expression) private ColumnReference resolveColumn(ColumnReference reference) { List parts = reference.parts(); - if (parts.size() == 2 && !canonical(parts.getFirst().value()).equals(canonical(logicalTable.name()))) { - throw resolutionError(reference, parts.getLast().value()); + if (parts.size() == 2) { + Optional binding = bindings.stream() + .filter(candidate -> candidate.qualifier().equals(canonical(parts.getFirst().value()))) + .findFirst(); + if (binding.isEmpty()) { + if (allRelationsLogical) { + throw resolutionError(reference, parts.getLast().value()); + } + return reference; + } + LogicalFieldDefinition field = binding.orElseThrow().fields().get(canonical(parts.getLast().value())); + if (field == null) { + throw resolutionError(reference, parts.getLast().value()); + } + return physicalColumn(field, Optional.of(binding.orElseThrow().outputQualifier()), reference.span()); } if (parts.size() > 2) { - throw resolutionError(reference, parts.getLast().value()); + if (allRelationsLogical) { + throw resolutionError(reference, parts.getLast().value()); + } + return reference; } String logicalName = parts.getLast().value(); - LogicalFieldDefinition field = fields.get(canonical(logicalName)); - if (field == null) { + List matches = bindings.stream() + .map(TableBinding::fields) + .map(fields -> fields.get(canonical(logicalName))) + .filter(field -> field != null) + .toList(); + if (matches.size() == 1 && (allRelationsLogical || bindings.size() == 1)) { + return physicalColumn(matches.getFirst(), Optional.empty(), reference.span()); + } + if (matches.size() > 1) { + throw ambiguousResolutionError(reference, logicalName); + } + if (allRelationsLogical) { throw resolutionError(reference, logicalName); } - return physicalColumn(field, reference.span()); + return reference; } - private static ColumnReference physicalColumn(LogicalFieldDefinition field, HogQlQuery.SourceSpan span) + private static ColumnReference physicalColumn(LogicalFieldDefinition field, Optional qualifier, HogQlQuery.SourceSpan span) { - return new ColumnReference(List.of(new Identifier(field.physicalColumn().value(), field.physicalColumn().delimited(), span)), span); + List parts = new ArrayList<>(); + qualifier.ifPresent(identifier -> parts.add(new Identifier(identifier.value(), identifier.delimited(), span))); + parts.add(new Identifier(field.physicalColumn().value(), field.physicalColumn().delimited(), span)); + return new ColumnReference(parts, span); } private static Identifier identifier(PhysicalIdentifier identifier, TableReference source) @@ -223,6 +351,76 @@ private static TrinoException resolutionError(ColumnReference reference, String null); } + private static TrinoException ambiguousResolutionError(ColumnReference reference, String name) + { + return new TrinoException( + HOGQL_RESOLUTION_ERROR, + Optional.of(new Location(reference.span().startLine(), reference.span().startColumn())), + "Ambiguous HogQL field: " + name, + null); + } + + private static TrinoException incompatibleUsingResolutionError(Identifier identifier) + { + return new TrinoException( + HOGQL_RESOLUTION_ERROR, + Optional.of(new Location(identifier.span().startLine(), identifier.span().startColumn())), + "HogQL USING field maps to different physical columns: " + identifier.value(), + null); + } + + private record ResolvedRelation(Relation relation, List bindings, boolean allLogical) + { + private ResolvedRelation + { + relation = requireNonNull(relation, "relation is null"); + bindings = List.copyOf(requireNonNull(bindings, "bindings is null")); + } + } + + private record TableBinding( + LogicalTableDefinition logicalTable, + String qualifier, + PhysicalIdentifier outputQualifier, + boolean aliased, + Map fields) + { + private TableBinding(LogicalTableDefinition logicalTable, String qualifier, PhysicalIdentifier outputQualifier, boolean aliased) + { + this(logicalTable, qualifier, outputQualifier, aliased, fieldMap(logicalTable)); + } + + private TableBinding + { + logicalTable = requireNonNull(logicalTable, "logicalTable is null"); + qualifier = requireNonNull(qualifier, "qualifier is null"); + outputQualifier = requireNonNull(outputQualifier, "outputQualifier is null"); + fields = Map.copyOf(requireNonNull(fields, "fields is null")); + } + + private TableBinding withAlias(Identifier alias) + { + return new TableBinding( + logicalTable, + canonical(alias.value()), + new PhysicalIdentifier(alias.value(), alias.delimited()), + true, + fields); + } + + private Optional starQualifier(int relationCount) + { + return aliased || relationCount > 1 ? Optional.of(outputQualifier) : Optional.empty(); + } + + private static Map fieldMap(LogicalTableDefinition table) + { + Map fields = new HashMap<>(); + table.fields().forEach(field -> fields.put(canonical(field.name()), field)); + return fields; + } + } + record ResolvedQuery(HogQlQuery query) { ResolvedQuery diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 6b2ec7e8f9b1..cc9d3a323429 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -14,6 +14,7 @@ package io.trino.hogql.compiler; import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; @@ -25,6 +26,9 @@ import io.trino.hogql.parser.tree.HogQlQuery.Identifier; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; @@ -94,7 +98,7 @@ public static Statement createStatement(HogQlQuery query, Map createSelectItem(projection, parameterIds)) .toList()), - query.from().map(TrinoAstFactory::createRelation), + query.from().map(relation -> createRelation(relation, parameterIds)), query.where().map(expression -> createExpression(expression, parameterIds)), createGroupBy(query, parameterIds), query.having().map(expression -> createExpression(expression, parameterIds)), @@ -318,9 +322,31 @@ private static Expression createColumnReference(ColumnReference reference) return expression; } - private static io.trino.sql.tree.Relation createRelation(Relation relation) + private static io.trino.sql.tree.Relation createRelation(Relation relation, Map parameterIds) { return switch (relation) { + case AliasedRelation alias -> new io.trino.sql.tree.AliasedRelation( + location(alias.span()), + createRelation(alias.relation(), parameterIds), + createIdentifier(alias.alias()), + null); + case JoinRelation join -> new io.trino.sql.tree.Join( + location(join.span()), + switch (join.type()) { + case CROSS -> io.trino.sql.tree.Join.Type.CROSS; + case INNER -> io.trino.sql.tree.Join.Type.INNER; + case LEFT -> io.trino.sql.tree.Join.Type.LEFT; + case RIGHT -> io.trino.sql.tree.Join.Type.RIGHT; + case FULL -> io.trino.sql.tree.Join.Type.FULL; + }, + createRelation(join.left(), parameterIds), + createRelation(join.right(), parameterIds), + join.criteria().map(criteria -> switch (criteria) { + case JoinOn on -> new io.trino.sql.tree.JoinOn(createExpression(on.expression(), parameterIds)); + case JoinUsing using -> new io.trino.sql.tree.JoinUsing(using.columns().stream() + .map(TrinoAstFactory::createIdentifier) + .toList()); + })); case TablePlaceholder _ -> throw new IllegalArgumentException("table placeholder was not validated"); case TableReference table -> createTable(table); }; diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index 67d6bb3d71ab..5daf0df52677 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -18,8 +18,11 @@ import io.trino.spi.TrinoException; import io.trino.sql.SqlFormatter; import io.trino.sql.parser.SqlParser; +import io.trino.sql.tree.AliasedRelation; import io.trino.sql.tree.AllColumns; import io.trino.sql.tree.Identifier; +import io.trino.sql.tree.Join; +import io.trino.sql.tree.JoinOn; import io.trino.sql.tree.Node; import io.trino.sql.tree.NodeLocation; import io.trino.sql.tree.Parameter; @@ -151,6 +154,63 @@ public void testBindsPlaceholdersAcrossGroupingHavingAndAggregateModifiers() .containsExactly(0, 1, 2, 3, 4, 5); } + @ParameterizedTest + @ValueSource(strings = { + "SELECT e.id FROM events AS e", + "SELECT date.id FROM events AS date", + "SELECT \"e\".id FROM events AS \"e\"", + "SELECT e.id FROM events e JOIN persons p ON e.person_id = p.id", + "SELECT * FROM events LEFT OUTER JOIN persons USING (id)", + "SELECT * FROM events RIGHT JOIN persons ON events.id = persons.id", + "SELECT * FROM events FULL OUTER JOIN persons USING (id, team_id)", + "SELECT * FROM events CROSS JOIN persons", + "SELECT * FROM (events CROSS JOIN persons)", + }) + public void testLowersAliasesAndStockJoins(String hogql) + { + Statement statement = compiler.compile(hogql); + + assertThat(statement).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @Test + public void testLowersUnparenthesizedUsing() + { + assertThat(compiler.compile("SELECT * FROM events JOIN persons USING id")) + .isEqualTo(sqlParser.createStatement("SELECT * FROM events JOIN persons USING (id)")); + } + + @Test + public void testPreservesAliasedJoinSourceLocations() + { + Query query = (Query) compiler.compile("SELECT e.id\nFROM events AS e\nLEFT JOIN persons AS p ON e.person_id = p.id"); + Join join = (Join) ((QuerySpecification) query.getQueryBody()).getFrom().orElseThrow(); + AliasedRelation left = (AliasedRelation) join.getLeft(); + JoinOn criteria = (JoinOn) join.getCriteria().orElseThrow(); + + assertThat(join.getLocation()).contains(new NodeLocation(2, 6)); + assertThat(left.getLocation()).contains(new NodeLocation(2, 6)); + assertThat(left.getAlias().getLocation()).contains(new NodeLocation(2, 16)); + assertThat(criteria.getExpression().getLocation()).contains(new NodeLocation(3, 27)); + } + + @Test + public void testBindsPlaceholdersInJoinCriteriaBySourceOrder() + { + HogQlCompilationResult result = compiler.compile( + "SELECT {projection} FROM events e JOIN persons p ON e.id = {join_value} WHERE {where_value}", + Map.of( + "projection", typedValue("projection"), + "join_value", typedValue("join"), + "where_value", typedValue("where"))); + + assertThat(result.parameterNames()).containsExactly("projection", "join_value", "where_value"); + assertThat(parameters(result.statement())) + .extracting(Parameter::getId) + .containsExactly(0, 1, 2); + } + @ParameterizedTest @ValueSource(strings = { "SELECT value BETWEEN 1 AND 10", diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java index 85bd7f921601..0e8351af1750 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -24,6 +24,7 @@ import io.trino.hogql.parser.HogQlLanguageContract; import io.trino.spi.Location; import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; import io.trino.sql.tree.Identifier; import io.trino.sql.tree.Query; import io.trino.sql.tree.QuerySpecification; @@ -50,20 +51,33 @@ public class TestHogQlSemanticResolution HogQlLanguageContract.current().languageVersion(), CATALOG, 7, - List.of(new LogicalTableDefinition( - "events", - new PhysicalQualifiedName( - CATALOG, - new PhysicalIdentifier("Hog Data", true), - new PhysicalIdentifier("raw-events", true)), - List.of( - new LogicalFieldDefinition("event", new PhysicalIdentifier("event_name", false), "varchar", LogicalType.STRING, false, true), - new LogicalFieldDefinition("personId", new PhysicalIdentifier("Person ID", true), "varchar", LogicalType.STRING, false, true), - new LogicalFieldDefinition("hidden", new PhysicalIdentifier("hidden", false), "bigint", LogicalType.INTEGER, true, false)), - List.of(), - List.of()))); + List.of( + new LogicalTableDefinition( + "events", + new PhysicalQualifiedName( + CATALOG, + new PhysicalIdentifier("Hog Data", true), + new PhysicalIdentifier("raw-events", true)), + List.of( + new LogicalFieldDefinition("event", new PhysicalIdentifier("event_name", false), "varchar", LogicalType.STRING, false, true), + new LogicalFieldDefinition("personId", new PhysicalIdentifier("Person ID", true), "varchar", LogicalType.STRING, false, true), + new LogicalFieldDefinition("hidden", new PhysicalIdentifier("hidden", false), "bigint", LogicalType.INTEGER, true, false)), + List.of(), + List.of()), + new LogicalTableDefinition( + "persons", + new PhysicalQualifiedName( + CATALOG, + new PhysicalIdentifier("Hog Data", true), + new PhysicalIdentifier("raw-persons", true)), + List.of( + new LogicalFieldDefinition("personId", new PhysicalIdentifier("person_id", false), "varchar", LogicalType.STRING, false, true), + new LogicalFieldDefinition("name", new PhysicalIdentifier("full_name", false), "varchar", LogicalType.STRING, false, true)), + List.of(), + List.of()))); private final HogQlCompiler compiler = new HogQlCompiler(); + private final SqlParser sqlParser = new SqlParser(); @Test public void testPinsAndResolvesLogicalTableFieldsAndStar() @@ -129,6 +143,37 @@ public void testUnknownLogicalFieldFailsAtOriginalLocation() assertThat(exception).hasMessage("line 1:8: Unknown HogQL field: missing"); } + @Test + public void testResolvesAliasedLogicalTablesAndJoinCriteria() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compile( + envelope("SELECT e.event, p.name FROM events e JOIN persons p ON e.personId = p.personId", OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.catalogGeneration()).hasValue(7); + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT e.event_name AS event, p.full_name AS name " + + "FROM analytics.\"Hog Data\".\"raw-events\" e " + + "JOIN analytics.\"Hog Data\".\"raw-persons\" p ON e.\"Person ID\" = p.person_id")); + } + + @Test + public void testRejectsLogicalUsingWhenPhysicalColumnNamesDiffer() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + String hogql = "SELECT e.event FROM events e JOIN persons p USING (personId)"; + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope(hogql, OptionalLong.empty()), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, hogql.lastIndexOf("personId") + 1)); + assertThat(exception).hasMessageContaining("HogQL USING field maps to different physical columns: personId"); + } + private static QuerySpecification querySpecification(HogQlCompilationResult result) { return (QuerySpecification) ((Query) result.statement()).getQueryBody(); diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 988ce7b64ef7..fbfc210a4aa8 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -16,6 +16,7 @@ import io.trino.hogql.parser.canonical.HogQLLexer; import io.trino.hogql.parser.canonical.HogQLParser; import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; @@ -30,6 +31,11 @@ import io.trino.hogql.parser.tree.HogQlQuery.Identifier; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinCriteria; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.JoinType; +import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; @@ -452,7 +458,7 @@ public HogQlQuery build(HogQLParser.SelectContext context) .toList(); Optional from = Optional.ofNullable(select.fromClause()) .map(HogQLParser.FromClauseContext::joinExpr) - .map(this::buildTableReference); + .map(this::buildRelation); Optional where = Optional.ofNullable(select.whereClause()) .map(HogQLParser.WhereClauseContext::columnExpr) .map(this::buildExpression); @@ -943,18 +949,110 @@ private Placeholder buildPlaceholder(HogQLParser.PlaceholderContext context) return new Placeholder(reference.parts().getFirst().value(), sourceSpan(context)); } - private Relation buildTableReference(HogQLParser.JoinExprContext context) + private Relation buildRelation(HogQLParser.JoinExprContext context) { - if (!(context instanceof HogQLParser.JoinExprTableContext table) || table.FINAL() != null || table.sampleClause() != null) { - throw unsupported(context, "table expression or join"); + if (context instanceof HogQLParser.JoinExprTableContext table) { + if (table.FINAL() != null || table.sampleClause() != null) { + throw unsupported(context, "HogQL-specific table modifier"); + } + return buildTableExpression(table.tableExpr()); + } + if (context instanceof HogQLParser.JoinExprParensContext parens) { + return buildRelation(parens.joinExpr()); + } + if (context instanceof HogQLParser.JoinExprCrossOpContext cross) { + if (cross.joinOpCross().COMMA() != null) { + throw unsupported(context, "implicit comma join"); + } + return new JoinRelation( + JoinType.CROSS, + buildRelation(cross.joinExpr(0)), + buildRelation(cross.joinExpr(1)), + Optional.empty(), + sourceSpan(context)); + } + if (context instanceof HogQLParser.JoinExprOpContext join) { + if (join.NATURAL() != null) { + throw unsupported(context, "natural join"); + } + if (join.joinConstraintClause() == null) { + throw unsupported(context, "join without ON or USING"); + } + return new JoinRelation( + buildJoinType(join.joinOp()), + buildRelation(join.joinExpr(0)), + buildRelation(join.joinExpr(1)), + Optional.of(buildJoinCriteria(join.joinConstraintClause())), + sourceSpan(context)); } - if (table.tableExpr() instanceof HogQLParser.TableExprIdentifierContext identifier) { + throw unsupported(context, "HogQL-specific join variant"); + } + + private Relation buildTableExpression(HogQLParser.TableExprContext context) + { + if (context instanceof HogQLParser.TableExprIdentifierContext identifier) { return new TableReference(buildIdentifiers(identifier.tableIdentifier()), sourceSpan(context)); } - if (table.tableExpr() instanceof HogQLParser.TableExprPlaceholderContext placeholder) { + if (context instanceof HogQLParser.TableExprPlaceholderContext placeholder) { return new TablePlaceholder(buildPlaceholder(placeholder.placeholder())); } - throw unsupported(context, "table expression or join"); + if (context instanceof HogQLParser.TableExprAliasContext alias) { + if (alias.columnAliases() != null) { + throw unsupported(context, "table column aliases"); + } + Identifier identifier = alias.identifier() != null + ? buildIdentifier(alias.identifier()) + : buildIdentifier(alias.alias().getText(), alias.alias()); + return new AliasedRelation(buildTableExpression(alias.tableExpr()), identifier, sourceSpan(context)); + } + throw unsupported(context, "table expression"); + } + + private JoinType buildJoinType(HogQLParser.JoinOpContext context) + { + if (context == null) { + return JoinType.INNER; + } + if (context instanceof HogQLParser.JoinOpInnerContext inner) { + if (inner.ANTI() != null || inner.SEMI() != null || inner.ASOF() != null || inner.ALL() != null || inner.ANY() != null) { + throw unsupported(context, "HogQL-specific inner join modifier"); + } + return JoinType.INNER; + } + if (context instanceof HogQLParser.JoinOpLeftRightContext outer) { + if (outer.ANTI() != null || outer.SEMI() != null || outer.ASOF() != null || outer.ALL() != null || outer.ANY() != null) { + throw unsupported(context, "HogQL-specific outer join modifier"); + } + return outer.LEFT() != null ? JoinType.LEFT : JoinType.RIGHT; + } + if (context instanceof HogQLParser.JoinOpFullContext full) { + if (full.ASOF() != null || full.ALL() != null || full.ANY() != null) { + throw unsupported(context, "HogQL-specific full join modifier"); + } + return JoinType.FULL; + } + throw unsupported(context, "join operator"); + } + + private JoinCriteria buildJoinCriteria(HogQLParser.JoinConstraintClauseContext context) + { + if (context.ON() != null) { + List expressions = context.columnExprList().columnExpr(); + if (expressions.size() != 1) { + throw unsupported(context, "multiple ON expressions"); + } + return new JoinOn(buildExpression(expressions.getFirst()), sourceSpan(context)); + } + List columns = context.columnExprList().columnExpr().stream() + .map(this::buildExpression) + .map(expression -> { + if (!(expression instanceof ColumnReference reference) || reference.parts().size() != 1) { + throw unsupported(context, "non-identifier USING column"); + } + return reference.parts().getFirst(); + }) + .toList(); + return new JoinUsing(columns, sourceSpan(context)); } private List buildIdentifiers(HogQLParser.TableIdentifierContext context) diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index 9303a537ceaa..443c9e811b56 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -326,11 +326,79 @@ public enum LiteralKind } public sealed interface Relation - permits TablePlaceholder, TableReference + permits AliasedRelation, + JoinRelation, + TablePlaceholder, + TableReference { SourceSpan span(); } + public record AliasedRelation(Relation relation, Identifier alias, SourceSpan span) + implements Relation + { + public AliasedRelation + { + relation = requireNonNull(relation, "relation is null"); + alias = requireNonNull(alias, "alias is null"); + span = requireNonNull(span, "span is null"); + } + } + + public enum JoinType + { + CROSS, + INNER, + LEFT, + RIGHT, + FULL, + } + + public sealed interface JoinCriteria + permits JoinOn, JoinUsing + { + SourceSpan span(); + } + + public record JoinOn(Expression expression, SourceSpan span) + implements JoinCriteria + { + public JoinOn + { + expression = requireNonNull(expression, "expression is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record JoinUsing(List columns, SourceSpan span) + implements JoinCriteria + { + public JoinUsing + { + columns = List.copyOf(requireNonNull(columns, "columns is null")); + if (columns.isEmpty()) { + throw new IllegalArgumentException("columns is empty"); + } + span = requireNonNull(span, "span is null"); + } + } + + public record JoinRelation(JoinType type, Relation left, Relation right, Optional criteria, SourceSpan span) + implements Relation + { + public JoinRelation + { + type = requireNonNull(type, "type is null"); + left = requireNonNull(left, "left is null"); + right = requireNonNull(right, "right is null"); + criteria = requireNonNull(criteria, "criteria is null"); + span = requireNonNull(span, "span is null"); + if ((type == JoinType.CROSS) == criteria.isPresent()) { + throw new IllegalArgumentException("cross joins must omit criteria and qualified joins must provide criteria"); + } + } + } + public record TableReference(List parts, SourceSpan span) implements Relation { diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json index 683c0629a07e..61893bdff17a 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json @@ -114,6 +114,30 @@ } ], "features": [ + { + "id": "alternative:alias:keyword", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:alias:quoted-identifier", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:alias:unquoted-identifier", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, { "id": "alternative:columnExpr:ColumnExprAnd", "loweringHandler": "TrinoAstFactory#createExpression", @@ -301,6 +325,102 @@ "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" ] }, + { + "id": "alternative:joinConstraintClause:on", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinConstraintClause:using", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", + "testCaseIds": [ + "TestHogQlCompiler#testLowersUnparenthesizedUsing" + ] + }, + { + "id": "alternative:joinConstraintClause:using-parenthesized", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinExpr:JoinExprCrossOp", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinExpr:JoinExprOp", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinExpr:JoinExprParens", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinExpr:JoinExprTable", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinOp:JoinOpFull", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinOp:JoinOpInner", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinOp:JoinOpLeftRight", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinOpCross:cross-join", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:keywordForAlias", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, { "id": "alternative:nestedIdentifier", "loweringHandler": "TrinoAstFactory#createIdentifier", @@ -333,10 +453,18 @@ "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" ] }, + { + "id": "alternative:tableExpr:TableExprAlias", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, { "id": "alternative:tableExpr:TableExprIdentifier", "loweringHandler": "TrinoAstFactory#createTable", - "parseHandler": "AstBuilder#buildTableReference", + "parseHandler": "AstBuilder#buildTableExpression", "testCaseIds": [ "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" ] @@ -357,6 +485,14 @@ "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause" ] }, + { + "id": "rule:alias", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, { "id": "rule:columnExprList", "loweringHandler": "TrinoAstFactory#createExpression", @@ -376,8 +512,8 @@ }, { "id": "rule:fromClause", - "loweringHandler": "TrinoAstFactory#createTable", - "parseHandler": "AstBuilder#buildTableReference", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", "testCaseIds": [ "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" ] @@ -390,6 +526,31 @@ "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" ] }, + { + "id": "rule:joinConstraintClause", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins", + "TestHogQlCompiler#testLowersUnparenthesizedUsing" + ] + }, + { + "id": "rule:joinOp", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "rule:keywordForAlias", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, { "id": "rule:nestedIdentifier", "loweringHandler": "TrinoAstFactory#createIdentifier", diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json index d485d90d73f5..c7fd27d720e5 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json @@ -3,34 +3,34 @@ { "id": "alternative:alias:keyword", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { "id": "alternative:alias:quoted-identifier", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { "id": "alternative:alias:unquoted-identifier", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { @@ -1499,67 +1499,67 @@ { "id": "alternative:joinConstraintClause:on", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { "id": "alternative:joinConstraintClause:using", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersUnparenthesizedUsing" ] }, { "id": "alternative:joinConstraintClause:using-parenthesized", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { "id": "alternative:joinExpr:JoinExprCrossOp", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { "id": "alternative:joinExpr:JoinExprOp", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { "id": "alternative:joinExpr:JoinExprParens", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { @@ -1587,12 +1587,12 @@ { "id": "alternative:joinExpr:JoinExprTable", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { @@ -1609,34 +1609,34 @@ { "id": "alternative:joinOp:JoinOpFull", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { "id": "alternative:joinOp:JoinOpInner", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { "id": "alternative:joinOp:JoinOpLeftRight", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { @@ -1653,12 +1653,12 @@ { "id": "alternative:joinOpCross:cross-join", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { @@ -1675,12 +1675,12 @@ { "id": "alternative:keywordForAlias", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { @@ -2423,12 +2423,12 @@ { "id": "alternative:tableExpr:TableExprAlias", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { @@ -2446,7 +2446,7 @@ "id": "alternative:tableExpr:TableExprIdentifier", "kind": "parserAlternative", "loweringHandler": "TrinoAstFactory#createTable", - "parseHandler": "AstBuilder#buildTableReference", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, "status": "supported", "testCaseIds": [ @@ -2841,12 +2841,12 @@ { "id": "rule:alias", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { @@ -3128,8 +3128,8 @@ { "id": "rule:fromClause", "kind": "parserRule", - "loweringHandler": "TrinoAstFactory#createTable", - "parseHandler": "AstBuilder#buildTableReference", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", "queryReachable": true, "status": "supported", "testCaseIds": [ @@ -3326,12 +3326,13 @@ { "id": "rule:joinConstraintClause", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins", + "TestHogQlCompiler#testLowersUnparenthesizedUsing" ] }, { @@ -3348,12 +3349,12 @@ { "id": "rule:joinOp", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { @@ -3381,12 +3382,12 @@ { "id": "rule:keywordForAlias", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, { diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 187656790d3f..cf87b5d7a9e1 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -14,10 +14,14 @@ package io.trino.hogql.parser; import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.JoinType; import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; @@ -41,6 +45,13 @@ public class TestHogQlParser @ValueSource(strings = { "SELECT", "SELECT * FROM first JOIN second", + "SELECT * FROM first NATURAL JOIN second", + "SELECT * FROM first ANY INNER JOIN second ON first.id = second.id", + "SELECT * FROM first LEFT SEMI JOIN second ON first.id = second.id", + "SELECT * FROM first ASOF JOIN second ON first.id = second.id", + "SELECT * FROM first POSITIONAL JOIN second", + "SELECT * FROM first, second", + "SELECT * FROM events AS e(id)", "SELECT count(*) OVER ()", "SELECT function(1)(value)", "DROP TABLE events", @@ -110,6 +121,21 @@ public void testBuildsOrdinaryGroupingHavingAndAggregateFunctionAst() assertThat(count.arguments()).singleElement().isInstanceOf(ColumnReference.class); } + @Test + public void testBuildsAliasedJoinAstWithSourceSpans() + { + HogQlQuery query = parser.parseStatement("SELECT e.id\nFROM events AS e\nLEFT JOIN persons AS p ON e.person_id = p.id"); + + JoinRelation join = (JoinRelation) query.from().orElseThrow(); + assertThat(join.type()).isEqualTo(JoinType.LEFT); + assertThat(join.span()).isEqualTo(new HogQlQuery.SourceSpan(17, 73, 2, 6, 3, 45)); + AliasedRelation left = (AliasedRelation) join.left(); + assertThat(left.alias().value()).isEqualTo("e"); + assertThat(left.span()).isEqualTo(new HogQlQuery.SourceSpan(17, 28, 2, 6, 2, 17)); + JoinOn criteria = (JoinOn) join.criteria().orElseThrow(); + assertThat(criteria.span()).isEqualTo(new HogQlQuery.SourceSpan(52, 73, 3, 24, 3, 45)); + } + @Test public void testSourceSpansUseUnicodeCodePointOffsetsAndEndPositions() { From c1ce38c79daa599415bd5a87836130cebd5e7a67 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 11:42:17 -0400 Subject: [PATCH 040/136] test(hogql): verify semantic catalog wiring --- .../TestHogQlSemanticCatalogGuiceWiring.java | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogGuiceWiring.java diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogGuiceWiring.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogGuiceWiring.java new file mode 100644 index 000000000000..b4eb9beb8ef5 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogGuiceWiring.java @@ -0,0 +1,58 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.Key; +import com.google.inject.TypeLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotCache; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider; +import io.trino.server.testing.TestingTrinoServer; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlSemanticCatalogGuiceWiring +{ + private static final Key> OPTIONAL_PROVIDER_KEY = + Key.get(new TypeLiteral<>() {}); + + @Test + public void testOptionalProviderFollowsHogQlAndCatalogConfiguration() + throws IOException + { + try (TestingTrinoServer server = TestingTrinoServer.builder() + .setProperties(Map.of("hogql.enabled", "false")) + .build()) { + assertThat(server.getInstance(OPTIONAL_PROVIDER_KEY)).isEmpty(); + } + + try (TestingTrinoServer server = TestingTrinoServer.builder() + .setProperties(Map.of( + "hogql.enabled", "true", + "hogql.semantic-catalog.uri", "https://duckgres.example/metadata")) + .build()) { + Optional provider = server.getInstance(OPTIONAL_PROVIDER_KEY); + HogQlSemanticCatalogManager manager = server.getInstance(Key.get(HogQlSemanticCatalogManager.class)); + HogQlSemanticCatalogSnapshotCache cache = server.getInstance(Key.get(HogQlSemanticCatalogSnapshotCache.class)); + + assertThat(provider).isPresent(); + assertThat(server.getInstance(OPTIONAL_PROVIDER_KEY)).containsSame(provider.orElseThrow()); + assertThat(cache).isSameAs(manager.cache()); + } + } +} From af4390158c0a27a8382f427541295ddc7ffe2a78 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 11:44:20 -0400 Subject: [PATCH 041/136] docs(hogql): document native coordinator frontend --- docs/src/main/sphinx/admin.md | 2 + docs/src/main/sphinx/admin/hogql.md | 76 +++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 docs/src/main/sphinx/admin/hogql.md diff --git a/docs/src/main/sphinx/admin.md b/docs/src/main/sphinx/admin.md index 52edb4e4c26c..c6dee22d0f10 100644 --- a/docs/src/main/sphinx/admin.md +++ b/docs/src/main/sphinx/admin.md @@ -12,6 +12,7 @@ admin/tuning admin/jmx admin/opentelemetry admin/openmetrics +admin/hogql admin/properties admin/spill admin/resource-groups @@ -64,6 +65,7 @@ admin/properties ``` * [Properties reference overview](admin/properties) +* [](admin/hogql) * [](admin/properties-general) * [](admin/properties-client-protocol) * [](admin/properties-http-server) diff --git a/docs/src/main/sphinx/admin/hogql.md b/docs/src/main/sphinx/admin/hogql.md new file mode 100644 index 000000000000..deb43b60e918 --- /dev/null +++ b/docs/src/main/sphinx/admin/hogql.md @@ -0,0 +1,76 @@ +# Native HogQL queries + +The PostHog Trino distribution can accept read-only HogQL through +`POST /v1/hogql`. The coordinator parses and compiles the query to the standard +Trino SQL AST before analysis and planning. Workers, connectors, access control, +and the optimizer receive standard Trino plans. + +Set the following coordinator property to enable the endpoint: + +```properties +hogql.enabled=true +``` + +The endpoint accepts a JSON request envelope. Identity, catalog, schema, time +zone, source, tags, and allowed session properties continue to use the standard +Trino request headers. + +```json +{ + "query": "SELECT event FROM events WHERE person_id = {person_id}", + "protocolVersion": 1, + "languageVersion": "1.0.0", + "parameters": { + "person_id": {"type": "uuid", "value": "00000000-0000-0000-0000-000000000000"} + }, + "variables": {}, + "filters": {}, + "modifiers": {} +} +``` + +Responses use the standard Trino query results protocol, including result +paging and cancellation. The coordinator rejects request bodies larger than 2 +MiB and more than 1,000 total entries across `parameters`, `variables`, +`filters`, and `modifiers`. + +## Semantic catalog manifests + +Logical tables, fields, quoted physical names, and star expansion use an +immutable semantic catalog manifest published by Duckgres. Configure its base +URI on the coordinator: + +```properties +hogql.semantic-catalog.uri=https://duckgres.example +``` + +The coordinator refreshes manifests asynchronously and compiles each query +against one pinned generation. A query does not wait for a remote metadata +request. A missing, expired, malformed, or mismatched manifest fails the query +with a catalog readiness or compatibility error. + +The compatibility request path is +`/v1/hogql/compatibility/semantic-catalog`. The coordinator identifies the +selected Trino catalog, HogQL language version, and optional requested +generation. Manifests contain typed identifiers and definitions, not SQL text. + +The Trino transport does not currently send an application authentication +credential. Duckgres requires `X-Duckgres-Internal-Secret` by default, so the +production connection remains incomplete until the coordinator credential +mechanism is configured. + +## Semantic catalog properties + +| Property | Default | Description | +| --- | --- | --- | +| `hogql.semantic-catalog.maximum-entries` | `100` | Maximum catalog snapshots retained by one coordinator. | +| `hogql.semantic-catalog.refresh-after` | `1m` | Age at which a cached manifest is refreshed in the background. | +| `hogql.semantic-catalog.expire-after` | `5m` | Age after which a cached manifest cannot be used. | +| `hogql.semantic-catalog.failure-backoff` | `10s` | Minimum delay before retrying a failed refresh. | +| `hogql.semantic-catalog.loader-threads` | `4` | Number of background manifest loader threads. | +| `hogql.semantic-catalog.loader-queue-capacity` | `64` | Maximum queued background manifest loads. | +| `hogql.semantic-catalog.request-timeout` | `10s` | Timeout for a background compatibility request. | +| `hogql.semantic-catalog.maximum-response-size` | `8MB` | Maximum accepted manifest response size. | + +Keep `refresh-after` below `expire-after`. Size the entry count, loader threads, +and queue capacity for the number of catalogs assigned to one coordinator. From c22aaed3beb1fa640047ee502b1a6bb36905e5a5 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 12:05:52 -0400 Subject: [PATCH 042/136] feat(hogql): prewarm catalog snapshots --- .../connector/CatalogLifecycleListener.java | 25 +++ .../connector/CatalogLifecycleListeners.java | 48 ++++++ .../trino/connector/CatalogManagerModule.java | 3 + .../CoordinatorDynamicCatalogManager.java | 9 +- .../trino/connector/StaticCatalogManager.java | 5 +- .../io/trino/execution/QueryPreparer.java | 10 +- .../trino/hogql/HogQlCatalogIdentifiers.java | 35 ++++ .../hogql/HogQlSemanticCatalogModule.java | 6 + .../HogQlSemanticCatalogPrewarmListener.java | 47 ++++++ .../java/io/trino/testing/PlanTester.java | 8 +- .../TestCatalogLifecycleListeners.java | 155 ++++++++++++++++++ .../TestHogQlSemanticCatalogGuiceWiring.java | 6 + .../TestHogQlSemanticCatalogManager.java | 33 ++++ 13 files changed, 379 insertions(+), 11 deletions(-) create mode 100644 core/trino-main/src/main/java/io/trino/connector/CatalogLifecycleListener.java create mode 100644 core/trino-main/src/main/java/io/trino/connector/CatalogLifecycleListeners.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlCatalogIdentifiers.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogPrewarmListener.java create mode 100644 core/trino-main/src/test/java/io/trino/connector/TestCatalogLifecycleListeners.java diff --git a/core/trino-main/src/main/java/io/trino/connector/CatalogLifecycleListener.java b/core/trino-main/src/main/java/io/trino/connector/CatalogLifecycleListener.java new file mode 100644 index 000000000000..24c20a3ae608 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/connector/CatalogLifecycleListener.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.connector; + +import io.trino.spi.catalog.CatalogName; + +@FunctionalInterface +public interface CatalogLifecycleListener +{ + /** + * Implementations run inline with catalog registration and must return without waiting for external work. + */ + void catalogLoaded(CatalogName catalogName); +} diff --git a/core/trino-main/src/main/java/io/trino/connector/CatalogLifecycleListeners.java b/core/trino-main/src/main/java/io/trino/connector/CatalogLifecycleListeners.java new file mode 100644 index 000000000000..c32526836196 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/connector/CatalogLifecycleListeners.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.connector; + +import com.google.inject.Inject; +import io.airlift.log.Logger; +import io.trino.spi.catalog.CatalogName; + +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public final class CatalogLifecycleListeners +{ + private static final Logger log = Logger.get(CatalogLifecycleListeners.class); + + private final Set listeners; + + @Inject + public CatalogLifecycleListeners(Set listeners) + { + this.listeners = Set.copyOf(requireNonNull(listeners, "listeners is null")); + } + + public void catalogLoaded(CatalogName catalogName) + { + requireNonNull(catalogName, "catalogName is null"); + for (CatalogLifecycleListener listener : listeners) { + try { + listener.catalogLoaded(catalogName); + } + catch (RuntimeException failure) { + log.warn(failure, "Catalog lifecycle listener failed for %s", catalogName); + } + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/connector/CatalogManagerModule.java b/core/trino-main/src/main/java/io/trino/connector/CatalogManagerModule.java index d792824432b2..421be6f98923 100644 --- a/core/trino-main/src/main/java/io/trino/connector/CatalogManagerModule.java +++ b/core/trino-main/src/main/java/io/trino/connector/CatalogManagerModule.java @@ -17,6 +17,7 @@ import com.google.inject.Scopes; import io.airlift.configuration.AbstractConfigurationAwareModule; +import static com.google.inject.multibindings.Multibinder.newSetBinder; import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; public class CatalogManagerModule @@ -28,6 +29,8 @@ protected void setup(Binder binder) binder.bind(DefaultCatalogFactory.class).in(Scopes.SINGLETON); binder.bind(LazyCatalogFactory.class).in(Scopes.SINGLETON); binder.bind(CatalogFactory.class).to(LazyCatalogFactory.class).in(Scopes.SINGLETON); + binder.bind(CatalogLifecycleListeners.class).in(Scopes.SINGLETON); + newSetBinder(binder, CatalogLifecycleListener.class); newOptionalBinder(binder, CatalogStoreManager.class); CatalogManagerConfig config = buildConfigObject(CatalogManagerConfig.class); diff --git a/core/trino-main/src/main/java/io/trino/connector/CoordinatorDynamicCatalogManager.java b/core/trino-main/src/main/java/io/trino/connector/CoordinatorDynamicCatalogManager.java index 89f06274c8dc..5f297d455199 100644 --- a/core/trino-main/src/main/java/io/trino/connector/CoordinatorDynamicCatalogManager.java +++ b/core/trino-main/src/main/java/io/trino/connector/CoordinatorDynamicCatalogManager.java @@ -43,6 +43,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executor; import static com.google.common.base.Preconditions.checkArgument; @@ -74,6 +75,7 @@ private enum State private final CatalogStore catalogStore; private final CatalogFactory catalogFactory; private final CacheManagerRegistry cacheManagerRegistry; + private final CatalogLifecycleListeners catalogLifecycleListeners; private final Executor executor; private final Object catalogsUpdateLock = new Object(); @@ -92,12 +94,13 @@ private enum State private State state = State.CREATED; @Inject - public CoordinatorDynamicCatalogManager(CatalogStore catalogStore, CatalogFactory catalogFactory, CacheManagerRegistry cacheManagerRegistry, @ForStartup Executor executor) + public CoordinatorDynamicCatalogManager(CatalogStore catalogStore, CatalogFactory catalogFactory, CacheManagerRegistry cacheManagerRegistry, @ForStartup Executor executor, CatalogLifecycleListeners catalogLifecycleListeners) { this.catalogStore = requireNonNull(catalogStore, "catalogStore is null"); this.catalogFactory = requireNonNull(catalogFactory, "catalogFactory is null"); this.cacheManagerRegistry = requireNonNull(cacheManagerRegistry, "cacheManagerRegistry is null"); this.executor = requireNonNull(executor, "executor is null"); + this.catalogLifecycleListeners = requireNonNull(catalogLifecycleListeners, "catalogLifecycleListeners is null"); } @PreDestroy @@ -124,6 +127,7 @@ public void stop() @Override public void loadInitialCatalogs() { + List loadedCatalogs = new CopyOnWriteArrayList<>(); synchronized (catalogsUpdateLock) { if (state == State.INITIALIZED) { return; @@ -142,6 +146,7 @@ public void loadInitialCatalogs() CatalogConnector newCatalog = catalogFactory.createCatalog(catalog); activeCatalogs.put(storedCatalog.name(), newCatalog.getCatalog()); allCatalogs.put(newCatalog.getCatalogHandle(), new RegisteredCatalog(new RegistrationToken(), newCatalog)); + loadedCatalogs.add(storedCatalog.name()); log.debug("-- Added catalog %s using connector %s --", storedCatalog.name(), catalog.connectorName()); } catch (Throwable e) { @@ -154,6 +159,7 @@ public void loadInitialCatalogs() }) .collect(toImmutableList())); } + loadedCatalogs.forEach(catalogLifecycleListeners::catalogLoaded); } @Override @@ -285,6 +291,7 @@ public void createCatalog(CatalogName catalogName, ConnectorName connectorName, log.debug("Added catalog: %s", catalog.getCatalogHandle()); } + catalogLifecycleListeners.catalogLoaded(catalogName); } public void registerGlobalSystemConnector(GlobalSystemConnector connector) diff --git a/core/trino-main/src/main/java/io/trino/connector/StaticCatalogManager.java b/core/trino-main/src/main/java/io/trino/connector/StaticCatalogManager.java index f3d54ff9a5a1..f56e826e76e2 100644 --- a/core/trino-main/src/main/java/io/trino/connector/StaticCatalogManager.java +++ b/core/trino-main/src/main/java/io/trino/connector/StaticCatalogManager.java @@ -69,6 +69,7 @@ private enum State } private final CatalogFactory catalogFactory; + private final CatalogLifecycleListeners catalogLifecycleListeners; private final List catalogProperties; private final Executor executor; @@ -77,9 +78,10 @@ private enum State private final AtomicReference state = new AtomicReference<>(State.CREATED); @Inject - public StaticCatalogManager(CatalogFactory catalogFactory, StaticCatalogManagerConfig config, @ForStartup Executor executor) + public StaticCatalogManager(CatalogFactory catalogFactory, StaticCatalogManagerConfig config, @ForStartup Executor executor, CatalogLifecycleListeners catalogLifecycleListeners) { this.catalogFactory = requireNonNull(catalogFactory, "catalogFactory is null"); + this.catalogLifecycleListeners = requireNonNull(catalogLifecycleListeners, "catalogLifecycleListeners is null"); List disabledCatalogs = requireNonNullElse(config.getDisabledCatalogs(), ImmutableList.of()); ImmutableList.Builder catalogProperties = ImmutableList.builder(); @@ -161,6 +163,7 @@ public void loadInitialCatalogs() log.info("-- Loading catalog %s --", catalogName); CatalogConnector newCatalog = catalogFactory.createCatalog(catalog); catalogs.put(catalogName, newCatalog); + catalogLifecycleListeners.catalogLoaded(catalogName); log.info("-- Added catalog %s using connector %s --", catalogName, catalog.connectorName()); return null; }) diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java index 2b932bdc2203..d998387999a5 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java @@ -23,7 +23,6 @@ import io.trino.hogql.compiler.HogQlCompileEnvelope; import io.trino.hogql.compiler.HogQlCompiler; import io.trino.hogql.compiler.HogQlSemanticCatalogContext; -import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider; import io.trino.spi.TrinoException; import io.trino.spi.resourcegroups.QueryType; @@ -36,11 +35,10 @@ import io.trino.sql.tree.Statement; import java.util.List; -import java.util.Locale; import java.util.Optional; -import java.util.regex.Pattern; import static io.trino.execution.ParameterExtractor.getParameterCount; +import static io.trino.hogql.HogQlCatalogIdentifiers.physicalCatalog; import static io.trino.hogql.HogQlCompilationEvent.Phase.COMPILATION; import static io.trino.hogql.HogQlCompilationEvent.Phase.PARAMETER_BINDING; import static io.trino.hogql.HogQlCompilationObserver.NOOP; @@ -54,8 +52,6 @@ public class QueryPreparer { - private static final Pattern UNDELIMITED_IDENTIFIER = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*"); - private final SqlParser sqlParser; private final Optional hogQlCompiler; private final HogQlParameterDecoder hogQlParameterDecoder; @@ -143,10 +139,8 @@ private Optional semanticCatalogContext(Session ses if (hogQlSemanticCatalogSnapshotProvider.isEmpty() || session.getCatalog().isEmpty()) { return Optional.empty(); } - String catalog = session.getCatalog().orElseThrow(); - boolean delimited = !UNDELIMITED_IDENTIFIER.matcher(catalog).matches() || !catalog.equals(catalog.toLowerCase(Locale.ENGLISH)); return Optional.of(new HogQlSemanticCatalogContext( - new PhysicalIdentifier(catalog, delimited), + physicalCatalog(session.getCatalog().orElseThrow()), hogQlSemanticCatalogSnapshotProvider.orElseThrow())); } diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCatalogIdentifiers.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCatalogIdentifiers.java new file mode 100644 index 000000000000..4096ebe8ae22 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCatalogIdentifiers.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; + +import java.util.Locale; +import java.util.regex.Pattern; + +import static java.util.Objects.requireNonNull; + +public final class HogQlCatalogIdentifiers +{ + private static final Pattern UNDELIMITED_IDENTIFIER = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*"); + + private HogQlCatalogIdentifiers() {} + + public static PhysicalIdentifier physicalCatalog(String catalog) + { + requireNonNull(catalog, "catalog is null"); + boolean delimited = !UNDELIMITED_IDENTIFIER.matcher(catalog).matches() || !catalog.equals(catalog.toLowerCase(Locale.ENGLISH)); + return new PhysicalIdentifier(catalog, delimited); + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogModule.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogModule.java index c49f92c5539b..7a986c367b88 100644 --- a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogModule.java +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogModule.java @@ -21,11 +21,13 @@ import com.google.inject.Singleton; import com.google.inject.multibindings.OptionalBinder; import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.connector.CatalogLifecycleListener; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotCache; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider; +import static com.google.inject.multibindings.Multibinder.newSetBinder; import static io.airlift.http.client.HttpClientBinder.httpClientBinder; public class HogQlSemanticCatalogModule @@ -43,6 +45,10 @@ protected void setup(Binder binder) .setMaxRequestsQueuedPerDestination(config.getLoaderQueueCapacity())); binder.bind(HogQlSemanticCatalogHttpTransport.class).in(Scopes.SINGLETON); binder.bind(HogQlSemanticCatalogManager.class).asEagerSingleton(); + newSetBinder(binder, CatalogLifecycleListener.class) + .addBinding() + .to(HogQlSemanticCatalogPrewarmListener.class) + .in(Scopes.SINGLETON); OptionalBinder.newOptionalBinder(binder, HogQlSemanticCatalogSnapshotProvider.class) .setBinding() .toProvider(SnapshotProviderFactory.class) diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogPrewarmListener.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogPrewarmListener.java new file mode 100644 index 000000000000..2347641418d0 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogPrewarmListener.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.Inject; +import io.airlift.log.Logger; +import io.trino.connector.CatalogLifecycleListener; +import io.trino.spi.catalog.CatalogName; + +import static io.trino.hogql.HogQlCatalogIdentifiers.physicalCatalog; +import static java.util.Objects.requireNonNull; + +public final class HogQlSemanticCatalogPrewarmListener + implements CatalogLifecycleListener +{ + private static final Logger log = Logger.get(HogQlSemanticCatalogPrewarmListener.class); + + private final HogQlSemanticCatalogManager semanticCatalogManager; + + @Inject + public HogQlSemanticCatalogPrewarmListener(HogQlSemanticCatalogManager semanticCatalogManager) + { + this.semanticCatalogManager = requireNonNull(semanticCatalogManager, "semanticCatalogManager is null"); + } + + @Override + public void catalogLoaded(CatalogName catalogName) + { + semanticCatalogManager.prewarm(physicalCatalog(catalogName.toString())) + .whenComplete((_, failure) -> { + if (failure != null) { + log.warn(failure, "Failed to prewarm HogQL semantic catalog for %s", catalogName); + } + }); + } +} diff --git a/core/trino-main/src/main/java/io/trino/testing/PlanTester.java b/core/trino-main/src/main/java/io/trino/testing/PlanTester.java index f4f9d6adbed9..9e62e7aabd34 100644 --- a/core/trino-main/src/main/java/io/trino/testing/PlanTester.java +++ b/core/trino-main/src/main/java/io/trino/testing/PlanTester.java @@ -36,6 +36,7 @@ import io.trino.cache.CacheManagerRegistry; import io.trino.connector.CatalogFactory; import io.trino.connector.CatalogHandle; +import io.trino.connector.CatalogLifecycleListeners; import io.trino.connector.CatalogServiceProviderModule; import io.trino.connector.ConnectorServicesProvider; import io.trino.connector.CoordinatorDynamicCatalogManager; @@ -386,7 +387,12 @@ private PlanTester(Session defaultSession, int nodeCountForStats) this.catalogFactory = catalogFactory; SecretsResolver secretsResolver = new SecretsResolver(ImmutableMap.of()); this.cacheManagerRegistry = new CacheManagerRegistry(noop(), noopTracer(), secretsResolver, new CacheManagerConfig()); - this.catalogManager = new CoordinatorDynamicCatalogManager(new InMemoryCatalogStore(), catalogFactory, cacheManagerRegistry, directExecutor()); + this.catalogManager = new CoordinatorDynamicCatalogManager( + new InMemoryCatalogStore(), + catalogFactory, + cacheManagerRegistry, + directExecutor(), + new CatalogLifecycleListeners(Set.of())); this.transactionManager = InMemoryTransactionManager.create( new TransactionManagerConfig().setIdleTimeout(new Duration(1, TimeUnit.DAYS)), yieldExecutor, diff --git a/core/trino-main/src/test/java/io/trino/connector/TestCatalogLifecycleListeners.java b/core/trino-main/src/test/java/io/trino/connector/TestCatalogLifecycleListeners.java new file mode 100644 index 000000000000..99aa3857cb9d --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/connector/TestCatalogLifecycleListeners.java @@ -0,0 +1,155 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.connector; + +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.MoreExecutors; +import io.airlift.configuration.secrets.SecretsResolver; +import io.opentelemetry.api.OpenTelemetry; +import io.trino.cache.CacheManagerConfig; +import io.trino.cache.CacheManagerRegistry; +import io.trino.spi.catalog.CatalogName; +import io.trino.spi.catalog.CatalogProperties; +import io.trino.spi.connector.CatalogVersion; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorFactory; +import io.trino.spi.connector.ConnectorName; +import io.trino.testing.TestingConnectorContext; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static io.airlift.tracing.Tracing.noopTracer; +import static io.trino.connector.CatalogHandle.createRootCatalogHandle; +import static org.assertj.core.api.Assertions.assertThat; + +class TestCatalogLifecycleListeners +{ + @Test + void testSuccessfulStaticAndDynamicCatalogLifecycleNotifications(@TempDir Path catalogDirectory) + throws IOException + { + List loadedCatalogs = new ArrayList<>(); + CatalogLifecycleListeners listeners = new CatalogLifecycleListeners( + Set.of(loadedCatalogs::add, _ -> { throw new RuntimeException("listener failure"); })); + + Files.writeString(catalogDirectory.resolve("static_catalog.properties"), "connector.name=mock\n"); + StaticCatalogManager staticManager = new StaticCatalogManager( + new TestingCatalogFactory(Set.of()), + new StaticCatalogManagerConfig().setCatalogConfigurationDir(catalogDirectory.toFile()), + MoreExecutors.directExecutor(), + listeners); + staticManager.loadInitialCatalogs(); + + InMemoryCatalogStore catalogStore = new InMemoryCatalogStore(); + catalogStore.addOrReplaceCatalog(catalogProperties("dynamic_initial")); + CoordinatorDynamicCatalogManager dynamicManager = new CoordinatorDynamicCatalogManager( + catalogStore, + new TestingCatalogFactory(Set.of()), + cacheManagerRegistry(), + MoreExecutors.directExecutor(), + listeners); + dynamicManager.loadInitialCatalogs(); + dynamicManager.createCatalog(new CatalogName("created_catalog"), new ConnectorName("mock"), Map.of(), false); + dynamicManager.createCatalog(new CatalogName("created_catalog"), new ConnectorName("mock"), Map.of(), true); + dynamicManager.dropCatalog(new CatalogName("created_catalog"), false); + dynamicManager.createCatalog(new CatalogName("created_catalog"), new ConnectorName("mock"), Map.of(), false); + + assertThat(loadedCatalogs).containsExactly( + new CatalogName("static_catalog"), + new CatalogName("dynamic_initial"), + new CatalogName("created_catalog"), + new CatalogName("created_catalog")); + + dynamicManager.stop(); + staticManager.stop(); + } + + @Test + void testFailedCatalogLoadDoesNotNotify() + { + List loadedCatalogs = new ArrayList<>(); + CatalogLifecycleListeners listeners = new CatalogLifecycleListeners(Set.of(loadedCatalogs::add)); + InMemoryCatalogStore catalogStore = new InMemoryCatalogStore(); + CatalogProperties broken = catalogProperties("broken_catalog"); + catalogStore.addOrReplaceCatalog(broken); + + CoordinatorDynamicCatalogManager manager = new CoordinatorDynamicCatalogManager( + catalogStore, + new TestingCatalogFactory(Set.of(broken.name())), + cacheManagerRegistry(), + MoreExecutors.directExecutor(), + listeners); + manager.loadInitialCatalogs(); + + assertThat(loadedCatalogs).isEmpty(); + manager.stop(); + } + + private static CatalogProperties catalogProperties(String name) + { + return new CatalogProperties(new CatalogName(name), new CatalogVersion("1"), new ConnectorName("mock"), ImmutableMap.of()); + } + + private static CacheManagerRegistry cacheManagerRegistry() + { + return new CacheManagerRegistry(OpenTelemetry.noop(), noopTracer(), new SecretsResolver(ImmutableMap.of()), new CacheManagerConfig()); + } + + private static class TestingCatalogFactory + implements CatalogFactory + { + private final Set failures; + + private TestingCatalogFactory(Set failures) + { + this.failures = Set.copyOf(failures); + } + + @Override + public void addConnectorFactory(ConnectorFactory connectorFactory) {} + + @Override + public CatalogConnector createCatalog(CatalogProperties catalogProperties) + { + if (failures.contains(catalogProperties.name())) { + throw new RuntimeException("catalog load failure"); + } + Connector connector = MockConnectorFactory.create().create(catalogProperties.name().toString(), catalogProperties.properties(), new TestingConnectorContext()); + CatalogHandle catalogHandle = createRootCatalogHandle(catalogProperties.name(), catalogProperties.version()); + ConnectorServices connectorServices = new ConnectorServices(noopTracer(), catalogHandle, connector); + return new CatalogConnector( + catalogHandle, + catalogProperties.connectorName(), + connectorServices, + connectorServices, + connectorServices, + Optional.of(catalogProperties)); + } + + @Override + public CatalogConnector createCatalog(CatalogHandle catalogHandle, ConnectorName connectorName, Connector connector) + { + throw new UnsupportedOperationException(); + } + } +} diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogGuiceWiring.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogGuiceWiring.java index b4eb9beb8ef5..72d995d534f8 100644 --- a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogGuiceWiring.java +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogGuiceWiring.java @@ -15,6 +15,7 @@ import com.google.inject.Key; import com.google.inject.TypeLiteral; +import io.trino.connector.CatalogLifecycleListener; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotCache; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider; import io.trino.server.testing.TestingTrinoServer; @@ -23,6 +24,7 @@ import java.io.IOException; import java.util.Map; import java.util.Optional; +import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; @@ -30,6 +32,8 @@ public class TestHogQlSemanticCatalogGuiceWiring { private static final Key> OPTIONAL_PROVIDER_KEY = Key.get(new TypeLiteral<>() {}); + private static final Key> CATALOG_LISTENERS_KEY = + Key.get(new TypeLiteral<>() {}); @Test public void testOptionalProviderFollowsHogQlAndCatalogConfiguration() @@ -39,6 +43,7 @@ public void testOptionalProviderFollowsHogQlAndCatalogConfiguration() .setProperties(Map.of("hogql.enabled", "false")) .build()) { assertThat(server.getInstance(OPTIONAL_PROVIDER_KEY)).isEmpty(); + assertThat(server.getInstance(CATALOG_LISTENERS_KEY)).isEmpty(); } try (TestingTrinoServer server = TestingTrinoServer.builder() @@ -51,6 +56,7 @@ public void testOptionalProviderFollowsHogQlAndCatalogConfiguration() HogQlSemanticCatalogSnapshotCache cache = server.getInstance(Key.get(HogQlSemanticCatalogSnapshotCache.class)); assertThat(provider).isPresent(); + assertThat(server.getInstance(CATALOG_LISTENERS_KEY)).hasOnlyElementsOfType(HogQlSemanticCatalogPrewarmListener.class); assertThat(server.getInstance(OPTIONAL_PROVIDER_KEY)).containsSame(provider.orElseThrow()); assertThat(cache).isSameAs(manager.cache()); } diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java index 46b5063a4a4a..3d5ebd9ec727 100644 --- a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java @@ -18,6 +18,7 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader; import io.trino.hogql.parser.HogQlLanguageVersion; +import io.trino.spi.catalog.CatalogName; import org.junit.jupiter.api.Test; import java.util.List; @@ -92,6 +93,38 @@ public void testShutdownTerminatesDedicatedLoaderExecutor() assertThat(manager.isLoaderExecutorShutdown()).isTrue(); } + @Test + public void testCatalogLifecyclePrewarmsWithoutWaitingForMetadata() + { + PhysicalIdentifier unusualCatalog = new PhysicalIdentifier("sales-data", true); + AtomicInteger loads = new AtomicInteger(); + CompletableFuture loadStarted = new CompletableFuture<>(); + CompletableFuture metadata = new CompletableFuture<>(); + HogQlSemanticCatalogManager manager = new HogQlSemanticCatalogManager( + config(), + request -> { + assertThat(request.catalog()).isEqualTo(unusualCatalog); + loads.incrementAndGet(); + loadStarted.complete(null); + return metadata; + }, + LANGUAGE_VERSION, + System::nanoTime); + try { + CompletableFuture listenerInvocation = CompletableFuture.runAsync( + () -> new HogQlSemanticCatalogPrewarmListener(manager).catalogLoaded(new CatalogName("sales-data"))); + + assertThat(listenerInvocation).succeedsWithin(10, SECONDS); + loadStarted.join(); + assertThat(loads).hasValue(1); + assertThat(manager.cache().currentSnapshot(unusualCatalog)).isEmpty(); + } + finally { + metadata.complete(new HogQlSemanticCatalogSnapshot(1, LANGUAGE_VERSION, unusualCatalog, 1, List.of())); + manager.shutdown(); + } + } + private static HogQlSemanticCatalogConfig config() { return new HogQlSemanticCatalogConfig() From c50099cd8307e3d45e5eed5618b8fe2e0a62192c Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 12:08:47 -0400 Subject: [PATCH 043/136] feat(hogql): consume semantic catalog schema v2 --- .../catalog/HogQlSemanticCatalogSnapshot.java | 782 +++++++++++++++--- ...gQlSemanticCatalogSnapshotJsonDecoder.java | 315 ++++++- .../compiler/TestHogQlSemanticResolution.java | 2 +- ...syncHogQlSemanticCatalogSnapshotCache.java | 2 +- .../TestHogQlSemanticCatalogSnapshot.java | 110 ++- ...gQlSemanticCatalogSnapshotJsonDecoder.java | 150 +++- ...estHogQlSemanticCatalogSnapshotLoader.java | 10 +- ...tHogQlSemanticCatalogSnapshotProvider.java | 2 +- 8 files changed, 1240 insertions(+), 133 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index d97f25ae51ad..ff2d7d31cae0 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -13,8 +13,13 @@ */ package io.trino.hogql.compiler.catalog; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.ObjectMapper; import io.trino.hogql.parser.HogQlLanguageVersion; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Base64; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -27,187 +32,419 @@ import static java.util.Objects.requireNonNull; public record HogQlSemanticCatalogSnapshot( + int protocolVersion, int schemaVersion, HogQlLanguageVersion languageVersion, PhysicalIdentifier catalog, long generation, - List logicalTables) + List logicalTables, + List expressionFields, + List virtualTables, + List savedQueries, + List materializedViews, + List functions, + List modifierDefaults) { + public static final int PROTOCOL_VERSION = 1; + public static final int SCHEMA_VERSION = 2; + private static final int MAX_SEMANTIC_DEFINITIONS = 10_000; + private static final int MAX_RECIPE_DEPTH = 64; + private static final int MAX_RECIPE_NODES = 4_096; + private static final int MAX_RELATION_DEPTH = 64; + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + public HogQlSemanticCatalogSnapshot { - if (schemaVersion != 1) { - throw new IllegalArgumentException("unsupported HogQL semantic catalog schema: " + schemaVersion); + if (protocolVersion != PROTOCOL_VERSION) { + throw new IllegalArgumentException("unsupported HogQL semantic catalog protocol"); + } + if (schemaVersion != SCHEMA_VERSION) { + throw new IllegalArgumentException("unsupported HogQL semantic catalog schema"); } if (generation <= 0) { throw new IllegalArgumentException("HogQL semantic catalog generation must be positive"); } languageVersion = requireNonNull(languageVersion, "languageVersion is null"); catalog = requireNonNull(catalog, "catalog is null"); - logicalTables = List.copyOf(requireNonNull(logicalTables, "logicalTables is null")); + logicalTables = copy(logicalTables, "logicalTables"); + expressionFields = copy(expressionFields, "expressionFields"); + virtualTables = copy(virtualTables, "virtualTables"); + savedQueries = copy(savedQueries, "savedQueries"); + materializedViews = copy(materializedViews, "materializedViews"); + functions = copy(functions, "functions"); + modifierDefaults = copy(modifierDefaults, "modifierDefaults"); + int definitions = expressionFields.size() + virtualTables.size() + savedQueries.size() + materializedViews.size() + functions.size() + modifierDefaults.size(); + if (definitions > MAX_SEMANTIC_DEFINITIONS) { + throw new IllegalArgumentException("semantic metadata exceeds definition limit"); + } + + Map tables = indexTables(catalog, logicalTables); + validateLogicalReferences(tables); + Set declaredFunctions = validateFunctions(functions); + Map expressions = indexExpressionFields(expressionFields, tables); + validateExpressionRecipes(expressionFields, tables, expressions, declaredFunctions); + validateModifiers(modifierDefaults); + validateRelations(catalog, tables, expressions, virtualTables, savedQueries, materializedViews); + } - Map tablesByName = indexTables(catalog, logicalTables); - validateReferences(tablesByName); + public HogQlSemanticCatalogSnapshot( + int schemaVersion, + HogQlLanguageVersion languageVersion, + PhysicalIdentifier catalog, + long generation, + List logicalTables) + { + this(PROTOCOL_VERSION, schemaVersion, languageVersion, catalog, generation, logicalTables, List.of(), List.of(), List.of(), List.of(), List.of(), List.of()); } public Optional logicalTable(String name) { - String canonicalName = canonicalName(name, "logical table"); - return logicalTables.stream() - .filter(table -> canonicalName(table.name(), "logical table").equals(canonicalName)) - .findFirst(); + String canonical = canonical(name, "logical table"); + return logicalTables.stream().filter(table -> canonical(table.name(), "logical table").equals(canonical)).findFirst(); } - private static Map indexTables(PhysicalIdentifier catalog, List tables) + private static Map indexTables(PhysicalIdentifier catalog, List definitions) { - Map tablesByName = new HashMap<>(); - for (LogicalTableDefinition table : tables) { + Map tables = new HashMap<>(); + for (LogicalTableDefinition table : definitions) { if (!table.physicalTable().catalog().equals(catalog)) { - throw new IllegalArgumentException("logical table physical reference uses another catalog: " + table.name()); + throw new IllegalArgumentException("logical table physical reference uses another catalog"); } - String canonicalName = canonicalName(table.name(), "logical table"); - if (tablesByName.put(canonicalName, table) != null) { - throw new IllegalArgumentException("duplicate logical table: " + table.name()); + if (tables.put(canonical(table.name(), "logical table"), table) != null) { + throw new IllegalArgumentException("duplicate logical table"); } - validateMemberNames(table); + Set members = new HashSet<>(); + table.fields().forEach(field -> addUnique(members, field.name(), "duplicate logical member")); + table.properties().forEach(property -> addUnique(members, property.name(), "duplicate logical member")); + table.relationships().forEach(relationship -> addUnique(members, relationship.name(), "duplicate logical member")); } - return Map.copyOf(tablesByName); + return Map.copyOf(tables); } - private static void validateMemberNames(LogicalTableDefinition table) + private static void validateLogicalReferences(Map tables) { - Set names = new HashSet<>(); - table.fields().forEach(field -> addMemberName(table, names, field.name())); - table.properties().forEach(property -> addMemberName(table, names, property.name())); - table.relationships().forEach(relationship -> addMemberName(table, names, relationship.name())); + for (LogicalTableDefinition table : tables.values()) { + Set fields = fieldNames(table); + for (PropertyDefinition property : table.properties()) { + requireReference(fields, property.sourceField(), "property has unknown source field"); + } + for (RelationshipDefinition relationship : table.relationships()) { + LogicalTableDefinition target = tables.get(canonical(relationship.targetTable(), "relationship target table")); + if (target == null) { + throw new IllegalArgumentException("relationship has unknown target table"); + } + Set targetFields = fieldNames(target); + for (JoinKey joinKey : relationship.joinKeys()) { + requireReference(fields, joinKey.sourceField(), "relationship has unknown source field"); + requireReference(targetFields, joinKey.targetField(), "relationship has unknown target field"); + } + } + } } - private static void addMemberName(LogicalTableDefinition table, Set names, String name) + private static Set validateFunctions(List definitions) { - if (!names.add(canonicalName(name, "logical member"))) { - throw new IllegalArgumentException("duplicate logical member in table " + table.name() + ": " + name); + Set functions = new HashSet<>(); + for (FunctionCapabilityDefinition function : definitions) { + addUnique(functions, function.name(), "duplicate function"); + if (function.implementation() == FunctionImplementation.REWRITE && !function.trinoName().isEmpty()) { + throw new IllegalArgumentException("rewrite function cannot name a Trino function"); + } + if (function.implementation() != FunctionImplementation.REWRITE && function.trinoName().isEmpty()) { + throw new IllegalArgumentException("function must name a Trino function"); + } } + return Set.copyOf(functions); } - private static void validateReferences(Map tablesByName) + private static Map indexExpressionFields(List definitions, Map tables) { - for (LogicalTableDefinition table : tablesByName.values()) { - Set fieldNames = fieldNames(table); - for (PropertyDefinition property : table.properties()) { - if (!fieldNames.contains(canonicalName(property.sourceField(), "property source field"))) { - throw new IllegalArgumentException("property has unknown source field in table " + table.name() + ": " + property.name()); - } + Map fields = new HashMap<>(); + for (ExpressionFieldDefinition field : definitions) { + LogicalTableDefinition table = tables.get(canonical(field.table(), "expression field table")); + if (table == null) { + throw new IllegalArgumentException("expression field references an unknown table"); } - for (RelationshipDefinition relationship : table.relationships()) { - LogicalTableDefinition target = tablesByName.get(canonicalName(relationship.targetTable(), "relationship target table")); - if (target == null) { - throw new IllegalArgumentException("relationship has unknown target table in table " + table.name() + ": " + relationship.name()); + Set members = new HashSet<>(fieldNames(table)); + table.properties().forEach(property -> members.add(canonical(property.name(), "property name"))); + table.relationships().forEach(relationship -> members.add(canonical(relationship.name(), "relationship name"))); + if (members.contains(canonical(field.name(), "expression field name"))) { + throw new IllegalArgumentException("expression field conflicts with an existing logical member"); + } + if (fields.put(expressionKey(field.table(), field.name()), field) != null) { + throw new IllegalArgumentException("duplicate expression field"); + } + } + return Map.copyOf(fields); + } + + private static void validateExpressionRecipes( + List definitions, + Map tables, + Map expressions, + Set functions) + { + Map> dependencies = new HashMap<>(); + RecipeCounter counter = new RecipeCounter(); + for (ExpressionFieldDefinition definition : definitions) { + List fieldDependencies = new ArrayList<>(); + validateRecipe(definition.recipe(), definition.table(), 1, counter, tables, expressions, functions, fieldDependencies); + dependencies.put(expressionKey(definition.table(), definition.name()), List.copyOf(fieldDependencies)); + } + Map states = new HashMap<>(); + dependencies.keySet().forEach(name -> visitDependency(name, 1, dependencies, states)); + } + + private static void validateRecipe( + ExpressionRecipe recipe, + String ownerTable, + int depth, + RecipeCounter counter, + Map tables, + Map expressions, + Set functions, + List dependencies) + { + requireNonNull(recipe, "expression recipe is null"); + if (depth > MAX_RECIPE_DEPTH) { + throw new IllegalArgumentException("expression recipe exceeds depth limit"); + } + counter.add(); + switch (recipe) { + case FieldReferenceRecipe reference -> { + if (!canonical(reference.table(), "field reference table").equals(canonical(ownerTable, "expression field table"))) { + throw new IllegalArgumentException("field reference crosses tables without a relationship path"); } - Set targetFieldNames = fieldNames(target); - for (JoinKey joinKey : relationship.joinKeys()) { - if (!fieldNames.contains(canonicalName(joinKey.sourceField(), "relationship source field"))) { - throw new IllegalArgumentException("relationship has unknown source field in table " + table.name() + ": " + relationship.name()); - } - if (!targetFieldNames.contains(canonicalName(joinKey.targetField(), "relationship target field"))) { - throw new IllegalArgumentException("relationship has unknown target field in table " + table.name() + ": " + relationship.name()); - } + LogicalTableDefinition table = tables.get(canonical(reference.table(), "field reference table")); + if (table == null) { + throw new IllegalArgumentException("field reference has unknown table"); + } + String key = expressionKey(reference.table(), reference.field()); + if (expressions.containsKey(key)) { + dependencies.add(key); } + else { + requireReference(fieldNames(table), reference.field(), "field reference has unknown field"); + } + } + case LiteralRecipe _ -> {} + case FunctionCallRecipe call -> { + requireReference(functions, call.name(), "function call references an undeclared function"); + call.arguments().forEach(argument -> validateRecipe(argument, ownerTable, depth + 1, counter, tables, expressions, functions, dependencies)); + } + case OperatorRecipe operator -> operator.arguments().forEach(argument -> validateRecipe(argument, ownerTable, depth + 1, counter, tables, expressions, functions, dependencies)); + case CastRecipe cast -> validateRecipe(cast.expression(), ownerTable, depth + 1, counter, tables, expressions, functions, dependencies); + } + } + + private static void visitDependency(String name, int depth, Map> dependencies, Map states) + { + if (depth > MAX_RECIPE_DEPTH) { + throw new IllegalArgumentException("expression field dependency exceeds depth limit"); + } + if (states.get(name) == VisitState.VISITING) { + throw new IllegalArgumentException("expression field dependency cycle"); + } + if (states.putIfAbsent(name, VisitState.VISITING) == VisitState.VISITED) { + return; + } + dependencies.getOrDefault(name, List.of()).forEach(dependency -> visitDependency(dependency, depth + 1, dependencies, states)); + states.put(name, VisitState.VISITED); + } + + private static void validateModifiers(List modifiers) + { + Set names = new HashSet<>(); + for (SemanticModifierDefault modifier : modifiers) { + addUnique(names, modifier.name(), "duplicate modifier"); + if (modifier.behavior() == ModifierBehavior.TRINO_SESSION_PROPERTY && modifier.sessionProperty().isEmpty()) { + throw new IllegalArgumentException("modifier must name a session property"); + } + if (modifier.behavior() != ModifierBehavior.TRINO_SESSION_PROPERTY && !modifier.sessionProperty().isEmpty()) { + throw new IllegalArgumentException("modifier cannot name a session property"); } } } + private static void validateRelations( + PhysicalIdentifier catalog, + Map tables, + Map expressions, + List virtualTables, + List savedQueries, + List materializedViews) + { + Map relations = new HashMap<>(); + tables.forEach((name, table) -> { + Set fields = new HashSet<>(fieldNames(table)); + expressions.values().stream() + .filter(expression -> canonical(expression.table(), "expression field table").equals(name)) + .forEach(expression -> fields.add(canonical(expression.name(), "expression field name"))); + relations.put(name, new SemanticRelation(RelationKind.LOGICAL_TABLE, fields, null, null)); + }); + savedQueries.forEach(saved -> addRelation(relations, saved.name(), new SemanticRelation(RelationKind.SAVED_QUERY, referencedFields(saved.fields()), null, saved))); + materializedViews.forEach(view -> { + if (!view.physicalView().catalog().equals(catalog)) { + throw new IllegalArgumentException("materialized view physical reference uses another catalog"); + } + addRelation(relations, view.name(), new SemanticRelation(RelationKind.MATERIALIZED_VIEW, referencedFields(view.fields()), null, null)); + }); + virtualTables.forEach(virtual -> addRelation(relations, virtual.name(), new SemanticRelation(RelationKind.VIRTUAL_TABLE, null, virtual, null))); + Map states = new HashMap<>(); + virtualTables.forEach(virtual -> resolveRelation(canonical(virtual.name(), "relation name"), relations, states, 1)); + savedQueries.forEach(saved -> resolveRelation(canonical(saved.name(), "relation name"), relations, states, 1)); + } + + private static void addRelation(Map relations, String name, SemanticRelation relation) + { + if (relations.put(canonical(name, "relation name"), relation) != null) { + throw new IllegalArgumentException("duplicate relation"); + } + } + + private static Set resolveRelation(String name, Map relations, Map states, int depth) + { + if (depth > MAX_RELATION_DEPTH) { + throw new IllegalArgumentException("virtual table reference exceeds depth limit"); + } + SemanticRelation relation = relations.get(name); + if (relation == null) { + throw new IllegalArgumentException("invalid semantic relation reference"); + } + if (relation.kind != RelationKind.VIRTUAL_TABLE && relation.kind != RelationKind.SAVED_QUERY) { + return relation.fields; + } + if (states.get(name) == VisitState.VISITING) { + throw new IllegalArgumentException("semantic relation reference cycle"); + } + if (states.get(name) == VisitState.VISITED) { + return requireNonNull(relation.fields, "resolved relation fields are null"); + } + states.put(name, VisitState.VISITING); + RelationReference reference = relation.kind == RelationKind.VIRTUAL_TABLE ? relation.virtualTable.source() : relation.savedQuery.target(); + if (relation.kind == RelationKind.SAVED_QUERY && reference.kind() == RelationKind.SAVED_QUERY) { + throw new IllegalArgumentException("saved query target must be logical, virtual, or materialized"); + } + SemanticRelation source = relations.get(canonical(reference.name(), "semantic relation source")); + if (source == null || source.kind != reference.kind()) { + throw new IllegalArgumentException("semantic relation references an unknown or mismatched source"); + } + Set sourceFields = source.kind == RelationKind.VIRTUAL_TABLE || source.kind == RelationKind.SAVED_QUERY + ? resolveRelation(canonical(reference.name(), "semantic relation source"), relations, states, depth + 1) + : requireNonNull(source.fields, "source relation fields are null"); + if (relation.kind == RelationKind.SAVED_QUERY) { + if (!sourceFields.containsAll(relation.fields)) { + throw new IllegalArgumentException("saved query declares a field missing from its target"); + } + } + else { + Set fields = new HashSet<>(); + for (VirtualProjection projection : relation.virtualTable.projections()) { + requireReference(sourceFields, projection.sourceField(), "virtual table projection references an unknown source field"); + addUnique(fields, projection.name(), "duplicate projection on virtual table"); + } + relation.fields = Set.copyOf(fields); + } + states.put(name, VisitState.VISITED); + return requireNonNull(relation.fields, "resolved relation fields are null"); + } + + private static Set referencedFields(List fields) + { + Set names = new HashSet<>(); + fields.forEach(field -> addUnique(names, field.name(), "duplicate referenced field")); + return names; + } + private static Set fieldNames(LogicalTableDefinition table) { - Set fieldNames = new HashSet<>(); - for (LogicalFieldDefinition field : table.fields()) { - fieldNames.add(canonicalName(field.name(), "logical field")); + Set fields = new HashSet<>(); + table.fields().forEach(field -> fields.add(canonical(field.name(), "logical field"))); + return fields; + } + + private static void requireReference(Set names, String name, String message) + { + if (!names.contains(canonical(name, "reference"))) { + throw new IllegalArgumentException(message); } - return Set.copyOf(fieldNames); } - private static String canonicalName(String name, String kind) + private static void addUnique(Set names, String name, String message) { - String value = validateDefinitionText(name, kind); - return value.toLowerCase(Locale.ENGLISH); + if (!names.add(canonical(name, "definition name"))) { + throw new IllegalArgumentException(message); + } } - private static String validateDefinitionText(String value, String kind) + private static String expressionKey(String table, String field) + { + return canonical(table, "expression field table") + "." + canonical(field, "expression field name"); + } + + private static String canonical(String value, String kind) + { + return definition(value, kind).toLowerCase(Locale.ENGLISH); + } + + private static String definition(String value, String kind) { requireNonNull(value, kind + " is null"); - if (value.isBlank() || containsExecutableDelimiter(value)) { - throw new IllegalArgumentException("invalid " + kind + ": " + value); + if (value.isBlank() || value.indexOf(';') >= 0 || value.indexOf('\0') >= 0 || value.indexOf('\n') >= 0 || value.indexOf('\r') >= 0 || value.contains("--") || value.contains("/*") || value.contains("*/")) { + throw new IllegalArgumentException("invalid " + kind); } return value; } - private static boolean containsExecutableDelimiter(String value) + private static List copy(List values, String name) { - return value.indexOf(';') >= 0 || value.indexOf('\0') >= 0 || value.indexOf('\n') >= 0 || value.indexOf('\r') >= 0 || value.contains("--") || value.contains("/*") || value.contains("*/"); + return List.copyOf(requireNonNull(values, name + " is null")); } - public record LogicalTableDefinition( - String name, - PhysicalQualifiedName physicalTable, - List fields, - List properties, - List relationships) + public record LogicalTableDefinition(String name, PhysicalQualifiedName physicalTable, List fields, List properties, List relationships) { public LogicalTableDefinition { - name = validateDefinitionText(name, "logical table name"); + name = definition(name, "logical table name"); physicalTable = requireNonNull(physicalTable, "physicalTable is null"); - fields = List.copyOf(requireNonNull(fields, "fields is null")); - properties = List.copyOf(requireNonNull(properties, "properties is null")); - relationships = List.copyOf(requireNonNull(relationships, "relationships is null")); + fields = copy(fields, "fields"); + properties = copy(properties, "properties"); + relationships = copy(relationships, "relationships"); } } - public record LogicalFieldDefinition( - String name, - PhysicalIdentifier physicalColumn, - String trinoTypeSignature, - LogicalType logicalType, - boolean nullable, - boolean starVisible) + public record LogicalFieldDefinition(String name, PhysicalIdentifier physicalColumn, String trinoTypeSignature, LogicalType logicalType, boolean nullable, boolean starVisible) { public LogicalFieldDefinition { - name = validateDefinitionText(name, "logical field name"); + name = definition(name, "logical field name"); physicalColumn = requireNonNull(physicalColumn, "physicalColumn is null"); - trinoTypeSignature = validateDefinitionText(trinoTypeSignature, "Trino type signature"); + trinoTypeSignature = definition(trinoTypeSignature, "Trino type signature"); logicalType = requireNonNull(logicalType, "logicalType is null"); } } - public record PropertyDefinition( - String name, - String sourceField, - PropertyStorage storage, - LogicalType logicalType, - boolean nullable) + public record PropertyDefinition(String name, String sourceField, PropertyStorage storage, LogicalType logicalType, boolean nullable) { public PropertyDefinition { - name = validateDefinitionText(name, "property name"); - sourceField = validateDefinitionText(sourceField, "property source field"); + name = definition(name, "property name"); + sourceField = definition(sourceField, "property source field"); storage = requireNonNull(storage, "storage is null"); logicalType = requireNonNull(logicalType, "logicalType is null"); } } - public record RelationshipDefinition( - String name, - String targetTable, - RelationshipCardinality cardinality, - List joinKeys) + public record RelationshipDefinition(String name, String targetTable, RelationshipCardinality cardinality, List joinKeys) { public RelationshipDefinition { - name = validateDefinitionText(name, "relationship name"); - targetTable = validateDefinitionText(targetTable, "relationship target table"); + name = definition(name, "relationship name"); + targetTable = definition(targetTable, "relationship target table"); cardinality = requireNonNull(cardinality, "cardinality is null"); - joinKeys = List.copyOf(requireNonNull(joinKeys, "joinKeys is null")); + joinKeys = copy(joinKeys, "joinKeys"); if (joinKeys.isEmpty()) { - throw new IllegalArgumentException("relationship must have at least one join key: " + name); + throw new IllegalArgumentException("relationship must have at least one join key"); } } } @@ -216,8 +453,262 @@ public record JoinKey(String sourceField, String targetField) { public JoinKey { - sourceField = validateDefinitionText(sourceField, "relationship source field"); - targetField = validateDefinitionText(targetField, "relationship target field"); + sourceField = definition(sourceField, "relationship source field"); + targetField = definition(targetField, "relationship target field"); + } + } + + public record ExpressionFieldDefinition(String table, String name, String trinoTypeSignature, LogicalType logicalType, boolean nullable, boolean starVisible, ExpressionRecipe recipe) + { + public ExpressionFieldDefinition + { + table = definition(table, "expression field table"); + name = definition(name, "expression field name"); + trinoTypeSignature = definition(trinoTypeSignature, "expression field type signature"); + logicalType = requireNonNull(logicalType, "logicalType is null"); + recipe = requireNonNull(recipe, "recipe is null"); + } + } + + public sealed interface ExpressionRecipe + permits CastRecipe, + FieldReferenceRecipe, + FunctionCallRecipe, + LiteralRecipe, + OperatorRecipe + { + ExpressionRecipeKind kind(); + } + + public record FieldReferenceRecipe(String table, String field) + implements ExpressionRecipe + { + public FieldReferenceRecipe + { + table = definition(table, "field reference table"); + field = definition(field, "field reference field"); + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.FIELD_REFERENCE; + } + } + + public record LiteralRecipe(TypedLiteral literal) + implements ExpressionRecipe + { + public LiteralRecipe + { + literal = requireNonNull(literal, "literal is null"); + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.LITERAL; + } + } + + public record FunctionCallRecipe(String name, List arguments) + implements ExpressionRecipe + { + public FunctionCallRecipe + { + name = definition(name, "function name"); + arguments = copy(arguments, "arguments"); + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.FUNCTION_CALL; + } + } + + public record OperatorRecipe(SemanticOperator operator, List arguments) + implements ExpressionRecipe + { + public OperatorRecipe + { + operator = requireNonNull(operator, "operator is null"); + arguments = copy(arguments, "arguments"); + if (arguments.isEmpty()) { + throw new IllegalArgumentException("operator recipe requires arguments"); + } + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.OPERATOR; + } + } + + public record CastRecipe(ExpressionRecipe expression, String targetTypeSignature) + implements ExpressionRecipe + { + public CastRecipe + { + expression = requireNonNull(expression, "expression is null"); + targetTypeSignature = definition(targetTypeSignature, "cast target type signature"); + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.CAST; + } + } + + public record TypedLiteral(String typeSignature, LiteralEncoding encoding, String value) + { + private static final Pattern DECIMAL = Pattern.compile("[+-]?(0|[1-9][0-9]*)(\\.[0-9]+)?"); + + public TypedLiteral + { + typeSignature = definition(typeSignature, "literal type signature"); + encoding = requireNonNull(encoding, "encoding is null"); + value = requireNonNull(value, "value is null"); + if (value.indexOf('\0') >= 0) { + throw new IllegalArgumentException("literal value contains NUL"); + } + try { + switch (encoding) { + case NULL -> { + if (!value.isEmpty()) { + throw new IllegalArgumentException(); + } + } + case STRING -> {} + case BOOLEAN -> { + if (!value.equals("true") && !value.equals("false")) { + throw new IllegalArgumentException(); + } + } + case INTEGER -> Long.parseLong(value); + case DECIMAL -> { + if (!DECIMAL.matcher(value).matches()) { + throw new IllegalArgumentException(); + } + } + case FLOAT -> { + if (!Double.isFinite(Double.parseDouble(value))) { + throw new IllegalArgumentException(); + } + } + case JSON -> validateJson(value); + case BASE64 -> Base64.getDecoder().decode(value); + } + } + catch (IOException | IllegalArgumentException e) { + throw new IllegalArgumentException("invalid typed literal"); + } + } + + private static void validateJson(String value) + throws IOException + { + try (JsonParser parser = JSON_MAPPER.createParser(value)) { + if (JSON_MAPPER.readTree(parser) == null || parser.nextToken() != null) { + throw new IllegalArgumentException(); + } + } + } + } + + public record VirtualTableDefinition(String name, RelationReference source, List projections) + { + public VirtualTableDefinition + { + name = definition(name, "virtual table name"); + source = requireNonNull(source, "source is null"); + projections = copy(projections, "projections"); + } + } + + public record RelationReference(RelationKind kind, String name) + { + public RelationReference + { + kind = requireNonNull(kind, "kind is null"); + name = definition(name, "semantic relation source"); + } + } + + public record VirtualProjection(String name, String sourceField, boolean starVisible) + { + public VirtualProjection + { + name = definition(name, "virtual projection name"); + sourceField = definition(sourceField, "virtual projection source field"); + } + } + + public record SavedQueryReference(String name, String queryId, RelationReference target, List fields) + { + public SavedQueryReference + { + name = definition(name, "saved query name"); + queryId = definition(queryId, "saved query ID"); + target = requireNonNull(target, "target is null"); + fields = copy(fields, "fields"); + } + } + + public record MaterializedViewReference(String name, PhysicalQualifiedName physicalView, List fields) + { + public MaterializedViewReference + { + name = definition(name, "materialized view name"); + physicalView = requireNonNull(physicalView, "physicalView is null"); + fields = copy(fields, "fields"); + } + } + + public record ReferencedField(String name, String trinoTypeSignature, LogicalType logicalType, boolean nullable, boolean starVisible) + { + public ReferencedField + { + name = definition(name, "referenced field name"); + trinoTypeSignature = definition(trinoTypeSignature, "referenced field type signature"); + logicalType = requireNonNull(logicalType, "logicalType is null"); + } + } + + public record FunctionCapabilityDefinition(String name, FunctionKind kind, FunctionImplementation implementation, List trinoName, List signatures, boolean deterministic, boolean supportsDistinct, boolean supportsOrderBy, boolean supportsFilter, boolean supportsWindow) + { + public FunctionCapabilityDefinition + { + name = definition(name, "function name"); + kind = requireNonNull(kind, "kind is null"); + implementation = requireNonNull(implementation, "implementation is null"); + trinoName = copy(trinoName, "trinoName"); + signatures = copy(signatures, "signatures"); + if (signatures.isEmpty()) { + throw new IllegalArgumentException("function must include signatures"); + } + } + } + + public record FunctionSignature(List argumentTypes, String returnType, boolean variadic) + { + public FunctionSignature + { + argumentTypes = copy(argumentTypes, "argumentTypes").stream().map(value -> definition(value, "function argument type")).toList(); + returnType = definition(returnType, "function return type"); + } + } + + public record SemanticModifierDefault(String name, ModifierBehavior behavior, TypedLiteral defaultValue, List sessionProperty) + { + public SemanticModifierDefault + { + name = definition(name, "modifier name"); + behavior = requireNonNull(behavior, "behavior is null"); + defaultValue = requireNonNull(defaultValue, "defaultValue is null"); + sessionProperty = copy(sessionProperty, "sessionProperty"); } } @@ -233,13 +724,13 @@ public record PhysicalQualifiedName(PhysicalIdentifier catalog, PhysicalIdentifi public record PhysicalIdentifier(String value, boolean delimited) { - private static final Pattern UNDELIMITED_IDENTIFIER = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*"); + private static final Pattern UNDELIMITED = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*"); public PhysicalIdentifier { - value = validateDefinitionText(value, "physical identifier"); - if (!delimited && !UNDELIMITED_IDENTIFIER.matcher(value).matches()) { - throw new IllegalArgumentException("invalid physical identifier: " + value); + value = definition(value, "physical identifier"); + if (!delimited && !UNDELIMITED.matcher(value).matches()) { + throw new IllegalArgumentException("invalid physical identifier"); } if (!delimited) { value = value.toLowerCase(Locale.ENGLISH); @@ -247,35 +738,86 @@ public record PhysicalIdentifier(String value, boolean delimited) } } + public enum ExpressionRecipeKind + { + FIELD_REFERENCE, LITERAL, FUNCTION_CALL, OPERATOR, CAST + } + + public enum SemanticOperator + { + ADD, SUBTRACT, MULTIPLY, DIVIDE, MODULUS, EQUAL, NOT_EQUAL, LESS_THAN, LESS_THAN_OR_EQUAL, GREATER_THAN, GREATER_THAN_OR_EQUAL, AND, OR, NOT, NEGATE, IS_NULL, IS_NOT_NULL + } + + public enum LiteralEncoding + { + NULL, STRING, BOOLEAN, INTEGER, DECIMAL, FLOAT, JSON, BASE64 + } + + public enum RelationKind + { + LOGICAL_TABLE, VIRTUAL_TABLE, SAVED_QUERY, MATERIALIZED_VIEW + } + + public enum FunctionKind + { + SCALAR, AGGREGATE, WINDOW, TABLE + } + + public enum FunctionImplementation + { + STOCK, UDF, REWRITE + } + + public enum ModifierBehavior + { + COMPILER, TRINO_SESSION_PROPERTY, SAFE_NOOP, UNSUPPORTED + } + public enum LogicalType { - UNKNOWN, - BOOLEAN, - INTEGER, - FLOAT, - DECIMAL, - STRING, - DATE, - TIMESTAMP, - INTERVAL, - UUID, - JSON, - ARRAY, - MAP, - ROW, + UNKNOWN, BOOLEAN, INTEGER, FLOAT, DECIMAL, STRING, DATE, TIMESTAMP, INTERVAL, UUID, JSON, ARRAY, MAP, ROW } public enum PropertyStorage { - JSON_OBJECT, - MAP, + JSON_OBJECT, MAP } public enum RelationshipCardinality { - ONE_TO_ONE, - ONE_TO_MANY, - MANY_TO_ONE, - MANY_TO_MANY, + ONE_TO_ONE, ONE_TO_MANY, MANY_TO_ONE, MANY_TO_MANY + } + + private enum VisitState + { + VISITING, VISITED + } + + private static final class RecipeCounter + { + private int nodes; + + private void add() + { + if (++nodes > MAX_RECIPE_NODES) { + throw new IllegalArgumentException("expression recipes exceed node limit"); + } + } + } + + private static final class SemanticRelation + { + private final RelationKind kind; + private Set fields; + private final VirtualTableDefinition virtualTable; + private final SavedQueryReference savedQuery; + + private SemanticRelation(RelationKind kind, Set fields, VirtualTableDefinition virtualTable, SavedQueryReference savedQuery) + { + this.kind = requireNonNull(kind, "kind is null"); + this.fields = fields == null ? null : Set.copyOf(fields); + this.virtualTable = virtualTable; + this.savedQuery = savedQuery; + } } } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java index f720c71b0ec7..821a83447e76 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java @@ -22,16 +22,40 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CastRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionRecipeKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCallRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.MaterializedViewReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ModifierBehavior; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ReferencedField; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationReference; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SavedQueryReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticModifierDefault; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.VirtualProjection; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.VirtualTableDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; import io.trino.hogql.parser.HogQlLanguageVersion; @@ -48,8 +72,8 @@ public final class HogQlSemanticCatalogSnapshotJsonDecoder public static final int PROTOCOL_VERSION = 1; public static final int MAXIMUM_PAYLOAD_BYTES = 8 * 1024 * 1024; - private static final int SCHEMA_VERSION = 1; - private static final Limits DEFAULT_LIMITS = new Limits(MAXIMUM_PAYLOAD_BYTES, 64, 100_000); + private static final int SCHEMA_VERSION = 2; + private static final Limits DEFAULT_LIMITS = new Limits(MAXIMUM_PAYLOAD_BYTES, 256, 100_000); private static final Set SNAPSHOT_FIELDS = Set.of( "protocolVersion", @@ -57,7 +81,13 @@ public final class HogQlSemanticCatalogSnapshotJsonDecoder "languageVersion", "catalog", "generation", - "logicalTables"); + "logicalTables", + "expressionFields", + "virtualTables", + "savedQueries", + "materializedViews", + "functions", + "modifierDefaults"); private static final Set IDENTIFIER_FIELDS = Set.of("value", "delimited"); private static final Set QUALIFIED_NAME_FIELDS = Set.of("catalog", "schema", "table"); private static final Set TABLE_FIELDS = Set.of("name", "physicalTable", "fields", "properties", "relationships"); @@ -65,6 +95,27 @@ public final class HogQlSemanticCatalogSnapshotJsonDecoder private static final Set PROPERTY_FIELDS = Set.of("name", "sourceField", "storage", "logicalType", "nullable"); private static final Set RELATIONSHIP_FIELDS = Set.of("name", "targetTable", "cardinality", "joinKeys"); private static final Set JOIN_KEY_FIELDS = Set.of("sourceField", "targetField"); + private static final Set EXPRESSION_FIELD_FIELDS = Set.of("table", "name", "trinoTypeSignature", "logicalType", "nullable", "starVisible", "recipe"); + private static final Set RECIPE_FIELD_REFERENCE_FIELDS = Set.of("kind", "fieldReference"); + private static final Set RECIPE_LITERAL_FIELDS = Set.of("kind", "literal"); + private static final Set RECIPE_FUNCTION_CALL_FIELDS = Set.of("kind", "functionCall"); + private static final Set RECIPE_OPERATOR_FIELDS = Set.of("kind", "operator"); + private static final Set RECIPE_CAST_FIELDS = Set.of("kind", "cast"); + private static final Set FIELD_REFERENCE_FIELDS = Set.of("table", "field"); + private static final Set TYPED_LITERAL_FIELDS = Set.of("typeSignature", "encoding", "value"); + private static final Set FUNCTION_CALL_FIELDS = Set.of("name", "arguments"); + private static final Set OPERATOR_FIELDS = Set.of("operator", "arguments"); + private static final Set CAST_FIELDS = Set.of("expression", "targetTypeSignature"); + private static final Set VIRTUAL_TABLE_FIELDS = Set.of("name", "source", "projections"); + private static final Set RELATION_REFERENCE_FIELDS = Set.of("kind", "name"); + private static final Set VIRTUAL_PROJECTION_FIELDS = Set.of("name", "sourceField", "starVisible"); + private static final Set SAVED_QUERY_FIELDS = Set.of("name", "queryId", "target", "fields"); + private static final Set MATERIALIZED_VIEW_FIELDS = Set.of("name", "physicalView", "fields"); + private static final Set REFERENCED_FIELD_FIELDS = Set.of("name", "trinoTypeSignature", "logicalType", "nullable", "starVisible"); + private static final Set FUNCTION_FIELDS = Set.of("name", "kind", "implementation", "trinoName", "signatures", "deterministic", "supportsDistinct", "supportsOrderBy", "supportsFilter", "supportsWindow"); + private static final Set FUNCTION_SIGNATURE_FIELDS = Set.of("argumentTypes", "returnType", "variadic"); + private static final Set MODIFIER_FIELDS = Set.of("name", "behavior", "defaultValue", "sessionProperty"); + private static final Set MODIFIER_FIELDS_WITHOUT_SESSION_PROPERTY = Set.of("name", "behavior", "defaultValue"); private final Limits limits; private final ObjectMapper objectMapper; @@ -128,7 +179,25 @@ public HogQlSemanticCatalogSnapshot decode(byte[] payload, LoadRequest request) CollectionBudget budget = new CollectionBudget(limits.maximumCollectionEntries()); List tables = logicalTables(required(root, "logicalTables"), budget); - return new HogQlSemanticCatalogSnapshot(schemaVersion, languageVersion, catalog, generation, tables); + List expressionFields = expressionFields(required(root, "expressionFields"), budget); + List virtualTables = virtualTables(required(root, "virtualTables"), budget); + List savedQueries = savedQueries(required(root, "savedQueries"), budget); + List materializedViews = materializedViews(required(root, "materializedViews"), budget); + List functions = functions(required(root, "functions"), budget); + List modifierDefaults = modifierDefaults(required(root, "modifierDefaults"), budget); + return new HogQlSemanticCatalogSnapshot( + protocolVersion, + schemaVersion, + languageVersion, + catalog, + generation, + tables, + expressionFields, + virtualTables, + savedQueries, + materializedViews, + functions, + modifierDefaults); } catch (DecodeException e) { throw e; @@ -247,6 +316,244 @@ private static List joinKeys(JsonNode node, CollectionBudget budget) return List.copyOf(joinKeys); } + private static List expressionFields(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List definitions = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode field = object(element); + validateFields(field, EXPRESSION_FIELD_FIELDS); + definitions.add(new ExpressionFieldDefinition( + text(field, "table"), + text(field, "name"), + text(field, "trinoTypeSignature"), + enumValue(field, "logicalType", LogicalType.class), + bool(field, "nullable"), + bool(field, "starVisible"), + expressionRecipe(required(field, "recipe"), budget))); + } + return List.copyOf(definitions); + } + + private static ExpressionRecipe expressionRecipe(JsonNode node, CollectionBudget budget) + { + ObjectNode recipe = object(node); + ExpressionRecipeKind kind = enumValue(recipe, "kind", ExpressionRecipeKind.class); + return switch (kind) { + case FIELD_REFERENCE -> { + validateFields(recipe, RECIPE_FIELD_REFERENCE_FIELDS); + ObjectNode reference = object(required(recipe, "fieldReference")); + validateFields(reference, FIELD_REFERENCE_FIELDS); + yield new FieldReferenceRecipe(text(reference, "table"), text(reference, "field")); + } + case LITERAL -> { + validateFields(recipe, RECIPE_LITERAL_FIELDS); + yield new LiteralRecipe(typedLiteral(required(recipe, "literal"))); + } + case FUNCTION_CALL -> { + validateFields(recipe, RECIPE_FUNCTION_CALL_FIELDS); + ObjectNode call = object(required(recipe, "functionCall")); + validateFields(call, FUNCTION_CALL_FIELDS); + yield new FunctionCallRecipe(text(call, "name"), expressionRecipes(required(call, "arguments"), budget)); + } + case OPERATOR -> { + validateFields(recipe, RECIPE_OPERATOR_FIELDS); + ObjectNode operator = object(required(recipe, "operator")); + validateFields(operator, OPERATOR_FIELDS); + yield new OperatorRecipe(enumValue(operator, "operator", SemanticOperator.class), expressionRecipes(required(operator, "arguments"), budget)); + } + case CAST -> { + validateFields(recipe, RECIPE_CAST_FIELDS); + ObjectNode cast = object(required(recipe, "cast")); + validateFields(cast, CAST_FIELDS); + yield new CastRecipe(expressionRecipe(required(cast, "expression"), budget), text(cast, "targetTypeSignature")); + } + }; + } + + private static List expressionRecipes(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List recipes = new ArrayList<>(array.size()); + for (JsonNode element : array) { + recipes.add(expressionRecipe(element, budget)); + } + return List.copyOf(recipes); + } + + private static TypedLiteral typedLiteral(JsonNode node) + { + ObjectNode literal = object(node); + validateFields(literal, TYPED_LITERAL_FIELDS); + return new TypedLiteral( + text(literal, "typeSignature"), + enumValue(literal, "encoding", LiteralEncoding.class), + text(literal, "value")); + } + + private static List virtualTables(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List definitions = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode table = object(element); + validateFields(table, VIRTUAL_TABLE_FIELDS); + definitions.add(new VirtualTableDefinition( + text(table, "name"), + relationReference(required(table, "source")), + virtualProjections(required(table, "projections"), budget))); + } + return List.copyOf(definitions); + } + + private static RelationReference relationReference(JsonNode node) + { + ObjectNode reference = object(node); + validateFields(reference, RELATION_REFERENCE_FIELDS); + return new RelationReference(enumValue(reference, "kind", RelationKind.class), text(reference, "name")); + } + + private static List virtualProjections(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List projections = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode projection = object(element); + validateFields(projection, VIRTUAL_PROJECTION_FIELDS); + projections.add(new VirtualProjection(text(projection, "name"), text(projection, "sourceField"), bool(projection, "starVisible"))); + } + return List.copyOf(projections); + } + + private static List savedQueries(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List definitions = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode savedQuery = object(element); + validateFields(savedQuery, SAVED_QUERY_FIELDS); + definitions.add(new SavedQueryReference( + text(savedQuery, "name"), + text(savedQuery, "queryId"), + relationReference(required(savedQuery, "target")), + referencedFields(required(savedQuery, "fields"), budget))); + } + return List.copyOf(definitions); + } + + private static List materializedViews(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List definitions = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode view = object(element); + validateFields(view, MATERIALIZED_VIEW_FIELDS); + definitions.add(new MaterializedViewReference( + text(view, "name"), + qualifiedName(required(view, "physicalView")), + referencedFields(required(view, "fields"), budget))); + } + return List.copyOf(definitions); + } + + private static List referencedFields(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List fields = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode field = object(element); + validateFields(field, REFERENCED_FIELD_FIELDS); + fields.add(new ReferencedField( + text(field, "name"), + text(field, "trinoTypeSignature"), + enumValue(field, "logicalType", LogicalType.class), + bool(field, "nullable"), + bool(field, "starVisible"))); + } + return List.copyOf(fields); + } + + private static List functions(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List definitions = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode function = object(element); + validateFields(function, FUNCTION_FIELDS); + definitions.add(new FunctionCapabilityDefinition( + text(function, "name"), + enumValue(function, "kind", FunctionKind.class), + enumValue(function, "implementation", FunctionImplementation.class), + physicalIdentifiers(required(function, "trinoName"), budget), + functionSignatures(required(function, "signatures"), budget), + bool(function, "deterministic"), + bool(function, "supportsDistinct"), + bool(function, "supportsOrderBy"), + bool(function, "supportsFilter"), + bool(function, "supportsWindow"))); + } + return List.copyOf(definitions); + } + + private static List physicalIdentifiers(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List identifiers = new ArrayList<>(array.size()); + for (JsonNode element : array) { + identifiers.add(physicalIdentifier(element)); + } + return List.copyOf(identifiers); + } + + private static List functionSignatures(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List signatures = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode signature = object(element); + validateFields(signature, FUNCTION_SIGNATURE_FIELDS); + signatures.add(new FunctionSignature( + strings(required(signature, "argumentTypes"), budget), + text(signature, "returnType"), + bool(signature, "variadic"))); + } + return List.copyOf(signatures); + } + + private static List strings(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List values = new ArrayList<>(array.size()); + for (JsonNode element : array) { + if (!element.isTextual()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + values.add(element.textValue()); + } + return List.copyOf(values); + } + + private static List modifierDefaults(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List modifiers = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode modifier = object(element); + if (modifier.has("sessionProperty")) { + validateFields(modifier, MODIFIER_FIELDS); + } + else { + validateFields(modifier, MODIFIER_FIELDS_WITHOUT_SESSION_PROPERTY); + } + modifiers.add(new SemanticModifierDefault( + text(modifier, "name"), + enumValue(modifier, "behavior", ModifierBehavior.class), + typedLiteral(required(modifier, "defaultValue")), + modifier.has("sessionProperty") ? physicalIdentifiers(required(modifier, "sessionProperty"), budget) : List.of())); + } + return List.copyOf(modifiers); + } + private static PhysicalQualifiedName qualifiedName(JsonNode node) { ObjectNode name = object(node); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java index 0e8351af1750..3f3a1fe7639a 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -47,7 +47,7 @@ public class TestHogQlSemanticResolution { private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); private static final HogQlSemanticCatalogSnapshot SNAPSHOT = new HogQlSemanticCatalogSnapshot( - 1, + 2, HogQlLanguageContract.current().languageVersion(), CATALOG, 7, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java index 4cc37f149c33..f7c527be7a35 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java @@ -268,7 +268,7 @@ private static PhysicalIdentifier catalog(String name) private static HogQlSemanticCatalogSnapshot snapshot(PhysicalIdentifier catalog, long generation) { return new HogQlSemanticCatalogSnapshot( - 1, + 2, LANGUAGE_VERSION, catalog, generation, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java index 400f05fdf723..872204832510 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java @@ -13,7 +13,17 @@ */ package io.trino.hogql.compiler.catalog; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CastRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCallRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; @@ -23,6 +33,7 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; import io.trino.hogql.parser.HogQlLanguageVersion; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -99,6 +110,64 @@ public void testAcceptsBidirectionalLazyRelationships() .satisfies(relationships -> assertThat(relationships).singleElement().extracting(RelationshipDefinition::targetTable).isEqualTo("events")); } + @Test + public void testSemanticDefinitionsAreDeeplyImmutable() + { + List arguments = new ArrayList<>(List.of(literal())); + FunctionCallRecipe recipe = new FunctionCallRecipe("identity", arguments); + List expressionFields = new ArrayList<>(List.of( + new ExpressionFieldDefinition("events", "derived", "bigint", LogicalType.INTEGER, false, true, recipe))); + List signatures = new ArrayList<>(List.of(new FunctionSignature(List.of("bigint"), "bigint", false))); + List functions = new ArrayList<>(List.of( + new FunctionCapabilityDefinition( + "identity", + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of(new PhysicalIdentifier("identity", false)), + signatures, + true, + false, + false, + false, + false))); + + HogQlSemanticCatalogSnapshot snapshot = semanticSnapshot(expressionFields, functions); + arguments.clear(); + expressionFields.clear(); + signatures.clear(); + functions.clear(); + + assertThat(snapshot.expressionFields()).singleElement().extracting(ExpressionFieldDefinition::recipe).isEqualTo(recipe); + assertThat(recipe.arguments()).singleElement().isEqualTo(literal()); + assertThat(snapshot.functions()).singleElement().extracting(FunctionCapabilityDefinition::signatures).satisfies(values -> assertThat(values).hasSize(1)); + assertThatThrownBy(() -> recipe.arguments().clear()).isInstanceOf(UnsupportedOperationException.class); + } + + @Test + public void testEnforcesExpressionRecipeDepthAndNodeLimits() + { + ExpressionRecipe deepRecipe = literal(); + for (int depth = 1; depth < 65; depth++) { + deepRecipe = new CastRecipe(deepRecipe, "bigint"); + } + ExpressionRecipe finalDeepRecipe = deepRecipe; + assertThatThrownBy(() -> semanticSnapshot( + List.of(new ExpressionFieldDefinition("events", "derived", "bigint", LogicalType.INTEGER, false, true, finalDeepRecipe)), + List.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("depth limit"); + + List arguments = new ArrayList<>(); + for (int node = 0; node < 4_096; node++) { + arguments.add(literal()); + } + assertThatThrownBy(() -> semanticSnapshot( + List.of(new ExpressionFieldDefinition("events", "derived", "bigint", LogicalType.INTEGER, false, true, new FunctionCallRecipe("identity", arguments))), + List.of(function("identity")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("node limit"); + } + private static Stream invalidSnapshots() { LogicalTableDefinition events = table("events"); @@ -149,7 +218,46 @@ public void testRejectsRawExecutablePhysicalDefinitions(String definition) private static HogQlSemanticCatalogSnapshot snapshot(List tables) { - return new HogQlSemanticCatalogSnapshot(1, LANGUAGE_VERSION, CATALOG, 7, tables); + return new HogQlSemanticCatalogSnapshot(2, LANGUAGE_VERSION, CATALOG, 7, tables); + } + + private static HogQlSemanticCatalogSnapshot semanticSnapshot( + List expressionFields, + List functions) + { + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + LANGUAGE_VERSION, + CATALOG, + 7, + List.of(table("events")), + expressionFields, + List.of(), + List.of(), + List.of(), + functions, + List.of()); + } + + private static LiteralRecipe literal() + { + return new LiteralRecipe(new TypedLiteral("bigint", LiteralEncoding.INTEGER, "1")); + } + + private static FunctionCapabilityDefinition function(String name) + { + return new FunctionCapabilityDefinition( + name, + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of(new PhysicalIdentifier(name, false)), + List.of(new FunctionSignature(List.of("bigint"), "bigint", false)), + true, + false, + false, + false, + false); } private static LogicalTableDefinition table(String name) diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java index 472856c12a30..1e5255b5df41 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java @@ -14,8 +14,10 @@ package io.trino.hogql.compiler.catalog; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ModifierBehavior; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder.DecodeFailure; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder.Limits; @@ -44,7 +46,7 @@ public void testDecodesCompleteSnapshot() HogQlSemanticCatalogSnapshot snapshot = new HogQlSemanticCatalogSnapshotJsonDecoder() .decode(bytes(validSnapshotJson()), LoadRequest.latest(CATALOG, LANGUAGE_VERSION)); - assertThat(snapshot.schemaVersion()).isEqualTo(1); + assertThat(snapshot.schemaVersion()).isEqualTo(2); assertThat(snapshot.languageVersion()).isEqualTo(LANGUAGE_VERSION); assertThat(snapshot.catalog()).isEqualTo(CATALOG); assertThat(snapshot.generation()).isEqualTo(7); @@ -71,6 +73,32 @@ public void testDecodesCompleteSnapshot() }); }); }); + assertThat(snapshot.expressionFields()).hasSize(2); + assertThat(snapshot.virtualTables()).singleElement().satisfies(table -> { + assertThat(table.name()).isEqualTo("visible_events"); + assertThat(table.source().kind()).isEqualTo(RelationKind.LOGICAL_TABLE); + assertThat(table.projections()).hasSize(2); + }); + assertThat(snapshot.savedQueries()).singleElement().satisfies(savedQuery -> { + assertThat(savedQuery.queryId()).isEqualTo("query-7"); + assertThat(savedQuery.target().kind()).isEqualTo(RelationKind.VIRTUAL_TABLE); + assertThat(savedQuery.fields()).hasSize(2); + }); + assertThat(snapshot.materializedViews()).singleElement().satisfies(view -> { + assertThat(view.name()).isEqualTo("daily_events"); + assertThat(view.fields()).singleElement().extracting(HogQlSemanticCatalogSnapshot.ReferencedField::name).isEqualTo("day"); + }); + assertThat(snapshot.functions()).singleElement().satisfies(function -> { + assertThat(function.name()).isEqualTo("length"); + assertThat(function.signatures()).singleElement().satisfies(signature -> { + assertThat(signature.argumentTypes()).containsExactly("json"); + assertThat(signature.returnType()).isEqualTo("bigint"); + }); + }); + assertThat(snapshot.modifierDefaults()).singleElement().satisfies(modifier -> { + assertThat(modifier.behavior()).isEqualTo(ModifierBehavior.TRINO_SESSION_PROPERTY); + assertThat(modifier.sessionProperty()).extracting(PhysicalIdentifier::value).containsExactly("hogql", "sampling"); + }); } @ParameterizedTest(name = "{0}") @@ -90,7 +118,7 @@ private static Stream compatibilityFailures() DecodeFailure.UNSUPPORTED_PROTOCOL), Arguments.of( "schema", - validSnapshotJson().replace("\"schemaVersion\": 1", "\"schemaVersion\": 2"), + validSnapshotJson().replace("\"schemaVersion\": 2", "\"schemaVersion\": 1"), LoadRequest.latest(CATALOG, LANGUAGE_VERSION), DecodeFailure.UNSUPPORTED_SCHEMA), Arguments.of( @@ -131,6 +159,20 @@ private static Stream strictFailures() return Stream.of( Arguments.of("unknown root field", validSnapshotJson().replace("\"logicalTables\":", "\"unknown\": true, \"logicalTables\":")), Arguments.of("unknown nested field", validSnapshotJson().replace("\"starVisible\": false", "\"starVisible\": false, \"unknown\": true")), + Arguments.of("unknown recipe field", validSnapshotJson().replace("\"fieldReference\": {", "\"unknown\": true, \"fieldReference\": {")), + Arguments.of("unknown recipe kind", validSnapshotJson().replace("\"kind\": \"FUNCTION_CALL\"", "\"kind\": \"" + SECRET + "\"")), + Arguments.of("mismatched recipe payload", validSnapshotJson().replace("\"functionCall\": {", "\"literal\": {")), + Arguments.of( + "unknown function reference", + validSnapshotJson().replaceFirst("\"name\": \"length\",", "\"name\": \"missing\",")), + Arguments.of("unknown virtual source", validSnapshotJson().replace("\"kind\": \"LOGICAL_TABLE\", \"name\": \"events\"", "\"kind\": \"LOGICAL_TABLE\", \"name\": \"missing\"")), + Arguments.of("saved query target missing field", validSnapshotJson().replace("\"name\": \"browser_length\", \"trinoTypeSignature\": \"bigint\"", "\"name\": \"missing\", \"trinoTypeSignature\": \"bigint\"")), + Arguments.of("invalid literal encoding", validSnapshotJson().replace("\"encoding\": \"INTEGER\", \"value\": \"1\"", "\"encoding\": \"INTEGER\", \"value\": \"" + SECRET + "\"")), + Arguments.of( + "invalid modifier session property", + validSnapshotJson().replace( + "\"sessionProperty\": [", + "\"unknown\": true,\n \"sessionProperty\": [")), Arguments.of("duplicate field", validSnapshotJson().replace("\"protocolVersion\": 1", "\"protocolVersion\": 1, \"protocolVersion\": 1")), Arguments.of("missing field", validSnapshotJson().replace("\"generation\": 7,", "")), Arguments.of("wrong field type", validSnapshotJson().replace("\"nullable\": true", "\"nullable\": \"true\"")), @@ -192,7 +234,7 @@ private static String validSnapshotJson() return """ { "protocolVersion": 1, - "schemaVersion": 1, + "schemaVersion": 2, "languageVersion": "1.0.0", "catalog": {"value": "ducklake", "delimited": false}, "generation": 7, @@ -232,6 +274,108 @@ private static String validSnapshotJson() } ] } + ], + "expressionFields": [ + { + "table": "events", + "name": "browser_length", + "trinoTypeSignature": "bigint", + "logicalType": "INTEGER", + "nullable": true, + "starVisible": false, + "recipe": { + "kind": "FUNCTION_CALL", + "functionCall": { + "name": "length", + "arguments": [ + {"kind": "FIELD_REFERENCE", "fieldReference": {"table": "events", "field": "properties"}} + ] + } + } + }, + { + "table": "events", + "name": "adjusted_count", + "trinoTypeSignature": "bigint", + "logicalType": "INTEGER", + "nullable": false, + "starVisible": false, + "recipe": { + "kind": "CAST", + "cast": { + "targetTypeSignature": "bigint", + "expression": { + "kind": "OPERATOR", + "operator": { + "operator": "ADD", + "arguments": [ + {"kind": "LITERAL", "literal": {"typeSignature": "bigint", "encoding": "INTEGER", "value": "1"}}, + {"kind": "LITERAL", "literal": {"typeSignature": "bigint", "encoding": "INTEGER", "value": "2"}} + ] + } + } + } + } + } + ], + "virtualTables": [ + { + "name": "visible_events", + "source": {"kind": "LOGICAL_TABLE", "name": "events"}, + "projections": [ + {"name": "properties", "sourceField": "properties", "starVisible": true}, + {"name": "browser_length", "sourceField": "browser_length", "starVisible": false} + ] + } + ], + "savedQueries": [ + { + "name": "saved_events", + "queryId": "query-7", + "target": {"kind": "VIRTUAL_TABLE", "name": "visible_events"}, + "fields": [ + {"name": "properties", "trinoTypeSignature": "json", "logicalType": "JSON", "nullable": true, "starVisible": true}, + {"name": "browser_length", "trinoTypeSignature": "bigint", "logicalType": "INTEGER", "nullable": true, "starVisible": false} + ] + } + ], + "materializedViews": [ + { + "name": "daily_events", + "physicalView": { + "catalog": {"value": "ducklake", "delimited": false}, + "schema": {"value": "Analytics", "delimited": true}, + "table": {"value": "daily_events", "delimited": false} + }, + "fields": [ + {"name": "day", "trinoTypeSignature": "date", "logicalType": "DATE", "nullable": false, "starVisible": true} + ] + } + ], + "functions": [ + { + "name": "length", + "kind": "SCALAR", + "implementation": "STOCK", + "trinoName": [{"value": "length", "delimited": false}], + "signatures": [{"argumentTypes": ["json"], "returnType": "bigint", "variadic": false}], + "deterministic": true, + "supportsDistinct": false, + "supportsOrderBy": false, + "supportsFilter": false, + "supportsWindow": false + } + ], + "modifierDefaults": [ + { + "name": "sampling", + "behavior": "TRINO_SESSION_PROPERTY", + "defaultValue": {"typeSignature": "bigint", "encoding": "INTEGER", "value": "1"}, + "sessionProperty": [ + {"value": "hogql", "delimited": false}, + {"value": "sampling", "delimited": false} + ] + } ] } """; diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotLoader.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotLoader.java index 79cec3f77253..04d2df65a820 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotLoader.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotLoader.java @@ -67,11 +67,17 @@ private static String snapshotJson(long generation) return """ { "protocolVersion": 1, - "schemaVersion": 1, + "schemaVersion": 2, "languageVersion": "1.0.0", "catalog": {"value": "ducklake", "delimited": false}, "generation": %s, - "logicalTables": [] + "logicalTables": [], + "expressionFields": [], + "virtualTables": [], + "savedQueries": [], + "materializedViews": [], + "functions": [], + "modifierDefaults": [] } """.formatted(generation); } diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java index 8ec41619ff95..e0ed504dfd23 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java @@ -107,7 +107,7 @@ private static HogQlSemanticCatalogSnapshot snapshot(long generation) private static HogQlSemanticCatalogSnapshot snapshot(PhysicalIdentifier catalog, HogQlLanguageVersion languageVersion, long generation) { return new HogQlSemanticCatalogSnapshot( - 1, + 2, languageVersion, catalog, generation, From bca52ce1a03c933a625ace6c8474f5da568574aa Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 12:10:03 -0400 Subject: [PATCH 044/136] feat(hogql): lower ctes and derived tables --- .../trino/hogql/compiler/HogQlCompiler.java | 54 ++- .../hogql/compiler/HogQlSemanticResolver.java | 52 ++- .../trino/hogql/compiler/TrinoAstFactory.java | 43 ++- .../hogql/compiler/TestHogQlCompiler.java | 51 +++ .../compiler/TestHogQlSemanticResolution.java | 19 + .../io/trino/hogql/parser/HogQlParser.java | 331 ++++++++++++++++-- .../trino/hogql/parser/tree/HogQlQuery.java | 35 ++ .../1.0.0/trino-compatibility-overrides.json | 83 ++++- .../language/1.0.0/trino-compatibility.json | 83 ++--- .../trino/hogql/parser/TestHogQlParser.java | 32 ++ 10 files changed, 681 insertions(+), 102 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index 951dff8f1aee..598a92d96b2f 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -27,6 +27,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.Expression; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; @@ -37,6 +38,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Relation; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; @@ -124,21 +126,10 @@ private static HogQlCompilationResult compile( OptionalLong expectedCatalogGeneration) { validateParameters(query, parameters); - query.from().ifPresent(HogQlCompiler::validateRelation); + validateQuery(query); List placeholders = new ArrayList<>(); - query.projections().forEach(projection -> { - if (projection instanceof HogQlQuery.ExpressionProjection expressionProjection) { - collectPlaceholders(expressionProjection.expression(), placeholders); - } - }); - query.from().ifPresent(relation -> collectPlaceholders(relation, placeholders)); - query.where().ifPresent(expression -> collectPlaceholders(expression, placeholders)); - query.groupBy().forEach(expression -> collectPlaceholders(expression, placeholders)); - query.having().ifPresent(expression -> collectPlaceholders(expression, placeholders)); - query.orderBy().forEach(sortItem -> collectPlaceholders(sortItem.expression(), placeholders)); - query.limit().ifPresent(expression -> collectPlaceholders(expression, placeholders)); - query.offset().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + collectPlaceholders(query, placeholders); placeholders.sort(Comparator.comparingInt(placeholder -> placeholder.span().startOffset())); List missing = placeholders.stream() @@ -177,13 +168,30 @@ private static HogQlCompilationResult compile( .toList(), resolved.catalogGeneration()); } + private static void collectPlaceholders(HogQlQuery query, List placeholders) + { + query.with().forEach(commonTable -> collectPlaceholders(commonTable.query(), placeholders)); + query.projections().forEach(projection -> { + if (projection instanceof HogQlQuery.ExpressionProjection expressionProjection) { + collectPlaceholders(expressionProjection.expression(), placeholders); + } + }); + query.from().ifPresent(relation -> collectPlaceholders(relation, placeholders)); + query.where().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + query.groupBy().forEach(expression -> collectPlaceholders(expression, placeholders)); + query.having().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + query.orderBy().forEach(sortItem -> collectPlaceholders(sortItem.expression(), placeholders)); + query.limit().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + query.offset().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + } + private static ResolvedQuery resolveQuery( HogQlQuery query, Optional catalogContext, HogQlLanguageVersion languageVersion, OptionalLong expectedCatalogGeneration) { - if (query.from().isEmpty() || !containsSemanticCandidate(query.from().orElseThrow()) || catalogContext.isEmpty()) { + if (!containsSemanticCandidate(query) || catalogContext.isEmpty()) { return new ResolvedQuery(query, OptionalLong.empty()); } HogQlSemanticCatalogContext context = catalogContext.orElseThrow(); @@ -197,11 +205,19 @@ private static ResolvedQuery resolveQuery( return new ResolvedQuery(resolved, OptionalLong.of(pinned.generation())); } + private static boolean containsSemanticCandidate(HogQlQuery query) + { + return query.with().stream().anyMatch(commonTable -> containsSemanticCandidate(commonTable.query())) || + query.from().map(HogQlCompiler::containsSemanticCandidate).orElse(false); + } + private static boolean containsSemanticCandidate(Relation relation) { return switch (relation) { case AliasedRelation alias -> containsSemanticCandidate(alias.relation()); + case CommonTableReference _ -> false; case JoinRelation join -> containsSemanticCandidate(join.left()) || containsSemanticCandidate(join.right()); + case SubqueryRelation subquery -> containsSemanticCandidate(subquery.query()); case TablePlaceholder _ -> false; case HogQlQuery.TableReference table -> table.parts().size() == 1; }; @@ -228,6 +244,12 @@ private static void validateParameters(HogQlQuery query, Map validateQuery(commonTable.query())); + query.from().ifPresent(HogQlCompiler::validateRelation); + } + private static void collectPlaceholders(Expression expression, List placeholders) { switch (expression) { @@ -272,10 +294,12 @@ private static void validateRelation(Relation relation) { switch (relation) { case AliasedRelation alias -> validateRelation(alias.relation()); + case CommonTableReference _ -> {} case JoinRelation join -> { validateRelation(join.left()); validateRelation(join.right()); } + case SubqueryRelation subquery -> validateQuery(subquery.query()); case TablePlaceholder tablePlaceholder -> throw bindingError( tablePlaceholder.span(), "HogQL parameter placeholders are not supported in table positions: " + tablePlaceholder.placeholder().name()); @@ -287,6 +311,7 @@ private static void collectPlaceholders(Relation relation, List pla { switch (relation) { case AliasedRelation alias -> collectPlaceholders(alias.relation(), placeholders); + case CommonTableReference _ -> {} case JoinRelation join -> { collectPlaceholders(join.left(), placeholders); collectPlaceholders(join.right(), placeholders); @@ -296,6 +321,7 @@ private static void collectPlaceholders(Relation relation, List pla } }); } + case SubqueryRelation subquery -> collectPlaceholders(subquery.query(), placeholders); case TablePlaceholder _ -> {} case HogQlQuery.TableReference _ -> {} } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index 5f731457e3f6..039eaa74761c 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -26,6 +26,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.CaseWhen; import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.Expression; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; @@ -41,6 +43,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.Relation; import io.trino.hogql.parser.tree.HogQlQuery.SortItem; import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; @@ -73,24 +76,46 @@ public static Optional resolve(PinnedSnapshot snapshot, HogQlQuer { requireNonNull(snapshot, "snapshot is null"); requireNonNull(query, "query is null"); - if (query.from().isEmpty()) { - return Optional.empty(); - } HogQlSemanticResolver resolver = new HogQlSemanticResolver(snapshot); - ResolvedRelation relation = resolver.resolveRelation(query.from().orElseThrow()); - if (relation.bindings().isEmpty()) { - return Optional.empty(); + HogQlQuery resolved = resolver.resolveNestedQuery(query); + return resolved.equals(query) ? Optional.empty() : Optional.of(new ResolvedQuery(resolved)); + } + + private HogQlQuery resolveNestedQuery(HogQlQuery query) + { + List commonTables = query.with().stream() + .map(commonTable -> new CommonTableExpression( + commonTable.name(), + commonTable.columnAliases(), + new HogQlSemanticResolver(snapshot).resolveNestedQuery(commonTable.query()), + commonTable.span())) + .toList(); + Optional relation = query.from().map(this::resolveRelation); + bindings = relation.map(ResolvedRelation::bindings).orElse(List.of()); + allRelationsLogical = relation.map(ResolvedRelation::allLogical).orElse(false); + if (bindings.isEmpty()) { + return new HogQlQuery( + commonTables, + query.distinct(), + query.projections(), + relation.map(ResolvedRelation::relation), + query.where(), + query.groupBy(), + query.having(), + query.orderBy(), + query.limit(), + query.offset(), + query.span()); } - resolver.bindings = relation.bindings(); - resolver.allRelationsLogical = relation.allLogical(); - return Optional.of(resolver.resolveQuery(query, relation.relation())); + return resolveQuery(query, commonTables, relation.orElseThrow().relation()); } - private ResolvedQuery resolveQuery(HogQlQuery query, Relation relation) + private HogQlQuery resolveQuery(HogQlQuery query, List commonTables, Relation relation) { List projections = new ArrayList<>(); query.projections().forEach(projection -> projections.addAll(resolveProjection(projection))); HogQlQuery resolved = new HogQlQuery( + commonTables, query.distinct(), projections, Optional.of(relation), @@ -101,7 +126,7 @@ private ResolvedQuery resolveQuery(HogQlQuery query, Relation relation) query.limit().map(this::resolveExpression), query.offset().map(this::resolveExpression), query.span()); - return new ResolvedQuery(resolved); + return resolved; } private List resolveProjection(Projection projection) @@ -144,6 +169,7 @@ yield new ResolvedRelation( aliasedBindings, child.allLogical()); } + case CommonTableReference commonTable -> new ResolvedRelation(commonTable, List.of(), false); case JoinRelation join -> { ResolvedRelation left = resolveRelation(join.left()); ResolvedRelation right = resolveRelation(join.right()); @@ -164,6 +190,10 @@ yield new ResolvedRelation( joinBindings, left.allLogical() && right.allLogical()); } + case SubqueryRelation subquery -> new ResolvedRelation( + new SubqueryRelation(new HogQlSemanticResolver(snapshot).resolveNestedQuery(subquery.query()), subquery.span()), + List.of(), + false); case TablePlaceholder placeholder -> new ResolvedRelation(placeholder, List.of(), false); case TableReference table -> resolveTable(table); }; diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index cc9d3a323429..72af5139bfda 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -21,6 +21,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.Identifier; @@ -35,6 +37,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.Relation; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; @@ -80,7 +83,10 @@ import io.trino.sql.tree.Statement; import io.trino.sql.tree.StringLiteral; import io.trino.sql.tree.Table; +import io.trino.sql.tree.TableSubquery; import io.trino.sql.tree.WhenClause; +import io.trino.sql.tree.With; +import io.trino.sql.tree.WithQuery; import java.util.List; import java.util.Map; @@ -91,6 +97,11 @@ final class TrinoAstFactory private TrinoAstFactory() {} public static Statement createStatement(HogQlQuery query, Map parameterIds) + { + return createQuery(query, parameterIds); + } + + private static Query createQuery(HogQlQuery query, Map parameterIds) { NodeLocation location = location(query.span()); QuerySpecification querySpecification = new QuerySpecification( @@ -110,13 +121,39 @@ public static Statement createStatement(HogQlQuery query, Map createWith(HogQlQuery query, Map parameterIds) + { + if (query.with().isEmpty()) { + return Optional.empty(); + } + return Optional.of(new With( + location(query.span()), + false, + query.with().stream() + .map(commonTable -> createWithQuery(commonTable, parameterIds)) + .toList())); + } + + private static WithQuery createWithQuery(CommonTableExpression commonTable, Map parameterIds) + { + return new WithQuery( + location(commonTable.span()), + createIdentifier(commonTable.name()), + createQuery(commonTable.query(), parameterIds), + commonTable.columnAliases().isEmpty() + ? Optional.empty() + : Optional.of(commonTable.columnAliases().stream() + .map(TrinoAstFactory::createIdentifier) + .toList())); + } + private static Optional createOrderBy(HogQlQuery query, Map parameterIds) { return createOrderBy(query.orderBy(), parameterIds); @@ -330,6 +367,9 @@ private static io.trino.sql.tree.Relation createRelation(Relation relation, Map< createRelation(alias.relation(), parameterIds), createIdentifier(alias.alias()), null); + case CommonTableReference commonTable -> new Table( + location(commonTable.span()), + QualifiedName.of(List.of(createIdentifier(commonTable.name())))); case JoinRelation join -> new io.trino.sql.tree.Join( location(join.span()), switch (join.type()) { @@ -347,6 +387,7 @@ private static io.trino.sql.tree.Relation createRelation(Relation relation, Map< .map(TrinoAstFactory::createIdentifier) .toList()); })); + case SubqueryRelation subquery -> new TableSubquery(location(subquery.span()), createQuery(subquery.query(), parameterIds)); case TablePlaceholder _ -> throw new IllegalArgumentException("table placeholder was not validated"); case TableReference table -> createTable(table); }; diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index 5daf0df52677..a973b2e65470 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -32,6 +32,8 @@ import io.trino.sql.tree.SingleColumn; import io.trino.sql.tree.Statement; import io.trino.sql.tree.Table; +import io.trino.sql.tree.TableSubquery; +import io.trino.sql.tree.WithQuery; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -181,6 +183,55 @@ public void testLowersUnparenthesizedUsing() .isEqualTo(sqlParser.createStatement("SELECT * FROM events JOIN persons USING (id)")); } + @ParameterizedTest + @ValueSource(strings = { + "WITH base AS (SELECT 1 AS id) SELECT id FROM base", + "WITH base(id) AS (SELECT 1), next AS (SELECT id FROM base) SELECT id FROM next", + "WITH base AS (WITH base AS (SELECT 1 AS id) SELECT id FROM base) SELECT id FROM base", + "SELECT derived.id FROM (SELECT id FROM events) AS derived", + "WITH base AS (SELECT id FROM events) SELECT derived.id FROM (SELECT id FROM base) derived", + }) + public void testLowersCtesAndDerivedTables(String hogql) + { + Statement statement = compiler.compile(hogql); + + assertThat(statement).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @Test + public void testBindsPlaceholdersAcrossCteAndDerivedQueryScopes() + { + HogQlCompilationResult result = compiler.compile( + "WITH base AS (SELECT {cte}) SELECT {outer} FROM (SELECT {derived} FROM base) nested WHERE {where}", + Map.of( + "cte", typedValue("cte"), + "outer", typedValue("outer"), + "derived", typedValue("derived"), + "where", typedValue("where"))); + + assertThat(result.parameterNames()).containsExactly("cte", "outer", "derived", "where"); + assertThat(parameters(result.statement())) + .extracting(Parameter::getId) + .containsExactly(0, 1, 2, 3); + } + + @Test + public void testPreservesCteAndDerivedTableSourceLocations() + { + Query query = (Query) compiler.compile("WITH base AS (SELECT id FROM events)\nSELECT d.id FROM (SELECT id FROM base) AS d"); + WithQuery commonTable = query.getWith().orElseThrow().getQueries().getFirst(); + AliasedRelation aliasedRelation = (AliasedRelation) ((QuerySpecification) query.getQueryBody()).getFrom().orElseThrow(); + TableSubquery derivedTable = (TableSubquery) aliasedRelation.getRelation(); + + assertThat(query.getWith().orElseThrow().getLocation()).contains(new NodeLocation(1, 1)); + assertThat(commonTable.getLocation()).contains(new NodeLocation(1, 6)); + assertThat(commonTable.getQuery().getLocation()).contains(new NodeLocation(1, 15)); + assertThat(aliasedRelation.getLocation()).contains(new NodeLocation(2, 18)); + assertThat(derivedTable.getLocation()).contains(new NodeLocation(2, 18)); + assertThat(derivedTable.getQuery().getLocation()).contains(new NodeLocation(2, 19)); + } + @Test public void testPreservesAliasedJoinSourceLocations() { diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java index 3f3a1fe7639a..17e64f2efe9e 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -129,6 +129,25 @@ public void testCatalogIndependentAndPhysicalQueriesDoNotFetchSemanticMetadata() assertThat(physical.catalogGeneration()).isEmpty(); } + @Test + public void testCteScopeShadowsLogicalCatalogNamesWithoutFetchingMetadata() + { + AtomicInteger pins = new AtomicInteger(); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + }); + + HogQlCompilationResult result = compiler.compile( + envelope("WITH events AS (SELECT 1 AS event), next AS (SELECT event FROM events) SELECT event FROM next", OptionalLong.empty()), + Optional.of(context)); + + assertThat(pins).hasValue(0); + assertThat(result.catalogGeneration()).isEmpty(); + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "WITH events AS (SELECT 1 AS event), next AS (SELECT event FROM events) SELECT event FROM next")); + } + @Test public void testUnknownLogicalFieldFailsAtOriginalLocation() { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index fbfc210a4aa8..0a399654ceee 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -25,6 +25,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.CaseWhen; import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.Expression; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; @@ -45,6 +47,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.SortItem; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; @@ -70,9 +73,15 @@ import org.antlr.v4.runtime.tree.ParseTreeListener; import org.antlr.v4.runtime.tree.TerminalNode; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Deque; +import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Optional; +import java.util.Set; import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.ADD; import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.AND; @@ -441,6 +450,8 @@ private SourceSpan sourceSpan(Token start, Token stop) private static final class AstBuilder { private final SourcePositions sourcePositions; + private final Deque> commonTableScopes = new ArrayDeque<>(); + private final Deque> prohibitedCommonTableScopes = new ArrayDeque<>(); private AstBuilder(String source) { @@ -449,39 +460,100 @@ private AstBuilder(String source) public HogQlQuery build(HogQLParser.SelectContext context) { - SelectedQuery selectedQuery = extractPlainSelect(context); + HogQlQuery query = buildQuery(extractPlainSelect(context)); + validateUncorrelatedSubqueries(query); + return query; + } + + private HogQlQuery buildQuery(SelectedQuery selectedQuery) + { HogQLParser.SelectStmtContext select = selectedQuery.statement(); rejectUnsupportedClauses(select); - - List projections = selectColumns(select.selectColumnExprListBeforeFrom()).stream() - .map(this::buildProjection) - .toList(); - Optional from = Optional.ofNullable(select.fromClause()) - .map(HogQLParser.FromClauseContext::joinExpr) - .map(this::buildRelation); - Optional where = Optional.ofNullable(select.whereClause()) - .map(HogQLParser.WhereClauseContext::columnExpr) - .map(this::buildExpression); - List groupBy = buildGroupBy(select); - Optional having = Optional.ofNullable(select.havingClause()) - .map(HogQLParser.HavingClauseContext::columnExpr) - .map(this::buildExpression); - List orderBy = buildOrderBy(selectedQuery.orderBy() != null ? selectedQuery.orderBy() : select.orderByClause()); - Pagination pagination = mergePagination( - buildPagination(select.limitAndOffsetClause(), select.offsetOnlyClause()), - selectedQuery.pagination() == null ? new Pagination(Optional.empty(), Optional.empty()) : buildPagination(selectedQuery.pagination()), - select); - return new HogQlQuery( - select.DISTINCT() != null, - projections, - from, - where, - groupBy, - having, - orderBy, - pagination.limit(), - pagination.offset(), - sourceSpan(select)); + commonTableScopes.push(new LinkedHashSet<>()); + prohibitedCommonTableScopes.push(new LinkedHashSet<>()); + try { + List with = buildCommonTables(select.withClause()); + + List projections = selectColumns(select.selectColumnExprListBeforeFrom()).stream() + .map(this::buildProjection) + .toList(); + Optional from = Optional.ofNullable(select.fromClause()) + .map(HogQLParser.FromClauseContext::joinExpr) + .map(this::buildRelation); + Optional where = Optional.ofNullable(select.whereClause()) + .map(HogQLParser.WhereClauseContext::columnExpr) + .map(this::buildExpression); + List groupBy = buildGroupBy(select); + Optional having = Optional.ofNullable(select.havingClause()) + .map(HogQLParser.HavingClauseContext::columnExpr) + .map(this::buildExpression); + List orderBy = buildOrderBy(selectedQuery.orderBy() != null ? selectedQuery.orderBy() : select.orderByClause()); + Pagination pagination = mergePagination( + buildPagination(select.limitAndOffsetClause(), select.offsetOnlyClause()), + selectedQuery.pagination() == null ? new Pagination(Optional.empty(), Optional.empty()) : buildPagination(selectedQuery.pagination()), + select); + return new HogQlQuery( + with, + select.DISTINCT() != null, + projections, + from, + where, + groupBy, + having, + orderBy, + pagination.limit(), + pagination.offset(), + sourceSpan(select)); + } + finally { + prohibitedCommonTableScopes.pop(); + commonTableScopes.pop(); + } + } + + private List buildCommonTables(HogQLParser.WithClauseContext context) + { + if (context == null) { + return List.of(); + } + if (context.RECURSIVE() != null) { + throw unsupported(context, "recursive CTE"); + } + List commonTables = new ArrayList<>(); + Set localNames = commonTableScopes.getFirst(); + for (HogQLParser.WithExprContext expression : context.withExprList().withExpr()) { + if (!(expression instanceof HogQLParser.WithExprSubqueryContext subquery)) { + throw unsupported(expression, "non-query WITH expression"); + } + if (subquery.USING() != null) { + throw unsupported(subquery, "CTE USING KEY"); + } + if (subquery.MATERIALIZED() != null) { + throw unsupported(subquery, "materialized CTE"); + } + Identifier name = buildIdentifier(subquery.identifier()); + String canonicalName = canonicalName(name); + if (localNames.contains(canonicalName)) { + throw unsupported(subquery, "duplicate CTE name"); + } + Set prohibitedNames = prohibitedCommonTableScopes.getFirst(); + prohibitedNames.add(canonicalName); + HogQlQuery query; + try { + query = buildQuery(extractPlainSelect(subquery.selectSetStmt())); + } + finally { + prohibitedNames.remove(canonicalName); + } + List columnAliases = subquery.withExprColumnNameList().isEmpty() + ? List.of() + : subquery.withExprColumnNameList().getFirst().identifier().stream() + .map(this::buildIdentifier) + .toList(); + commonTables.add(new CommonTableExpression(name, columnAliases, query, sourceSpan(subquery))); + localNames.add(canonicalName); + } + return List.copyOf(commonTables); } private List selectColumns(HogQLParser.SelectColumnExprListBeforeFromContext context) @@ -511,6 +583,24 @@ private SelectedQuery extractPlainSelect(HogQLParser.SelectContext context) return new SelectedQuery(wrapped.selectStmt(), set.orderByClause(), set.limitAndOffsetClauseOptional()); } + private SelectedQuery extractPlainSelect(HogQLParser.SelectSetStmtContext context) + { + if (!context.subsequentSelectSetClause().isEmpty()) { + throw unsupported(context, "set query"); + } + HogQLParser.SelectStmtWithParensContext wrapped = context.selectStmtWithParens(); + if (wrapped.selectStmt() != null) { + return new SelectedQuery(wrapped.selectStmt(), context.orderByClause(), context.limitAndOffsetClauseOptional()); + } + if (wrapped.withClause() != null) { + throw unsupported(wrapped, "non-standard WITH query wrapper"); + } + if (wrapped.selectSetStmt() != null && context.orderByClause() == null && context.limitAndOffsetClauseOptional() == null) { + return extractPlainSelect(wrapped.selectSetStmt()); + } + throw unsupported(wrapped, "parenthesized, placeholder, or decorated query"); + } + private record SelectedQuery( HogQLParser.SelectStmtContext statement, HogQLParser.OrderByClauseContext orderBy, @@ -519,7 +609,6 @@ private record SelectedQuery( private void rejectUnsupportedClauses(HogQLParser.SelectStmtContext context) { List clauses = new ArrayList<>(); - clauses.add(context.withClause()); clauses.add(context.topClause()); clauses.add(context.arrayJoinClause()); clauses.add(context.prewhereClause()); @@ -991,11 +1080,25 @@ private Relation buildRelation(HogQLParser.JoinExprContext context) private Relation buildTableExpression(HogQLParser.TableExprContext context) { if (context instanceof HogQLParser.TableExprIdentifierContext identifier) { - return new TableReference(buildIdentifiers(identifier.tableIdentifier()), sourceSpan(context)); + List parts = buildIdentifiers(identifier.tableIdentifier()); + if (parts.size() == 1) { + String name = canonicalName(parts.getFirst()); + switch (resolveCommonTableName(name)) { + case PROHIBITED -> throw unsupported(context, "recursive CTE reference"); + case VISIBLE -> { + return new CommonTableReference(parts.getFirst(), sourceSpan(context)); + } + case ABSENT -> {} + } + } + return new TableReference(parts, sourceSpan(context)); } if (context instanceof HogQLParser.TableExprPlaceholderContext placeholder) { return new TablePlaceholder(buildPlaceholder(placeholder.placeholder())); } + if (context instanceof HogQLParser.TableExprSubqueryContext subquery) { + return new SubqueryRelation(buildQuery(extractPlainSelect(subquery.selectSetStmt())), sourceSpan(context)); + } if (context instanceof HogQLParser.TableExprAliasContext alias) { if (alias.columnAliases() != null) { throw unsupported(context, "table column aliases"); @@ -1008,6 +1111,162 @@ private Relation buildTableExpression(HogQLParser.TableExprContext context) throw unsupported(context, "table expression"); } + private CommonTableNameResolution resolveCommonTableName(String name) + { + Iterator> visibleScopes = commonTableScopes.iterator(); + Iterator> prohibitedScopes = prohibitedCommonTableScopes.iterator(); + while (visibleScopes.hasNext()) { + Set visibleNames = visibleScopes.next(); + Set prohibitedNames = prohibitedScopes.next(); + if (prohibitedNames.contains(name)) { + return CommonTableNameResolution.PROHIBITED; + } + if (visibleNames.contains(name)) { + return CommonTableNameResolution.VISIBLE; + } + } + return CommonTableNameResolution.ABSENT; + } + + private enum CommonTableNameResolution + { + ABSENT, + PROHIBITED, + VISIBLE, + } + + private static String canonicalName(Identifier identifier) + { + return identifier.delimited() ? identifier.value() : identifier.value().toLowerCase(Locale.ENGLISH); + } + + private void validateUncorrelatedSubqueries(HogQlQuery query) + { + validateQueryScope(query, Set.of()); + } + + private void validateQueryScope(HogQlQuery query, Set forbiddenOuterRelations) + { + Set localRelations = relationNames(query.from()); + query.projections().forEach(projection -> { + if (projection instanceof ExpressionProjection expression) { + validateExpressionScope(expression.expression(), forbiddenOuterRelations, localRelations); + } + }); + query.where().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); + query.groupBy().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); + query.having().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); + query.orderBy().forEach(item -> validateExpressionScope(item.expression(), forbiddenOuterRelations, localRelations)); + query.limit().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); + query.offset().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); + query.from().ifPresent(relation -> validateRelationExpressionScopes(relation, forbiddenOuterRelations, localRelations)); + + query.with().forEach(commonTable -> validateQueryScope(commonTable.query(), forbiddenOuterRelations)); + Set nestedForbiddenRelations = new LinkedHashSet<>(forbiddenOuterRelations); + nestedForbiddenRelations.addAll(localRelations); + query.from().ifPresent(relation -> validateNestedRelationScopes(relation, Set.copyOf(nestedForbiddenRelations))); + } + + private void validateRelationExpressionScopes(Relation relation, Set forbiddenOuterRelations, Set localRelations) + { + switch (relation) { + case AliasedRelation alias -> validateRelationExpressionScopes(alias.relation(), forbiddenOuterRelations, localRelations); + case JoinRelation join -> { + validateRelationExpressionScopes(join.left(), forbiddenOuterRelations, localRelations); + validateRelationExpressionScopes(join.right(), forbiddenOuterRelations, localRelations); + join.criteria().ifPresent(criteria -> { + if (criteria instanceof JoinOn on) { + validateExpressionScope(on.expression(), forbiddenOuterRelations, localRelations); + } + }); + } + case CommonTableReference _, SubqueryRelation _, TablePlaceholder _, TableReference _ -> {} + } + } + + private void validateNestedRelationScopes(Relation relation, Set forbiddenOuterRelations) + { + switch (relation) { + case AliasedRelation alias -> validateNestedRelationScopes(alias.relation(), forbiddenOuterRelations); + case JoinRelation join -> { + validateNestedRelationScopes(join.left(), forbiddenOuterRelations); + validateNestedRelationScopes(join.right(), forbiddenOuterRelations); + } + case SubqueryRelation subquery -> validateQueryScope(subquery.query(), forbiddenOuterRelations); + case CommonTableReference _, TablePlaceholder _, TableReference _ -> {} + } + } + + private Set relationNames(Optional relation) + { + Set names = new LinkedHashSet<>(); + relation.ifPresent(value -> collectRelationNames(value, names)); + return Set.copyOf(names); + } + + private void collectRelationNames(Relation relation, Set names) + { + switch (relation) { + case AliasedRelation alias -> names.add(canonicalName(alias.alias())); + case CommonTableReference commonTable -> names.add(canonicalName(commonTable.name())); + case JoinRelation join -> { + collectRelationNames(join.left(), names); + collectRelationNames(join.right(), names); + } + case SubqueryRelation _, TablePlaceholder _ -> {} + case TableReference table -> names.add(canonicalName(table.parts().getLast())); + } + } + + private void validateExpressionScope(Expression expression, Set forbiddenOuterRelations, Set localRelations) + { + switch (expression) { + case ArrayExpression array -> array.values().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); + case BetweenExpression between -> { + validateExpressionScope(between.value(), forbiddenOuterRelations, localRelations); + validateExpressionScope(between.min(), forbiddenOuterRelations, localRelations); + validateExpressionScope(between.max(), forbiddenOuterRelations, localRelations); + } + case BinaryExpression binary -> { + validateExpressionScope(binary.left(), forbiddenOuterRelations, localRelations); + validateExpressionScope(binary.right(), forbiddenOuterRelations, localRelations); + } + case CaseExpression caseExpression -> { + caseExpression.operand().ifPresent(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); + caseExpression.whenClauses().forEach(when -> { + validateExpressionScope(when.operand(), forbiddenOuterRelations, localRelations); + validateExpressionScope(when.result(), forbiddenOuterRelations, localRelations); + }); + caseExpression.defaultValue().ifPresent(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); + } + case CastExpression cast -> validateExpressionScope(cast.value(), forbiddenOuterRelations, localRelations); + case ColumnReference reference -> { + if (reference.parts().size() == 1 && !forbiddenOuterRelations.isEmpty() && localRelations.isEmpty()) { + throw unsupported(reference.span(), "correlated relation subquery"); + } + if (reference.parts().size() > 1) { + String qualifier = canonicalName(reference.parts().getFirst()); + if (forbiddenOuterRelations.contains(qualifier) && !localRelations.contains(qualifier)) { + throw unsupported(reference.span(), "correlated relation subquery"); + } + } + } + case FunctionCall function -> { + function.arguments().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); + function.orderBy().forEach(item -> validateExpressionScope(item.expression(), forbiddenOuterRelations, localRelations)); + function.filter().ifPresent(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); + } + case InExpression in -> { + validateExpressionScope(in.value(), forbiddenOuterRelations, localRelations); + in.values().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); + } + case IsNullExpression isNull -> validateExpressionScope(isNull.value(), forbiddenOuterRelations, localRelations); + case Literal _, Placeholder _ -> {} + case TupleExpression tuple -> tuple.values().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); + case UnaryExpression unary -> validateExpressionScope(unary.operand(), forbiddenOuterRelations, localRelations); + } + } + private JoinType buildJoinType(HogQLParser.JoinOpContext context) { if (context == null) { @@ -1103,7 +1362,11 @@ private SourceSpan sourceSpan(Token start, Token stop) private HogQlParsingException unsupported(ParserRuleContext context, String feature) { - SourceSpan span = sourceSpan(context); + return unsupported(sourceSpan(context), feature); + } + + private HogQlParsingException unsupported(SourceSpan span, String feature) + { return new HogQlParsingException("HogQL feature is not lowered yet: " + feature, null, span.startLine(), span.startColumn()); } diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index 443c9e811b56..13d73f02c28a 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -19,6 +19,7 @@ import static java.util.Objects.requireNonNull; public record HogQlQuery( + List with, boolean distinct, List projections, Optional from, @@ -32,6 +33,7 @@ public record HogQlQuery( { public HogQlQuery { + with = List.copyOf(requireNonNull(with, "with is null")); projections = List.copyOf(requireNonNull(projections, "projections is null")); from = requireNonNull(from, "from is null"); where = requireNonNull(where, "where is null"); @@ -43,6 +45,17 @@ public record HogQlQuery( span = requireNonNull(span, "span is null"); } + public record CommonTableExpression(Identifier name, List columnAliases, HogQlQuery query, SourceSpan span) + { + public CommonTableExpression + { + name = requireNonNull(name, "name is null"); + columnAliases = List.copyOf(requireNonNull(columnAliases, "columnAliases is null")); + query = requireNonNull(query, "query is null"); + span = requireNonNull(span, "span is null"); + } + } + public record SortItem(Expression expression, SortDirection direction, NullPlacement nullPlacement, SourceSpan span) { public SortItem @@ -327,7 +340,9 @@ public enum LiteralKind public sealed interface Relation permits AliasedRelation, + CommonTableReference, JoinRelation, + SubqueryRelation, TablePlaceholder, TableReference { @@ -345,6 +360,26 @@ public record AliasedRelation(Relation relation, Identifier alias, SourceSpan sp } } + public record CommonTableReference(Identifier name, SourceSpan span) + implements Relation + { + public CommonTableReference + { + name = requireNonNull(name, "name is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record SubqueryRelation(HogQlQuery query, SourceSpan span) + implements Relation + { + public SubqueryRelation + { + query = requireNonNull(query, "query is null"); + span = requireNonNull(span, "span is null"); + } + } + public enum JoinType { CROSS, diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json index 61893bdff17a..bea2b9f5ab1a 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json @@ -453,12 +453,29 @@ "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" ] }, + { + "id": "alternative:selectStmtWithParens:direct", + "loweringHandler": "TrinoAstFactory#createQuery", + "parseHandler": "AstBuilder#extractPlainSelect", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:selectStmtWithParens:parenthesized", + "loweringHandler": "TrinoAstFactory#createQuery", + "parseHandler": "AstBuilder#extractPlainSelect", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, { "id": "alternative:tableExpr:TableExprAlias", "loweringHandler": "TrinoAstFactory#createRelation", "parseHandler": "AstBuilder#buildTableExpression", "testCaseIds": [ - "TestHogQlCompiler#testLowersAliasesAndStockJoins" + "TestHogQlCompiler#testLowersAliasesAndStockJoins", + "TestHogQlCompiler#testLowersCtesAndDerivedTables" ] }, { @@ -469,6 +486,14 @@ "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" ] }, + { + "id": "alternative:tableExpr:TableExprSubquery", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, { "id": "alternative:tableIdentifier", "loweringHandler": "TrinoAstFactory#createIdentifier", @@ -485,6 +510,38 @@ "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause" ] }, + { + "id": "alternative:withClause", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:withExpr:WithExprSubquery", + "loweringHandler": "TrinoAstFactory#createWithQuery", + "parseHandler": "AstBuilder#buildCommonTables", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:withExprColumnNameList", + "loweringHandler": "TrinoAstFactory#createWithQuery", + "parseHandler": "AstBuilder#buildCommonTables", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:withExprList", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, { "id": "rule:alias", "loweringHandler": "TrinoAstFactory#createRelation", @@ -590,6 +647,30 @@ "testCaseIds": [ "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause" ] + }, + { + "id": "rule:withClause", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "rule:withExprColumnNameList", + "loweringHandler": "TrinoAstFactory#createWithQuery", + "parseHandler": "AstBuilder#buildCommonTables", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "rule:withExprList", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] } ], "schemaVersion": 1 diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json index c7fd27d720e5..d358d1d7c7f7 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json @@ -2137,23 +2137,23 @@ { "id": "alternative:selectStmtWithParens:direct", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createQuery", + "parseHandler": "AstBuilder#extractPlainSelect", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersCtesAndDerivedTables" ] }, { "id": "alternative:selectStmtWithParens:parenthesized", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createQuery", + "parseHandler": "AstBuilder#extractPlainSelect", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersCtesAndDerivedTables" ] }, { @@ -2428,7 +2428,8 @@ "queryReachable": true, "status": "supported", "testCaseIds": [ - "TestHogQlCompiler#testLowersAliasesAndStockJoins" + "TestHogQlCompiler#testLowersAliasesAndStockJoins", + "TestHogQlCompiler#testLowersCtesAndDerivedTables" ] }, { @@ -2478,12 +2479,12 @@ { "id": "alternative:tableExpr:TableExprSubquery", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersCtesAndDerivedTables" ] }, { @@ -2764,12 +2765,12 @@ { "id": "alternative:withClause", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersCtesAndDerivedTables" ] }, { @@ -2786,34 +2787,34 @@ { "id": "alternative:withExpr:WithExprSubquery", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWithQuery", + "parseHandler": "AstBuilder#buildCommonTables", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersCtesAndDerivedTables" ] }, { "id": "alternative:withExprColumnNameList", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWithQuery", + "parseHandler": "AstBuilder#buildCommonTables", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersCtesAndDerivedTables" ] }, { "id": "alternative:withExprList", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersCtesAndDerivedTables" ] }, { @@ -4086,12 +4087,12 @@ { "id": "rule:withClause", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersCtesAndDerivedTables" ] }, { @@ -4108,23 +4109,23 @@ { "id": "rule:withExprColumnNameList", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWithQuery", + "parseHandler": "AstBuilder#buildCommonTables", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersCtesAndDerivedTables" ] }, { "id": "rule:withExprList", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersCtesAndDerivedTables" ] }, { diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index cf87b5d7a9e1..78c08445f35d 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -17,6 +17,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; @@ -25,6 +26,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; import io.trino.hogql.parser.tree.HogQlSyntaxTree; import io.trino.hogql.parser.tree.HogQlSyntaxTree.Element; import io.trino.hogql.parser.tree.HogQlSyntaxTree.Node; @@ -59,6 +61,16 @@ public class TestHogQlParser "SELECT * FROM", "SELECT {1 + 2}", "SELECT 1 GROUP BY ALL", + "WITH RECURSIVE x AS (SELECT 1) SELECT * FROM x", + "WITH x AS MATERIALIZED (SELECT 1) SELECT * FROM x", + "WITH x AS NOT MATERIALIZED (SELECT 1) SELECT * FROM x", + "WITH x USING KEY (id) AS (SELECT 1) SELECT * FROM x", + "WITH 1 AS x SELECT x", + "WITH x AS (SELECT * FROM x) SELECT * FROM x", + "WITH x AS (SELECT 1), y AS (WITH x AS (SELECT * FROM x) SELECT * FROM x) SELECT * FROM y", + "WITH x AS (SELECT 1), x AS (SELECT 2) SELECT * FROM x", + "SELECT * FROM events e JOIN (SELECT * FROM persons WHERE personId = e.person_id) p ON true", + "SELECT * FROM events e JOIN (SELECT person_id) p ON true", }) public void testRejectsSyntaxWithoutAnAstMapping(String hogql) { @@ -136,6 +148,26 @@ public void testBuildsAliasedJoinAstWithSourceSpans() assertThat(criteria.span()).isEqualTo(new HogQlQuery.SourceSpan(52, 73, 3, 24, 3, 45)); } + @Test + public void testBuildsCteAndDerivedTableAstWithSourceSpans() + { + String hogql = "WITH base(id) AS (SELECT {cte}), next AS (SELECT id FROM base)\n" + + "SELECT d.id FROM (SELECT id FROM next WHERE id = {inner}) AS d WHERE d.id = {outer}"; + + HogQlQuery query = parser.parseStatement(hogql); + + assertThat(query.with()).hasSize(2); + assertThat(query.with().getFirst().name().value()).isEqualTo("base"); + assertThat(query.with().getFirst().columnAliases()).singleElement().satisfies(alias -> assertThat(alias.value()).isEqualTo("id")); + assertThat(query.with().getFirst().span()).isEqualTo(new HogQlQuery.SourceSpan(5, 31, 1, 6, 1, 32)); + assertThat(query.with().get(1).query().from()).get().isInstanceOf(CommonTableReference.class); + AliasedRelation derived = (AliasedRelation) query.from().orElseThrow(); + assertThat(derived.alias().value()).isEqualTo("d"); + SubqueryRelation subquery = (SubqueryRelation) derived.relation(); + assertThat(subquery.query().from()).get().isInstanceOf(CommonTableReference.class); + assertThat(subquery.span()).isEqualTo(new HogQlQuery.SourceSpan(80, 120, 2, 18, 2, 58)); + } + @Test public void testSourceSpansUseUnicodeCodePointOffsetsAndEndPositions() { From 09bc3485080949328bbfebcc87394384b13831f6 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 12:24:34 -0400 Subject: [PATCH 045/136] test(hogql): harden semantic catalog refresh --- ...syncHogQlSemanticCatalogSnapshotCache.java | 96 +++++++++++++++++++ .../TestHogQlSemanticCatalogManager.java | 77 ++++++++++++++- 2 files changed, 171 insertions(+), 2 deletions(-) diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java index f7c527be7a35..915dee91a580 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java @@ -20,15 +20,20 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinRequest; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; import io.trino.hogql.parser.HogQlLanguageVersion; import org.junit.jupiter.api.Test; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayDeque; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.CountDownLatch; @@ -219,6 +224,77 @@ public void testConcurrentRequestsShareOneRefresh() } } + @Test + public void testPinnedSnapshotRemainsStableAcrossRefreshPublication() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, snapshot(CATALOG, 1)); + HogQlSemanticCatalogSnapshotProvider provider = HogQlSemanticCatalogSnapshotProvider.fromCache(cache); + + CompletableFuture load = loader.expect(CATALOG); + ticker.set(REFRESH_AFTER.toNanos()); + PinnedSnapshot pinned = provider.pin(new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.empty())); + load.complete(snapshot(CATALOG, 2)); + + assertThat(pinned.generation()).isEqualTo(1); + assertThat(pinned.snapshot().generation()).isEqualTo(1); + assertThat(provider.pin(new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.of(2))).generation()).isEqualTo(2); + } + + @Test + public void testOrdinaryAndDelimitedCatalogIdentifiersHaveIsolatedEntries() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + PhysicalIdentifier ordinary = new PhysicalIdentifier("sales", false); + PhysicalIdentifier delimited = new PhysicalIdentifier("sales", true); + + completePrewarm(cache, loader, snapshot(ordinary, 1)); + completePrewarm(cache, loader, snapshot(delimited, 2)); + + assertThat(cache.currentSnapshot(ordinary)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + assertThat(cache.currentSnapshot(delimited)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(2L); + cache.invalidate(ordinary); + assertThat(cache.currentSnapshot(delimited)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(2L); + } + + @Test + public void testMalformedSchemaV2RefreshRetainsLastKnownGoodSnapshot() + { + AtomicLong ticker = new AtomicLong(); + ArrayDeque> payloads = new ArrayDeque<>(); + HogQlSemanticCatalogSnapshotLoader jsonLoader = HogQlSemanticCatalogSnapshotLoader.fromJsonTransport( + _ -> payloads.remove(), + new HogQlSemanticCatalogSnapshotJsonDecoder()); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache( + 2, + ticker, + catalog -> jsonLoader.load(LoadRequest.latest(catalog, LANGUAGE_VERSION))); + + CompletableFuture initialPayload = new CompletableFuture<>(); + payloads.add(initialPayload); + CompletionStage initialRefresh = cache.prewarm(CATALOG); + initialPayload.complete(snapshotJson(1).getBytes(StandardCharsets.UTF_8)); + assertThat(initialRefresh.toCompletableFuture()).isCompletedWithValueMatching(snapshot -> snapshot.generation() == 1); + + CompletableFuture malformedPayload = new CompletableFuture<>(); + payloads.add(malformedPayload); + ticker.set(REFRESH_AFTER.toNanos()); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + CompletionStage malformedRefresh = cache.prewarm(CATALOG); + malformedPayload.complete(snapshotJson(2) + .replace("\"logicalTables\": []", "\"logicalTables\": {}") + .getBytes(StandardCharsets.UTF_8)); + + assertThatThrownBy(malformedRefresh.toCompletableFuture()::join) + .cause() + .isInstanceOf(HogQlSemanticCatalogSnapshotJsonDecoder.DecodeException.class); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + } + private static BoundedAsyncHogQlSemanticCatalogSnapshotCache cache(int maximumEntries, AtomicLong ticker, SnapshotLoader loader) { return new BoundedAsyncHogQlSemanticCatalogSnapshotCache( @@ -286,6 +362,26 @@ private static HogQlSemanticCatalogSnapshot snapshot(PhysicalIdentifier catalog, List.of()))); } + private static String snapshotJson(long generation) + { + return """ + { + "protocolVersion": 1, + "schemaVersion": 2, + "languageVersion": "1.0.0", + "catalog": {"value": "ducklake", "delimited": false}, + "generation": %s, + "logicalTables": [], + "expressionFields": [], + "virtualTables": [], + "savedQueries": [], + "materializedViews": [], + "functions": [], + "modifierDefaults": [] + } + """.formatted(generation); + } + private static final class ControlledLoader implements SnapshotLoader { diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java index 3d5ebd9ec727..3a3dd480b2cc 100644 --- a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java @@ -24,11 +24,14 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TestHogQlSemanticCatalogManager { @@ -120,7 +123,72 @@ public void testCatalogLifecyclePrewarmsWithoutWaitingForMetadata() assertThat(manager.cache().currentSnapshot(unusualCatalog)).isEmpty(); } finally { - metadata.complete(new HogQlSemanticCatalogSnapshot(1, LANGUAGE_VERSION, unusualCatalog, 1, List.of())); + metadata.complete(new HogQlSemanticCatalogSnapshot(2, LANGUAGE_VERSION, unusualCatalog, 1, List.of())); + manager.shutdown(); + } + } + + @Test + public void testRejectedLoaderWorkIsBackedOffBeforeRetry() + { + AtomicLong ticker = new AtomicLong(); + PhysicalIdentifier runningCatalog = new PhysicalIdentifier("running", false); + PhysicalIdentifier queuedCatalog = new PhysicalIdentifier("queued", false); + PhysicalIdentifier rejectedCatalog = new PhysicalIdentifier("rejected", false); + CompletableFuture runningStarted = new CompletableFuture<>(); + CompletableFuture releaseRunning = new CompletableFuture<>(); + CompletableFuture rejectedStarted = new CompletableFuture<>(); + CompletableFuture rejectedLoad = new CompletableFuture<>(); + AtomicInteger rejectedLoads = new AtomicInteger(); + HogQlSemanticCatalogSnapshotLoader loader = request -> switch (request.catalog().value()) { + case "running" -> { + runningStarted.complete(null); + releaseRunning.join(); + yield CompletableFuture.completedFuture(snapshot(runningCatalog, 1)); + } + case "queued" -> CompletableFuture.completedFuture(snapshot(queuedCatalog, 1)); + case "rejected" -> { + rejectedLoads.incrementAndGet(); + rejectedStarted.complete(null); + yield rejectedLoad; + } + default -> throw new IllegalStateException("unexpected catalog load"); + }; + HogQlSemanticCatalogConfig config = config() + .setMaximumEntries(4) + .setFailureBackoff(new Duration(10, SECONDS)) + .setLoaderQueueCapacity(1); + HogQlSemanticCatalogManager manager = new HogQlSemanticCatalogManager(config, loader, LANGUAGE_VERSION, ticker::get); + try { + CompletionStage runningRefresh = manager.prewarm(runningCatalog); + runningStarted.join(); + CompletionStage queuedRefresh = manager.prewarm(queuedCatalog); + CompletionStage rejectedRefresh = manager.prewarm(rejectedCatalog); + + assertThatThrownBy(rejectedRefresh.toCompletableFuture()::join) + .cause() + .isInstanceOf(RejectedExecutionException.class); + assertThat(rejectedLoads).hasValue(0); + assertThat(manager.cache().currentSnapshot(rejectedCatalog)).isEmpty(); + + releaseRunning.complete(null); + runningRefresh.toCompletableFuture().join(); + queuedRefresh.toCompletableFuture().join(); + + ticker.set(SECONDS.toNanos(10) - 1); + assertThat(manager.cache().currentSnapshot(rejectedCatalog)).isEmpty(); + assertThat(rejectedLoads).hasValue(0); + + ticker.set(SECONDS.toNanos(10)); + assertThat(manager.cache().currentSnapshot(rejectedCatalog)).isEmpty(); + rejectedStarted.join(); + assertThat(rejectedLoads).hasValue(1); + rejectedLoad.complete(snapshot(rejectedCatalog, 1)); + assertThat(manager.cache().currentSnapshot(rejectedCatalog)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + } + finally { + releaseRunning.complete(null); + rejectedLoad.completeExceptionally(new IllegalStateException("test shutdown")); manager.shutdown(); } } @@ -138,6 +206,11 @@ private static HogQlSemanticCatalogConfig config() private static HogQlSemanticCatalogSnapshot snapshot(long generation) { - return new HogQlSemanticCatalogSnapshot(1, LANGUAGE_VERSION, CATALOG, generation, List.of()); + return snapshot(CATALOG, generation); + } + + private static HogQlSemanticCatalogSnapshot snapshot(PhysicalIdentifier catalog, long generation) + { + return new HogQlSemanticCatalogSnapshot(2, LANGUAGE_VERSION, catalog, generation, List.of()); } } From 1cbd28e808e8416a684af183504833331bd7c83d Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 12:39:32 -0400 Subject: [PATCH 046/136] feat(hogql): lower sets and semantic catalog recipes --- .../trino/hogql/compiler/HogQlCompiler.java | 43 +- .../hogql/compiler/HogQlSemanticResolver.java | 558 ++++++++++++++++-- .../trino/hogql/compiler/TrinoAstFactory.java | 135 ++++- .../catalog/HogQlSemanticCatalogSnapshot.java | 7 + .../hogql/compiler/TestHogQlCompiler.java | 70 +++ .../compiler/TestHogQlSemanticExpansion.java | 244 ++++++++ .../TestHogQlSemanticCatalogSnapshot.java | 38 ++ .../io/trino/hogql/parser/HogQlParser.java | 319 ++++++++-- .../trino/hogql/parser/tree/HogQlQuery.java | 180 +++++- .../1.0.0/trino-compatibility-overrides.json | 88 +++ .../language/1.0.0/trino-compatibility.json | 88 +-- .../trino/hogql/parser/TestHogQlParser.java | 40 +- 12 files changed, 1630 insertions(+), 180 deletions(-) create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticExpansion.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index 598a92d96b2f..8a207431ee34 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -37,11 +37,14 @@ import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; import io.trino.spi.Location; import io.trino.spi.TrinoException; import io.trino.sql.tree.Statement; @@ -171,15 +174,23 @@ private static HogQlCompilationResult compile( private static void collectPlaceholders(HogQlQuery query, List placeholders) { query.with().forEach(commonTable -> collectPlaceholders(commonTable.query(), placeholders)); - query.projections().forEach(projection -> { - if (projection instanceof HogQlQuery.ExpressionProjection expressionProjection) { - collectPlaceholders(expressionProjection.expression(), placeholders); + switch (query.body()) { + case SelectQueryBody select -> { + select.projections().forEach(projection -> { + if (projection instanceof HogQlQuery.ExpressionProjection expressionProjection) { + collectPlaceholders(expressionProjection.expression(), placeholders); + } + }); + select.from().ifPresent(relation -> collectPlaceholders(relation, placeholders)); + select.where().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + select.groupBy().forEach(expression -> collectPlaceholders(expression, placeholders)); + select.having().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + } + case SetOperation setOperation -> { + collectPlaceholders(setOperation.left(), placeholders); + collectPlaceholders(setOperation.right(), placeholders); } - }); - query.from().ifPresent(relation -> collectPlaceholders(relation, placeholders)); - query.where().ifPresent(expression -> collectPlaceholders(expression, placeholders)); - query.groupBy().forEach(expression -> collectPlaceholders(expression, placeholders)); - query.having().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + } query.orderBy().forEach(sortItem -> collectPlaceholders(sortItem.expression(), placeholders)); query.limit().ifPresent(expression -> collectPlaceholders(expression, placeholders)); query.offset().ifPresent(expression -> collectPlaceholders(expression, placeholders)); @@ -208,7 +219,10 @@ private static ResolvedQuery resolveQuery( private static boolean containsSemanticCandidate(HogQlQuery query) { return query.with().stream().anyMatch(commonTable -> containsSemanticCandidate(commonTable.query())) || - query.from().map(HogQlCompiler::containsSemanticCandidate).orElse(false); + switch (query.body()) { + case SelectQueryBody select -> select.from().map(HogQlCompiler::containsSemanticCandidate).orElse(false); + case SetOperation setOperation -> containsSemanticCandidate(setOperation.left()) || containsSemanticCandidate(setOperation.right()); + }; } private static boolean containsSemanticCandidate(Relation relation) @@ -220,6 +234,7 @@ private static boolean containsSemanticCandidate(Relation relation) case SubqueryRelation subquery -> containsSemanticCandidate(subquery.query()); case TablePlaceholder _ -> false; case HogQlQuery.TableReference table -> table.parts().size() == 1; + case ValuesRelation _ -> false; }; } @@ -247,7 +262,13 @@ private static void validateParameters(HogQlQuery query, Map validateQuery(commonTable.query())); - query.from().ifPresent(HogQlCompiler::validateRelation); + switch (query.body()) { + case SelectQueryBody select -> select.from().ifPresent(HogQlCompiler::validateRelation); + case SetOperation setOperation -> { + validateQuery(setOperation.left()); + validateQuery(setOperation.right()); + } + } } private static void collectPlaceholders(Expression expression, List placeholders) @@ -304,6 +325,7 @@ private static void validateRelation(Relation relation) tablePlaceholder.span(), "HogQL parameter placeholders are not supported in table positions: " + tablePlaceholder.placeholder().name()); case HogQlQuery.TableReference _ -> {} + case ValuesRelation _ -> {} } } @@ -324,6 +346,7 @@ private static void collectPlaceholders(Relation relation, List pla case SubqueryRelation subquery -> collectPlaceholders(subquery.query(), placeholders); case TablePlaceholder _ -> {} case HogQlQuery.TableReference _ -> {} + case ValuesRelation values -> values.rows().forEach(row -> row.forEach(expression -> collectPlaceholders(expression, placeholders))); } } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index 039eaa74761c..2c33510fcaba 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -13,9 +13,25 @@ */ package io.trino.hogql.compiler; -import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CastRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCallRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.MaterializedViewReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ReferencedField; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SavedQueryReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.VirtualTableDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; import io.trino.hogql.parser.tree.HogQlQuery; import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; @@ -41,6 +57,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; import io.trino.hogql.parser.tree.HogQlQuery.SortItem; import io.trino.hogql.parser.tree.HogQlQuery.Star; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; @@ -48,6 +66,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; import io.trino.spi.Location; import io.trino.spi.TrinoException; @@ -58,18 +77,27 @@ import java.util.Map; import java.util.Optional; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_COMPILER_LIMIT_EXCEEDED; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; import static java.util.Objects.requireNonNull; final class HogQlSemanticResolver { private final PinnedSnapshot snapshot; + private final ExpansionBudget expansionBudget; private List bindings = List.of(); private boolean allRelationsLogical; private HogQlSemanticResolver(PinnedSnapshot snapshot) + { + this(snapshot, new ExpansionBudget()); + } + + private HogQlSemanticResolver(PinnedSnapshot snapshot, ExpansionBudget expansionBudget) { this.snapshot = requireNonNull(snapshot, "snapshot is null"); + this.expansionBudget = requireNonNull(expansionBudget, "expansionBudget is null"); } public static Optional resolve(PinnedSnapshot snapshot, HogQlQuery query) @@ -87,21 +115,34 @@ private HogQlQuery resolveNestedQuery(HogQlQuery query) .map(commonTable -> new CommonTableExpression( commonTable.name(), commonTable.columnAliases(), - new HogQlSemanticResolver(snapshot).resolveNestedQuery(commonTable.query()), + new HogQlSemanticResolver(snapshot, expansionBudget).resolveNestedQuery(commonTable.query()), commonTable.span())) .toList(); - Optional relation = query.from().map(this::resolveRelation); + if (query.body() instanceof SetOperation setOperation) { + SetOperation resolved = new SetOperation( + setOperation.type(), + setOperation.distinct(), + new HogQlSemanticResolver(snapshot, expansionBudget).resolveNestedQuery(setOperation.left()), + new HogQlSemanticResolver(snapshot, expansionBudget).resolveNestedQuery(setOperation.right()), + setOperation.leftParenthesized(), + setOperation.rightParenthesized(), + setOperation.operatorSpan(), + setOperation.span()); + return new HogQlQuery(commonTables, resolved, query.orderBy(), query.limit(), query.offset(), query.span()); + } + SelectQueryBody select = (SelectQueryBody) query.body(); + Optional relation = select.from().map(this::resolveRelation); bindings = relation.map(ResolvedRelation::bindings).orElse(List.of()); allRelationsLogical = relation.map(ResolvedRelation::allLogical).orElse(false); if (bindings.isEmpty()) { return new HogQlQuery( commonTables, - query.distinct(), - query.projections(), + select.distinct(), + select.projections(), relation.map(ResolvedRelation::relation), - query.where(), - query.groupBy(), - query.having(), + select.where(), + select.groupBy(), + select.having(), query.orderBy(), query.limit(), query.offset(), @@ -137,10 +178,10 @@ private List resolveProjection(Projection projection) yield List.of(star); } yield bindings.stream() - .flatMap(binding -> binding.logicalTable().fields().stream() - .filter(LogicalFieldDefinition::starVisible) + .flatMap(binding -> binding.orderedFields().stream() + .filter(BoundField::starVisible) .map(field -> new ExpressionProjection( - physicalColumn(field, binding.starQualifier(bindings.size()), star.span()), + resolveBoundField(binding, field, binding.starQualifier(bindings.size()), star.span(), expansionBudget), Optional.of(new Identifier(field.name(), true, star.span()))))) .map(Projection.class::cast) .toList(); @@ -165,7 +206,7 @@ private ResolvedRelation resolveRelation(Relation relation) .map(binding -> binding.withAlias(alias.alias())) .toList(); yield new ResolvedRelation( - new AliasedRelation(child.relation(), alias.alias(), alias.span()), + new AliasedRelation(child.relation(), alias.alias(), alias.columnAliases(), alias.span()), aliasedBindings, child.allLogical()); } @@ -191,11 +232,12 @@ yield new ResolvedRelation( left.allLogical() && right.allLogical()); } case SubqueryRelation subquery -> new ResolvedRelation( - new SubqueryRelation(new HogQlSemanticResolver(snapshot).resolveNestedQuery(subquery.query()), subquery.span()), + new SubqueryRelation(new HogQlSemanticResolver(snapshot, expansionBudget).resolveNestedQuery(subquery.query()), subquery.span()), List.of(), false); case TablePlaceholder placeholder -> new ResolvedRelation(placeholder, List.of(), false); case TableReference table -> resolveTable(table); + case ValuesRelation values -> new ResolvedRelation(values, List.of(), false); }; } @@ -203,8 +245,8 @@ private JoinUsing resolveJoinUsing(JoinUsing using, List leftBindi { List columns = using.columns().stream() .map(column -> { - List leftFields = matchingFields(leftBindings, column.value()); - List rightFields = matchingFields(rightBindings, column.value()); + List leftFields = matchingFields(leftBindings, column.value()); + List rightFields = matchingFields(rightBindings, column.value()); if (leftFields.size() != 1 || rightFields.size() != 1) { throw resolutionError(new ColumnReference(List.of(column), column.span()), column.value()); } @@ -219,7 +261,7 @@ private JoinUsing resolveJoinUsing(JoinUsing using, List leftBindi return new JoinUsing(columns, using.span()); } - private static List matchingFields(List tableBindings, String name) + private static List matchingFields(List tableBindings, String name) { return tableBindings.stream() .map(TableBinding::fields) @@ -233,27 +275,193 @@ private ResolvedRelation resolveTable(TableReference table) if (table.parts().size() != 1) { return new ResolvedRelation(table, List.of(), false); } - Optional logicalTable = snapshot.logicalTable(table.parts().getFirst().value()); - if (logicalTable.isEmpty()) { - return new ResolvedRelation(table, List.of(), false); + String name = table.parts().getFirst().value(); + return resolveSemanticRelation(name, table.span(), new RelationExpansionBudget()) + .orElseGet(() -> new ResolvedRelation(table, List.of(), false)); + } + + private Optional resolveSemanticRelation(String name, HogQlQuery.SourceSpan span, RelationExpansionBudget budget) + { + budget.enter(span); + try { + Optional logicalTable = snapshot.logicalTable(name); + if (logicalTable.isPresent()) { + return Optional.of(resolveLogicalTable(logicalTable.orElseThrow(), span)); + } + Optional materializedView = snapshot.snapshot().materializedViews().stream() + .filter(view -> canonical(view.name()).equals(canonical(name))) + .findFirst(); + if (materializedView.isPresent()) { + return Optional.of(resolveMaterializedView(materializedView.orElseThrow(), span)); + } + Optional virtualTable = snapshot.snapshot().virtualTables().stream() + .filter(table -> canonical(table.name()).equals(canonical(name))) + .findFirst(); + if (virtualTable.isPresent()) { + return Optional.of(resolveVirtualTable(virtualTable.orElseThrow(), span, budget)); + } + Optional savedQuery = snapshot.snapshot().savedQueries().stream() + .filter(query -> canonical(query.name()).equals(canonical(name))) + .findFirst(); + if (savedQuery.isPresent()) { + return Optional.of(resolveSavedQuery(savedQuery.orElseThrow(), span, budget)); + } + return Optional.empty(); } - LogicalTableDefinition definition = logicalTable.orElseThrow(); + finally { + budget.exit(); + } + } + + private ResolvedRelation resolveLogicalTable(LogicalTableDefinition definition, HogQlQuery.SourceSpan span) + { TableReference physicalTable = new TableReference( List.of( - identifier(definition.physicalTable().catalog(), table), - identifier(definition.physicalTable().schema(), table), - identifier(definition.physicalTable().table(), table)), - table.span()); + identifier(definition.physicalTable().catalog(), span), + identifier(definition.physicalTable().schema(), span), + identifier(definition.physicalTable().table(), span)), + span); + List fields = new ArrayList<>(); + definition.fields().forEach(field -> fields.add(new BoundField(field.name(), field.physicalColumn(), field.starVisible(), Optional.empty()))); + snapshot.snapshot().expressionFields().stream() + .filter(field -> canonical(field.table()).equals(canonical(definition.name()))) + .forEach(field -> fields.add(new BoundField( + field.name(), + new PhysicalIdentifier(field.name(), true), + field.starVisible(), + Optional.of(field)))); return new ResolvedRelation( physicalTable, - List.of(new TableBinding( + List.of(tableBinding( definition, canonical(definition.name()), definition.physicalTable().table(), - false)), + false, + fields)), + true); + } + + private ResolvedRelation resolveMaterializedView(MaterializedViewReference definition, HogQlQuery.SourceSpan span) + { + TableReference physicalTable = new TableReference( + List.of( + identifier(definition.physicalView().catalog(), span), + identifier(definition.physicalView().schema(), span), + identifier(definition.physicalView().table(), span)), + span); + List fields = definition.fields().stream() + .map(field -> referencedField(field, Optional.empty())) + .toList(); + return new ResolvedRelation( + physicalTable, + List.of(new TableBinding( + definition.name(), + canonical(definition.name()), + new PhysicalIdentifier(definition.physicalView().table().value(), definition.physicalView().table().delimited()), + false, + fields, + TableBinding.fieldMap(fields))), + true); + } + + private ResolvedRelation resolveVirtualTable(VirtualTableDefinition definition, HogQlQuery.SourceSpan span, RelationExpansionBudget budget) + { + ResolvedRelation source = resolveRelationReference(definition.source(), span, budget); + List projections = definition.projections().stream() + .map(projection -> new ProjectedField(projection.name(), projection.sourceField(), projection.starVisible())) + .toList(); + return projectRelation(definition.name(), source, projections, span); + } + + private ResolvedRelation resolveSavedQuery(SavedQueryReference definition, HogQlQuery.SourceSpan span, RelationExpansionBudget budget) + { + ResolvedRelation source = resolveRelationReference(definition.target(), span, budget); + List projections = definition.fields().stream() + .map(field -> new ProjectedField(field.name(), field.name(), field.starVisible())) + .toList(); + return projectRelation(definition.name(), source, projections, span); + } + + private ResolvedRelation resolveRelationReference(RelationReference reference, HogQlQuery.SourceSpan span, RelationExpansionBudget budget) + { + ResolvedRelation relation = resolveSemanticRelation(reference.name(), span, budget) + .orElseThrow(() -> expansionError(span, "HogQL semantic relation references an unavailable target")); + RelationKind actualKind = semanticRelationKind(reference.name()); + if (actualKind != reference.kind()) { + throw expansionError(span, "HogQL semantic relation target kind does not match the catalog"); + } + return relation; + } + + private RelationKind semanticRelationKind(String name) + { + if (snapshot.logicalTable(name).isPresent()) { + return RelationKind.LOGICAL_TABLE; + } + if (snapshot.snapshot().virtualTables().stream().anyMatch(table -> canonical(table.name()).equals(canonical(name)))) { + return RelationKind.VIRTUAL_TABLE; + } + if (snapshot.snapshot().savedQueries().stream().anyMatch(query -> canonical(query.name()).equals(canonical(name)))) { + return RelationKind.SAVED_QUERY; + } + return RelationKind.MATERIALIZED_VIEW; + } + + private ResolvedRelation projectRelation(String relationName, ResolvedRelation source, List projectedFields, HogQlQuery.SourceSpan span) + { + List previousBindings = bindings; + boolean previousAllRelationsLogical = allRelationsLogical; + bindings = source.bindings(); + allRelationsLogical = source.allLogical(); + List projections; + try { + projections = projectedFields.stream() + .map(field -> new ExpressionProjection( + resolveColumn(new ColumnReference(List.of(new Identifier(field.sourceField(), true, span)), span)), + Optional.of(new Identifier(field.name(), true, span)))) + .map(Projection.class::cast) + .toList(); + } + finally { + bindings = previousBindings; + allRelationsLogical = previousAllRelationsLogical; + } + HogQlQuery projectedQuery = new HogQlQuery( + List.of(), + false, + projections, + Optional.of(source.relation()), + Optional.empty(), + List.of(), + Optional.empty(), + List.of(), + Optional.empty(), + Optional.empty(), + span); + List fields = projectedFields.stream() + .map(field -> new BoundField( + field.name(), + new PhysicalIdentifier(field.name(), true), + field.starVisible(), + Optional.empty())) + .toList(); + return new ResolvedRelation( + new SubqueryRelation(projectedQuery, span), + List.of(new TableBinding( + relationName, + canonical(relationName), + new PhysicalIdentifier(relationName, true), + false, + fields, + TableBinding.fieldMap(fields))), true); } + private static BoundField referencedField(ReferencedField field, Optional expression) + { + return new BoundField(field.name(), new PhysicalIdentifier(field.name(), true), field.starVisible(), expression); + } + private List resolveSortItems(List sortItems) { return sortItems.stream() @@ -287,7 +495,7 @@ private Expression resolveExpression(Expression expression) case CastExpression cast -> new CastExpression(resolveExpression(cast.value()), cast.type(), cast.safe(), cast.span()); case ColumnReference reference -> resolveColumn(reference); case FunctionCall function -> new FunctionCall( - function.name(), + function.nameParts(), function.arguments().stream().map(this::resolveExpression).toList(), function.distinct(), resolveSortItems(function.orderBy()), @@ -311,7 +519,7 @@ private Expression resolveExpression(Expression expression) }; } - private ColumnReference resolveColumn(ColumnReference reference) + private Expression resolveColumn(ColumnReference reference) { List parts = reference.parts(); if (parts.size() == 2) { @@ -324,11 +532,11 @@ private ColumnReference resolveColumn(ColumnReference reference) } return reference; } - LogicalFieldDefinition field = binding.orElseThrow().fields().get(canonical(parts.getLast().value())); + BoundField field = binding.orElseThrow().fields().get(canonical(parts.getLast().value())); if (field == null) { throw resolutionError(reference, parts.getLast().value()); } - return physicalColumn(field, Optional.of(binding.orElseThrow().outputQualifier()), reference.span()); + return resolveBoundField(binding.orElseThrow(), field, Optional.of(binding.orElseThrow().outputQualifier()), reference.span(), expansionBudget); } if (parts.size() > 2) { if (allRelationsLogical) { @@ -337,13 +545,14 @@ private ColumnReference resolveColumn(ColumnReference reference) return reference; } String logicalName = parts.getLast().value(); - List matches = bindings.stream() - .map(TableBinding::fields) - .map(fields -> fields.get(canonical(logicalName))) - .filter(field -> field != null) + List matches = bindings.stream() + .map(binding -> new FieldMatch(binding, binding.fields().get(canonical(logicalName)))) + .filter(match -> match.field() != null) .toList(); if (matches.size() == 1 && (allRelationsLogical || bindings.size() == 1)) { - return physicalColumn(matches.getFirst(), Optional.empty(), reference.span()); + FieldMatch match = matches.getFirst(); + Optional qualifier = bindings.size() > 1 ? Optional.of(match.binding().outputQualifier()) : Optional.empty(); + return resolveBoundField(match.binding(), match.field(), qualifier, reference.span(), expansionBudget); } if (matches.size() > 1) { throw ambiguousResolutionError(reference, logicalName); @@ -354,7 +563,20 @@ private ColumnReference resolveColumn(ColumnReference reference) return reference; } - private static ColumnReference physicalColumn(LogicalFieldDefinition field, Optional qualifier, HogQlQuery.SourceSpan span) + private Expression resolveBoundField( + TableBinding binding, + BoundField field, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget) + { + budget.add(span); + return field.expression() + .map(expression -> expandRecipe(binding, expression.recipe(), qualifier, span, budget)) + .orElseGet(() -> physicalColumn(field, qualifier, span)); + } + + private static ColumnReference physicalColumn(BoundField field, Optional qualifier, HogQlQuery.SourceSpan span) { List parts = new ArrayList<>(); qualifier.ifPresent(identifier -> parts.add(new Identifier(identifier.value(), identifier.delimited(), span))); @@ -362,11 +584,182 @@ private static ColumnReference physicalColumn(LogicalFieldDefinition field, Opti return new ColumnReference(parts, span); } + private Expression expandRecipe( + TableBinding binding, + ExpressionRecipe recipe, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget) + { + budget.enter(span); + try { + return switch (recipe) { + case FieldReferenceRecipe reference -> { + BoundField field = binding.fields().get(canonical(reference.field())); + if (field == null) { + throw expansionError(span, "HogQL expression recipe references an unavailable field"); + } + yield resolveBoundField(binding, field, qualifier, span, budget); + } + case LiteralRecipe literal -> typedLiteral(literal.literal(), span); + case FunctionCallRecipe function -> expandFunction(binding, function, qualifier, span, budget); + case OperatorRecipe operator -> expandOperator(binding, operator, qualifier, span, budget); + case CastRecipe cast -> new CastExpression( + expandRecipe(binding, cast.expression(), qualifier, span, budget), + new Identifier(cast.targetTypeSignature(), false, span), + false, + span); + }; + } + finally { + budget.exit(); + } + } + + private Expression expandFunction( + TableBinding binding, + FunctionCallRecipe function, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget) + { + FunctionCapabilityDefinition capability = snapshot.snapshot().functions().stream() + .filter(candidate -> canonical(candidate.name()).equals(canonical(function.name()))) + .findFirst() + .orElseThrow(() -> expansionError(span, "HogQL expression recipe references an unavailable function")); + if (capability.implementation() == FunctionImplementation.REWRITE || capability.trinoName().isEmpty()) { + throw unsupportedExpansion(span, "HogQL function recipe requires an unavailable compiler rewrite"); + } + boolean supportedArity = capability.signatures().stream() + .anyMatch(signature -> signature.variadic() + ? function.arguments().size() >= Math.max(0, signature.argumentTypes().size() - 1) + : function.arguments().size() == signature.argumentTypes().size()); + if (!supportedArity) { + throw expansionError(span, "HogQL function recipe does not match a declared signature"); + } + return new FunctionCall( + capability.trinoName().stream() + .map(name -> new Identifier(name.value(), name.delimited(), span)) + .toList(), + function.arguments().stream() + .map(argument -> expandRecipe(binding, argument, qualifier, span, budget)) + .toList(), + false, + List.of(), + Optional.empty(), + span); + } + + private Expression expandOperator( + TableBinding binding, + OperatorRecipe operator, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget) + { + List arguments = operator.arguments().stream() + .map(argument -> expandRecipe(binding, argument, qualifier, span, budget)) + .toList(); + return switch (operator.operator()) { + case NOT -> new UnaryExpression(HogQlQuery.UnaryOperator.NOT, arguments.getFirst(), span); + case NEGATE -> new UnaryExpression(HogQlQuery.UnaryOperator.NEGATE, arguments.getFirst(), span); + case IS_NULL -> new IsNullExpression(arguments.getFirst(), false, span, span); + case IS_NOT_NULL -> new IsNullExpression(arguments.getFirst(), true, span, span); + default -> new BinaryExpression(binaryOperator(operator.operator()), arguments.getFirst(), arguments.getLast(), span); + }; + } + + private static HogQlQuery.BinaryOperator binaryOperator(SemanticOperator operator) + { + return switch (operator) { + case ADD -> HogQlQuery.BinaryOperator.ADD; + case SUBTRACT -> HogQlQuery.BinaryOperator.SUBTRACT; + case MULTIPLY -> HogQlQuery.BinaryOperator.MULTIPLY; + case DIVIDE -> HogQlQuery.BinaryOperator.DIVIDE; + case MODULUS -> HogQlQuery.BinaryOperator.MODULO; + case EQUAL -> HogQlQuery.BinaryOperator.EQUAL; + case NOT_EQUAL -> HogQlQuery.BinaryOperator.NOT_EQUAL; + case LESS_THAN -> HogQlQuery.BinaryOperator.LESS_THAN; + case LESS_THAN_OR_EQUAL -> HogQlQuery.BinaryOperator.LESS_THAN_OR_EQUAL; + case GREATER_THAN -> HogQlQuery.BinaryOperator.GREATER_THAN; + case GREATER_THAN_OR_EQUAL -> HogQlQuery.BinaryOperator.GREATER_THAN_OR_EQUAL; + case AND -> HogQlQuery.BinaryOperator.AND; + case OR -> HogQlQuery.BinaryOperator.OR; + case NOT, NEGATE, IS_NULL, IS_NOT_NULL -> throw new IllegalArgumentException("unary operator cannot be lowered as binary"); + }; + } + + private static Expression typedLiteral(TypedLiteral literal, HogQlQuery.SourceSpan span) + { + Expression value = switch (literal.encoding()) { + case NULL -> new Literal(HogQlQuery.LiteralKind.NULL, "", span); + case BOOLEAN -> new Literal(HogQlQuery.LiteralKind.BOOLEAN, literal.value(), span); + case INTEGER -> new Literal(HogQlQuery.LiteralKind.INTEGER, literal.value(), span); + case STRING, DECIMAL, FLOAT, JSON -> new Literal(HogQlQuery.LiteralKind.STRING, literal.value(), span); + case BASE64 -> new FunctionCall( + new Identifier("from_base64", false, span), + List.of(new Literal(HogQlQuery.LiteralKind.STRING, literal.value(), span)), + false, + List.of(), + Optional.empty(), + span); + }; + return new CastExpression(value, new Identifier(literal.typeSignature(), false, span), false, span); + } + + private static TrinoException expansionError(HogQlQuery.SourceSpan span, String message) + { + return new TrinoException( + HOGQL_RESOLUTION_ERROR, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } + + private static TrinoException unsupportedExpansion(HogQlQuery.SourceSpan span, String message) + { + return new TrinoException( + HOGQL_UNSUPPORTED_FEATURE, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } + + private static TrinoException limitError(HogQlQuery.SourceSpan span, String message) + { + return new TrinoException( + HOGQL_COMPILER_LIMIT_EXCEEDED, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } + private static Identifier identifier(PhysicalIdentifier identifier, TableReference source) { return new Identifier(identifier.value(), identifier.delimited(), source.span()); } + private static Identifier identifier(PhysicalIdentifier identifier, HogQlQuery.SourceSpan span) + { + return new Identifier(identifier.value(), identifier.delimited(), span); + } + + private static TableBinding tableBinding( + LogicalTableDefinition definition, + String qualifier, + PhysicalIdentifier outputQualifier, + boolean aliased, + List fields) + { + return new TableBinding( + definition.name(), + qualifier, + outputQualifier, + aliased, + fields, + TableBinding.fieldMap(fields)); + } + private static String canonical(String value) { return value.toLowerCase(Locale.ENGLISH); @@ -409,32 +802,35 @@ private record ResolvedRelation(Relation relation, List bindings, } private record TableBinding( - LogicalTableDefinition logicalTable, + String relationName, String qualifier, PhysicalIdentifier outputQualifier, boolean aliased, - Map fields) + List orderedFields, + Map fields) { private TableBinding(LogicalTableDefinition logicalTable, String qualifier, PhysicalIdentifier outputQualifier, boolean aliased) { - this(logicalTable, qualifier, outputQualifier, aliased, fieldMap(logicalTable)); + this(logicalTable.name(), qualifier, outputQualifier, aliased, boundFields(logicalTable), fieldMap(boundFields(logicalTable))); } private TableBinding { - logicalTable = requireNonNull(logicalTable, "logicalTable is null"); + relationName = requireNonNull(relationName, "relationName is null"); qualifier = requireNonNull(qualifier, "qualifier is null"); outputQualifier = requireNonNull(outputQualifier, "outputQualifier is null"); + orderedFields = List.copyOf(requireNonNull(orderedFields, "orderedFields is null")); fields = Map.copyOf(requireNonNull(fields, "fields is null")); } private TableBinding withAlias(Identifier alias) { return new TableBinding( - logicalTable, + relationName, canonical(alias.value()), new PhysicalIdentifier(alias.value(), alias.delimited()), true, + orderedFields, fields); } @@ -443,14 +839,88 @@ private Optional starQualifier(int relationCount) return aliased || relationCount > 1 ? Optional.of(outputQualifier) : Optional.empty(); } - private static Map fieldMap(LogicalTableDefinition table) + private static List boundFields(LogicalTableDefinition table) { - Map fields = new HashMap<>(); - table.fields().forEach(field -> fields.put(canonical(field.name()), field)); + return table.fields().stream() + .map(field -> new BoundField(field.name(), field.physicalColumn(), field.starVisible(), Optional.empty())) + .toList(); + } + + private static Map fieldMap(List orderedFields) + { + Map fields = new HashMap<>(); + orderedFields.forEach(field -> fields.put(canonical(field.name()), field)); return fields; } } + private record BoundField(String name, PhysicalIdentifier physicalColumn, boolean starVisible, Optional expression) + { + private BoundField + { + name = requireNonNull(name, "name is null"); + physicalColumn = requireNonNull(physicalColumn, "physicalColumn is null"); + expression = requireNonNull(expression, "expression is null"); + } + } + + private record FieldMatch(TableBinding binding, BoundField field) {} + + private record ProjectedField(String name, String sourceField, boolean starVisible) + { + private ProjectedField + { + name = requireNonNull(name, "name is null"); + sourceField = requireNonNull(sourceField, "sourceField is null"); + } + } + + private static final class ExpansionBudget + { + private static final int MAXIMUM_NODES = 10_000; + private static final int MAXIMUM_DEPTH = 64; + private int nodes; + private int depth; + + private void add(HogQlQuery.SourceSpan span) + { + if (++nodes > MAXIMUM_NODES) { + throw limitError(span, "HogQL semantic expansion exceeded node limit"); + } + } + + private void enter(HogQlQuery.SourceSpan span) + { + add(span); + if (++depth > MAXIMUM_DEPTH) { + throw limitError(span, "HogQL semantic expansion exceeded depth limit"); + } + } + + private void exit() + { + depth--; + } + } + + private static final class RelationExpansionBudget + { + private static final int MAXIMUM_DEPTH = 64; + private int depth; + + private void enter(HogQlQuery.SourceSpan span) + { + if (++depth > MAXIMUM_DEPTH) { + throw limitError(span, "HogQL semantic relation expansion exceeded depth limit"); + } + } + + private void exit() + { + depth--; + } + } + record ResolvedQuery(HogQlQuery query) { ResolvedQuery diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 72af5139bfda..4cb61d76545c 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -35,6 +35,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.Star; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; @@ -42,6 +44,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.sql.parser.SqlParser; import io.trino.sql.tree.AllColumns; import io.trino.sql.tree.ArithmeticBinaryExpression; import io.trino.sql.tree.ArithmeticUnaryExpression; @@ -52,11 +56,12 @@ import io.trino.sql.tree.Cast; import io.trino.sql.tree.ComparisonPredicate; import io.trino.sql.tree.DereferenceExpression; +import io.trino.sql.tree.Except; import io.trino.sql.tree.Expression; -import io.trino.sql.tree.GenericDataType; import io.trino.sql.tree.GroupBy; import io.trino.sql.tree.InListExpression; import io.trino.sql.tree.InPredicate; +import io.trino.sql.tree.Intersect; import io.trino.sql.tree.IsNullPredicate; import io.trino.sql.tree.Limit; import io.trino.sql.tree.LogicalExpression; @@ -70,6 +75,7 @@ import io.trino.sql.tree.Predicated; import io.trino.sql.tree.QualifiedName; import io.trino.sql.tree.Query; +import io.trino.sql.tree.QueryBody; import io.trino.sql.tree.QuerySpecification; import io.trino.sql.tree.Row; import io.trino.sql.tree.SearchedCaseExpression; @@ -84,6 +90,8 @@ import io.trino.sql.tree.StringLiteral; import io.trino.sql.tree.Table; import io.trino.sql.tree.TableSubquery; +import io.trino.sql.tree.Union; +import io.trino.sql.tree.Values; import io.trino.sql.tree.WhenClause; import io.trino.sql.tree.With; import io.trino.sql.tree.WithQuery; @@ -94,6 +102,8 @@ final class TrinoAstFactory { + private static final SqlParser SQL_PARSER = new SqlParser(); + private TrinoAstFactory() {} public static Statement createStatement(HogQlQuery query, Map parameterIds) @@ -104,28 +114,70 @@ public static Statement createStatement(HogQlQuery query, Map parameterIds) { NodeLocation location = location(query.span()); + QueryBody queryBody = createQueryBody(query, parameterIds); + boolean setOperation = query.body() instanceof SetOperation; + return new Query( + location, + List.of(), + List.of(), + createWith(query, parameterIds), + queryBody, + setOperation ? createOrderBy(query, parameterIds) : Optional.empty(), + setOperation ? query.offset().map(offset -> new Offset(location(offset.span()), createExpression(offset, parameterIds))) : Optional.empty(), + setOperation ? query.limit().map(limit -> new Limit(location(limit.span()), createExpression(limit, parameterIds))) : Optional.empty()); + } + + private static QueryBody createQueryBody(HogQlQuery query, Map parameterIds) + { + return switch (query.body()) { + case SelectQueryBody select -> createQuerySpecification(query, select, parameterIds); + case SetOperation setOperation -> createSetOperation(setOperation, parameterIds); + }; + } + + private static QuerySpecification createQuerySpecification( + HogQlQuery query, + SelectQueryBody select, + Map parameterIds) + { + NodeLocation location = location(select.span()); QuerySpecification querySpecification = new QuerySpecification( location, - new Select(location, query.distinct(), query.projections().stream() + new Select(location, select.distinct(), select.projections().stream() .map(projection -> createSelectItem(projection, parameterIds)) .toList()), - query.from().map(relation -> createRelation(relation, parameterIds)), - query.where().map(expression -> createExpression(expression, parameterIds)), - createGroupBy(query, parameterIds), - query.having().map(expression -> createExpression(expression, parameterIds)), + select.from().map(relation -> createRelation(relation, parameterIds)), + select.where().map(expression -> createExpression(expression, parameterIds)), + createGroupBy(select.groupBy(), parameterIds), + select.having().map(expression -> createExpression(expression, parameterIds)), List.of(), createOrderBy(query, parameterIds), query.offset().map(offset -> new Offset(location(offset.span()), createExpression(offset, parameterIds))), query.limit().map(limit -> new Limit(location(limit.span()), createExpression(limit, parameterIds)))); - return new Query( - location, - List.of(), - List.of(), - createWith(query, parameterIds), - querySpecification, - Optional.empty(), - Optional.empty(), - Optional.empty()); + return querySpecification; + } + + private static QueryBody createSetOperation(SetOperation setOperation, Map parameterIds) + { + NodeLocation location = location(setOperation.operatorSpan()); + QueryBody left = createSetOperand(setOperation.left(), setOperation.leftParenthesized(), parameterIds); + QueryBody right = createSetOperand(setOperation.right(), setOperation.rightParenthesized(), parameterIds); + return switch (setOperation.type()) { + case EXCEPT -> new Except(location, left, right, setOperation.distinct(), Optional.empty()); + case INTERSECT -> new Intersect(location, List.of(left, right), setOperation.distinct(), Optional.empty()); + case UNION -> new Union(location, List.of(left, right), setOperation.distinct(), Optional.empty()); + }; + } + + private static QueryBody createSetOperand( + HogQlQuery query, + boolean parenthesized, + Map parameterIds) + { + if (parenthesized || !query.with().isEmpty() || !query.orderBy().isEmpty() || query.limit().isPresent() || query.offset().isPresent()) { + return new TableSubquery(location(query.span()), createQuery(query, parameterIds)); + } + return createQueryBody(query, parameterIds); } private static Optional createWith(HogQlQuery query, Map parameterIds) @@ -182,18 +234,18 @@ private static Optional createOrderBy(List sortIte .toList())); } - private static Optional createGroupBy(HogQlQuery query, Map parameterIds) + private static Optional createGroupBy(List expressions, Map parameterIds) { - if (query.groupBy().isEmpty()) { + if (expressions.isEmpty()) { return Optional.empty(); } - NodeLocation location = location(query.groupBy().getFirst().span()); + NodeLocation location = location(expressions.getFirst().span()); return Optional.of(new GroupBy( location, false, List.of(new SimpleGroupBy( location, - query.groupBy().stream() + expressions.stream() .map(expression -> createExpression(expression, parameterIds)) .toList())))); } @@ -223,7 +275,7 @@ private static Expression createExpression(HogQlQuery.Expression expression, Map case CastExpression cast -> new Cast( location(cast.span()), createExpression(cast.value(), parameterIds), - new GenericDataType(location(cast.type().span()), createIdentifier(cast.type()), List.of()), + SQL_PARSER.createType(cast.type().value()), cast.safe()); case ColumnReference reference -> createColumnReference(reference); case FunctionCall function -> createFunctionCall(function, parameterIds); @@ -251,7 +303,9 @@ private static Expression createFunctionCall(FunctionCall function, Map createExpression(filter, parameterIds)), createOrderBy(function.orderBy(), parameterIds), @@ -366,7 +420,11 @@ private static io.trino.sql.tree.Relation createRelation(Relation relation, Map< location(alias.span()), createRelation(alias.relation(), parameterIds), createIdentifier(alias.alias()), - null); + alias.columnAliases().isEmpty() + ? null + : alias.columnAliases().stream() + .map(TrinoAstFactory::createIdentifier) + .toList()); case CommonTableReference commonTable -> new Table( location(commonTable.span()), QualifiedName.of(List.of(createIdentifier(commonTable.name())))); @@ -390,9 +448,42 @@ private static io.trino.sql.tree.Relation createRelation(Relation relation, Map< case SubqueryRelation subquery -> new TableSubquery(location(subquery.span()), createQuery(subquery.query(), parameterIds)); case TablePlaceholder _ -> throw new IllegalArgumentException("table placeholder was not validated"); case TableReference table -> createTable(table); + case ValuesRelation values -> createValuesRelation(values, parameterIds); }; } + private static TableSubquery createValuesRelation(ValuesRelation values, Map parameterIds) + { + NodeLocation valuesLocation = new NodeLocation(values.span().startLine(), values.span().startColumn() + 1); + Values body = new Values( + valuesLocation, + values.rows().stream() + .map(row -> createValuesRow(row, parameterIds)) + .toList()); + Query query = new Query( + valuesLocation, + List.of(), + List.of(), + Optional.empty(), + body, + Optional.empty(), + Optional.empty(), + Optional.empty()); + return new TableSubquery(location(values.span()), query); + } + + private static Expression createValuesRow(List row, Map parameterIds) + { + if (row.size() == 1) { + return createExpression(row.getFirst(), parameterIds); + } + return new Row( + location(row.getFirst().span()), + row.stream() + .map(value -> new Row.Field(location(value.span()), Optional.empty(), createExpression(value, parameterIds))) + .toList()); + } + private static Table createTable(TableReference table) { List parts = table.parts().stream() diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index ff2d7d31cae0..4b4fade32df2 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -537,6 +537,13 @@ public record OperatorRecipe(SemanticOperator operator, List a if (arguments.isEmpty()) { throw new IllegalArgumentException("operator recipe requires arguments"); } + int expectedArguments = switch (operator) { + case NOT, NEGATE, IS_NULL, IS_NOT_NULL -> 1; + default -> 2; + }; + if (arguments.size() != expectedArguments) { + throw new IllegalArgumentException("invalid operator arity"); + } } @Override diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index a973b2e65470..aaea37efb5e5 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -33,6 +33,8 @@ import io.trino.sql.tree.Statement; import io.trino.sql.tree.Table; import io.trino.sql.tree.TableSubquery; +import io.trino.sql.tree.Union; +import io.trino.sql.tree.Values; import io.trino.sql.tree.WithQuery; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -232,6 +234,74 @@ public void testPreservesCteAndDerivedTableSourceLocations() assertThat(derivedTable.getQuery().getLocation()).contains(new NodeLocation(2, 19)); } + @ParameterizedTest + @ValueSource(strings = { + "SELECT 1 UNION SELECT 2", + "SELECT 1 UNION ALL SELECT 2", + "SELECT 1 UNION DISTINCT SELECT 2", + "SELECT 1 INTERSECT SELECT 2", + "SELECT 1 INTERSECT ALL SELECT 2", + "SELECT 1 INTERSECT DISTINCT SELECT 2", + "SELECT 1 EXCEPT SELECT 2", + "SELECT 1 EXCEPT ALL SELECT 2", + "SELECT 1 UNION SELECT 2 INTERSECT SELECT 2", + "SELECT 1 EXCEPT SELECT 2 UNION SELECT 3", + "SELECT 1 UNION (SELECT 2 EXCEPT SELECT 3)", + "WITH base AS (SELECT 1 AS id) SELECT id FROM base UNION SELECT id FROM base", + "WITH base AS (SELECT 1 UNION ALL SELECT 2) SELECT * FROM base", + "SELECT * FROM (SELECT 1 UNION ALL SELECT 2) AS data", + "SELECT * FROM (VALUES (1, 'first'), (2, 'second')) AS data(id, label)", + }) + public void testLowersSetOperationsAndValuesToStockTrinoAst(String hogql) + { + Statement statement = compiler.compile(hogql); + + assertThat(statement).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @Test + public void testLowersSetOperationOrderLimitAndOffset() + { + Statement statement = compiler.compile("SELECT 1 UNION SELECT 2 ORDER BY 1 LIMIT 3 OFFSET 1"); + + assertThat(statement).isEqualTo(sqlParser.createStatement("SELECT 1 UNION SELECT 2 ORDER BY 1 OFFSET 1 ROW LIMIT 3")); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @Test + public void testBindsPlaceholdersAcrossSetBranchesAndValuesRows() + { + HogQlCompilationResult result = compiler.compile( + "SELECT {left} UNION ALL SELECT {right} FROM (VALUES ({first}), ({second})) AS data(value)", + Map.of( + "left", typedValue("left"), + "right", typedValue("right"), + "first", typedValue("first"), + "second", typedValue("second"))); + + assertThat(result.parameterNames()).containsExactly("left", "right", "first", "second"); + assertThat(parameters(result.statement())) + .extracting(Parameter::getId) + .containsExactly(0, 1, 2, 3); + } + + @Test + public void testPreservesSetOperationAndValuesSourceLocations() + { + Query query = (Query) compiler.compile("SELECT 1\nUNION ALL\nSELECT * FROM (VALUES (2), (3)) data(value)"); + Union union = (Union) query.getQueryBody(); + QuerySpecification right = (QuerySpecification) union.getRelations().get(1); + AliasedRelation alias = (AliasedRelation) right.getFrom().orElseThrow(); + TableSubquery valuesSubquery = (TableSubquery) alias.getRelation(); + Values values = (Values) valuesSubquery.getQuery().getQueryBody(); + + assertThat(union.getLocation()).contains(new NodeLocation(2, 1)); + assertThat(alias.getLocation()).contains(new NodeLocation(3, 15)); + assertThat(valuesSubquery.getLocation()).contains(new NodeLocation(3, 15)); + assertThat(values.getLocation()).contains(new NodeLocation(3, 16)); + } + @Test public void testPreservesAliasedJoinSourceLocations() { diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticExpansion.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticExpansion.java new file mode 100644 index 000000000000..e4ad1bf27c35 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticExpansion.java @@ -0,0 +1,244 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CastRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCallRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.MaterializedViewReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ReferencedField; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SavedQueryReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.VirtualProjection; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.VirtualTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_COMPILER_LIMIT_EXCEEDED; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlSemanticExpansion +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); + private static final HogQlSemanticCatalogSnapshot SNAPSHOT = snapshot(); + + private final HogQlCompiler compiler = new HogQlCompiler(); + private final SqlParser sqlParser = new SqlParser(); + + @Test + public void testExpandsEveryExpressionRecipeToStockAst() + { + HogQlCompilationResult result = compile("SELECT constant, upperEvent, added, castEvent, missingEvent FROM events"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT CAST(7 AS bigint) AS constant, " + + "system.builtin.upper(event_name) AS upperEvent, " + + "CAST(7 AS bigint) + CAST(1 AS bigint) AS added, " + + "CAST(event_name AS varchar) AS castEvent, " + + "event_name IS NULL AS missingEvent " + + "FROM analytics.data.raw_events")); + } + + @Test + public void testExpandsVirtualSavedAndMaterializedRelationsWithDeclaredOutputs() + { + assertThat(compile("SELECT * FROM event_view").statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"eventTitle\" AS \"eventTitle\", \"constant\" AS \"constant\" " + + "FROM (SELECT system.builtin.upper(event_name) AS \"eventTitle\", CAST(7 AS bigint) AS \"constant\" " + + "FROM analytics.data.raw_events)")); + assertThat(compile("SELECT * FROM saved_view").statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"eventTitle\" AS \"eventTitle\" FROM (" + + "SELECT \"eventTitle\" AS \"eventTitle\" FROM (" + + "SELECT system.builtin.upper(event_name) AS \"eventTitle\", CAST(7 AS bigint) AS \"constant\" " + + "FROM analytics.data.raw_events))")); + assertThat(compile("SELECT * FROM daily_events").statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"day\" AS \"day\" FROM analytics.data.daily_events")); + } + + @Test + public void testPinsOnceAcrossRecursiveExpansionAndPreservesPlaceholderOrder() + { + AtomicInteger pins = new AtomicInteger(); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + }); + HogQlCompileEnvelope envelope = new HogQlCompileEnvelope( + "SELECT {first}, eventTitle, {second} FROM saved_view", + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of( + "first", new HogQlTypedValue("bigint", new HogQlTypedValue.NumberValue("1")), + "second", new HogQlTypedValue("bigint", new HogQlTypedValue.NumberValue("2"))), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(7)); + + HogQlCompilationResult result = compiler.compile(envelope, Optional.of(context)); + + assertThat(pins).hasValue(1); + assertThat(result.catalogGeneration()).hasValue(7); + assertThat(result.parameterNames()).containsExactly("first", "second"); + } + + @Test + public void testBoundsRecursiveExpressionExpansionAtOriginalLocation() + { + List expressions = new ArrayList<>(); + expressions.add(expression("branch0", new LiteralRecipe(new TypedLiteral("bigint", LiteralEncoding.INTEGER, "1")))); + for (int index = 1; index <= 14; index++) { + String previous = "branch" + (index - 1); + expressions.add(expression("branch" + index, new OperatorRecipe(SemanticOperator.ADD, List.of( + new FieldReferenceRecipe("events", previous), + new FieldReferenceRecipe("events", previous))))); + } + HogQlSemanticCatalogSnapshot expansiveSnapshot = new HogQlSemanticCatalogSnapshot( + SNAPSHOT.protocolVersion(), + SNAPSHOT.schemaVersion(), + SNAPSHOT.languageVersion(), + SNAPSHOT.catalog(), + SNAPSHOT.generation(), + SNAPSHOT.logicalTables(), + expressions, + List.of(), + List.of(), + List.of(), + List.of(), + List.of()); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compile("SELECT branch14 FROM events", expansiveSnapshot)); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_COMPILER_LIMIT_EXCEEDED.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, 8)); + assertThat(exception).hasMessage("line 1:8: HogQL semantic expansion exceeded node limit"); + } + + private HogQlCompilationResult compile(String query) + { + return compile(query, SNAPSHOT); + } + + private HogQlCompilationResult compile(String query, HogQlSemanticCatalogSnapshot snapshot) + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(snapshot)); + return compiler.compile(new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(7)), Optional.of(context)); + } + + private static HogQlSemanticCatalogSnapshot snapshot() + { + LogicalFieldDefinition event = new LogicalFieldDefinition("event", new PhysicalIdentifier("event_name", false), "varchar", LogicalType.STRING, false, true); + List expressions = List.of( + expression("constant", new LiteralRecipe(new TypedLiteral("bigint", LiteralEncoding.INTEGER, "7"))), + expression("upperEvent", new FunctionCallRecipe("hogUpper", List.of(new FieldReferenceRecipe("events", "event")))), + expression("added", new OperatorRecipe(SemanticOperator.ADD, List.of( + new FieldReferenceRecipe("events", "constant"), + new LiteralRecipe(new TypedLiteral("bigint", LiteralEncoding.INTEGER, "1"))))), + expression("castEvent", new CastRecipe(new FieldReferenceRecipe("events", "event"), "varchar")), + expression("missingEvent", new OperatorRecipe(SemanticOperator.IS_NULL, List.of(new FieldReferenceRecipe("events", "event"))))); + ReferencedField eventTitle = new ReferencedField("eventTitle", "varchar", LogicalType.STRING, false, true); + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 7, + List.of(new LogicalTableDefinition( + "events", + physicalName("raw_events"), + List.of(event), + List.of(), + List.of())), + expressions, + List.of(new VirtualTableDefinition( + "event_view", + new RelationReference(RelationKind.LOGICAL_TABLE, "events"), + List.of( + new VirtualProjection("eventTitle", "upperEvent", true), + new VirtualProjection("constant", "constant", true)))), + List.of(new SavedQueryReference( + "saved_view", + "saved-7", + new RelationReference(RelationKind.VIRTUAL_TABLE, "event_view"), + List.of(eventTitle))), + List.of(new MaterializedViewReference( + "daily_events", + physicalName("daily_events"), + List.of(new ReferencedField("day", "date", LogicalType.DATE, false, true)))), + List.of(new FunctionCapabilityDefinition( + "hogUpper", + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of( + new PhysicalIdentifier("system", false), + new PhysicalIdentifier("builtin", false), + new PhysicalIdentifier("upper", false)), + List.of(new FunctionSignature(List.of("varchar"), "varchar", false)), + true, + false, + false, + false, + false)), + List.of()); + } + + private static ExpressionFieldDefinition expression(String name, HogQlSemanticCatalogSnapshot.ExpressionRecipe recipe) + { + return new ExpressionFieldDefinition("events", name, "bigint", LogicalType.INTEGER, false, true, recipe); + } + + private static PhysicalQualifiedName physicalName(String table) + { + return new PhysicalQualifiedName(CATALOG, new PhysicalIdentifier("data", false), new PhysicalIdentifier(table, false)); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java index 872204832510..7ca98b667f89 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java @@ -27,12 +27,14 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; import io.trino.hogql.parser.HogQlLanguageVersion; import org.junit.jupiter.api.Test; @@ -168,6 +170,42 @@ public void testEnforcesExpressionRecipeDepthAndNodeLimits() .hasMessageContaining("node limit"); } + @ParameterizedTest(name = "{0} with {1} arguments") + @MethodSource("invalidOperatorArities") + public void testRejectsInvalidOperatorArity(SemanticOperator operator, int argumentCount) + { + List arguments = new ArrayList<>(); + for (int index = 0; index < argumentCount; index++) { + arguments.add(literal()); + } + + assertThatThrownBy(() -> semanticSnapshot( + List.of(new ExpressionFieldDefinition( + "events", + "derived", + "bigint", + LogicalType.INTEGER, + false, + true, + new OperatorRecipe(operator, arguments))), + List.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("operator arity"); + } + + private static Stream invalidOperatorArities() + { + return Stream.concat( + Stream.of(SemanticOperator.values()) + .filter(operator -> switch (operator) { + case NOT, NEGATE, IS_NULL, IS_NOT_NULL -> false; + default -> true; + }) + .map(operator -> Arguments.of(operator, 1)), + Stream.of(SemanticOperator.NOT, SemanticOperator.NEGATE, SemanticOperator.IS_NULL, SemanticOperator.IS_NOT_NULL) + .map(operator -> Arguments.of(operator, 2))); + } + private static Stream invalidSnapshots() { LogicalTableDefinition events = table("events"); diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 0a399654ceee..64895dd877ef 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -43,6 +43,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperationType; import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; import io.trino.hogql.parser.tree.HogQlQuery.SortItem; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; @@ -52,6 +54,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; import io.trino.hogql.parser.tree.HogQlSyntaxTree; import org.antlr.v4.runtime.ANTLRErrorListener; import org.antlr.v4.runtime.BailErrorStrategy; @@ -460,11 +463,243 @@ private AstBuilder(String source) public HogQlQuery build(HogQLParser.SelectContext context) { - HogQlQuery query = buildQuery(extractPlainSelect(context)); + HogQlQuery query; + if (context.selectStmt() != null) { + query = buildQuery(new SelectedQuery(context.selectStmt(), null, null)); + } + else if (context.selectSetStmt() != null) { + query = buildSetQuery(context.selectSetStmt()); + } + else { + throw unsupported(context, "query"); + } validateUncorrelatedSubqueries(query); return query; } + private HogQlQuery buildSetQuery(HogQLParser.SelectSetStmtContext context) + { + commonTableScopes.push(new LinkedHashSet<>()); + prohibitedCommonTableScopes.push(new LinkedHashSet<>()); + try { + QueryOperand first = buildSetOperand(context.selectStmtWithParens()); + if (context.subsequentSelectSetClause().isEmpty()) { + return attachSetLevelClauses(first.query(), context); + } + + List commonTables = List.of(); + if (!first.parenthesized() && !first.query().with().isEmpty()) { + commonTables = first.query().with(); + commonTables.stream() + .map(CommonTableExpression::name) + .map(HogQlParser.AstBuilder::canonicalName) + .forEach(commonTableScopes.getFirst()::add); + first = first.withQuery(copyQuery(List.of(), first.query().body(), first.query().orderBy(), first.query().limit(), first.query().offset(), first.query().span())); + } + + List operands = new ArrayList<>(); + List operators = new ArrayList<>(); + operands.add(first); + for (HogQLParser.SubsequentSelectSetClauseContext clause : context.subsequentSelectSetClause()) { + SetOperator operator = buildSetOperator(clause); + QueryOperand operand = buildSetOperand(clause.selectStmtWithParens()); + if (!operand.parenthesized() && !operand.query().with().isEmpty()) { + throw unsupported(clause.selectStmtWithParens(), "unparenthesized WITH set operand"); + } + operators.add(operator); + operands.add(operand); + } + + for (QueryOperand operand : operands.subList(0, operands.size() - 1)) { + if (!operand.parenthesized() && hasQueryClauses(operand.query())) { + throw unsupported(context, "unparenthesized set operand ORDER BY or pagination"); + } + } + List orderBy = buildOrderBy(context.orderByClause()); + Pagination pagination = context.limitAndOffsetClauseOptional() == null + ? new Pagination(Optional.empty(), Optional.empty()) + : buildPagination(context.limitAndOffsetClauseOptional()); + QueryOperand last = operands.getLast(); + if (!last.parenthesized() && hasQueryClauses(last.query())) { + if (!orderBy.isEmpty() && !last.query().orderBy().isEmpty()) { + throw unsupported(context.orderByClause(), "multiple ORDER BY clauses"); + } + orderBy = orderBy.isEmpty() ? last.query().orderBy() : orderBy; + pagination = mergePagination( + new Pagination(last.query().limit(), last.query().offset()), + pagination, + context); + operands.set( + operands.size() - 1, + last.withQuery(copyQuery( + last.query().with(), + last.query().body(), + List.of(), + Optional.empty(), + Optional.empty(), + last.query().span()))); + } + QueryOperand operation = applySetOperationPrecedence(operands, operators); + return copyQuery( + commonTables, + operation.query().body(), + orderBy, + pagination.limit(), + pagination.offset(), + sourceSpan(context)); + } + finally { + prohibitedCommonTableScopes.pop(); + commonTableScopes.pop(); + } + } + + private static boolean hasQueryClauses(HogQlQuery query) + { + return !query.orderBy().isEmpty() || query.limit().isPresent() || query.offset().isPresent(); + } + + private QueryOperand buildSetOperand(HogQLParser.SelectStmtWithParensContext context) + { + if (context.selectStmt() != null) { + return new QueryOperand(buildQuery(new SelectedQuery(context.selectStmt(), null, null)), false, sourceSpan(context)); + } + if (context.withClause() != null) { + throw unsupported(context, "non-standard WITH query wrapper"); + } + if (context.selectSetStmt() != null) { + return new QueryOperand(buildSetQuery(context.selectSetStmt()), true, sourceSpan(context)); + } + throw unsupported(context, "placeholder query operand"); + } + + private SetOperator buildSetOperator(HogQLParser.SubsequentSelectSetClauseContext context) + { + if (context.BY() != null) { + throw unsupported(context, "set operation BY NAME"); + } + TerminalNode operator; + SetOperationType type; + if (context.UNION() != null) { + operator = context.UNION(); + type = SetOperationType.UNION; + } + else if (context.INTERSECT() != null) { + operator = context.INTERSECT(); + type = SetOperationType.INTERSECT; + } + else { + operator = context.EXCEPT(); + type = SetOperationType.EXCEPT; + } + Token stop = context.ALL() != null + ? context.ALL().getSymbol() + : context.DISTINCT() != null ? context.DISTINCT().getSymbol() : operator.getSymbol(); + return new SetOperator(type, context.ALL() == null, sourceSpan(operator.getSymbol(), stop)); + } + + private QueryOperand applySetOperationPrecedence(List operands, List operators) + { + List unionAndExceptOperands = new ArrayList<>(); + List unionAndExceptOperators = new ArrayList<>(); + QueryOperand current = operands.getFirst(); + for (int index = 0; index < operators.size(); index++) { + SetOperator operator = operators.get(index); + QueryOperand right = operands.get(index + 1); + if (operator.type() == SetOperationType.INTERSECT) { + current = combineSetOperation(current, operator, right); + } + else { + unionAndExceptOperands.add(current); + unionAndExceptOperators.add(operator); + current = right; + } + } + unionAndExceptOperands.add(current); + + current = unionAndExceptOperands.getFirst(); + for (int index = 0; index < unionAndExceptOperators.size(); index++) { + current = combineSetOperation(current, unionAndExceptOperators.get(index), unionAndExceptOperands.get(index + 1)); + } + return current; + } + + private QueryOperand combineSetOperation(QueryOperand left, SetOperator operator, QueryOperand right) + { + SourceSpan span = enclosingSpan(left.span(), right.span()); + SetOperation body = new SetOperation( + operator.type(), + operator.distinct(), + left.query(), + right.query(), + left.parenthesized(), + right.parenthesized(), + operator.span(), + span); + HogQlQuery query = new HogQlQuery( + List.of(), + body, + List.of(), + Optional.empty(), + Optional.empty(), + span); + return new QueryOperand(query, false, span); + } + + private HogQlQuery attachSetLevelClauses(HogQlQuery query, HogQLParser.SelectSetStmtContext context) + { + List outerOrderBy = buildOrderBy(context.orderByClause()); + if (!outerOrderBy.isEmpty() && !query.orderBy().isEmpty()) { + throw unsupported(context.orderByClause(), "multiple ORDER BY clauses"); + } + Pagination outerPagination = context.limitAndOffsetClauseOptional() == null + ? new Pagination(Optional.empty(), Optional.empty()) + : buildPagination(context.limitAndOffsetClauseOptional()); + Pagination pagination = mergePagination( + new Pagination(query.limit(), query.offset()), + outerPagination, + context); + return copyQuery( + query.with(), + query.body(), + outerOrderBy.isEmpty() ? query.orderBy() : outerOrderBy, + pagination.limit(), + pagination.offset(), + sourceSpan(context)); + } + + private static HogQlQuery copyQuery( + List commonTables, + HogQlQuery.QueryBody body, + List orderBy, + Optional limit, + Optional offset, + SourceSpan span) + { + return new HogQlQuery(commonTables, body, orderBy, limit, offset, span); + } + + private static SourceSpan enclosingSpan(SourceSpan left, SourceSpan right) + { + return new SourceSpan( + left.startOffset(), + right.endOffset(), + left.startLine(), + left.startColumn(), + right.endLine(), + right.endColumn()); + } + + private record QueryOperand(HogQlQuery query, boolean parenthesized, SourceSpan span) + { + private QueryOperand withQuery(HogQlQuery query) + { + return new QueryOperand(query, parenthesized, span); + } + } + + private record SetOperator(SetOperationType type, boolean distinct, SourceSpan span) {} + private HogQlQuery buildQuery(SelectedQuery selectedQuery) { HogQLParser.SelectStmtContext select = selectedQuery.statement(); @@ -540,7 +775,7 @@ private List buildCommonTables(HogQLParser.WithClauseCont prohibitedNames.add(canonicalName); HogQlQuery query; try { - query = buildQuery(extractPlainSelect(subquery.selectSetStmt())); + query = buildSetQuery(subquery.selectSetStmt()); } finally { prohibitedNames.remove(canonicalName); @@ -567,40 +802,6 @@ private List selectColumns(HogQLParser.Sele throw unsupported(context, "select list"); } - private SelectedQuery extractPlainSelect(HogQLParser.SelectContext context) - { - if (context.selectStmt() != null) { - return new SelectedQuery(context.selectStmt(), null, null); - } - HogQLParser.SelectSetStmtContext set = context.selectSetStmt(); - if (set == null || !set.subsequentSelectSetClause().isEmpty()) { - throw unsupported(context, "set query"); - } - HogQLParser.SelectStmtWithParensContext wrapped = set.selectStmtWithParens(); - if (wrapped.selectStmt() == null) { - throw unsupported(wrapped, "parenthesized or placeholder query"); - } - return new SelectedQuery(wrapped.selectStmt(), set.orderByClause(), set.limitAndOffsetClauseOptional()); - } - - private SelectedQuery extractPlainSelect(HogQLParser.SelectSetStmtContext context) - { - if (!context.subsequentSelectSetClause().isEmpty()) { - throw unsupported(context, "set query"); - } - HogQLParser.SelectStmtWithParensContext wrapped = context.selectStmtWithParens(); - if (wrapped.selectStmt() != null) { - return new SelectedQuery(wrapped.selectStmt(), context.orderByClause(), context.limitAndOffsetClauseOptional()); - } - if (wrapped.withClause() != null) { - throw unsupported(wrapped, "non-standard WITH query wrapper"); - } - if (wrapped.selectSetStmt() != null && context.orderByClause() == null && context.limitAndOffsetClauseOptional() == null) { - return extractPlainSelect(wrapped.selectSetStmt()); - } - throw unsupported(wrapped, "parenthesized, placeholder, or decorated query"); - } - private record SelectedQuery( HogQLParser.SelectStmtContext statement, HogQLParser.OrderByClauseContext orderBy, @@ -1097,16 +1298,36 @@ private Relation buildTableExpression(HogQLParser.TableExprContext context) return new TablePlaceholder(buildPlaceholder(placeholder.placeholder())); } if (context instanceof HogQLParser.TableExprSubqueryContext subquery) { - return new SubqueryRelation(buildQuery(extractPlainSelect(subquery.selectSetStmt())), sourceSpan(context)); + return new SubqueryRelation(buildSetQuery(subquery.selectSetStmt()), sourceSpan(context)); } - if (context instanceof HogQLParser.TableExprAliasContext alias) { - if (alias.columnAliases() != null) { - throw unsupported(context, "table column aliases"); + if (context instanceof HogQLParser.TableExprValuesContext values) { + List> rows = values.valuesClause().valuesRow().stream() + .map(row -> row.columnExpr().stream() + .map(this::buildExpression) + .toList()) + .toList(); + int width = rows.getFirst().size(); + for (int index = 1; index < rows.size(); index++) { + if (rows.get(index).size() != width) { + throw unsupported(values.valuesClause().valuesRow(index), "VALUES rows with different column counts"); + } } + return new ValuesRelation(rows, sourceSpan(context)); + } + if (context instanceof HogQLParser.TableExprAliasContext alias) { + Relation relation = buildTableExpression(alias.tableExpr()); Identifier identifier = alias.identifier() != null ? buildIdentifier(alias.identifier()) : buildIdentifier(alias.alias().getText(), alias.alias()); - return new AliasedRelation(buildTableExpression(alias.tableExpr()), identifier, sourceSpan(context)); + List columnAliases = alias.columnAliases() == null + ? List.of() + : alias.columnAliases().identifier().stream() + .map(this::buildIdentifier) + .toList(); + if (relation instanceof ValuesRelation values && !columnAliases.isEmpty() && columnAliases.size() != values.columnCount()) { + throw unsupported(alias.columnAliases(), "VALUES column alias count"); + } + return new AliasedRelation(relation, identifier, columnAliases, sourceSpan(context)); } throw unsupported(context, "table expression"); } @@ -1147,6 +1368,16 @@ private void validateUncorrelatedSubqueries(HogQlQuery query) private void validateQueryScope(HogQlQuery query, Set forbiddenOuterRelations) { + query.with().forEach(commonTable -> validateQueryScope(commonTable.query(), forbiddenOuterRelations)); + if (query.body() instanceof SetOperation setOperation) { + query.orderBy().forEach(item -> validateExpressionScope(item.expression(), forbiddenOuterRelations, Set.of())); + query.limit().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, Set.of())); + query.offset().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, Set.of())); + validateQueryScope(setOperation.left(), forbiddenOuterRelations); + validateQueryScope(setOperation.right(), forbiddenOuterRelations); + return; + } + Set localRelations = relationNames(query.from()); query.projections().forEach(projection -> { if (projection instanceof ExpressionProjection expression) { @@ -1161,7 +1392,6 @@ private void validateQueryScope(HogQlQuery query, Set forbiddenOuterRela query.offset().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); query.from().ifPresent(relation -> validateRelationExpressionScopes(relation, forbiddenOuterRelations, localRelations)); - query.with().forEach(commonTable -> validateQueryScope(commonTable.query(), forbiddenOuterRelations)); Set nestedForbiddenRelations = new LinkedHashSet<>(forbiddenOuterRelations); nestedForbiddenRelations.addAll(localRelations); query.from().ifPresent(relation -> validateNestedRelationScopes(relation, Set.copyOf(nestedForbiddenRelations))); @@ -1180,7 +1410,7 @@ private void validateRelationExpressionScopes(Relation relation, Set for } }); } - case CommonTableReference _, SubqueryRelation _, TablePlaceholder _, TableReference _ -> {} + case CommonTableReference _, SubqueryRelation _, TablePlaceholder _, TableReference _, ValuesRelation _ -> {} } } @@ -1193,6 +1423,7 @@ private void validateNestedRelationScopes(Relation relation, Set forbidd validateNestedRelationScopes(join.right(), forbiddenOuterRelations); } case SubqueryRelation subquery -> validateQueryScope(subquery.query(), forbiddenOuterRelations); + case ValuesRelation values -> values.rows().forEach(row -> row.forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, Set.of()))); case CommonTableReference _, TablePlaceholder _, TableReference _ -> {} } } @@ -1213,7 +1444,7 @@ private void collectRelationNames(Relation relation, Set names) collectRelationNames(join.left(), names); collectRelationNames(join.right(), names); } - case SubqueryRelation _, TablePlaceholder _ -> {} + case SubqueryRelation _, TablePlaceholder _, ValuesRelation _ -> {} case TableReference table -> names.add(canonicalName(table.parts().getLast())); } } diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index 13d73f02c28a..205c191ef160 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -20,12 +20,7 @@ public record HogQlQuery( List with, - boolean distinct, - List projections, - Optional from, - Optional where, - List groupBy, - Optional having, + QueryBody body, List orderBy, Optional limit, Optional offset, @@ -34,17 +29,122 @@ public record HogQlQuery( public HogQlQuery { with = List.copyOf(requireNonNull(with, "with is null")); - projections = List.copyOf(requireNonNull(projections, "projections is null")); - from = requireNonNull(from, "from is null"); - where = requireNonNull(where, "where is null"); - groupBy = List.copyOf(requireNonNull(groupBy, "groupBy is null")); - having = requireNonNull(having, "having is null"); + body = requireNonNull(body, "body is null"); orderBy = List.copyOf(requireNonNull(orderBy, "orderBy is null")); limit = requireNonNull(limit, "limit is null"); offset = requireNonNull(offset, "offset is null"); span = requireNonNull(span, "span is null"); } + public HogQlQuery( + List with, + boolean distinct, + List projections, + Optional from, + Optional where, + List groupBy, + Optional having, + List orderBy, + Optional limit, + Optional offset, + SourceSpan span) + { + this(with, new SelectQueryBody(distinct, projections, from, where, groupBy, having, span), orderBy, limit, offset, span); + } + + public boolean distinct() + { + return selectBody().distinct(); + } + + public List projections() + { + return selectBody().projections(); + } + + public Optional from() + { + return selectBody().from(); + } + + public Optional where() + { + return selectBody().where(); + } + + public List groupBy() + { + return selectBody().groupBy(); + } + + public Optional having() + { + return selectBody().having(); + } + + private SelectQueryBody selectBody() + { + if (body instanceof SelectQueryBody select) { + return select; + } + throw new IllegalStateException("query body is not a SELECT"); + } + + public sealed interface QueryBody + permits SelectQueryBody, SetOperation + { + SourceSpan span(); + } + + public record SelectQueryBody( + boolean distinct, + List projections, + Optional from, + Optional where, + List groupBy, + Optional having, + SourceSpan span) + implements QueryBody + { + public SelectQueryBody + { + projections = List.copyOf(requireNonNull(projections, "projections is null")); + from = requireNonNull(from, "from is null"); + where = requireNonNull(where, "where is null"); + groupBy = List.copyOf(requireNonNull(groupBy, "groupBy is null")); + having = requireNonNull(having, "having is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record SetOperation( + SetOperationType type, + boolean distinct, + HogQlQuery left, + HogQlQuery right, + boolean leftParenthesized, + boolean rightParenthesized, + SourceSpan operatorSpan, + SourceSpan span) + implements QueryBody + { + public SetOperation + { + type = requireNonNull(type, "type is null"); + left = requireNonNull(left, "left is null"); + right = requireNonNull(right, "right is null"); + operatorSpan = requireNonNull(operatorSpan, "operatorSpan is null"); + span = requireNonNull(span, "span is null"); + } + } + + public enum SetOperationType + { + EXCEPT, + INTERSECT, + UNION, + } + public record CommonTableExpression(Identifier name, List columnAliases, HogQlQuery query, SourceSpan span) { public CommonTableExpression @@ -251,7 +351,7 @@ public record BinaryExpression(BinaryOperator operator, Expression left, Express } public record FunctionCall( - Identifier name, + List nameParts, List arguments, boolean distinct, List orderBy, @@ -261,12 +361,31 @@ public record FunctionCall( { public FunctionCall { - name = requireNonNull(name, "name is null"); + nameParts = List.copyOf(requireNonNull(nameParts, "nameParts is null")); + if (nameParts.isEmpty()) { + throw new IllegalArgumentException("nameParts is empty"); + } arguments = List.copyOf(requireNonNull(arguments, "arguments is null")); orderBy = List.copyOf(requireNonNull(orderBy, "orderBy is null")); filter = requireNonNull(filter, "filter is null"); span = requireNonNull(span, "span is null"); } + + public FunctionCall( + Identifier name, + List arguments, + boolean distinct, + List orderBy, + Optional filter, + SourceSpan span) + { + this(List.of(name), arguments, distinct, orderBy, filter, span); + } + + public Identifier name() + { + return nameParts.getLast(); + } } public enum UnaryOperator @@ -344,18 +463,25 @@ public sealed interface Relation JoinRelation, SubqueryRelation, TablePlaceholder, - TableReference + TableReference, + ValuesRelation { SourceSpan span(); } - public record AliasedRelation(Relation relation, Identifier alias, SourceSpan span) + public record AliasedRelation(Relation relation, Identifier alias, List columnAliases, SourceSpan span) implements Relation { + public AliasedRelation(Relation relation, Identifier alias, SourceSpan span) + { + this(relation, alias, List.of(), span); + } + public AliasedRelation { relation = requireNonNull(relation, "relation is null"); alias = requireNonNull(alias, "alias is null"); + columnAliases = List.copyOf(requireNonNull(columnAliases, "columnAliases is null")); span = requireNonNull(span, "span is null"); } } @@ -380,6 +506,30 @@ public record SubqueryRelation(HogQlQuery query, SourceSpan span) } } + public record ValuesRelation(List> rows, SourceSpan span) + implements Relation + { + public ValuesRelation + { + rows = requireNonNull(rows, "rows is null").stream() + .map(row -> List.copyOf(requireNonNull(row, "row is null"))) + .toList(); + if (rows.isEmpty()) { + throw new IllegalArgumentException("rows is empty"); + } + int columnCount = rows.getFirst().size(); + if (columnCount == 0 || rows.stream().anyMatch(row -> row.size() != columnCount)) { + throw new IllegalArgumentException("VALUES rows must have the same non-zero column count"); + } + span = requireNonNull(span, "span is null"); + } + + public int columnCount() + { + return rows.getFirst().size(); + } + } + public enum JoinType { CROSS, diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json index bea2b9f5ab1a..e400a1720a6a 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json @@ -138,6 +138,14 @@ "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, + { + "id": "alternative:columnAliases", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, { "id": "alternative:columnExpr:ColumnExprAnd", "loweringHandler": "TrinoAstFactory#createExpression", @@ -453,6 +461,14 @@ "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" ] }, + { + "id": "alternative:selectSetStmt", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, { "id": "alternative:selectStmtWithParens:direct", "loweringHandler": "TrinoAstFactory#createQuery", @@ -469,6 +485,14 @@ "TestHogQlCompiler#testLowersCtesAndDerivedTables" ] }, + { + "id": "alternative:subsequentSelectSetClause", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, { "id": "alternative:tableExpr:TableExprAlias", "loweringHandler": "TrinoAstFactory#createRelation", @@ -494,6 +518,14 @@ "TestHogQlCompiler#testLowersCtesAndDerivedTables" ] }, + { + "id": "alternative:tableExpr:TableExprValues", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, { "id": "alternative:tableIdentifier", "loweringHandler": "TrinoAstFactory#createIdentifier", @@ -502,6 +534,22 @@ "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" ] }, + { + "id": "alternative:valuesClause", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "alternative:valuesRow", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, { "id": "alternative:whereClause", "loweringHandler": "TrinoAstFactory#createStatement", @@ -550,6 +598,14 @@ "TestHogQlCompiler#testLowersAliasesAndStockJoins" ] }, + { + "id": "rule:columnAliases", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, { "id": "rule:columnExprList", "loweringHandler": "TrinoAstFactory#createExpression", @@ -632,6 +688,22 @@ "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" ] }, + { + "id": "rule:selectSetStmt", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "rule:subsequentSelectSetClause", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, { "id": "rule:tableIdentifier", "loweringHandler": "TrinoAstFactory#createIdentifier", @@ -640,6 +712,22 @@ "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" ] }, + { + "id": "rule:valuesClause", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "rule:valuesRow", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, { "id": "rule:whereClause", "loweringHandler": "TrinoAstFactory#createStatement", diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json index d358d1d7c7f7..b32dfaef6609 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json @@ -69,12 +69,12 @@ { "id": "alternative:columnAliases", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" ] }, { @@ -2115,12 +2115,12 @@ { "id": "alternative:selectSetStmt", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" ] }, { @@ -2401,12 +2401,12 @@ { "id": "alternative:subsequentSelectSetClause", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" ] }, { @@ -2512,12 +2512,12 @@ { "id": "alternative:tableExpr:TableExprValues", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" ] }, { @@ -2611,23 +2611,23 @@ { "id": "alternative:valuesClause", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" ] }, { "id": "alternative:valuesRow", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" ] }, { @@ -2886,12 +2886,12 @@ { "id": "rule:columnAliases", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" ] }, { @@ -3713,12 +3713,12 @@ { "id": "rule:selectSetStmt", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" ] }, { @@ -3823,12 +3823,12 @@ { "id": "rule:subsequentSelectSetClause", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" ] }, { @@ -3944,23 +3944,23 @@ { "id": "rule:valuesClause", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" ] }, { "id": "rule:valuesRow", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" ] }, { diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 78c08445f35d..fc9f2affccc3 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -25,8 +25,11 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinType; import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperationType; import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; import io.trino.hogql.parser.tree.HogQlSyntaxTree; import io.trino.hogql.parser.tree.HogQlSyntaxTree.Element; import io.trino.hogql.parser.tree.HogQlSyntaxTree.Node; @@ -53,7 +56,6 @@ public class TestHogQlParser "SELECT * FROM first ASOF JOIN second ON first.id = second.id", "SELECT * FROM first POSITIONAL JOIN second", "SELECT * FROM first, second", - "SELECT * FROM events AS e(id)", "SELECT count(*) OVER ()", "SELECT function(1)(value)", "DROP TABLE events", @@ -71,6 +73,12 @@ public class TestHogQlParser "WITH x AS (SELECT 1), x AS (SELECT 2) SELECT * FROM x", "SELECT * FROM events e JOIN (SELECT * FROM persons WHERE personId = e.person_id) p ON true", "SELECT * FROM events e JOIN (SELECT person_id) p ON true", + "SELECT 1 UNION BY NAME SELECT 1", + "SELECT 1 UNION ALL BY NAME SELECT 1", + "SELECT 1 INTERSECT BY NAME SELECT 1", + "SELECT 1 EXCEPT BY NAME SELECT 1", + "SELECT * FROM (VALUES (1, 2), (3)) AS data(first, second)", + "SELECT * FROM (VALUES (1, 2)) AS data(first)", }) public void testRejectsSyntaxWithoutAnAstMapping(String hogql) { @@ -168,6 +176,36 @@ public void testBuildsCteAndDerivedTableAstWithSourceSpans() assertThat(subquery.span()).isEqualTo(new HogQlQuery.SourceSpan(80, 120, 2, 18, 2, 58)); } + @Test + public void testBuildsSetOperationAstWithPrecedenceAndSourceSpans() + { + HogQlQuery query = parser.parseStatement("SELECT {left}\nUNION ALL\nSELECT {middle}\nINTERSECT\nSELECT {right}"); + + SetOperation union = (SetOperation) query.body(); + assertThat(union.type()).isEqualTo(SetOperationType.UNION); + assertThat(union.distinct()).isFalse(); + assertThat(union.operatorSpan()).isEqualTo(new HogQlQuery.SourceSpan(14, 23, 2, 1, 2, 10)); + SetOperation intersect = (SetOperation) union.right().body(); + assertThat(intersect.type()).isEqualTo(SetOperationType.INTERSECT); + assertThat(intersect.distinct()).isTrue(); + assertThat(intersect.operatorSpan()).isEqualTo(new HogQlQuery.SourceSpan(40, 49, 4, 1, 4, 10)); + } + + @Test + public void testBuildsValuesRelationWithColumnSchema() + { + HogQlQuery query = parser.parseStatement("SELECT *\nFROM (VALUES (1, {first}), (2, {second})) AS data(id, label)"); + + AliasedRelation alias = (AliasedRelation) query.from().orElseThrow(); + assertThat(alias.columnAliases()) + .extracting(HogQlQuery.Identifier::value) + .containsExactly("id", "label"); + ValuesRelation values = (ValuesRelation) alias.relation(); + assertThat(values.rows()).hasSize(2).allSatisfy(row -> assertThat(row).hasSize(2)); + assertThat(values.span().startLine()).isEqualTo(2); + assertThat(values.span().startColumn()).isEqualTo(6); + } + @Test public void testSourceSpansUseUnicodeCodePointOffsetsAndEndPositions() { From 7f0120ed10ed304db2bd367f58f3ed47d9080c4c Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 12:47:37 -0400 Subject: [PATCH 047/136] feat(hogql): enforce function capabilities --- .../trino/hogql/compiler/HogQlCompiler.java | 67 +++- .../hogql/compiler/HogQlFunctionResolver.java | 285 ++++++++++++++++ .../compiler/TestHogQlFunctionResolver.java | 313 ++++++++++++++++++ .../compiler/TestHogQlSemanticResolution.java | 23 +- 4 files changed, 684 insertions(+), 4 deletions(-) create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index 8a207431ee34..fc850b65576c 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -29,6 +29,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; @@ -210,15 +211,17 @@ private static ResolvedQuery resolveQuery( context.catalog(), requireNonNull(languageVersion, "languageVersion is null"), expectedCatalogGeneration)); - HogQlQuery resolved = HogQlSemanticResolver.resolve(pinned, query) + HogQlQuery functionsResolved = HogQlFunctionResolver.resolve(pinned, query); + HogQlQuery resolved = HogQlSemanticResolver.resolve(pinned, functionsResolved) .map(HogQlSemanticResolver.ResolvedQuery::query) - .orElse(query); + .orElse(functionsResolved); return new ResolvedQuery(resolved, OptionalLong.of(pinned.generation())); } private static boolean containsSemanticCandidate(HogQlQuery query) { - return query.with().stream().anyMatch(commonTable -> containsSemanticCandidate(commonTable.query())) || + return containsFunctionCall(query) || + query.with().stream().anyMatch(commonTable -> containsSemanticCandidate(commonTable.query())) || switch (query.body()) { case SelectQueryBody select -> select.from().map(HogQlCompiler::containsSemanticCandidate).orElse(false); case SetOperation setOperation -> containsSemanticCandidate(setOperation.left()) || containsSemanticCandidate(setOperation.right()); @@ -238,6 +241,64 @@ private static boolean containsSemanticCandidate(Relation relation) }; } + private static boolean containsFunctionCall(HogQlQuery query) + { + return query.with().stream().anyMatch(commonTable -> containsFunctionCall(commonTable.query())) || + switch (query.body()) { + case SelectQueryBody select -> select.projections().stream() + .filter(ExpressionProjection.class::isInstance) + .map(ExpressionProjection.class::cast) + .anyMatch(projection -> containsFunctionCall(projection.expression())) || + select.from().map(HogQlCompiler::containsFunctionCall).orElse(false) || + select.where().map(HogQlCompiler::containsFunctionCall).orElse(false) || + select.groupBy().stream().anyMatch(HogQlCompiler::containsFunctionCall) || + select.having().map(HogQlCompiler::containsFunctionCall).orElse(false); + case SetOperation set -> containsFunctionCall(set.left()) || containsFunctionCall(set.right()); + } || + query.orderBy().stream().anyMatch(sortItem -> containsFunctionCall(sortItem.expression())) || + query.limit().map(HogQlCompiler::containsFunctionCall).orElse(false) || + query.offset().map(HogQlCompiler::containsFunctionCall).orElse(false); + } + + private static boolean containsFunctionCall(Relation relation) + { + return switch (relation) { + case AliasedRelation alias -> containsFunctionCall(alias.relation()); + case CommonTableReference _ -> false; + case JoinRelation join -> containsFunctionCall(join.left()) || + containsFunctionCall(join.right()) || + join.criteria().filter(JoinOn.class::isInstance) + .map(JoinOn.class::cast) + .map(JoinOn::expression) + .map(HogQlCompiler::containsFunctionCall) + .orElse(false); + case SubqueryRelation subquery -> containsFunctionCall(subquery.query()); + case TablePlaceholder _, HogQlQuery.TableReference _ -> false; + case ValuesRelation values -> values.rows().stream() + .flatMap(List::stream) + .anyMatch(HogQlCompiler::containsFunctionCall); + }; + } + + private static boolean containsFunctionCall(Expression expression) + { + return switch (expression) { + case ArrayExpression array -> array.values().stream().anyMatch(HogQlCompiler::containsFunctionCall); + case BetweenExpression between -> containsFunctionCall(between.value()) || containsFunctionCall(between.min()) || containsFunctionCall(between.max()); + case BinaryExpression binary -> containsFunctionCall(binary.left()) || containsFunctionCall(binary.right()); + case CaseExpression caseExpression -> caseExpression.operand().map(HogQlCompiler::containsFunctionCall).orElse(false) || + caseExpression.whenClauses().stream().anyMatch(when -> containsFunctionCall(when.operand()) || containsFunctionCall(when.result())) || + caseExpression.defaultValue().map(HogQlCompiler::containsFunctionCall).orElse(false); + case CastExpression cast -> containsFunctionCall(cast.value()); + case ColumnReference _, Literal _, Placeholder _ -> false; + case FunctionCall _ -> true; + case InExpression in -> containsFunctionCall(in.value()) || in.values().stream().anyMatch(HogQlCompiler::containsFunctionCall); + case IsNullExpression isNull -> containsFunctionCall(isNull.value()); + case TupleExpression tuple -> tuple.values().stream().anyMatch(HogQlCompiler::containsFunctionCall); + case UnaryExpression unary -> containsFunctionCall(unary.operand()); + }; + } + private record ResolvedQuery(HogQlQuery query, OptionalLong catalogGeneration) { private ResolvedQuery diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java new file mode 100644 index 000000000000..6fac944f98f5 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -0,0 +1,285 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseWhen; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; +import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; +import io.trino.hogql.parser.tree.HogQlQuery.SortItem; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; +import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; + +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static java.util.Objects.requireNonNull; + +final class HogQlFunctionResolver +{ + private final Map functions; + + private HogQlFunctionResolver(PinnedSnapshot snapshot) + { + functions = snapshot.snapshot().functions().stream() + .collect(Collectors.toUnmodifiableMap(function -> canonical(function.name()), Function.identity())); + } + + public static HogQlQuery resolve(PinnedSnapshot snapshot, HogQlQuery query) + { + requireNonNull(snapshot, "snapshot is null"); + requireNonNull(query, "query is null"); + return new HogQlFunctionResolver(snapshot).resolveQuery(query); + } + + private HogQlQuery resolveQuery(HogQlQuery query) + { + List commonTables = query.with().stream() + .map(commonTable -> new CommonTableExpression( + commonTable.name(), + commonTable.columnAliases(), + resolveQuery(commonTable.query()), + commonTable.span())) + .toList(); + HogQlQuery.QueryBody body = switch (query.body()) { + case SelectQueryBody select -> new SelectQueryBody( + select.distinct(), + select.projections().stream().map(this::resolveProjection).toList(), + select.from().map(this::resolveRelation), + select.where().map(this::resolveExpression), + select.groupBy().stream().map(this::resolveExpression).toList(), + select.having().map(this::resolveExpression), + select.span()); + case SetOperation set -> new SetOperation( + set.type(), + set.distinct(), + resolveQuery(set.left()), + resolveQuery(set.right()), + set.leftParenthesized(), + set.rightParenthesized(), + set.operatorSpan(), + set.span()); + }; + return new HogQlQuery( + commonTables, + body, + resolveSortItems(query.orderBy()), + query.limit().map(this::resolveExpression), + query.offset().map(this::resolveExpression), + query.span()); + } + + private Projection resolveProjection(Projection projection) + { + return switch (projection) { + case ExpressionProjection expression -> new ExpressionProjection(resolveExpression(expression.expression()), expression.alias()); + case Star star -> star; + }; + } + + private Relation resolveRelation(Relation relation) + { + return switch (relation) { + case AliasedRelation alias -> new AliasedRelation(resolveRelation(alias.relation()), alias.alias(), alias.columnAliases(), alias.span()); + case CommonTableReference commonTable -> commonTable; + case JoinRelation join -> new JoinRelation( + join.type(), + resolveRelation(join.left()), + resolveRelation(join.right()), + join.criteria().map(criteria -> switch (criteria) { + case JoinOn on -> new JoinOn(resolveExpression(on.expression()), on.span()); + case JoinUsing using -> using; + }), + join.span()); + case SubqueryRelation subquery -> new SubqueryRelation(resolveQuery(subquery.query()), subquery.span()); + case TablePlaceholder placeholder -> placeholder; + case TableReference table -> table; + case ValuesRelation values -> new ValuesRelation( + values.rows().stream() + .map(row -> row.stream().map(this::resolveExpression).toList()) + .toList(), + values.span()); + }; + } + + private List resolveSortItems(List sortItems) + { + return sortItems.stream() + .map(sortItem -> new SortItem(resolveExpression(sortItem.expression()), sortItem.direction(), sortItem.nullPlacement(), sortItem.span())) + .toList(); + } + + private Expression resolveExpression(Expression expression) + { + return switch (expression) { + case ArrayExpression array -> new ArrayExpression(array.values().stream().map(this::resolveExpression).toList(), array.span()); + case BetweenExpression between -> new BetweenExpression( + resolveExpression(between.value()), + resolveExpression(between.min()), + resolveExpression(between.max()), + between.negated(), + between.predicateSpan(), + between.span()); + case BinaryExpression binary -> new BinaryExpression( + binary.operator(), + resolveExpression(binary.left()), + resolveExpression(binary.right()), + binary.span()); + case CaseExpression caseExpression -> new CaseExpression( + caseExpression.operand().map(this::resolveExpression), + caseExpression.whenClauses().stream() + .map(when -> new CaseWhen(resolveExpression(when.operand()), resolveExpression(when.result()), when.span())) + .toList(), + caseExpression.defaultValue().map(this::resolveExpression), + caseExpression.span()); + case CastExpression cast -> new CastExpression(resolveExpression(cast.value()), cast.type(), cast.safe(), cast.span()); + case ColumnReference reference -> reference; + case FunctionCall function -> resolveFunction(function, false); + case InExpression in -> new InExpression( + resolveExpression(in.value()), + in.values().stream().map(this::resolveExpression).toList(), + in.negated(), + in.predicateSpan(), + in.span()); + case IsNullExpression isNull -> new IsNullExpression( + resolveExpression(isNull.value()), + isNull.negated(), + isNull.predicateSpan(), + isNull.span()); + case Literal literal -> literal; + case Placeholder placeholder -> placeholder; + case TupleExpression tuple -> new TupleExpression(tuple.values().stream().map(this::resolveExpression).toList(), tuple.span()); + case UnaryExpression unary -> new UnaryExpression(unary.operator(), resolveExpression(unary.operand()), unary.span()); + }; + } + + private FunctionCall resolveFunction(FunctionCall function, boolean windowInvocation) + { + String name = function.name().value(); + FunctionCapabilityDefinition capability = functions.get(canonical(name)); + if (capability == null) { + throw resolutionError(function, "Unknown HogQL function: " + name); + } + if (capability.implementation() == FunctionImplementation.REWRITE) { + throw unsupportedError(function, "HogQL function " + name + " requires a compiler rewrite"); + } + if (capability.kind() == FunctionKind.TABLE) { + throw unsupportedError(function, "HogQL table function " + name + " cannot be used as an expression"); + } + if (windowInvocation && !capability.supportsWindow()) { + throw unsupportedError(function, "HogQL function " + name + " does not support OVER"); + } + if (!windowInvocation && capability.kind() == FunctionKind.WINDOW) { + throw unsupportedError(function, "HogQL window function " + name + " requires an OVER clause"); + } + if (!matchesArity(capability.signatures(), function.arguments().size())) { + throw resolutionError(function, "HogQL function " + name + " does not accept " + function.arguments().size() + " arguments"); + } + if (function.distinct() && !capability.supportsDistinct()) { + throw unsupportedError(function, "HogQL function " + name + " does not support DISTINCT"); + } + if (!function.orderBy().isEmpty() && !capability.supportsOrderBy()) { + throw unsupportedError(function, "HogQL function " + name + " does not support ORDER BY"); + } + if (function.filter().isPresent() && !capability.supportsFilter()) { + throw unsupportedError(function, "HogQL function " + name + " does not support FILTER"); + } + return new FunctionCall( + capability.trinoName().stream() + .map(identifier -> identifier(identifier, function.span())) + .toList(), + function.arguments().stream().map(this::resolveExpression).toList(), + function.distinct(), + resolveSortItems(function.orderBy()), + function.filter().map(this::resolveExpression), + function.span()); + } + + private static boolean matchesArity(List signatures, int arity) + { + return signatures.stream().anyMatch(signature -> signature.variadic() + ? arity >= Math.max(0, signature.argumentTypes().size() - 1) + : arity == signature.argumentTypes().size()); + } + + private static Identifier identifier(PhysicalIdentifier identifier, HogQlQuery.SourceSpan span) + { + return new Identifier(identifier.value(), identifier.delimited(), span); + } + + private static String canonical(String value) + { + return value.toLowerCase(Locale.ENGLISH); + } + + private static TrinoException resolutionError(FunctionCall function, String message) + { + return error(HOGQL_RESOLUTION_ERROR, function, message); + } + + private static TrinoException unsupportedError(FunctionCall function, String message) + { + return error(HOGQL_UNSUPPORTED_FEATURE, function, message); + } + + private static TrinoException error(HogQlErrorCode errorCode, FunctionCall function, String message) + { + return new TrinoException( + errorCode, + Optional.of(new Location(function.span().startLine(), function.span().startColumn())), + message, + null); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java new file mode 100644 index 000000000000..7050923b129c --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -0,0 +1,313 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.hogql.parser.HogQlParser; +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlFunctionResolver +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); + + private final HogQlParser parser = new HogQlParser(); + private final SqlParser sqlParser = new SqlParser(); + + @Test + public void testMapsStockAndCompatibilityFunctionsToQualifiedTrinoNames() + { + HogQlQuery query = resolve( + "SELECT hogUpper('one'), hogCompat('two')", + function("hogUpper", FunctionKind.SCALAR, FunctionImplementation.STOCK, List.of("system", "builtin", "upper"), signature(1), false, false, false, false), + function("hogCompat", FunctionKind.SCALAR, FunctionImplementation.UDF, List.of("ducklake", "compat", "hog_compat"), signature(1), false, false, false, false)); + + assertThat(TrinoAstFactory.createStatement(query, Map.of())).isEqualTo(sqlParser.createStatement( + "SELECT system.builtin.upper('one'), ducklake.compat.hog_compat('two')")); + } + + @Test + public void testCompilerPinsOnceAndResolvesUserFunctions() + { + AtomicInteger pins = new AtomicInteger(); + HogQlSemanticCatalogSnapshot snapshot = snapshot(List.of( + function("hogUpper", FunctionKind.SCALAR, FunctionImplementation.STOCK, List.of("system", "builtin", "upper"), signature(1), false, false, false, false))); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(snapshot); + }); + + HogQlCompilationResult result = new HogQlCompiler().compile(envelope("SELECT hogUpper('one')"), Optional.of(context)); + + assertThat(pins).hasValue(1); + assertThat(result.catalogGeneration()).hasValue(7); + assertThat(result.statement()).isEqualTo(sqlParser.createStatement("SELECT system.builtin.upper('one')")); + } + + @Test + public void testCompilerPreservesOrdinaryFunctionsWithoutCatalogContext() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope("SELECT ordinary(1)")); + + assertThat(result.catalogGeneration()).isEmpty(); + assertThat(result.statement()).isEqualTo(sqlParser.createStatement("SELECT ordinary(1)")); + } + + @ParameterizedTest + @MethodSource("nestedFunctionQueries") + public void testResolvesFunctionsThroughoutQueryTree(String query, String expected) + { + HogQlQuery resolved = resolve( + query, + function("hogUpper", FunctionKind.SCALAR, FunctionImplementation.STOCK, List.of("system", "builtin", "upper"), signature(1), false, false, false, false)); + + assertThat(TrinoAstFactory.createStatement(resolved, Map.of())).isEqualTo(sqlParser.createStatement(expected)); + } + + private static Stream nestedFunctionQueries() + { + return Stream.of( + Arguments.of( + "WITH cte AS (SELECT hogUpper('cte') AS value) " + + "SELECT hogUpper(value) FROM cte WHERE hogUpper(value) = 'x' " + + "GROUP BY hogUpper(value) HAVING hogUpper(value) = 'y' ORDER BY hogUpper(value)", + "WITH cte AS (SELECT system.builtin.upper('cte') AS value) " + + "SELECT system.builtin.upper(value) FROM cte WHERE system.builtin.upper(value) = 'x' " + + "GROUP BY system.builtin.upper(value) HAVING system.builtin.upper(value) = 'y' ORDER BY system.builtin.upper(value)"), + Arguments.of( + "SELECT hogUpper('left') UNION ALL SELECT hogUpper('right')", + "SELECT system.builtin.upper('left') UNION ALL SELECT system.builtin.upper('right')"), + Arguments.of( + "SELECT * FROM (VALUES (hogUpper('value')))", + "SELECT * FROM (VALUES (system.builtin.upper('value')))")); + } + + @Test + public void testAcceptsExactAndVariadicArities() + { + FunctionCapabilityDefinition exact = function("exact", FunctionKind.SCALAR, FunctionImplementation.STOCK, List.of("exact"), signature(2), false, false, false, false); + FunctionCapabilityDefinition variadic = function( + "variadic", + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of("variadic"), + new FunctionSignature(List.of("varchar", "varchar"), "varchar", true), + false, + false, + false, + false); + + resolve("SELECT exact('one', 'two'), variadic('one'), variadic('one', 'two', 'three')", exact, variadic); + } + + @Test + public void testRejectsCallsOutsideEveryDeclaredArity() + { + FunctionCapabilityDefinition function = function( + "exact", + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of("exact"), + List.of(signature(1), signature(3)), + false, + false, + false, + false); + + assertResolutionError("SELECT exact('one', 'two')", function, "HogQL function exact does not accept 2 arguments"); + } + + @ParameterizedTest + @MethodSource("unsupportedInvocationFeatures") + public void testRejectsUnsupportedInvocationFeatures( + String query, + FunctionCapabilityDefinition function, + String message) + { + assertUnsupportedError(query, function, message); + } + + private static Stream unsupportedInvocationFeatures() + { + return Stream.of( + Arguments.of( + "SELECT aggregate(DISTINCT value)", + function("aggregate", FunctionKind.AGGREGATE, FunctionImplementation.STOCK, List.of("aggregate"), signature(1), false, false, false, true), + "HogQL function aggregate does not support DISTINCT"), + Arguments.of( + "SELECT aggregate(value ORDER BY value)", + function("aggregate", FunctionKind.AGGREGATE, FunctionImplementation.STOCK, List.of("aggregate"), signature(1), true, false, false, true), + "HogQL function aggregate does not support ORDER BY"), + Arguments.of( + "SELECT aggregate(value) FILTER (WHERE true)", + function("aggregate", FunctionKind.AGGREGATE, FunctionImplementation.STOCK, List.of("aggregate"), signature(1), true, true, false, true), + "HogQL function aggregate does not support FILTER"), + Arguments.of( + "SELECT windowOnly(value)", + function("windowOnly", FunctionKind.WINDOW, FunctionImplementation.STOCK, List.of("window_only"), signature(1), false, false, false, true), + "HogQL window function windowOnly requires an OVER clause"), + Arguments.of( + "SELECT tableOnly(value)", + function("tableOnly", FunctionKind.TABLE, FunctionImplementation.STOCK, List.of("table_only"), signature(1), false, false, false, false), + "HogQL table function tableOnly cannot be used as an expression")); + } + + @Test + public void testRejectsUnknownAndRewriteOnlyFunctionsAtCallLocation() + { + assertResolutionError("SELECT missing('one')", "Unknown HogQL function: missing"); + + FunctionCapabilityDefinition rewrite = function( + "rewriteMe", + FunctionKind.SCALAR, + FunctionImplementation.REWRITE, + List.of(), + signature(1), + false, + false, + false, + false); + assertUnsupportedError("SELECT rewriteMe('one')", rewrite, "HogQL function rewriteMe requires a compiler rewrite"); + } + + private HogQlQuery resolve(String query, FunctionCapabilityDefinition... functions) + { + return HogQlFunctionResolver.resolve(new PinnedSnapshot(snapshot(List.of(functions))), parser.parseStatement(query)); + } + + private void assertResolutionError(String query, FunctionCapabilityDefinition function, String message) + { + TrinoException exception = catchThrowableOfType(TrinoException.class, () -> resolve(query, function)); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, 8)); + assertThat(exception).hasMessage("line 1:8: " + message); + } + + private void assertResolutionError(String query, String message) + { + TrinoException exception = catchThrowableOfType(TrinoException.class, () -> resolve(query)); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, 8)); + assertThat(exception).hasMessage("line 1:8: " + message); + } + + private void assertUnsupportedError(String query, FunctionCapabilityDefinition function, String message) + { + TrinoException exception = catchThrowableOfType(TrinoException.class, () -> resolve(query, function)); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, 8)); + assertThat(exception).hasMessage("line 1:8: " + message); + } + + private static FunctionCapabilityDefinition function( + String name, + FunctionKind kind, + FunctionImplementation implementation, + List trinoName, + FunctionSignature signature, + boolean supportsDistinct, + boolean supportsOrderBy, + boolean supportsFilter, + boolean supportsWindow) + { + return function(name, kind, implementation, trinoName, List.of(signature), supportsDistinct, supportsOrderBy, supportsFilter, supportsWindow); + } + + private static FunctionCapabilityDefinition function( + String name, + FunctionKind kind, + FunctionImplementation implementation, + List trinoName, + List signatures, + boolean supportsDistinct, + boolean supportsOrderBy, + boolean supportsFilter, + boolean supportsWindow) + { + return new FunctionCapabilityDefinition( + name, + kind, + implementation, + trinoName.stream().map(value -> new PhysicalIdentifier(value, false)).toList(), + signatures, + true, + supportsDistinct, + supportsOrderBy, + supportsFilter, + supportsWindow); + } + + private static FunctionSignature signature(int arity) + { + return new FunctionSignature(Stream.generate(() -> "varchar").limit(arity).toList(), "varchar", false); + } + + private static HogQlSemanticCatalogSnapshot snapshot(List functions) + { + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 7, + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + functions, + List.of()); + } + + private static HogQlCompileEnvelope envelope(String query) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.empty()); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java index 17e64f2efe9e..bf54e257cdd1 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -14,6 +14,10 @@ package io.trino.hogql.compiler; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; @@ -47,6 +51,7 @@ public class TestHogQlSemanticResolution { private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); private static final HogQlSemanticCatalogSnapshot SNAPSHOT = new HogQlSemanticCatalogSnapshot( + 1, 2, HogQlLanguageContract.current().languageVersion(), CATALOG, @@ -74,7 +79,23 @@ public class TestHogQlSemanticResolution new LogicalFieldDefinition("personId", new PhysicalIdentifier("person_id", false), "varchar", LogicalType.STRING, false, true), new LogicalFieldDefinition("name", new PhysicalIdentifier("full_name", false), "varchar", LogicalType.STRING, false, true)), List.of(), - List.of()))); + List.of())), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(new FunctionCapabilityDefinition( + "count", + FunctionKind.AGGREGATE, + FunctionImplementation.STOCK, + List.of(new PhysicalIdentifier("count", false)), + List.of(new FunctionSignature(List.of(), "bigint", false)), + true, + true, + true, + true, + true)), + List.of()); private final HogQlCompiler compiler = new HogQlCompiler(); private final SqlParser sqlParser = new SqlParser(); From 84b9d76206ab7b8da42d274f0ab8b197fb7a8013 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 12:54:36 -0400 Subject: [PATCH 048/136] feat(hogql): expose native explain plans --- .../io/trino/dispatcher/HogQlRequest.java | 35 ++++++++++- .../dispatcher/HogQlStatementResource.java | 9 +-- .../io/trino/execution/QueryPreparer.java | 22 ++++++- .../io/trino/execution/QuerySubmission.java | 31 ++++++++-- .../TestHogQlStatementResource.java | 61 ++++++++++++++++++- docs/src/main/sphinx/admin/hogql.md | 16 +++++ 6 files changed, 160 insertions(+), 14 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java index 2e5ecf1b0352..96af362b02ab 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java @@ -16,6 +16,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Utf8; import io.airlift.json.JsonCodec; +import io.trino.execution.QuerySubmission.HogQlExplain; import io.trino.hogql.compiler.HogQlCompileEnvelope; import io.trino.hogql.compiler.HogQlTypedValue; import io.trino.hogql.compiler.HogQlTypedValue.ArrayValue; @@ -26,12 +27,15 @@ import io.trino.hogql.compiler.HogQlTypedValue.StringValue; import io.trino.hogql.compiler.HogQlTypedValue.Value; import io.trino.hogql.parser.HogQlLanguageVersion; +import io.trino.sql.tree.ExplainFormat; +import io.trino.sql.tree.ExplainType; import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.OptionalLong; import java.util.Set; @@ -46,7 +50,8 @@ record HogQlRequest( Map variables, Map filters, Map modifiers, - OptionalLong catalogGeneration) + OptionalLong catalogGeneration, + Optional explain) { private static final JsonCodec JSON_CODEC = jsonCodec(JsonNode.class); private static final Set FIELDS = Set.of( @@ -57,8 +62,10 @@ record HogQlRequest( "variables", "filters", "modifiers", - "catalogGeneration"); + "catalogGeneration", + "explain"); private static final Set TYPED_VALUE_FIELDS = Set.of("type", "value"); + private static final Set EXPLAIN_FIELDS = Set.of("type", "format"); private static final int MAX_VALUE_DEPTH = 64; private static final int MAX_REQUEST_BYTES = 2 * 1024 * 1024; private static final int MAX_BINDINGS_PER_FIELD = 1_000; @@ -73,6 +80,7 @@ record HogQlRequest( filters = Map.copyOf(requireNonNull(filters, "filters is null")); modifiers = Map.copyOf(requireNonNull(modifiers, "modifiers is null")); catalogGeneration = requireNonNull(catalogGeneration, "catalogGeneration is null"); + explain = requireNonNull(explain, "explain is null"); if (parameters.size() + variables.size() + filters.size() + modifiers.size() > MAX_TOTAL_BINDINGS) { throw new IllegalArgumentException("HogQL request has too many bindings"); } @@ -98,7 +106,8 @@ static HogQlRequest fromJson(String requestBody) typedValues(root, "variables"), typedValues(root, "filters"), typedValues(root, "modifiers"), - catalogGeneration(root)); + catalogGeneration(root), + explain(root)); } HogQlCompileEnvelope toCompileEnvelope() @@ -136,6 +145,26 @@ private static OptionalLong catalogGeneration(JsonNode root) return OptionalLong.of(value.longValue()); } + private static Optional explain(JsonNode root) + { + JsonNode value = root.get("explain"); + if (value == null) { + return Optional.empty(); + } + if (!value.isObject()) { + throw new IllegalArgumentException("invalid HogQL explain request"); + } + rejectUnknownFields(value, EXPLAIN_FIELDS, "unknown HogQL explain field"); + try { + return Optional.of(new HogQlExplain( + ExplainType.Type.valueOf(requiredText(value, "type")), + ExplainFormat.Type.valueOf(requiredText(value, "format")))); + } + catch (RuntimeException _) { + throw new IllegalArgumentException("invalid HogQL explain request"); + } + } + private static Map typedValues(JsonNode root, String field) { JsonNode values = root.get(field); diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java index 610e8ddfb0df..7c49ca6eed53 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java @@ -14,7 +14,7 @@ package io.trino.dispatcher; import com.google.inject.Inject; -import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.execution.QuerySubmission; import io.trino.server.ExternalUriInfo; import io.trino.server.security.ResourceSecurity; import jakarta.servlet.http.HttpServletRequest; @@ -54,13 +54,14 @@ public Response postStatement( @Context HttpHeaders httpHeaders, @BeanParam ExternalUriInfo externalUriInfo) { - HogQlCompileEnvelope envelope; + QuerySubmission submission; try { - envelope = HogQlRequest.fromJson(requestBody).toCompileEnvelope(); + HogQlRequest request = HogQlRequest.fromJson(requestBody); + submission = hogQl(request.toCompileEnvelope(), request.explain()); } catch (RuntimeException _) { throw new BadRequestException("Invalid HogQL request"); } - return queuedStatementResource.postStatement(hogQl(envelope), servletRequest, httpHeaders, externalUriInfo); + return queuedStatementResource.postStatement(submission, servletRequest, httpHeaders, externalUriInfo); } } diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java index d998387999a5..93346e7fc0c6 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java @@ -30,8 +30,12 @@ import io.trino.sql.parser.SqlParser; import io.trino.sql.tree.Execute; import io.trino.sql.tree.ExecuteImmediate; +import io.trino.sql.tree.Explain; import io.trino.sql.tree.ExplainAnalyze; +import io.trino.sql.tree.ExplainFormat; +import io.trino.sql.tree.ExplainType; import io.trino.sql.tree.Expression; +import io.trino.sql.tree.NodeLocation; import io.trino.sql.tree.Statement; import java.util.List; @@ -119,9 +123,10 @@ public PreparedQuery prepareQuery(Session session, QuerySubmission submission) HogQlCompilationResult result = tracker.observe(COMPILATION, () -> hogQlCompiler .orElseThrow(() -> new TrinoException(NOT_SUPPORTED, "HogQL query submission is disabled")) .compile(envelope, semanticCatalogContext(session))); + Statement statement = explain(result.statement(), submission.hogQlExplain()); preparedQuery = tracker.observe(PARAMETER_BINDING, () -> prepareQuery( session, - result.statement(), + statement, Optional.of(hogQlParameterDecoder.decode(result, envelope.parameters())))); } catch (RuntimeException | Error failure) { @@ -134,6 +139,21 @@ public PreparedQuery prepareQuery(Session session, QuerySubmission submission) }; } + private static Statement explain(Statement statement, Optional explain) + { + if (explain.isEmpty()) { + return statement; + } + NodeLocation location = statement.getLocation().orElseThrow(); + QuerySubmission.HogQlExplain options = explain.orElseThrow(); + return new Explain( + location, + statement, + List.of( + new ExplainType(location, options.type()), + new ExplainFormat(location, options.format()))); + } + private Optional semanticCatalogContext(Session session) { if (hogQlSemanticCatalogSnapshotProvider.isEmpty() || session.getCatalog().isEmpty()) { diff --git a/core/trino-main/src/main/java/io/trino/execution/QuerySubmission.java b/core/trino-main/src/main/java/io/trino/execution/QuerySubmission.java index 42c7776101db..c44ab011e3d9 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QuerySubmission.java +++ b/core/trino-main/src/main/java/io/trino/execution/QuerySubmission.java @@ -14,6 +14,8 @@ package io.trino.execution; import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.sql.tree.ExplainFormat; +import io.trino.sql.tree.ExplainType; import java.util.Optional; @@ -21,15 +23,20 @@ import static io.trino.execution.QueryLanguage.TRINO; import static java.util.Objects.requireNonNull; -public record QuerySubmission(QueryLanguage language, String originalText, Optional hogQlEnvelope) +public record QuerySubmission( + QueryLanguage language, + String originalText, + Optional hogQlEnvelope, + Optional hogQlExplain) { public QuerySubmission { requireNonNull(language, "language is null"); requireNonNull(originalText, "originalText is null"); hogQlEnvelope = requireNonNull(hogQlEnvelope, "hogQlEnvelope is null"); - if (language == TRINO && hogQlEnvelope.isPresent()) { - throw new IllegalArgumentException("Trino submission has a HogQL envelope"); + hogQlExplain = requireNonNull(hogQlExplain, "hogQlExplain is null"); + if (language == TRINO && (hogQlEnvelope.isPresent() || hogQlExplain.isPresent())) { + throw new IllegalArgumentException("Trino submission has HogQL context"); } if (language == HOGQL && (hogQlEnvelope.isEmpty() || !hogQlEnvelope.orElseThrow().query().equals(originalText))) { throw new IllegalArgumentException("HogQL submission envelope does not match its query"); @@ -38,12 +45,26 @@ public record QuerySubmission(QueryLanguage language, String originalText, Optio public static QuerySubmission trino(String originalText) { - return new QuerySubmission(TRINO, originalText, Optional.empty()); + return new QuerySubmission(TRINO, originalText, Optional.empty(), Optional.empty()); } public static QuerySubmission hogQl(HogQlCompileEnvelope envelope) + { + return hogQl(envelope, Optional.empty()); + } + + public static QuerySubmission hogQl(HogQlCompileEnvelope envelope, Optional explain) { requireNonNull(envelope, "envelope is null"); - return new QuerySubmission(HOGQL, envelope.query(), Optional.of(envelope)); + return new QuerySubmission(HOGQL, envelope.query(), Optional.of(envelope), requireNonNull(explain, "explain is null")); + } + + public record HogQlExplain(ExplainType.Type type, ExplainFormat.Type format) + { + public HogQlExplain + { + requireNonNull(type, "type is null"); + requireNonNull(format, "format is null"); + } } } diff --git a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java index fe61699c478f..c4c247d522be 100644 --- a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java +++ b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Stream; @@ -59,6 +60,7 @@ class TestHogQlStatementResource { private static final HeaderName REQUEST_USER_HEADER = HeaderName.of(TRINO_HEADERS.requestUser()); + private static final HeaderName REQUEST_SESSION_HEADER = HeaderName.of(TRINO_HEADERS.requestSession()); private static final HeaderName CONTENT_TYPE_HEADER = HeaderName.of("Content-Type"); private static final JsonCodec QUERY_RESULTS_CODEC = new JsonCodecFactory(new JsonMapperProvider() .withModules(Set.of(new QueryDataJacksonModule())) @@ -154,6 +156,33 @@ public void testQueuedQueryCanBeCancelled() assertThat(response.getStatusCode()).isEqualTo(204); } + @Test + public void testQueryUsesNativeExecutionTimeout() + throws Exception + { + List results = runHogQlToCompletion( + hogQlRequest("SELECT count(*) FROM tpch.tiny.lineitem a CROSS JOIN tpch.tiny.lineitem b CROSS JOIN tpch.tiny.lineitem c"), + "query_max_execution_time=1ms"); + + assertThat(results.getLast().getError().getErrorName()) + .describedAs("query error: %s", results.getLast().getError()) + .isEqualTo("EXCEEDED_TIME_LIMIT"); + } + + @Test + public void testExplainUsesNativePlanner() + throws Exception + { + List results = runHogQlToCompletion(hogQlExplainRequest( + "SELECT nationkey FROM tpch.tiny.nation", + "DISTRIBUTED", + "TEXT")); + + assertThat(results.getLast().getError()).isNull(); + assertThat(rows(results)).singleElement().satisfies(row -> + assertThat(row).singleElement().asString().contains("TableScan")); + } + @Test public void testEndpointRequiresJsonContentType() { @@ -201,6 +230,14 @@ private static Stream invalidRequests() """ {"query":"SELECT '%s'","protocolVersion":1,"languageVersion":"%s","catalogGeneration":0} """.formatted(SECRET, languageVersion)), + Arguments.of("unknown explain field", + """ + {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","explain":{"type":"LOGICAL","format":"TEXT","unknown":"%s"}} + """.formatted(languageVersion, SECRET)), + Arguments.of("invalid explain type", + """ + {"query":"SELECT '%s'","protocolVersion":1,"languageVersion":"%s","explain":{"type":"UNKNOWN","format":"TEXT"}} + """.formatted(SECRET, languageVersion)), Arguments.of("too many parameter bindings", requestWithParameterCount(languageVersion, 1_001)), Arguments.of("too many total bindings", requestWithTotalBindingCount(languageVersion)), Arguments.of("request exceeds byte limit", oversizedRequest(languageVersion)), @@ -239,10 +276,20 @@ private static String oversizedRequest(String languageVersion) private static List runHogQlToCompletion(String request) { - return runToCompletion("/v1/hogql", request, true); + return runToCompletion("/v1/hogql", request, true, Optional.empty()); + } + + private static List runHogQlToCompletion(String request, String session) + { + return runToCompletion("/v1/hogql", request, true, Optional.of(session)); } private static List runToCompletion(String path, String body, boolean json) + { + return runToCompletion(path, body, json, Optional.empty()); + } + + private static List runToCompletion(String path, String body, boolean json, Optional session) { ImmutableList.Builder results = ImmutableList.builder(); Request.Builder request = preparePost() @@ -252,6 +299,7 @@ private static List runToCompletion(String path, String body, bool if (json) { request.setHeader(CONTENT_TYPE_HEADER, APPLICATION_JSON); } + session.ifPresent(value -> request.setHeader(REQUEST_SESSION_HEADER, value)); QueryResults current = client.execute(request.build(), createJsonResponseHandler(QUERY_RESULTS_CODEC)); results.add(current); @@ -314,6 +362,17 @@ private static String hogQlRequestWithTypedValues(String query) """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); } + private static String hogQlExplainRequest(String query, String type, String format) + { + return """ + {"query":%s,"protocolVersion":1,"languageVersion":"%s","explain":{"type":%s,"format":%s}} + """.formatted( + STRING_CODEC.toJson(query), + HogQlLanguageContract.current().languageVersion(), + STRING_CODEC.toJson(type), + STRING_CODEC.toJson(format)); + } + private static List> rows(List results) throws Exception { diff --git a/docs/src/main/sphinx/admin/hogql.md b/docs/src/main/sphinx/admin/hogql.md index deb43b60e918..9fddab880e7d 100644 --- a/docs/src/main/sphinx/admin/hogql.md +++ b/docs/src/main/sphinx/admin/hogql.md @@ -34,6 +34,22 @@ paging and cancellation. The coordinator rejects request bodies larger than 2 MiB and more than 1,000 total entries across `parameters`, `variables`, `filters`, and `modifiers`. +To return a native Trino plan instead of executing the query, add an `explain` +object: + +```json +{ + "query": "SELECT event FROM events", + "protocolVersion": 1, + "languageVersion": "1.0.0", + "explain": {"type": "DISTRIBUTED", "format": "TEXT"} +} +``` + +The supported plan types are `LOGICAL`, `DISTRIBUTED`, `VALIDATE`, and `IO`. +The supported formats are `TEXT`, `GRAPHVIZ`, and `JSON`. Execution timeouts +continue to use Trino session properties such as `query_max_execution_time`. + ## Semantic catalog manifests Logical tables, fields, quoted physical names, and star expansion use an From 13eb5e1717cc23f7322d10bb10e5c0adc9a3e747 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 13:08:21 -0400 Subject: [PATCH 049/136] feat(hogql): lower window semantics --- .../trino/hogql/compiler/HogQlCompiler.java | 40 ++++- .../hogql/compiler/HogQlFunctionResolver.java | 42 ++++- .../hogql/compiler/HogQlSemanticResolver.java | 41 +++++ .../trino/hogql/compiler/TrinoAstFactory.java | 80 +++++++++- .../compiler/TestHogQlFunctionResolver.java | 15 ++ .../compiler/TestHogQlSemanticResolution.java | 17 +++ .../compiler/TestHogQlWindowCompiler.java | 121 +++++++++++++++ .../io/trino/hogql/parser/HogQlParser.java | 136 ++++++++++++++++- .../trino/hogql/parser/tree/HogQlQuery.java | 142 ++++++++++++++++- .../1.0.0/trino-compatibility-overrides.json | 144 ++++++++++++++++++ .../language/1.0.0/trino-compatibility.json | 144 +++++++++--------- .../trino/hogql/parser/TestHogQlParser.java | 1 - .../hogql/parser/TestHogQlWindowParser.java | 118 ++++++++++++++ 13 files changed, 960 insertions(+), 81 deletions(-) create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlWindowCompiler.java create mode 100644 core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlWindowParser.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index fc850b65576c..ac37a1ea06cc 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -46,6 +46,10 @@ import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; import io.trino.spi.Location; import io.trino.spi.TrinoException; import io.trino.sql.tree.Statement; @@ -186,6 +190,7 @@ private static void collectPlaceholders(HogQlQuery query, List plac select.where().ifPresent(expression -> collectPlaceholders(expression, placeholders)); select.groupBy().forEach(expression -> collectPlaceholders(expression, placeholders)); select.having().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + select.windows().forEach(window -> collectPlaceholders(window.specification(), placeholders)); } case SetOperation setOperation -> { collectPlaceholders(setOperation.left(), placeholders); @@ -252,7 +257,8 @@ private static boolean containsFunctionCall(HogQlQuery query) select.from().map(HogQlCompiler::containsFunctionCall).orElse(false) || select.where().map(HogQlCompiler::containsFunctionCall).orElse(false) || select.groupBy().stream().anyMatch(HogQlCompiler::containsFunctionCall) || - select.having().map(HogQlCompiler::containsFunctionCall).orElse(false); + select.having().map(HogQlCompiler::containsFunctionCall).orElse(false) || + select.windows().stream().map(WindowDefinition::specification).anyMatch(HogQlCompiler::containsFunctionCall); case SetOperation set -> containsFunctionCall(set.left()) || containsFunctionCall(set.right()); } || query.orderBy().stream().anyMatch(sortItem -> containsFunctionCall(sortItem.expression())) || @@ -299,6 +305,22 @@ private static boolean containsFunctionCall(Expression expression) }; } + private static boolean containsFunctionCall(Window window) + { + return switch (window) { + case WindowReference _ -> false; + case WindowSpecification specification -> specification.partitionBy().stream().anyMatch(HogQlCompiler::containsFunctionCall) || + specification.orderBy().stream().anyMatch(sortItem -> containsFunctionCall(sortItem.expression())) || + specification.frame().map(frame -> containsFunctionCall(frame.start().value()) || + frame.end().map(bound -> containsFunctionCall(bound.value())).orElse(false)).orElse(false); + }; + } + + private static boolean containsFunctionCall(Optional expression) + { + return expression.map(HogQlCompiler::containsFunctionCall).orElse(false); + } + private record ResolvedQuery(HogQlQuery query, OptionalLong catalogGeneration) { private ResolvedQuery @@ -359,6 +381,7 @@ private static void collectPlaceholders(Expression expression, List function.arguments().forEach(argument -> collectPlaceholders(argument, placeholders)); function.orderBy().forEach(sortItem -> collectPlaceholders(sortItem.expression(), placeholders)); function.filter().ifPresent(filter -> collectPlaceholders(filter, placeholders)); + function.window().ifPresent(window -> collectPlaceholders(window, placeholders)); } case InExpression in -> { collectPlaceholders(in.value(), placeholders); @@ -372,6 +395,21 @@ private static void collectPlaceholders(Expression expression, List } } + private static void collectPlaceholders(Window window, List placeholders) + { + switch (window) { + case WindowReference _ -> {} + case WindowSpecification specification -> { + specification.partitionBy().forEach(expression -> collectPlaceholders(expression, placeholders)); + specification.orderBy().forEach(sortItem -> collectPlaceholders(sortItem.expression(), placeholders)); + specification.frame().ifPresent(frame -> { + frame.start().value().ifPresent(value -> collectPlaceholders(value, placeholders)); + frame.end().flatMap(HogQlQuery.FrameBound::value).ifPresent(value -> collectPlaceholders(value, placeholders)); + }); + } + } + } + private static void validateRelation(Relation relation) { switch (relation) { diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 6fac944f98f5..4247faeaab3a 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -53,6 +53,11 @@ import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; +import io.trino.hogql.parser.tree.HogQlQuery.WindowFrame; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; import io.trino.spi.Location; import io.trino.spi.TrinoException; @@ -101,6 +106,7 @@ private HogQlQuery resolveQuery(HogQlQuery query) select.where().map(this::resolveExpression), select.groupBy().stream().map(this::resolveExpression).toList(), select.having().map(this::resolveExpression), + select.windows().stream().map(this::resolveWindowDefinition).toList(), select.span()); case SetOperation set -> new SetOperation( set.type(), @@ -186,7 +192,7 @@ private Expression resolveExpression(Expression expression) caseExpression.span()); case CastExpression cast -> new CastExpression(resolveExpression(cast.value()), cast.type(), cast.safe(), cast.span()); case ColumnReference reference -> reference; - case FunctionCall function -> resolveFunction(function, false); + case FunctionCall function -> resolveFunction(function, function.window().isPresent()); case InExpression in -> new InExpression( resolveExpression(in.value()), in.values().stream().map(this::resolveExpression).toList(), @@ -244,9 +250,43 @@ private FunctionCall resolveFunction(FunctionCall function, boolean windowInvoca function.distinct(), resolveSortItems(function.orderBy()), function.filter().map(this::resolveExpression), + function.nullTreatment(), + function.window().map(this::resolveWindow), function.span()); } + private WindowDefinition resolveWindowDefinition(WindowDefinition definition) + { + return new WindowDefinition(definition.name(), (WindowSpecification) resolveWindow(definition.specification()), definition.span()); + } + + private Window resolveWindow(Window window) + { + return switch (window) { + case WindowReference reference -> reference; + case WindowSpecification specification -> new WindowSpecification( + specification.partitionBy().stream().map(this::resolveExpression).toList(), + resolveSortItems(specification.orderBy()), + specification.frame().map(this::resolveWindowFrame), + specification.span()); + }; + } + + private WindowFrame resolveWindowFrame(WindowFrame frame) + { + return new WindowFrame( + frame.type(), + new HogQlQuery.FrameBound( + frame.start().type(), + frame.start().value().map(this::resolveExpression), + frame.start().span()), + frame.end().map(bound -> new HogQlQuery.FrameBound( + bound.type(), + bound.value().map(this::resolveExpression), + bound.span())), + frame.span()); + } + private static boolean matchesArity(List signatures, int arity) { return signatures.stream().anyMatch(signature -> signature.variadic() diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index 2c33510fcaba..63fe183132c1 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -67,6 +67,11 @@ import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; +import io.trino.hogql.parser.tree.HogQlQuery.WindowFrame; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; import io.trino.spi.Location; import io.trino.spi.TrinoException; @@ -143,6 +148,7 @@ private HogQlQuery resolveNestedQuery(HogQlQuery query) select.where(), select.groupBy(), select.having(), + select.windows(), query.orderBy(), query.limit(), query.offset(), @@ -163,6 +169,7 @@ private HogQlQuery resolveQuery(HogQlQuery query, List co query.where().map(this::resolveExpression), query.groupBy().stream().map(this::resolveExpression).toList(), query.having().map(this::resolveExpression), + query.windows().stream().map(this::resolveWindowDefinition).toList(), resolveSortItems(query.orderBy()), query.limit().map(this::resolveExpression), query.offset().map(this::resolveExpression), @@ -500,6 +507,8 @@ private Expression resolveExpression(Expression expression) function.distinct(), resolveSortItems(function.orderBy()), function.filter().map(this::resolveExpression), + function.nullTreatment(), + function.window().map(this::resolveWindow), function.span()); case InExpression in -> new InExpression( resolveExpression(in.value()), @@ -519,6 +528,38 @@ private Expression resolveExpression(Expression expression) }; } + private WindowDefinition resolveWindowDefinition(WindowDefinition definition) + { + return new WindowDefinition(definition.name(), (WindowSpecification) resolveWindow(definition.specification()), definition.span()); + } + + private Window resolveWindow(Window window) + { + return switch (window) { + case WindowReference reference -> reference; + case WindowSpecification specification -> new WindowSpecification( + specification.partitionBy().stream().map(this::resolveExpression).toList(), + resolveSortItems(specification.orderBy()), + specification.frame().map(this::resolveWindowFrame), + specification.span()); + }; + } + + private WindowFrame resolveWindowFrame(WindowFrame frame) + { + return new WindowFrame( + frame.type(), + new HogQlQuery.FrameBound( + frame.start().type(), + frame.start().value().map(this::resolveExpression), + frame.start().span()), + frame.end().map(bound -> new HogQlQuery.FrameBound( + bound.type(), + bound.value().map(this::resolveExpression), + bound.span())), + frame.span()); + } + private Expression resolveColumn(ColumnReference reference) { List parts = reference.parts(); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 4cb61d76545c..69a2072c5b29 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -24,6 +24,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.CommonTableExpression; import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FrameBound; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.Identifier; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; @@ -45,6 +46,11 @@ import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; +import io.trino.hogql.parser.tree.HogQlQuery.WindowFrame; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; import io.trino.sql.parser.SqlParser; import io.trino.sql.tree.AllColumns; import io.trino.sql.tree.ArithmeticBinaryExpression; @@ -150,7 +156,9 @@ private static QuerySpecification createQuerySpecification( select.where().map(expression -> createExpression(expression, parameterIds)), createGroupBy(select.groupBy(), parameterIds), select.having().map(expression -> createExpression(expression, parameterIds)), - List.of(), + select.windows().stream() + .map(window -> createWindowDefinition(window, parameterIds)) + .toList(), createOrderBy(query, parameterIds), query.offset().map(offset -> new Offset(location(offset.span()), createExpression(offset, parameterIds))), query.limit().map(limit -> new Limit(location(limit.span()), createExpression(limit, parameterIds)))); @@ -306,11 +314,11 @@ private static Expression createFunctionCall(FunctionCall function, Map createWindow(window, parameterIds)), function.filter().map(filter -> createExpression(filter, parameterIds)), createOrderBy(function.orderBy(), parameterIds), function.distinct(), - Optional.empty(), + function.nullTreatment().map(_ -> io.trino.sql.tree.FunctionCall.NullTreatment.IGNORE), Optional.empty(), function.arguments().stream() .map(argument -> new CallArgument( @@ -320,6 +328,72 @@ private static Expression createFunctionCall(FunctionCall function, Map parameterIds) + { + return new io.trino.sql.tree.WindowDefinition( + location(definition.span()), + createIdentifier(definition.name()), + (io.trino.sql.tree.WindowSpecification) createWindow(definition.specification(), parameterIds)); + } + + private static io.trino.sql.tree.Window createWindow(Window window, Map parameterIds) + { + return switch (window) { + case WindowReference reference -> new io.trino.sql.tree.WindowReference( + location(reference.span()), + createIdentifier(reference.name())); + case WindowSpecification specification -> new io.trino.sql.tree.WindowSpecification( + location(specification.span()), + Optional.empty(), + specification.partitionBy().stream() + .map(expression -> createExpression(expression, parameterIds)) + .toList(), + createOrderBy(specification.orderBy(), parameterIds), + specification.frame().map(frame -> createWindowFrame(frame, parameterIds))); + }; + } + + private static io.trino.sql.tree.WindowFrame createWindowFrame( + WindowFrame frame, + Map parameterIds) + { + return new io.trino.sql.tree.WindowFrame( + location(frame.span()), + switch (frame.type()) { + case RANGE -> io.trino.sql.tree.WindowFrame.Type.RANGE; + case ROWS -> io.trino.sql.tree.WindowFrame.Type.ROWS; + }, + createFrameBound(frame.start(), parameterIds), + frame.end().map(bound -> createFrameBound(bound, parameterIds)), + List.of(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of(), + List.of()); + } + + private static io.trino.sql.tree.FrameBound createFrameBound( + FrameBound bound, + Map parameterIds) + { + io.trino.sql.tree.FrameBound.Type type = switch (bound.type()) { + case CURRENT_ROW -> io.trino.sql.tree.FrameBound.Type.CURRENT_ROW; + case FOLLOWING -> io.trino.sql.tree.FrameBound.Type.FOLLOWING; + case PRECEDING -> io.trino.sql.tree.FrameBound.Type.PRECEDING; + case UNBOUNDED_FOLLOWING -> io.trino.sql.tree.FrameBound.Type.UNBOUNDED_FOLLOWING; + case UNBOUNDED_PRECEDING -> io.trino.sql.tree.FrameBound.Type.UNBOUNDED_PRECEDING; + }; + return bound.value() + .map(value -> new io.trino.sql.tree.FrameBound( + location(bound.span()), + type, + createExpression(value, parameterIds))) + .orElseGet(() -> new io.trino.sql.tree.FrameBound(location(bound.span()), type)); + } + private static Expression createCaseExpression(CaseExpression caseExpression, Map parameterIds) { NodeLocation location = location(caseExpression.span()); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 7050923b129c..dc138be5530c 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -136,6 +136,17 @@ public void testAcceptsExactAndVariadicArities() resolve("SELECT exact('one', 'two'), variadic('one'), variadic('one', 'two', 'three')", exact, variadic); } + @Test + public void testMapsWindowFunctionsAndPreservesWindowSpecifications() + { + HogQlQuery query = resolve( + "SELECT hogRank(value) OVER (PARTITION BY team_id ORDER BY timestamp ROWS CURRENT ROW)", + function("hogRank", FunctionKind.WINDOW, FunctionImplementation.STOCK, List.of("analytics", "rank_value"), signature(1), false, false, false, true)); + + assertThat(TrinoAstFactory.createStatement(query, Map.of())).isEqualTo(sqlParser.createStatement( + "SELECT analytics.rank_value(value) OVER (PARTITION BY team_id ORDER BY timestamp ROWS CURRENT ROW)")); + } + @Test public void testRejectsCallsOutsideEveryDeclaredArity() { @@ -182,6 +193,10 @@ private static Stream unsupportedInvocationFeatures() "SELECT windowOnly(value)", function("windowOnly", FunctionKind.WINDOW, FunctionImplementation.STOCK, List.of("window_only"), signature(1), false, false, false, true), "HogQL window function windowOnly requires an OVER clause"), + Arguments.of( + "SELECT aggregate(value) OVER ()", + function("aggregate", FunctionKind.AGGREGATE, FunctionImplementation.STOCK, List.of("aggregate"), signature(1), false, false, false, false), + "HogQL function aggregate does not support OVER"), Arguments.of( "SELECT tableOnly(value)", function("tableOnly", FunctionKind.TABLE, FunctionImplementation.STOCK, List.of("table_only"), signature(1), false, false, false, false), diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java index bf54e257cdd1..54fb33abc746 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -133,6 +133,23 @@ public void testPinsAndResolvesLogicalTableFieldsAndStar() .containsExactly("event_name", "Person ID"); } + @Test + public void testResolvesLogicalFieldsInsideNamedWindows() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compile( + envelope( + "SELECT count(*) OVER recent FROM events " + + "WINDOW recent AS (PARTITION BY event ORDER BY personId ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT count(*) OVER recent FROM analytics.\"Hog Data\".\"raw-events\" " + + "WINDOW recent AS (PARTITION BY event_name ORDER BY \"Person ID\" ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)")); + } + @Test public void testCatalogIndependentAndPhysicalQueriesDoNotFetchSemanticMetadata() { diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlWindowCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlWindowCompiler.java new file mode 100644 index 000000000000..b38113040bcd --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlWindowCompiler.java @@ -0,0 +1,121 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.sql.SqlFormatter; +import io.trino.sql.parser.SqlParser; +import io.trino.sql.tree.Parameter; +import io.trino.sql.tree.Statement; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Deque; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlWindowCompiler +{ + private final HogQlCompiler compiler = new HogQlCompiler(); + private final SqlParser sqlParser = new SqlParser(); + + @ParameterizedTest + @ValueSource(strings = { + "SELECT row_number() OVER ()", + "SELECT sum(value) FILTER (WHERE enabled) OVER (PARTITION BY team_id ORDER BY timestamp DESC NULLS LAST)", + "SELECT avg(value) OVER (ORDER BY timestamp ROWS BETWEEN 2 PRECEDING AND CURRENT ROW)", + "SELECT first_value(value) OVER (ORDER BY timestamp RANGE UNBOUNDED PRECEDING)", + "SELECT row_number() OVER recent FROM events WINDOW recent AS (PARTITION BY team_id ORDER BY timestamp)", + "SELECT sum(value) OVER first_window, avg(value) OVER second_window FROM events " + + "WINDOW first_window AS (ROWS CURRENT ROW), second_window AS (RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)", + }) + public void testLowersWindowsToEquivalentStockTrinoAst(String hogql) + { + Statement statement = compiler.compile(hogql); + + assertThat(statement).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @ParameterizedTest + @ValueSource(strings = { + "ROWS UNBOUNDED PRECEDING", + "ROWS 2 PRECEDING", + "ROWS CURRENT ROW", + "ROWS 2 FOLLOWING", + "ROWS UNBOUNDED FOLLOWING", + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW", + "RANGE BETWEEN 2 PRECEDING AND 3 FOLLOWING", + }) + public void testLowersEveryCanonicalFrameBound(String frame) + { + String hogql = "SELECT sum(value) OVER (ORDER BY timestamp " + frame + ")"; + + assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(hogql)); + } + + @Test + public void testLowersCanonicalWindowNullTreatmentOrder() + { + assertThat(compiler.compile("SELECT first_value(value) OVER (ORDER BY timestamp) IGNORE NULLS")) + .isEqualTo(sqlParser.createStatement("SELECT first_value(value) IGNORE NULLS OVER (ORDER BY timestamp)")); + } + + @Test + public void testBindsPlaceholdersAcrossWindowComponents() + { + String hogql = "SELECT sum({value}) FILTER (WHERE {filter}) OVER " + + "(PARTITION BY {partition} ORDER BY {sort} ROWS BETWEEN {lower} PRECEDING AND {upper} FOLLOWING)"; + Map bindings = Map.of( + "value", typedValue("value"), + "filter", typedValue("filter"), + "partition", typedValue("partition"), + "sort", typedValue("sort"), + "lower", typedValue("lower"), + "upper", typedValue("upper")); + + HogQlCompilationResult result = compiler.compile(hogql, bindings); + + assertThat(result.parameterNames()).containsExactly("value", "filter", "partition", "sort", "lower", "upper"); + assertThat(parameters(result.statement())).extracting(Parameter::getId).containsExactly(0, 1, 2, 3, 4, 5); + } + + private static HogQlTypedValue typedValue(String value) + { + return new HogQlTypedValue("varchar", new StringValue(value)); + } + + private static List parameters(Statement statement) + { + List parameters = new ArrayList<>(); + Deque nodes = new ArrayDeque<>(); + nodes.add(statement); + while (!nodes.isEmpty()) { + io.trino.sql.tree.Node node = nodes.removeFirst(); + if (node instanceof Parameter parameter) { + parameters.add(parameter); + } + nodes.addAll(node.getChildren()); + } + return parameters.stream() + .sorted(Comparator.comparingInt(Parameter::getId)) + .toList(); + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 64895dd877ef..f1fa9ec6d703 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -29,6 +29,9 @@ import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.Expression; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FrameBound; +import io.trino.hogql.parser.tree.HogQlQuery.FrameBoundType; +import io.trino.hogql.parser.tree.HogQlQuery.FrameType; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.Identifier; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; @@ -40,6 +43,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; +import io.trino.hogql.parser.tree.HogQlQuery.NullTreatment; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; @@ -55,6 +59,11 @@ import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; +import io.trino.hogql.parser.tree.HogQlQuery.WindowFrame; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; import io.trino.hogql.parser.tree.HogQlSyntaxTree; import org.antlr.v4.runtime.ANTLRErrorListener; import org.antlr.v4.runtime.BailErrorStrategy; @@ -722,6 +731,7 @@ private HogQlQuery buildQuery(SelectedQuery selectedQuery) Optional having = Optional.ofNullable(select.havingClause()) .map(HogQLParser.HavingClauseContext::columnExpr) .map(this::buildExpression); + List windows = buildWindowDefinitions(select.windowClause()); List orderBy = buildOrderBy(selectedQuery.orderBy() != null ? selectedQuery.orderBy() : select.orderByClause()); Pagination pagination = mergePagination( buildPagination(select.limitAndOffsetClause(), select.offsetOnlyClause()), @@ -735,6 +745,7 @@ private HogQlQuery buildQuery(SelectedQuery selectedQuery) where, groupBy, having, + windows, orderBy, pagination.limit(), pagination.offset(), @@ -815,7 +826,6 @@ private void rejectUnsupportedClauses(HogQLParser.SelectStmtContext context) clauses.add(context.prewhereClause()); clauses.addAll(context.sampleClause()); clauses.add(context.qualifyClause()); - clauses.add(context.windowClause()); clauses.add(context.limitByClause()); clauses.add(context.settingsClause()); Optional firstClause = clauses.stream() @@ -1080,9 +1090,45 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) sourceSpan(between.NOT() == null ? between.BETWEEN().getSymbol() : between.NOT().getSymbol(), between.getStop()), sourceSpan(between)); } + if (context instanceof HogQLParser.ColumnExprIgnoreNullsContext ignoreNulls) { + Expression expression = buildExpression(ignoreNulls.columnExprValue()); + if (!(expression instanceof FunctionCall function) || function.window().isEmpty()) { + throw unsupported(ignoreNulls, "IGNORE NULLS outside window function"); + } + if (function.nullTreatment().isPresent()) { + throw unsupported(ignoreNulls, "duplicate window null treatment"); + } + return new FunctionCall( + function.nameParts(), + function.arguments(), + function.distinct(), + function.orderBy(), + function.filter(), + Optional.of(NullTreatment.IGNORE), + function.window(), + sourceSpan(ignoreNulls)); + } if (context instanceof HogQLParser.ColumnExprFunctionContext function) { return buildFunction(function); } + if (context instanceof HogQLParser.ColumnExprWinFunctionContext function) { + return buildWindowFunction( + function, + function.identifier(), + function.columnExprs, + function.columnArgList, + function.filterExpr, + buildWindowSpecification(function.windowExpr())); + } + if (context instanceof HogQLParser.ColumnExprWinFunctionTargetContext function) { + return buildWindowFunction( + function, + function.identifier(0), + function.columnExprs, + function.columnArgList, + function.filterExpr, + new WindowReference(buildIdentifier(function.identifier(1)), sourceSpan(function.identifier(1)))); + } throw unsupported(context, "expression " + context.getClass().getSimpleName()); } @@ -1161,6 +1207,94 @@ private FunctionCall buildFunction(HogQLParser.ColumnExprFunctionContext context sourceSpan(context)); } + private FunctionCall buildWindowFunction( + ParserRuleContext context, + HogQLParser.IdentifierContext name, + HogQLParser.ColumnExprListContext arguments, + HogQLParser.ColumnExprListContext parametricArguments, + HogQLParser.ColumnExprContext filter, + Window window) + { + if (parametricArguments != null) { + throw unsupported(context, "parametric window function"); + } + List functionArguments; + if (arguments == null) { + functionArguments = List.of(); + } + else if (arguments.columnExpr().size() == 1 && isUnqualifiedStar(arguments.columnExpr().getFirst())) { + functionArguments = List.of(); + } + else { + functionArguments = buildExpressions(arguments); + } + Optional functionFilter = Optional.ofNullable(filter).map(this::buildExpression); + return new FunctionCall( + List.of(buildIdentifier(name)), + functionArguments, + false, + List.of(), + functionFilter, + Optional.empty(), + Optional.of(window), + sourceSpan(context)); + } + + private List buildWindowDefinitions(HogQLParser.WindowClauseContext context) + { + if (context == null) { + return List.of(); + } + List definitions = new ArrayList<>(); + for (int index = 0; index < context.identifier().size(); index++) { + Identifier name = buildIdentifier(context.identifier(index)); + WindowSpecification specification = buildWindowSpecification(context.windowExpr(index)); + definitions.add(new WindowDefinition(name, specification, enclosingSpan(name.span(), specification.span()))); + } + return List.copyOf(definitions); + } + + private WindowSpecification buildWindowSpecification(HogQLParser.WindowExprContext context) + { + List partitionBy = context.winPartitionByClause() == null + ? List.of() + : buildExpressions(context.winPartitionByClause().columnExprList()); + List orderBy = context.winOrderByClause() == null + ? List.of() + : buildSortItems(context.winOrderByClause().orderExprList()); + Optional frame = Optional.ofNullable(context.winFrameClause()).map(this::buildWindowFrame); + return new WindowSpecification(partitionBy, orderBy, frame, sourceSpan(context)); + } + + private WindowFrame buildWindowFrame(HogQLParser.WinFrameClauseContext context) + { + FrameType type = context.ROWS() != null ? FrameType.ROWS : FrameType.RANGE; + if (context.winFrameExtend() instanceof HogQLParser.FrameStartContext start) { + return new WindowFrame(type, buildFrameBound(start.winFrameBound()), Optional.empty(), sourceSpan(context)); + } + HogQLParser.FrameBetweenContext between = (HogQLParser.FrameBetweenContext) context.winFrameExtend(); + return new WindowFrame( + type, + buildFrameBound(between.winFrameBound(0)), + Optional.of(buildFrameBound(between.winFrameBound(1))), + sourceSpan(context)); + } + + private FrameBound buildFrameBound(HogQLParser.WinFrameBoundContext context) + { + if (context.CURRENT() != null) { + return new FrameBound(FrameBoundType.CURRENT_ROW, Optional.empty(), sourceSpan(context)); + } + if (context.UNBOUNDED() != null) { + FrameBoundType type = context.PRECEDING() != null + ? FrameBoundType.UNBOUNDED_PRECEDING + : FrameBoundType.UNBOUNDED_FOLLOWING; + return new FrameBound(type, Optional.empty(), sourceSpan(context)); + } + FrameBoundType type = context.PRECEDING() != null ? FrameBoundType.PRECEDING : FrameBoundType.FOLLOWING; + return new FrameBound(type, Optional.of(buildExpression(context.columnExpr())), sourceSpan(context)); + } + private List buildFunctionArguments(HogQLParser.ColumnExprFunctionContext context) { if (context.columnArgList == null) { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index 205c191ef160..db95f1439e1b 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -44,12 +44,29 @@ public HogQlQuery( Optional where, List groupBy, Optional having, + List windows, List orderBy, Optional limit, Optional offset, SourceSpan span) { - this(with, new SelectQueryBody(distinct, projections, from, where, groupBy, having, span), orderBy, limit, offset, span); + this(with, new SelectQueryBody(distinct, projections, from, where, groupBy, having, windows, span), orderBy, limit, offset, span); + } + + public HogQlQuery( + List with, + boolean distinct, + List projections, + Optional from, + Optional where, + List groupBy, + Optional having, + List orderBy, + Optional limit, + Optional offset, + SourceSpan span) + { + this(with, distinct, projections, from, where, groupBy, having, List.of(), orderBy, limit, offset, span); } public boolean distinct() @@ -82,6 +99,11 @@ public Optional having() return selectBody().having(); } + public List windows() + { + return selectBody().windows(); + } + private SelectQueryBody selectBody() { if (body instanceof SelectQueryBody select) { @@ -103,6 +125,7 @@ public record SelectQueryBody( Optional where, List groupBy, Optional having, + List windows, SourceSpan span) implements QueryBody { @@ -113,8 +136,21 @@ public record SelectQueryBody( where = requireNonNull(where, "where is null"); groupBy = List.copyOf(requireNonNull(groupBy, "groupBy is null")); having = requireNonNull(having, "having is null"); + windows = List.copyOf(requireNonNull(windows, "windows is null")); span = requireNonNull(span, "span is null"); } + + public SelectQueryBody( + boolean distinct, + List projections, + Optional from, + Optional where, + List groupBy, + Optional having, + SourceSpan span) + { + this(distinct, projections, from, where, groupBy, having, List.of(), span); + } } public record SetOperation( @@ -356,6 +392,8 @@ public record FunctionCall( boolean distinct, List orderBy, Optional filter, + Optional nullTreatment, + Optional window, SourceSpan span) implements Expression { @@ -368,9 +406,22 @@ public record FunctionCall( arguments = List.copyOf(requireNonNull(arguments, "arguments is null")); orderBy = List.copyOf(requireNonNull(orderBy, "orderBy is null")); filter = requireNonNull(filter, "filter is null"); + nullTreatment = requireNonNull(nullTreatment, "nullTreatment is null"); + window = requireNonNull(window, "window is null"); span = requireNonNull(span, "span is null"); } + public FunctionCall( + List nameParts, + List arguments, + boolean distinct, + List orderBy, + Optional filter, + SourceSpan span) + { + this(nameParts, arguments, distinct, orderBy, filter, Optional.empty(), Optional.empty(), span); + } + public FunctionCall( Identifier name, List arguments, @@ -379,7 +430,7 @@ public FunctionCall( Optional filter, SourceSpan span) { - this(List.of(name), arguments, distinct, orderBy, filter, span); + this(List.of(name), arguments, distinct, orderBy, filter, Optional.empty(), Optional.empty(), span); } public Identifier name() @@ -388,6 +439,93 @@ public Identifier name() } } + public enum NullTreatment + { + IGNORE + } + + public record WindowDefinition(Identifier name, WindowSpecification specification, SourceSpan span) + { + public WindowDefinition + { + name = requireNonNull(name, "name is null"); + specification = requireNonNull(specification, "specification is null"); + span = requireNonNull(span, "span is null"); + } + } + + public sealed interface Window + permits WindowReference, WindowSpecification + { + SourceSpan span(); + } + + public record WindowReference(Identifier name, SourceSpan span) + implements Window + { + public WindowReference + { + name = requireNonNull(name, "name is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record WindowSpecification( + List partitionBy, + List orderBy, + Optional frame, + SourceSpan span) + implements Window + { + public WindowSpecification + { + partitionBy = List.copyOf(requireNonNull(partitionBy, "partitionBy is null")); + orderBy = List.copyOf(requireNonNull(orderBy, "orderBy is null")); + frame = requireNonNull(frame, "frame is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record WindowFrame(FrameType type, FrameBound start, Optional end, SourceSpan span) + { + public WindowFrame + { + type = requireNonNull(type, "type is null"); + start = requireNonNull(start, "start is null"); + end = requireNonNull(end, "end is null"); + span = requireNonNull(span, "span is null"); + } + } + + public enum FrameType + { + RANGE, + ROWS, + } + + public record FrameBound(FrameBoundType type, Optional value, SourceSpan span) + { + public FrameBound + { + type = requireNonNull(type, "type is null"); + value = requireNonNull(value, "value is null"); + span = requireNonNull(span, "span is null"); + boolean valueRequired = type == FrameBoundType.PRECEDING || type == FrameBoundType.FOLLOWING; + if (value.isPresent() != valueRequired) { + throw new IllegalArgumentException("window frame bound value does not match bound type"); + } + } + } + + public enum FrameBoundType + { + CURRENT_ROW, + FOLLOWING, + PRECEDING, + UNBOUNDED_FOLLOWING, + UNBOUNDED_PRECEDING, + } + public enum UnaryOperator { NEGATE, diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json index e400a1720a6a..53df67543603 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json @@ -759,6 +759,150 @@ "testCaseIds": [ "TestHogQlCompiler#testLowersCtesAndDerivedTables" ] + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunction", + "loweringHandler": "TrinoAstFactory#createFunctionCall", + "parseHandler": "AstBuilder#buildWindowFunction", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunctionTarget", + "loweringHandler": "TrinoAstFactory#createFunctionCall", + "parseHandler": "AstBuilder#buildWindowFunction", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:winFrameBound", + "loweringHandler": "TrinoAstFactory#createFrameBound", + "parseHandler": "AstBuilder#buildFrameBound", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "alternative:winFrameClause", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "alternative:winFrameExtend:frameBetween", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "alternative:winFrameExtend:frameStart", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "alternative:winOrderByClause", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:winPartitionByClause", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:windowClause", + "loweringHandler": "TrinoAstFactory#createWindowDefinition", + "parseHandler": "AstBuilder#buildWindowDefinitions", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:windowExpr", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:winFrameBound", + "loweringHandler": "TrinoAstFactory#createFrameBound", + "parseHandler": "AstBuilder#buildFrameBound", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "rule:winFrameClause", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "rule:winFrameExtend", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "rule:winOrderByClause", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:winPartitionByClause", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:windowClause", + "loweringHandler": "TrinoAstFactory#createWindowDefinition", + "parseHandler": "AstBuilder#buildWindowDefinitions", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:windowExpr", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIgnoreNulls", + "loweringHandler": "TrinoAstFactory#createFunctionCall", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersCanonicalWindowNullTreatmentOrder" + ] } ], "schemaVersion": 1 diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json index b32dfaef6609..287c025d8e11 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json @@ -449,12 +449,12 @@ { "id": "alternative:columnExprValue:ColumnExprIgnoreNulls", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createFunctionCall", + "parseHandler": "AstBuilder#buildExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersCanonicalWindowNullTreatmentOrder" ] }, { @@ -816,23 +816,23 @@ { "id": "alternative:columnExprValue:ColumnExprWinFunction", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createFunctionCall", + "parseHandler": "AstBuilder#buildWindowFunction", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" ] }, { "id": "alternative:columnExprValue:ColumnExprWinFunctionTarget", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createFunctionCall", + "parseHandler": "AstBuilder#buildWindowFunction", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" ] }, { @@ -2677,89 +2677,89 @@ { "id": "alternative:winFrameBound", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createFrameBound", + "parseHandler": "AstBuilder#buildFrameBound", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" ] }, { "id": "alternative:winFrameClause", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" ] }, { "id": "alternative:winFrameExtend:frameBetween", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" ] }, { "id": "alternative:winFrameExtend:frameStart", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" ] }, { "id": "alternative:winOrderByClause", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" ] }, { "id": "alternative:winPartitionByClause", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" ] }, { "id": "alternative:windowClause", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWindowDefinition", + "parseHandler": "AstBuilder#buildWindowDefinitions", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" ] }, { "id": "alternative:windowExpr", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" ] }, { @@ -4010,78 +4010,78 @@ { "id": "rule:winFrameBound", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createFrameBound", + "parseHandler": "AstBuilder#buildFrameBound", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" ] }, { "id": "rule:winFrameClause", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" ] }, { "id": "rule:winFrameExtend", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" ] }, { "id": "rule:winOrderByClause", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" ] }, { "id": "rule:winPartitionByClause", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" ] }, { "id": "rule:windowClause", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWindowDefinition", + "parseHandler": "AstBuilder#buildWindowDefinitions", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" ] }, { "id": "rule:windowExpr", "kind": "parserRule", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" ] }, { diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index fc9f2affccc3..38f92ca78115 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -56,7 +56,6 @@ public class TestHogQlParser "SELECT * FROM first ASOF JOIN second ON first.id = second.id", "SELECT * FROM first POSITIONAL JOIN second", "SELECT * FROM first, second", - "SELECT count(*) OVER ()", "SELECT function(1)(value)", "DROP TABLE events", "SELECT 1; SELECT 2", diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlWindowParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlWindowParser.java new file mode 100644 index 000000000000..72d81f64ef18 --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlWindowParser.java @@ -0,0 +1,118 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FrameBoundType; +import io.trino.hogql.parser.tree.HogQlQuery.FrameType; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.NullTreatment; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlWindowParser +{ + private final HogQlParser parser = new HogQlParser(); + + @Test + public void testBuildsInlineWindowSpecificationAndFrame() + { + String hogql = "SELECT sum({value}) FILTER (WHERE {filter}) OVER " + + "(PARTITION BY {partition}, team_id ORDER BY {sort} DESC NULLS LAST " + + "ROWS BETWEEN {preceding} PRECEDING AND CURRENT ROW)"; + + HogQlQuery query = parser.parseStatement(hogql); + FunctionCall function = (FunctionCall) ((ExpressionProjection) query.projections().getFirst()).expression(); + + assertThat(function.name().value()).isEqualTo("sum"); + assertThat(function.arguments()).singleElement().isInstanceOf(Placeholder.class); + assertThat(function.filter()).get().isInstanceOf(Placeholder.class); + WindowSpecification window = (WindowSpecification) function.window().orElseThrow(); + assertThat(window.partitionBy()).hasSize(2); + assertThat(window.orderBy()).singleElement().satisfies(sortItem -> { + assertThat(sortItem.expression()).isInstanceOf(Placeholder.class); + assertThat(sortItem.direction()).isEqualTo(SortDirection.DESCENDING); + }); + assertThat(window.span().startOffset()).isEqualTo(hogql.indexOf("PARTITION")); + assertThat(window.span().endOffset()).isEqualTo(hogql.length() - 1); + assertThat(window.frame()).get().satisfies(frame -> { + assertThat(frame.type()).isEqualTo(FrameType.ROWS); + assertThat(frame.start().type()).isEqualTo(FrameBoundType.PRECEDING); + assertThat(frame.start().value()).get().isInstanceOf(Placeholder.class); + assertThat(frame.end()).get().extracting(HogQlQuery.FrameBound::type).isEqualTo(FrameBoundType.CURRENT_ROW); + }); + } + + @Test + public void testBuildsNamedWindowReferenceAndDefinitions() + { + HogQlQuery query = parser.parseStatement( + "SELECT row_number() OVER recent " + + "WINDOW recent AS (PARTITION BY team_id ORDER BY timestamp), " + + "trailing AS (ORDER BY timestamp RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)"); + + FunctionCall function = (FunctionCall) ((ExpressionProjection) query.projections().getFirst()).expression(); + assertThat(function.window()).get().isInstanceOf(WindowReference.class); + WindowReference reference = (WindowReference) function.window().orElseThrow(); + assertThat(reference.name().value()).isEqualTo("recent"); + assertThat(query.windows()).hasSize(2); + assertThat(query.windows()).extracting(definition -> definition.name().value()) + .containsExactly("recent", "trailing"); + assertThat(query.windows().getFirst().specification().partitionBy()).hasSize(1); + assertThat(query.windows().get(1).specification().frame()).get().satisfies(frame -> { + assertThat(frame.type()).isEqualTo(FrameType.RANGE); + assertThat(frame.start().type()).isEqualTo(FrameBoundType.CURRENT_ROW); + assertThat(frame.end()).get().extracting(HogQlQuery.FrameBound::type).isEqualTo(FrameBoundType.UNBOUNDED_FOLLOWING); + }); + } + + @Test + public void testBuildsCanonicalWindowNullTreatment() + { + HogQlQuery query = parser.parseStatement("SELECT first_value(value) OVER (ORDER BY timestamp) IGNORE NULLS"); + + FunctionCall function = (FunctionCall) ((ExpressionProjection) query.projections().getFirst()).expression(); + assertThat(function.nullTreatment()).contains(NullTreatment.IGNORE); + assertThat(function.window()).get().isInstanceOf(WindowSpecification.class); + } + + @ParameterizedTest + @ValueSource(strings = { + "SELECT quantile(0.9)(value) OVER ()", + "SELECT count()(DISTINCT value) OVER named WINDOW named AS ()", + }) + public void testRejectsParametricWindowFunctions(String hogql) + { + assertThatThrownBy(() -> parser.parseStatement(hogql)) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("parametric window function"); + } + + @Test + public void testRejectsNullTreatmentOutsideWindowFunction() + { + assertThatThrownBy(() -> parser.parseStatement("SELECT first_value(value) IGNORE NULLS")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("IGNORE NULLS outside window function"); + } +} From 4ea65e57e178607172f3fd4f459b1a13d66b370a Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 13:13:19 -0400 Subject: [PATCH 050/136] feat(hogql): bound compiler concurrency --- .../execution/HogQlCompilationExecutor.java | 129 ++++++++++++ .../io/trino/execution/QueryPreparer.java | 34 +++- .../main/java/io/trino/hogql/HogQlConfig.java | 31 +++ .../hogql/HogQlCoordinatorErrorCode.java | 40 ++++ .../io/trino/server/CoordinatorModule.java | 2 + .../TestHogQlCompilationExecutor.java | 186 ++++++++++++++++++ .../java/io/trino/hogql/TestHogQlConfig.java | 14 +- docs/src/main/sphinx/admin/hogql.md | 16 ++ 8 files changed, 441 insertions(+), 11 deletions(-) create mode 100644 core/trino-main/src/main/java/io/trino/execution/HogQlCompilationExecutor.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlCoordinatorErrorCode.java create mode 100644 core/trino-main/src/test/java/io/trino/execution/TestHogQlCompilationExecutor.java diff --git a/core/trino-main/src/main/java/io/trino/execution/HogQlCompilationExecutor.java b/core/trino-main/src/main/java/io/trino/execution/HogQlCompilationExecutor.java new file mode 100644 index 000000000000..7b10681922d2 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/execution/HogQlCompilationExecutor.java @@ -0,0 +1,129 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +import com.google.inject.Inject; +import io.trino.hogql.HogQlConfig; +import io.trino.spi.TrinoException; +import jakarta.annotation.PreDestroy; + +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.FutureTask; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.function.Supplier; + +import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static io.trino.hogql.HogQlCoordinatorErrorCode.HOGQL_COMPILATION_QUEUE_FULL; +import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +public final class HogQlCompilationExecutor +{ + private final Executor executor; + private final Optional ownedExecutor; + private final Set> tasks = ConcurrentHashMap.newKeySet(); + + @Inject + public HogQlCompilationExecutor(HogQlConfig config) + { + requireNonNull(config, "config is null"); + BlockingQueue queue = config.getCompilationQueueCapacity() == 0 + ? new SynchronousQueue<>() + : new ArrayBlockingQueue<>(config.getCompilationQueueCapacity()); + ThreadPoolExecutor executor = new ThreadPoolExecutor( + config.getCompilationThreads(), + config.getCompilationThreads(), + 0, + MILLISECONDS, + queue, + daemonThreadsNamed("hogql-compilation-%s"), + new ThreadPoolExecutor.AbortPolicy()); + this.executor = executor; + this.ownedExecutor = Optional.of(executor); + } + + private HogQlCompilationExecutor(Executor executor) + { + this.executor = requireNonNull(executor, "executor is null"); + ownedExecutor = Optional.empty(); + } + + static HogQlCompilationExecutor directExecutor() + { + return new HogQlCompilationExecutor(Runnable::run); + } + + public T execute(Supplier compilation) + { + requireNonNull(compilation, "compilation is null"); + FutureTask task = new FutureTask<>(compilation::get); + tasks.add(task); + try { + try { + executor.execute(task); + } + catch (java.util.concurrent.RejectedExecutionException failure) { + throw new TrinoException(HOGQL_COMPILATION_QUEUE_FULL, "HogQL compilation capacity is exhausted; retry later", failure); + } + + try { + return task.get(); + } + catch (InterruptedException failure) { + task.cancel(true); + Thread.currentThread().interrupt(); + throw new TrinoException(GENERIC_INTERNAL_ERROR, "HogQL compilation was interrupted", failure); + } + catch (CancellationException failure) { + throw new TrinoException(HOGQL_COMPILATION_QUEUE_FULL, "HogQL compilation capacity is unavailable; retry later", failure); + } + catch (ExecutionException failure) { + Throwable cause = failure.getCause(); + if (cause instanceof RuntimeException runtimeException) { + throw runtimeException; + } + if (cause instanceof Error error) { + throw error; + } + throw new TrinoException(GENERIC_INTERNAL_ERROR, "HogQL compilation failed", cause); + } + } + finally { + tasks.remove(task); + } + } + + @PreDestroy + public void shutdown() + { + ownedExecutor.ifPresent(executor -> { + executor.shutdownNow(); + tasks.forEach(task -> task.cancel(true)); + }); + } + + boolean isShutdown() + { + return ownedExecutor.map(ThreadPoolExecutor::isShutdown).orElse(false); + } +} diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java index 93346e7fc0c6..9a8a40339345 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java @@ -61,23 +61,23 @@ public class QueryPreparer private final HogQlParameterDecoder hogQlParameterDecoder; private final HogQlCompilationObserver hogQlCompilationObserver; private final Optional hogQlSemanticCatalogSnapshotProvider; + private final HogQlCompilationExecutor hogQlCompilationExecutor; public QueryPreparer(SqlParser sqlParser) { - this(sqlParser, Optional.empty(), NOOP, Optional.empty()); + this(sqlParser, Optional.empty(), NOOP, Optional.empty(), HogQlCompilationExecutor.directExecutor()); } public QueryPreparer(SqlParser sqlParser, HogQlCompiler hogQlCompiler) { - this(sqlParser, Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), NOOP, Optional.empty()); + this(sqlParser, Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), NOOP, Optional.empty(), HogQlCompilationExecutor.directExecutor()); } public QueryPreparer(SqlParser sqlParser, HogQlCompiler hogQlCompiler, HogQlCompilationObserver hogQlCompilationObserver) { - this(sqlParser, Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), hogQlCompilationObserver, Optional.empty()); + this(sqlParser, Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), hogQlCompilationObserver, Optional.empty(), HogQlCompilationExecutor.directExecutor()); } - @Inject public QueryPreparer( SqlParser sqlParser, HogQlCompiler hogQlCompiler, @@ -87,20 +87,38 @@ public QueryPreparer( this(sqlParser, Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), hogQlCompilationObserver, - hogQlSemanticCatalogSnapshotProvider); + hogQlSemanticCatalogSnapshotProvider, + HogQlCompilationExecutor.directExecutor()); + } + + @Inject + public QueryPreparer( + SqlParser sqlParser, + HogQlCompiler hogQlCompiler, + HogQlCompilationObserver hogQlCompilationObserver, + Optional hogQlSemanticCatalogSnapshotProvider, + HogQlCompilationExecutor hogQlCompilationExecutor) + { + this(sqlParser, + Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), + hogQlCompilationObserver, + hogQlSemanticCatalogSnapshotProvider, + hogQlCompilationExecutor); } private QueryPreparer( SqlParser sqlParser, Optional hogQlCompiler, HogQlCompilationObserver hogQlCompilationObserver, - Optional hogQlSemanticCatalogSnapshotProvider) + Optional hogQlSemanticCatalogSnapshotProvider, + HogQlCompilationExecutor hogQlCompilationExecutor) { this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); this.hogQlCompiler = requireNonNull(hogQlCompiler, "hogQlCompiler is null"); this.hogQlParameterDecoder = new HogQlParameterDecoder(sqlParser); this.hogQlCompilationObserver = requireNonNull(hogQlCompilationObserver, "hogQlCompilationObserver is null"); this.hogQlSemanticCatalogSnapshotProvider = requireNonNull(hogQlSemanticCatalogSnapshotProvider, "hogQlSemanticCatalogSnapshotProvider is null"); + this.hogQlCompilationExecutor = requireNonNull(hogQlCompilationExecutor, "hogQlCompilationExecutor is null"); } public PreparedQuery prepareQuery(Session session, String query) @@ -120,9 +138,9 @@ public PreparedQuery prepareQuery(Session session, QuerySubmission submission) HogQlCompilationTracker tracker = new HogQlCompilationTracker(hogQlCompilationObserver, Dimensions.fromEnvelope(envelope)); PreparedQuery preparedQuery; try { - HogQlCompilationResult result = tracker.observe(COMPILATION, () -> hogQlCompiler + HogQlCompilationResult result = tracker.observe(COMPILATION, () -> hogQlCompilationExecutor.execute(() -> hogQlCompiler .orElseThrow(() -> new TrinoException(NOT_SUPPORTED, "HogQL query submission is disabled")) - .compile(envelope, semanticCatalogContext(session))); + .compile(envelope, semanticCatalogContext(session)))); Statement statement = explain(result.statement(), submission.hogQlExplain()); preparedQuery = tracker.observe(PARAMETER_BINDING, () -> prepareQuery( session, diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlConfig.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlConfig.java index 525350a90d02..e5051febc03c 100644 --- a/core/trino-main/src/main/java/io/trino/hogql/HogQlConfig.java +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlConfig.java @@ -15,10 +15,13 @@ import io.airlift.configuration.Config; import io.airlift.configuration.ConfigDescription; +import jakarta.validation.constraints.Min; public class HogQlConfig { private boolean enabled; + private int compilationThreads = 2; + private int compilationQueueCapacity = 32; public boolean isEnabled() { @@ -32,4 +35,32 @@ public HogQlConfig setEnabled(boolean enabled) this.enabled = enabled; return this; } + + @Min(1) + public int getCompilationThreads() + { + return compilationThreads; + } + + @Config("hogql.compilation-threads") + @ConfigDescription("Number of coordinator threads dedicated to HogQL compilation") + public HogQlConfig setCompilationThreads(int compilationThreads) + { + this.compilationThreads = compilationThreads; + return this; + } + + @Min(0) + public int getCompilationQueueCapacity() + { + return compilationQueueCapacity; + } + + @Config("hogql.compilation-queue-capacity") + @ConfigDescription("Maximum number of HogQL compilations waiting for a compiler thread") + public HogQlConfig setCompilationQueueCapacity(int compilationQueueCapacity) + { + this.compilationQueueCapacity = compilationQueueCapacity; + return this; + } } diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCoordinatorErrorCode.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCoordinatorErrorCode.java new file mode 100644 index 000000000000..612a412c1374 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCoordinatorErrorCode.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.trino.spi.ErrorCode; +import io.trino.spi.ErrorCodeSupplier; +import io.trino.spi.ErrorType; + +import static io.trino.spi.ErrorType.INSUFFICIENT_RESOURCES; + +public enum HogQlCoordinatorErrorCode + implements ErrorCodeSupplier +{ + HOGQL_COMPILATION_QUEUE_FULL(0, INSUFFICIENT_RESOURCES), + ; + + private final ErrorCode errorCode; + + HogQlCoordinatorErrorCode(int code, ErrorType type) + { + errorCode = new ErrorCode(code + 0x0522_0000, name(), type); + } + + @Override + public ErrorCode toErrorCode() + { + return errorCode; + } +} diff --git a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java index b96d242e1b05..1ce8e3213cf4 100644 --- a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java +++ b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java @@ -52,6 +52,7 @@ import io.trino.execution.ExecutionFailureInfo; import io.trino.execution.ExplainAnalyzeContext; import io.trino.execution.ForQueryExecution; +import io.trino.execution.HogQlCompilationExecutor; import io.trino.execution.NodeTaskMap; import io.trino.execution.QueryExecution; import io.trino.execution.QueryExecutionMBean; @@ -186,6 +187,7 @@ protected void setup(Binder binder) configBinder(binder).bindConfig(HogQlConfig.class); configBinder(binder).bindConfig(HogQlSemanticCatalogConfig.class); binder.bind(HogQlCompiler.class).in(Scopes.SINGLETON); + binder.bind(HogQlCompilationExecutor.class).in(Scopes.SINGLETON); binder.bind(HogQlCompilationStats.class).in(Scopes.SINGLETON); binder.bind(HogQlCompilationObserver.class).to(HogQlCompilationStats.class); OptionalBinder.newOptionalBinder(binder, HogQlSemanticCatalogSnapshotProvider.class); diff --git a/core/trino-main/src/test/java/io/trino/execution/TestHogQlCompilationExecutor.java b/core/trino-main/src/test/java/io/trino/execution/TestHogQlCompilationExecutor.java new file mode 100644 index 000000000000..e6bdf6e72ad9 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/execution/TestHogQlCompilationExecutor.java @@ -0,0 +1,186 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +import io.trino.hogql.HogQlCompilationEvent; +import io.trino.hogql.HogQlConfig; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +import static io.trino.execution.QuerySubmission.hogQl; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.INSUFFICIENT_RESOURCES; +import static io.trino.hogql.HogQlCompilationEvent.Phase.COMPILATION; +import static io.trino.hogql.HogQlCoordinatorErrorCode.HOGQL_COMPILATION_QUEUE_FULL; +import static io.trino.testing.TestingSession.testSessionBuilder; +import static java.util.concurrent.Executors.newSingleThreadExecutor; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlCompilationExecutor +{ + @Test + public void testCompilationRunsOnDedicatedWorker() + { + HogQlCompilationExecutor executor = new HogQlCompilationExecutor(config(1, 0)); + try { + assertThat(executor.execute(() -> Thread.currentThread().getName())) + .startsWith("hogql-compilation-"); + } + finally { + executor.shutdown(); + } + } + + @Test + public void testSaturationRejectsHogQlWithoutAffectingTrinoSql() + throws Exception + { + HogQlCompilationExecutor executor = new HogQlCompilationExecutor(config(1, 0)); + CountDownLatch workerStarted = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + List events = new ArrayList<>(); + QueryPreparer queryPreparer = new QueryPreparer( + new SqlParser(), + new HogQlCompiler(), + events::add, + Optional.empty(), + executor); + + ExecutorService caller = newSingleThreadExecutor(); + try { + Future occupiedWorker = caller.submit(() -> executor.execute(() -> { + workerStarted.countDown(); + await(releaseWorker); + return null; + })); + assertThat(workerStarted.await(10, SECONDS)).isTrue(); + + try { + assertThatThrownBy(() -> queryPreparer.prepareQuery(testSessionBuilder().build(), hogQl(envelope("SELECT 1")))) + .isInstanceOfSatisfying(TrinoException.class, exception -> { + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_COMPILATION_QUEUE_FULL.toErrorCode()); + assertThat(exception).hasMessageContaining("retry"); + }); + assertThat(queryPreparer.prepareQuery(testSessionBuilder().build(), "SELECT 1").getStatement()).isNotNull(); + + assertThat(events).singleElement().satisfies(event -> { + assertThat(event.outcome()).isEqualTo(INSUFFICIENT_RESOURCES); + assertThat(event.failedPhase()).contains(COMPILATION); + }); + } + finally { + releaseWorker.countDown(); + } + + occupiedWorker.get(10, SECONDS); + } + finally { + releaseWorker.countDown(); + caller.shutdownNow(); + executor.shutdown(); + } + } + + @Test + public void testShutdownInterruptsWorkAndRejectsNewCompilations() + throws Exception + { + HogQlCompilationExecutor executor = new HogQlCompilationExecutor(config(1, 1)); + CountDownLatch workerStarted = new CountDownLatch(1); + CountDownLatch workerInterrupted = new CountDownLatch(1); + + ExecutorService caller = newSingleThreadExecutor(); + try { + Future occupiedWorker = caller.submit(() -> executor.execute(() -> { + workerStarted.countDown(); + try { + new CountDownLatch(1).await(); + } + catch (InterruptedException _) { + workerInterrupted.countDown(); + Thread.currentThread().interrupt(); + } + return null; + })); + assertThat(workerStarted.await(10, SECONDS)).isTrue(); + + executor.shutdown(); + + assertThat(workerInterrupted.await(10, SECONDS)).isTrue(); + try { + occupiedWorker.get(10, SECONDS); + } + catch (ExecutionException exception) { + assertThat(exception.getCause()) + .isInstanceOfSatisfying(TrinoException.class, cause -> + assertThat(cause.getErrorCode()).isEqualTo(HOGQL_COMPILATION_QUEUE_FULL.toErrorCode())); + } + assertThat(occupiedWorker).isDone(); + assertThat(executor.isShutdown()).isTrue(); + assertThatThrownBy(() -> executor.execute(() -> null)) + .isInstanceOfSatisfying(TrinoException.class, exception -> + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_COMPILATION_QUEUE_FULL.toErrorCode())); + } + finally { + caller.shutdownNow(); + executor.shutdown(); + } + } + + private static HogQlConfig config(int threads, int queueCapacity) + { + return new HogQlConfig() + .setCompilationThreads(threads) + .setCompilationQueueCapacity(queueCapacity); + } + + private static HogQlCompileEnvelope envelope(String query) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.empty()); + } + + private static void await(CountDownLatch latch) + { + try { + latch.await(); + } + catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new RuntimeException(exception); + } + } +} diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlConfig.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlConfig.java index bdbb8e2bf1ef..f14202bafbb7 100644 --- a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlConfig.java +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlConfig.java @@ -27,14 +27,22 @@ public class TestHogQlConfig public void testDefaults() { assertRecordedDefaults(recordDefaults(HogQlConfig.class) - .setEnabled(false)); + .setEnabled(false) + .setCompilationThreads(2) + .setCompilationQueueCapacity(32)); } @Test public void testExplicitPropertyMappings() { assertFullMapping( - Map.of("hogql.enabled", "true"), - new HogQlConfig().setEnabled(true)); + Map.of( + "hogql.enabled", "true", + "hogql.compilation-threads", "4", + "hogql.compilation-queue-capacity", "128"), + new HogQlConfig() + .setEnabled(true) + .setCompilationThreads(4) + .setCompilationQueueCapacity(128)); } } diff --git a/docs/src/main/sphinx/admin/hogql.md b/docs/src/main/sphinx/admin/hogql.md index 9fddab880e7d..e63f25fa5c95 100644 --- a/docs/src/main/sphinx/admin/hogql.md +++ b/docs/src/main/sphinx/admin/hogql.md @@ -50,6 +50,22 @@ The supported plan types are `LOGICAL`, `DISTRIBUTED`, `VALIDATE`, and `IO`. The supported formats are `TEXT`, `GRAPHVIZ`, and `JSON`. Execution timeouts continue to use Trino session properties such as `query_max_execution_time`. +## Compilation capacity + +HogQL compilation runs on coordinator threads that are separate from standard +query dispatch work. The following properties set the maximum concurrent and +queued compilation work: + +| Property | Default | Description | +| --- | --- | --- | +| `hogql.compilation-threads` | `2` | Number of coordinator threads dedicated to HogQL compilation. The value must be at least `1`. | +| `hogql.compilation-queue-capacity` | `32` | Maximum compilations waiting for a compiler thread. Set this to `0` to reject requests instead of queuing them when every compiler thread is busy. | + +When the workers and queue are full, the coordinator fails the HogQL query with +`HOGQL_COMPILATION_QUEUE_FULL`. This is an insufficient-resources error, so the +caller can retry the query. Standard Trino SQL does not use this executor or +queue. + ## Semantic catalog manifests Logical tables, fields, quoted physical names, and star expansion use an From 73ce634e4591e8017930e1df4304866c3cc30475 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 13:31:37 -0400 Subject: [PATCH 051/136] feat(hogql): lower collection access --- .../trino/hogql/compiler/HogQlCompiler.java | 9 +++++ .../hogql/compiler/HogQlFunctionResolver.java | 10 +++++ .../hogql/compiler/HogQlSemanticResolver.java | 10 +++++ .../trino/hogql/compiler/TrinoAstFactory.java | 10 +++++ .../hogql/compiler/TestHogQlCompiler.java | 38 +++++++++++++++++++ .../compiler/TestHogQlFunctionResolver.java | 12 ++++-- .../compiler/TestHogQlSemanticResolution.java | 14 +++++++ .../io/trino/hogql/parser/HogQlParser.java | 28 ++++++++++++++ .../trino/hogql/parser/tree/HogQlQuery.java | 24 ++++++++++++ .../1.0.0/trino-compatibility-overrides.json | 27 +++++++++++++ .../language/1.0.0/trino-compatibility.json | 27 +++++++------ .../trino/hogql/parser/TestHogQlParser.java | 31 +++++++++++++++ 12 files changed, 225 insertions(+), 15 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index ac37a1ea06cc..01674044863a 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -36,12 +36,14 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Relation; import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; @@ -300,6 +302,8 @@ private static boolean containsFunctionCall(Expression expression) case FunctionCall _ -> true; case InExpression in -> containsFunctionCall(in.value()) || in.values().stream().anyMatch(HogQlCompiler::containsFunctionCall); case IsNullExpression isNull -> containsFunctionCall(isNull.value()); + case MemberAccessExpression memberAccess -> containsFunctionCall(memberAccess.base()); + case SubscriptExpression subscript -> containsFunctionCall(subscript.base()) || containsFunctionCall(subscript.index()); case TupleExpression tuple -> tuple.values().stream().anyMatch(HogQlCompiler::containsFunctionCall); case UnaryExpression unary -> containsFunctionCall(unary.operand()); }; @@ -389,7 +393,12 @@ private static void collectPlaceholders(Expression expression, List } case IsNullExpression isNull -> collectPlaceholders(isNull.value(), placeholders); case Literal _ -> {} + case MemberAccessExpression memberAccess -> collectPlaceholders(memberAccess.base(), placeholders); case Placeholder placeholder -> placeholders.add(placeholder); + case SubscriptExpression subscript -> { + collectPlaceholders(subscript.base(), placeholders); + collectPlaceholders(subscript.index(), placeholders); + } case TupleExpression tuple -> tuple.values().forEach(value -> collectPlaceholders(value, placeholders)); case UnaryExpression unary -> collectPlaceholders(unary.operand(), placeholders); } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 4247faeaab3a..bc2ad64c23bb 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -40,6 +40,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; @@ -48,6 +49,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.SortItem; import io.trino.hogql.parser.tree.HogQlQuery.Star; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; @@ -205,7 +207,15 @@ private Expression resolveExpression(Expression expression) isNull.predicateSpan(), isNull.span()); case Literal literal -> literal; + case MemberAccessExpression memberAccess -> new MemberAccessExpression( + resolveExpression(memberAccess.base()), + memberAccess.member(), + memberAccess.span()); case Placeholder placeholder -> placeholder; + case SubscriptExpression subscript -> new SubscriptExpression( + resolveExpression(subscript.base()), + resolveExpression(subscript.index()), + subscript.span()); case TupleExpression tuple -> new TupleExpression(tuple.values().stream().map(this::resolveExpression).toList(), tuple.span()); case UnaryExpression unary -> new UnaryExpression(unary.operator(), resolveExpression(unary.operand()), unary.span()); }; diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index 63fe183132c1..0e4ed02b3f16 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -54,6 +54,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; @@ -62,6 +63,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.SortItem; import io.trino.hogql.parser.tree.HogQlQuery.Star; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; @@ -522,7 +524,15 @@ private Expression resolveExpression(Expression expression) isNull.predicateSpan(), isNull.span()); case Literal literal -> literal; + case MemberAccessExpression memberAccess -> new MemberAccessExpression( + resolveExpression(memberAccess.base()), + memberAccess.member(), + memberAccess.span()); case Placeholder placeholder -> placeholder; + case SubscriptExpression subscript -> new SubscriptExpression( + resolveExpression(subscript.base()), + resolveExpression(subscript.index()), + subscript.span()); case TupleExpression tuple -> new TupleExpression(tuple.values().stream().map(this::resolveExpression).toList(), tuple.span()); case UnaryExpression unary -> new UnaryExpression(unary.operator(), resolveExpression(unary.operand()), unary.span()); }; diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 69a2072c5b29..fea9cb02cf13 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -33,6 +33,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; @@ -41,6 +42,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.Star; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; @@ -293,7 +295,15 @@ private static Expression createExpression(HogQlQuery.Expression expression, Map createExpression(isNull.value(), parameterIds), new IsNullPredicate(location(isNull.predicateSpan()), isNull.negated())); case Literal literal -> createLiteral(literal); + case MemberAccessExpression memberAccess -> new DereferenceExpression( + location(memberAccess.span()), + createExpression(memberAccess.base(), parameterIds), + createIdentifier(memberAccess.member())); case Placeholder placeholder -> new Parameter(location(placeholder.span()), parameterIds.get(placeholder.span())); + case SubscriptExpression subscript -> new io.trino.sql.tree.SubscriptExpression( + location(subscript.span()), + createExpression(subscript.base(), parameterIds), + createExpression(subscript.index(), parameterIds)); case TupleExpression tuple -> new Row( location(tuple.span()), tuple.values().stream() diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index aaea37efb5e5..cd3eeea81280 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -31,6 +31,7 @@ import io.trino.sql.tree.QuerySpecification; import io.trino.sql.tree.SingleColumn; import io.trino.sql.tree.Statement; +import io.trino.sql.tree.SubscriptExpression; import io.trino.sql.tree.Table; import io.trino.sql.tree.TableSubquery; import io.trino.sql.tree.Union; @@ -97,6 +98,11 @@ public void testLowersDistinctOrderingAndClickHousePaginationOrder() SELECT ARRAY[] | SELECT ARRAY[] SELECT (1, 'example') | SELECT (1, 'example') SELECT (1,) | SELECT ROW(1) + SELECT [1, 2, 3][1] | SELECT ARRAY[1, 2, 3][1] + SELECT (1, 'example').2 | SELECT ROW(1, 'example')[2] + SELECT attributes['plan'] | SELECT attributes['plan'] + SELECT nested[position[1]] | SELECT nested[position[1]] + SELECT [payload][1].plan | SELECT ARRAY[payload][1].plan SELECT value BETWEEN 1 AND 10 | SELECT value BETWEEN 1 AND 10 SELECT value NOT BETWEEN 1 AND 10| SELECT value NOT BETWEEN 1 AND 10 SELECT value IS NULL | SELECT value IS NULL @@ -110,6 +116,38 @@ public void testLowersCollectionAndPredicateExpressions(String hogql, String tri assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(trinoSql)); } + @Test + public void testBindsPlaceholdersInsideCollectionSubscriptsBySourceOrder() + { + HogQlCompilationResult result = compiler.compile( + "SELECT [{element}][{index}], ({left}, {right}).2, {object}.field", + Map.of( + "element", typedValue("element"), + "index", typedValue("index"), + "left", typedValue("left"), + "right", typedValue("right"), + "object", typedValue("object"))); + + assertThat(result.parameterNames()).containsExactly("element", "index", "left", "right", "object"); + assertThat(parameters(result.statement())) + .extracting(Parameter::getId) + .containsExactly(0, 1, 2, 3, 4); + } + + @Test + public void testPreservesCollectionSubscriptSourceLocations() + { + Query query = (Query) compiler.compile("SELECT [10, 20][2], (1, 'two').2"); + QuerySpecification querySpecification = (QuerySpecification) query.getQueryBody(); + SubscriptExpression arrayAccess = (SubscriptExpression) ((SingleColumn) querySpecification.getSelect().getSelectItems().getFirst()).getExpression(); + SubscriptExpression tupleAccess = (SubscriptExpression) ((SingleColumn) querySpecification.getSelect().getSelectItems().get(1)).getExpression(); + + assertThat(arrayAccess.getLocation()).contains(new NodeLocation(1, 8)); + assertThat(arrayAccess.getIndex().getLocation()).contains(new NodeLocation(1, 17)); + assertThat(tupleAccess.getLocation()).contains(new NodeLocation(1, 21)); + assertThat(tupleAccess.getIndex().getLocation()).contains(new NodeLocation(1, 32)); + } + @ParameterizedTest @ValueSource(strings = { "SELECT CASE WHEN enabled THEN 1 ELSE 0 END", diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index dc138be5530c..b381abe287e3 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -73,11 +73,11 @@ public void testCompilerPinsOnceAndResolvesUserFunctions() return new PinnedSnapshot(snapshot); }); - HogQlCompilationResult result = new HogQlCompiler().compile(envelope("SELECT hogUpper('one')"), Optional.of(context)); + HogQlCompilationResult result = new HogQlCompiler().compile(envelope("SELECT [hogUpper('one')][1]"), Optional.of(context)); assertThat(pins).hasValue(1); assertThat(result.catalogGeneration()).hasValue(7); - assertThat(result.statement()).isEqualTo(sqlParser.createStatement("SELECT system.builtin.upper('one')")); + assertThat(result.statement()).isEqualTo(sqlParser.createStatement("SELECT ARRAY[system.builtin.upper('one')][1]")); } @Test @@ -115,7 +115,13 @@ private static Stream nestedFunctionQueries() "SELECT system.builtin.upper('left') UNION ALL SELECT system.builtin.upper('right')"), Arguments.of( "SELECT * FROM (VALUES (hogUpper('value')))", - "SELECT * FROM (VALUES (system.builtin.upper('value')))")); + "SELECT * FROM (VALUES (system.builtin.upper('value')))"), + Arguments.of( + "SELECT [hogUpper('value')][hogUpper('1')]", + "SELECT ARRAY[system.builtin.upper('value')][system.builtin.upper('1')]"), + Arguments.of( + "SELECT hogUpper('value').field", + "SELECT system.builtin.upper('value').field")); } @Test diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java index 54fb33abc746..ae7a1b026c5e 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -150,6 +150,20 @@ public void testResolvesLogicalFieldsInsideNamedWindows() "WINDOW recent AS (PARTITION BY event_name ORDER BY \"Person ID\" ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)")); } + @Test + public void testResolvesLogicalFieldsInsideCollectionSubscripts() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compile( + envelope("SELECT [event][1], [personId][1], [event][1].label FROM events", OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT ARRAY[event_name][1], ARRAY[\"Person ID\"][1], ARRAY[event_name][1].label " + + "FROM analytics.\"Hog Data\".\"raw-events\"")); + } + @Test public void testCatalogIndependentAndPhysicalQueriesDoNotFetchSemanticMetadata() { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index f1fa9ec6d703..5378418bcec0 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -42,6 +42,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinType; import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; import io.trino.hogql.parser.tree.HogQlQuery.NullTreatment; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; @@ -54,6 +55,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.Star; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; @@ -1028,6 +1030,27 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) if (context instanceof HogQLParser.ColumnExprTupleContext tuple) { return new TupleExpression(buildExpressions(tuple.columnExprList()), sourceSpan(tuple)); } + if (context instanceof HogQLParser.ColumnExprArrayAccessContext access) { + return new SubscriptExpression( + buildExpression(access.columnExprValue()), + buildExpression(access.columnExpr()), + sourceSpan(access)); + } + if (context instanceof HogQLParser.ColumnExprTupleAccessContext access) { + if (access.DECIMAL_LITERAL().getText().equals("0")) { + throw unsupported(access, "tuple index zero"); + } + return new SubscriptExpression( + buildExpression(access.columnExprValue()), + new Literal(INTEGER, access.DECIMAL_LITERAL().getText(), sourceSpan(access.DECIMAL_LITERAL().getSymbol(), access.DECIMAL_LITERAL().getSymbol())), + sourceSpan(access)); + } + if (context instanceof HogQLParser.ColumnExprPropertyAccessContext access) { + return new MemberAccessExpression( + buildExpression(access.columnExprValue()), + buildIdentifier(access.identifier()), + sourceSpan(access)); + } if (context instanceof HogQLParser.ColumnExprNegateContext negate) { return new UnaryExpression(NEGATE, buildExpression(negate.columnExprValue()), sourceSpan(negate)); } @@ -1627,6 +1650,11 @@ private void validateExpressionScope(Expression expression, Set forbidde } case IsNullExpression isNull -> validateExpressionScope(isNull.value(), forbiddenOuterRelations, localRelations); case Literal _, Placeholder _ -> {} + case MemberAccessExpression memberAccess -> validateExpressionScope(memberAccess.base(), forbiddenOuterRelations, localRelations); + case SubscriptExpression subscript -> { + validateExpressionScope(subscript.base(), forbiddenOuterRelations, localRelations); + validateExpressionScope(subscript.index(), forbiddenOuterRelations, localRelations); + } case TupleExpression tuple -> tuple.values().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); case UnaryExpression unary -> validateExpressionScope(unary.operand(), forbiddenOuterRelations, localRelations); } diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index db95f1439e1b..d636688c2839 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -258,7 +258,9 @@ public sealed interface Expression InExpression, IsNullExpression, Literal, + MemberAccessExpression, Placeholder, + SubscriptExpression, TupleExpression, UnaryExpression { @@ -324,6 +326,28 @@ public record TupleExpression(List values, SourceSpan span) } } + public record SubscriptExpression(Expression base, Expression index, SourceSpan span) + implements Expression + { + public SubscriptExpression + { + base = requireNonNull(base, "base is null"); + index = requireNonNull(index, "index is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record MemberAccessExpression(Expression base, Identifier member, SourceSpan span) + implements Expression + { + public MemberAccessExpression + { + base = requireNonNull(base, "base is null"); + member = requireNonNull(member, "member is null"); + span = requireNonNull(span, "span is null"); + } + } + public record BetweenExpression(Expression value, Expression min, Expression max, boolean negated, SourceSpan predicateSpan, SourceSpan span) implements Expression { diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json index 53df67543603..a8bd21e615b6 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json @@ -189,6 +189,15 @@ "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" ] }, + { + "id": "alternative:columnExprValue:ColumnExprArrayAccess", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlParser#testBuildsCollectionAccessWithSourceSpans", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, { "id": "alternative:columnExprValue:ColumnExprBetween", "loweringHandler": "TrinoAstFactory#createExpression", @@ -250,6 +259,15 @@ "TestHogQlCompatibilityCorpus#expr.accept.boolean" ] }, + { + "id": "alternative:columnExprValue:ColumnExprPropertyAccess", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlParser#testBuildsCollectionAccessWithSourceSpans", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, { "id": "alternative:columnExprValue:ColumnExprPrecedence1", "loweringHandler": "TrinoAstFactory#createExpression", @@ -276,6 +294,15 @@ "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" ] }, + { + "id": "alternative:columnExprValue:ColumnExprTupleAccess", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlParser#testBuildsCollectionAccessWithSourceSpans", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, { "id": "alternative:columnTypeExpr:ColumnTypeExprSimple", "loweringHandler": "TrinoAstFactory#createExpression", diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json index 287c025d8e11..2d52c1795d90 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json @@ -182,12 +182,13 @@ { "id": "alternative:columnExprValue:ColumnExprArrayAccess", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlParser#testBuildsCollectionAccessWithSourceSpans", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" ] }, { @@ -672,12 +673,13 @@ { "id": "alternative:columnExprValue:ColumnExprPropertyAccess", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlParser#testBuildsCollectionAccessWithSourceSpans", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" ] }, { @@ -794,12 +796,13 @@ { "id": "alternative:columnExprValue:ColumnExprTupleAccess", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlParser#testBuildsCollectionAccessWithSourceSpans", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" ] }, { diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 38f92ca78115..d29446aee7b2 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -23,12 +23,15 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; import io.trino.hogql.parser.tree.HogQlQuery.JoinType; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; import io.trino.hogql.parser.tree.HogQlQuery.SetOperationType; import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; import io.trino.hogql.parser.tree.HogQlSyntaxTree; import io.trino.hogql.parser.tree.HogQlSyntaxTree.Element; @@ -57,6 +60,10 @@ public class TestHogQlParser "SELECT * FROM first POSITIONAL JOIN second", "SELECT * FROM first, second", "SELECT function(1)(value)", + "SELECT tuple_value.0", + "SELECT array_value[1:2]", + "SELECT array_value?.[1]", + "SELECT tuple_value?.1", "DROP TABLE events", "SELECT 1; SELECT 2", "SELECT * FROM", @@ -127,6 +134,30 @@ public void testBuildsDistinctOrderingAndPaginationAst() assertThat(query.offset()).get().extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("2"); } + @Test + public void testBuildsCollectionAccessWithSourceSpans() + { + HogQlQuery query = parser.parseStatement("SELECT [10,{element}][{index}], (1, 'two').2, {payload}.plan"); + + SubscriptExpression arrayAccess = (SubscriptExpression) ((ExpressionProjection) query.projections().getFirst()).expression(); + assertThat(arrayAccess.base()).isInstanceOf(HogQlQuery.ArrayExpression.class); + assertThat(arrayAccess.index()).isInstanceOf(Placeholder.class); + assertThat(arrayAccess.span()).isEqualTo(new HogQlQuery.SourceSpan(7, 30, 1, 8, 1, 31)); + assertThat(arrayAccess.index().span()).isEqualTo(new HogQlQuery.SourceSpan(22, 29, 1, 23, 1, 30)); + + SubscriptExpression tupleAccess = (SubscriptExpression) ((ExpressionProjection) query.projections().get(1)).expression(); + assertThat(tupleAccess.base()).isInstanceOf(TupleExpression.class); + assertThat(tupleAccess.index()).isInstanceOf(HogQlQuery.Literal.class); + assertThat(tupleAccess.span()).isEqualTo(new HogQlQuery.SourceSpan(32, 44, 1, 33, 1, 45)); + assertThat(tupleAccess.index().span()).isEqualTo(new HogQlQuery.SourceSpan(43, 44, 1, 44, 1, 45)); + + MemberAccessExpression memberAccess = (MemberAccessExpression) ((ExpressionProjection) query.projections().get(2)).expression(); + assertThat(memberAccess.base()).isInstanceOf(Placeholder.class); + assertThat(memberAccess.member().value()).isEqualTo("plan"); + assertThat(memberAccess.span()).isEqualTo(new HogQlQuery.SourceSpan(46, 60, 1, 47, 1, 61)); + assertThat(memberAccess.member().span()).isEqualTo(new HogQlQuery.SourceSpan(56, 60, 1, 57, 1, 61)); + } + @Test public void testBuildsOrdinaryGroupingHavingAndAggregateFunctionAst() { From f25513df4096db7f5fb2869c1de5b67dc32f7ecd Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 13:32:28 -0400 Subject: [PATCH 052/136] feat(hogql): consume semantic relation recipes --- .../hogql/compiler/HogQlSemanticResolver.java | 9 +- .../catalog/HogQlSemanticCatalogSnapshot.java | 529 ++++++++++++++++-- ...gQlSemanticCatalogSnapshotJsonDecoder.java | 194 ++++++- .../TestHogQlSemanticCatalogSnapshot.java | 224 +++++++- ...gQlSemanticCatalogSnapshotJsonDecoder.java | 123 +++- 5 files changed, 1034 insertions(+), 45 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index 0e4ed02b3f16..df9b5a005d8f 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -13,6 +13,7 @@ */ package io.trino.hogql.compiler; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CastRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionRecipe; @@ -25,10 +26,12 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.MaterializedViewReference; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyLookupRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ReferencedField; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationReference; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SavedQueryReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ScopedFieldReferenceRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.VirtualTableDefinition; @@ -660,6 +663,9 @@ private Expression expandRecipe( new Identifier(cast.targetTypeSignature(), false, span), false, span); + case ArgumentReferenceRecipe _ -> throw unsupportedExpansion(span, "HogQL recipe argument is only valid inside a property lookup recipe"); + case ScopedFieldReferenceRecipe _ -> throw unsupportedExpansion(span, "HogQL scoped field recipe is only valid inside a relationship predicate"); + case PropertyLookupRecipe _ -> throw unsupportedExpansion(span, "HogQL property lookup recipe expansion is unavailable"); }; } finally { @@ -716,6 +722,7 @@ private Expression expandOperator( case NEGATE -> new UnaryExpression(HogQlQuery.UnaryOperator.NEGATE, arguments.getFirst(), span); case IS_NULL -> new IsNullExpression(arguments.getFirst(), false, span, span); case IS_NOT_NULL -> new IsNullExpression(arguments.getFirst(), true, span, span); + case SUBSCRIPT -> new SubscriptExpression(arguments.getFirst(), arguments.getLast(), span); default -> new BinaryExpression(binaryOperator(operator.operator()), arguments.getFirst(), arguments.getLast(), span); }; } @@ -736,7 +743,7 @@ private static HogQlQuery.BinaryOperator binaryOperator(SemanticOperator operato case GREATER_THAN_OR_EQUAL -> HogQlQuery.BinaryOperator.GREATER_THAN_OR_EQUAL; case AND -> HogQlQuery.BinaryOperator.AND; case OR -> HogQlQuery.BinaryOperator.OR; - case NOT, NEGATE, IS_NULL, IS_NOT_NULL -> throw new IllegalArgumentException("unary operator cannot be lowered as binary"); + case NOT, NEGATE, IS_NULL, IS_NOT_NULL, SUBSCRIPT -> throw new IllegalArgumentException("operator cannot be lowered as binary"); }; } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 4b4fade32df2..6eaf4bb04e0f 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -43,7 +43,10 @@ public record HogQlSemanticCatalogSnapshot( List savedQueries, List materializedViews, List functions, - List modifierDefaults) + List modifierDefaults, + List lazyTables, + List actions, + List cohorts) { public static final int PROTOCOL_VERSION = 1; public static final int SCHEMA_VERSION = 2; @@ -73,18 +76,43 @@ public record HogQlSemanticCatalogSnapshot( materializedViews = copy(materializedViews, "materializedViews"); functions = copy(functions, "functions"); modifierDefaults = copy(modifierDefaults, "modifierDefaults"); - int definitions = expressionFields.size() + virtualTables.size() + savedQueries.size() + materializedViews.size() + functions.size() + modifierDefaults.size(); + lazyTables = copy(lazyTables, "lazyTables"); + actions = copy(actions, "actions"); + cohorts = copy(cohorts, "cohorts"); + int definitions = expressionFields.size() + virtualTables.size() + savedQueries.size() + materializedViews.size() + functions.size() + modifierDefaults.size() + lazyTables.size() + actions.size() + cohorts.size(); if (definitions > MAX_SEMANTIC_DEFINITIONS) { throw new IllegalArgumentException("semantic metadata exceeds definition limit"); } Map tables = indexTables(catalog, logicalTables); validateLogicalReferences(tables); - Set declaredFunctions = validateFunctions(functions); + Map declaredFunctions = validateFunctions(functions); Map expressions = indexExpressionFields(expressionFields, tables); - validateExpressionRecipes(expressionFields, tables, expressions, declaredFunctions); + RecipeCounter counter = new RecipeCounter(); + validatePropertyRecipes(tables, expressions, declaredFunctions, counter); + validateExpressionRecipes(expressionFields, tables, expressions, declaredFunctions, counter); validateModifiers(modifierDefaults); - validateRelations(catalog, tables, expressions, virtualTables, savedQueries, materializedViews); + validateRelationshipPredicates(tables, expressions, declaredFunctions, counter); + Map relations = validateRelations(catalog, tables, expressions, virtualTables, savedQueries, materializedViews); + validateLazyTables(lazyTables, tables, expressions, declaredFunctions, counter); + validateSemanticEntities(actions, cohorts, tables, expressions, declaredFunctions, relations, counter); + } + + public HogQlSemanticCatalogSnapshot( + int protocolVersion, + int schemaVersion, + HogQlLanguageVersion languageVersion, + PhysicalIdentifier catalog, + long generation, + List logicalTables, + List expressionFields, + List virtualTables, + List savedQueries, + List materializedViews, + List functions, + List modifierDefaults) + { + this(protocolVersion, schemaVersion, languageVersion, catalog, generation, logicalTables, expressionFields, virtualTables, savedQueries, materializedViews, functions, modifierDefaults, List.of(), List.of(), List.of()); } public HogQlSemanticCatalogSnapshot( @@ -94,7 +122,7 @@ public HogQlSemanticCatalogSnapshot( long generation, List logicalTables) { - this(PROTOCOL_VERSION, schemaVersion, languageVersion, catalog, generation, logicalTables, List.of(), List.of(), List.of(), List.of(), List.of(), List.of()); + this(PROTOCOL_VERSION, schemaVersion, languageVersion, catalog, generation, logicalTables, List.of(), List.of(), List.of(), List.of(), List.of(), List.of(), List.of(), List.of(), List.of()); } public Optional logicalTable(String name) @@ -142,11 +170,67 @@ private static void validateLogicalReferences(Map validateFunctions(List definitions) + private static void validatePropertyRecipes( + Map tables, + Map expressions, + Map functions, + RecipeCounter counter) + { + for (LogicalTableDefinition table : tables.values()) { + for (PropertyDefinition property : table.properties()) { + if (property.lookupRecipe().isEmpty()) { + continue; + } + Map arguments = new HashMap<>(); + validateRecipe( + property.lookupRecipe().orElseThrow(), + 1, + counter, + new RecipeValidationContext(table.name(), tables, expressions, functions, null, false, false, arguments, Map.of())); + if (!arguments.containsKey(ExpressionArgument.PROPERTY_SOURCE) || !arguments.containsKey(ExpressionArgument.PROPERTY_KEY)) { + throw new IllegalArgumentException("property lookup recipe must reference source and key arguments"); + } + } + } + } + + private static void validateRelationshipPredicates( + Map tables, + Map expressions, + Map functions, + RecipeCounter counter) + { + for (LogicalTableDefinition source : tables.values()) { + for (RelationshipDefinition relationship : source.relationships()) { + if (relationship.joinPredicate().isEmpty()) { + continue; + } + LogicalTableDefinition target = tables.get(canonical(relationship.targetTable(), "relationship target table")); + validateRecipe( + relationship.joinPredicate().orElseThrow(), + 1, + counter, + new RecipeValidationContext( + source.name(), + tables, + expressions, + functions, + null, + false, + true, + null, + Map.of(RelationshipJoinSide.SOURCE, source.name(), RelationshipJoinSide.TARGET, target.name()))); + } + } + } + + private static Map validateFunctions(List definitions) { - Set functions = new HashSet<>(); + Map functions = new HashMap<>(); for (FunctionCapabilityDefinition function : definitions) { - addUnique(functions, function.name(), "duplicate function"); + if (functions.put(canonical(function.name(), "function name"), function) != null) { + throw new IllegalArgumentException("duplicate function"); + } if (function.implementation() == FunctionImplementation.REWRITE && !function.trinoName().isEmpty()) { throw new IllegalArgumentException("rewrite function cannot name a Trino function"); } @@ -154,7 +238,7 @@ private static Set validateFunctions(List throw new IllegalArgumentException("function must name a Trino function"); } } - return Set.copyOf(functions); + return Map.copyOf(functions); } private static Map indexExpressionFields(List definitions, Map tables) @@ -182,13 +266,17 @@ private static void validateExpressionRecipes( List definitions, Map tables, Map expressions, - Set functions) + Map functions, + RecipeCounter counter) { Map> dependencies = new HashMap<>(); - RecipeCounter counter = new RecipeCounter(); for (ExpressionFieldDefinition definition : definitions) { List fieldDependencies = new ArrayList<>(); - validateRecipe(definition.recipe(), definition.table(), 1, counter, tables, expressions, functions, fieldDependencies); + validateRecipe( + definition.recipe(), + 1, + counter, + new RecipeValidationContext(definition.table(), tables, expressions, functions, fieldDependencies, true, true, null, Map.of())); dependencies.put(expressionKey(definition.table(), definition.name()), List.copyOf(fieldDependencies)); } Map states = new HashMap<>(); @@ -197,13 +285,9 @@ private static void validateExpressionRecipes( private static void validateRecipe( ExpressionRecipe recipe, - String ownerTable, int depth, RecipeCounter counter, - Map tables, - Map expressions, - Set functions, - List dependencies) + RecipeValidationContext context) { requireNonNull(recipe, "expression recipe is null"); if (depth > MAX_RECIPE_DEPTH) { @@ -212,16 +296,21 @@ private static void validateRecipe( counter.add(); switch (recipe) { case FieldReferenceRecipe reference -> { - if (!canonical(reference.table(), "field reference table").equals(canonical(ownerTable, "expression field table"))) { + if (!context.allowFieldReferences()) { + throw new IllegalArgumentException("field reference is not valid in this recipe"); + } + if (!canonical(reference.table(), "field reference table").equals(canonical(context.ownerTable(), "expression field table"))) { throw new IllegalArgumentException("field reference crosses tables without a relationship path"); } - LogicalTableDefinition table = tables.get(canonical(reference.table(), "field reference table")); + LogicalTableDefinition table = context.tables().get(canonical(reference.table(), "field reference table")); if (table == null) { throw new IllegalArgumentException("field reference has unknown table"); } String key = expressionKey(reference.table(), reference.field()); - if (expressions.containsKey(key)) { - dependencies.add(key); + if (context.expressions().containsKey(key)) { + if (context.dependencies() != null) { + context.dependencies().add(key); + } } else { requireReference(fieldNames(table), reference.field(), "field reference has unknown field"); @@ -229,14 +318,62 @@ private static void validateRecipe( } case LiteralRecipe _ -> {} case FunctionCallRecipe call -> { - requireReference(functions, call.name(), "function call references an undeclared function"); - call.arguments().forEach(argument -> validateRecipe(argument, ownerTable, depth + 1, counter, tables, expressions, functions, dependencies)); + FunctionCapabilityDefinition function = context.functions().get(canonical(call.name(), "function name")); + if (function == null) { + throw new IllegalArgumentException("function call references an undeclared function"); + } + if (!acceptsArity(function, call.arguments().size())) { + throw new IllegalArgumentException("function call has an unsupported argument count"); + } + call.arguments().forEach(argument -> validateRecipe(argument, depth + 1, counter, context)); + } + case OperatorRecipe operator -> operator.arguments().forEach(argument -> validateRecipe(argument, depth + 1, counter, context)); + case CastRecipe cast -> validateRecipe(cast.expression(), depth + 1, counter, context); + case ArgumentReferenceRecipe reference -> { + if (context.arguments() == null) { + throw new IllegalArgumentException("argument reference is not valid in this recipe"); + } + context.arguments().merge(reference.argument(), 1, Integer::sum); + } + case ScopedFieldReferenceRecipe reference -> { + String tableName = context.scopedTables().get(reference.side()); + if (tableName == null) { + throw new IllegalArgumentException("scoped field reference is not valid in this recipe"); + } + LogicalTableDefinition table = context.tables().get(canonical(tableName, "scoped field table")); + requireReference(semanticFieldNames(table, context.expressions()), reference.field(), "scoped field reference has unknown field"); + } + case PropertyLookupRecipe lookup -> { + if (!context.allowPropertyLookups() || !contextAllowsTable(context, lookup.table())) { + throw new IllegalArgumentException("property lookup is not valid in this recipe"); + } + LogicalTableDefinition table = context.tables().get(canonical(lookup.table(), "property lookup table")); + if (table == null || table.properties().stream().noneMatch(property -> canonical(property.name(), "property name").equals(canonical(lookup.property(), "property lookup name")))) { + throw new IllegalArgumentException("property lookup references an unknown property"); + } + validateRecipe(lookup.key(), depth + 1, counter, context); } - case OperatorRecipe operator -> operator.arguments().forEach(argument -> validateRecipe(argument, ownerTable, depth + 1, counter, tables, expressions, functions, dependencies)); - case CastRecipe cast -> validateRecipe(cast.expression(), ownerTable, depth + 1, counter, tables, expressions, functions, dependencies); } } + private static boolean contextAllowsTable(RecipeValidationContext context, String table) + { + String canonicalTable = canonical(table, "recipe table"); + if (canonicalTable.equals(canonical(context.ownerTable(), "recipe owner table"))) { + return true; + } + return context.scopedTables().values().stream() + .map(value -> canonical(value, "scoped recipe table")) + .anyMatch(canonicalTable::equals); + } + + private static boolean acceptsArity(FunctionCapabilityDefinition function, int arity) + { + return function.signatures().stream().anyMatch(signature -> signature.variadic() + ? arity >= signature.argumentTypes().size() - 1 + : arity == signature.argumentTypes().size()); + } + private static void visitDependency(String name, int depth, Map> dependencies, Map states) { if (depth > MAX_RECIPE_DEPTH) { @@ -266,7 +403,7 @@ private static void validateModifiers(List modifiers) } } - private static void validateRelations( + private static Map validateRelations( PhysicalIdentifier catalog, Map tables, Map expressions, @@ -293,6 +430,108 @@ private static void validateRelations( Map states = new HashMap<>(); virtualTables.forEach(virtual -> resolveRelation(canonical(virtual.name(), "relation name"), relations, states, 1)); savedQueries.forEach(saved -> resolveRelation(canonical(saved.name(), "relation name"), relations, states, 1)); + return Map.copyOf(relations); + } + + private static void validateLazyTables( + List definitions, + Map tables, + Map expressions, + Map functions, + RecipeCounter counter) + { + Map> members = new HashMap<>(); + tables.forEach((name, table) -> members.put(name, semanticMemberNames(table, expressions))); + for (LazyTableDefinition definition : definitions) { + LogicalTableDefinition owner = tables.get(canonical(definition.table(), "lazy table owner")); + if (owner == null) { + throw new IllegalArgumentException("lazy table references an unknown owner table"); + } + if (!members.get(canonical(owner.name(), "lazy table owner")).add(canonical(definition.name(), "lazy table name"))) { + throw new IllegalArgumentException("lazy table conflicts with an existing logical member"); + } + if (definition.relationshipPath().isEmpty()) { + throw new IllegalArgumentException("lazy table must include a relationship path"); + } + if (definition.relationshipPath().size() > MAX_RELATION_DEPTH) { + throw new IllegalArgumentException("lazy table relationship path exceeds depth limit"); + } + LogicalTableDefinition terminal = owner; + for (String relationshipName : definition.relationshipPath()) { + LogicalTableDefinition pathSource = terminal; + RelationshipDefinition relationship = pathSource.relationships().stream() + .filter(candidate -> canonical(candidate.name(), "relationship name").equals(canonical(relationshipName, "relationship path"))) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("lazy table references an unknown relationship")); + terminal = tables.get(canonical(relationship.targetTable(), "relationship target table")); + } + if (definition.projections().isEmpty()) { + throw new IllegalArgumentException("lazy table must include projections"); + } + Set projectionNames = new HashSet<>(); + for (LazyProjectionDefinition projection : definition.projections()) { + addUnique(projectionNames, projection.name(), "duplicate projection on lazy table"); + validateRecipe( + projection.recipe(), + 1, + counter, + new RecipeValidationContext(terminal.name(), tables, expressions, functions, null, true, true, null, Map.of())); + } + } + } + + private static void validateSemanticEntities( + List actions, + List cohorts, + Map tables, + Map expressions, + Map functions, + Map relations, + RecipeCounter counter) + { + Set actionNames = new HashSet<>(); + for (ActionReference action : actions) { + addUnique(actionNames, action.name(), "duplicate action"); + validateSemanticEntity(action.name(), action.table(), action.representation(), "action", tables, expressions, functions, relations, counter); + } + Set cohortNames = new HashSet<>(); + for (CohortReference cohort : cohorts) { + addUnique(cohortNames, cohort.name(), "duplicate cohort"); + validateSemanticEntity(cohort.name(), cohort.table(), cohort.representation(), "cohort", tables, expressions, functions, relations, counter); + } + } + + private static void validateSemanticEntity( + String name, + String tableName, + SemanticEntityRepresentation representation, + String kind, + Map tables, + Map expressions, + Map functions, + Map relations, + RecipeCounter counter) + { + LogicalTableDefinition owner = tables.get(canonical(tableName, kind + " table")); + if (owner == null) { + throw new IllegalArgumentException(kind + " references an unknown table"); + } + switch (representation) { + case PredicateRepresentation predicate -> validateRecipe( + predicate.predicate(), + 1, + counter, + new RecipeValidationContext(owner.name(), tables, expressions, functions, null, true, true, null, Map.of())); + case RelationMembershipRepresentation membershipRepresentation -> { + RelationMembershipRecipe membership = membershipRepresentation.relation(); + SemanticRelation relation = relations.get(canonical(membership.relation().name(), kind + " relation")); + if (relation == null || relation.kind != membership.relation().kind()) { + throw new IllegalArgumentException(kind + " references an unknown or mismatched relation"); + } + requireReference(semanticFieldNames(owner, expressions), membership.sourceField(), kind + " references an unknown source field"); + requireReference(requireNonNull(relation.fields, "resolved relation fields are null"), membership.targetField(), kind + " references an unknown target field"); + } + } } private static void addRelation(Map relations, String name, SemanticRelation relation) @@ -363,6 +602,23 @@ private static Set fieldNames(LogicalTableDefinition table) return fields; } + private static Set semanticFieldNames(LogicalTableDefinition table, Map expressions) + { + Set fields = fieldNames(requireNonNull(table, "logical table is null")); + expressions.values().stream() + .filter(expression -> canonical(expression.table(), "expression table").equals(canonical(table.name(), "logical table"))) + .forEach(expression -> fields.add(canonical(expression.name(), "expression field"))); + return fields; + } + + private static Set semanticMemberNames(LogicalTableDefinition table, Map expressions) + { + Set members = semanticFieldNames(table, expressions); + table.properties().forEach(property -> members.add(canonical(property.name(), "property name"))); + table.relationships().forEach(relationship -> members.add(canonical(relationship.name(), "relationship name"))); + return members; + } + private static void requireReference(Set names, String name, String message) { if (!names.contains(canonical(name, "reference"))) { @@ -424,7 +680,15 @@ public record LogicalFieldDefinition(String name, PhysicalIdentifier physicalCol } } - public record PropertyDefinition(String name, String sourceField, PropertyStorage storage, LogicalType logicalType, boolean nullable) + public record PropertyDefinition( + String name, + String sourceField, + PropertyStorage storage, + LogicalType logicalType, + boolean nullable, + Optional keyTypeSignature, + Optional valueTypeSignature, + Optional lookupRecipe) { public PropertyDefinition { @@ -432,10 +696,21 @@ public record PropertyDefinition(String name, String sourceField, PropertyStorag sourceField = definition(sourceField, "property source field"); storage = requireNonNull(storage, "storage is null"); logicalType = requireNonNull(logicalType, "logicalType is null"); + keyTypeSignature = requireNonNull(keyTypeSignature, "keyTypeSignature is null").map(value -> definition(value, "property key type signature")); + valueTypeSignature = requireNonNull(valueTypeSignature, "valueTypeSignature is null").map(value -> definition(value, "property value type signature")); + lookupRecipe = requireNonNull(lookupRecipe, "lookupRecipe is null"); + if (lookupRecipe.isPresent() != keyTypeSignature.isPresent() || lookupRecipe.isPresent() != valueTypeSignature.isPresent()) { + throw new IllegalArgumentException("property lookup recipe and type signatures must be declared together"); + } + } + + public PropertyDefinition(String name, String sourceField, PropertyStorage storage, LogicalType logicalType, boolean nullable) + { + this(name, sourceField, storage, logicalType, nullable, Optional.empty(), Optional.empty(), Optional.empty()); } } - public record RelationshipDefinition(String name, String targetTable, RelationshipCardinality cardinality, List joinKeys) + public record RelationshipDefinition(String name, String targetTable, RelationshipCardinality cardinality, List joinKeys, Optional joinPredicate) { public RelationshipDefinition { @@ -446,6 +721,12 @@ public record RelationshipDefinition(String name, String targetTable, Relationsh if (joinKeys.isEmpty()) { throw new IllegalArgumentException("relationship must have at least one join key"); } + joinPredicate = requireNonNull(joinPredicate, "joinPredicate is null"); + } + + public RelationshipDefinition(String name, String targetTable, RelationshipCardinality cardinality, List joinKeys) + { + this(name, targetTable, cardinality, joinKeys, Optional.empty()); } } @@ -471,11 +752,14 @@ public record ExpressionFieldDefinition(String table, String name, String trinoT } public sealed interface ExpressionRecipe - permits CastRecipe, + permits ArgumentReferenceRecipe, + CastRecipe, FieldReferenceRecipe, FunctionCallRecipe, LiteralRecipe, - OperatorRecipe + OperatorRecipe, + PropertyLookupRecipe, + ScopedFieldReferenceRecipe { ExpressionRecipeKind kind(); } @@ -569,6 +853,54 @@ public ExpressionRecipeKind kind() } } + public record ArgumentReferenceRecipe(ExpressionArgument argument) + implements ExpressionRecipe + { + public ArgumentReferenceRecipe + { + argument = requireNonNull(argument, "argument is null"); + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.ARGUMENT_REFERENCE; + } + } + + public record ScopedFieldReferenceRecipe(RelationshipJoinSide side, String field) + implements ExpressionRecipe + { + public ScopedFieldReferenceRecipe + { + side = requireNonNull(side, "side is null"); + field = definition(field, "scoped field reference"); + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.SCOPED_FIELD_REFERENCE; + } + } + + public record PropertyLookupRecipe(String table, String property, ExpressionRecipe key) + implements ExpressionRecipe + { + public PropertyLookupRecipe + { + table = definition(table, "property lookup table"); + property = definition(property, "property lookup name"); + key = requireNonNull(key, "property lookup key is null"); + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.PROPERTY_LOOKUP; + } + } + public record TypedLiteral(String typeSignature, LiteralEncoding encoding, String value) { private static final Pattern DECIMAL = Pattern.compile("[+-]?(0|[1-9][0-9]*)(\\.[0-9]+)?"); @@ -705,6 +1037,9 @@ public record FunctionSignature(List argumentTypes, String returnType, b { argumentTypes = copy(argumentTypes, "argumentTypes").stream().map(value -> definition(value, "function argument type")).toList(); returnType = definition(returnType, "function return type"); + if (variadic && argumentTypes.isEmpty()) { + throw new IllegalArgumentException("variadic function signature must declare an argument"); + } } } @@ -719,6 +1054,98 @@ public record SemanticModifierDefault(String name, ModifierBehavior behavior, Ty } } + public record LazyTableDefinition(String table, String name, List relationshipPath, List projections) + { + public LazyTableDefinition + { + table = definition(table, "lazy table owner"); + name = definition(name, "lazy table name"); + relationshipPath = copy(relationshipPath, "relationshipPath").stream() + .map(value -> definition(value, "lazy table relationship path")) + .toList(); + projections = copy(projections, "projections"); + } + } + + public record LazyProjectionDefinition(String name, String trinoTypeSignature, LogicalType logicalType, boolean nullable, boolean starVisible, ExpressionRecipe recipe) + { + public LazyProjectionDefinition + { + name = definition(name, "lazy projection name"); + trinoTypeSignature = definition(trinoTypeSignature, "lazy projection type signature"); + logicalType = requireNonNull(logicalType, "logicalType is null"); + recipe = requireNonNull(recipe, "recipe is null"); + } + } + + public record ActionReference(String name, String actionId, String table, SemanticEntityRepresentation representation) + { + public ActionReference + { + name = definition(name, "action name"); + actionId = definition(actionId, "action ID"); + table = definition(table, "action table"); + representation = requireNonNull(representation, "representation is null"); + } + } + + public record CohortReference(String name, String cohortId, String table, SemanticEntityRepresentation representation) + { + public CohortReference + { + name = definition(name, "cohort name"); + cohortId = definition(cohortId, "cohort ID"); + table = definition(table, "cohort table"); + representation = requireNonNull(representation, "representation is null"); + } + } + + public sealed interface SemanticEntityRepresentation + permits PredicateRepresentation, RelationMembershipRepresentation + { + SemanticEntityKind kind(); + } + + public record PredicateRepresentation(ExpressionRecipe predicate) + implements SemanticEntityRepresentation + { + public PredicateRepresentation + { + predicate = requireNonNull(predicate, "predicate is null"); + } + + @Override + public SemanticEntityKind kind() + { + return SemanticEntityKind.PREDICATE; + } + } + + public record RelationMembershipRepresentation(RelationMembershipRecipe relation) + implements SemanticEntityRepresentation + { + public RelationMembershipRepresentation + { + relation = requireNonNull(relation, "relation is null"); + } + + @Override + public SemanticEntityKind kind() + { + return SemanticEntityKind.RELATION; + } + } + + public record RelationMembershipRecipe(RelationReference relation, String sourceField, String targetField) + { + public RelationMembershipRecipe + { + relation = requireNonNull(relation, "relation is null"); + sourceField = definition(sourceField, "membership source field"); + targetField = definition(targetField, "membership target field"); + } + } + public record PhysicalQualifiedName(PhysicalIdentifier catalog, PhysicalIdentifier schema, PhysicalIdentifier table) { public PhysicalQualifiedName @@ -747,12 +1174,12 @@ public record PhysicalIdentifier(String value, boolean delimited) public enum ExpressionRecipeKind { - FIELD_REFERENCE, LITERAL, FUNCTION_CALL, OPERATOR, CAST + FIELD_REFERENCE, LITERAL, FUNCTION_CALL, OPERATOR, CAST, ARGUMENT_REFERENCE, SCOPED_FIELD_REFERENCE, PROPERTY_LOOKUP } public enum SemanticOperator { - ADD, SUBTRACT, MULTIPLY, DIVIDE, MODULUS, EQUAL, NOT_EQUAL, LESS_THAN, LESS_THAN_OR_EQUAL, GREATER_THAN, GREATER_THAN_OR_EQUAL, AND, OR, NOT, NEGATE, IS_NULL, IS_NOT_NULL + ADD, SUBTRACT, MULTIPLY, DIVIDE, MODULUS, EQUAL, NOT_EQUAL, LESS_THAN, LESS_THAN_OR_EQUAL, GREATER_THAN, GREATER_THAN_OR_EQUAL, AND, OR, NOT, NEGATE, IS_NULL, IS_NOT_NULL, SUBSCRIPT } public enum LiteralEncoding @@ -795,6 +1222,21 @@ public enum RelationshipCardinality ONE_TO_ONE, ONE_TO_MANY, MANY_TO_ONE, MANY_TO_MANY } + public enum ExpressionArgument + { + PROPERTY_SOURCE, PROPERTY_KEY + } + + public enum RelationshipJoinSide + { + SOURCE, TARGET + } + + public enum SemanticEntityKind + { + PREDICATE, RELATION + } + private enum VisitState { VISITING, VISITED @@ -812,6 +1254,27 @@ private void add() } } + private record RecipeValidationContext( + String ownerTable, + Map tables, + Map expressions, + Map functions, + List dependencies, + boolean allowFieldReferences, + boolean allowPropertyLookups, + Map arguments, + Map scopedTables) + { + private RecipeValidationContext + { + ownerTable = definition(ownerTable, "recipe owner table"); + tables = requireNonNull(tables, "tables is null"); + expressions = requireNonNull(expressions, "expressions is null"); + functions = requireNonNull(functions, "functions is null"); + scopedTables = Map.copyOf(requireNonNull(scopedTables, "scopedTables is null")); + } + } + private static final class SemanticRelation { private final RelationKind kind; diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java index 821a83447e76..f45a9ec5cb92 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java @@ -22,7 +22,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ActionReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CastRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CohortReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionArgument; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionRecipeKind; @@ -33,6 +37,8 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyProjectionDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyTableDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; @@ -43,14 +49,21 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PredicateRepresentation; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyLookupRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ReferencedField; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRepresentation; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationReference; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipJoinSide; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SavedQueryReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ScopedFieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticEntityKind; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticModifierDefault; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; @@ -63,6 +76,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Optional; import java.util.Set; import static java.util.Objects.requireNonNull; @@ -76,6 +90,22 @@ public final class HogQlSemanticCatalogSnapshotJsonDecoder private static final Limits DEFAULT_LIMITS = new Limits(MAXIMUM_PAYLOAD_BYTES, 256, 100_000); private static final Set SNAPSHOT_FIELDS = Set.of( + "protocolVersion", + "schemaVersion", + "languageVersion", + "catalog", + "generation", + "logicalTables", + "expressionFields", + "virtualTables", + "savedQueries", + "materializedViews", + "functions", + "modifierDefaults", + "lazyTables", + "actions", + "cohorts"); + private static final Set REQUIRED_SNAPSHOT_FIELDS = Set.of( "protocolVersion", "schemaVersion", "languageVersion", @@ -92,8 +122,10 @@ public final class HogQlSemanticCatalogSnapshotJsonDecoder private static final Set QUALIFIED_NAME_FIELDS = Set.of("catalog", "schema", "table"); private static final Set TABLE_FIELDS = Set.of("name", "physicalTable", "fields", "properties", "relationships"); private static final Set LOGICAL_FIELD_FIELDS = Set.of("name", "physicalColumn", "trinoTypeSignature", "logicalType", "nullable", "starVisible"); - private static final Set PROPERTY_FIELDS = Set.of("name", "sourceField", "storage", "logicalType", "nullable"); - private static final Set RELATIONSHIP_FIELDS = Set.of("name", "targetTable", "cardinality", "joinKeys"); + private static final Set PROPERTY_FIELDS = Set.of("name", "sourceField", "storage", "logicalType", "nullable", "keyTypeSignature", "valueTypeSignature", "lookupRecipe"); + private static final Set REQUIRED_PROPERTY_FIELDS = Set.of("name", "sourceField", "storage", "logicalType", "nullable"); + private static final Set RELATIONSHIP_FIELDS = Set.of("name", "targetTable", "cardinality", "joinKeys", "joinPredicate"); + private static final Set REQUIRED_RELATIONSHIP_FIELDS = Set.of("name", "targetTable", "cardinality", "joinKeys"); private static final Set JOIN_KEY_FIELDS = Set.of("sourceField", "targetField"); private static final Set EXPRESSION_FIELD_FIELDS = Set.of("table", "name", "trinoTypeSignature", "logicalType", "nullable", "starVisible", "recipe"); private static final Set RECIPE_FIELD_REFERENCE_FIELDS = Set.of("kind", "fieldReference"); @@ -101,11 +133,17 @@ public final class HogQlSemanticCatalogSnapshotJsonDecoder private static final Set RECIPE_FUNCTION_CALL_FIELDS = Set.of("kind", "functionCall"); private static final Set RECIPE_OPERATOR_FIELDS = Set.of("kind", "operator"); private static final Set RECIPE_CAST_FIELDS = Set.of("kind", "cast"); + private static final Set RECIPE_ARGUMENT_REFERENCE_FIELDS = Set.of("kind", "argumentReference"); + private static final Set RECIPE_SCOPED_FIELD_REFERENCE_FIELDS = Set.of("kind", "scopedFieldReference"); + private static final Set RECIPE_PROPERTY_LOOKUP_FIELDS = Set.of("kind", "propertyLookup"); private static final Set FIELD_REFERENCE_FIELDS = Set.of("table", "field"); private static final Set TYPED_LITERAL_FIELDS = Set.of("typeSignature", "encoding", "value"); private static final Set FUNCTION_CALL_FIELDS = Set.of("name", "arguments"); private static final Set OPERATOR_FIELDS = Set.of("operator", "arguments"); private static final Set CAST_FIELDS = Set.of("expression", "targetTypeSignature"); + private static final Set ARGUMENT_REFERENCE_FIELDS = Set.of("argument"); + private static final Set SCOPED_FIELD_REFERENCE_FIELDS = Set.of("side", "field"); + private static final Set PROPERTY_LOOKUP_FIELDS = Set.of("table", "property", "key"); private static final Set VIRTUAL_TABLE_FIELDS = Set.of("name", "source", "projections"); private static final Set RELATION_REFERENCE_FIELDS = Set.of("kind", "name"); private static final Set VIRTUAL_PROJECTION_FIELDS = Set.of("name", "sourceField", "starVisible"); @@ -116,6 +154,13 @@ public final class HogQlSemanticCatalogSnapshotJsonDecoder private static final Set FUNCTION_SIGNATURE_FIELDS = Set.of("argumentTypes", "returnType", "variadic"); private static final Set MODIFIER_FIELDS = Set.of("name", "behavior", "defaultValue", "sessionProperty"); private static final Set MODIFIER_FIELDS_WITHOUT_SESSION_PROPERTY = Set.of("name", "behavior", "defaultValue"); + private static final Set LAZY_TABLE_FIELDS = Set.of("table", "name", "relationshipPath", "projections"); + private static final Set LAZY_PROJECTION_FIELDS = Set.of("name", "trinoTypeSignature", "logicalType", "nullable", "starVisible", "recipe"); + private static final Set ACTION_FIELDS = Set.of("name", "actionId", "table", "representation"); + private static final Set COHORT_FIELDS = Set.of("name", "cohortId", "table", "representation"); + private static final Set PREDICATE_REPRESENTATION_FIELDS = Set.of("kind", "predicate"); + private static final Set RELATION_REPRESENTATION_FIELDS = Set.of("kind", "relation"); + private static final Set RELATION_MEMBERSHIP_FIELDS = Set.of("relation", "sourceField", "targetField"); private final Limits limits; private final ObjectMapper objectMapper; @@ -151,7 +196,7 @@ public HogQlSemanticCatalogSnapshot decode(byte[] payload, LoadRequest request) try { ObjectNode root = parse(payload); - validateFields(root, SNAPSHOT_FIELDS); + validateFields(root, SNAPSHOT_FIELDS, REQUIRED_SNAPSHOT_FIELDS); int protocolVersion = integer(root, "protocolVersion"); if (protocolVersion != PROTOCOL_VERSION) { @@ -185,6 +230,9 @@ public HogQlSemanticCatalogSnapshot decode(byte[] payload, LoadRequest request) List materializedViews = materializedViews(required(root, "materializedViews"), budget); List functions = functions(required(root, "functions"), budget); List modifierDefaults = modifierDefaults(required(root, "modifierDefaults"), budget); + List lazyTables = root.has("lazyTables") ? lazyTables(required(root, "lazyTables"), budget) : List.of(); + List actions = root.has("actions") ? actions(required(root, "actions"), budget) : List.of(); + List cohorts = root.has("cohorts") ? cohorts(required(root, "cohorts"), budget) : List.of(); return new HogQlSemanticCatalogSnapshot( protocolVersion, schemaVersion, @@ -197,7 +245,10 @@ public HogQlSemanticCatalogSnapshot decode(byte[] payload, LoadRequest request) savedQueries, materializedViews, functions, - modifierDefaults); + modifierDefaults, + lazyTables, + actions, + cohorts); } catch (DecodeException e) { throw e; @@ -277,13 +328,16 @@ private static List properties(JsonNode node, CollectionBudg List properties = new ArrayList<>(array.size()); for (JsonNode element : array) { ObjectNode property = object(element); - validateFields(property, PROPERTY_FIELDS); + validateFields(property, PROPERTY_FIELDS, REQUIRED_PROPERTY_FIELDS); properties.add(new PropertyDefinition( text(property, "name"), text(property, "sourceField"), enumValue(property, "storage", PropertyStorage.class), enumValue(property, "logicalType", LogicalType.class), - bool(property, "nullable"))); + bool(property, "nullable"), + property.has("keyTypeSignature") ? Optional.of(text(property, "keyTypeSignature")) : Optional.empty(), + property.has("valueTypeSignature") ? Optional.of(text(property, "valueTypeSignature")) : Optional.empty(), + property.has("lookupRecipe") ? Optional.of(expressionRecipe(required(property, "lookupRecipe"), budget)) : Optional.empty())); } return List.copyOf(properties); } @@ -294,12 +348,13 @@ private static List relationships(JsonNode node, Collect List relationships = new ArrayList<>(array.size()); for (JsonNode element : array) { ObjectNode relationship = object(element); - validateFields(relationship, RELATIONSHIP_FIELDS); + validateFields(relationship, RELATIONSHIP_FIELDS, REQUIRED_RELATIONSHIP_FIELDS); relationships.add(new RelationshipDefinition( text(relationship, "name"), text(relationship, "targetTable"), enumValue(relationship, "cardinality", RelationshipCardinality.class), - joinKeys(required(relationship, "joinKeys"), budget))); + joinKeys(required(relationship, "joinKeys"), budget), + relationship.has("joinPredicate") ? Optional.of(expressionRecipe(required(relationship, "joinPredicate"), budget)) : Optional.empty())); } return List.copyOf(relationships); } @@ -368,6 +423,29 @@ private static ExpressionRecipe expressionRecipe(JsonNode node, CollectionBudget validateFields(cast, CAST_FIELDS); yield new CastRecipe(expressionRecipe(required(cast, "expression"), budget), text(cast, "targetTypeSignature")); } + case ARGUMENT_REFERENCE -> { + validateFields(recipe, RECIPE_ARGUMENT_REFERENCE_FIELDS); + ObjectNode reference = object(required(recipe, "argumentReference")); + validateFields(reference, ARGUMENT_REFERENCE_FIELDS); + yield new ArgumentReferenceRecipe(enumValue(reference, "argument", ExpressionArgument.class)); + } + case SCOPED_FIELD_REFERENCE -> { + validateFields(recipe, RECIPE_SCOPED_FIELD_REFERENCE_FIELDS); + ObjectNode reference = object(required(recipe, "scopedFieldReference")); + validateFields(reference, SCOPED_FIELD_REFERENCE_FIELDS); + yield new ScopedFieldReferenceRecipe( + enumValue(reference, "side", RelationshipJoinSide.class), + text(reference, "field")); + } + case PROPERTY_LOOKUP -> { + validateFields(recipe, RECIPE_PROPERTY_LOOKUP_FIELDS); + ObjectNode lookup = object(required(recipe, "propertyLookup")); + validateFields(lookup, PROPERTY_LOOKUP_FIELDS); + yield new PropertyLookupRecipe( + text(lookup, "table"), + text(lookup, "property"), + expressionRecipe(required(lookup, "key"), budget)); + } }; } @@ -554,6 +632,93 @@ private static List modifierDefaults(JsonNode node, Col return List.copyOf(modifiers); } + private static List lazyTables(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List definitions = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode table = object(element); + validateFields(table, LAZY_TABLE_FIELDS); + definitions.add(new LazyTableDefinition( + text(table, "table"), + text(table, "name"), + strings(required(table, "relationshipPath"), budget), + lazyProjections(required(table, "projections"), budget))); + } + return List.copyOf(definitions); + } + + private static List lazyProjections(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List projections = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode projection = object(element); + validateFields(projection, LAZY_PROJECTION_FIELDS); + projections.add(new LazyProjectionDefinition( + text(projection, "name"), + text(projection, "trinoTypeSignature"), + enumValue(projection, "logicalType", LogicalType.class), + bool(projection, "nullable"), + bool(projection, "starVisible"), + expressionRecipe(required(projection, "recipe"), budget))); + } + return List.copyOf(projections); + } + + private static List actions(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List actions = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode action = object(element); + validateFields(action, ACTION_FIELDS); + actions.add(new ActionReference( + text(action, "name"), + text(action, "actionId"), + text(action, "table"), + semanticEntityRepresentation(required(action, "representation"), budget))); + } + return List.copyOf(actions); + } + + private static List cohorts(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List cohorts = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode cohort = object(element); + validateFields(cohort, COHORT_FIELDS); + cohorts.add(new CohortReference( + text(cohort, "name"), + text(cohort, "cohortId"), + text(cohort, "table"), + semanticEntityRepresentation(required(cohort, "representation"), budget))); + } + return List.copyOf(cohorts); + } + + private static HogQlSemanticCatalogSnapshot.SemanticEntityRepresentation semanticEntityRepresentation(JsonNode node, CollectionBudget budget) + { + ObjectNode representation = object(node); + SemanticEntityKind kind = enumValue(representation, "kind", SemanticEntityKind.class); + return switch (kind) { + case PREDICATE -> { + validateFields(representation, PREDICATE_REPRESENTATION_FIELDS); + yield new PredicateRepresentation(expressionRecipe(required(representation, "predicate"), budget)); + } + case RELATION -> { + validateFields(representation, RELATION_REPRESENTATION_FIELDS); + ObjectNode membership = object(required(representation, "relation")); + validateFields(membership, RELATION_MEMBERSHIP_FIELDS); + yield new RelationMembershipRepresentation(new RelationMembershipRecipe( + relationReference(required(membership, "relation")), + text(membership, "sourceField"), + text(membership, "targetField"))); + } + }; + } + private static PhysicalQualifiedName qualifiedName(JsonNode node) { ObjectNode name = object(node); @@ -642,6 +807,19 @@ private static void validateFields(ObjectNode object, Set allowedFields) } } + private static void validateFields(ObjectNode object, Set allowedFields, Set requiredFields) + { + Iterator fieldNames = object.fieldNames(); + while (fieldNames.hasNext()) { + if (!allowedFields.contains(fieldNames.next())) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + } + if (!requiredFields.stream().allMatch(object::has)) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + } + private static DecodeException failure(DecodeFailure failure) { return new DecodeException(failure); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java index 7ca98b667f89..610f5d436c9d 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java @@ -13,7 +13,11 @@ */ package io.trino.hogql.compiler.catalog; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ActionReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CastRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CohortReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionArgument; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCallRecipe; @@ -22,6 +26,8 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyProjectionDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyTableDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; @@ -30,10 +36,18 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PredicateRepresentation; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyLookupRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRepresentation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationReference; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipJoinSide; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ScopedFieldReferenceRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; import io.trino.hogql.parser.HogQlLanguageVersion; @@ -45,6 +59,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -145,6 +160,111 @@ public void testSemanticDefinitionsAreDeeplyImmutable() assertThatThrownBy(() -> recipe.arguments().clear()).isInstanceOf(UnsupportedOperationException.class); } + @Test + public void testLogicalSemanticRecipesAreDeeplyImmutable() + { + List lookupArguments = new ArrayList<>(List.of( + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE), + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_KEY))); + PropertyDefinition property = propertyWithLookup(new OperatorRecipe(SemanticOperator.SUBSCRIPT, lookupArguments)); + List relationshipPath = new ArrayList<>(List.of("self")); + List projections = new ArrayList<>(List.of(new LazyProjectionDefinition( + "browser", + "varchar", + LogicalType.STRING, + true, + true, + propertyLookup("events", "browser")))); + List lazyTables = new ArrayList<>(List.of(new LazyTableDefinition("events", "profile", relationshipPath, projections))); + List actions = new ArrayList<>(List.of(new ActionReference( + "paid", + "action-1", + "events", + new PredicateRepresentation(propertyLookup("events", "browser"))))); + List cohorts = new ArrayList<>(List.of(new CohortReference( + "active", + "cohort-1", + "events", + new RelationMembershipRepresentation(new RelationMembershipRecipe( + new RelationReference(RelationKind.LOGICAL_TABLE, "events"), + "properties", + "properties"))))); + + HogQlSemanticCatalogSnapshot snapshot = logicalSemanticSnapshot(property, relationshipPathPredicate(), lazyTables, actions, cohorts); + lookupArguments.clear(); + relationshipPath.clear(); + projections.clear(); + lazyTables.clear(); + actions.clear(); + cohorts.clear(); + + assertThat(snapshot.logicalTables().getFirst().properties().getFirst().lookupRecipe()).isPresent(); + assertThat(snapshot.logicalTables().getFirst().relationships().getFirst().joinPredicate()).isPresent(); + assertThat(snapshot.lazyTables()).singleElement().satisfies(lazy -> { + assertThat(lazy.relationshipPath()).containsExactly("self"); + assertThat(lazy.projections()).hasSize(1); + }); + assertThat(snapshot.actions()).hasSize(1); + assertThat(snapshot.cohorts()).hasSize(1); + assertThatThrownBy(() -> snapshot.lazyTables().getFirst().relationshipPath().clear()) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + public void testRejectsInvalidLogicalSemanticScopes() + { + assertThatThrownBy(() -> logicalSemanticSnapshot( + propertyWithLookup(new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE)), + relationshipPathPredicate(), + List.of(), + List.of(), + List.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("source and key arguments"); + + assertThatThrownBy(() -> logicalSemanticSnapshot( + propertyWithLookup(validLookupRecipe()), + Optional.empty(), + List.of(), + List.of(new ActionReference( + "invalid", + "action-1", + "events", + new PredicateRepresentation(new ScopedFieldReferenceRecipe(RelationshipJoinSide.SOURCE, "properties")))), + List.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("scoped field reference"); + + assertThatThrownBy(() -> logicalSemanticSnapshot( + propertyWithLookup(validLookupRecipe()), + relationshipPathPredicate(), + List.of(new LazyTableDefinition( + "events", + "profile", + List.of("missing"), + List.of(new LazyProjectionDefinition("browser", "varchar", LogicalType.STRING, true, true, propertyLookup("events", "browser"))))), + List.of(), + List.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("unknown relationship"); + + assertThatThrownBy(() -> logicalSemanticSnapshot( + propertyWithLookup(validLookupRecipe()), + relationshipPathPredicate(), + List.of(), + List.of(), + List.of(new CohortReference( + "invalid", + "cohort-1", + "events", + new RelationMembershipRepresentation(new RelationMembershipRecipe( + new RelationReference(RelationKind.LOGICAL_TABLE, "events"), + "properties", + "missing")))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("unknown target field"); + } + @Test public void testEnforcesExpressionRecipeDepthAndNodeLimits() { @@ -165,7 +285,7 @@ public void testEnforcesExpressionRecipeDepthAndNodeLimits() } assertThatThrownBy(() -> semanticSnapshot( List.of(new ExpressionFieldDefinition("events", "derived", "bigint", LogicalType.INTEGER, false, true, new FunctionCallRecipe("identity", arguments))), - List.of(function("identity")))) + List.of(variadicFunction("identity")))) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("node limit"); } @@ -193,6 +313,23 @@ public void testRejectsInvalidOperatorArity(SemanticOperator operator, int argum .hasMessageContaining("operator arity"); } + @Test + public void testRejectsUnsupportedRecipeFunctionArity() + { + assertThatThrownBy(() -> semanticSnapshot( + List.of(new ExpressionFieldDefinition( + "events", + "derived", + "bigint", + LogicalType.INTEGER, + false, + true, + new FunctionCallRecipe("identity", List.of()))), + List.of(function("identity")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("unsupported argument count"); + } + private static Stream invalidOperatorArities() { return Stream.concat( @@ -298,6 +435,91 @@ private static FunctionCapabilityDefinition function(String name) false); } + private static FunctionCapabilityDefinition variadicFunction(String name) + { + return new FunctionCapabilityDefinition( + name, + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of(new PhysicalIdentifier(name, false)), + List.of(new FunctionSignature(List.of("bigint"), "bigint", true)), + true, + false, + false, + false, + false); + } + + private static HogQlSemanticCatalogSnapshot logicalSemanticSnapshot( + PropertyDefinition property, + Optional joinPredicate, + List lazyTables, + List actions, + List cohorts) + { + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + LANGUAGE_VERSION, + CATALOG, + 7, + List.of(table( + "events", + List.of(field("properties")), + List.of(property), + List.of(new RelationshipDefinition( + "self", + "events", + RelationshipCardinality.MANY_TO_ONE, + List.of(new JoinKey("properties", "properties")), + joinPredicate)))), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + lazyTables, + actions, + cohorts); + } + + private static PropertyDefinition propertyWithLookup(ExpressionRecipe recipe) + { + return new PropertyDefinition( + "browser", + "properties", + PropertyStorage.JSON_OBJECT, + LogicalType.STRING, + true, + Optional.of("varchar"), + Optional.of("varchar"), + Optional.of(recipe)); + } + + private static OperatorRecipe validLookupRecipe() + { + return new OperatorRecipe( + SemanticOperator.SUBSCRIPT, + List.of( + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE), + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_KEY))); + } + + private static Optional relationshipPathPredicate() + { + return Optional.of(new OperatorRecipe( + SemanticOperator.EQUAL, + List.of( + new ScopedFieldReferenceRecipe(RelationshipJoinSide.SOURCE, "properties"), + new ScopedFieldReferenceRecipe(RelationshipJoinSide.TARGET, "properties")))); + } + + private static PropertyLookupRecipe propertyLookup(String table, String property) + { + return new PropertyLookupRecipe(table, property, new LiteralRecipe(new TypedLiteral("varchar", LiteralEncoding.STRING, "browser"))); + } + private static LogicalTableDefinition table(String name) { return table(name, List.of(field("id")), List.of(), List.of()); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java index 1e5255b5df41..0f5523032504 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java @@ -13,6 +13,9 @@ */ package io.trino.hogql.compiler.catalog; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ModifierBehavior; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; @@ -29,6 +32,7 @@ import org.junit.jupiter.params.provider.MethodSource; import java.nio.charset.StandardCharsets; +import java.util.List; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -99,6 +103,40 @@ public void testDecodesCompleteSnapshot() assertThat(modifier.behavior()).isEqualTo(ModifierBehavior.TRINO_SESSION_PROPERTY); assertThat(modifier.sessionProperty()).extracting(PhysicalIdentifier::value).containsExactly("hogql", "sampling"); }); + assertThat(snapshot.logicalTables().getFirst().properties().getFirst()).satisfies(property -> { + assertThat(property.keyTypeSignature()).contains("varchar"); + assertThat(property.valueTypeSignature()).contains("varchar"); + assertThat(property.lookupRecipe()).isPresent(); + }); + assertThat(snapshot.logicalTables().getFirst().relationships().getFirst().joinPredicate()).isPresent(); + assertThat(snapshot.lazyTables()).singleElement().satisfies(table -> { + assertThat(table.relationshipPath()).containsExactly("self"); + assertThat(table.projections()).singleElement().extracting(HogQlSemanticCatalogSnapshot.LazyProjectionDefinition::name).isEqualTo("browser"); + }); + assertThat(snapshot.actions()).singleElement().extracting(HogQlSemanticCatalogSnapshot.ActionReference::actionId).isEqualTo("action-7"); + assertThat(snapshot.cohorts()).singleElement().extracting(HogQlSemanticCatalogSnapshot.CohortReference::cohortId).isEqualTo("cohort-7"); + } + + @Test + public void testDecodesSnapshotFromOlderSchemaVersionTwoPublisher() + throws Exception + { + ObjectNode payload = (ObjectNode) new ObjectMapper().readTree(validSnapshotJson()); + payload.remove(List.of("lazyTables", "actions", "cohorts")); + ObjectNode table = (ObjectNode) ((ArrayNode) payload.get("logicalTables")).get(0); + ObjectNode property = (ObjectNode) ((ArrayNode) table.get("properties")).get(0); + property.remove(List.of("keyTypeSignature", "valueTypeSignature", "lookupRecipe")); + ObjectNode relationship = (ObjectNode) ((ArrayNode) table.get("relationships")).get(0); + relationship.remove("joinPredicate"); + + HogQlSemanticCatalogSnapshot snapshot = new HogQlSemanticCatalogSnapshotJsonDecoder() + .decode(bytes(payload.toString()), LoadRequest.latest(CATALOG, LANGUAGE_VERSION)); + + assertThat(snapshot.lazyTables()).isEmpty(); + assertThat(snapshot.actions()).isEmpty(); + assertThat(snapshot.cohorts()).isEmpty(); + assertThat(snapshot.logicalTables().getFirst().properties().getFirst().lookupRecipe()).isEmpty(); + assertThat(snapshot.logicalTables().getFirst().relationships().getFirst().joinPredicate()).isEmpty(); } @ParameterizedTest(name = "{0}") @@ -162,6 +200,8 @@ private static Stream strictFailures() Arguments.of("unknown recipe field", validSnapshotJson().replace("\"fieldReference\": {", "\"unknown\": true, \"fieldReference\": {")), Arguments.of("unknown recipe kind", validSnapshotJson().replace("\"kind\": \"FUNCTION_CALL\"", "\"kind\": \"" + SECRET + "\"")), Arguments.of("mismatched recipe payload", validSnapshotJson().replace("\"functionCall\": {", "\"literal\": {")), + Arguments.of("partial property lookup metadata", validSnapshotJson().replace("\"valueTypeSignature\": \"varchar\",\n", "")), + Arguments.of("mismatched entity representation", validSnapshotJson().replace("\"kind\": \"PREDICATE\"", "\"kind\": \"RELATION\"")), Arguments.of( "unknown function reference", validSnapshotJson().replaceFirst("\"name\": \"length\",", "\"name\": \"missing\",")), @@ -262,7 +302,19 @@ private static String validSnapshotJson() "sourceField": "properties", "storage": "JSON_OBJECT", "logicalType": "STRING", - "nullable": true + "nullable": true, + "keyTypeSignature": "varchar", + "valueTypeSignature": "varchar", + "lookupRecipe": { + "kind": "OPERATOR", + "operator": { + "operator": "SUBSCRIPT", + "arguments": [ + {"kind": "ARGUMENT_REFERENCE", "argumentReference": {"argument": "PROPERTY_SOURCE"}}, + {"kind": "ARGUMENT_REFERENCE", "argumentReference": {"argument": "PROPERTY_KEY"}} + ] + } + } } ], "relationships": [ @@ -270,7 +322,17 @@ private static String validSnapshotJson() "name": "self", "targetTable": "events", "cardinality": "MANY_TO_ONE", - "joinKeys": [{"sourceField": "properties", "targetField": "properties"}] + "joinKeys": [{"sourceField": "properties", "targetField": "properties"}], + "joinPredicate": { + "kind": "OPERATOR", + "operator": { + "operator": "EQUAL", + "arguments": [ + {"kind": "SCOPED_FIELD_REFERENCE", "scopedFieldReference": {"side": "SOURCE", "field": "properties"}}, + {"kind": "SCOPED_FIELD_REFERENCE", "scopedFieldReference": {"side": "TARGET", "field": "properties"}} + ] + } + } } ] } @@ -376,6 +438,63 @@ private static String validSnapshotJson() {"value": "sampling", "delimited": false} ] } + ], + "lazyTables": [ + { + "table": "events", + "name": "profile", + "relationshipPath": ["self"], + "projections": [ + { + "name": "browser", + "trinoTypeSignature": "varchar", + "logicalType": "STRING", + "nullable": true, + "starVisible": true, + "recipe": { + "kind": "PROPERTY_LOOKUP", + "propertyLookup": { + "table": "events", + "property": "browser", + "key": {"kind": "LITERAL", "literal": {"typeSignature": "varchar", "encoding": "STRING", "value": "browser"}} + } + } + } + ] + } + ], + "actions": [ + { + "name": "paid_event", + "actionId": "action-7", + "table": "events", + "representation": { + "kind": "PREDICATE", + "predicate": { + "kind": "PROPERTY_LOOKUP", + "propertyLookup": { + "table": "events", + "property": "browser", + "key": {"kind": "LITERAL", "literal": {"typeSignature": "varchar", "encoding": "STRING", "value": "browser"}} + } + } + } + } + ], + "cohorts": [ + { + "name": "active_people", + "cohortId": "cohort-7", + "table": "events", + "representation": { + "kind": "RELATION", + "relation": { + "relation": {"kind": "MATERIALIZED_VIEW", "name": "daily_events"}, + "sourceField": "properties", + "targetField": "day" + } + } + } ] } """; From 801a9be622311598745471ecc430b4fd4bbca55c Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 13:51:23 -0400 Subject: [PATCH 053/136] perf(hogql): benchmark native compilation --- core/trino-hogql-compiler/pom.xml | 18 ++ .../compiler/BenchmarkHogQlCompiler.java | 169 ++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/BenchmarkHogQlCompiler.java diff --git a/core/trino-hogql-compiler/pom.xml b/core/trino-hogql-compiler/pom.xml index 6d86f84703c3..711f62ad8eee 100644 --- a/core/trino-hogql-compiler/pom.xml +++ b/core/trino-hogql-compiler/pom.xml @@ -39,6 +39,12 @@ trino-spi + + io.trino + trino-testing-services + test + + org.assertj assertj-core @@ -62,5 +68,17 @@ junit-jupiter-params test + + + org.openjdk.jmh + jmh-core + test + + + + org.openjdk.jmh + jmh-generator-annprocess + test + diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/BenchmarkHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/BenchmarkHogQlCompiler.java new file mode 100644 index 000000000000..6e9edeb9569d --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/BenchmarkHogQlCompiler.java @@ -0,0 +1,169 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCallRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.sql.tree.Statement; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.profile.GCProfiler; +import org.openjdk.jmh.runner.RunnerException; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.TimeUnit; + +import static io.trino.jmh.Benchmarks.benchmark; + +@State(Scope.Thread) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) +public class BenchmarkHogQlCompiler +{ + private static final String SIMPLE_QUERY = "SELECT event, distinct_id FROM events WHERE team_id = 42"; + private static final String JOIN_WINDOW_QUERY = + """ + SELECT e.event, p.person_id, + row_number() OVER (PARTITION BY e.distinct_id ORDER BY e.timestamp) AS row_number + FROM events AS e + LEFT JOIN persons AS p ON e.distinct_id = p.distinct_id + WHERE e.event = 'signup' + """; + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("ducklake", false); + private static final HogQlSemanticCatalogSnapshot SEMANTIC_SNAPSHOT = semanticSnapshot(); + private static final Optional SEMANTIC_CONTEXT = Optional.of( + new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SEMANTIC_SNAPSHOT))); + private static final HogQlCompileEnvelope SEMANTIC_QUERY = new HogQlCompileEnvelope( + "SELECT lowerEvent FROM events WHERE event = 'signup'", + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(1)); + + private final HogQlCompiler compiler = new HogQlCompiler(); + + @Benchmark + public Statement compileSimpleQuery() + { + return compiler.compile(SIMPLE_QUERY); + } + + @Benchmark + public Statement compileJoinWindowQuery() + { + return compiler.compile(JOIN_WINDOW_QUERY); + } + + @Benchmark + public HogQlCompilationResult compileSemanticCatalogQuery() + { + return compiler.compile(SEMANTIC_QUERY, SEMANTIC_CONTEXT); + } + + static void main() + throws RunnerException + { + benchmark(BenchmarkHogQlCompiler.class) + .withOptions(options -> options.addProfiler(GCProfiler.class)) + .run(); + } + + private static HogQlSemanticCatalogSnapshot semanticSnapshot() + { + LogicalFieldDefinition event = new LogicalFieldDefinition( + "event", + new PhysicalIdentifier("event_name", false), + "varchar", + LogicalType.STRING, + false, + true); + LogicalFieldDefinition distinctId = new LogicalFieldDefinition( + "distinct_id", + new PhysicalIdentifier("distinct_id", false), + "varchar", + LogicalType.STRING, + false, + true); + LogicalTableDefinition events = new LogicalTableDefinition( + "events", + new PhysicalQualifiedName( + CATALOG, + new PhysicalIdentifier("analytics", false), + new PhysicalIdentifier("events_data", false)), + List.of(event, distinctId), + List.of(), + List.of()); + ExpressionFieldDefinition lowerEvent = new ExpressionFieldDefinition( + "events", + "lowerEvent", + "varchar", + LogicalType.STRING, + false, + true, + new FunctionCallRecipe("lower", List.of(new FieldReferenceRecipe("events", "event")))); + FunctionCapabilityDefinition lower = new FunctionCapabilityDefinition( + "lower", + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of(new PhysicalIdentifier("lower", false)), + List.of(new FunctionSignature(List.of("varchar"), "varchar", false)), + true, + false, + false, + false, + false); + return new HogQlSemanticCatalogSnapshot( + HogQlSemanticCatalogSnapshot.PROTOCOL_VERSION, + HogQlSemanticCatalogSnapshot.SCHEMA_VERSION, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 1, + List.of(events), + List.of(lowerEvent), + List.of(), + List.of(), + List.of(), + List.of(lower), + List.of()); + } +} From 6a554937800c54b2f09e0c76e5772a7fd7854fd7 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 13:54:45 -0400 Subject: [PATCH 054/136] feat(hogql): expand generic properties --- .../hogql/compiler/HogQlSemanticResolver.java | 229 ++++++++++++++++-- .../catalog/HogQlSemanticCatalogSnapshot.java | 12 +- .../compiler/TestHogQlSemanticExpansion.java | 145 +++++++++-- .../TestHogQlSemanticCatalogSnapshot.java | 12 +- ...gQlSemanticCatalogSnapshotJsonDecoder.java | 6 +- .../trino/hogql/parser/TestHogQlParser.java | 14 ++ 6 files changed, 374 insertions(+), 44 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index df9b5a005d8f..00e042679801 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -15,6 +15,7 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CastRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionArgument; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; @@ -26,6 +27,7 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.MaterializedViewReference; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyLookupRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ReferencedField; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; @@ -527,15 +529,9 @@ private Expression resolveExpression(Expression expression) isNull.predicateSpan(), isNull.span()); case Literal literal -> literal; - case MemberAccessExpression memberAccess -> new MemberAccessExpression( - resolveExpression(memberAccess.base()), - memberAccess.member(), - memberAccess.span()); + case MemberAccessExpression memberAccess -> resolveMemberAccess(memberAccess); case Placeholder placeholder -> placeholder; - case SubscriptExpression subscript -> new SubscriptExpression( - resolveExpression(subscript.base()), - resolveExpression(subscript.index()), - subscript.span()); + case SubscriptExpression subscript -> resolveSubscript(subscript); case TupleExpression tuple -> new TupleExpression(tuple.values().stream().map(this::resolveExpression).toList(), tuple.span()); case UnaryExpression unary -> new UnaryExpression(unary.operator(), resolveExpression(unary.operand()), unary.span()); }; @@ -576,6 +572,10 @@ private WindowFrame resolveWindowFrame(WindowFrame frame) private Expression resolveColumn(ColumnReference reference) { List parts = reference.parts(); + Optional propertyAccess = resolveDottedPropertyAccess(reference); + if (propertyAccess.isPresent()) { + return propertyAccess.orElseThrow(); + } if (parts.size() == 2) { Optional binding = bindings.stream() .filter(candidate -> candidate.qualifier().equals(canonical(parts.getFirst().value()))) @@ -617,6 +617,114 @@ private Expression resolveColumn(ColumnReference reference) return reference; } + private Expression resolveMemberAccess(MemberAccessExpression memberAccess) + { + if (memberAccess.base() instanceof ColumnReference reference) { + Optional propertyAccess = resolvePropertyAccess( + reference, + new Literal(HogQlQuery.LiteralKind.STRING, memberAccess.member().value(), memberAccess.member().span()), + memberAccess.span()); + if (propertyAccess.isPresent()) { + return propertyAccess.orElseThrow(); + } + } + return new MemberAccessExpression( + resolveExpression(memberAccess.base()), + memberAccess.member(), + memberAccess.span()); + } + + private Expression resolveSubscript(SubscriptExpression subscript) + { + if (subscript.base() instanceof ColumnReference reference) { + Optional propertyAccess = resolvePropertyAccess( + reference, + subscript.index(), + subscript.span()); + if (propertyAccess.isPresent()) { + return propertyAccess.orElseThrow(); + } + } + return new SubscriptExpression( + resolveExpression(subscript.base()), + resolveExpression(subscript.index()), + subscript.span()); + } + + private Optional resolveDottedPropertyAccess(ColumnReference reference) + { + List parts = reference.parts(); + if (parts.size() == 2 && bindings.stream().noneMatch(binding -> binding.qualifier().equals(canonical(parts.getFirst().value())))) { + return resolvePropertyAccess( + new ColumnReference(List.of(parts.getFirst()), parts.getFirst().span()), + new Literal(HogQlQuery.LiteralKind.STRING, parts.getLast().value(), parts.getLast().span()), + reference.span()); + } + if (parts.size() == 3) { + return resolvePropertyAccess( + new ColumnReference(parts.subList(0, 2), reference.span()), + new Literal(HogQlQuery.LiteralKind.STRING, parts.getLast().value(), parts.getLast().span()), + reference.span()); + } + return Optional.empty(); + } + + private Optional resolvePropertyAccess( + ColumnReference propertyReference, + Expression key, + HogQlQuery.SourceSpan span) + { + List parts = propertyReference.parts(); + List candidateBindings; + String propertyName; + boolean qualified; + if (parts.size() == 1) { + candidateBindings = bindings; + propertyName = parts.getFirst().value(); + qualified = false; + } + else if (parts.size() == 2) { + candidateBindings = bindings.stream() + .filter(binding -> binding.qualifier().equals(canonical(parts.getFirst().value()))) + .toList(); + propertyName = parts.getLast().value(); + qualified = true; + } + else { + return Optional.empty(); + } + + List propertyMatches = candidateBindings.stream() + .flatMap(binding -> properties(binding).stream() + .filter(property -> canonical(property.name()).equals(canonical(propertyName))) + .map(property -> new PropertyMatch(binding, property))) + .toList(); + if (propertyMatches.isEmpty()) { + return Optional.empty(); + } + List matchedBindings = propertyMatches.stream() + .map(PropertyMatch::binding) + .distinct() + .toList(); + if (matchedBindings.size() > 1) { + throw ambiguousPropertyResolutionError(span, propertyName); + } + + PropertyDefinition property = propertyMatches.getFirst().property(); + TableBinding binding = matchedBindings.getFirst(); + Optional qualifier = qualified || bindings.size() > 1 + ? Optional.of(binding.outputQualifier()) + : Optional.empty(); + return Optional.of(expandProperty(binding, property, resolveExpression(key), qualifier, span, expansionBudget)); + } + + private List properties(TableBinding binding) + { + return snapshot.logicalTable(binding.relationName()) + .map(LogicalTableDefinition::properties) + .orElse(List.of()); + } + private Expression resolveBoundField( TableBinding binding, BoundField field, @@ -644,6 +752,17 @@ private Expression expandRecipe( Optional qualifier, HogQlQuery.SourceSpan span, ExpansionBudget budget) + { + return expandRecipe(binding, recipe, qualifier, span, budget, Map.of()); + } + + private Expression expandRecipe( + TableBinding binding, + ExpressionRecipe recipe, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget, + Map arguments) { budget.enter(span); try { @@ -656,16 +775,22 @@ private Expression expandRecipe( yield resolveBoundField(binding, field, qualifier, span, budget); } case LiteralRecipe literal -> typedLiteral(literal.literal(), span); - case FunctionCallRecipe function -> expandFunction(binding, function, qualifier, span, budget); - case OperatorRecipe operator -> expandOperator(binding, operator, qualifier, span, budget); + case FunctionCallRecipe function -> expandFunction(binding, function, qualifier, span, budget, arguments); + case OperatorRecipe operator -> expandOperator(binding, operator, qualifier, span, budget, arguments); case CastRecipe cast -> new CastExpression( - expandRecipe(binding, cast.expression(), qualifier, span, budget), + expandRecipe(binding, cast.expression(), qualifier, span, budget, arguments), new Identifier(cast.targetTypeSignature(), false, span), false, span); - case ArgumentReferenceRecipe _ -> throw unsupportedExpansion(span, "HogQL recipe argument is only valid inside a property lookup recipe"); + case ArgumentReferenceRecipe reference -> { + Expression argument = arguments.get(reference.argument()); + if (argument == null) { + throw unsupportedExpansion(span, "HogQL recipe argument is unavailable in this expansion context"); + } + yield argument; + } case ScopedFieldReferenceRecipe _ -> throw unsupportedExpansion(span, "HogQL scoped field recipe is only valid inside a relationship predicate"); - case PropertyLookupRecipe _ -> throw unsupportedExpansion(span, "HogQL property lookup recipe expansion is unavailable"); + case PropertyLookupRecipe lookup -> expandPropertyLookup(binding, lookup, qualifier, span, budget, arguments); }; } finally { @@ -678,7 +803,8 @@ private Expression expandFunction( FunctionCallRecipe function, Optional qualifier, HogQlQuery.SourceSpan span, - ExpansionBudget budget) + ExpansionBudget budget, + Map arguments) { FunctionCapabilityDefinition capability = snapshot.snapshot().functions().stream() .filter(candidate -> canonical(candidate.name()).equals(canonical(function.name()))) @@ -699,7 +825,7 @@ private Expression expandFunction( .map(name -> new Identifier(name.value(), name.delimited(), span)) .toList(), function.arguments().stream() - .map(argument -> expandRecipe(binding, argument, qualifier, span, budget)) + .map(argument -> expandRecipe(binding, argument, qualifier, span, budget, arguments)) .toList(), false, List.of(), @@ -712,10 +838,11 @@ private Expression expandOperator( OperatorRecipe operator, Optional qualifier, HogQlQuery.SourceSpan span, - ExpansionBudget budget) + ExpansionBudget budget, + Map recipeArguments) { List arguments = operator.arguments().stream() - .map(argument -> expandRecipe(binding, argument, qualifier, span, budget)) + .map(argument -> expandRecipe(binding, argument, qualifier, span, budget, recipeArguments)) .toList(); return switch (operator.operator()) { case NOT -> new UnaryExpression(HogQlQuery.UnaryOperator.NOT, arguments.getFirst(), span); @@ -727,6 +854,63 @@ private Expression expandOperator( }; } + private Expression expandPropertyLookup( + TableBinding binding, + PropertyLookupRecipe lookup, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget, + Map arguments) + { + if (!canonical(binding.relationName()).equals(canonical(lookup.table()))) { + throw expansionError(span, "HogQL property lookup recipe references an unavailable table"); + } + PropertyDefinition property = properties(binding).stream() + .filter(candidate -> canonical(candidate.name()).equals(canonical(lookup.property()))) + .findFirst() + .orElseThrow(() -> expansionError(span, "HogQL property lookup recipe references an unavailable property")); + Expression key = expandRecipe(binding, lookup.key(), qualifier, span, budget, arguments); + return expandProperty(binding, property, key, qualifier, span, budget); + } + + private Expression expandProperty( + TableBinding binding, + PropertyDefinition property, + Expression key, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget) + { + budget.enter(span); + try { + ExpressionRecipe recipe = property.lookupRecipe() + .orElseThrow(() -> unsupportedExpansion(span, "HogQL property lookup has no declared compiler recipe")); + String keyType = property.keyTypeSignature() + .orElseThrow(() -> unsupportedExpansion(span, "HogQL property lookup has no declared key type")); + String valueType = property.valueTypeSignature() + .orElseThrow(() -> unsupportedExpansion(span, "HogQL property lookup has no declared value type")); + BoundField source = binding.fields().get(canonical(property.sourceField())); + if (source == null) { + throw expansionError(span, "HogQL property lookup references an unavailable source field"); + } + Expression sourceExpression = resolveBoundField(binding, source, qualifier, span, budget); + Expression typedKey = new CastExpression(key, new Identifier(keyType, false, key.span()), false, key.span()); + Expression value = expandRecipe( + binding, + recipe, + qualifier, + span, + budget, + Map.of( + ExpressionArgument.PROPERTY_SOURCE, sourceExpression, + ExpressionArgument.PROPERTY_KEY, typedKey)); + return new CastExpression(value, new Identifier(valueType, false, span), false, span); + } + finally { + budget.exit(); + } + } + private static HogQlQuery.BinaryOperator binaryOperator(SemanticOperator operator) { return switch (operator) { @@ -841,6 +1025,15 @@ private static TrinoException ambiguousResolutionError(ColumnReference reference null); } + private static TrinoException ambiguousPropertyResolutionError(HogQlQuery.SourceSpan span, String name) + { + return new TrinoException( + HOGQL_RESOLUTION_ERROR, + Optional.of(new Location(span.startLine(), span.startColumn())), + "Ambiguous HogQL property source: " + name, + null); + } + private static TrinoException incompatibleUsingResolutionError(Identifier identifier) { return new TrinoException( @@ -924,6 +1117,8 @@ private record BoundField(String name, PhysicalIdentifier physicalColumn, boolea private record FieldMatch(TableBinding binding, BoundField field) {} + private record PropertyMatch(TableBinding binding, PropertyDefinition property) {} + private record ProjectedField(String name, String sourceField, boolean starVisible) { private ProjectedField diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 6eaf4bb04e0f..4a480007d57c 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -143,7 +143,17 @@ private static Map indexTables(PhysicalIdentifie } Set members = new HashSet<>(); table.fields().forEach(field -> addUnique(members, field.name(), "duplicate logical member")); - table.properties().forEach(property -> addUnique(members, property.name(), "duplicate logical member")); + Set properties = new HashSet<>(); + for (PropertyDefinition property : table.properties()) { + String name = canonical(property.name(), "property name"); + if (!properties.add(name)) { + throw new IllegalArgumentException("duplicate logical member"); + } + if (members.contains(name) && !name.equals(canonical(property.sourceField(), "property source field"))) { + throw new IllegalArgumentException("duplicate logical member"); + } + members.add(name); + } table.relationships().forEach(relationship -> addUnique(members, relationship.name(), "duplicate logical member")); } return Map.copyOf(tables); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticExpansion.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticExpansion.java index e4ad1bf27c35..1772abd77e74 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticExpansion.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticExpansion.java @@ -14,7 +14,9 @@ package io.trino.hogql.compiler; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CastRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionArgument; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCallRecipe; @@ -31,6 +33,9 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyLookupRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ReferencedField; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationReference; @@ -54,6 +59,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_COMPILER_LIMIT_EXCEEDED; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowableOfType; @@ -123,7 +129,61 @@ public void testPinsOnceAcrossRecursiveExpansionAndPreservesPlaceholderOrder() } @Test - public void testBoundsRecursiveExpressionExpansionAtOriginalLocation() + public void testExpandsStaticAndQualifiedPropertyAccessThroughDeclaredRecipe() + { + HogQlCompilationResult result = compile("SELECT properties.browser, e.properties.browser, (properties).browser FROM events e"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT CAST(event_properties_blob[CAST('browser' AS varchar)] AS varchar) AS browser, " + + "CAST(e.event_properties_blob[CAST('browser' AS varchar)] AS varchar) AS browser, " + + "CAST(event_properties_blob[CAST('browser' AS varchar)] AS varchar) " + + "FROM analytics.data.raw_events e")); + } + + @Test + public void testExpandsDynamicPropertyAccessAndPreservesPlaceholder() + { + HogQlCompilationResult result = compile( + "SELECT properties[{key}] FROM events", + SNAPSHOT, + Map.of("key", new HogQlTypedValue("varchar", new HogQlTypedValue.StringValue("browser")))); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT CAST(event_properties_blob[CAST(? AS varchar)] AS varchar) FROM analytics.data.raw_events")); + assertThat(result.parameterNames()).containsExactly("key"); + } + + @Test + public void testExpandsPropertyLookupRecipeThroughDeclaredPropertyRecipe() + { + HogQlCompilationResult result = compile("SELECT browserProperty FROM events"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT CAST(event_properties_blob[CAST(CAST('browser' AS varchar) AS varchar)] AS varchar) AS browserProperty " + + "FROM analytics.data.raw_events")); + } + + @Test + public void testRejectsPropertyAccessWithoutDeclaredRecipeAtOriginalLocation() + { + PropertyDefinition property = new PropertyDefinition( + "properties", + "eventProperties", + PropertyStorage.JSON_OBJECT, + LogicalType.STRING, + true); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compile("SELECT properties.browser FROM events", propertySnapshot(property, List.of()))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, 8)); + assertThat(exception).hasMessage("line 1:8: HogQL property lookup has no declared compiler recipe"); + } + + @Test + public void testBoundsPropertyLookupExpansionAtOriginalLocation() { List expressions = new ArrayList<>(); expressions.add(expression("branch0", new LiteralRecipe(new TypedLiteral("bigint", LiteralEncoding.INTEGER, "1")))); @@ -133,23 +193,15 @@ public void testBoundsRecursiveExpressionExpansionAtOriginalLocation() new FieldReferenceRecipe("events", previous), new FieldReferenceRecipe("events", previous))))); } - HogQlSemanticCatalogSnapshot expansiveSnapshot = new HogQlSemanticCatalogSnapshot( - SNAPSHOT.protocolVersion(), - SNAPSHOT.schemaVersion(), - SNAPSHOT.languageVersion(), - SNAPSHOT.catalog(), - SNAPSHOT.generation(), - SNAPSHOT.logicalTables(), - expressions, - List.of(), - List.of(), - List.of(), - List.of(), - List.of()); + expressions.add(expression("lookupBranch", new PropertyLookupRecipe( + "events", + "properties", + new FieldReferenceRecipe("events", "branch14")))); + HogQlSemanticCatalogSnapshot expansiveSnapshot = propertySnapshot(propertiesProperty(), expressions); TrinoException exception = catchThrowableOfType( TrinoException.class, - () -> compile("SELECT branch14 FROM events", expansiveSnapshot)); + () -> compile("SELECT lookupBranch FROM events", expansiveSnapshot)); assertThat(exception.getErrorCode()).isEqualTo(HOGQL_COMPILER_LIMIT_EXCEEDED.toErrorCode()); assertThat(exception.getLocation()).contains(new Location(1, 8)); @@ -162,13 +214,18 @@ private HogQlCompilationResult compile(String query) } private HogQlCompilationResult compile(String query, HogQlSemanticCatalogSnapshot snapshot) + { + return compile(query, snapshot, Map.of()); + } + + private HogQlCompilationResult compile(String query, HogQlSemanticCatalogSnapshot snapshot, Map parameters) { HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(snapshot)); return compiler.compile(new HogQlCompileEnvelope( query, HogQlCompileEnvelope.PROTOCOL_VERSION, HogQlLanguageContract.current().languageVersion(), - Map.of(), + parameters, Map.of(), Map.of(), Map.of(), @@ -178,6 +235,7 @@ private HogQlCompilationResult compile(String query, HogQlSemanticCatalogSnapsho private static HogQlSemanticCatalogSnapshot snapshot() { LogicalFieldDefinition event = new LogicalFieldDefinition("event", new PhysicalIdentifier("event_name", false), "varchar", LogicalType.STRING, false, true); + LogicalFieldDefinition properties = new LogicalFieldDefinition("eventProperties", new PhysicalIdentifier("event_properties_blob", false), "map(varchar, varchar)", LogicalType.MAP, true, false); List expressions = List.of( expression("constant", new LiteralRecipe(new TypedLiteral("bigint", LiteralEncoding.INTEGER, "7"))), expression("upperEvent", new FunctionCallRecipe("hogUpper", List.of(new FieldReferenceRecipe("events", "event")))), @@ -185,7 +243,11 @@ private static HogQlSemanticCatalogSnapshot snapshot() new FieldReferenceRecipe("events", "constant"), new LiteralRecipe(new TypedLiteral("bigint", LiteralEncoding.INTEGER, "1"))))), expression("castEvent", new CastRecipe(new FieldReferenceRecipe("events", "event"), "varchar")), - expression("missingEvent", new OperatorRecipe(SemanticOperator.IS_NULL, List.of(new FieldReferenceRecipe("events", "event"))))); + expression("missingEvent", new OperatorRecipe(SemanticOperator.IS_NULL, List.of(new FieldReferenceRecipe("events", "event")))), + expression("browserProperty", new PropertyLookupRecipe( + "events", + "properties", + new LiteralRecipe(new TypedLiteral("varchar", LiteralEncoding.STRING, "browser"))))); ReferencedField eventTitle = new ReferencedField("eventTitle", "varchar", LogicalType.STRING, false, true); return new HogQlSemanticCatalogSnapshot( 1, @@ -196,8 +258,8 @@ private static HogQlSemanticCatalogSnapshot snapshot() List.of(new LogicalTableDefinition( "events", physicalName("raw_events"), - List.of(event), - List.of(), + List.of(event, properties), + List.of(propertiesProperty()), List.of())), expressions, List.of(new VirtualTableDefinition( @@ -232,6 +294,51 @@ private static HogQlSemanticCatalogSnapshot snapshot() List.of()); } + private static HogQlSemanticCatalogSnapshot propertySnapshot(PropertyDefinition property, List expressions) + { + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 7, + List.of(new LogicalTableDefinition( + "events", + physicalName("raw_events"), + List.of(new LogicalFieldDefinition( + "eventProperties", + new PhysicalIdentifier("event_properties_blob", false), + "map(varchar, varchar)", + LogicalType.MAP, + true, + false)), + List.of(property), + List.of())), + expressions, + List.of(), + List.of(), + List.of(), + List.of(), + List.of()); + } + + private static PropertyDefinition propertiesProperty() + { + return new PropertyDefinition( + "properties", + "eventProperties", + PropertyStorage.JSON_OBJECT, + LogicalType.STRING, + true, + Optional.of("varchar"), + Optional.of("varchar"), + Optional.of(new OperatorRecipe( + SemanticOperator.SUBSCRIPT, + List.of( + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE), + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_KEY))))); + } + private static ExpressionFieldDefinition expression(String name, HogQlSemanticCatalogSnapshot.ExpressionRecipe recipe) { return new ExpressionFieldDefinition("events", name, "bigint", LogicalType.INTEGER, false, true, recipe); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java index 610f5d436c9d..e172e758c471 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java @@ -174,13 +174,13 @@ public void testLogicalSemanticRecipesAreDeeplyImmutable() LogicalType.STRING, true, true, - propertyLookup("events", "browser")))); + propertyLookup("events", "properties")))); List lazyTables = new ArrayList<>(List.of(new LazyTableDefinition("events", "profile", relationshipPath, projections))); List actions = new ArrayList<>(List.of(new ActionReference( "paid", "action-1", "events", - new PredicateRepresentation(propertyLookup("events", "browser"))))); + new PredicateRepresentation(propertyLookup("events", "properties"))))); List cohorts = new ArrayList<>(List.of(new CohortReference( "active", "cohort-1", @@ -242,7 +242,7 @@ public void testRejectsInvalidLogicalSemanticScopes() "events", "profile", List.of("missing"), - List.of(new LazyProjectionDefinition("browser", "varchar", LogicalType.STRING, true, true, propertyLookup("events", "browser"))))), + List.of(new LazyProjectionDefinition("browser", "varchar", LogicalType.STRING, true, true, propertyLookup("events", "properties"))))), List.of(), List.of())) .isInstanceOf(IllegalArgumentException.class) @@ -353,6 +353,10 @@ private static Stream invalidSnapshots() "duplicate members", List.of(table("events", List.of(field("id"), field("ID")), List.of(), List.of())), "duplicate logical member"), + Arguments.of( + "property collides with another field", + List.of(table("events", List.of(field("id"), field("properties")), List.of(property("id", "properties")), List.of())), + "duplicate logical member"), Arguments.of( "missing property field", List.of(table("events", List.of(field("id")), List.of(property("properties", "missing")), List.of())), @@ -487,7 +491,7 @@ private static HogQlSemanticCatalogSnapshot logicalSemanticSnapshot( private static PropertyDefinition propertyWithLookup(ExpressionRecipe recipe) { return new PropertyDefinition( - "browser", + "properties", "properties", PropertyStorage.JSON_OBJECT, LogicalType.STRING, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java index 0f5523032504..f4257d789c05 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java @@ -298,7 +298,7 @@ private static String validSnapshotJson() ], "properties": [ { - "name": "browser", + "name": "properties", "sourceField": "properties", "storage": "JSON_OBJECT", "logicalType": "STRING", @@ -455,7 +455,7 @@ private static String validSnapshotJson() "kind": "PROPERTY_LOOKUP", "propertyLookup": { "table": "events", - "property": "browser", + "property": "properties", "key": {"kind": "LITERAL", "literal": {"typeSignature": "varchar", "encoding": "STRING", "value": "browser"}} } } @@ -474,7 +474,7 @@ private static String validSnapshotJson() "kind": "PROPERTY_LOOKUP", "propertyLookup": { "table": "events", - "property": "browser", + "property": "properties", "key": {"kind": "LITERAL", "literal": {"typeSignature": "varchar", "encoding": "STRING", "value": "browser"}} } } diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index d29446aee7b2..818fd3fddb83 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -158,6 +158,20 @@ public void testBuildsCollectionAccessWithSourceSpans() assertThat(memberAccess.member().span()).isEqualTo(new HogQlQuery.SourceSpan(56, 60, 1, 57, 1, 61)); } + @Test + public void testBuildsPropertyAccessPaths() + { + HogQlQuery query = parser.parseStatement("SELECT properties.browser, e.properties.browser FROM events e"); + + ColumnReference unqualified = (ColumnReference) ((ExpressionProjection) query.projections().getFirst()).expression(); + assertThat(unqualified.parts()).extracting(HogQlQuery.Identifier::value).containsExactly("properties", "browser"); + assertThat(unqualified.span()).isEqualTo(new HogQlQuery.SourceSpan(7, 25, 1, 8, 1, 26)); + + ColumnReference qualified = (ColumnReference) ((ExpressionProjection) query.projections().get(1)).expression(); + assertThat(qualified.parts()).extracting(HogQlQuery.Identifier::value).containsExactly("e", "properties", "browser"); + assertThat(qualified.span()).isEqualTo(new HogQlQuery.SourceSpan(27, 47, 1, 28, 1, 48)); + } + @Test public void testBuildsOrdinaryGroupingHavingAndAggregateFunctionAst() { From 2cf9e2b4f5f06298f22af2393a3428442e4b7432 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 14:10:23 -0400 Subject: [PATCH 055/136] feat(hogql): lower null predicate rewrites --- .../hogql/compiler/HogQlFunctionResolver.java | 19 +++- .../catalog/HogQlSemanticCatalogSnapshot.java | 61 ++++++++++- ...gQlSemanticCatalogSnapshotJsonDecoder.java | 7 +- .../compiler/TestHogQlFunctionResolver.java | 84 ++++++++++++--- .../TestHogQlSemanticCatalogSnapshot.java | 100 ++++++++++++++++++ ...gQlSemanticCatalogSnapshotJsonDecoder.java | 85 +++++++++++++++ 6 files changed, 332 insertions(+), 24 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index bc2ad64c23bb..0bc89b4d55ab 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -16,6 +16,7 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionRewrite; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; @@ -221,15 +222,15 @@ private Expression resolveExpression(Expression expression) }; } - private FunctionCall resolveFunction(FunctionCall function, boolean windowInvocation) + private Expression resolveFunction(FunctionCall function, boolean windowInvocation) { String name = function.name().value(); FunctionCapabilityDefinition capability = functions.get(canonical(name)); if (capability == null) { throw resolutionError(function, "Unknown HogQL function: " + name); } - if (capability.implementation() == FunctionImplementation.REWRITE) { - throw unsupportedError(function, "HogQL function " + name + " requires a compiler rewrite"); + if (capability.implementation() == FunctionImplementation.REWRITE && function.nullTreatment().isPresent()) { + throw unsupportedError(function, "HogQL function " + name + " does not support null treatment"); } if (capability.kind() == FunctionKind.TABLE) { throw unsupportedError(function, "HogQL table function " + name + " cannot be used as an expression"); @@ -252,11 +253,21 @@ private FunctionCall resolveFunction(FunctionCall function, boolean windowInvoca if (function.filter().isPresent() && !capability.supportsFilter()) { throw unsupportedError(function, "HogQL function " + name + " does not support FILTER"); } + List arguments = function.arguments().stream().map(this::resolveExpression).toList(); + if (capability.implementation() == FunctionImplementation.REWRITE) { + FunctionRewrite rewrite = capability.rewrite() + .orElseThrow(() -> unsupportedError(function, "HogQL function " + name + " has no compiler rewrite")); + boolean negated = switch (rewrite) { + case IS_NULL -> false; + case IS_NOT_NULL -> true; + }; + return new IsNullExpression(arguments.getFirst(), negated, function.span(), function.span()); + } return new FunctionCall( capability.trinoName().stream() .map(identifier -> identifier(identifier, function.span())) .toList(), - function.arguments().stream().map(this::resolveExpression).toList(), + arguments, function.distinct(), resolveSortItems(function.orderBy()), function.filter().map(this::resolveExpression), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 4a480007d57c..fc47af398580 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -244,8 +244,42 @@ private static Map validateFunctions(List< if (function.implementation() == FunctionImplementation.REWRITE && !function.trinoName().isEmpty()) { throw new IllegalArgumentException("rewrite function cannot name a Trino function"); } - if (function.implementation() != FunctionImplementation.REWRITE && function.trinoName().isEmpty()) { - throw new IllegalArgumentException("function must name a Trino function"); + if (function.implementation() == FunctionImplementation.REWRITE) { + if (function.rewrite().isEmpty()) { + throw new IllegalArgumentException("rewrite function must declare a rewrite"); + } + if (function.kind() != FunctionKind.SCALAR) { + throw new IllegalArgumentException("rewrite function must be scalar"); + } + if (!function.deterministic()) { + throw new IllegalArgumentException("rewrite function must be deterministic"); + } + if (function.supportsDistinct()) { + throw new IllegalArgumentException("rewrite function cannot support DISTINCT"); + } + if (function.supportsOrderBy()) { + throw new IllegalArgumentException("rewrite function cannot support ORDER BY"); + } + if (function.supportsFilter()) { + throw new IllegalArgumentException("rewrite function cannot support FILTER"); + } + if (function.supportsWindow()) { + throw new IllegalArgumentException("rewrite function cannot support window invocation"); + } + if (function.signatures().stream().anyMatch(signature -> signature.variadic() || signature.argumentTypes().size() != 1)) { + throw new IllegalArgumentException("rewrite function must declare unary signatures"); + } + if (function.signatures().stream().anyMatch(signature -> !signature.returnType().equalsIgnoreCase("boolean"))) { + throw new IllegalArgumentException("rewrite function signatures must return boolean"); + } + } + else { + if (function.rewrite().isPresent()) { + throw new IllegalArgumentException("non-rewrite function cannot declare a rewrite"); + } + if (function.trinoName().isEmpty()) { + throw new IllegalArgumentException("function must name a Trino function"); + } } } return Map.copyOf(functions); @@ -1026,7 +1060,7 @@ public record ReferencedField(String name, String trinoTypeSignature, LogicalTyp } } - public record FunctionCapabilityDefinition(String name, FunctionKind kind, FunctionImplementation implementation, List trinoName, List signatures, boolean deterministic, boolean supportsDistinct, boolean supportsOrderBy, boolean supportsFilter, boolean supportsWindow) + public record FunctionCapabilityDefinition(String name, FunctionKind kind, FunctionImplementation implementation, List trinoName, Optional rewrite, List signatures, boolean deterministic, boolean supportsDistinct, boolean supportsOrderBy, boolean supportsFilter, boolean supportsWindow) { public FunctionCapabilityDefinition { @@ -1034,11 +1068,27 @@ public record FunctionCapabilityDefinition(String name, FunctionKind kind, Funct kind = requireNonNull(kind, "kind is null"); implementation = requireNonNull(implementation, "implementation is null"); trinoName = copy(trinoName, "trinoName"); + rewrite = requireNonNull(rewrite, "rewrite is null"); signatures = copy(signatures, "signatures"); if (signatures.isEmpty()) { throw new IllegalArgumentException("function must include signatures"); } } + + public FunctionCapabilityDefinition( + String name, + FunctionKind kind, + FunctionImplementation implementation, + List trinoName, + List signatures, + boolean deterministic, + boolean supportsDistinct, + boolean supportsOrderBy, + boolean supportsFilter, + boolean supportsWindow) + { + this(name, kind, implementation, trinoName, Optional.empty(), signatures, deterministic, supportsDistinct, supportsOrderBy, supportsFilter, supportsWindow); + } } public record FunctionSignature(List argumentTypes, String returnType, boolean variadic) @@ -1212,6 +1262,11 @@ public enum FunctionImplementation STOCK, UDF, REWRITE } + public enum FunctionRewrite + { + IS_NULL, IS_NOT_NULL + } + public enum ModifierBehavior { COMPILER, TRINO_SESSION_PROPERTY, SAFE_NOOP, UNSUPPORTED diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java index f45a9ec5cb92..e9417cacca22 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java @@ -35,6 +35,7 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionRewrite; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyProjectionDefinition; @@ -150,7 +151,8 @@ public final class HogQlSemanticCatalogSnapshotJsonDecoder private static final Set SAVED_QUERY_FIELDS = Set.of("name", "queryId", "target", "fields"); private static final Set MATERIALIZED_VIEW_FIELDS = Set.of("name", "physicalView", "fields"); private static final Set REFERENCED_FIELD_FIELDS = Set.of("name", "trinoTypeSignature", "logicalType", "nullable", "starVisible"); - private static final Set FUNCTION_FIELDS = Set.of("name", "kind", "implementation", "trinoName", "signatures", "deterministic", "supportsDistinct", "supportsOrderBy", "supportsFilter", "supportsWindow"); + private static final Set FUNCTION_FIELDS = Set.of("name", "kind", "implementation", "trinoName", "rewrite", "signatures", "deterministic", "supportsDistinct", "supportsOrderBy", "supportsFilter", "supportsWindow"); + private static final Set REQUIRED_FUNCTION_FIELDS = Set.of("name", "kind", "implementation", "trinoName", "signatures", "deterministic", "supportsDistinct", "supportsOrderBy", "supportsFilter", "supportsWindow"); private static final Set FUNCTION_SIGNATURE_FIELDS = Set.of("argumentTypes", "returnType", "variadic"); private static final Set MODIFIER_FIELDS = Set.of("name", "behavior", "defaultValue", "sessionProperty"); private static final Set MODIFIER_FIELDS_WITHOUT_SESSION_PROPERTY = Set.of("name", "behavior", "defaultValue"); @@ -557,12 +559,13 @@ private static List functions(JsonNode node, Colle List definitions = new ArrayList<>(array.size()); for (JsonNode element : array) { ObjectNode function = object(element); - validateFields(function, FUNCTION_FIELDS); + validateFields(function, FUNCTION_FIELDS, REQUIRED_FUNCTION_FIELDS); definitions.add(new FunctionCapabilityDefinition( text(function, "name"), enumValue(function, "kind", FunctionKind.class), enumValue(function, "implementation", FunctionImplementation.class), physicalIdentifiers(required(function, "trinoName"), budget), + function.has("rewrite") ? Optional.of(enumValue(function, "rewrite", FunctionRewrite.class)) : Optional.empty(), functionSignatures(required(function, "signatures"), budget), bool(function, "deterministic"), bool(function, "supportsDistinct"), diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index b381abe287e3..675500a6250e 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -17,6 +17,7 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionRewrite; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; @@ -80,6 +81,26 @@ public void testCompilerPinsOnceAndResolvesUserFunctions() assertThat(result.statement()).isEqualTo(sqlParser.createStatement("SELECT ARRAY[system.builtin.upper('one')][1]")); } + @Test + public void testCompilerRewritesNullPredicatesAndPreservesPlaceholders() + { + HogQlSemanticCatalogSnapshot snapshot = snapshot(List.of( + rewriteFunction("hogIsNull", FunctionRewrite.IS_NULL), + rewriteFunction("hogIsNotNull", FunctionRewrite.IS_NOT_NULL))); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(snapshot)); + + HogQlCompilationResult result = new HogQlCompiler().compile( + envelope( + "SELECT hogIsNull({first}), hogIsNotNull({second})", + Map.of( + "first", new HogQlTypedValue("varchar", new HogQlTypedValue.StringValue("one")), + "second", new HogQlTypedValue("varchar", new HogQlTypedValue.StringValue("two")))), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement("SELECT ? IS NULL, ? IS NOT NULL")); + assertThat(result.parameterNames()).containsExactly("first", "second"); + } + @Test public void testCompilerPreservesOrdinaryFunctionsWithoutCatalogContext() { @@ -168,6 +189,10 @@ public void testRejectsCallsOutsideEveryDeclaredArity() false); assertResolutionError("SELECT exact('one', 'two')", function, "HogQL function exact does not accept 2 arguments"); + assertResolutionError( + "SELECT hogIsNull('one', 'two')", + rewriteFunction("hogIsNull", FunctionRewrite.IS_NULL), + "HogQL function hogIsNull does not accept 2 arguments"); } @ParameterizedTest @@ -206,25 +231,33 @@ private static Stream unsupportedInvocationFeatures() Arguments.of( "SELECT tableOnly(value)", function("tableOnly", FunctionKind.TABLE, FunctionImplementation.STOCK, List.of("table_only"), signature(1), false, false, false, false), - "HogQL table function tableOnly cannot be used as an expression")); + "HogQL table function tableOnly cannot be used as an expression"), + Arguments.of( + "SELECT hogIsNull(DISTINCT value)", + rewriteFunction("hogIsNull", FunctionRewrite.IS_NULL), + "HogQL function hogIsNull does not support DISTINCT"), + Arguments.of( + "SELECT hogIsNull(value ORDER BY value)", + rewriteFunction("hogIsNull", FunctionRewrite.IS_NULL), + "HogQL function hogIsNull does not support ORDER BY"), + Arguments.of( + "SELECT hogIsNull(value) FILTER (WHERE true)", + rewriteFunction("hogIsNull", FunctionRewrite.IS_NULL), + "HogQL function hogIsNull does not support FILTER"), + Arguments.of( + "SELECT hogIsNull(value) OVER ()", + rewriteFunction("hogIsNull", FunctionRewrite.IS_NULL), + "HogQL function hogIsNull does not support OVER"), + Arguments.of( + "SELECT hogIsNull(value) OVER () IGNORE NULLS", + rewriteFunction("hogIsNull", FunctionRewrite.IS_NULL), + "HogQL function hogIsNull does not support null treatment")); } @Test - public void testRejectsUnknownAndRewriteOnlyFunctionsAtCallLocation() + public void testRejectsUnknownFunctionsAtCallLocation() { assertResolutionError("SELECT missing('one')", "Unknown HogQL function: missing"); - - FunctionCapabilityDefinition rewrite = function( - "rewriteMe", - FunctionKind.SCALAR, - FunctionImplementation.REWRITE, - List.of(), - signature(1), - false, - false, - false, - false); - assertUnsupportedError("SELECT rewriteMe('one')", rewrite, "HogQL function rewriteMe requires a compiler rewrite"); } private HogQlQuery resolve(String query, FunctionCapabilityDefinition... functions) @@ -302,6 +335,22 @@ private static FunctionSignature signature(int arity) return new FunctionSignature(Stream.generate(() -> "varchar").limit(arity).toList(), "varchar", false); } + private static FunctionCapabilityDefinition rewriteFunction(String name, FunctionRewrite rewrite) + { + return new FunctionCapabilityDefinition( + name, + FunctionKind.SCALAR, + FunctionImplementation.REWRITE, + List.of(), + Optional.of(rewrite), + List.of(new FunctionSignature(List.of("varchar"), "boolean", false)), + true, + false, + false, + false, + false); + } + private static HogQlSemanticCatalogSnapshot snapshot(List functions) { return new HogQlSemanticCatalogSnapshot( @@ -320,12 +369,17 @@ private static HogQlSemanticCatalogSnapshot snapshot(List parameters) { return new HogQlCompileEnvelope( query, HogQlCompileEnvelope.PROTOCOL_VERSION, HogQlLanguageContract.current().languageVersion(), - Map.of(), + parameters, Map.of(), Map.of(), Map.of(), diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java index e172e758c471..7947d692adc2 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java @@ -24,6 +24,7 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionRewrite; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyProjectionDefinition; @@ -160,6 +161,55 @@ public void testSemanticDefinitionsAreDeeplyImmutable() assertThatThrownBy(() -> recipe.arguments().clear()).isInstanceOf(UnsupportedOperationException.class); } + @Test + public void testValidatesFunctionRewriteContract() + { + FunctionCapabilityDefinition isNull = rewriteFunction("isNull", FunctionKind.SCALAR, List.of(), Optional.of(FunctionRewrite.IS_NULL), List.of(new FunctionSignature(List.of("varchar"), "boolean", false))); + FunctionCapabilityDefinition isNotNull = rewriteFunction("isNotNull", FunctionKind.SCALAR, List.of(), Optional.of(FunctionRewrite.IS_NOT_NULL), List.of(new FunctionSignature(List.of("varchar"), "boolean", false))); + + HogQlSemanticCatalogSnapshot snapshot = semanticSnapshot(List.of(), List.of(isNull, isNotNull)); + + assertThat(snapshot.functions()).extracting(FunctionCapabilityDefinition::rewrite) + .containsExactly(Optional.of(FunctionRewrite.IS_NULL), Optional.of(FunctionRewrite.IS_NOT_NULL)); + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( + rewriteFunction("missing", FunctionKind.SCALAR, List.of(), Optional.empty(), List.of(new FunctionSignature(List.of("varchar"), "boolean", false)))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("must declare a rewrite"); + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( + rewriteFunction("named", FunctionKind.SCALAR, List.of(new PhysicalIdentifier("named", false)), Optional.of(FunctionRewrite.IS_NULL), List.of(new FunctionSignature(List.of("varchar"), "boolean", false)))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot name a Trino function"); + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( + new FunctionCapabilityDefinition( + "stock", + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of(new PhysicalIdentifier("stock", false)), + Optional.of(FunctionRewrite.IS_NULL), + List.of(new FunctionSignature(List.of("varchar"), "boolean", false)), + true, + false, + false, + false, + false)))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot declare a rewrite"); + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( + rewriteFunction("aggregate", FunctionKind.AGGREGATE, List.of(), Optional.of(FunctionRewrite.IS_NULL), List.of(new FunctionSignature(List.of("varchar"), "boolean", false)))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("must be scalar"); + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( + rewriteFunction("binary", FunctionKind.SCALAR, List.of(), Optional.of(FunctionRewrite.IS_NULL), List.of(new FunctionSignature(List.of("varchar", "varchar"), "boolean", false)))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("unary signatures"); + assertInvalidRewrite(rewriteFunction(false, false, false, false, false, "boolean"), "must be deterministic"); + assertInvalidRewrite(rewriteFunction(true, true, false, false, false, "boolean"), "cannot support DISTINCT"); + assertInvalidRewrite(rewriteFunction(true, false, true, false, false, "boolean"), "cannot support ORDER BY"); + assertInvalidRewrite(rewriteFunction(true, false, false, true, false, "boolean"), "cannot support FILTER"); + assertInvalidRewrite(rewriteFunction(true, false, false, false, true, "boolean"), "cannot support window invocation"); + assertInvalidRewrite(rewriteFunction(true, false, false, false, false, "varchar"), "must return boolean"); + } + @Test public void testLogicalSemanticRecipesAreDeeplyImmutable() { @@ -454,6 +504,56 @@ private static FunctionCapabilityDefinition variadicFunction(String name) false); } + private static FunctionCapabilityDefinition rewriteFunction( + String name, + FunctionKind kind, + List trinoName, + Optional rewrite, + List signatures) + { + return new FunctionCapabilityDefinition( + name, + kind, + FunctionImplementation.REWRITE, + trinoName, + rewrite, + signatures, + true, + false, + false, + false, + false); + } + + private static FunctionCapabilityDefinition rewriteFunction( + boolean deterministic, + boolean supportsDistinct, + boolean supportsOrderBy, + boolean supportsFilter, + boolean supportsWindow, + String returnType) + { + return new FunctionCapabilityDefinition( + "rewrite", + FunctionKind.SCALAR, + FunctionImplementation.REWRITE, + List.of(), + Optional.of(FunctionRewrite.IS_NULL), + List.of(new FunctionSignature(List.of("varchar"), returnType, false)), + deterministic, + supportsDistinct, + supportsOrderBy, + supportsFilter, + supportsWindow); + } + + private static void assertInvalidRewrite(FunctionCapabilityDefinition function, String message) + { + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of(function))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(message); + } + private static HogQlSemanticCatalogSnapshot logicalSemanticSnapshot( PropertyDefinition property, Optional joinPredicate, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java index f4257d789c05..ab19efaa56e7 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java @@ -16,6 +16,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionRewrite; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ModifierBehavior; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; @@ -33,6 +34,7 @@ import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.Optional; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -94,6 +96,7 @@ public void testDecodesCompleteSnapshot() }); assertThat(snapshot.functions()).singleElement().satisfies(function -> { assertThat(function.name()).isEqualTo("length"); + assertThat(function.rewrite()).isEmpty(); assertThat(function.signatures()).singleElement().satisfies(signature -> { assertThat(signature.argumentTypes()).containsExactly("json"); assertThat(signature.returnType()).isEqualTo("bigint"); @@ -117,6 +120,45 @@ public void testDecodesCompleteSnapshot() assertThat(snapshot.cohorts()).singleElement().extracting(HogQlSemanticCatalogSnapshot.CohortReference::cohortId).isEqualTo("cohort-7"); } + @Test + public void testDecodesClosedFunctionRewrite() + throws Exception + { + HogQlSemanticCatalogSnapshot snapshot = new HogQlSemanticCatalogSnapshotJsonDecoder() + .decode(bytes(functionPayload("REWRITE", Optional.of("IS_NULL"), true, false)), LoadRequest.latest(CATALOG, LANGUAGE_VERSION)); + + assertThat(snapshot.functions()).singleElement().satisfies(function -> { + assertThat(function.implementation()).isEqualTo(HogQlSemanticCatalogSnapshot.FunctionImplementation.REWRITE); + assertThat(function.rewrite()).contains(FunctionRewrite.IS_NULL); + assertThat(function.trinoName()).isEmpty(); + }); + } + + @Test + public void testRejectsInvalidFunctionRewritePayloads() + throws Exception + { + List payloads = List.of( + functionPayload("REWRITE", Optional.empty(), true, false), + functionPayload("REWRITE", Optional.of("IS_NULL"), false, false), + functionPayload("STOCK", Optional.of("IS_NULL"), false, false), + functionPayload("UDF", Optional.of("IS_NOT_NULL"), false, false), + functionPayload("REWRITE", Optional.of(SECRET), true, false), + functionPayload("REWRITE", Optional.empty(), true, true), + functionPayloadWithBoolean("deterministic", false), + functionPayloadWithBoolean("supportsDistinct", true), + functionPayloadWithBoolean("supportsOrderBy", true), + functionPayloadWithBoolean("supportsFilter", true), + functionPayloadWithBoolean("supportsWindow", true), + functionPayloadWithReturnType("varchar")); + + payloads.forEach(payload -> assertDecodeFailure( + new HogQlSemanticCatalogSnapshotJsonDecoder(), + payload, + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + DecodeFailure.INVALID_PAYLOAD)); + } + @Test public void testDecodesSnapshotFromOlderSchemaVersionTwoPublisher() throws Exception @@ -269,6 +311,49 @@ private static byte[] bytes(String value) return value.getBytes(StandardCharsets.UTF_8); } + private static String functionPayload(String implementation, Optional rewrite, boolean emptyTrinoName, boolean nullRewrite) + throws Exception + { + ObjectNode payload = (ObjectNode) new ObjectMapper().readTree(validSnapshotJson()); + ObjectNode function = (ObjectNode) ((ArrayNode) payload.get("functions")).get(0); + function.put("implementation", implementation); + rewrite.ifPresentOrElse(value -> function.put("rewrite", value), () -> { + if (nullRewrite) { + function.putNull("rewrite"); + } + else { + function.remove("rewrite"); + } + }); + if (emptyTrinoName) { + ((ArrayNode) function.get("trinoName")).removeAll(); + } + if (implementation.equals("REWRITE")) { + ObjectNode signature = (ObjectNode) ((ArrayNode) function.get("signatures")).get(0); + signature.put("returnType", "boolean"); + } + return payload.toString(); + } + + private static String functionPayloadWithBoolean(String field, boolean value) + throws Exception + { + ObjectNode payload = (ObjectNode) new ObjectMapper().readTree(functionPayload("REWRITE", Optional.of("IS_NULL"), true, false)); + ObjectNode function = (ObjectNode) ((ArrayNode) payload.get("functions")).get(0); + function.put(field, value); + return payload.toString(); + } + + private static String functionPayloadWithReturnType(String returnType) + throws Exception + { + ObjectNode payload = (ObjectNode) new ObjectMapper().readTree(functionPayload("REWRITE", Optional.of("IS_NULL"), true, false)); + ObjectNode function = (ObjectNode) ((ArrayNode) payload.get("functions")).get(0); + ObjectNode signature = (ObjectNode) ((ArrayNode) function.get("signatures")).get(0); + signature.put("returnType", returnType); + return payload.toString(); + } + private static String validSnapshotJson() { return """ From 3bd5fe51013f65919eee3f6495dbb85313bc0ba3 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 14:32:52 -0400 Subject: [PATCH 056/136] feat(hogql): expand relationship paths --- .../hogql/compiler/HogQlSemanticResolver.java | 340 ++++++++++++++++-- .../TestHogQlRelationshipExpansion.java | 241 +++++++++++++ 2 files changed, 552 insertions(+), 29 deletions(-) create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index 00e042679801..b81c88070bab 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -22,6 +22,9 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCallRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyProjectionDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyTableDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.MaterializedViewReference; @@ -32,6 +35,8 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ReferencedField; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipJoinSide; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SavedQueryReference; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ScopedFieldReferenceRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; @@ -84,6 +89,7 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -98,8 +104,11 @@ final class HogQlSemanticResolver { private final PinnedSnapshot snapshot; private final ExpansionBudget expansionBudget; + private final Map relationshipPaths = new LinkedHashMap<>(); private List bindings = List.of(); private boolean allRelationsLogical; + private Optional expandedRelation = Optional.empty(); + private int generatedRelationId; private HogQlSemanticResolver(PinnedSnapshot snapshot) { @@ -146,6 +155,7 @@ private HogQlQuery resolveNestedQuery(HogQlQuery query) Optional relation = select.from().map(this::resolveRelation); bindings = relation.map(ResolvedRelation::bindings).orElse(List.of()); allRelationsLogical = relation.map(ResolvedRelation::allLogical).orElse(false); + expandedRelation = relation.map(ResolvedRelation::relation); if (bindings.isEmpty()) { return new HogQlQuery( commonTables, @@ -161,27 +171,33 @@ private HogQlQuery resolveNestedQuery(HogQlQuery query) query.offset(), query.span()); } - return resolveQuery(query, commonTables, relation.orElseThrow().relation()); + return resolveQuery(query, commonTables); } - private HogQlQuery resolveQuery(HogQlQuery query, List commonTables, Relation relation) + private HogQlQuery resolveQuery(HogQlQuery query, List commonTables) { List projections = new ArrayList<>(); query.projections().forEach(projection -> projections.addAll(resolveProjection(projection))); - HogQlQuery resolved = new HogQlQuery( + Optional where = query.where().map(this::resolveExpression); + List groupBy = query.groupBy().stream().map(this::resolveExpression).toList(); + Optional having = query.having().map(this::resolveExpression); + List windows = query.windows().stream().map(this::resolveWindowDefinition).toList(); + List orderBy = resolveSortItems(query.orderBy()); + Optional limit = query.limit().map(this::resolveExpression); + Optional offset = query.offset().map(this::resolveExpression); + return new HogQlQuery( commonTables, query.distinct(), projections, - Optional.of(relation), - query.where().map(this::resolveExpression), - query.groupBy().stream().map(this::resolveExpression).toList(), - query.having().map(this::resolveExpression), - query.windows().stream().map(this::resolveWindowDefinition).toList(), - resolveSortItems(query.orderBy()), - query.limit().map(this::resolveExpression), - query.offset().map(this::resolveExpression), + expandedRelation, + where, + groupBy, + having, + windows, + orderBy, + limit, + offset, query.span()); - return resolved; } private List resolveProjection(Projection projection) @@ -572,6 +588,10 @@ private WindowFrame resolveWindowFrame(WindowFrame frame) private Expression resolveColumn(ColumnReference reference) { List parts = reference.parts(); + Optional semanticPath = resolveSemanticPath(reference); + if (semanticPath.isPresent()) { + return semanticPath.orElseThrow(); + } Optional propertyAccess = resolveDottedPropertyAccess(reference); if (propertyAccess.isPresent()) { return propertyAccess.orElseThrow(); @@ -617,6 +637,214 @@ private Expression resolveColumn(ColumnReference reference) return reference; } + private Optional resolveSemanticPath(ColumnReference reference) + { + List parts = reference.parts(); + if (parts.size() < 2) { + return Optional.empty(); + } + + List candidates = new ArrayList<>(); + List qualifiedBindings = bindings.stream() + .filter(binding -> binding.qualifier().equals(canonical(parts.getFirst().value()))) + .toList(); + if (!qualifiedBindings.isEmpty()) { + if (qualifiedBindings.size() == 1 && isSemanticPathMember(qualifiedBindings.getFirst(), parts.get(1).value())) { + candidates.add(new PathCandidate(qualifiedBindings.getFirst(), parts.subList(1, parts.size()))); + } + } + else { + bindings.stream() + .filter(binding -> isSemanticPathMember(binding, parts.getFirst().value())) + .map(binding -> new PathCandidate(binding, parts)) + .forEach(candidates::add); + } + if (candidates.isEmpty()) { + return Optional.empty(); + } + if (candidates.size() > 1) { + throw ambiguousResolutionError(reference, parts.getFirst().value()); + } + if (expandedRelation.isEmpty()) { + throw unsupportedExpansion(reference.span(), "HogQL relationship paths are not supported inside explicit join criteria"); + } + return Optional.of(resolveSemanticPath(reference, candidates.getFirst())); + } + + private boolean isSemanticPathMember(TableBinding binding, String name) + { + return relationship(binding, name).isPresent() || lazyTable(binding, name).isPresent(); + } + + private Expression resolveSemanticPath(ColumnReference reference, PathCandidate candidate) + { + TableBinding owner = candidate.owner(); + List path = candidate.path(); + Optional lazyTable = lazyTable(owner, path.getFirst().value()); + if (lazyTable.isPresent()) { + if (path.size() != 2) { + throw resolutionError(reference, path.getLast().value()); + } + LazyTableDefinition definition = lazyTable.orElseThrow(); + LazyProjectionDefinition projection = definition.projections().stream() + .filter(candidateProjection -> canonical(candidateProjection.name()).equals(canonical(path.getLast().value()))) + .findFirst() + .orElseThrow(() -> resolutionError(reference, path.getLast().value())); + TableBinding terminal = ensureRelationshipPath(owner, definition.relationshipPath(), reference.span()); + return expandRecipe( + terminal, + projection.recipe(), + Optional.of(terminal.outputQualifier()), + reference.span(), + expansionBudget); + } + + LogicalTableDefinition currentTable = snapshot.logicalTable(owner.relationName()).orElseThrow(); + List relationshipPath = new ArrayList<>(); + int index = 0; + while (index < path.size() - 1) { + Optional relationship = relationship(currentTable, path.get(index).value()); + if (relationship.isEmpty()) { + break; + } + RelationshipDefinition definition = relationship.orElseThrow(); + relationshipPath.add(definition.name()); + currentTable = snapshot.logicalTable(definition.targetTable()) + .orElseThrow(() -> expansionError(reference.span(), "HogQL relationship references an unavailable target table")); + index++; + } + if (relationshipPath.isEmpty()) { + return reference; + } + + TableBinding terminal = ensureRelationshipPath(owner, relationshipPath, reference.span()); + List remaining = path.subList(index, path.size()); + if (remaining.size() == 1) { + BoundField field = terminal.fields().get(canonical(remaining.getFirst().value())); + if (field == null) { + throw resolutionError(reference, remaining.getFirst().value()); + } + return resolveBoundField(terminal, field, Optional.of(terminal.outputQualifier()), reference.span(), expansionBudget); + } + if (remaining.size() == 2) { + Optional property = properties(terminal).stream() + .filter(candidateProperty -> canonical(candidateProperty.name()).equals(canonical(remaining.getFirst().value()))) + .findFirst(); + if (property.isPresent()) { + return expandProperty( + terminal, + property.orElseThrow(), + new Literal(HogQlQuery.LiteralKind.STRING, remaining.getLast().value(), remaining.getLast().span()), + Optional.of(terminal.outputQualifier()), + reference.span(), + expansionBudget); + } + } + throw resolutionError(reference, remaining.getLast().value()); + } + + private TableBinding ensureRelationshipPath(TableBinding owner, List path, HogQlQuery.SourceSpan span) + { + TableBinding source = owner; + List prefix = new ArrayList<>(); + for (String relationshipName : path) { + prefix.add(canonical(relationshipName)); + RelationshipPathKey key = new RelationshipPathKey(owner.qualifier(), prefix); + TableBinding cached = relationshipPaths.get(key); + if (cached != null) { + source = cached; + continue; + } + RelationshipDefinition relationship = relationship(source, relationshipName) + .orElseThrow(() -> expansionError(span, "HogQL lazy table references an unavailable relationship")); + LogicalTableDefinition target = snapshot.logicalTable(relationship.targetTable()) + .orElseThrow(() -> expansionError(span, "HogQL relationship references an unavailable target table")); + source = addRelationshipJoin(source, target, relationship, span); + relationshipPaths.put(key, source); + } + return source; + } + + private TableBinding addRelationshipJoin( + TableBinding source, + LogicalTableDefinition target, + RelationshipDefinition relationship, + HogQlQuery.SourceSpan span) + { + expansionBudget.add(span); + ResolvedRelation targetRelation = resolveLogicalTable(target, span); + Identifier alias = nextGeneratedRelationAlias(span); + TableBinding targetBinding = targetRelation.bindings().getFirst().withAlias(alias); + Relation aliasedTarget = new AliasedRelation(targetRelation.relation(), alias, span); + + List predicates = relationship.joinKeys().stream() + .map(joinKey -> joinKeyPredicate(source, targetBinding, joinKey, span)) + .collect(java.util.stream.Collectors.toCollection(ArrayList::new)); + relationship.joinPredicate().ifPresent(predicate -> predicates.add(expandRecipe( + source, + predicate, + Optional.of(source.outputQualifier()), + span, + expansionBudget, + Map.of(), + Map.of(RelationshipJoinSide.SOURCE, source, RelationshipJoinSide.TARGET, targetBinding)))); + Expression criteria = predicates.stream() + .reduce((left, right) -> new BinaryExpression(HogQlQuery.BinaryOperator.AND, left, right, span)) + .orElseThrow(); + expandedRelation = Optional.of(new JoinRelation( + HogQlQuery.JoinType.LEFT, + expandedRelation.orElseThrow(), + aliasedTarget, + Optional.of(new JoinOn(criteria, span)), + span)); + return targetBinding; + } + + private Expression joinKeyPredicate(TableBinding source, TableBinding target, JoinKey joinKey, HogQlQuery.SourceSpan span) + { + BoundField sourceField = Optional.ofNullable(source.fields().get(canonical(joinKey.sourceField()))) + .orElseThrow(() -> expansionError(span, "HogQL relationship references an unavailable source field")); + BoundField targetField = Optional.ofNullable(target.fields().get(canonical(joinKey.targetField()))) + .orElseThrow(() -> expansionError(span, "HogQL relationship references an unavailable target field")); + return new BinaryExpression( + HogQlQuery.BinaryOperator.EQUAL, + physicalColumn(sourceField, Optional.of(source.outputQualifier()), span), + physicalColumn(targetField, Optional.of(target.outputQualifier()), span), + span); + } + + private Identifier nextGeneratedRelationAlias(HogQlQuery.SourceSpan span) + { + while (true) { + Identifier candidate = new Identifier("__hogql_lazy_" + ++generatedRelationId, true, span); + boolean used = bindings.stream().anyMatch(binding -> binding.qualifier().equals(canonical(candidate.value()))) || + relationshipPaths.values().stream().anyMatch(binding -> binding.qualifier().equals(canonical(candidate.value()))); + if (!used) { + return candidate; + } + } + } + + private Optional relationship(TableBinding binding, String name) + { + return snapshot.logicalTable(binding.relationName()).flatMap(table -> relationship(table, name)); + } + + private static Optional relationship(LogicalTableDefinition table, String name) + { + return table.relationships().stream() + .filter(candidate -> canonical(candidate.name()).equals(canonical(name))) + .findFirst(); + } + + private Optional lazyTable(TableBinding binding, String name) + { + return snapshot.snapshot().lazyTables().stream() + .filter(candidate -> canonical(candidate.table()).equals(canonical(binding.relationName()))) + .filter(candidate -> canonical(candidate.name()).equals(canonical(name))) + .findFirst(); + } + private Expression resolveMemberAccess(MemberAccessExpression memberAccess) { if (memberAccess.base() instanceof ColumnReference reference) { @@ -753,7 +981,7 @@ private Expression expandRecipe( HogQlQuery.SourceSpan span, ExpansionBudget budget) { - return expandRecipe(binding, recipe, qualifier, span, budget, Map.of()); + return expandRecipe(binding, recipe, qualifier, span, budget, Map.of(), Map.of()); } private Expression expandRecipe( @@ -763,6 +991,18 @@ private Expression expandRecipe( HogQlQuery.SourceSpan span, ExpansionBudget budget, Map arguments) + { + return expandRecipe(binding, recipe, qualifier, span, budget, arguments, Map.of()); + } + + private Expression expandRecipe( + TableBinding binding, + ExpressionRecipe recipe, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget, + Map arguments, + Map scopedBindings) { budget.enter(span); try { @@ -775,10 +1015,10 @@ private Expression expandRecipe( yield resolveBoundField(binding, field, qualifier, span, budget); } case LiteralRecipe literal -> typedLiteral(literal.literal(), span); - case FunctionCallRecipe function -> expandFunction(binding, function, qualifier, span, budget, arguments); - case OperatorRecipe operator -> expandOperator(binding, operator, qualifier, span, budget, arguments); + case FunctionCallRecipe function -> expandFunction(binding, function, qualifier, span, budget, arguments, scopedBindings); + case OperatorRecipe operator -> expandOperator(binding, operator, qualifier, span, budget, arguments, scopedBindings); case CastRecipe cast -> new CastExpression( - expandRecipe(binding, cast.expression(), qualifier, span, budget, arguments), + expandRecipe(binding, cast.expression(), qualifier, span, budget, arguments, scopedBindings), new Identifier(cast.targetTypeSignature(), false, span), false, span); @@ -789,8 +1029,23 @@ private Expression expandRecipe( } yield argument; } - case ScopedFieldReferenceRecipe _ -> throw unsupportedExpansion(span, "HogQL scoped field recipe is only valid inside a relationship predicate"); - case PropertyLookupRecipe lookup -> expandPropertyLookup(binding, lookup, qualifier, span, budget, arguments); + case ScopedFieldReferenceRecipe reference -> { + TableBinding scopedBinding = scopedBindings.get(reference.side()); + if (scopedBinding == null) { + throw unsupportedExpansion(span, "HogQL scoped field recipe is only valid inside a relationship predicate"); + } + BoundField field = scopedBinding.fields().get(canonical(reference.field())); + if (field == null) { + throw expansionError(span, "HogQL relationship predicate references an unavailable field"); + } + yield resolveBoundField( + scopedBinding, + field, + Optional.of(scopedBinding.outputQualifier()), + span, + budget); + } + case PropertyLookupRecipe lookup -> expandPropertyLookup(binding, lookup, qualifier, span, budget, arguments, scopedBindings); }; } finally { @@ -804,7 +1059,8 @@ private Expression expandFunction( Optional qualifier, HogQlQuery.SourceSpan span, ExpansionBudget budget, - Map arguments) + Map arguments, + Map scopedBindings) { FunctionCapabilityDefinition capability = snapshot.snapshot().functions().stream() .filter(candidate -> canonical(candidate.name()).equals(canonical(function.name()))) @@ -825,7 +1081,7 @@ private Expression expandFunction( .map(name -> new Identifier(name.value(), name.delimited(), span)) .toList(), function.arguments().stream() - .map(argument -> expandRecipe(binding, argument, qualifier, span, budget, arguments)) + .map(argument -> expandRecipe(binding, argument, qualifier, span, budget, arguments, scopedBindings)) .toList(), false, List.of(), @@ -839,10 +1095,11 @@ private Expression expandOperator( Optional qualifier, HogQlQuery.SourceSpan span, ExpansionBudget budget, - Map recipeArguments) + Map recipeArguments, + Map scopedBindings) { List arguments = operator.arguments().stream() - .map(argument -> expandRecipe(binding, argument, qualifier, span, budget, recipeArguments)) + .map(argument -> expandRecipe(binding, argument, qualifier, span, budget, recipeArguments, scopedBindings)) .toList(); return switch (operator.operator()) { case NOT -> new UnaryExpression(HogQlQuery.UnaryOperator.NOT, arguments.getFirst(), span); @@ -860,17 +1117,24 @@ private Expression expandPropertyLookup( Optional qualifier, HogQlQuery.SourceSpan span, ExpansionBudget budget, - Map arguments) + Map arguments, + Map scopedBindings) { - if (!canonical(binding.relationName()).equals(canonical(lookup.table()))) { - throw expansionError(span, "HogQL property lookup recipe references an unavailable table"); - } - PropertyDefinition property = properties(binding).stream() + TableBinding lookupBinding = canonical(binding.relationName()).equals(canonical(lookup.table())) + ? binding + : scopedBindings.values().stream() + .filter(candidate -> canonical(candidate.relationName()).equals(canonical(lookup.table()))) + .findFirst() + .orElseThrow(() -> expansionError(span, "HogQL property lookup recipe references an unavailable table")); + Optional lookupQualifier = lookupBinding == binding + ? qualifier + : Optional.of(lookupBinding.outputQualifier()); + PropertyDefinition property = properties(lookupBinding).stream() .filter(candidate -> canonical(candidate.name()).equals(canonical(lookup.property()))) .findFirst() .orElseThrow(() -> expansionError(span, "HogQL property lookup recipe references an unavailable property")); - Expression key = expandRecipe(binding, lookup.key(), qualifier, span, budget, arguments); - return expandProperty(binding, property, key, qualifier, span, budget); + Expression key = expandRecipe(binding, lookup.key(), qualifier, span, budget, arguments, scopedBindings); + return expandProperty(lookupBinding, property, key, lookupQualifier, span, budget); } private Expression expandProperty( @@ -1119,6 +1383,24 @@ private record FieldMatch(TableBinding binding, BoundField field) {} private record PropertyMatch(TableBinding binding, PropertyDefinition property) {} + private record PathCandidate(TableBinding owner, List path) + { + private PathCandidate + { + owner = requireNonNull(owner, "owner is null"); + path = List.copyOf(requireNonNull(path, "path is null")); + } + } + + private record RelationshipPathKey(String ownerQualifier, List path) + { + private RelationshipPathKey + { + ownerQualifier = requireNonNull(ownerQualifier, "ownerQualifier is null"); + path = List.copyOf(requireNonNull(path, "path is null")); + } + } + private record ProjectedField(String name, String sourceField, boolean starVisible) { private ProjectedField diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java new file mode 100644 index 000000000000..4eff489c9fbb --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java @@ -0,0 +1,241 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionArgument; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyProjectionDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyLookupRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipJoinSide; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ScopedFieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.stream.Stream; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlRelationshipExpansion +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); + private static final HogQlSemanticCatalogSnapshot SNAPSHOT = snapshot(); + + private final HogQlCompiler compiler = new HogQlCompiler(); + private final SqlParser sqlParser = new SqlParser(); + + @Test + public void testExpandsRelationshipAndLazyProjectionThroughOneJoin() + { + HogQlCompilationResult result = compile("SELECT person.name, e.personProfile.name, e.personProfile.plan FROM events e"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"__hogql_lazy_1\".full_name AS name, " + + "\"__hogql_lazy_1\".full_name AS name, " + + "CAST(\"__hogql_lazy_1\".properties_map[CAST(CAST('plan' AS varchar) AS varchar)] AS varchar) AS plan " + + "FROM analytics.data.raw_events e " + + "LEFT JOIN analytics.data.raw_persons \"__hogql_lazy_1\" " + + "ON e.person_id = \"__hogql_lazy_1\".person_id " + + "AND e.workspace_id = \"__hogql_lazy_1\".workspace_id")); + } + + @Test + public void testExpandsOnlyTheDeclaredPathThroughRelationshipCycle() + { + HogQlCompilationResult result = compile("SELECT personEvent.event FROM events"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"__hogql_lazy_2\".event_name AS event " + + "FROM analytics.data.raw_events " + + "LEFT JOIN analytics.data.raw_persons \"__hogql_lazy_1\" " + + "ON raw_events.person_id = \"__hogql_lazy_1\".person_id " + + "AND raw_events.workspace_id = \"__hogql_lazy_1\".workspace_id " + + "LEFT JOIN analytics.data.raw_events \"__hogql_lazy_2\" " + + "ON \"__hogql_lazy_1\".event_id = \"__hogql_lazy_2\".event_id")); + } + + @ParameterizedTest + @MethodSource("nonProjectionRelationshipQueries") + public void testExpandsRelationshipDemandedOutsideProjection(String hogql, String trinoSql) + { + assertThat(compile(hogql).statement()).isEqualTo(sqlParser.createStatement(trinoSql)); + } + + @Test + public void testFailsExplicitlyWhenRelationshipIsDemandedInsideJoinCriteria() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compile("SELECT e.event FROM events e JOIN persons p ON e.person.name = p.name")); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception).hasMessageContaining("HogQL relationship paths are not supported inside explicit join criteria"); + } + + private static Stream nonProjectionRelationshipQueries() + { + String join = "FROM analytics.data.raw_events " + + "LEFT JOIN analytics.data.raw_persons \"__hogql_lazy_1\" " + + "ON raw_events.person_id = \"__hogql_lazy_1\".person_id " + + "AND raw_events.workspace_id = \"__hogql_lazy_1\".workspace_id "; + return Stream.of( + Arguments.of( + "SELECT event FROM events WHERE person.name IS NOT NULL", + "SELECT event_name AS event " + join + "WHERE \"__hogql_lazy_1\".full_name IS NOT NULL"), + Arguments.of( + "SELECT event FROM events ORDER BY person.name", + "SELECT event_name AS event " + join + "ORDER BY \"__hogql_lazy_1\".full_name")); + } + + private HogQlCompilationResult compile(String query) + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + return compiler.compile(new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(7)), Optional.of(context)); + } + + private static HogQlSemanticCatalogSnapshot snapshot() + { + LogicalTableDefinition events = new LogicalTableDefinition( + "events", + physicalName("raw_events"), + List.of( + field("event", "event_name"), + field("eventId", "event_id"), + field("personId", "person_id"), + field("workspaceId", "workspace_id")), + List.of(), + List.of(new RelationshipDefinition( + "person", + "persons", + RelationshipCardinality.MANY_TO_ONE, + List.of(new JoinKey("personId", "personId")), + Optional.of(new OperatorRecipe( + SemanticOperator.EQUAL, + List.of( + new ScopedFieldReferenceRecipe(RelationshipJoinSide.SOURCE, "workspaceId"), + new ScopedFieldReferenceRecipe(RelationshipJoinSide.TARGET, "workspaceId"))))))); + LogicalTableDefinition persons = new LogicalTableDefinition( + "persons", + physicalName("raw_persons"), + List.of( + field("personId", "person_id"), + field("workspaceId", "workspace_id"), + field("eventId", "event_id"), + field("name", "full_name"), + new LogicalFieldDefinition("propertiesMap", new PhysicalIdentifier("properties_map", false), "map(varchar, varchar)", LogicalType.MAP, true, false)), + List.of(new PropertyDefinition( + "properties", + "propertiesMap", + PropertyStorage.MAP, + LogicalType.STRING, + true, + Optional.of("varchar"), + Optional.of("varchar"), + Optional.of(new OperatorRecipe( + SemanticOperator.SUBSCRIPT, + List.of( + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE), + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_KEY)))))), + List.of(new RelationshipDefinition( + "event", + "events", + RelationshipCardinality.MANY_TO_ONE, + List.of(new JoinKey("eventId", "eventId"))))); + List lazyTables = List.of( + new LazyTableDefinition( + "events", + "personProfile", + List.of("person"), + List.of( + new LazyProjectionDefinition("name", "varchar", LogicalType.STRING, true, true, new FieldReferenceRecipe("persons", "name")), + new LazyProjectionDefinition( + "plan", + "varchar", + LogicalType.STRING, + true, + true, + new PropertyLookupRecipe( + "persons", + "properties", + new LiteralRecipe(new TypedLiteral("varchar", LiteralEncoding.STRING, "plan")))))), + new LazyTableDefinition( + "events", + "personEvent", + List.of("person", "event"), + List.of(new LazyProjectionDefinition("event", "varchar", LogicalType.STRING, true, true, new FieldReferenceRecipe("events", "event"))))); + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 7, + List.of(events, persons), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + lazyTables, + List.of(), + List.of()); + } + + private static LogicalFieldDefinition field(String name, String physicalName) + { + return new LogicalFieldDefinition(name, new PhysicalIdentifier(physicalName, false), "varchar", LogicalType.STRING, false, true); + } + + private static PhysicalQualifiedName physicalName(String table) + { + return new PhysicalQualifiedName(CATALOG, new PhysicalIdentifier("data", false), new PhysicalIdentifier(table, false)); + } +} From b5b02ba7717cd122103d808d101c9ae5ce2d990f Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 14:40:07 -0400 Subject: [PATCH 057/136] feat(hogql): apply semantic modifiers --- .../compiler/HogQlCompilationResult.java | 12 +- .../trino/hogql/compiler/HogQlCompiler.java | 59 ++++-- .../hogql/compiler/HogQlModifierBinding.java | 38 ++++ .../hogql/compiler/HogQlModifierResolver.java | 177 ++++++++++++++++ .../catalog/HogQlSemanticCatalogSnapshot.java | 3 + .../hogql/compiler/TestHogQlModifiers.java | 196 ++++++++++++++++++ .../TestHogQlSemanticCatalogSnapshot.java | 34 +++ ...gQlSemanticCatalogSnapshotJsonDecoder.java | 5 + .../execution/HogQlParameterDecoder.java | 32 ++- .../io/trino/execution/QueryPreparer.java | 51 ++++- .../io/trino/sql/SessionPropertyResolver.java | 11 +- ...TestHogQlQueryPreparerSemanticCatalog.java | 141 ++++++++++++- .../io/trino/execution/TestQueryPreparer.java | 3 + docs/src/main/sphinx/admin/hogql.md | 20 ++ 14 files changed, 758 insertions(+), 24 deletions(-) create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlModifierBinding.java create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlModifierResolver.java create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlModifiers.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java index ff2f821f63f2..9aa9d2c5f5ca 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java @@ -20,12 +20,22 @@ import static java.util.Objects.requireNonNull; -public record HogQlCompilationResult(Statement statement, List parameterNames, OptionalLong catalogGeneration) +public record HogQlCompilationResult( + Statement statement, + List parameterNames, + List modifierBindings, + OptionalLong catalogGeneration) { public HogQlCompilationResult { statement = requireNonNull(statement, "statement is null"); parameterNames = List.copyOf(requireNonNull(parameterNames, "parameterNames is null")); + modifierBindings = List.copyOf(requireNonNull(modifierBindings, "modifierBindings is null")); catalogGeneration = requireNonNull(catalogGeneration, "catalogGeneration is null"); } + + public HogQlCompilationResult(Statement statement, List parameterNames, OptionalLong catalogGeneration) + { + this(statement, parameterNames, List.of(), catalogGeneration); + } } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index 01674044863a..fbce17e07919 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -13,6 +13,8 @@ */ package io.trino.hogql.compiler; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException.Failure; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinRequest; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; import io.trino.hogql.parser.HogQlLanguageContract; @@ -101,6 +103,7 @@ public HogQlCompilationResult compile(HogQlCompileEnvelope envelope, Optional { requireNonNull(hogql, "hogql is null"); requireNonNull(parameters, "parameters is null"); - return compile(parse(hogql), parameters, Optional.empty(), HogQlLanguageContract.current().languageVersion(), OptionalLong.empty()); + return compile(parse(hogql), parameters, Map.of(), Optional.empty(), HogQlLanguageContract.current().languageVersion(), OptionalLong.empty()); } private HogQlQuery parse(String hogql) @@ -131,6 +134,7 @@ private HogQlQuery parse(String hogql) private static HogQlCompilationResult compile( HogQlQuery query, Map parameters, + Map modifiers, Optional catalogContext, HogQlLanguageVersion languageVersion, OptionalLong expectedCatalogGeneration) @@ -171,11 +175,18 @@ private static HogQlCompilationResult compile( for (int index = 0; index < placeholders.size(); index++) { parameterIds.put(placeholders.get(index).span(), index); } - ResolvedQuery resolved = resolveQuery(query, catalogContext, languageVersion, expectedCatalogGeneration); + ResolvedQuery resolved = resolveQuery(query, catalogContext, languageVersion, expectedCatalogGeneration, !modifiers.isEmpty()); Statement statement = TrinoAstFactory.createStatement(resolved.query(), parameterIds); - return new HogQlCompilationResult(statement, placeholders.stream() - .map(Placeholder::name) - .toList(), resolved.catalogGeneration()); + List modifierBindings = resolved.pinnedSnapshot() + .map(snapshot -> HogQlModifierResolver.resolve(snapshot, modifiers, query.span())) + .orElseGet(List::of); + return new HogQlCompilationResult( + statement, + placeholders.stream() + .map(Placeholder::name) + .toList(), + modifierBindings, + resolved.catalogGeneration()); } private static void collectPlaceholders(HogQlQuery query, List placeholders) @@ -208,21 +219,32 @@ private static ResolvedQuery resolveQuery( HogQlQuery query, Optional catalogContext, HogQlLanguageVersion languageVersion, - OptionalLong expectedCatalogGeneration) + OptionalLong expectedCatalogGeneration, + boolean modifiersRequireSnapshot) { - if (!containsSemanticCandidate(query) || catalogContext.isEmpty()) { - return new ResolvedQuery(query, OptionalLong.empty()); + boolean semanticCandidate = containsSemanticCandidate(query); + if (!semanticCandidate && !modifiersRequireSnapshot) { + return new ResolvedQuery(query, Optional.empty()); + } + if (catalogContext.isEmpty()) { + if (modifiersRequireSnapshot) { + throw new HogQlSemanticCatalogException(Failure.UNAVAILABLE, "HogQL semantic catalog snapshot is required for modifiers"); + } + return new ResolvedQuery(query, Optional.empty()); } HogQlSemanticCatalogContext context = catalogContext.orElseThrow(); PinnedSnapshot pinned = context.snapshotProvider().pin(new PinRequest( context.catalog(), requireNonNull(languageVersion, "languageVersion is null"), expectedCatalogGeneration)); - HogQlQuery functionsResolved = HogQlFunctionResolver.resolve(pinned, query); - HogQlQuery resolved = HogQlSemanticResolver.resolve(pinned, functionsResolved) - .map(HogQlSemanticResolver.ResolvedQuery::query) - .orElse(functionsResolved); - return new ResolvedQuery(resolved, OptionalLong.of(pinned.generation())); + HogQlQuery resolved = query; + if (semanticCandidate) { + HogQlQuery functionsResolved = HogQlFunctionResolver.resolve(pinned, query); + resolved = HogQlSemanticResolver.resolve(pinned, functionsResolved) + .map(HogQlSemanticResolver.ResolvedQuery::query) + .orElse(functionsResolved); + } + return new ResolvedQuery(resolved, Optional.of(pinned)); } private static boolean containsSemanticCandidate(HogQlQuery query) @@ -325,12 +347,19 @@ private static boolean containsFunctionCall(Optional expression) return expression.map(HogQlCompiler::containsFunctionCall).orElse(false); } - private record ResolvedQuery(HogQlQuery query, OptionalLong catalogGeneration) + private record ResolvedQuery(HogQlQuery query, Optional pinnedSnapshot) { private ResolvedQuery { query = requireNonNull(query, "query is null"); - catalogGeneration = requireNonNull(catalogGeneration, "catalogGeneration is null"); + pinnedSnapshot = requireNonNull(pinnedSnapshot, "pinnedSnapshot is null"); + } + + public OptionalLong catalogGeneration() + { + return pinnedSnapshot + .map(snapshot -> OptionalLong.of(snapshot.generation())) + .orElseGet(OptionalLong::empty); } } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlModifierBinding.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlModifierBinding.java new file mode 100644 index 000000000000..6a8adaa74862 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlModifierBinding.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; + +import java.util.List; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public record HogQlModifierBinding(String modifierName, Optional> sessionProperty, HogQlTypedValue value) +{ + public HogQlModifierBinding + { + modifierName = requireNonNull(modifierName, "modifierName is null"); + if (modifierName.isBlank()) { + throw new IllegalArgumentException("modifier name is empty"); + } + sessionProperty = requireNonNull(sessionProperty, "sessionProperty is null") + .map(property -> List.copyOf(requireNonNull(property, "session property is null"))); + if (sessionProperty.isPresent() && sessionProperty.orElseThrow().isEmpty()) { + throw new IllegalArgumentException("session property name is empty"); + } + value = requireNonNull(value, "value is null"); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlModifierResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlModifierResolver.java new file mode 100644 index 000000000000..ed60b988c5b8 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlModifierResolver.java @@ -0,0 +1,177 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.trino.hogql.compiler.HogQlTypedValue.ArrayValue; +import io.trino.hogql.compiler.HogQlTypedValue.BooleanValue; +import io.trino.hogql.compiler.HogQlTypedValue.NullValue; +import io.trino.hogql.compiler.HogQlTypedValue.NumberValue; +import io.trino.hogql.compiler.HogQlTypedValue.ObjectValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.hogql.compiler.HogQlTypedValue.Value; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticModifierDefault; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; + +import java.util.ArrayList; +import java.util.Base64; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_BINDING_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_COMPILER_INTERNAL_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static java.util.Objects.requireNonNull; + +final class HogQlModifierResolver +{ + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + + private HogQlModifierResolver() {} + + public static List resolve( + PinnedSnapshot pinnedSnapshot, + Map suppliedModifiers, + SourceSpan querySpan) + { + requireNonNull(pinnedSnapshot, "pinnedSnapshot is null"); + requireNonNull(suppliedModifiers, "suppliedModifiers is null"); + requireNonNull(querySpan, "querySpan is null"); + + Map definitions = new HashMap<>(); + for (SemanticModifierDefault definition : pinnedSnapshot.snapshot().modifierDefaults()) { + definitions.put(canonical(definition.name()), definition); + } + + Map suppliedByCanonicalName = new LinkedHashMap<>(); + suppliedModifiers.entrySet().stream() + .sorted(Comparator.comparing(entry -> canonical(entry.getKey()))) + .forEach(entry -> { + String name = canonical(entry.getKey()); + if (suppliedByCanonicalName.put(name, entry.getValue()) != null) { + throw bindingError(querySpan, "Duplicate HogQL modifier: " + entry.getKey()); + } + SemanticModifierDefault definition = definitions.get(name); + if (definition == null) { + throw bindingError(querySpan, "Unknown HogQL modifier: " + entry.getKey()); + } + if (!definition.defaultValue().typeSignature().equalsIgnoreCase(entry.getValue().type())) { + throw bindingError(querySpan, "HogQL modifier has an incompatible type: " + definition.name()); + } + }); + + List bindings = new ArrayList<>(); + for (SemanticModifierDefault definition : pinnedSnapshot.snapshot().modifierDefaults()) { + Optional supplied = Optional.ofNullable(suppliedByCanonicalName.get(canonical(definition.name()))); + switch (definition.behavior()) { + case TRINO_SESSION_PROPERTY -> bindings.add(new HogQlModifierBinding( + definition.name(), + Optional.of(definition.sessionProperty()), + supplied.orElseGet(() -> typedValue(definition.defaultValue())))); + case COMPILER -> { + if (supplied.isPresent()) { + throw unsupportedError(querySpan, "HogQL compiler modifier is not implemented: " + definition.name()); + } + } + case SAFE_NOOP -> supplied.ifPresent(value -> bindings.add(new HogQlModifierBinding( + definition.name(), + Optional.empty(), + value))); + case UNSUPPORTED -> { + if (supplied.isPresent()) { + throw unsupportedError(querySpan, "HogQL modifier is not supported: " + definition.name()); + } + } + } + } + return List.copyOf(bindings); + } + + private static HogQlTypedValue typedValue(TypedLiteral literal) + { + Value value = switch (literal.encoding()) { + case NULL -> NullValue.NULL; + case STRING -> new StringValue(literal.value()); + case BOOLEAN -> new BooleanValue(Boolean.parseBoolean(literal.value())); + case INTEGER, DECIMAL, FLOAT -> new NumberValue(literal.value()); + case JSON -> jsonValue(literal.value()); + case BASE64 -> new StringValue(HexFormat.of().formatHex(Base64.getDecoder().decode(literal.value()))); + }; + return new HogQlTypedValue(literal.typeSignature(), value); + } + + private static Value jsonValue(String json) + { + try { + return jsonValue(JSON_MAPPER.readTree(json)); + } + catch (JsonProcessingException _) { + throw new TrinoException(HOGQL_COMPILER_INTERNAL_ERROR, "Invalid HogQL modifier default"); + } + } + + private static Value jsonValue(JsonNode node) + { + if (node.isNull()) { + return NullValue.NULL; + } + if (node.isBoolean()) { + return new BooleanValue(node.booleanValue()); + } + if (node.isNumber()) { + return new NumberValue(node.asText()); + } + if (node.isTextual()) { + return new StringValue(node.textValue()); + } + if (node.isArray()) { + List values = new ArrayList<>(node.size()); + node.forEach(element -> values.add(jsonValue(element))); + return new ArrayValue(values); + } + if (node.isObject()) { + Map values = new LinkedHashMap<>(); + node.properties().forEach(entry -> values.put(entry.getKey(), jsonValue(entry.getValue()))); + return new ObjectValue(values); + } + throw new TrinoException(HOGQL_COMPILER_INTERNAL_ERROR, "Invalid HogQL modifier default"); + } + + private static String canonical(String name) + { + return requireNonNull(name, "name is null").toLowerCase(Locale.ENGLISH); + } + + private static TrinoException bindingError(SourceSpan span, String message) + { + return new TrinoException(HOGQL_BINDING_ERROR, Optional.of(new Location(span.startLine(), span.startColumn())), message, null); + } + + private static TrinoException unsupportedError(SourceSpan span, String message) + { + return new TrinoException(HOGQL_UNSUPPORTED_FEATURE, Optional.of(new Location(span.startLine(), span.startColumn())), message, null); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index fc47af398580..297c6db5a578 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -441,6 +441,9 @@ private static void validateModifiers(List modifiers) if (modifier.behavior() == ModifierBehavior.TRINO_SESSION_PROPERTY && modifier.sessionProperty().isEmpty()) { throw new IllegalArgumentException("modifier must name a session property"); } + if (modifier.behavior() == ModifierBehavior.TRINO_SESSION_PROPERTY && modifier.sessionProperty().size() > 2) { + throw new IllegalArgumentException("modifier has an invalid session property name"); + } if (modifier.behavior() != ModifierBehavior.TRINO_SESSION_PROPERTY && !modifier.sessionProperty().isEmpty()) { throw new IllegalArgumentException("modifier cannot name a session property"); } diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlModifiers.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlModifiers.java new file mode 100644 index 000000000000..c642e4fae5cd --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlModifiers.java @@ -0,0 +1,196 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.HogQlTypedValue.BooleanValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ModifierBehavior; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticModifierDefault; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.ErrorCode; +import io.trino.spi.TrinoException; +import io.trino.sql.tree.Query; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_BINDING_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlModifiers +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); + private static final SemanticModifierDefault SESSION_MODIFIER = modifier( + "sampling", + ModifierBehavior.TRINO_SESSION_PROPERTY, + List.of(new PhysicalIdentifier("hogql", false), new PhysicalIdentifier("sampling", false))); + private static final SemanticModifierDefault COMPILER_MODIFIER = modifier("compilerMode", ModifierBehavior.COMPILER, List.of()); + private static final SemanticModifierDefault NOOP_MODIFIER = modifier("legacyMode", ModifierBehavior.SAFE_NOOP, List.of()); + private static final SemanticModifierDefault UNSUPPORTED_MODIFIER = modifier("futureMode", ModifierBehavior.UNSUPPORTED, List.of()); + private static final HogQlSemanticCatalogSnapshot SNAPSHOT = snapshot(List.of( + SESSION_MODIFIER, + COMPILER_MODIFIER, + NOOP_MODIFIER, + UNSUPPORTED_MODIFIER)); + + @Test + public void testExplicitModifierPinsLiteralQueryAndOverridesDefault() + { + AtomicInteger pins = new AtomicInteger(); + HogQlCompilationResult result = new HogQlCompiler().compile( + envelope("SELECT 1", Map.of( + "SAMPLING", new HogQlTypedValue("BOOLEAN", new BooleanValue(true)), + "legacyMode", new HogQlTypedValue("boolean", new BooleanValue(true)))), + Optional.of(context(pins))); + + assertThat(pins).hasValue(1); + assertThat(result.catalogGeneration()).hasValue(7); + assertThat(result.modifierBindings()).hasSize(2); + assertThat(result.modifierBindings()).filteredOn(binding -> binding.modifierName().equals("sampling")).singleElement().satisfies(binding -> { + assertThat(binding.sessionProperty().orElseThrow()).extracting(PhysicalIdentifier::value).containsExactly("hogql", "sampling"); + assertThat(binding.value()).isEqualTo(new HogQlTypedValue("BOOLEAN", new BooleanValue(true))); + }); + assertThat(result.modifierBindings()).filteredOn(binding -> binding.modifierName().equals("legacyMode")).singleElement().satisfies(binding -> { + assertThat(binding.sessionProperty()).isEmpty(); + assertThat(binding.value()).isEqualTo(new HogQlTypedValue("boolean", new BooleanValue(true))); + }); + assertThat(((Query) result.statement()).getSessionProperties()).isEmpty(); + } + + @Test + public void testPinnedQueryAppliesSessionDefaultAndLeavesOtherDefaultsInert() + { + HogQlCompilationResult result = new HogQlCompiler().compile( + envelope("SELECT 1", Map.of("legacyMode", new HogQlTypedValue("boolean", new BooleanValue(true)))), + Optional.of(context(new AtomicInteger()))); + + assertThat(result.modifierBindings()).hasSize(2); + assertThat(result.modifierBindings()).filteredOn(binding -> binding.modifierName().equals("sampling")).singleElement() + .extracting(HogQlModifierBinding::value) + .isEqualTo(new HogQlTypedValue("boolean", new BooleanValue(false))); + } + + @Test + public void testUnmodifiedLiteralQueryDoesNotPinSnapshot() + { + AtomicInteger pins = new AtomicInteger(); + HogQlCompilationResult result = new HogQlCompiler().compile( + envelope("SELECT 1", Map.of()), + Optional.of(context(pins))); + + assertThat(pins).hasValue(0); + assertThat(result.catalogGeneration()).isEmpty(); + assertThat(result.modifierBindings()).isEmpty(); + } + + @Test + public void testExplicitModifierRequiresSnapshotContext() + { + assertThatThrownBy(() -> new HogQlCompiler().compile(envelope( + "SELECT 1", + Map.of("sampling", new HogQlTypedValue("boolean", new BooleanValue(true)))))) + .isInstanceOf(HogQlSemanticCatalogException.class) + .hasMessageContaining("required for modifiers"); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("invalidModifiers") + public void testRejectsInvalidExplicitModifiers(String name, Map modifiers, ErrorCode errorCode, String message) + { + assertThatThrownBy(() -> new HogQlCompiler().compile( + envelope("SELECT 1", modifiers), + Optional.of(context(new AtomicInteger())))) + .isInstanceOfSatisfying(TrinoException.class, exception -> { + assertThat(exception.getErrorCode()).isEqualTo(errorCode); + assertThat(exception).hasMessageContaining(message); + }); + } + + private static Stream invalidModifiers() + { + HogQlTypedValue booleanValue = new HogQlTypedValue("boolean", new BooleanValue(true)); + return Stream.of( + Arguments.of("unknown", Map.of("missing", booleanValue), HOGQL_BINDING_ERROR.toErrorCode(), "Unknown HogQL modifier"), + Arguments.of("wrong type", Map.of("sampling", new HogQlTypedValue("varchar", new StringValue("true"))), HOGQL_BINDING_ERROR.toErrorCode(), "incompatible type"), + Arguments.of("compiler", Map.of("compilerMode", booleanValue), HOGQL_UNSUPPORTED_FEATURE.toErrorCode(), "not implemented"), + Arguments.of("unsupported", Map.of("futureMode", booleanValue), HOGQL_UNSUPPORTED_FEATURE.toErrorCode(), "not supported"), + Arguments.of("duplicate canonical name", Map.of("sampling", booleanValue, "SAMPLING", booleanValue), HOGQL_BINDING_ERROR.toErrorCode(), "Duplicate HogQL modifier")); + } + + private static HogQlSemanticCatalogContext context(AtomicInteger pins) + { + return new HogQlSemanticCatalogContext(CATALOG, _ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + }); + } + + private static HogQlCompileEnvelope envelope(String query, Map modifiers) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + modifiers, + OptionalLong.empty()); + } + + private static SemanticModifierDefault modifier(String name, ModifierBehavior behavior, List sessionProperty) + { + return new SemanticModifierDefault( + name, + behavior, + new TypedLiteral("boolean", LiteralEncoding.BOOLEAN, "false"), + sessionProperty); + } + + private static HogQlSemanticCatalogSnapshot snapshot(List modifiers) + { + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 7, + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + modifiers, + List.of(), + List.of(), + List.of()); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java index 7947d692adc2..e757decbccb4 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java @@ -34,6 +34,7 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ModifierBehavior; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; @@ -49,6 +50,7 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipJoinSide; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ScopedFieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticModifierDefault; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; import io.trino.hogql.parser.HogQlLanguageVersion; @@ -210,6 +212,21 @@ public void testValidatesFunctionRewriteContract() assertInvalidRewrite(rewriteFunction(true, false, false, false, false, "varchar"), "must return boolean"); } + @Test + public void testRejectsOverqualifiedModifierSessionProperty() + { + assertThatThrownBy(() -> semanticSnapshotWithModifiers(List.of(new SemanticModifierDefault( + "sampling", + ModifierBehavior.TRINO_SESSION_PROPERTY, + new TypedLiteral("boolean", LiteralEncoding.BOOLEAN, "false"), + List.of( + new PhysicalIdentifier("catalog", false), + new PhysicalIdentifier("schema", false), + new PhysicalIdentifier("property", false)))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("invalid session property name"); + } + @Test public void testLogicalSemanticRecipesAreDeeplyImmutable() { @@ -469,6 +486,23 @@ private static HogQlSemanticCatalogSnapshot semanticSnapshot( List.of()); } + private static HogQlSemanticCatalogSnapshot semanticSnapshotWithModifiers(List modifiers) + { + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + LANGUAGE_VERSION, + CATALOG, + 7, + List.of(table("events")), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + modifiers); + } + private static LiteralRecipe literal() { return new LiteralRecipe(new TypedLiteral("bigint", LiteralEncoding.INTEGER, "1")); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java index ab19efaa56e7..8b289ea58b2a 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java @@ -255,6 +255,11 @@ private static Stream strictFailures() validSnapshotJson().replace( "\"sessionProperty\": [", "\"unknown\": true,\n \"sessionProperty\": [")), + Arguments.of( + "overqualified modifier session property", + validSnapshotJson().replace( + "{\"value\": \"hogql\", \"delimited\": false},", + "{\"value\": \"catalog\", \"delimited\": false},\n {\"value\": \"schema\", \"delimited\": false},")), Arguments.of("duplicate field", validSnapshotJson().replace("\"protocolVersion\": 1", "\"protocolVersion\": 1, \"protocolVersion\": 1")), Arguments.of("missing field", validSnapshotJson().replace("\"generation\": 7,", "")), Arguments.of("wrong field type", validSnapshotJson().replace("\"nullable\": true", "\"nullable\": \"true\"")), diff --git a/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java b/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java index 077af3b42fc1..f156d20bd0e3 100644 --- a/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java +++ b/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.collect.ImmutableList; import io.trino.hogql.compiler.HogQlCompilationResult; +import io.trino.hogql.compiler.HogQlModifierBinding; import io.trino.hogql.compiler.HogQlTypedValue; import io.trino.hogql.compiler.HogQlTypedValue.ArrayValue; import io.trino.hogql.compiler.HogQlTypedValue.BooleanValue; @@ -112,8 +113,7 @@ List decode(HogQlCompilationResult result, Map decode(HogQlCompilationResult result, Map prepareQuery( session, statement, - Optional.of(hogQlParameterDecoder.decode(result, envelope.parameters())))); + Optional.of(hogQlParameterDecoder.decode(result, envelope.parameters())), + sessionPropertyOverrides(result, statement))); } catch (RuntimeException | Error failure) { tracker.failed(failure); @@ -190,6 +195,16 @@ public PreparedQuery prepareQuery(Session session, Statement wrappedStatement) private PreparedQuery prepareQuery(Session session, Statement wrappedStatement, Optional> suppliedParameters) throws ParsingException, TrinoException + { + return prepareQuery(session, wrappedStatement, suppliedParameters, List.of()); + } + + private PreparedQuery prepareQuery( + Session session, + Statement wrappedStatement, + Optional> suppliedParameters, + List sessionPropertyOverrides) + throws ParsingException, TrinoException { Statement statement = wrappedStatement; Optional prepareSql = Optional.empty(); @@ -225,7 +240,27 @@ else if (wrappedStatement instanceof ExecuteImmediate executeImmediateStatement) } validateParameters(statement, parameters); - return new PreparedQuery(statement, parameters, prepareSql); + return new PreparedQuery(statement, parameters, prepareSql, sessionPropertyOverrides); + } + + private List sessionPropertyOverrides(HogQlCompilationResult result, Statement statement) + { + NodeLocation location = statement.getLocation().orElseThrow(); + return result.modifierBindings().stream() + .map(binding -> modifierBinding(binding, location)) + .flatMap(Optional::stream) + .toList(); + } + + private Optional modifierBinding(HogQlModifierBinding binding, NodeLocation location) + { + Expression value = hogQlParameterDecoder.decode(binding, location); + return binding.sessionProperty().map(property -> { + QualifiedName name = QualifiedName.of(property.stream() + .map(part -> new Identifier(location, part.value(), part.delimited())) + .toList()); + return new SessionProperty(location, name, value); + }); } private static void validateParameters(Statement node, List parameterValues) @@ -244,12 +279,19 @@ public static class PreparedQuery private final Statement statement; private final List parameters; private final Optional prepareSql; + private final List sessionPropertyOverrides; public PreparedQuery(Statement statement, List parameters, Optional prepareSql) + { + this(statement, parameters, prepareSql, List.of()); + } + + public PreparedQuery(Statement statement, List parameters, Optional prepareSql, List sessionPropertyOverrides) { this.statement = requireNonNull(statement, "statement is null"); this.parameters = ImmutableList.copyOf(requireNonNull(parameters, "parameters is null")); this.prepareSql = requireNonNull(prepareSql, "prepareSql is null"); + this.sessionPropertyOverrides = ImmutableList.copyOf(requireNonNull(sessionPropertyOverrides, "sessionPropertyOverrides is null")); } public Statement getStatement() @@ -266,5 +308,10 @@ public Optional getPrepareSql() { return prepareSql; } + + public List getSessionPropertyOverrides() + { + return sessionPropertyOverrides; + } } } diff --git a/core/trino-main/src/main/java/io/trino/sql/SessionPropertyResolver.java b/core/trino-main/src/main/java/io/trino/sql/SessionPropertyResolver.java index 886585a8b68e..a528c0f41506 100644 --- a/core/trino-main/src/main/java/io/trino/sql/SessionPropertyResolver.java +++ b/core/trino-main/src/main/java/io/trino/sql/SessionPropertyResolver.java @@ -14,6 +14,7 @@ package io.trino.sql; import com.google.common.collect.HashBasedTable; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Table; import com.google.inject.Inject; @@ -57,10 +58,16 @@ public SessionPropertyResolver(SessionPropertyEvaluator sessionPropertyEvaluator public SessionPropertiesApplier getSessionPropertiesApplier(PreparedQuery preparedQuery) { - if (!(preparedQuery.getStatement() instanceof Query queryStatement)) { + ImmutableList.Builder sessionProperties = ImmutableList.builder(); + if (preparedQuery.getStatement() instanceof Query queryStatement) { + sessionProperties.addAll(queryStatement.getSessionProperties()); + } + sessionProperties.addAll(preparedQuery.getSessionPropertyOverrides()); + List properties = sessionProperties.build(); + if (properties.isEmpty()) { return session -> session; } - return session -> prepareSession(session, queryStatement.getSessionProperties(), bindParameters(preparedQuery.getStatement(), preparedQuery.getParameters())); + return session -> prepareSession(session, properties, bindParameters(preparedQuery.getStatement(), preparedQuery.getParameters())); } private Session prepareSession(Session session, List sessionProperties, Map, Expression> parameters) diff --git a/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerSemanticCatalog.java b/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerSemanticCatalog.java index 32bd191418c0..37b67fd3b2ab 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerSemanticCatalog.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerSemanticCatalog.java @@ -13,23 +13,37 @@ */ package io.trino.execution; +import io.trino.Session; import io.trino.execution.QueryPreparer.PreparedQuery; import io.trino.hogql.compiler.HogQlCompileEnvelope; import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.HogQlErrorCode; +import io.trino.hogql.compiler.HogQlTypedValue; +import io.trino.hogql.compiler.HogQlTypedValue.BooleanValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ModifierBehavior; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticModifierDefault; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinRequest; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.metadata.SessionPropertyManager; +import io.trino.spi.TrinoException; +import io.trino.sql.SessionPropertyResolver; +import io.trino.sql.SqlEnvironmentConfig; import io.trino.sql.parser.SqlParser; import io.trino.sql.tree.Identifier; import io.trino.sql.tree.Query; import io.trino.sql.tree.QuerySpecification; import io.trino.sql.tree.Table; +import io.trino.testing.StandaloneQueryRunner; import org.junit.jupiter.api.Test; import java.util.List; @@ -39,16 +53,19 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import static io.trino.SessionTestUtils.TEST_SESSION; import static io.trino.execution.QuerySubmission.hogQl; import static io.trino.hogql.HogQlCompilationObserver.NOOP; +import static io.trino.spi.session.PropertyMetadata.booleanProperty; import static io.trino.testing.TestingSession.testSessionBuilder; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TestHogQlQueryPreparerSemanticCatalog { private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); private static final HogQlSemanticCatalogSnapshot SNAPSHOT = new HogQlSemanticCatalogSnapshot( - 1, + HogQlSemanticCatalogSnapshot.SCHEMA_VERSION, HogQlLanguageContract.current().languageVersion(), CATALOG, 7, @@ -58,6 +75,31 @@ public class TestHogQlQueryPreparerSemanticCatalog List.of(new LogicalFieldDefinition("event", new PhysicalIdentifier("event_name", false), "varchar", LogicalType.STRING, false, true)), List.of(), List.of()))); + private static final HogQlSemanticCatalogSnapshot MODIFIER_SNAPSHOT = new HogQlSemanticCatalogSnapshot( + HogQlSemanticCatalogSnapshot.PROTOCOL_VERSION, + HogQlSemanticCatalogSnapshot.SCHEMA_VERSION, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 7, + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(new SemanticModifierDefault( + "sampling", + ModifierBehavior.TRINO_SESSION_PROPERTY, + new TypedLiteral("boolean", LiteralEncoding.BOOLEAN, "false"), + List.of(new PhysicalIdentifier("hogql_sampling", false))), + new SemanticModifierDefault( + "legacyMode", + ModifierBehavior.SAFE_NOOP, + new TypedLiteral("boolean", LiteralEncoding.BOOLEAN, "false"), + List.of())), + List.of(), + List.of(), + List.of()); @Test public void testSessionCatalogPinsSemanticSnapshotAndResolvesLogicalTable() @@ -103,7 +145,102 @@ public void testMissingSessionCatalogOrProviderDoesNotFetchMetadata() assertThat(pins).hasValue(0); } + @Test + public void testModifierOverrideIsAppliedOutsideStatementAst() + { + try (StandaloneQueryRunner queryRunner = new StandaloneQueryRunner(TEST_SESSION)) { + SessionPropertyManager sessionPropertyManager = queryRunner.getSessionPropertyManager(); + sessionPropertyManager.addSystemSessionProperty(booleanProperty("hogql_sampling", "HogQL sampling", false, false)); + Session session = testSessionBuilder(sessionPropertyManager).setCatalog("analytics").build(); + QueryPreparer queryPreparer = new QueryPreparer( + new SqlParser(), + new HogQlCompiler(), + NOOP, + Optional.of(_ -> new PinnedSnapshot(MODIFIER_SNAPSHOT))); + + PreparedQuery preparedQuery = queryPreparer.prepareQuery( + session, + hogQl(envelope( + "SELECT 1", + OptionalLong.empty(), + Map.of("sampling", new HogQlTypedValue("boolean", new BooleanValue(true)))))); + + assertThat(((Query) preparedQuery.getStatement()).getSessionProperties()).isEmpty(); + assertThat(preparedQuery.getSessionPropertyOverrides()).singleElement() + .extracting(property -> property.getName().toString()) + .isEqualTo("hogql_sampling"); + + SessionPropertyResolver resolver = new SessionPropertyResolver( + new SessionPropertyEvaluator( + queryRunner.getPlannerContext(), + queryRunner.getAccessControl(), + sessionPropertyManager, + new SqlEnvironmentConfig()), + queryRunner.getAccessControl()); + Session overridden = resolver.getSessionPropertiesApplier(preparedQuery).apply(session); + + assertThat(overridden.getSystemProperty("hogql_sampling", Boolean.class)).isTrue(); + } + } + + @Test + public void testInvalidModifierValueFailsAsRedactedBindingError() + { + QueryPreparer queryPreparer = new QueryPreparer( + new SqlParser(), + new HogQlCompiler(), + NOOP, + Optional.of(_ -> new PinnedSnapshot(MODIFIER_SNAPSHOT))); + + assertThatThrownBy(() -> queryPreparer.prepareQuery( + testSessionBuilder().setCatalog("analytics").build(), + hogQl(envelope( + "SELECT 1", + OptionalLong.empty(), + Map.of("sampling", new HogQlTypedValue("boolean", new StringValue("secret-value"))))))) + .isInstanceOfSatisfying(TrinoException.class, exception -> { + assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_BINDING_ERROR.toErrorCode()); + assertThat(exception.getLocation()).hasValueSatisfying(location -> { + assertThat(location.lineNumber()).isEqualTo(1); + assertThat(location.columnNumber()).isEqualTo(1); + }); + assertThat(exception).hasMessageContaining("Invalid HogQL modifier binding: sampling"); + assertThat(exception).hasMessageNotContaining("secret-value"); + }); + } + + @Test + public void testInvalidSafeNoopValueFailsAsRedactedBindingError() + { + QueryPreparer queryPreparer = new QueryPreparer( + new SqlParser(), + new HogQlCompiler(), + NOOP, + Optional.of(_ -> new PinnedSnapshot(MODIFIER_SNAPSHOT))); + + assertThatThrownBy(() -> queryPreparer.prepareQuery( + testSessionBuilder().setCatalog("analytics").build(), + hogQl(envelope( + "SELECT 1", + OptionalLong.empty(), + Map.of("legacyMode", new HogQlTypedValue("boolean", new StringValue("secret-value"))))))) + .isInstanceOfSatisfying(TrinoException.class, exception -> { + assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_BINDING_ERROR.toErrorCode()); + assertThat(exception.getLocation()).hasValueSatisfying(location -> { + assertThat(location.lineNumber()).isEqualTo(1); + assertThat(location.columnNumber()).isEqualTo(1); + }); + assertThat(exception).hasMessageContaining("Invalid HogQL modifier binding: legacyMode"); + assertThat(exception).hasMessageNotContaining("secret-value"); + }); + } + private static HogQlCompileEnvelope envelope(String query, OptionalLong generation) + { + return envelope(query, generation, Map.of()); + } + + private static HogQlCompileEnvelope envelope(String query, OptionalLong generation, Map modifiers) { return new HogQlCompileEnvelope( query, @@ -112,7 +249,7 @@ private static HogQlCompileEnvelope envelope(String query, OptionalLong generati Map.of(), Map.of(), Map.of(), - Map.of(), + modifiers, generation); } } diff --git a/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java b/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java index 135939343645..614507592331 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java @@ -78,6 +78,8 @@ public void testSelectStatement() PreparedQuery submittedQuery = QUERY_PREPARER.prepareQuery(TEST_SESSION, trino("SELECT * FROM foo")); assertThat(preparedQuery.getStatement()).isEqualTo(simpleQuery(selectList(new AllColumns()), table(QualifiedName.of("foo")))); assertThat(submittedQuery.getStatement()).isEqualTo(preparedQuery.getStatement()); + assertThat(preparedQuery.getSessionPropertyOverrides()).isEmpty(); + assertThat(submittedQuery.getSessionPropertyOverrides()).isEmpty(); } @Test @@ -87,6 +89,7 @@ public void testHogQlStatement() QuerySubmission submission = hogQl(envelope); PreparedQuery preparedQuery = HOGQL_QUERY_PREPARER.prepareQuery(TEST_SESSION, submission); assertThat(preparedQuery.getStatement()).isInstanceOf(Query.class); + assertThat(preparedQuery.getSessionPropertyOverrides()).isEmpty(); assertThat(submission.hogQlEnvelope()).containsSame(envelope); } diff --git a/docs/src/main/sphinx/admin/hogql.md b/docs/src/main/sphinx/admin/hogql.md index e63f25fa5c95..20894be91845 100644 --- a/docs/src/main/sphinx/admin/hogql.md +++ b/docs/src/main/sphinx/admin/hogql.md @@ -34,6 +34,26 @@ paging and cancellation. The coordinator rejects request bodies larger than 2 MiB and more than 1,000 total entries across `parameters`, `variables`, `filters`, and `modifiers`. +## Modifiers + +Modifiers are defined by the pinned semantic catalog manifest. An explicit +modifier causes the coordinator to pin a manifest even when the query does not +otherwise use semantic catalog definitions. Unknown modifiers and values whose +type does not exactly match the declared type are rejected before execution. + +Each declared modifier has one behavior: + +| Behavior | Result | +| --- | --- | +| `TRINO_SESSION_PROPERTY` | Applies the explicit value, or the manifest default when omitted, as a query-scoped Trino session property. Normal session-property validation and access control still apply. | +| `COMPILER` | Uses the compiler's default behavior when omitted. Explicit values are rejected until that modifier has a compiler handler. | +| `SAFE_NOOP` | Accepts a type-valid explicit value or default and does not change the query. | +| `UNSUPPORTED` | Does nothing when omitted and rejects explicit use. | + +Modifier values stay outside the generated statement AST. The coordinator +decodes them through the typed-value boundary and carries the resulting session +overrides separately during query preparation. + To return a native Trino plan instead of executing the query, add an `explain` object: From ca97146d5fcb639e8a3b44f631dde665d83b90cf Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 14:53:47 -0400 Subject: [PATCH 058/136] feat(hogql): expand actions and cohorts --- .../trino/hogql/compiler/HogQlCompiler.java | 12 + .../hogql/compiler/HogQlFunctionResolver.java | 27 +++ .../hogql/compiler/HogQlSemanticResolver.java | 229 +++++++++++++++++- .../trino/hogql/compiler/TrinoAstFactory.java | 32 +++ .../TestHogQlSemanticEntityExpansion.java | 223 +++++++++++++++++ .../io/trino/hogql/parser/HogQlParser.java | 22 +- .../trino/hogql/parser/tree/HogQlQuery.java | 26 ++ 7 files changed, 561 insertions(+), 10 deletions(-) create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticEntityExpansion.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index fbce17e07919..c40117f1e4bc 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -33,7 +33,9 @@ import io.trino.hogql.parser.tree.HogQlQuery.Expression; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; @@ -322,7 +324,9 @@ private static boolean containsFunctionCall(Expression expression) case CastExpression cast -> containsFunctionCall(cast.value()); case ColumnReference _, Literal _, Placeholder _ -> false; case FunctionCall _ -> true; + case InCohortExpression _ -> true; case InExpression in -> containsFunctionCall(in.value()) || in.values().stream().anyMatch(HogQlCompiler::containsFunctionCall); + case InSubqueryExpression in -> containsFunctionCall(in.value()) || containsFunctionCall(in.query()); case IsNullExpression isNull -> containsFunctionCall(isNull.value()); case MemberAccessExpression memberAccess -> containsFunctionCall(memberAccess.base()); case SubscriptExpression subscript -> containsFunctionCall(subscript.base()) || containsFunctionCall(subscript.index()); @@ -416,10 +420,18 @@ private static void collectPlaceholders(Expression expression, List function.filter().ifPresent(filter -> collectPlaceholders(filter, placeholders)); function.window().ifPresent(window -> collectPlaceholders(window, placeholders)); } + case InCohortExpression in -> { + collectPlaceholders(in.value(), placeholders); + collectPlaceholders(in.cohort(), placeholders); + } case InExpression in -> { collectPlaceholders(in.value(), placeholders); in.values().forEach(value -> collectPlaceholders(value, placeholders)); } + case InSubqueryExpression in -> { + collectPlaceholders(in.value(), placeholders); + collectPlaceholders(in.query(), placeholders); + } case IsNullExpression isNull -> collectPlaceholders(isNull.value(), placeholders); case Literal _ -> {} case MemberAccessExpression memberAccess -> collectPlaceholders(memberAccess.base(), placeholders); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 0bc89b4d55ab..9b0471437cb6 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -35,7 +35,9 @@ import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; @@ -77,6 +79,8 @@ final class HogQlFunctionResolver { + private static final String MATCHES_ACTION = "matchesaction"; + private final Map functions; private HogQlFunctionResolver(PinnedSnapshot snapshot) @@ -196,12 +200,24 @@ private Expression resolveExpression(Expression expression) case CastExpression cast -> new CastExpression(resolveExpression(cast.value()), cast.type(), cast.safe(), cast.span()); case ColumnReference reference -> reference; case FunctionCall function -> resolveFunction(function, function.window().isPresent()); + case InCohortExpression in -> new InCohortExpression( + resolveExpression(in.value()), + resolveExpression(in.cohort()), + in.negated(), + in.predicateSpan(), + in.span()); case InExpression in -> new InExpression( resolveExpression(in.value()), in.values().stream().map(this::resolveExpression).toList(), in.negated(), in.predicateSpan(), in.span()); + case InSubqueryExpression in -> new InSubqueryExpression( + resolveExpression(in.value()), + resolveQuery(in.query()), + in.negated(), + in.predicateSpan(), + in.span()); case IsNullExpression isNull -> new IsNullExpression( resolveExpression(isNull.value()), isNull.negated(), @@ -225,6 +241,17 @@ private Expression resolveExpression(Expression expression) private Expression resolveFunction(FunctionCall function, boolean windowInvocation) { String name = function.name().value(); + if (function.nameParts().size() == 1 && canonical(name).equals(MATCHES_ACTION)) { + return new FunctionCall( + function.nameParts(), + function.arguments().stream().map(this::resolveExpression).toList(), + function.distinct(), + resolveSortItems(function.orderBy()), + function.filter().map(this::resolveExpression), + function.nullTreatment(), + function.window().map(this::resolveWindow), + function.span()); + } FunctionCapabilityDefinition capability = functions.get(canonical(name)); if (capability == null) { throw resolutionError(function, "Unknown HogQL function: " + name); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index b81c88070bab..a9db609085ce 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -13,8 +13,10 @@ */ package io.trino.hogql.compiler; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ActionReference; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CastRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CohortReference; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionArgument; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionRecipe; @@ -30,10 +32,13 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.MaterializedViewReference; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PredicateRepresentation; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyLookupRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ReferencedField; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRepresentation; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationReference; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipJoinSide; @@ -58,7 +63,9 @@ import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; @@ -102,6 +109,8 @@ final class HogQlSemanticResolver { + private static final String MATCHES_ACTION = "matchesaction"; + private final PinnedSnapshot snapshot; private final ExpansionBudget expansionBudget; private final Map relationshipPaths = new LinkedHashMap<>(); @@ -524,21 +533,20 @@ private Expression resolveExpression(Expression expression) caseExpression.span()); case CastExpression cast -> new CastExpression(resolveExpression(cast.value()), cast.type(), cast.safe(), cast.span()); case ColumnReference reference -> resolveColumn(reference); - case FunctionCall function -> new FunctionCall( - function.nameParts(), - function.arguments().stream().map(this::resolveExpression).toList(), - function.distinct(), - resolveSortItems(function.orderBy()), - function.filter().map(this::resolveExpression), - function.nullTreatment(), - function.window().map(this::resolveWindow), - function.span()); + case FunctionCall function -> resolveFunctionExpression(function); + case InCohortExpression in -> resolveCohortExpression(in); case InExpression in -> new InExpression( resolveExpression(in.value()), in.values().stream().map(this::resolveExpression).toList(), in.negated(), in.predicateSpan(), in.span()); + case InSubqueryExpression in -> new InSubqueryExpression( + resolveExpression(in.value()), + new HogQlSemanticResolver(snapshot, expansionBudget).resolveNestedQuery(in.query()), + in.negated(), + in.predicateSpan(), + in.span()); case IsNullExpression isNull -> new IsNullExpression( resolveExpression(isNull.value()), isNull.negated(), @@ -553,6 +561,187 @@ private Expression resolveExpression(Expression expression) }; } + private Expression resolveFunctionExpression(FunctionCall function) + { + if (function.nameParts().size() == 1 && canonical(function.name().value()).equals(MATCHES_ACTION)) { + return resolveActionExpression(function); + } + return new FunctionCall( + function.nameParts(), + function.arguments().stream().map(this::resolveExpression).toList(), + function.distinct(), + resolveSortItems(function.orderBy()), + function.filter().map(this::resolveExpression), + function.nullTreatment(), + function.window().map(this::resolveWindow), + function.span()); + } + + private Expression resolveActionExpression(FunctionCall function) + { + if (function.arguments().size() != 1) { + throw semanticEntityError(HOGQL_RESOLUTION_ERROR, function.span(), "HogQL matchesAction requires exactly one argument"); + } + if (function.distinct() || !function.orderBy().isEmpty() || function.filter().isPresent() || function.nullTreatment().isPresent() || function.window().isPresent()) { + throw semanticEntityError(HOGQL_UNSUPPORTED_FEATURE, function.span(), "HogQL matchesAction does not support invocation modifiers"); + } + EntityLookup lookup = entityLookup(function.arguments().getFirst(), "action"); + List matches = snapshot.snapshot().actions().stream() + .filter(action -> lookup.matches(action.name(), action.actionId())) + .toList(); + ActionReference action = requireEntity(matches, "action", function.span()); + TableBinding binding = requireEntityBinding(action.table(), "action", function.span()); + return switch (action.representation()) { + case PredicateRepresentation predicate -> expandRecipe( + binding, + predicate.predicate(), + Optional.of(binding.outputQualifier()), + function.span(), + expansionBudget); + case RelationMembershipRepresentation membership -> membershipExpression(binding, membership.relation(), false, function.span()); + }; + } + + private Expression resolveCohortExpression(InCohortExpression in) + { + EntityLookup lookup = entityLookup(in.cohort(), "cohort"); + List matches = snapshot.snapshot().cohorts().stream() + .filter(cohort -> lookup.matches(cohort.name(), cohort.cohortId())) + .toList(); + CohortReference cohort = requireEntity(matches, "cohort", in.span()); + TableBinding binding = requireEntityBinding(cohort.table(), "cohort", in.span()); + return switch (cohort.representation()) { + case PredicateRepresentation predicate -> { + requireCohortSource(in.value(), binding, in.span()); + Expression expanded = expandRecipe( + binding, + predicate.predicate(), + Optional.of(binding.outputQualifier()), + in.span(), + expansionBudget); + yield in.negated() ? new UnaryExpression(HogQlQuery.UnaryOperator.NOT, expanded, in.span()) : expanded; + } + case RelationMembershipRepresentation membership -> { + requireMembershipSource(in.value(), binding, membership.relation(), in.span()); + yield membershipExpression(binding, membership.relation(), in.negated(), in.span()); + } + }; + } + + private void requireCohortSource(Expression value, TableBinding binding, HogQlQuery.SourceSpan span) + { + if (!(value instanceof ColumnReference reference)) { + throw unsupportedExpansion(span, "HogQL cohort membership source must be a declared field reference"); + } + List parts = reference.parts(); + boolean qualified = parts.size() == 2 && binding.qualifier().equals(canonical(parts.getFirst().value())); + boolean matches = (parts.size() == 1 || qualified) && binding.fields().containsKey(canonical(parts.getLast().value())); + if (!matches) { + throw unsupportedExpansion(span, "HogQL cohort membership source does not match the catalog table"); + } + } + + private InSubqueryExpression membershipExpression( + TableBinding sourceBinding, + RelationMembershipRecipe membership, + boolean negated, + HogQlQuery.SourceSpan span) + { + expansionBudget.enter(span); + try { + BoundField sourceField = Optional.ofNullable(sourceBinding.fields().get(canonical(membership.sourceField()))) + .orElseThrow(() -> expansionError(span, "HogQL semantic entity references an unavailable source field")); + Expression source = resolveBoundField( + sourceBinding, + sourceField, + sourceBinding.starQualifier(bindings.size()), + span, + expansionBudget); + ResolvedRelation target = resolveRelationReference(membership.relation(), span, new RelationExpansionBudget()); + if (target.bindings().size() != 1) { + throw unsupportedExpansion(span, "HogQL semantic entity membership requires a single target relation"); + } + TableBinding targetBinding = target.bindings().getFirst(); + BoundField targetField = Optional.ofNullable(targetBinding.fields().get(canonical(membership.targetField()))) + .orElseThrow(() -> expansionError(span, "HogQL semantic entity references an unavailable target field")); + Expression targetValue = resolveBoundField( + targetBinding, + targetField, + targetBinding.starQualifier(1), + span, + expansionBudget); + HogQlQuery membershipQuery = new HogQlQuery( + List.of(), + true, + List.of(new ExpressionProjection(targetValue, Optional.empty())), + Optional.of(target.relation()), + Optional.empty(), + List.of(), + Optional.empty(), + List.of(), + Optional.empty(), + Optional.empty(), + span); + return new InSubqueryExpression(source, membershipQuery, negated, span, span); + } + finally { + expansionBudget.exit(); + } + } + + private void requireMembershipSource(Expression value, TableBinding binding, RelationMembershipRecipe membership, HogQlQuery.SourceSpan span) + { + if (!(value instanceof ColumnReference reference)) { + throw unsupportedExpansion(span, "HogQL cohort membership source must be a declared field reference"); + } + List parts = reference.parts(); + boolean matches = switch (parts.size()) { + case 1 -> canonical(parts.getFirst().value()).equals(canonical(membership.sourceField())); + case 2 -> binding.qualifier().equals(canonical(parts.getFirst().value())) && + canonical(parts.getLast().value()).equals(canonical(membership.sourceField())); + default -> false; + }; + if (!matches) { + throw unsupportedExpansion(span, "HogQL cohort membership source does not match the catalog"); + } + } + + private TableBinding requireEntityBinding(String tableName, String kind, HogQlQuery.SourceSpan span) + { + List matches = bindings.stream() + .filter(binding -> canonical(binding.relationName()).equals(canonical(tableName))) + .toList(); + if (matches.size() != 1) { + throw semanticEntityError( + HOGQL_RESOLUTION_ERROR, + span, + "HogQL " + kind + " requires exactly one " + tableName + " relation in scope"); + } + return matches.getFirst(); + } + + private EntityLookup entityLookup(Expression expression, String kind) + { + if (!(expression instanceof Literal literal) || (literal.kind() != HogQlQuery.LiteralKind.STRING && literal.kind() != HogQlQuery.LiteralKind.INTEGER)) { + throw semanticEntityError( + HOGQL_UNSUPPORTED_FEATURE, + expression.span(), + "HogQL " + kind + " reference must be a string or integer literal"); + } + return new EntityLookup(literal.value(), literal.kind() == HogQlQuery.LiteralKind.INTEGER); + } + + private static T requireEntity(List matches, String kind, HogQlQuery.SourceSpan span) + { + if (matches.isEmpty()) { + throw semanticEntityError(HOGQL_RESOLUTION_ERROR, span, "Unknown HogQL " + kind); + } + if (matches.size() > 1) { + throw semanticEntityError(HOGQL_RESOLUTION_ERROR, span, "Ambiguous HogQL " + kind); + } + return matches.getFirst(); + } + private WindowDefinition resolveWindowDefinition(WindowDefinition definition) { return new WindowDefinition(definition.name(), (WindowSpecification) resolveWindow(definition.specification()), definition.span()); @@ -1231,6 +1420,15 @@ private static TrinoException unsupportedExpansion(HogQlQuery.SourceSpan span, S null); } + private static TrinoException semanticEntityError(HogQlErrorCode errorCode, HogQlQuery.SourceSpan span, String message) + { + return new TrinoException( + errorCode, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } + private static TrinoException limitError(HogQlQuery.SourceSpan span, String message) { return new TrinoException( @@ -1410,6 +1608,19 @@ private record ProjectedField(String name, String sourceField, boolean starVisib } } + private record EntityLookup(String value, boolean id) + { + private EntityLookup + { + value = requireNonNull(value, "value is null"); + } + + private boolean matches(String name, String entityId) + { + return id ? value.equals(entityId) : canonical(value).equals(canonical(name)); + } + } + private static final class ExpansionBudget { private static final int MAXIMUM_NODES = 10_000; diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index fea9cb02cf13..9a94849a87b2 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -27,7 +27,9 @@ import io.trino.hogql.parser.tree.HogQlQuery.FrameBound; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; @@ -53,6 +55,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.WindowFrame; import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; import io.trino.sql.parser.SqlParser; import io.trino.sql.tree.AllColumns; import io.trino.sql.tree.ArithmeticBinaryExpression; @@ -108,6 +112,8 @@ import java.util.Map; import java.util.Optional; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; + final class TrinoAstFactory { private static final SqlParser SQL_PARSER = new SqlParser(); @@ -289,7 +295,9 @@ private static Expression createExpression(HogQlQuery.Expression expression, Map cast.safe()); case ColumnReference reference -> createColumnReference(reference); case FunctionCall function -> createFunctionCall(function, parameterIds); + case InCohortExpression in -> throw unsupportedSemanticExpression(in.span(), "HogQL IN COHORT requires a semantic catalog snapshot"); case InExpression in -> createInExpression(in, parameterIds); + case InSubqueryExpression in -> createInSubqueryExpression(in, parameterIds); case IsNullExpression isNull -> new Predicated( location(isNull.predicateSpan()), createExpression(isNull.value(), parameterIds), @@ -319,6 +327,9 @@ private static Expression createExpression(HogQlQuery.Expression expression, Map private static Expression createFunctionCall(FunctionCall function, Map parameterIds) { + if (function.nameParts().size() == 1 && function.name().value().equalsIgnoreCase("matchesAction")) { + throw unsupportedSemanticExpression(function.span(), "HogQL matchesAction requires a semantic catalog snapshot and events relation"); + } return new io.trino.sql.tree.FunctionCall( location(function.span()), QualifiedName.of(function.nameParts().stream() @@ -449,6 +460,18 @@ private static Expression createInExpression(InExpression in, Map parameterIds) + { + NodeLocation location = location(in.predicateSpan()); + return new Predicated( + location, + createExpression(in.value(), parameterIds), + new InPredicate( + location, + in.negated(), + new io.trino.sql.tree.SubqueryExpression(location, createQuery(in.query(), parameterIds)))); + } + private static Expression createBinaryExpression(BinaryExpression binary, Map parameterIds) { NodeLocation location = location(binary.span()); @@ -585,4 +608,13 @@ private static NodeLocation location(SourceSpan span) { return new NodeLocation(span.startLine(), span.startColumn()); } + + private static TrinoException unsupportedSemanticExpression(SourceSpan span, String message) + { + return new TrinoException( + HOGQL_UNSUPPORTED_FEATURE, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } } diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticEntityExpansion.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticEntityExpansion.java new file mode 100644 index 000000000000..92f1bc60ee91 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticEntityExpansion.java @@ -0,0 +1,223 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ActionReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CohortReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.MaterializedViewReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PredicateRepresentation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ReferencedField; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRepresentation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.stream.Stream; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlSemanticEntityExpansion +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); + private static final HogQlSemanticCatalogSnapshot SNAPSHOT = snapshot(); + + private final HogQlCompiler compiler = new HogQlCompiler(); + private final SqlParser sqlParser = new SqlParser(); + + @ParameterizedTest + @ValueSource(strings = {"'Paid event'", "17"}) + public void testExpandsActionPredicateByNameOrId(String action) + { + assertThat(compile("SELECT event FROM events e WHERE matchesAction(" + action + ")").statement()) + .isEqualTo(sqlParser.createStatement( + "SELECT event_name AS event FROM analytics.data.raw_events e " + + "WHERE e.event_name = CAST('purchase' AS varchar)")); + } + + @ParameterizedTest + @ValueSource(strings = {"IN COHORT 23", "NOT IN COHORT 'Active people'"}) + public void testExpandsCohortMembershipAsInSubquery(String predicate) + { + assertThat(compile("SELECT event FROM events WHERE personId " + predicate).statement()) + .isEqualTo(sqlParser.createStatement( + "SELECT event_name AS event FROM analytics.data.raw_events " + + "WHERE person_id " + (predicate.startsWith("NOT") ? "NOT " : "") + + "IN (SELECT DISTINCT \"personId\" FROM analytics.data.active_people)")); + } + + @Test + public void testExpandsActionRelationMembership() + { + assertThat(compile("SELECT event FROM events WHERE matchesAction(18)").statement()) + .isEqualTo(sqlParser.createStatement( + "SELECT event_name AS event FROM analytics.data.raw_events " + + "WHERE person_id IN (SELECT DISTINCT \"personId\" FROM analytics.data.active_people)")); + } + + @Test + public void testExpandsCohortPredicate() + { + assertThat(compile("SELECT event FROM events WHERE event NOT IN COHORT 24").statement()) + .isEqualTo(sqlParser.createStatement( + "SELECT event_name AS event FROM analytics.data.raw_events " + + "WHERE NOT (raw_events.event_name = CAST('purchase' AS varchar))")); + } + + @Test + public void testRejectsUnknownActionWithTypedResolutionError() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compile("SELECT event FROM events WHERE matchesAction('Missing action')")); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception).hasMessageContaining("Unknown HogQL action"); + } + + @ParameterizedTest + @MethodSource("invalidCohortQueries") + public void testRejectsInvalidCohortUseWithTypedCompatibilityError(String query, String message) + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compile(query)); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception).hasMessageContaining(message); + } + + private static Stream invalidCohortQueries() + { + return Stream.of( + Arguments.of( + "SELECT event FROM events WHERE personId IN COHORT event", + "HogQL cohort reference must be a string or integer literal"), + Arguments.of( + "SELECT event FROM events WHERE event IN COHORT 23", + "HogQL cohort membership source does not match the catalog")); + } + + private HogQlCompilationResult compile(String query) + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + return compiler.compile(new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(11)), Optional.of(context)); + } + + private static HogQlSemanticCatalogSnapshot snapshot() + { + LogicalTableDefinition events = new LogicalTableDefinition( + "events", + physicalName("raw_events"), + List.of( + field("event", "event_name"), + field("personId", "person_id")), + List.of(), + List.of()); + MaterializedViewReference activePeople = new MaterializedViewReference( + "active_people", + physicalName("active_people"), + List.of(new ReferencedField("personId", "varchar", LogicalType.STRING, false, true))); + PredicateRepresentation purchasePredicate = new PredicateRepresentation(new OperatorRecipe( + SemanticOperator.EQUAL, + List.of( + new FieldReferenceRecipe("events", "event"), + new LiteralRecipe(new TypedLiteral("varchar", LiteralEncoding.STRING, "purchase"))))); + RelationMembershipRepresentation activeMembership = new RelationMembershipRepresentation(new RelationMembershipRecipe( + new RelationReference(RelationKind.MATERIALIZED_VIEW, "active_people"), + "personId", + "personId")); + ActionReference paidEvent = new ActionReference( + "Paid event", + "17", + "events", + purchasePredicate); + ActionReference precomputedAction = new ActionReference( + "Precomputed action", + "18", + "events", + activeMembership); + CohortReference activePeopleCohort = new CohortReference( + "Active people", + "23", + "events", + activeMembership); + CohortReference purchasers = new CohortReference( + "Purchasers", + "24", + "events", + purchasePredicate); + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 11, + List.of(events), + List.of(), + List.of(), + List.of(), + List.of(activePeople), + List.of(), + List.of(), + List.of(), + List.of(paidEvent, precomputedAction), + List.of(activePeopleCohort, purchasers)); + } + + private static LogicalFieldDefinition field(String name, String physicalName) + { + return new LogicalFieldDefinition(name, new PhysicalIdentifier(physicalName, false), "varchar", LogicalType.STRING, false, true); + } + + private static PhysicalQualifiedName physicalName(String table) + { + return new PhysicalQualifiedName(CATALOG, new PhysicalIdentifier("data", false), new PhysicalIdentifier(table, false)); + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 5378418bcec0..9c6d074652b3 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -34,7 +34,9 @@ import io.trino.hogql.parser.tree.HogQlQuery.FrameType; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.JoinCriteria; import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; @@ -1075,7 +1077,15 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) return binary(operator, binary.left, binary.right, binary); } if (context instanceof HogQLParser.ColumnExprPrecedence3Context binary) { - if (binary.IN() != null && binary.COHORT() == null) { + if (binary.IN() != null) { + if (binary.COHORT() != null) { + return new InCohortExpression( + buildExpression(binary.left), + buildExpression(binary.right), + binary.NOT() != null, + sourceSpan(binary.NOT() == null ? binary.IN().getSymbol() : binary.NOT().getSymbol(), binary.getStop()), + sourceSpan(binary)); + } return new InExpression( buildExpression(binary.left), buildInValues(binary.right), @@ -1648,6 +1658,16 @@ private void validateExpressionScope(Expression expression, Set forbidde validateExpressionScope(in.value(), forbiddenOuterRelations, localRelations); in.values().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); } + case InCohortExpression in -> { + validateExpressionScope(in.value(), forbiddenOuterRelations, localRelations); + validateExpressionScope(in.cohort(), forbiddenOuterRelations, localRelations); + } + case InSubqueryExpression in -> { + validateExpressionScope(in.value(), forbiddenOuterRelations, localRelations); + Set nestedForbiddenRelations = new LinkedHashSet<>(forbiddenOuterRelations); + nestedForbiddenRelations.addAll(localRelations); + validateQueryScope(in.query(), Set.copyOf(nestedForbiddenRelations)); + } case IsNullExpression isNull -> validateExpressionScope(isNull.value(), forbiddenOuterRelations, localRelations); case Literal _, Placeholder _ -> {} case MemberAccessExpression memberAccess -> validateExpressionScope(memberAccess.base(), forbiddenOuterRelations, localRelations); diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index d636688c2839..f0a47ab58cb3 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -255,7 +255,9 @@ public sealed interface Expression CastExpression, ColumnReference, FunctionCall, + InCohortExpression, InExpression, + InSubqueryExpression, IsNullExpression, Literal, MemberAccessExpression, @@ -376,6 +378,30 @@ public record InExpression(Expression value, List values, boolean ne } } + public record InCohortExpression(Expression value, Expression cohort, boolean negated, SourceSpan predicateSpan, SourceSpan span) + implements Expression + { + public InCohortExpression + { + value = requireNonNull(value, "value is null"); + cohort = requireNonNull(cohort, "cohort is null"); + predicateSpan = requireNonNull(predicateSpan, "predicateSpan is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record InSubqueryExpression(Expression value, HogQlQuery query, boolean negated, SourceSpan predicateSpan, SourceSpan span) + implements Expression + { + public InSubqueryExpression + { + value = requireNonNull(value, "value is null"); + query = requireNonNull(query, "query is null"); + predicateSpan = requireNonNull(predicateSpan, "predicateSpan is null"); + span = requireNonNull(span, "span is null"); + } + } + public record IsNullExpression(Expression value, boolean negated, SourceSpan predicateSpan, SourceSpan span) implements Expression { From fa8d7e70b3ec43a62f97d5ec1e35d598c14b9aac Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 15:15:03 -0400 Subject: [PATCH 059/136] feat(hogql): expand qualified stars --- .../hogql/compiler/HogQlSemanticResolver.java | 161 ++++++++++++++++-- .../trino/hogql/compiler/TrinoAstFactory.java | 17 +- .../compiler/TestHogQlSemanticResolution.java | 148 ++++++++++++++++ .../io/trino/hogql/parser/HogQlParser.java | 10 +- .../trino/hogql/parser/tree/HogQlQuery.java | 9 +- .../trino/hogql/parser/TestHogQlParser.java | 28 +++ 6 files changed, 353 insertions(+), 20 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index a9db609085ce..b1c79bdb6583 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -96,11 +96,13 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.Set; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_COMPILER_LIMIT_EXCEEDED; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; @@ -212,19 +214,7 @@ private HogQlQuery resolveQuery(HogQlQuery query, List co private List resolveProjection(Projection projection) { return switch (projection) { - case Star star -> { - if (!allRelationsLogical) { - yield List.of(star); - } - yield bindings.stream() - .flatMap(binding -> binding.orderedFields().stream() - .filter(BoundField::starVisible) - .map(field -> new ExpressionProjection( - resolveBoundField(binding, field, binding.starQualifier(bindings.size()), star.span(), expansionBudget), - Optional.of(new Identifier(field.name(), true, star.span()))))) - .map(Projection.class::cast) - .toList(); - } + case Star star -> resolveStar(star); case ExpressionProjection expressionProjection -> { Expression resolved = resolveExpression(expressionProjection.expression()); Optional alias = expressionProjection.alias(); @@ -236,6 +226,123 @@ private List resolveProjection(Projection projection) }; } + private List resolveStar(Star star) + { + List starBindings; + boolean qualified = !star.qualifier().isEmpty(); + if (qualified) { + starBindings = bindings.stream() + .filter(binding -> matchesStarQualifier(binding, star.qualifier())) + .toList(); + if (starBindings.size() > 1) { + throw starResolutionError(star.qualifier().getFirst(), "Ambiguous HogQL star qualifier: " + starQualifier(star)); + } + if (starBindings.isEmpty()) { + if (allRelationsLogical) { + throw starResolutionError(star.qualifier().getFirst(), "Unknown HogQL star qualifier: " + starQualifier(star)); + } + return List.of(star); + } + } + else { + if (!allRelationsLogical) { + return List.of(star); + } + starBindings = bindings; + } + + Set exclusions = resolveStarExclusions(star, starBindings); + return starBindings.stream() + .flatMap(binding -> binding.orderedFields().stream() + .filter(BoundField::starVisible) + .filter(field -> !exclusions.contains(new StarField(binding, field))) + .map(field -> new ExpressionProjection( + resolveBoundField( + binding, + field, + qualified ? Optional.of(binding.outputQualifier()) : binding.starQualifier(bindings.size()), + star.span(), + expansionBudget), + Optional.of(new Identifier(field.name(), true, star.span()))))) + .map(Projection.class::cast) + .toList(); + } + + private static Set resolveStarExclusions(Star star, List starBindings) + { + Set exclusions = new HashSet<>(); + for (ColumnReference exclusion : star.exclusions()) { + List parts = exclusion.parts(); + Identifier fieldName = parts.getLast(); + List exclusionBindings = parts.size() == 1 ? starBindings : starBindings.stream() + .filter(binding -> matchesStarQualifier(binding, parts.subList(0, parts.size() - 1))) + .toList(); + if (parts.size() > 1 && exclusionBindings.size() > 1) { + throw starResolutionError(exclusion.span(), "Ambiguous HogQL star exclusion qualifier: " + identifierPath(parts.subList(0, parts.size() - 1))); + } + List matchedFields = exclusionBindings.stream() + .flatMap(binding -> binding.orderedFields().stream() + .filter(BoundField::starVisible) + .filter(field -> matchesIdentifier(fieldName, field.name())) + .map(field -> new StarField(binding, field))) + .distinct() + .toList(); + if (matchedFields.isEmpty()) { + throw starResolutionError(exclusion.span(), "Unknown HogQL star exclusion: " + identifierPath(parts)); + } + if (matchedFields.stream().anyMatch(exclusions::contains)) { + throw starResolutionError(exclusion.span(), "Duplicate HogQL star exclusion: " + identifierPath(parts)); + } + exclusions.addAll(matchedFields); + } + return exclusions; + } + + private static boolean matchesStarQualifier(TableBinding binding, List qualifier) + { + if (binding.aliased()) { + return qualifier.size() == 1 && matchesIdentifier(qualifier.getFirst(), binding.outputQualifier()); + } + if (qualifier.size() == 1 && canonicalIdentifier(qualifier.getFirst().value(), qualifier.getFirst().delimited()).equals(canonical(binding.relationName()))) { + return true; + } + if (qualifier.size() > binding.physicalQualifier().size()) { + return false; + } + int offset = binding.physicalQualifier().size() - qualifier.size(); + for (int index = 0; index < qualifier.size(); index++) { + if (!matchesIdentifier(qualifier.get(index), binding.physicalQualifier().get(offset + index))) { + return false; + } + } + return !qualifier.isEmpty(); + } + + private static boolean matchesIdentifier(Identifier identifier, String value) + { + return identifier.delimited() ? identifier.value().equals(value) : canonical(identifier.value()).equals(canonical(value)); + } + + private static boolean matchesIdentifier(Identifier identifier, PhysicalIdentifier value) + { + return canonicalIdentifier(identifier.value(), identifier.delimited()).equals(canonicalIdentifier(value.value(), value.delimited())); + } + + private static String canonicalIdentifier(String value, boolean delimited) + { + return delimited ? value : canonical(value); + } + + private static String starQualifier(Star star) + { + return identifierPath(star.qualifier()); + } + + private static String identifierPath(List identifiers) + { + return String.join(".", identifiers.stream().map(Identifier::value).toList()); + } + private ResolvedRelation resolveRelation(Relation relation) { return switch (relation) { @@ -375,6 +482,7 @@ private ResolvedRelation resolveLogicalTable(LogicalTableDefinition definition, definition, canonical(definition.name()), definition.physicalTable().table(), + List.of(definition.physicalTable().catalog(), definition.physicalTable().schema(), definition.physicalTable().table()), false, fields)), true); @@ -397,6 +505,7 @@ private ResolvedRelation resolveMaterializedView(MaterializedViewReference defin definition.name(), canonical(definition.name()), new PhysicalIdentifier(definition.physicalView().table().value(), definition.physicalView().table().delimited()), + List.of(definition.physicalView().catalog(), definition.physicalView().schema(), definition.physicalView().table()), false, fields, TableBinding.fieldMap(fields))), @@ -490,6 +599,7 @@ private ResolvedRelation projectRelation(String relationName, ResolvedRelation s relationName, canonical(relationName), new PhysicalIdentifier(relationName, true), + List.of(), false, fields, TableBinding.fieldMap(fields))), @@ -1452,6 +1562,7 @@ private static TableBinding tableBinding( LogicalTableDefinition definition, String qualifier, PhysicalIdentifier outputQualifier, + List physicalQualifier, boolean aliased, List fields) { @@ -1459,6 +1570,7 @@ private static TableBinding tableBinding( definition.name(), qualifier, outputQualifier, + physicalQualifier, aliased, fields, TableBinding.fieldMap(fields)); @@ -1496,6 +1608,20 @@ private static TrinoException ambiguousPropertyResolutionError(HogQlQuery.Source null); } + private static TrinoException starResolutionError(Identifier identifier, String message) + { + return starResolutionError(identifier.span(), message); + } + + private static TrinoException starResolutionError(HogQlQuery.SourceSpan span, String message) + { + return new TrinoException( + HOGQL_RESOLUTION_ERROR, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } + private static TrinoException incompatibleUsingResolutionError(Identifier identifier) { return new TrinoException( @@ -1518,13 +1644,14 @@ private record TableBinding( String relationName, String qualifier, PhysicalIdentifier outputQualifier, + List physicalQualifier, boolean aliased, List orderedFields, Map fields) { - private TableBinding(LogicalTableDefinition logicalTable, String qualifier, PhysicalIdentifier outputQualifier, boolean aliased) + private TableBinding(LogicalTableDefinition logicalTable, String qualifier, PhysicalIdentifier outputQualifier, List physicalQualifier, boolean aliased) { - this(logicalTable.name(), qualifier, outputQualifier, aliased, boundFields(logicalTable), fieldMap(boundFields(logicalTable))); + this(logicalTable.name(), qualifier, outputQualifier, physicalQualifier, aliased, boundFields(logicalTable), fieldMap(boundFields(logicalTable))); } private TableBinding @@ -1532,6 +1659,7 @@ private TableBinding(LogicalTableDefinition logicalTable, String qualifier, Phys relationName = requireNonNull(relationName, "relationName is null"); qualifier = requireNonNull(qualifier, "qualifier is null"); outputQualifier = requireNonNull(outputQualifier, "outputQualifier is null"); + physicalQualifier = List.copyOf(requireNonNull(physicalQualifier, "physicalQualifier is null")); orderedFields = List.copyOf(requireNonNull(orderedFields, "orderedFields is null")); fields = Map.copyOf(requireNonNull(fields, "fields is null")); } @@ -1542,6 +1670,7 @@ private TableBinding withAlias(Identifier alias) relationName, canonical(alias.value()), new PhysicalIdentifier(alias.value(), alias.delimited()), + physicalQualifier, true, orderedFields, fields); @@ -1579,6 +1708,8 @@ private record BoundField(String name, PhysicalIdentifier physicalColumn, boolea private record FieldMatch(TableBinding binding, BoundField field) {} + private record StarField(TableBinding binding, BoundField field) {} + private record PropertyMatch(TableBinding binding, PropertyDefinition property) {} private record PathCandidate(TableBinding owner, List path) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 9a94849a87b2..57c09905a911 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -269,7 +269,7 @@ private static Optional createGroupBy(List expre private static SelectItem createSelectItem(Projection projection, Map parameterIds) { return switch (projection) { - case Star star -> new AllColumns(location(star.span())); + case Star star -> createAllColumns(star); case ExpressionProjection expression -> new SingleColumn( location(expression.span()), createExpression(expression.expression(), parameterIds), @@ -277,6 +277,21 @@ private static SelectItem createSelectItem(Projection projection, Map target = star.qualifier().isEmpty() + ? Optional.empty() + : Optional.of(createColumnReference(new ColumnReference(star.qualifier(), star.span()))); + return new AllColumns(location(star.span()), target, List.of()); + } + private static Expression createExpression(HogQlQuery.Expression expression, Map parameterIds) { return switch (expression) { diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java index ae7a1b026c5e..f5016c6ef44c 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -44,6 +44,7 @@ import java.util.concurrent.atomic.AtomicReference; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowableOfType; @@ -133,6 +134,142 @@ public void testPinsAndResolvesLogicalTableFieldsAndStar() .containsExactly("event_name", "Person ID"); } + @Test + public void testExpandsQualifiedLogicalStarsWithExclusionsInManifestOrder() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compile( + envelope( + "SELECT e.* EXCLUDE (event), p.* EXCLUDE (personId) " + + "FROM events e JOIN persons p ON e.personId = p.personId", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT e.\"Person ID\" AS \"personId\", p.full_name AS \"name\" " + + "FROM analytics.\"Hog Data\".\"raw-events\" e " + + "JOIN analytics.\"Hog Data\".\"raw-persons\" p ON e.\"Person ID\" = p.person_id")); + } + + @Test + public void testExpandsQuotedLogicalStarAndUnqualifiedExclusion() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult quoted = compiler.compile( + envelope("SELECT \"E\".* EXCLUDE (\"event\") FROM events AS \"E\"", OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult unqualified = compiler.compile( + envelope("SELECT * EXCLUDE (events.personId) FROM events", OptionalLong.of(7)), + Optional.of(context)); + + assertThat(quoted.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"E\".\"Person ID\" AS \"personId\" FROM analytics.\"Hog Data\".\"raw-events\" AS \"E\"")); + assertThat(unqualified.statement()).isEqualTo(sqlParser.createStatement( + "SELECT event_name AS \"event\" FROM analytics.\"Hog Data\".\"raw-events\"")); + } + + @Test + public void testMatchesLogicalAndPhysicalQualifiedStarSuffixes() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compile( + envelope( + "SELECT events.* EXCLUDE (personId), " + + "\"raw-events\".* EXCLUDE (personId), " + + "\"Hog Data\".\"raw-events\".* EXCLUDE (personId), " + + "analytics.\"Hog Data\".\"raw-events\".* EXCLUDE (personId) FROM events", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"raw-events\".event_name AS \"event\", " + + "\"raw-events\".event_name AS \"event\", " + + "\"raw-events\".event_name AS \"event\", " + + "\"raw-events\".event_name AS \"event\" FROM analytics.\"Hog Data\".\"raw-events\"")); + } + + @Test + public void testRejectsInvalidLogicalStarQualifierAndExclusionsAtSource() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + assertResolutionFailure( + context, + "SELECT e.* EXCLUDE (missing) FROM events e", + "missing", + "Unknown HogQL star exclusion: missing"); + assertResolutionFailure( + context, + "SELECT e.* EXCLUDE (event, \"event\") FROM events e", + "\"event\"", + "Duplicate HogQL star exclusion: event"); + assertResolutionFailure( + context, + "SELECT e.* FROM events e JOIN persons e ON e.personId = e.personId", + "e.*", + "Ambiguous HogQL star qualifier: e"); + assertResolutionFailure( + context, + "SELECT e.* FROM events AS \"E\"", + "e.*", + "Unknown HogQL star qualifier: e"); + assertResolutionFailure( + context, + "SELECT events.* FROM events e", + "events.*", + "Unknown HogQL star qualifier: events"); + assertResolutionFailure( + context, + "SELECT analytics.\"hog data\".\"raw-events\".* FROM events", + "analytics", + "Unknown HogQL star qualifier: analytics.hog data.raw-events"); + } + + @Test + public void testLowersQualifiedPhysicalStarWithoutFetchingSemanticMetadata() + { + AtomicInteger pins = new AtomicInteger(); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + }); + + HogQlCompilationResult result = compiler.compile( + envelope("SELECT p.* FROM analytics.default.raw_events p", OptionalLong.empty()), + Optional.of(context)); + + assertThat(pins).hasValue(0); + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT p.* FROM analytics.default.raw_events p")); + } + + @Test + public void testRejectsStarExclusionsWhenRelationSchemaIsUnavailable() + { + AtomicInteger pins = new AtomicInteger(); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + }); + List queries = List.of( + "SELECT p.* EXCLUDE (event_name) FROM analytics.default.raw_events p", + "WITH source AS (SELECT 1 AS event_name) SELECT s.* EXCLUDE (event_name) FROM source s"); + + for (String hogql : queries) { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope(hogql, OptionalLong.empty()), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, hogql.lastIndexOf("event_name") + 1)); + assertThat(exception).hasMessageContaining("HogQL star exclusions require a logical relation from the semantic catalog: event_name"); + } + assertThat(pins).hasValue(0); + } + @Test public void testResolvesLogicalFieldsInsideNamedWindows() { @@ -250,6 +387,17 @@ private static QuerySpecification querySpecification(HogQlCompilationResult resu return (QuerySpecification) ((Query) result.statement()).getQueryBody(); } + private void assertResolutionFailure(HogQlSemanticCatalogContext context, String hogql, String errorToken, String message) + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope(hogql, OptionalLong.empty()), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, hogql.indexOf(errorToken) + 1)); + assertThat(exception).hasMessage("line %s:%s: %s", 1, hogql.indexOf(errorToken) + 1, message); + } + private static HogQlCompileEnvelope envelope(String query, OptionalLong generation) { return new HogQlCompileEnvelope( diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 9c6d074652b3..48a087cff926 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -983,10 +983,14 @@ else if (context instanceof HogQLParser.ColumnExprSelectValueContext selected) { } if (expression instanceof HogQLParser.ColumnExprValuePassthroughContext passthrough && passthrough.columnExprValue() instanceof HogQLParser.ColumnExprAsteriskContext asterisk) { - if (alias.isPresent() || asterisk.tableIdentifier() != null || asterisk.EXCLUDE() != null) { - throw unsupported(asterisk, "qualified or transformed star"); + if (alias.isPresent()) { + throw unsupported(asterisk, "aliased star"); } - return new Star(sourceSpan(context)); + List qualifier = asterisk.tableIdentifier() == null ? List.of() : buildIdentifiers(asterisk.tableIdentifier()); + List exclusions = asterisk.identifierList() == null ? List.of() : asterisk.identifierList().nestedIdentifier().stream() + .map(identifier -> new ColumnReference(buildIdentifiers(identifier), sourceSpan(identifier))) + .toList(); + return new Star(qualifier, exclusions, sourceSpan(asterisk)); } return new ExpressionProjection(buildExpression(expression), alias); } diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index f0a47ab58cb3..a354fc1bb61f 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -222,13 +222,20 @@ public sealed interface Projection SourceSpan span(); } - public record Star(SourceSpan span) + public record Star(List qualifier, List exclusions, SourceSpan span) implements Projection { public Star { + qualifier = List.copyOf(requireNonNull(qualifier, "qualifier is null")); + exclusions = List.copyOf(requireNonNull(exclusions, "exclusions is null")); span = requireNonNull(span, "span is null"); } + + public Star(SourceSpan span) + { + this(List.of(), List.of(), span); + } } public record ExpressionProjection(Expression expression, Optional alias) diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 818fd3fddb83..3ccf1b18c116 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -29,6 +29,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; import io.trino.hogql.parser.tree.HogQlQuery.SetOperationType; import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; +import io.trino.hogql.parser.tree.HogQlQuery.Star; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; @@ -120,6 +121,33 @@ public void testPreservesNamedPlaceholderNamesAndSourceSpans() assertThat(repeated.span()).isEqualTo(new HogQlQuery.SourceSpan(27, 34, 2, 12, 2, 19)); } + @Test + public void testBuildsQualifiedStarWithExclusionsAndSourceSpans() + { + HogQlQuery query = parser.parseStatement("SELECT \"Event Alias\".* EXCLUDE (\"event\", personId) FROM events AS \"Event Alias\""); + + Star star = (Star) query.projections().getFirst(); + assertThat(star.qualifier()).singleElement().satisfies(qualifier -> { + assertThat(qualifier.value()).isEqualTo("Event Alias"); + assertThat(qualifier.delimited()).isTrue(); + assertThat(qualifier.span()).isEqualTo(new HogQlQuery.SourceSpan(7, 20, 1, 8, 1, 21)); + }); + assertThat(star.exclusions()).extracting(exclusion -> exclusion.parts().getFirst().value()).containsExactly("event", "personId"); + assertThat(star.exclusions()).extracting(exclusion -> exclusion.parts().getFirst().delimited()).containsExactly(true, false); + assertThat(star.exclusions().getFirst().span()).isEqualTo(new HogQlQuery.SourceSpan(32, 39, 1, 33, 1, 40)); + assertThat(star.exclusions().get(1).span()).isEqualTo(new HogQlQuery.SourceSpan(41, 49, 1, 42, 1, 50)); + assertThat(star.span()).isEqualTo(new HogQlQuery.SourceSpan(7, 50, 1, 8, 1, 51)); + } + + @Test + public void testPreservesQualifiedStarExclusionPath() + { + Star star = (Star) parser.parseStatement("SELECT * EXCLUDE (analytics.events.event) FROM events").projections().getFirst(); + + assertThat(star.exclusions()).singleElement().satisfies(exclusion -> + assertThat(exclusion.parts()).extracting(HogQlQuery.Identifier::value).containsExactly("analytics", "events", "event")); + } + @Test public void testBuildsDistinctOrderingAndPaginationAst() { From 08446d5c73e9c3bc47a2557821737eff17e8d4b5 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 15:17:10 -0400 Subject: [PATCH 060/136] feat(hogql): expose physical catalog metadata --- .../HogQlPhysicalCatalogResource.java | 99 +++++++ .../io/trino/hogql/HogQlPhysicalCatalog.java | 94 +++++++ .../hogql/HogQlPhysicalCatalogProvider.java | 263 ++++++++++++++++++ .../io/trino/server/CoordinatorModule.java | 4 + .../TestHogQlStatementResource.java | 162 ++++++++++- docs/src/main/sphinx/admin/hogql.md | 38 +++ 6 files changed, 658 insertions(+), 2 deletions(-) create mode 100644 core/trino-main/src/main/java/io/trino/dispatcher/HogQlPhysicalCatalogResource.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlPhysicalCatalog.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlPhysicalCatalogProvider.java diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlPhysicalCatalogResource.java b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlPhysicalCatalogResource.java new file mode 100644 index 000000000000..03b3e21e40eb --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlPhysicalCatalogResource.java @@ -0,0 +1,99 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.dispatcher; + +import com.google.inject.Inject; +import io.opentelemetry.api.trace.Span; +import io.trino.Session; +import io.trino.execution.QueryIdGenerator; +import io.trino.hogql.HogQlPhysicalCatalog; +import io.trino.hogql.HogQlPhysicalCatalogProvider; +import io.trino.server.HttpRequestSessionContextFactory; +import io.trino.server.SessionContext; +import io.trino.server.SessionSupplier; +import io.trino.server.security.InternalPrincipal; +import io.trino.server.security.ResourceSecurity; +import io.trino.spi.security.Identity; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.ForbiddenException; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.HttpHeaders; + +import java.util.Optional; + +import static io.trino.server.ServletSecurityUtils.authenticatedIdentity; +import static io.trino.server.security.ResourceSecurity.AccessType.AUTHENTICATED_USER; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +@Path("/v1/hogql/compatibility/physical-catalog") +public class HogQlPhysicalCatalogResource +{ + private final HttpRequestSessionContextFactory sessionContextFactory; + private final SessionSupplier sessionSupplier; + private final QueryIdGenerator queryIdGenerator; + private final HogQlPhysicalCatalogProvider physicalCatalogProvider; + + @Inject + public HogQlPhysicalCatalogResource( + HttpRequestSessionContextFactory sessionContextFactory, + SessionSupplier sessionSupplier, + QueryIdGenerator queryIdGenerator, + HogQlPhysicalCatalogProvider physicalCatalogProvider) + { + this.sessionContextFactory = requireNonNull(sessionContextFactory, "sessionContextFactory is null"); + this.sessionSupplier = requireNonNull(sessionSupplier, "sessionSupplier is null"); + this.queryIdGenerator = requireNonNull(queryIdGenerator, "queryIdGenerator is null"); + this.physicalCatalogProvider = requireNonNull(physicalCatalogProvider, "physicalCatalogProvider is null"); + } + + @ResourceSecurity(AUTHENTICATED_USER) + @GET + @Produces(APPLICATION_JSON) + public HogQlPhysicalCatalog getPhysicalCatalog( + @QueryParam("catalog") String catalog, + @QueryParam("protocolVersion") Integer protocolVersion, + @Context HttpServletRequest servletRequest, + @Context HttpHeaders httpHeaders) + { + if (catalog == null || catalog.isBlank()) { + throw new BadRequestException("Catalog is required"); + } + if (protocolVersion == null || protocolVersion != HogQlPhysicalCatalog.PROTOCOL_VERSION) { + throw new BadRequestException("Unsupported physical catalog protocol version"); + } + catalog = catalog.toLowerCase(ENGLISH); + + Optional identity = authenticatedIdentity(servletRequest); + if (identity.flatMap(Identity::getPrincipal).map(InternalPrincipal.class::isInstance).orElse(false)) { + throw new ForbiddenException("Internal communication can not be used to read a physical catalog"); + } + + SessionContext sessionContext = sessionContextFactory.createSessionContext( + httpHeaders.getRequestHeaders(), + Optional.ofNullable(servletRequest.getRemoteAddr()), + identity); + if (sessionContext.getTransactionId().isPresent()) { + throw new BadRequestException("A transaction ID is not supported for physical catalog requests"); + } + Session session = sessionSupplier.createSession(queryIdGenerator.createNextQueryId(), Span.getInvalid(), sessionContext); + return physicalCatalogProvider.load(session, catalog); + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlPhysicalCatalog.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlPhysicalCatalog.java new file mode 100644 index 000000000000..87e61acbde63 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlPhysicalCatalog.java @@ -0,0 +1,94 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; + +import static java.util.Objects.requireNonNull; + +public record HogQlPhysicalCatalog( + int protocolVersion, + int schemaVersion, + Identifier catalog, + String catalogHandleVersion, + List
tables) +{ + public static final int PROTOCOL_VERSION = 1; + public static final int SCHEMA_VERSION = 1; + + private static final Pattern UNDELIMITED_IDENTIFIER = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*"); + + public HogQlPhysicalCatalog + { + if (protocolVersion != PROTOCOL_VERSION) { + throw new IllegalArgumentException("unsupported protocolVersion"); + } + if (schemaVersion != SCHEMA_VERSION) { + throw new IllegalArgumentException("unsupported schemaVersion"); + } + requireNonNull(catalog, "catalog is null"); + requireNonNull(catalogHandleVersion, "catalogHandleVersion is null"); + tables = ImmutableList.copyOf(requireNonNull(tables, "tables is null")); + } + + public static Identifier identifier(String value) + { + requireNonNull(value, "value is null"); + boolean delimited = !UNDELIMITED_IDENTIFIER.matcher(value).matches() || !value.equals(value.toLowerCase(Locale.ENGLISH)); + return new Identifier(value, delimited); + } + + public record Identifier(String value, boolean delimited) + { + public Identifier + { + requireNonNull(value, "value is null"); + } + } + + public record Table(Identifier schema, Identifier table, List columns) + { + public Table + { + requireNonNull(schema, "schema is null"); + requireNonNull(table, "table is null"); + columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); + } + } + + public record Column( + Identifier name, + int ordinal, + String type, + boolean nullable, + boolean hidden, + boolean starVisible) + { + public Column + { + requireNonNull(name, "name is null"); + requireNonNull(type, "type is null"); + if (ordinal < 1) { + throw new IllegalArgumentException("ordinal must be positive"); + } + if (hidden == starVisible) { + throw new IllegalArgumentException("hidden and starVisible must be opposites"); + } + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlPhysicalCatalogProvider.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlPhysicalCatalogProvider.java new file mode 100644 index 000000000000..21dc6b2f014a --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlPhysicalCatalogProvider.java @@ -0,0 +1,263 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import io.trino.Session; +import io.trino.connector.CatalogHandle; +import io.trino.hogql.HogQlPhysicalCatalog.Column; +import io.trino.hogql.HogQlPhysicalCatalog.Table; +import io.trino.metadata.Catalog; +import io.trino.metadata.CatalogManager; +import io.trino.metadata.Metadata; +import io.trino.metadata.QualifiedTablePrefix; +import io.trino.security.AccessControl; +import io.trino.spi.catalog.CatalogName; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.connector.TableColumnsMetadata; +import io.trino.transaction.TransactionId; +import io.trino.transaction.TransactionManager; +import jakarta.ws.rs.NotFoundException; +import jakarta.ws.rs.WebApplicationException; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static io.airlift.concurrent.MoreFutures.getFutureValue; +import static io.trino.hogql.HogQlPhysicalCatalog.identifier; +import static io.trino.metadata.MetadataListing.handleListingException; +import static io.trino.spi.transaction.IsolationLevel.READ_UNCOMMITTED; +import static jakarta.ws.rs.core.Response.Status.CONFLICT; +import static jakarta.ws.rs.core.Response.Status.REQUEST_ENTITY_TOO_LARGE; +import static java.util.Objects.requireNonNull; + +public class HogQlPhysicalCatalogProvider +{ + static final int MAX_TABLES = 10_000; + static final int MAX_COLUMNS = 100_000; + static final int MAX_COLUMNS_PER_TABLE = 10_000; + static final int MAX_VALUE_CHARACTERS = 8 * 1024 * 1024; + + private final Metadata metadata; + private final AccessControl accessControl; + private final TransactionManager transactionManager; + private final CatalogManager catalogManager; + + @Inject + public HogQlPhysicalCatalogProvider( + Metadata metadata, + AccessControl accessControl, + TransactionManager transactionManager, + CatalogManager catalogManager) + { + this.metadata = requireNonNull(metadata, "metadata is null"); + this.accessControl = requireNonNull(accessControl, "accessControl is null"); + this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); + this.catalogManager = requireNonNull(catalogManager, "catalogManager is null"); + } + + public HogQlPhysicalCatalog load(Session session, String catalogName) + { + requireNonNull(session, "session is null"); + CatalogName requestedCatalog = new CatalogName(requireNonNull(catalogName, "catalogName is null")); + CatalogHandle expectedHandle = currentCatalogHandle(requestedCatalog); + + TransactionId transactionId = transactionManager.beginTransaction(READ_UNCOMMITTED, true, true); + boolean success = false; + try { + Session transactionSession = session.beginTransactionId(transactionId, transactionManager, accessControl); + metadata.beginQuery(transactionSession); + try { + HogQlPhysicalCatalog result = load(transactionSession, catalogName, requestedCatalog, expectedHandle); + success = true; + return result; + } + finally { + metadata.cleanupQuery(transactionSession); + } + } + finally { + if (transactionManager.transactionExists(transactionId)) { + if (success) { + getFutureValue(transactionManager.asyncCommit(transactionId)); + } + else { + getFutureValue(transactionManager.asyncAbort(transactionId)); + } + } + } + } + + private HogQlPhysicalCatalog load( + Session transactionSession, + String catalogName, + CatalogName requestedCatalog, + CatalogHandle expectedHandle) + { + CatalogHandle transactionHandle = metadata.getCatalogHandle(transactionSession, catalogName) + .orElseThrow(NotFoundException::new); + verifyCatalogUnchanged(requestedCatalog, expectedHandle, transactionHandle); + verifyCatalogVisible(transactionSession, catalogName); + + Map> tableColumns = listPhysicalTableColumns(transactionSession, catalogName); + HogQlPhysicalCatalog result = createCatalog(catalogName, expectedHandle, tableColumns); + verifyCatalogUnchanged(requestedCatalog, expectedHandle, currentCatalogHandle(requestedCatalog)); + return result; + } + + private void verifyCatalogVisible(Session session, String catalogName) + { + Set visibleCatalogs = accessControl.filterCatalogs(session.toSecurityContext(), Set.of(catalogName)); + if (!visibleCatalogs.contains(catalogName)) { + throw new NotFoundException(); + } + } + + private CatalogHandle currentCatalogHandle(CatalogName catalogName) + { + return catalogManager.getCatalog(catalogName) + .filter(catalog -> !catalog.isFailed()) + .map(Catalog::getCatalogHandle) + .orElseThrow(NotFoundException::new); + } + + private static void verifyCatalogUnchanged(CatalogName catalogName, CatalogHandle expected, CatalogHandle actual) + { + if (!expected.equals(actual)) { + throw new WebApplicationException("Catalog '%s' changed while its physical inventory was read".formatted(catalogName), CONFLICT); + } + } + + private Map> listPhysicalTableColumns(Session session, String catalogName) + { + QualifiedTablePrefix prefix = new QualifiedTablePrefix(catalogName); + AtomicInteger filteredCount = new AtomicInteger(); + List tables; + try { + tables = metadata.listTableColumns( + session, + prefix, + relationNames -> { + Set filtered = accessControl.filterTables(session.toSecurityContext(), catalogName, relationNames); + filteredCount.addAndGet(filtered.size()); + return filtered; + }); + } + catch (RuntimeException exception) { + throw handleListingException(exception, "table columns", catalogName); + } + checkState(filteredCount.get() >= tables.size(), "relation filter was not applied to every returned relation"); + + Map> columnsByTable = tables.stream() + .filter(table -> !table.getTable().getSchemaName().equals("information_schema")) + .collect(toImmutableMap( + TableColumnsMetadata::getTable, + table -> table.getColumns().orElseThrow(() -> new WebApplicationException( + "Redirected tables are not supported by the physical catalog endpoint", + CONFLICT)))); + Map> allowedColumns = accessControl.filterColumns( + session.toSecurityContext(), + catalogName, + columnsByTable.entrySet().stream() + .collect(toImmutableMap( + Map.Entry::getKey, + entry -> entry.getValue().stream() + .map(ColumnMetadata::getName) + .collect(toImmutableSet())))); + + Map> result = new HashMap<>(); + columnsByTable.forEach((table, columns) -> { + Set visible = allowedColumns.getOrDefault(table, Set.of()); + ImmutableList.Builder physicalColumns = ImmutableList.builder(); + for (int index = 0; index < columns.size(); index++) { + ColumnMetadata column = columns.get(index); + if (visible.contains(column.getName())) { + physicalColumns.add(new PhysicalColumn(column, index + 1)); + } + } + result.put(table, physicalColumns.build()); + }); + return Map.copyOf(result); + } + + private static HogQlPhysicalCatalog createCatalog( + String catalogName, + CatalogHandle catalogHandle, + Map> tableColumns) + { + if (tableColumns.size() > MAX_TABLES) { + throw tooLarge(); + } + + ImmutableList.Builder
tables = ImmutableList.builderWithExpectedSize(tableColumns.size()); + int totalColumns = 0; + int valueCharacters = catalogName.length() + catalogHandle.getVersion().toString().length(); + + List>> sortedTables = tableColumns.entrySet().stream() + .sorted(Map.Entry.comparingByKey(Comparator.comparing(SchemaTableName::getSchemaName) + .thenComparing(SchemaTableName::getTableName))) + .toList(); + for (Map.Entry> entry : sortedTables) { + SchemaTableName tableName = entry.getKey(); + List columnMetadata = entry.getValue(); + if (columnMetadata.size() > MAX_COLUMNS_PER_TABLE || totalColumns + columnMetadata.size() > MAX_COLUMNS) { + throw tooLarge(); + } + + ImmutableList.Builder columns = ImmutableList.builderWithExpectedSize(columnMetadata.size()); + for (int index = 0; index < columnMetadata.size(); index++) { + PhysicalColumn physicalColumn = columnMetadata.get(index); + ColumnMetadata column = physicalColumn.metadata(); + String typeSignature = column.getType().getTypeId().getId(); + valueCharacters += column.getName().length() + typeSignature.length(); + columns.add(new Column( + identifier(column.getName()), + physicalColumn.ordinal(), + typeSignature, + column.isNullable(), + column.isHidden(), + !column.isHidden())); + } + valueCharacters += tableName.getSchemaName().length() + tableName.getTableName().length(); + if (valueCharacters > MAX_VALUE_CHARACTERS) { + throw tooLarge(); + } + totalColumns += columnMetadata.size(); + tables.add(new Table(identifier(tableName.getSchemaName()), identifier(tableName.getTableName()), columns.build())); + } + + return new HogQlPhysicalCatalog( + HogQlPhysicalCatalog.PROTOCOL_VERSION, + HogQlPhysicalCatalog.SCHEMA_VERSION, + identifier(catalogName), + catalogHandle.getVersion().toString(), + tables.build()); + } + + private static WebApplicationException tooLarge() + { + return new WebApplicationException("Physical catalog inventory exceeds the compatibility endpoint limit", REQUEST_ENTITY_TOO_LARGE); + } + + private record PhysicalColumn(ColumnMetadata metadata, int ordinal) {} +} diff --git a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java index 1ce8e3213cf4..89e8bf635857 100644 --- a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java +++ b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java @@ -41,6 +41,7 @@ import io.trino.dispatcher.DispatchManager; import io.trino.dispatcher.DispatchQueryFactory; import io.trino.dispatcher.FailedDispatchQueryFactory; +import io.trino.dispatcher.HogQlPhysicalCatalogResource; import io.trino.dispatcher.HogQlStatementResource; import io.trino.dispatcher.LocalDispatchQueryFactory; import io.trino.dispatcher.QueuedStatementResource; @@ -102,6 +103,7 @@ import io.trino.hogql.HogQlCompilationObserver; import io.trino.hogql.HogQlCompilationStats; import io.trino.hogql.HogQlConfig; +import io.trino.hogql.HogQlPhysicalCatalogProvider; import io.trino.hogql.HogQlSemanticCatalogConfig; import io.trino.hogql.HogQlSemanticCatalogModule; import io.trino.hogql.compiler.HogQlCompiler; @@ -187,6 +189,7 @@ protected void setup(Binder binder) configBinder(binder).bindConfig(HogQlConfig.class); configBinder(binder).bindConfig(HogQlSemanticCatalogConfig.class); binder.bind(HogQlCompiler.class).in(Scopes.SINGLETON); + binder.bind(HogQlPhysicalCatalogProvider.class).in(Scopes.SINGLETON); binder.bind(HogQlCompilationExecutor.class).in(Scopes.SINGLETON); binder.bind(HogQlCompilationStats.class).in(Scopes.SINGLETON); binder.bind(HogQlCompilationObserver.class).to(HogQlCompilationStats.class); @@ -194,6 +197,7 @@ protected void setup(Binder binder) newExporter(binder).export(HogQlCompilationStats.class).withGeneratedName(); if (buildConfigObject(HogQlConfig.class).isEnabled()) { jaxrsBinder(binder).bind(HogQlStatementResource.class); + jaxrsBinder(binder).bind(HogQlPhysicalCatalogResource.class); if (buildConfigObject(HogQlSemanticCatalogConfig.class).getUri() != null) { install(new HogQlSemanticCatalogModule()); } diff --git a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java index c4c247d522be..3851451a6a20 100644 --- a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java +++ b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java @@ -26,12 +26,20 @@ import io.trino.client.QueryDataJacksonModule; import io.trino.client.QueryResults; import io.trino.client.ResultRowsDecoder; +import io.trino.connector.MockConnectorFactory; +import io.trino.connector.MockConnectorPlugin; +import io.trino.hogql.HogQlPhysicalCatalog; import io.trino.hogql.parser.HogQlLanguageContract; import io.trino.plugin.tpch.TpchPlugin; import io.trino.server.testing.TestingTrinoServer; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.RelationColumnsMetadata; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.type.ArrayType; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -51,12 +59,18 @@ import static io.airlift.http.client.StringResponseHandler.createStringResponseHandler; import static io.airlift.testing.Closeables.closeAll; import static io.trino.client.ProtocolHeaders.TRINO_HEADERS; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.VarcharType.createVarcharType; +import static io.trino.testing.TestingAccessControlManager.TestingPrivilegeType.SELECT_COLUMN; +import static io.trino.testing.TestingAccessControlManager.privilege; import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.stream.Collectors.joining; import static java.util.stream.IntStream.range; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; +@Execution(SAME_THREAD) class TestHogQlStatementResource { private static final HeaderName REQUEST_USER_HEADER = HeaderName.of(TRINO_HEADERS.requestUser()); @@ -67,6 +81,7 @@ class TestHogQlStatementResource .get()) .jsonCodec(QueryResults.class); private static final JsonCodec STRING_CODEC = JsonCodec.jsonCodec(String.class); + private static final JsonCodec PHYSICAL_CATALOG_CODEC = JsonCodec.jsonCodec(HogQlPhysicalCatalog.class); private static final String SECRET = "do-not-echo-this-value"; private static HttpClient client; @@ -78,10 +93,40 @@ static void setUp() { client = new JettyHttpClient(); server = TestingTrinoServer.builder() - .setProperties(Map.of("hogql.enabled", "true")) + .setProperties(Map.of( + "hogql.enabled", "true", + "sql.default-catalog", "tpch")) .build(); server.installPlugin(new TpchPlugin()); server.createCatalog("tpch", "tpch"); + SchemaTableName physicalTable = new SchemaTableName("analytics", "events"); + server.installPlugin(new MockConnectorPlugin(MockConnectorFactory.builder() + .withName("physical_metadata_connector") + .withListSchemaNames(_ -> ImmutableList.of(physicalTable.getSchemaName())) + .withStreamRelationColumns((_, schema, relationFilter) -> { + if (schema.isPresent() && !schema.orElseThrow().equals(physicalTable.getSchemaName())) { + return ImmutableList.of().iterator(); + } + if (!relationFilter.apply(Set.of(physicalTable)).contains(physicalTable)) { + return ImmutableList.of().iterator(); + } + return ImmutableList.of(RelationColumnsMetadata.forTable( + physicalTable, + ImmutableList.of( + ColumnMetadata.builder() + .setName("event_id") + .setType(BIGINT) + .setNullable(false) + .build(), + ColumnMetadata.builder() + .setName("tags") + .setType(new ArrayType(createVarcharType(7))) + .setHidden(true) + .build()))) + .iterator(); + }) + .build())); + server.createCatalog("physical_metadata", "physical_metadata_connector"); } @AfterAll @@ -108,6 +153,14 @@ public void testEndpointIsDisabledByDefault() createStatusResponseHandler()); assertThat(response.getStatusCode()).isEqualTo(404); + + StatusResponse physicalCatalogResponse = disabledClient.execute( + prepareGet() + .setUri(disabledServer.resolve("/v1/hogql/compatibility/physical-catalog?catalog=missing&protocolVersion=1")) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createStatusResponseHandler()); + assertThat(physicalCatalogResponse.getStatusCode()).isEqualTo(404); } finally { closeAll(disabledServer, disabledClient); @@ -129,6 +182,111 @@ public void testEnabledEndpointUsesStatementProtocol() assertThat(rows(sqlResults)).containsExactly(ImmutableList.of(2)); } + @Test + public void testPhysicalCatalogCompatibilityEndpoint() + { + HogQlPhysicalCatalog catalog = client.execute( + prepareGet() + .setUri(server.resolve("/v1/hogql/compatibility/physical-catalog?catalog=physical_metadata&protocolVersion=1")) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createJsonResponseHandler(PHYSICAL_CATALOG_CODEC)); + + assertThat(catalog.protocolVersion()).isEqualTo(1); + assertThat(catalog.schemaVersion()).isEqualTo(1); + assertThat(catalog.catalog()).isEqualTo(new HogQlPhysicalCatalog.Identifier("physical_metadata", false)); + assertThat(catalog.catalogHandleVersion()).isNotBlank(); + assertThat(catalog.tables()).singleElement().satisfies(table -> { + assertThat(table.schema()).isEqualTo(new HogQlPhysicalCatalog.Identifier("analytics", false)); + assertThat(table.table()).isEqualTo(new HogQlPhysicalCatalog.Identifier("events", false)); + assertThat(table.columns()).containsExactly( + new HogQlPhysicalCatalog.Column( + new HogQlPhysicalCatalog.Identifier("event_id", false), + 1, + "bigint", + false, + false, + true), + new HogQlPhysicalCatalog.Column( + new HogQlPhysicalCatalog.Identifier("tags", false), + 2, + "array(varchar(7))", + true, + true, + false)); + }); + } + + @Test + public void testPhysicalCatalogCompatibilityEndpointFailsClosed() + { + StatusResponse missingCatalog = client.execute( + prepareGet() + .setUri(server.resolve("/v1/hogql/compatibility/physical-catalog")) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createStatusResponseHandler()); + assertThat(missingCatalog.getStatusCode()).isEqualTo(400); + + StatusResponse missingProtocolVersion = client.execute( + prepareGet() + .setUri(server.resolve("/v1/hogql/compatibility/physical-catalog?catalog=physical_metadata")) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createStatusResponseHandler()); + assertThat(missingProtocolVersion.getStatusCode()).isEqualTo(400); + + StatusResponse unsupportedProtocolVersion = client.execute( + prepareGet() + .setUri(server.resolve("/v1/hogql/compatibility/physical-catalog?catalog=physical_metadata&protocolVersion=2")) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createStatusResponseHandler()); + assertThat(unsupportedProtocolVersion.getStatusCode()).isEqualTo(400); + + StatusResponse unknownCatalog = client.execute( + prepareGet() + .setUri(server.resolve("/v1/hogql/compatibility/physical-catalog?catalog=missing&protocolVersion=1")) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createStatusResponseHandler()); + assertThat(unknownCatalog.getStatusCode()).isEqualTo(404); + } + + @Test + public void testPhysicalCatalogCompatibilityEndpointRequiresAuthentication() + { + StatusResponse response = client.execute( + prepareGet() + .setUri(server.resolve("/v1/hogql/compatibility/physical-catalog?catalog=physical_metadata&protocolVersion=1")) + .build(), + createStatusResponseHandler()); + + assertThat(response.getStatusCode()).isEqualTo(401); + } + + @Test + public void testPhysicalCatalogCompatibilityEndpointAppliesVisibilityFilters() + { + server.getAccessControl().deny(privilege("events.event_id", SELECT_COLUMN)); + try { + HogQlPhysicalCatalog catalog = client.execute( + prepareGet() + .setUri(server.resolve("/v1/hogql/compatibility/physical-catalog?catalog=physical_metadata&protocolVersion=1")) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createJsonResponseHandler(PHYSICAL_CATALOG_CODEC)); + + assertThat(catalog.tables()).singleElement().satisfies(table -> { + assertThat(table.columns()).extracting(column -> column.name().value()).containsExactly("tags"); + assertThat(table.columns()).extracting(HogQlPhysicalCatalog.Column::ordinal).containsExactly(2); + }); + } + finally { + server.getAccessControl().reset(); + } + } + @Test public void testEmptyResultsAndCompilerDiagnostics() throws Exception @@ -356,7 +514,7 @@ private static String hogQlRequestWithTypedValues(String query) "parameters": {}, "variables": {"array": {"type": "array", "value": [true, "value", null]}}, "filters": {"object": {"type": "object", "value": {"nested": 2.5}}}, - "modifiers": {"missing": {"type": "nullable", "value": null}}, + "modifiers": {}, "catalogGeneration": 1 } """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); diff --git a/docs/src/main/sphinx/admin/hogql.md b/docs/src/main/sphinx/admin/hogql.md index 20894be91845..696de638b2fb 100644 --- a/docs/src/main/sphinx/admin/hogql.md +++ b/docs/src/main/sphinx/admin/hogql.md @@ -106,11 +106,49 @@ The compatibility request path is selected Trino catalog, HogQL language version, and optional requested generation. Manifests contain typed identifiers and definitions, not SQL text. +Qualified stars on relations without a manifest schema compile to standard +Trino `relation.*`. `EXCLUDE` needs the manifest's ordered, star-visible field +list, and fails with a source-located compatibility error when that schema is +unavailable. Duckgres should publish physical tables as physical-derived +semantic definitions from the physical catalog inventory when those tables +need HogQL star exclusion. Schema inference for CTE and subquery outputs is not +currently supported. + The Trino transport does not currently send an application authentication credential. Duckgres requires `X-Duckgres-Internal-Secret` by default, so the production connection remains incomplete until the coordinator credential mechanism is configured. +## Physical catalog inventory + +An authenticated client can read connector-facing table and column metadata +from `GET /v1/hogql/compatibility/physical-catalog?catalog=&protocolVersion=1`. +The endpoint is available only on coordinators when `hogql.enabled=true`. + +The versioned response contains `protocolVersion` and `schemaVersion` before +the structured catalog, schema, table, and column identifiers. Each column +includes its one-based ordinal, exact Trino type signature, nullability, hidden +state, and star visibility. Hidden columns are reported when the caller can see +them, but remain excluded from star expansion. The ordinal is the connector's +original column position. Column visibility filtering can therefore leave gaps +in the returned ordinals. + +The `catalogHandleVersion` identifies the active Trino catalog registration. +It detects a catalog replacement during the request, but it is not the +connector's metadata snapshot or a semantic catalog generation. Duckgres +assigns its own monotonically increasing generation when it publishes the +inventory. + +The endpoint applies the same catalog, table, and column visibility filters as +Trino metadata listings. It reads one connector transaction, so DuckLake +metadata is pinned to one snapshot. The request fails if the catalog is dropped +or replaced while the inventory is read. Results are limited to 10,000 tables, +10,000 columns per table, 100,000 columns in total, and 8 million characters of +identifier and type text. + +Use normal Trino HTTP authentication and request headers. The endpoint does not +accept a separate credential or an existing transaction ID. + ## Semantic catalog properties | Property | Default | Description | From 113370d9c95eec7b52499ffdd5e05df1e476a5b6 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 15:29:02 -0400 Subject: [PATCH 061/136] feat(hogql): expand advanced column selectors --- core/trino-hogql-compiler/pom.xml | 10 ++ .../trino/hogql/compiler/HogQlCompiler.java | 26 +++-- .../hogql/compiler/HogQlFunctionResolver.java | 13 ++- .../hogql/compiler/HogQlSemanticResolver.java | 94 +++++++++++++++++-- .../trino/hogql/compiler/TrinoAstFactory.java | 31 +++++- .../compiler/TestHogQlSemanticResolution.java | 87 +++++++++++++++++ .../io/trino/hogql/parser/HogQlParser.java | 93 ++++++++++++++++-- .../trino/hogql/parser/tree/HogQlQuery.java | 46 ++++++++- .../trino/hogql/parser/TestHogQlParser.java | 27 ++++++ 9 files changed, 396 insertions(+), 31 deletions(-) diff --git a/core/trino-hogql-compiler/pom.xml b/core/trino-hogql-compiler/pom.xml index 711f62ad8eee..d4ae8742ebfa 100644 --- a/core/trino-hogql-compiler/pom.xml +++ b/core/trino-hogql-compiler/pom.xml @@ -24,6 +24,11 @@ jackson-databind + + io.airlift + slice + + io.trino trino-hogql-parser @@ -34,6 +39,11 @@ trino-parser + + io.trino + trino-re2j + + io.trino trino-spi diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index c40117f1e4bc..f1db4beb41b1 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -29,6 +29,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.Expression; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; @@ -42,10 +44,12 @@ import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.hogql.parser.tree.HogQlQuery.Star; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; @@ -197,8 +201,11 @@ private static void collectPlaceholders(HogQlQuery query, List plac switch (query.body()) { case SelectQueryBody select -> { select.projections().forEach(projection -> { - if (projection instanceof HogQlQuery.ExpressionProjection expressionProjection) { - collectPlaceholders(expressionProjection.expression(), placeholders); + switch (projection) { + case ColumnsList columns -> columns.expressions().forEach(expression -> collectPlaceholders(expression, placeholders)); + case ColumnsRegex _ -> {} + case ExpressionProjection expression -> collectPlaceholders(expression.expression(), placeholders); + case Star star -> star.replacements().forEach(replacement -> collectPlaceholders(replacement.expression(), placeholders)); } }); select.from().ifPresent(relation -> collectPlaceholders(relation, placeholders)); @@ -276,10 +283,7 @@ private static boolean containsFunctionCall(HogQlQuery query) { return query.with().stream().anyMatch(commonTable -> containsFunctionCall(commonTable.query())) || switch (query.body()) { - case SelectQueryBody select -> select.projections().stream() - .filter(ExpressionProjection.class::isInstance) - .map(ExpressionProjection.class::cast) - .anyMatch(projection -> containsFunctionCall(projection.expression())) || + case SelectQueryBody select -> select.projections().stream().anyMatch(HogQlCompiler::containsFunctionCall) || select.from().map(HogQlCompiler::containsFunctionCall).orElse(false) || select.where().map(HogQlCompiler::containsFunctionCall).orElse(false) || select.groupBy().stream().anyMatch(HogQlCompiler::containsFunctionCall) || @@ -312,6 +316,16 @@ private static boolean containsFunctionCall(Relation relation) }; } + private static boolean containsFunctionCall(Projection projection) + { + return switch (projection) { + case ColumnsList columns -> columns.expressions().stream().anyMatch(HogQlCompiler::containsFunctionCall); + case ColumnsRegex _ -> false; + case ExpressionProjection expression -> containsFunctionCall(expression.expression()); + case Star star -> star.replacements().stream().anyMatch(replacement -> containsFunctionCall(replacement.expression())); + }; + } + private static boolean containsFunctionCall(Expression expression) { return switch (expression) { diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 9b0471437cb6..48451ba54dda 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -29,6 +29,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.CaseWhen; import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; import io.trino.hogql.parser.tree.HogQlQuery.CommonTableExpression; import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.Expression; @@ -51,6 +53,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; import io.trino.hogql.parser.tree.HogQlQuery.SortItem; import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.StarReplacement; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; @@ -137,8 +140,16 @@ private HogQlQuery resolveQuery(HogQlQuery query) private Projection resolveProjection(Projection projection) { return switch (projection) { + case ColumnsList columns -> new ColumnsList(columns.expressions().stream().map(this::resolveExpression).toList(), columns.span()); + case ColumnsRegex columns -> columns; case ExpressionProjection expression -> new ExpressionProjection(resolveExpression(expression.expression()), expression.alias()); - case Star star -> star; + case Star star -> new Star( + star.qualifier(), + star.exclusions(), + star.replacements().stream() + .map(replacement -> new StarReplacement(resolveExpression(replacement.expression()), replacement.target(), replacement.span())) + .toList(), + star.span()); }; } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index b1c79bdb6583..675e4b7cf515 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -57,6 +57,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.CaseWhen; import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; import io.trino.hogql.parser.tree.HogQlQuery.CommonTableExpression; import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.Expression; @@ -79,6 +81,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; import io.trino.hogql.parser.tree.HogQlQuery.SortItem; import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.StarReplacement; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; @@ -91,6 +94,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.WindowFrame; import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; +import io.trino.re2j.Pattern; +import io.trino.re2j.PatternSyntaxException; import io.trino.spi.Location; import io.trino.spi.TrinoException; @@ -104,6 +109,7 @@ import java.util.Optional; import java.util.Set; +import static io.airlift.slice.Slices.utf8Slice; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_COMPILER_LIMIT_EXCEEDED; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; @@ -214,6 +220,8 @@ private HogQlQuery resolveQuery(HogQlQuery query, List co private List resolveProjection(Projection projection) { return switch (projection) { + case ColumnsList columns -> resolveColumnsList(columns); + case ColumnsRegex columns -> resolveColumnsRegex(columns); case Star star -> resolveStar(star); case ExpressionProjection expressionProjection -> { Expression resolved = resolveExpression(expressionProjection.expression()); @@ -226,6 +234,40 @@ private List resolveProjection(Projection projection) }; } + private List resolveColumnsList(ColumnsList columns) + { + return columns.expressions().stream() + .flatMap(expression -> resolveProjection(new ExpressionProjection(expression, Optional.empty())).stream()) + .toList(); + } + + private List resolveColumnsRegex(ColumnsRegex columns) + { + if (!allRelationsLogical) { + throw unsupportedColumns(columns.span()); + } + Pattern pattern; + try { + pattern = Pattern.compile(columns.pattern()); + } + catch (PatternSyntaxException _) { + throw semanticEntityError(HOGQL_RESOLUTION_ERROR, columns.patternSpan(), "Invalid HogQL COLUMNS regex: " + columns.pattern()); + } + List projections = bindings.stream() + .flatMap(binding -> binding.orderedFields().stream() + .filter(BoundField::starVisible) + .filter(field -> pattern.find(utf8Slice(field.name()))) + .map(field -> new ExpressionProjection( + resolveBoundField(binding, field, binding.starQualifier(bindings.size()), columns.span(), expansionBudget), + Optional.of(new Identifier(field.name(), true, columns.span()))))) + .map(Projection.class::cast) + .toList(); + if (projections.isEmpty()) { + throw semanticEntityError(HOGQL_RESOLUTION_ERROR, columns.patternSpan(), "No HogQL fields matched COLUMNS regex: " + columns.pattern()); + } + return projections; + } + private List resolveStar(Star star) { List starBindings; @@ -252,22 +294,50 @@ private List resolveStar(Star star) } Set exclusions = resolveStarExclusions(star, starBindings); + Map replacements = resolveStarReplacements(star, starBindings, exclusions); return starBindings.stream() .flatMap(binding -> binding.orderedFields().stream() .filter(BoundField::starVisible) .filter(field -> !exclusions.contains(new StarField(binding, field))) - .map(field -> new ExpressionProjection( - resolveBoundField( - binding, - field, - qualified ? Optional.of(binding.outputQualifier()) : binding.starQualifier(bindings.size()), - star.span(), - expansionBudget), - Optional.of(new Identifier(field.name(), true, star.span()))))) + .map(field -> { + StarReplacement replacement = replacements.get(new StarField(binding, field)); + Expression expression = replacement == null + ? resolveBoundField( + binding, + field, + qualified ? Optional.of(binding.outputQualifier()) : binding.starQualifier(bindings.size()), + star.span(), + expansionBudget) + : resolveExpression(replacement.expression()); + return new ExpressionProjection(expression, Optional.of(new Identifier(field.name(), true, star.span()))); + })) .map(Projection.class::cast) .toList(); } + private static Map resolveStarReplacements(Star star, List starBindings, Set exclusions) + { + Map replacements = new HashMap<>(); + for (StarReplacement replacement : star.replacements()) { + List matchedFields = starBindings.stream() + .flatMap(binding -> binding.orderedFields().stream() + .filter(BoundField::starVisible) + .map(field -> new StarField(binding, field))) + .filter(field -> !exclusions.contains(field)) + .filter(field -> matchesIdentifier(replacement.target(), field.field().name())) + .distinct() + .toList(); + if (matchedFields.isEmpty()) { + throw starResolutionError(replacement.target(), "Unknown HogQL star replacement: " + replacement.target().value()); + } + if (matchedFields.stream().anyMatch(replacements::containsKey)) { + throw starResolutionError(replacement.target(), "Duplicate HogQL star replacement: " + replacement.target().value()); + } + matchedFields.forEach(field -> replacements.put(field, replacement)); + } + return replacements; + } + private static Set resolveStarExclusions(Star star, List starBindings) { Set exclusions = new HashSet<>(); @@ -1539,6 +1609,14 @@ private static TrinoException semanticEntityError(HogQlErrorCode errorCode, HogQ null); } + private static TrinoException unsupportedColumns(HogQlQuery.SourceSpan span) + { + return semanticEntityError( + HOGQL_UNSUPPORTED_FEATURE, + span, + "HogQL COLUMNS requires a logical relation from the semantic catalog"); + } + private static TrinoException limitError(HogQlQuery.SourceSpan span, String message) { return new TrinoException( diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 57c09905a911..64f16dbc9aff 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -21,6 +21,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; import io.trino.hogql.parser.tree.HogQlQuery.CommonTableExpression; import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; @@ -158,7 +160,7 @@ private static QuerySpecification createQuerySpecification( QuerySpecification querySpecification = new QuerySpecification( location, new Select(location, select.distinct(), select.projections().stream() - .map(projection -> createSelectItem(projection, parameterIds)) + .flatMap(projection -> createSelectItems(projection, parameterIds).stream()) .toList()), select.from().map(relation -> createRelation(relation, parameterIds)), select.where().map(expression -> createExpression(expression, parameterIds)), @@ -266,19 +268,33 @@ private static Optional createGroupBy(List expre .toList())))); } - private static SelectItem createSelectItem(Projection projection, Map parameterIds) + private static List createSelectItems(Projection projection, Map parameterIds) { return switch (projection) { - case Star star -> createAllColumns(star); - case ExpressionProjection expression -> new SingleColumn( + case ColumnsList columns -> columns.expressions().stream() + .map(expression -> new SingleColumn( + location(expression.span()), + createExpression(expression, parameterIds), + Optional.empty())) + .map(SelectItem.class::cast) + .toList(); + case ColumnsRegex columns -> throw unsupportedColumns(columns.span()); + case Star star -> List.of(createAllColumns(star)); + case ExpressionProjection expression -> List.of(new SingleColumn( location(expression.span()), createExpression(expression.expression(), parameterIds), - expression.alias().map(TrinoAstFactory::createIdentifier)); + expression.alias().map(TrinoAstFactory::createIdentifier))); }; } private static AllColumns createAllColumns(Star star) { + if (!star.replacements().isEmpty()) { + Identifier target = star.replacements().getFirst().target(); + throw unsupportedSemanticExpression( + target.span(), + "HogQL star replacement requires a logical relation from the semantic catalog: " + target.value()); + } if (!star.exclusions().isEmpty()) { ColumnReference exclusion = star.exclusions().getFirst(); throw unsupportedSemanticExpression( @@ -292,6 +308,11 @@ private static AllColumns createAllColumns(Star star) return new AllColumns(location(star.span()), target, List.of()); } + private static TrinoException unsupportedColumns(SourceSpan span) + { + return unsupportedSemanticExpression(span, "HogQL COLUMNS requires a logical relation from the semantic catalog"); + } + private static Expression createExpression(HogQlQuery.Expression expression, Map parameterIds) { return switch (expression) { diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java index f5016c6ef44c..6a3b2a8c389d 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -191,6 +191,93 @@ public void testMatchesLogicalAndPhysicalQualifiedStarSuffixes() "\"raw-events\".event_name AS \"event\" FROM analytics.\"Hog Data\".\"raw-events\"")); } + @Test + public void testExpandsColumnsSelectorsAndReplacements() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult regex = compiler.compile( + envelope("SELECT COLUMNS('event|person') FROM events", OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult explicit = compiler.compile( + envelope("SELECT COLUMNS(personId, event) FROM events", OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult replaced = compiler.compile( + envelope("SELECT COLUMNS(events.* REPLACE (personId AS event)) FROM events", OptionalLong.of(7)), + Optional.of(context)); + + assertThat(regex.statement()).isEqualTo(sqlParser.createStatement( + "SELECT event_name AS \"event\", \"Person ID\" AS \"personId\" FROM analytics.\"Hog Data\".\"raw-events\"")); + assertThat(explicit.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"Person ID\" AS personId, event_name AS event FROM analytics.\"Hog Data\".\"raw-events\"")); + assertThat(replaced.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"Person ID\" AS \"event\", \"raw-events\".\"Person ID\" AS \"personId\" FROM analytics.\"Hog Data\".\"raw-events\"")); + } + + @Test + public void testRejectsInvalidColumnsSelectorsAndReplacementsAtSource() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + assertResolutionFailure( + context, + "SELECT COLUMNS('^missing$') FROM events", + "'^missing$'", + "No HogQL fields matched COLUMNS regex: ^missing$"); + assertResolutionFailure( + context, + "SELECT COLUMNS('(') FROM events", + "'('", + "Invalid HogQL COLUMNS regex: ("); + assertResolutionFailure( + context, + "SELECT COLUMNS(* REPLACE (personId AS event, event AS \"event\")) FROM events", + "\"event\"", + "Duplicate HogQL star replacement: event"); + assertResolutionFailure( + context, + "SELECT COLUMNS(* REPLACE (personId AS \"Event\")) FROM events", + "\"Event\"", + "Unknown HogQL star replacement: Event"); + } + + @Test + public void testFlattensExplicitColumnsWhenRelationSchemaIsUnavailable() + { + List queries = List.of( + "SELECT COLUMNS(event_name, abs(event_id)) FROM analytics.default.raw_events", + "WITH source AS (SELECT event_name FROM analytics.default.raw_events) SELECT COLUMNS(event_name) FROM source", + "SELECT COLUMNS(source.event_name) FROM (SELECT event_name FROM analytics.default.raw_events) source"); + List expected = List.of( + "SELECT event_name, abs(event_id) FROM analytics.default.raw_events", + "WITH source AS (SELECT event_name FROM analytics.default.raw_events) SELECT event_name FROM source", + "SELECT source.event_name FROM (SELECT event_name FROM analytics.default.raw_events) source"); + + for (int index = 0; index < queries.size(); index++) { + HogQlCompilationResult result = compiler.compile(envelope(queries.get(index), OptionalLong.empty()), Optional.empty()); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement(expected.get(index))); + } + } + + @Test + public void testRejectsSchemaDependentColumnsWhenRelationSchemaIsUnavailable() + { + List queries = List.of( + "SELECT COLUMNS('event') FROM analytics.default.raw_events", + "SELECT COLUMNS(* REPLACE (event_name AS event_name)) FROM analytics.default.raw_events"); + + for (String hogql : queries) { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope(hogql, OptionalLong.empty()), Optional.empty())); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception.getLocation()).isPresent(); + assertThat(exception).hasMessageContaining("requires a logical relation from the semantic catalog"); + } + } + @Test public void testRejectsInvalidLogicalStarQualifierAndExclusionsAtSource() { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 48a087cff926..b9975baa239d 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -25,6 +25,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.CaseWhen; import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; import io.trino.hogql.parser.tree.HogQlQuery.CommonTableExpression; import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.Expression; @@ -56,6 +58,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.SortItem; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.StarReplacement; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; @@ -982,19 +985,93 @@ else if (context instanceof HogQLParser.ColumnExprSelectValueContext selected) { throw unsupported(context, "projection"); } - if (expression instanceof HogQLParser.ColumnExprValuePassthroughContext passthrough && passthrough.columnExprValue() instanceof HogQLParser.ColumnExprAsteriskContext asterisk) { - if (alias.isPresent()) { - throw unsupported(asterisk, "aliased star"); + if (expression instanceof HogQLParser.ColumnExprValuePassthroughContext passthrough) { + Projection columns = buildColumnsProjection(passthrough.columnExprValue(), alias); + if (columns != null) { + return columns; } - List qualifier = asterisk.tableIdentifier() == null ? List.of() : buildIdentifiers(asterisk.tableIdentifier()); - List exclusions = asterisk.identifierList() == null ? List.of() : asterisk.identifierList().nestedIdentifier().stream() - .map(identifier -> new ColumnReference(buildIdentifiers(identifier), sourceSpan(identifier))) - .toList(); - return new Star(qualifier, exclusions, sourceSpan(asterisk)); } return new ExpressionProjection(buildExpression(expression), alias); } + private Projection buildColumnsProjection(HogQLParser.ColumnExprValueContext context, Optional alias) + { + if (context instanceof HogQLParser.ColumnExprAsteriskContext asterisk) { + requireUnaliasedColumnsProjection(alias, asterisk); + List qualifier = asterisk.tableIdentifier() == null ? List.of() : buildIdentifiers(asterisk.tableIdentifier()); + return new Star(qualifier, buildStarExclusions(asterisk.identifierList()), List.of(), sourceSpan(asterisk)); + } + if (context instanceof HogQLParser.ColumnExprColumnsRegexContext regex) { + requireUnaliasedColumnsProjection(alias, regex); + return new ColumnsRegex(decodeQuoted(regex.STRING_LITERAL().getText()), sourceSpan(regex.STRING_LITERAL().getSymbol(), regex.STRING_LITERAL().getSymbol()), sourceSpan(regex)); + } + if (context instanceof HogQLParser.ColumnExprColumnsListContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new ColumnsList(buildExpressions(columns.columnExprList()), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsAllContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star(List.of(), List.of(), List.of(), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsExcludeContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star(List.of(), buildStarExclusions(columns.identifierList()), List.of(), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsReplaceContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star(List.of(), List.of(), buildStarReplacements(columns.columnsReplaceList()), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsExcludeReplaceContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star(List.of(), buildStarExclusions(columns.identifierList()), buildStarReplacements(columns.columnsReplaceList()), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsQualifiedAllContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star(List.of(buildIdentifier(columns.identifier())), List.of(), List.of(), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsQualifiedExcludeContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star(List.of(buildIdentifier(columns.identifier())), buildStarExclusions(columns.identifierList()), List.of(), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsQualifiedReplaceContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star(List.of(buildIdentifier(columns.identifier())), List.of(), buildStarReplacements(columns.columnsReplaceList()), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsQualifiedExcludeReplaceContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star( + List.of(buildIdentifier(columns.identifier())), + buildStarExclusions(columns.identifierList()), + buildStarReplacements(columns.columnsReplaceList()), + sourceSpan(columns)); + } + return null; + } + + private void requireUnaliasedColumnsProjection(Optional alias, ParserRuleContext context) + { + if (alias.isPresent()) { + throw unsupported(context, "aliased columns projection"); + } + } + + private List buildStarExclusions(HogQLParser.IdentifierListContext exclusions) + { + return exclusions == null ? List.of() : exclusions.nestedIdentifier().stream() + .map(identifier -> new ColumnReference(buildIdentifiers(identifier), sourceSpan(identifier))) + .toList(); + } + + private List buildStarReplacements(HogQLParser.ColumnsReplaceListContext replacements) + { + return replacements.columnsReplaceItem().stream() + .map(replacement -> new StarReplacement( + buildExpression(replacement.columnExpr()), + buildIdentifier(replacement.identifier()), + sourceSpan(replacement))) + .toList(); + } + private Expression buildExpression(HogQLParser.ColumnExprContext context) { if (context instanceof HogQLParser.ColumnExprValuePassthroughContext passthrough) { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index a354fc1bb61f..71336bc56fe6 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -217,24 +217,64 @@ public enum NullPlacement } public sealed interface Projection - permits ExpressionProjection, Star + permits ColumnsList, + ColumnsRegex, + ExpressionProjection, + Star { SourceSpan span(); } - public record Star(List qualifier, List exclusions, SourceSpan span) + public record ColumnsRegex(String pattern, SourceSpan patternSpan, SourceSpan span) + implements Projection + { + public ColumnsRegex + { + pattern = requireNonNull(pattern, "pattern is null"); + patternSpan = requireNonNull(patternSpan, "patternSpan is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record ColumnsList(List expressions, SourceSpan span) + implements Projection + { + public ColumnsList + { + expressions = List.copyOf(requireNonNull(expressions, "expressions is null")); + span = requireNonNull(span, "span is null"); + } + } + + public record Star(List qualifier, List exclusions, List replacements, SourceSpan span) implements Projection { public Star { qualifier = List.copyOf(requireNonNull(qualifier, "qualifier is null")); exclusions = List.copyOf(requireNonNull(exclusions, "exclusions is null")); + replacements = List.copyOf(requireNonNull(replacements, "replacements is null")); span = requireNonNull(span, "span is null"); } + public Star(List qualifier, List exclusions, SourceSpan span) + { + this(qualifier, exclusions, List.of(), span); + } + public Star(SourceSpan span) { - this(List.of(), List.of(), span); + this(List.of(), List.of(), List.of(), span); + } + } + + public record StarReplacement(Expression expression, Identifier target, SourceSpan span) + { + public StarReplacement + { + expression = requireNonNull(expression, "expression is null"); + target = requireNonNull(target, "target is null"); + span = requireNonNull(span, "span is null"); } } diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 3ccf1b18c116..0a5bfb4eeb30 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -17,6 +17,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; @@ -30,6 +32,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.SetOperationType; import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.StarReplacement; import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; @@ -148,6 +151,30 @@ public void testPreservesQualifiedStarExclusionPath() assertThat(exclusion.parts()).extracting(HogQlQuery.Identifier::value).containsExactly("analytics", "events", "event")); } + @Test + public void testBuildsColumnsSelectorsAndReplacementAst() + { + HogQlQuery query = parser.parseStatement( + "SELECT COLUMNS('^(event|person)'), COLUMNS(event, personId + 1), " + + "COLUMNS(events.* REPLACE (personId AS event, event AS \"personId\")) FROM events"); + + ColumnsRegex regex = (ColumnsRegex) query.projections().getFirst(); + assertThat(regex.pattern()).isEqualTo("^(event|person)"); + assertThat(regex.patternSpan()).isEqualTo(new HogQlQuery.SourceSpan(15, 32, 1, 16, 1, 33)); + + ColumnsList explicit = (ColumnsList) query.projections().get(1); + assertThat(explicit.expressions()).hasSize(2); + assertThat(explicit.expressions().getFirst()).isInstanceOf(ColumnReference.class); + assertThat(explicit.expressions().get(1)).isInstanceOf(BinaryExpression.class); + + Star star = (Star) query.projections().get(2); + assertThat(star.qualifier()).extracting(HogQlQuery.Identifier::value).containsExactly("events"); + assertThat(star.replacements()).extracting(replacement -> replacement.target().value()).containsExactly("event", "personId"); + assertThat(star.replacements()).extracting(replacement -> replacement.target().delimited()).containsExactly(false, true); + assertThat(star.replacements()).extracting(StarReplacement::expression).allSatisfy(expression -> + assertThat(expression).isInstanceOf(ColumnReference.class)); + } + @Test public void testBuildsDistinctOrderingAndPaginationAst() { From c3efdde3054796a56943aebcfc72a5cf0f9a5ee1 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 15:47:33 -0400 Subject: [PATCH 062/136] feat(hogql): translate cast type syntax --- .../compiler/HogQlCastTypeTranslator.java | 350 ++++++++++++++++++ .../hogql/compiler/HogQlFunctionResolver.java | 2 +- .../hogql/compiler/HogQlSemanticResolver.java | 2 +- .../trino/hogql/compiler/TrinoAstFactory.java | 18 +- .../hogql/compiler/TestHogQlCompiler.java | 86 ++++- .../compiler/TestHogQlSemanticExpansion.java | 15 + .../io/trino/hogql/parser/HogQlParser.java | 15 +- .../trino/hogql/parser/tree/HogQlQuery.java | 14 +- .../trino/hogql/parser/TestHogQlParser.java | 19 + 9 files changed, 508 insertions(+), 13 deletions(-) create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCastTypeTranslator.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCastTypeTranslator.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCastTypeTranslator.java new file mode 100644 index 000000000000..8a56220954b7 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCastTypeTranslator.java @@ -0,0 +1,350 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +import static java.lang.Integer.parseInt; +import static java.util.Objects.requireNonNull; + +final class HogQlCastTypeTranslator +{ + private HogQlCastTypeTranslator() {} + + public static String translate(String type) + { + Translation translation = translateType(requireNonNull(type, "type is null").trim()); + return translation.sql(); + } + + private static Translation translateType(String type) + { + if (type.isEmpty()) { + throw unsupported("empty type"); + } + if (type.endsWith("[]")) { + return composite("array(" + translateType(type.substring(0, type.length() - 2)).sql() + ")"); + } + if (type.matches(".*\\[\\s*\\d+\\s*]$")) { + throw unsupported("fixed-size array syntax"); + } + + Optional call = parseCall(type); + String name = call.map(Call::name).orElse(type).trim().toLowerCase(Locale.ENGLISH); + List arguments = call.map(Call::arguments).orElseGet(List::of); + String suffix = call.map(Call::suffix).orElse("").trim().toLowerCase(Locale.ENGLISH); + + if (name.matches("u?int(8|16|32|64|128|256)")) { + if (name.startsWith("u")) { + throw unsupported("unsigned integer range"); + } + return switch (name) { + case "int8" -> scalar("tinyint"); + case "int16" -> scalar("smallint"); + case "int32" -> scalar("integer"); + case "int64" -> scalar("bigint"); + default -> throw unsupported("integer width exceeds Trino bigint"); + }; + } + if (name.matches("decimal(32|64|128|256)")) { + requireArity(name, arguments, 1); + int precision = switch (name) { + case "decimal32" -> 9; + case "decimal64" -> 18; + case "decimal128" -> 38; + default -> throw unsupported("decimal precision exceeds Trino decimal(38, s)"); + }; + return decimal(precision, parseUnsigned(arguments.getFirst(), "decimal scale")); + } + + return switch (name) { + case "int", "integer" -> noArguments(name, arguments, scalar("integer")); + case "bigint" -> noArguments(name, arguments, scalar("bigint")); + case "smallint" -> noArguments(name, arguments, scalar("smallint")); + case "tinyint" -> noArguments(name, arguments, scalar("tinyint")); + case "float", "real" -> noArguments(name, arguments, scalar("real")); + case "float64", "double", "double precision" -> noArguments(name, arguments, scalar("double")); + case "float32" -> noArguments(name, arguments, scalar("real")); + case "string", "text" -> noArguments(name, arguments, scalar("varchar")); + case "varchar", "char" -> characterType(name, arguments); + case "bool", "boolean" -> noArguments(name, arguments, scalar("boolean")); + case "date" -> noArguments(name, arguments, scalar("date")); + case "uuid" -> noArguments(name, arguments, scalar("uuid")); + case "json" -> noArguments(name, arguments, scalar("json")); + case "decimal", "numeric" -> decimal(arguments); + case "nullable" -> nullable(arguments); + case "array" -> unaryComposite("array", arguments); + case "map" -> map(arguments); + case "tuple", "row" -> row(arguments); + case "timestamp", "time" -> temporal(name, arguments, suffix); + case "timestamp with time zone" -> noArguments(name, arguments, scalar("timestamp(3) with time zone")); + case "time with time zone" -> noArguments(name, arguments, scalar("time(3) with time zone")); + case "timestamp with local time zone", "time with local time zone" -> throw unsupported("WITH LOCAL TIME ZONE semantics differ from Trino WITH TIME ZONE"); + case "interval day to second", "interval year to month" -> noArguments(name, arguments, scalar(name)); + case "fixedstring" -> throw unsupported("FixedString padding semantics"); + case "date32" -> throw unsupported("Date32 range"); + case "datetime", "datetime64", "timestamptz" -> throw unsupported("ClickHouse time-zone and range semantics"); + case "interval" -> throw unsupported("interval qualifier is required"); + default -> throw unsupported("unknown type family"); + }; + } + + private static Translation characterType(String name, List arguments) + { + if (arguments.isEmpty()) { + return scalar(name); + } + requireArity(name, arguments, 1); + int length = parseUnsigned(arguments.getFirst(), name + " length"); + if (length < 1) { + throw unsupported(name + " length must be positive"); + } + return scalar(name + "(" + length + ")"); + } + + private static Translation decimal(List arguments) + { + requireArity("decimal", arguments, 2); + return decimal( + parseUnsigned(arguments.getFirst(), "decimal precision"), + parseUnsigned(arguments.get(1), "decimal scale")); + } + + private static Translation decimal(int precision, int scale) + { + if (precision < 1 || precision > 38 || scale > precision) { + throw unsupported("decimal requires 1 <= precision <= 38 and 0 <= scale <= precision"); + } + return scalar("decimal(" + precision + ", " + scale + ")"); + } + + private static Translation nullable(List arguments) + { + requireArity("nullable", arguments, 1); + Translation nested = translateType(arguments.getFirst()); + if (nested.composite()) { + throw unsupported("ClickHouse Nullable cannot wrap Array, Map, Tuple, or Row"); + } + return nested; + } + + private static Translation unaryComposite(String name, List arguments) + { + requireArity(name, arguments, 1); + return composite(name + "(" + translateType(arguments.getFirst()).sql() + ")"); + } + + private static Translation map(List arguments) + { + requireArity("map", arguments, 2); + return composite("map(" + translateType(arguments.getFirst()).sql() + ", " + translateType(arguments.get(1)).sql() + ")"); + } + + private static Translation row(List arguments) + { + if (arguments.isEmpty()) { + throw unsupported("row requires at least one field"); + } + return composite("row(" + String.join(", ", arguments.stream().map(HogQlCastTypeTranslator::rowField).toList()) + ")"); + } + + private static String rowField(String argument) + { + try { + return translateType(argument).sql(); + } + catch (IllegalArgumentException _) { + int separator = topLevelWhitespace(argument); + if (separator < 0) { + throw unsupported("invalid tuple or row field"); + } + String fieldName = argument.substring(0, separator).trim(); + String fieldType = argument.substring(separator).trim(); + if (!fieldName.matches("[A-Za-z_][A-Za-z0-9_]*|\"(?:\"\"|[^\"])+\"")) { + throw unsupported("invalid tuple or row field name"); + } + return fieldName + " " + translateType(fieldType).sql(); + } + } + + private static Translation temporal(String name, List arguments, String suffix) + { + if (arguments.size() > 1) { + throw unsupported(name + " accepts at most one precision"); + } + int precision = arguments.isEmpty() ? 3 : parseUnsigned(arguments.getFirst(), name + " precision"); + if (precision > 12) { + throw unsupported(name + " precision exceeds 12"); + } + if (suffix.equals("with local time zone")) { + throw unsupported("WITH LOCAL TIME ZONE semantics differ from Trino WITH TIME ZONE"); + } + if (!suffix.isEmpty() && !suffix.equals("with time zone")) { + throw unsupported("invalid time-zone qualifier"); + } + String qualifier = suffix.isEmpty() ? "" : " with time zone"; + return scalar(name + "(" + precision + ")" + qualifier); + } + + private static Optional parseCall(String type) + { + int opening = type.indexOf('('); + if (opening < 0) { + return Optional.empty(); + } + int closing = matchingParenthesis(type, opening); + if (closing < 0) { + throw unsupported("unbalanced parentheses"); + } + String suffix = type.substring(closing + 1); + if (suffix.contains("(") || suffix.contains(")")) { + throw unsupported("invalid type suffix"); + } + return Optional.of(new Call( + type.substring(0, opening).trim(), + splitArguments(type.substring(opening + 1, closing)), + suffix)); + } + + private static List splitArguments(String input) + { + if (input.isBlank()) { + return List.of(); + } + List arguments = new ArrayList<>(); + int depth = 0; + boolean quoted = false; + int start = 0; + for (int index = 0; index < input.length(); index++) { + char character = input.charAt(index); + if (character == '\'' || character == '"') { + quoted = !quoted; + } + else if (!quoted && character == '(') { + depth++; + } + else if (!quoted && character == ')') { + depth--; + } + else if (!quoted && depth == 0 && character == ',') { + arguments.add(nonEmpty(input.substring(start, index))); + start = index + 1; + } + } + if (depth != 0 || quoted) { + throw unsupported("unbalanced nested type"); + } + arguments.add(nonEmpty(input.substring(start))); + return List.copyOf(arguments); + } + + private static int matchingParenthesis(String input, int opening) + { + int depth = 0; + boolean quoted = false; + for (int index = opening; index < input.length(); index++) { + char character = input.charAt(index); + if (character == '\'' || character == '"') { + quoted = !quoted; + } + else if (!quoted && character == '(') { + depth++; + } + else if (!quoted && character == ')' && --depth == 0) { + return index; + } + } + return -1; + } + + private static int topLevelWhitespace(String input) + { + int depth = 0; + boolean quoted = false; + for (int index = 0; index < input.length(); index++) { + char character = input.charAt(index); + if (character == '"') { + quoted = !quoted; + } + else if (!quoted && character == '(') { + depth++; + } + else if (!quoted && character == ')') { + depth--; + } + else if (!quoted && depth == 0 && Character.isWhitespace(character)) { + return index; + } + } + return -1; + } + + private static int parseUnsigned(String value, String description) + { + String normalized = value.trim(); + if (!normalized.matches("\\d+")) { + throw unsupported(description + " must be an unsigned integer"); + } + try { + return parseInt(normalized); + } + catch (NumberFormatException _) { + throw unsupported(description + " is too large"); + } + } + + private static Translation noArguments(String name, List arguments, Translation translation) + { + requireArity(name, arguments, 0); + return translation; + } + + private static void requireArity(String name, List arguments, int expected) + { + if (arguments.size() != expected) { + throw unsupported(name + " requires " + expected + " type argument(s)"); + } + } + + private static String nonEmpty(String value) + { + String normalized = value.trim(); + if (normalized.isEmpty()) { + throw unsupported("empty type argument"); + } + return normalized; + } + + private static Translation scalar(String sql) + { + return new Translation(sql, false); + } + + private static Translation composite(String sql) + { + return new Translation(sql, true); + } + + private static IllegalArgumentException unsupported(String reason) + { + return new IllegalArgumentException(reason); + } + + private record Call(String name, List arguments, String suffix) {} + + private record Translation(String sql, boolean composite) {} +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 48451ba54dda..826d53d61b81 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -208,7 +208,7 @@ private Expression resolveExpression(Expression expression) .toList(), caseExpression.defaultValue().map(this::resolveExpression), caseExpression.span()); - case CastExpression cast -> new CastExpression(resolveExpression(cast.value()), cast.type(), cast.safe(), cast.span()); + case CastExpression cast -> new CastExpression(resolveExpression(cast.value()), cast.type(), cast.safe(), cast.typeDialect(), cast.span()); case ColumnReference reference -> reference; case FunctionCall function -> resolveFunction(function, function.window().isPresent()); case InCohortExpression in -> new InCohortExpression( diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index 675e4b7cf515..7cda9d591a15 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -711,7 +711,7 @@ private Expression resolveExpression(Expression expression) .toList(), caseExpression.defaultValue().map(this::resolveExpression), caseExpression.span()); - case CastExpression cast -> new CastExpression(resolveExpression(cast.value()), cast.type(), cast.safe(), cast.span()); + case CastExpression cast -> new CastExpression(resolveExpression(cast.value()), cast.type(), cast.safe(), cast.typeDialect(), cast.span()); case ColumnReference reference -> resolveColumn(reference); case FunctionCall function -> resolveFunctionExpression(function); case InCohortExpression in -> resolveCohortExpression(in); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 64f16dbc9aff..688a8815e56d 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -20,6 +20,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CastTypeDialect; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; @@ -327,7 +328,7 @@ private static Expression createExpression(HogQlQuery.Expression expression, Map case CastExpression cast -> new Cast( location(cast.span()), createExpression(cast.value(), parameterIds), - SQL_PARSER.createType(cast.type().value()), + createCastType(cast.type(), cast.typeDialect()), cast.safe()); case ColumnReference reference -> createColumnReference(reference); case FunctionCall function -> createFunctionCall(function, parameterIds); @@ -361,6 +362,21 @@ private static Expression createExpression(HogQlQuery.Expression expression, Map }; } + private static io.trino.sql.tree.DataType createCastType(Identifier type, CastTypeDialect typeDialect) + { + if (typeDialect == CastTypeDialect.TRINO) { + return SQL_PARSER.createType(type.value()); + } + try { + return SQL_PARSER.createType(HogQlCastTypeTranslator.translate(type.value())); + } + catch (IllegalArgumentException exception) { + throw unsupportedSemanticExpression( + type.span(), + "HogQL cast type cannot be represented exactly in Trino: " + type.value() + " (" + exception.getMessage() + ")"); + } + } + private static Expression createFunctionCall(FunctionCall function, Map parameterIds) { if (function.nameParts().size() == 1 && function.name().value().equalsIgnoreCase("matchesAction")) { diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index cd3eeea81280..306c24a9438d 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -152,14 +152,34 @@ public void testPreservesCollectionSubscriptSourceLocations() @ValueSource(strings = { "SELECT CASE WHEN enabled THEN 1 ELSE 0 END", "SELECT CASE value WHEN 1 THEN 'one' WHEN 2 THEN 'two' ELSE 'other' END", - "SELECT CAST(value AS INT)", - "SELECT TRY_CAST(value AS VARCHAR)", }) public void testLowersCaseAndCastExpressions(String hogql) { assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(hogql)); } + @ParameterizedTest + @MethodSource("representableHogQlCastTypes") + public void testLowersRepresentableHogQlCastTypes(String hogql, String trinoSql) + { + assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(trinoSql)); + } + + @ParameterizedTest + @MethodSource("unsupportedHogQlCastTypes") + public void testRejectsHogQlCastTypesWithoutExactTrinoRepresentation(String type, String reason) + { + String hogql = "SELECT CAST(value AS " + type + ")"; + + TrinoException exception = catchThrowableOfType(TrinoException.class, () -> compiler.compile(hogql)); + + assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, hogql.indexOf(type) + 1)); + assertThat(exception) + .hasMessageContaining("HogQL cast type cannot be represented exactly in Trino") + .hasMessageContaining(reason); + } + @ParameterizedTest @ValueSource(strings = { "SELECT event, count(*) AS total FROM events GROUP BY event HAVING count(*) > 1", @@ -519,6 +539,68 @@ private static Stream bindingErrors() "sensitive-table-value")); } + private static Stream representableHogQlCastTypes() + { + return Stream.of( + arguments("SELECT CAST(value AS Int8)", "SELECT CAST(value AS tinyint)"), + arguments("SELECT CAST(value AS Int16)", "SELECT CAST(value AS smallint)"), + arguments("SELECT CAST(value AS Int32)", "SELECT CAST(value AS integer)"), + arguments("SELECT CAST(value AS Int64)", "SELECT CAST(value AS bigint)"), + arguments("SELECT CAST(value AS Int)", "SELECT CAST(value AS integer)"), + arguments("SELECT CAST(value AS Integer)", "SELECT CAST(value AS integer)"), + arguments("SELECT CAST(value AS Float32)", "SELECT CAST(value AS real)"), + arguments("SELECT CAST(value AS Float64)", "SELECT CAST(value AS double)"), + arguments("SELECT CAST(value AS Float)", "SELECT CAST(value AS real)"), + arguments("SELECT CAST(value AS Real)", "SELECT CAST(value AS real)"), + arguments("SELECT CAST(value AS String)", "SELECT CAST(value AS varchar)"), + arguments("SELECT TRY_CAST(value AS VARCHAR)", "SELECT TRY_CAST(value AS varchar)"), + arguments("SELECT CAST(value AS Bool)", "SELECT CAST(value AS boolean)"), + arguments("SELECT CAST(value AS Date)", "SELECT CAST(value AS date)"), + arguments("SELECT CAST(value AS UUID)", "SELECT CAST(value AS uuid)"), + arguments("SELECT CAST(value AS JSON)", "SELECT CAST(value AS json)"), + arguments("SELECT CAST(value AS Decimal(18, 4))", "SELECT CAST(value AS decimal(18, 4))"), + arguments("SELECT CAST(value AS Decimal32(2))", "SELECT CAST(value AS decimal(9, 2))"), + arguments("SELECT CAST(value AS Decimal64(4))", "SELECT CAST(value AS decimal(18, 4))"), + arguments("SELECT CAST(value AS Decimal128(8))", "SELECT CAST(value AS decimal(38, 8))"), + arguments("SELECT TRY_CAST(value AS Nullable(Decimal(18, 4)))", "SELECT TRY_CAST(value AS decimal(18, 4))"), + arguments("SELECT CAST(value AS Array(Nullable(Int32)))", "SELECT CAST(value AS array(integer))"), + arguments("SELECT CAST(value AS Map(String, Float64))", "SELECT CAST(value AS map(varchar, double))"), + arguments("SELECT CAST(value AS Tuple(Int64, String))", "SELECT CAST(value AS row(bigint, varchar))"), + arguments("SELECT CAST(value AS Tuple(id Int64, label String))", "SELECT CAST(value AS row(id bigint, label varchar))"), + arguments("SELECT CAST(value AS Timestamp(6))", "SELECT CAST(value AS timestamp(6))"), + arguments("SELECT CAST(value AS Timestamp WITH TIME ZONE)", "SELECT CAST(value AS timestamp(3) with time zone)"), + arguments("SELECT CAST(value AS Time(12))", "SELECT CAST(value AS time(12))"), + arguments("SELECT CAST(value AS Interval Day To Second)", "SELECT CAST(value AS interval day to second)"), + arguments("SELECT CAST(value AS Interval Year To Month)", "SELECT CAST(value AS interval year to month)")); + } + + private static Stream unsupportedHogQlCastTypes() + { + return Stream.of( + arguments("UInt8", "unsigned integer range"), + arguments("UInt16", "unsigned integer range"), + arguments("UInt32", "unsigned integer range"), + arguments("UInt64", "unsigned integer range"), + arguments("UInt128", "unsigned integer range"), + arguments("UInt256", "unsigned integer range"), + arguments("Int128", "integer width exceeds Trino bigint"), + arguments("Int256", "integer width exceeds Trino bigint"), + arguments("Decimal256(2)", "decimal precision exceeds Trino"), + arguments("Decimal(39, 2)", "precision <= 38"), + arguments("Decimal(4, 5)", "scale <= precision"), + arguments("Nullable(Array(Int32))", "Nullable cannot wrap"), + arguments("Nullable(Map(String, Int32))", "Nullable cannot wrap"), + arguments("Nullable(Tuple(Int32))", "Nullable cannot wrap"), + arguments("FixedString(16)", "padding semantics"), + arguments("Date32", "Date32 range"), + arguments("DateTime", "time-zone and range semantics"), + arguments("DateTime64(3, 'UTC')", "time-zone and range semantics"), + arguments("Timestamp(13)", "precision exceeds 12"), + arguments("Time(13)", "precision exceeds 12"), + arguments("Timestamp WITH LOCAL TIME ZONE", "WITH LOCAL TIME ZONE semantics"), + arguments("Interval", "interval qualifier is required")); + } + private static HogQlTypedValue typedValue(String value) { return new HogQlTypedValue("varchar", new StringValue(value)); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticExpansion.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticExpansion.java index 1772abd77e74..08ca77b2e2d4 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticExpansion.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticExpansion.java @@ -85,6 +85,18 @@ public void testExpandsEveryExpressionRecipeToStockAst() "FROM analytics.data.raw_events")); } + @Test + public void testPreservesCatalogProvidedTrinoCastTypes() + { + HogQlCompilationResult result = compile("SELECT castReal, castInteger, castBigint FROM events"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT CAST(event_name AS real) AS castReal, " + + "CAST(event_name AS integer) AS castInteger, " + + "CAST(event_name AS bigint) AS castBigint " + + "FROM analytics.data.raw_events")); + } + @Test public void testExpandsVirtualSavedAndMaterializedRelationsWithDeclaredOutputs() { @@ -243,6 +255,9 @@ private static HogQlSemanticCatalogSnapshot snapshot() new FieldReferenceRecipe("events", "constant"), new LiteralRecipe(new TypedLiteral("bigint", LiteralEncoding.INTEGER, "1"))))), expression("castEvent", new CastRecipe(new FieldReferenceRecipe("events", "event"), "varchar")), + expression("castReal", new CastRecipe(new FieldReferenceRecipe("events", "event"), "real")), + expression("castInteger", new CastRecipe(new FieldReferenceRecipe("events", "event"), "integer")), + expression("castBigint", new CastRecipe(new FieldReferenceRecipe("events", "event"), "bigint")), expression("missingEvent", new OperatorRecipe(SemanticOperator.IS_NULL, List.of(new FieldReferenceRecipe("events", "event")))), expression("browserProperty", new PropertyLookupRecipe( "events", diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index b9975baa239d..b829f064fe92 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -24,6 +24,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; import io.trino.hogql.parser.tree.HogQlQuery.CaseWhen; import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CastTypeDialect; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; @@ -468,12 +469,14 @@ private SourceSpan sourceSpan(Token start, Token stop) private static final class AstBuilder { + private final String source; private final SourcePositions sourcePositions; private final Deque> commonTableScopes = new ArrayDeque<>(); private final Deque> prohibitedCommonTableScopes = new ArrayDeque<>(); private AstBuilder(String source) { + this.source = requireNonNull(source, "source is null"); sourcePositions = new SourcePositions(source); } @@ -1095,10 +1098,10 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) return buildCaseExpression(caseExpression); } if (context instanceof HogQLParser.ColumnExprCastContext cast) { - return new CastExpression(buildExpression(cast.columnExpr()), buildSimpleType(cast.columnTypeExpr()), false, sourceSpan(cast)); + return new CastExpression(buildExpression(cast.columnExpr()), buildType(cast.columnTypeExpr()), false, CastTypeDialect.HOGQL, sourceSpan(cast)); } if (context instanceof HogQLParser.ColumnExprTryCastContext cast) { - return new CastExpression(buildExpression(cast.columnExpr()), buildSimpleType(cast.columnTypeExpr()), true, sourceSpan(cast)); + return new CastExpression(buildExpression(cast.columnExpr()), buildType(cast.columnTypeExpr()), true, CastTypeDialect.HOGQL, sourceSpan(cast)); } if (context instanceof HogQLParser.ColumnExprIdentifierContext identifier) { return buildColumnReference(identifier.columnIdentifier()); @@ -1268,12 +1271,10 @@ private CaseExpression buildCaseExpression(HogQLParser.ColumnExprCaseContext con return new CaseExpression(operand, whenClauses, defaultValue, sourceSpan(context)); } - private Identifier buildSimpleType(HogQLParser.ColumnTypeExprContext context) + private Identifier buildType(HogQLParser.ColumnTypeExprContext context) { - if (context instanceof HogQLParser.ColumnTypeExprSimpleContext simple) { - return buildIdentifier(simple.identifier()); - } - throw unsupported(context, "complex cast type"); + SourceSpan span = sourceSpan(context); + return new Identifier(source.substring(span.startOffset(), span.endOffset()), false, span); } private List buildExpressions(HogQLParser.ColumnExprListContext context) diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index 71336bc56fe6..5990e9a3f1ca 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -341,15 +341,27 @@ public record CaseWhen(Expression operand, Expression result, SourceSpan span) } } - public record CastExpression(Expression value, Identifier type, boolean safe, SourceSpan span) + public enum CastTypeDialect + { + HOGQL, + TRINO, + } + + public record CastExpression(Expression value, Identifier type, boolean safe, CastTypeDialect typeDialect, SourceSpan span) implements Expression { public CastExpression { value = requireNonNull(value, "value is null"); type = requireNonNull(type, "type is null"); + typeDialect = requireNonNull(typeDialect, "typeDialect is null"); span = requireNonNull(span, "span is null"); } + + public CastExpression(Expression value, Identifier type, boolean safe, SourceSpan span) + { + this(value, type, safe, CastTypeDialect.TRINO, span); + } } public record ArrayExpression(List values, SourceSpan span) diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 0a5bfb4eeb30..28b2aad77a33 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -16,6 +16,7 @@ import io.trino.hogql.parser.tree.HogQlQuery; import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; @@ -175,6 +176,24 @@ public void testBuildsColumnsSelectorsAndReplacementAst() assertThat(expression).isInstanceOf(ColumnReference.class)); } + @Test + public void testPreservesParameterizedAndNestedCastTypeSyntax() + { + HogQlQuery query = parser.parseStatement( + "SELECT CAST(value AS Nullable(Decimal(18, 4))), " + + "TRY_CAST(value AS Tuple(id Int64, payload Array(String)))"); + + CastExpression decimal = (CastExpression) ((ExpressionProjection) query.projections().getFirst()).expression(); + CastExpression tuple = (CastExpression) ((ExpressionProjection) query.projections().get(1)).expression(); + + assertThat(decimal.type().value()).isEqualTo("Nullable(Decimal(18, 4))"); + assertThat(decimal.type().span()).isEqualTo(new HogQlQuery.SourceSpan(21, 45, 1, 22, 1, 46)); + assertThat(decimal.safe()).isFalse(); + assertThat(decimal.typeDialect()).isEqualTo(HogQlQuery.CastTypeDialect.HOGQL); + assertThat(tuple.type().value()).isEqualTo("Tuple(id Int64, payload Array(String))"); + assertThat(tuple.safe()).isTrue(); + } + @Test public void testBuildsDistinctOrderingAndPaginationAst() { From 965bdb11d1926667c34def16478468b7b124bc40 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 15:55:13 -0400 Subject: [PATCH 063/136] test(hogql): execute endpoint differentials --- .../TestHogQlStatementResource.java | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java index 3851451a6a20..4dd52a41abf3 100644 --- a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java +++ b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java @@ -23,6 +23,7 @@ import io.airlift.json.JsonCodec; import io.airlift.json.JsonCodecFactory; import io.airlift.json.JsonMapperProvider; +import io.trino.client.Column; import io.trino.client.QueryDataJacksonModule; import io.trino.client.QueryResults; import io.trino.client.ResultRowsDecoder; @@ -182,6 +183,55 @@ public void testEnabledEndpointUsesStatementProtocol() assertThat(rows(sqlResults)).containsExactly(ImmutableList.of(2)); } + @ParameterizedTest(name = "{0}") + @MethodSource("differentialQueries") + public void testHogQlExecutionMatchesTrinoSql(String name, String hogql, String trinoSql) + throws Exception + { + List hogqlResults = runHogQlToCompletion(hogQlRequest(hogql)); + List trinoResults = runToCompletion("/v1/statement", trinoSql, false); + + assertThat(hogqlResults.getLast().getError()).describedAs("HogQL result for %s", name).isNull(); + assertThat(trinoResults.getLast().getError()).describedAs("Trino SQL result for %s", name).isNull(); + assertThat(resultColumns(hogqlResults)).isEqualTo(resultColumns(trinoResults)); + assertThat(rows(hogqlResults)).isEqualTo(rows(trinoResults)); + } + + private static Stream differentialQueries() + { + return Stream.of( + Arguments.of( + "nulls, cases, and nested casts", + "SELECT CAST(NULL AS Nullable(Int32)) AS value, CASE WHEN NULL IS NULL THEN 'yes' ELSE 'no' END AS marker", + "SELECT CAST(NULL AS integer) AS value, CASE WHEN NULL IS NULL THEN 'yes' ELSE 'no' END AS marker"), + Arguments.of( + "joins, grouping, and ordering", + "SELECT r.regionkey, count(n.nationkey) AS nations " + + "FROM tpch.tiny.region r LEFT JOIN tpch.tiny.nation n ON r.regionkey = n.regionkey " + + "GROUP BY r.regionkey ORDER BY r.regionkey", + "SELECT r.regionkey, count(n.nationkey) AS nations " + + "FROM tpch.tiny.region r LEFT JOIN tpch.tiny.nation n ON r.regionkey = n.regionkey " + + "GROUP BY r.regionkey ORDER BY r.regionkey"), + Arguments.of( + "window frames", + "SELECT nationkey, sum(nationkey) OVER (ORDER BY nationkey ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS running " + + "FROM tpch.tiny.nation WHERE nationkey < 5 ORDER BY nationkey", + "SELECT nationkey, sum(nationkey) OVER (ORDER BY nationkey ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS running " + + "FROM tpch.tiny.nation WHERE nationkey < 5 ORDER BY nationkey"), + Arguments.of( + "values and set operations", + "SELECT value FROM (VALUES (2), (1), (2)) AS numbers(value) UNION SELECT 3 ORDER BY value", + "SELECT value FROM (VALUES (2), (1), (2)) AS numbers(value) UNION SELECT 3 ORDER BY value"), + Arguments.of( + "arrays, maps, and rows", + "SELECT ARRAY[1, 2, 3][2] AS array_value, MAP(ARRAY['k'], ARRAY[7])['k'] AS map_value, (11, 'x').1 AS row_value", + "SELECT ARRAY[1, 2, 3][2] AS array_value, MAP(ARRAY['k'], ARRAY[7])['k'] AS map_value, ROW(11, 'x')[1] AS row_value"), + Arguments.of( + "timestamp precision", + "SELECT CAST('2024-03-10 01:59:59.123456' AS Timestamp(6)) AS value", + "SELECT CAST('2024-03-10 01:59:59.123456' AS timestamp(6)) AS value")); + } + @Test public void testPhysicalCatalogCompatibilityEndpoint() { @@ -544,4 +594,13 @@ private static List> rows(List results) } return rows.build(); } + + private static List resultColumns(List results) + { + return results.stream() + .map(QueryResults::getColumns) + .filter(columns -> columns != null) + .findFirst() + .orElseThrow(); + } } From 4471ff5a61a00045926dc7e8a22106e6ff1e26d5 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 16:10:58 -0400 Subject: [PATCH 064/136] feat(hogql): prune derived lazy projections --- .../hogql/compiler/HogQlProjectionDemand.java | 272 ++++++++++++++++++ .../hogql/compiler/HogQlSemanticResolver.java | 151 +++++++++- .../compiler/TestHogQlProjectionDemand.java | 38 +++ .../TestHogQlRelationshipExpansion.java | 23 ++ .../planner/TestHogQlProjectionPruning.java | 180 ++++++++++++ 5 files changed, 655 insertions(+), 9 deletions(-) create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlProjectionDemand.java create mode 100644 core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java new file mode 100644 index 000000000000..ed4667b63c2b --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java @@ -0,0 +1,272 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; +import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowFrame; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +final class HogQlProjectionDemand +{ + private final boolean all; + private final Set unqualified; + private final Set allQualifiers; + private final Map> qualified; + + private HogQlProjectionDemand(boolean all, Set unqualified, Set allQualifiers, Map> qualified) + { + this.all = all; + this.unqualified = Set.copyOf(requireNonNull(unqualified, "unqualified is null")); + this.allQualifiers = Set.copyOf(requireNonNull(allQualifiers, "allQualifiers is null")); + requireNonNull(qualified, "qualified is null"); + Map> copied = new HashMap<>(); + qualified.forEach((key, value) -> copied.put(key, Set.copyOf(value))); + this.qualified = Map.copyOf(copied); + } + + public static HogQlProjectionDemand collect(HogQlQuery query) + { + Builder builder = new Builder(); + if (query.body() instanceof SelectQueryBody select) { + select.projections().forEach(projection -> collect(projection, builder)); + select.where().ifPresent(expression -> collect(expression, builder)); + select.groupBy().forEach(expression -> collect(expression, builder)); + select.having().ifPresent(expression -> collect(expression, builder)); + select.windows().forEach(window -> collect(window.specification(), builder)); + select.from().ifPresent(relation -> collectJoinCriteria(relation, builder)); + } + query.orderBy().forEach(sortItem -> collect(sortItem.expression(), builder)); + query.limit().ifPresent(expression -> collect(expression, builder)); + query.offset().ifPresent(expression -> collect(expression, builder)); + return builder.build(); + } + + public static HogQlProjectionDemand preserveAll() + { + return new HogQlProjectionDemand(true, Set.of(), Set.of(), Map.of()); + } + + public RequiredOutputs forAlias(Identifier alias) + { + String qualifier = canonical(alias.value()); + if (all || allQualifiers.contains(qualifier)) { + return RequiredOutputs.allOutputs(); + } + Set names = new HashSet<>(unqualified); + names.addAll(qualified.getOrDefault(qualifier, Set.of())); + return new RequiredOutputs(false, names); + } + + public RequiredOutputs unqualified() + { + if (all) { + return RequiredOutputs.allOutputs(); + } + return new RequiredOutputs(false, unqualified); + } + + private static void collect(Projection projection, Builder builder) + { + switch (projection) { + case ColumnsList columns -> columns.expressions().forEach(expression -> collect(expression, builder)); + case ColumnsRegex _ -> builder.all = true; + case ExpressionProjection expression -> collect(expression.expression(), builder); + case Star star -> { + if (star.qualifier().isEmpty()) { + builder.all = true; + } + else { + builder.allQualifiers.add(canonical(star.qualifier().getFirst().value())); + } + star.replacements().forEach(replacement -> collect(replacement.expression(), builder)); + } + } + } + + private static void collectJoinCriteria(Relation relation, Builder builder) + { + switch (relation) { + case HogQlQuery.AliasedRelation alias -> collectJoinCriteria(alias.relation(), builder); + case HogQlQuery.CommonTableReference _, HogQlQuery.SubqueryRelation _, HogQlQuery.TablePlaceholder _, HogQlQuery.TableReference _, HogQlQuery.ValuesRelation _ -> {} + case JoinRelation join -> { + collectJoinCriteria(join.left(), builder); + collectJoinCriteria(join.right(), builder); + join.criteria().ifPresent(criteria -> { + switch (criteria) { + case JoinOn on -> collect(on.expression(), builder); + case JoinUsing using -> using.columns().forEach(builder::addUnqualified); + } + }); + } + } + } + + private static void collect(Expression expression, Builder builder) + { + switch (expression) { + case ArrayExpression array -> array.values().forEach(value -> collect(value, builder)); + case BetweenExpression between -> { + collect(between.value(), builder); + collect(between.min(), builder); + collect(between.max(), builder); + } + case BinaryExpression binary -> { + collect(binary.left(), builder); + collect(binary.right(), builder); + } + case CaseExpression caseExpression -> { + caseExpression.operand().ifPresent(value -> collect(value, builder)); + caseExpression.whenClauses().forEach(when -> { + collect(when.operand(), builder); + collect(when.result(), builder); + }); + caseExpression.defaultValue().ifPresent(value -> collect(value, builder)); + } + case CastExpression cast -> collect(cast.value(), builder); + case ColumnReference reference -> builder.add(reference); + case FunctionCall function -> { + function.arguments().forEach(argument -> collect(argument, builder)); + function.orderBy().forEach(sortItem -> collect(sortItem.expression(), builder)); + function.filter().ifPresent(filter -> collect(filter, builder)); + function.window().ifPresent(window -> collect(window, builder)); + } + case InCohortExpression cohort -> collect(cohort.value(), builder); + case InExpression in -> { + collect(in.value(), builder); + in.values().forEach(value -> collect(value, builder)); + } + case InSubqueryExpression in -> { + collect(in.value(), builder); + builder.all = true; + } + case IsNullExpression isNull -> collect(isNull.value(), builder); + case Literal _, Placeholder _ -> {} + case MemberAccessExpression member -> collect(member.base(), builder); + case SubscriptExpression subscript -> { + collect(subscript.base(), builder); + collect(subscript.index(), builder); + } + case TupleExpression tuple -> tuple.values().forEach(value -> collect(value, builder)); + case UnaryExpression unary -> collect(unary.operand(), builder); + } + } + + private static void collect(Window window, Builder builder) + { + switch (window) { + case WindowReference _ -> {} + case WindowSpecification specification -> { + specification.partitionBy().forEach(expression -> collect(expression, builder)); + specification.orderBy().forEach(sortItem -> collect(sortItem.expression(), builder)); + specification.frame().ifPresent(frame -> collect(frame, builder)); + } + } + } + + private static void collect(WindowFrame frame, Builder builder) + { + frame.start().value().ifPresent(value -> collect(value, builder)); + frame.end().flatMap(HogQlQuery.FrameBound::value).ifPresent(value -> collect(value, builder)); + } + + private static String canonical(String value) + { + return value.toLowerCase(java.util.Locale.ENGLISH); + } + + public record RequiredOutputs(boolean all, Set names) + { + public RequiredOutputs + { + names = Set.copyOf(requireNonNull(names, "names is null")); + } + + public static RequiredOutputs allOutputs() + { + return new RequiredOutputs(true, Set.of()); + } + + public boolean includes(String name) + { + return all || names.contains(canonical(name)); + } + } + + private static final class Builder + { + private boolean all; + private final Set unqualified = new HashSet<>(); + private final Set allQualifiers = new HashSet<>(); + private final Map> qualified = new HashMap<>(); + + public void add(ColumnReference reference) + { + if (reference.parts().size() == 1) { + unqualified.add(canonical(reference.parts().getFirst().value())); + return; + } + String qualifier = canonical(reference.parts().getFirst().value()); + String field = canonical(reference.parts().get(1).value()); + qualified.computeIfAbsent(qualifier, _ -> new HashSet<>()).add(field); + } + + public void addUnqualified(Identifier identifier) + { + unqualified.add(canonical(identifier.value())); + } + + public HogQlProjectionDemand build() + { + return new HogQlProjectionDemand(all, unqualified, allQualifiers, qualified); + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index 7cda9d591a15..32e1e2bb69f7 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -13,6 +13,7 @@ */ package io.trino.hogql.compiler; +import io.trino.hogql.compiler.HogQlProjectionDemand.RequiredOutputs; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ActionReference; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CastRecipe; @@ -121,6 +122,7 @@ final class HogQlSemanticResolver private final PinnedSnapshot snapshot; private final ExpansionBudget expansionBudget; + private final Optional requiredOutputs; private final Map relationshipPaths = new LinkedHashMap<>(); private List bindings = List.of(); private boolean allRelationsLogical; @@ -129,13 +131,19 @@ final class HogQlSemanticResolver private HogQlSemanticResolver(PinnedSnapshot snapshot) { - this(snapshot, new ExpansionBudget()); + this(snapshot, new ExpansionBudget(), Optional.empty()); } private HogQlSemanticResolver(PinnedSnapshot snapshot, ExpansionBudget expansionBudget) + { + this(snapshot, expansionBudget, Optional.empty()); + } + + private HogQlSemanticResolver(PinnedSnapshot snapshot, ExpansionBudget expansionBudget, Optional requiredOutputs) { this.snapshot = requireNonNull(snapshot, "snapshot is null"); this.expansionBudget = requireNonNull(expansionBudget, "expansionBudget is null"); + this.requiredOutputs = requireNonNull(requiredOutputs, "requiredOutputs is null"); } public static Optional resolve(PinnedSnapshot snapshot, HogQlQuery query) @@ -169,7 +177,8 @@ private HogQlQuery resolveNestedQuery(HogQlQuery query) return new HogQlQuery(commonTables, resolved, query.orderBy(), query.limit(), query.offset(), query.span()); } SelectQueryBody select = (SelectQueryBody) query.body(); - Optional relation = select.from().map(this::resolveRelation); + HogQlProjectionDemand projectionDemand = HogQlProjectionDemand.collect(query); + Optional relation = select.from().map(value -> resolveRelation(value, projectionDemand)); bindings = relation.map(ResolvedRelation::bindings).orElse(List.of()); allRelationsLogical = relation.map(ResolvedRelation::allLogical).orElse(false); expandedRelation = relation.map(ResolvedRelation::relation); @@ -195,6 +204,11 @@ private HogQlQuery resolveQuery(HogQlQuery query, List co { List projections = new ArrayList<>(); query.projections().forEach(projection -> projections.addAll(resolveProjection(projection))); + if (projections.isEmpty() && requiredOutputs.isPresent()) { + projections.add(new ExpressionProjection( + new Literal(HogQlQuery.LiteralKind.INTEGER, "1", query.span()), + Optional.of(new Identifier("__hogql_pruned", true, query.span())))); + } Optional where = query.where().map(this::resolveExpression); List groupBy = query.groupBy().stream().map(this::resolveExpression).toList(); Optional having = query.having().map(this::resolveExpression); @@ -270,6 +284,11 @@ private List resolveColumnsRegex(ColumnsRegex columns) private List resolveStar(Star star) { + Optional> lazyStar = resolveLazyStar(star); + if (lazyStar.isPresent()) { + return lazyStar.orElseThrow(); + } + List starBindings; boolean qualified = !star.qualifier().isEmpty(); if (qualified) { @@ -299,6 +318,7 @@ private List resolveStar(Star star) .flatMap(binding -> binding.orderedFields().stream() .filter(BoundField::starVisible) .filter(field -> !exclusions.contains(new StarField(binding, field))) + .filter(field -> projectionDemanded(field.name())) .map(field -> { StarReplacement replacement = replacements.get(new StarField(binding, field)); Expression expression = replacement == null @@ -315,6 +335,99 @@ private List resolveStar(Star star) .toList(); } + private Optional> resolveLazyStar(Star star) + { + if (star.qualifier().isEmpty() || star.qualifier().size() > 2) { + return Optional.empty(); + } + + List matches; + if (star.qualifier().size() == 1) { + String lazyName = star.qualifier().getFirst().value(); + matches = bindings.stream() + .flatMap(binding -> lazyTable(binding, lazyName) + .map(definition -> java.util.stream.Stream.of(new LazyStar(binding, definition))) + .orElseGet(java.util.stream.Stream::empty)) + .toList(); + } + else { + Identifier owner = star.qualifier().getFirst(); + String lazyName = star.qualifier().getLast().value(); + matches = bindings.stream() + .filter(binding -> matchesStarQualifier(binding, List.of(owner))) + .flatMap(binding -> lazyTable(binding, lazyName) + .map(definition -> java.util.stream.Stream.of(new LazyStar(binding, definition))) + .orElseGet(java.util.stream.Stream::empty)) + .toList(); + } + if (matches.isEmpty()) { + return Optional.empty(); + } + if (matches.size() > 1) { + throw starResolutionError(star.qualifier().getFirst(), "Ambiguous HogQL star qualifier: " + starQualifier(star)); + } + + LazyStar lazyStar = matches.getFirst(); + List visible = lazyStar.definition().projections().stream() + .filter(LazyProjectionDefinition::starVisible) + .toList(); + Set exclusions = new HashSet<>(); + for (ColumnReference exclusion : star.exclusions()) { + Identifier name = exclusion.parts().getLast(); + LazyProjectionDefinition matched = visible.stream() + .filter(projection -> matchesIdentifier(name, projection.name())) + .findFirst() + .orElseThrow(() -> starResolutionError(exclusion.span(), "Unknown HogQL star exclusion: " + identifierPath(exclusion.parts()))); + if (!exclusions.add(canonical(matched.name()))) { + throw starResolutionError(exclusion.span(), "Duplicate HogQL star exclusion: " + identifierPath(exclusion.parts())); + } + } + Map replacements = new HashMap<>(); + for (StarReplacement replacement : star.replacements()) { + LazyProjectionDefinition matched = visible.stream() + .filter(projection -> !exclusions.contains(canonical(projection.name()))) + .filter(projection -> matchesIdentifier(replacement.target(), projection.name())) + .findFirst() + .orElseThrow(() -> starResolutionError(replacement.target(), "Unknown HogQL star replacement: " + replacement.target().value())); + if (replacements.putIfAbsent(canonical(matched.name()), replacement) != null) { + throw starResolutionError(replacement.target(), "Duplicate HogQL star replacement: " + replacement.target().value()); + } + } + + return Optional.of(visible.stream() + .filter(projection -> !exclusions.contains(canonical(projection.name()))) + .filter(projection -> projectionDemanded(projection.name())) + .map(projection -> { + StarReplacement replacement = replacements.get(canonical(projection.name())); + Expression expression = replacement == null + ? resolveLazyProjection(lazyStar.binding(), lazyStar.definition(), projection, star.span()) + : resolveExpression(replacement.expression()); + return new ExpressionProjection(expression, Optional.of(new Identifier(projection.name(), true, star.span()))); + }) + .map(Projection.class::cast) + .toList()); + } + + private Expression resolveLazyProjection( + TableBinding owner, + LazyTableDefinition definition, + LazyProjectionDefinition projection, + HogQlQuery.SourceSpan span) + { + TableBinding terminal = ensureRelationshipPath(owner, definition.relationshipPath(), span); + return expandRecipe( + terminal, + projection.recipe(), + Optional.of(terminal.outputQualifier()), + span, + expansionBudget); + } + + private boolean projectionDemanded(String name) + { + return requiredOutputs.map(outputs -> outputs.includes(name)).orElse(true); + } + private static Map resolveStarReplacements(Star star, List starBindings, Set exclusions) { Map replacements = new HashMap<>(); @@ -414,10 +527,21 @@ private static String identifierPath(List identifiers) } private ResolvedRelation resolveRelation(Relation relation) + { + return resolveRelation(relation, HogQlProjectionDemand.preserveAll()); + } + + private ResolvedRelation resolveRelation(Relation relation, HogQlProjectionDemand projectionDemand) { return switch (relation) { case AliasedRelation alias -> { - ResolvedRelation child = resolveRelation(alias.relation()); + ResolvedRelation child; + if (alias.relation() instanceof SubqueryRelation subquery && alias.columnAliases().isEmpty()) { + child = resolveSubquery(subquery, projectionDemand.forAlias(alias.alias())); + } + else { + child = resolveRelation(alias.relation(), HogQlProjectionDemand.preserveAll()); + } List aliasedBindings = child.bindings().stream() .map(binding -> binding.withAlias(alias.alias())) .toList(); @@ -428,8 +552,8 @@ yield new ResolvedRelation( } case CommonTableReference commonTable -> new ResolvedRelation(commonTable, List.of(), false); case JoinRelation join -> { - ResolvedRelation left = resolveRelation(join.left()); - ResolvedRelation right = resolveRelation(join.right()); + ResolvedRelation left = resolveRelation(join.left(), projectionDemand); + ResolvedRelation right = resolveRelation(join.right(), projectionDemand); List joinBindings = new ArrayList<>(left.bindings()); joinBindings.addAll(right.bindings()); List previousBindings = bindings; @@ -447,16 +571,23 @@ yield new ResolvedRelation( joinBindings, left.allLogical() && right.allLogical()); } - case SubqueryRelation subquery -> new ResolvedRelation( - new SubqueryRelation(new HogQlSemanticResolver(snapshot, expansionBudget).resolveNestedQuery(subquery.query()), subquery.span()), - List.of(), - false); + case SubqueryRelation subquery -> resolveSubquery(subquery, projectionDemand.unqualified()); case TablePlaceholder placeholder -> new ResolvedRelation(placeholder, List.of(), false); case TableReference table -> resolveTable(table); case ValuesRelation values -> new ResolvedRelation(values, List.of(), false); }; } + private ResolvedRelation resolveSubquery(SubqueryRelation subquery, RequiredOutputs requiredOutputs) + { + return new ResolvedRelation( + new SubqueryRelation( + new HogQlSemanticResolver(snapshot, expansionBudget, Optional.of(requiredOutputs)).resolveNestedQuery(subquery.query()), + subquery.span()), + List.of(), + false); + } + private JoinUsing resolveJoinUsing(JoinUsing using, List leftBindings, List rightBindings) { List columns = using.columns().stream() @@ -1788,6 +1919,8 @@ private record FieldMatch(TableBinding binding, BoundField field) {} private record StarField(TableBinding binding, BoundField field) {} + private record LazyStar(TableBinding binding, LazyTableDefinition definition) {} + private record PropertyMatch(TableBinding binding, PropertyDefinition property) {} private record PathCandidate(TableBinding owner, List path) diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlProjectionDemand.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlProjectionDemand.java new file mode 100644 index 000000000000..bbc0ab106958 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlProjectionDemand.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.parser.HogQlParser; +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlProjectionDemand +{ + private final HogQlParser parser = new HogQlParser(); + + @Test + public void testJoinUsingDemandsTheSharedDerivedColumn() + { + HogQlQuery query = parser.parseStatement("SELECT sub.event FROM (SELECT * FROM events) sub JOIN persons USING (name)"); + + HogQlProjectionDemand.RequiredOutputs outputs = HogQlProjectionDemand.collect(query) + .forAlias(new Identifier("sub", false, query.span())); + + assertThat(outputs.includes("event")).isTrue(); + assertThat(outputs.includes("name")).isTrue(); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java index 4eff489c9fbb..b4fcfa817abe 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java @@ -94,6 +94,29 @@ public void testExpandsOnlyTheDeclaredPathThroughRelationshipCycle() "ON \"__hogql_lazy_1\".event_id = \"__hogql_lazy_2\".event_id")); } + @Test + public void testPrunesUnusedLazyStarProjectionBeforeExpansion() + { + HogQlCompilationResult result = compile("SELECT sub.event FROM (SELECT e.personProfile.*, e.* FROM events e) sub"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT sub.event FROM (SELECT e.event_name AS event FROM analytics.data.raw_events e) sub")); + } + + @Test + public void testRetainsDemandedLazyStarProjectionAsStockJoin() + { + HogQlCompilationResult result = compile("SELECT sub.name FROM (SELECT e.personProfile.*, e.* FROM events e) sub"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT sub.name FROM (" + + "SELECT \"__hogql_lazy_1\".full_name AS name " + + "FROM analytics.data.raw_events e " + + "LEFT JOIN analytics.data.raw_persons \"__hogql_lazy_1\" " + + "ON e.person_id = \"__hogql_lazy_1\".person_id " + + "AND e.workspace_id = \"__hogql_lazy_1\".workspace_id) sub")); + } + @ParameterizedTest @MethodSource("nonProjectionRelationshipQueries") public void testExpandsRelationshipDemandedOutsideProjection(String hogql, String trinoSql) diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java new file mode 100644 index 000000000000..b58bb7583218 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java @@ -0,0 +1,180 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.sql.planner; + +import com.google.common.collect.ImmutableMap; +import io.trino.hogql.compiler.HogQlCompilationResult; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.HogQlSemanticCatalogContext; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyProjectionDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.sql.SqlFormatter; +import io.trino.sql.planner.assertions.BasePlanTest; +import io.trino.sql.planner.optimizations.PlanNodeSearcher; +import io.trino.sql.planner.plan.FilterNode; +import io.trino.sql.planner.plan.JoinNode; +import io.trino.sql.planner.plan.PlanNode; +import io.trino.sql.planner.plan.TableScanNode; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; + +import static io.trino.SystemSessionProperties.ENABLE_DYNAMIC_FILTERING; +import static io.trino.SystemSessionProperties.JOIN_REORDERING_STRATEGY; +import static io.trino.sql.planner.plan.JoinType.INNER; +import static io.trino.testing.TestingHandles.TEST_CATALOG_NAME; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlProjectionPruning + extends BasePlanTest +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier(TEST_CATALOG_NAME, false); + private static final HogQlSemanticCatalogSnapshot SNAPSHOT = snapshot(); + + private final HogQlCompiler compiler = new HogQlCompiler(); + + public TestHogQlProjectionPruning() + { + super(ImmutableMap.of( + ENABLE_DYNAMIC_FILTERING, "false", + JOIN_REORDERING_STRATEGY, "NONE")); + } + + @Test + public void testUnusedLazyProjectionDoesNotReachStockPlan() + { + Plan plan = plan(compile("SELECT sub.event FROM (SELECT e.personProfile.*, e.* FROM events e) sub")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(1); + assertThat(nodes(plan, JoinNode.class)).isEmpty(); + } + + @Test + public void testUsedLazyProjectionRemainsVisibleToStockPredicatePushdown() + { + Plan plan = plan(compile( + "SELECT sub.name FROM (SELECT e.personProfile.*, e.* FROM events e) sub " + + "WHERE sub.name = 'Customer#000000001'")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(2); + List joins = nodes(plan, JoinNode.class); + assertThat(joins).hasSize(1); + JoinNode join = joins.getFirst(); + assertThat(join.getType()).isEqualTo(INNER); + assertThat(PlanNodeSearcher.searchFrom(join.getRight()) + .whereIsInstanceOfAny(FilterNode.class) + .findFirst()).isPresent(); + } + + private String compile(String query) + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + HogQlCompilationResult result = compiler.compile(new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(1)), Optional.of(context)); + return SqlFormatter.formatSql(result.statement()); + } + + private static List nodes(Plan plan, Class type) + { + return PlanNodeSearcher.searchFrom(plan.getRoot()) + .whereIsInstanceOfAny(type) + .findAll().stream() + .map(type::cast) + .toList(); + } + + private static HogQlSemanticCatalogSnapshot snapshot() + { + LogicalTableDefinition events = new LogicalTableDefinition( + "events", + physicalName("orders"), + List.of( + field("event", "orderkey", "bigint", LogicalType.INTEGER), + field("personId", "custkey", "bigint", LogicalType.INTEGER)), + List.of(), + List.of(new RelationshipDefinition( + "person", + "persons", + RelationshipCardinality.MANY_TO_ONE, + List.of(new JoinKey("personId", "personId"))))); + LogicalTableDefinition persons = new LogicalTableDefinition( + "persons", + physicalName("customer"), + List.of( + field("personId", "custkey", "bigint", LogicalType.INTEGER), + field("name", "name", "varchar", LogicalType.STRING)), + List.of(), + List.of()); + LazyTableDefinition personProfile = new LazyTableDefinition( + "events", + "personProfile", + List.of("person"), + List.of(new LazyProjectionDefinition( + "name", + "varchar", + LogicalType.STRING, + false, + true, + new FieldReferenceRecipe("persons", "name")))); + return new HogQlSemanticCatalogSnapshot( + HogQlSemanticCatalogSnapshot.PROTOCOL_VERSION, + HogQlSemanticCatalogSnapshot.SCHEMA_VERSION, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 1, + List.of(events, persons), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(personProfile), + List.of(), + List.of()); + } + + private static LogicalFieldDefinition field(String name, String physicalName, String type, LogicalType logicalType) + { + return new LogicalFieldDefinition(name, new PhysicalIdentifier(physicalName, false), type, logicalType, false, true); + } + + private static PhysicalQualifiedName physicalName(String table) + { + return new PhysicalQualifiedName(CATALOG, new PhysicalIdentifier("tiny", false), new PhysicalIdentifier(table, false)); + } +} From 64a1dc90472fe95e10a259193c16482ff92db0be Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 16:23:04 -0400 Subject: [PATCH 065/136] feat(hogql): compile canonical interval expressions --- .../trino/hogql/compiler/HogQlCompiler.java | 3 + .../hogql/compiler/HogQlFunctionResolver.java | 2 + .../hogql/compiler/HogQlProjectionDemand.java | 2 + .../hogql/compiler/HogQlSemanticResolver.java | 2 + .../trino/hogql/compiler/TrinoAstFactory.java | 34 ++++++++++ .../hogql/compiler/TestHogQlCompiler.java | 20 ++++++ .../io/trino/hogql/parser/HogQlParser.java | 68 +++++++++++++++++++ .../trino/hogql/parser/tree/HogQlQuery.java | 24 +++++++ .../1.0.0/trino-compatibility-overrides.json | 18 +++++ .../language/1.0.0/trino-compatibility.json | 18 ++--- .../trino/hogql/parser/TestHogQlParser.java | 37 ++++++++++ .../TestHogQlStatementResource.java | 8 ++- 12 files changed, 227 insertions(+), 9 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index f1db4beb41b1..26bcbdda7387 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -38,6 +38,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; @@ -341,6 +342,7 @@ private static boolean containsFunctionCall(Expression expression) case InCohortExpression _ -> true; case InExpression in -> containsFunctionCall(in.value()) || in.values().stream().anyMatch(HogQlCompiler::containsFunctionCall); case InSubqueryExpression in -> containsFunctionCall(in.value()) || containsFunctionCall(in.query()); + case IntervalExpression interval -> containsFunctionCall(interval.value()); case IsNullExpression isNull -> containsFunctionCall(isNull.value()); case MemberAccessExpression memberAccess -> containsFunctionCall(memberAccess.base()); case SubscriptExpression subscript -> containsFunctionCall(subscript.base()) || containsFunctionCall(subscript.index()); @@ -446,6 +448,7 @@ private static void collectPlaceholders(Expression expression, List collectPlaceholders(in.value(), placeholders); collectPlaceholders(in.query(), placeholders); } + case IntervalExpression interval -> collectPlaceholders(interval.value(), placeholders); case IsNullExpression isNull -> collectPlaceholders(isNull.value(), placeholders); case Literal _ -> {} case MemberAccessExpression memberAccess -> collectPlaceholders(memberAccess.base(), placeholders); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 826d53d61b81..1356301f5678 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -40,6 +40,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; @@ -229,6 +230,7 @@ private Expression resolveExpression(Expression expression) in.negated(), in.predicateSpan(), in.span()); + case IntervalExpression interval -> new IntervalExpression(resolveExpression(interval.value()), interval.unit(), interval.span()); case IsNullExpression isNull -> new IsNullExpression( resolveExpression(isNull.value()), isNull.negated(), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java index ed4667b63c2b..594aa33c3b6b 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java @@ -29,6 +29,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; @@ -188,6 +189,7 @@ private static void collect(Expression expression, Builder builder) collect(in.value(), builder); builder.all = true; } + case IntervalExpression interval -> collect(interval.value(), builder); case IsNullExpression isNull -> collect(isNull.value(), builder); case Literal _, Placeholder _ -> {} case MemberAccessExpression member -> collect(member.base(), builder); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index 32e1e2bb69f7..ef738571c4e9 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -69,6 +69,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; @@ -858,6 +859,7 @@ private Expression resolveExpression(Expression expression) in.negated(), in.predicateSpan(), in.span()); + case IntervalExpression interval -> new IntervalExpression(resolveExpression(interval.value()), interval.unit(), interval.span()); case IsNullExpression isNull -> new IsNullExpression( resolveExpression(isNull.value()), isNull.negated(), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 688a8815e56d..72022a07ce57 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -33,6 +33,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; @@ -77,6 +78,8 @@ import io.trino.sql.tree.InListExpression; import io.trino.sql.tree.InPredicate; import io.trino.sql.tree.Intersect; +import io.trino.sql.tree.IntervalField; +import io.trino.sql.tree.IntervalLiteral; import io.trino.sql.tree.IsNullPredicate; import io.trino.sql.tree.Limit; import io.trino.sql.tree.LogicalExpression; @@ -98,6 +101,7 @@ import io.trino.sql.tree.SelectItem; import io.trino.sql.tree.SimpleCaseExpression; import io.trino.sql.tree.SimpleGroupBy; +import io.trino.sql.tree.SimpleIntervalQualifier; import io.trino.sql.tree.SingleColumn; import io.trino.sql.tree.SortItem.NullOrdering; import io.trino.sql.tree.SortItem.Ordering; @@ -114,6 +118,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; @@ -335,6 +340,7 @@ private static Expression createExpression(HogQlQuery.Expression expression, Map case InCohortExpression in -> throw unsupportedSemanticExpression(in.span(), "HogQL IN COHORT requires a semantic catalog snapshot"); case InExpression in -> createInExpression(in, parameterIds); case InSubqueryExpression in -> createInSubqueryExpression(in, parameterIds); + case IntervalExpression interval -> createIntervalExpression(interval, parameterIds); case IsNullExpression isNull -> new Predicated( location(isNull.predicateSpan()), createExpression(isNull.value(), parameterIds), @@ -362,6 +368,34 @@ private static Expression createExpression(HogQlQuery.Expression expression, Map }; } + private static Expression createIntervalExpression(IntervalExpression interval, Map parameterIds) + { + NodeLocation location = location(interval.span()); + int multiplier = switch (interval.unit()) { + case WEEK -> 7; + case QUARTER -> 3; + default -> 1; + }; + IntervalField field = switch (interval.unit()) { + case SECOND -> new IntervalField.Second(OptionalInt.empty()); + case MINUTE -> new IntervalField.Minute(); + case HOUR -> new IntervalField.Hour(); + case DAY, WEEK -> new IntervalField.Day(); + case MONTH, QUARTER -> new IntervalField.Month(); + case YEAR -> new IntervalField.Year(); + }; + IntervalLiteral unitInterval = new IntervalLiteral( + location, + Integer.toString(multiplier), + IntervalLiteral.Sign.POSITIVE, + new SimpleIntervalQualifier(location, OptionalInt.empty(), field)); + return new ArithmeticBinaryExpression( + location, + ArithmeticBinaryExpression.Operator.MULTIPLY, + createExpression(interval.value(), parameterIds), + unitInterval); + } + private static io.trino.sql.tree.DataType createCastType(Identifier type, CastTypeDialect typeDialect) { if (typeDialect == CastTypeDialect.TRINO) { diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index 306c24a9438d..78230473667d 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -116,6 +116,26 @@ public void testLowersCollectionAndPredicateExpressions(String hogql, String tri assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(trinoSql)); } + @ParameterizedTest + @CsvSource(delimiter = '|', textBlock = """ + SELECT INTERVAL 2 SECOND | SELECT 2 * INTERVAL '1' SECOND + SELECT INTERVAL value MINUTE | SELECT value * INTERVAL '1' MINUTE + SELECT INTERVAL 2 HOUR | SELECT 2 * INTERVAL '1' HOUR + SELECT INTERVAL 2 DAY | SELECT 2 * INTERVAL '1' DAY + SELECT INTERVAL 2 WEEK | SELECT 2 * INTERVAL '7' DAY + SELECT INTERVAL 2 MONTH | SELECT 2 * INTERVAL '1' MONTH + SELECT INTERVAL 2 QUARTER | SELECT 2 * INTERVAL '3' MONTH + SELECT INTERVAL value YEAR | SELECT value * INTERVAL '1' YEAR + SELECT INTERVAL '5 months' | SELECT 5 * INTERVAL '1' MONTH + """) + public void testLowersCanonicalIntervals(String hogql, String trinoSql) + { + Statement statement = compiler.compile(hogql); + + assertThat(statement).isEqualTo(sqlParser.createStatement(trinoSql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + @Test public void testBindsPlaceholdersInsideCollectionSubscriptsBySourceOrder() { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index b829f064fe92..596a79c692f9 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -40,6 +40,8 @@ import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; import io.trino.hogql.parser.tree.HogQlQuery.InExpression; import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalUnit; import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; import io.trino.hogql.parser.tree.HogQlQuery.JoinCriteria; import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; @@ -1103,6 +1105,15 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) if (context instanceof HogQLParser.ColumnExprTryCastContext cast) { return new CastExpression(buildExpression(cast.columnExpr()), buildType(cast.columnTypeExpr()), true, CastTypeDialect.HOGQL, sourceSpan(cast)); } + if (context instanceof HogQLParser.ColumnExprIntervalContext interval) { + return new IntervalExpression( + buildExpression(interval.columnExpr()), + buildIntervalUnit(interval.interval().getText(), interval.interval()), + sourceSpan(interval)); + } + if (context instanceof HogQLParser.ColumnExprIntervalStringContext interval) { + return buildStringInterval(interval); + } if (context instanceof HogQLParser.ColumnExprIdentifierContext identifier) { return buildColumnReference(identifier.columnIdentifier()); } @@ -1249,6 +1260,56 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) throw unsupported(context, "expression " + context.getClass().getSimpleName()); } + private IntervalExpression buildStringInterval(HogQLParser.ColumnExprIntervalStringContext context) + { + String value = decodeQuoted(context.STRING_LITERAL().getText()); + int separator = value.indexOf(' '); + if (separator < 0) { + throw intervalError(context, "Unsupported interval type: must be in the format ' '"); + } + + String count = value.substring(0, separator); + String unit = value.substring(separator + 1); + if (!count.matches("[0-9]+")) { + throw intervalError(context, "Unsupported interval count: '" + count + "' is not a valid integer"); + } + try { + Long.parseLong(count); + } + catch (NumberFormatException _) { + throw intervalError(context, "Unsupported interval count: '" + count + "' is too large"); + } + + String singularUnit = unit.endsWith("s") ? unit.substring(0, unit.length() - 1) : unit; + IntervalUnit intervalUnit; + try { + intervalUnit = IntervalUnit.valueOf(singularUnit.toUpperCase(Locale.ENGLISH)); + } + catch (IllegalArgumentException _) { + throw intervalError(context, "Unsupported interval unit: " + unit); + } + if (!unit.equals(singularUnit) && !unit.equals(singularUnit + "s")) { + throw intervalError(context, "Unsupported interval unit: " + unit); + } + if (!unit.equals(unit.toLowerCase(Locale.ENGLISH))) { + throw intervalError(context, "Unsupported interval unit: " + unit); + } + return new IntervalExpression( + new Literal(INTEGER, normalizeInteger(count), sourceSpan(context.STRING_LITERAL().getSymbol(), context.STRING_LITERAL().getSymbol())), + intervalUnit, + sourceSpan(context)); + } + + private IntervalUnit buildIntervalUnit(String value, ParserRuleContext context) + { + try { + return IntervalUnit.valueOf(value.toUpperCase(Locale.ENGLISH)); + } + catch (IllegalArgumentException _) { + throw intervalError(context, "Unsupported interval unit: " + value); + } + } + private CaseExpression buildCaseExpression(HogQLParser.ColumnExprCaseContext context) { List expressions = context.columnExpr(); @@ -1750,6 +1811,7 @@ private void validateExpressionScope(Expression expression, Set forbidde nestedForbiddenRelations.addAll(localRelations); validateQueryScope(in.query(), Set.copyOf(nestedForbiddenRelations)); } + case IntervalExpression interval -> validateExpressionScope(interval.value(), forbiddenOuterRelations, localRelations); case IsNullExpression isNull -> validateExpressionScope(isNull.value(), forbiddenOuterRelations, localRelations); case Literal _, Placeholder _ -> {} case MemberAccessExpression memberAccess -> validateExpressionScope(memberAccess.base(), forbiddenOuterRelations, localRelations); @@ -1865,6 +1927,12 @@ private HogQlParsingException unsupported(SourceSpan span, String feature) return new HogQlParsingException("HogQL feature is not lowered yet: " + feature, null, span.startLine(), span.startColumn()); } + private HogQlParsingException intervalError(ParserRuleContext context, String message) + { + SourceSpan span = sourceSpan(context); + return new HogQlParsingException(message, null, span.startLine(), span.startColumn()); + } + private static String normalizeInteger(String value) { int firstNonZero = 0; diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index 5990e9a3f1ca..bf82d3b3d541 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -305,6 +305,7 @@ public sealed interface Expression InCohortExpression, InExpression, InSubqueryExpression, + IntervalExpression, IsNullExpression, Literal, MemberAccessExpression, @@ -364,6 +365,29 @@ public CastExpression(Expression value, Identifier type, boolean safe, SourceSpa } } + public enum IntervalUnit + { + SECOND, + MINUTE, + HOUR, + DAY, + WEEK, + MONTH, + QUARTER, + YEAR, + } + + public record IntervalExpression(Expression value, IntervalUnit unit, SourceSpan span) + implements Expression + { + public IntervalExpression + { + value = requireNonNull(value, "value is null"); + unit = requireNonNull(unit, "unit is null"); + span = requireNonNull(span, "span is null"); + } + } + public record ArrayExpression(List values, SourceSpan span) implements Expression { diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json index a8bd21e615b6..c4868978b0ad 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json @@ -930,6 +930,24 @@ "testCaseIds": [ "TestHogQlWindowCompiler#testLowersCanonicalWindowNullTreatmentOrder" ] + }, + { + "id": "alternative:columnExprValue:ColumnExprInterval", + "loweringHandler": "TrinoAstFactory#createIntervalExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCanonicalIntervals", + "TestHogQlParser#testBuildsCanonicalIntervalExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIntervalString", + "loweringHandler": "TrinoAstFactory#createIntervalExpression", + "parseHandler": "AstBuilder#buildStringInterval", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCanonicalIntervals", + "TestHogQlParser#testRejectsInvalidCombinedStringIntervals" + ] } ], "schemaVersion": 1 diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json index 2d52c1795d90..a62ebf43b88c 100644 --- a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json @@ -461,23 +461,25 @@ { "id": "alternative:columnExprValue:ColumnExprInterval", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createIntervalExpression", + "parseHandler": "AstBuilder#buildExpression", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersCanonicalIntervals", + "TestHogQlParser#testBuildsCanonicalIntervalExpressions" ] }, { "id": "alternative:columnExprValue:ColumnExprIntervalString", "kind": "parserAlternative", - "loweringHandler": "UnsupportedFeature", - "parseHandler": "CanonicalAntlrParser", + "loweringHandler": "TrinoAstFactory#createIntervalExpression", + "parseHandler": "AstBuilder#buildStringInterval", "queryReachable": true, - "status": "explicitCurrentError", + "status": "supported", "testCaseIds": [ - "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + "TestHogQlCompiler#testLowersCanonicalIntervals", + "TestHogQlParser#testRejectsInvalidCombinedStringIntervals" ] }, { diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 28b2aad77a33..f1338b4137f2 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -23,9 +23,12 @@ import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalUnit; import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; import io.trino.hogql.parser.tree.HogQlQuery.JoinType; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; @@ -108,6 +111,40 @@ public void testUsesCanonicalGrammarForExpressionsAndWhereClause() assertThat(query.where()).isPresent(); } + @Test + public void testBuildsCanonicalIntervalExpressions() + { + HogQlQuery query = parser.parseStatement("SELECT INTERVAL 1 WEEK, INTERVAL event QUARTER, INTERVAL '5 months'"); + + assertThat(query.projections()).extracting(projection -> ((IntervalExpression) ((ExpressionProjection) projection).expression()).unit()) + .containsExactly(IntervalUnit.WEEK, IntervalUnit.QUARTER, IntervalUnit.MONTH); + IntervalExpression stringInterval = (IntervalExpression) ((ExpressionProjection) query.projections().get(2)).expression(); + assertThat(stringInterval.value()).isInstanceOfSatisfying(Literal.class, value -> { + assertThat(value.kind()).isEqualTo(HogQlQuery.LiteralKind.INTEGER); + assertThat(value.value()).isEqualTo("5"); + }); + } + + @Test + public void testRejectsInvalidCombinedStringIntervals() + { + assertThatThrownBy(() -> parser.parseStatement("SELECT INTERVAL 'twenty days'")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("Unsupported interval count: 'twenty' is not a valid integer"); + assertThatThrownBy(() -> parser.parseStatement("SELECT INTERVAL '9223372036854775808 day'")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("Unsupported interval count: '9223372036854775808' is too large"); + assertThatThrownBy(() -> parser.parseStatement("SELECT INTERVAL '1 SECOND'")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("Unsupported interval unit: SECOND"); + assertThatThrownBy(() -> parser.parseStatement("SELECT INTERVAL '1 dayss'")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("Unsupported interval unit: dayss"); + assertThatThrownBy(() -> parser.parseStatement("SELECT INTERVAL 'x'")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("Unsupported interval type: must be in the format ' '"); + } + @Test public void testPreservesNamedPlaceholderNamesAndSourceSpans() { diff --git a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java index 4dd52a41abf3..6f0686a3a18f 100644 --- a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java +++ b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java @@ -229,7 +229,13 @@ private static Stream differentialQueries() Arguments.of( "timestamp precision", "SELECT CAST('2024-03-10 01:59:59.123456' AS Timestamp(6)) AS value", - "SELECT CAST('2024-03-10 01:59:59.123456' AS timestamp(6)) AS value")); + "SELECT CAST('2024-03-10 01:59:59.123456' AS timestamp(6)) AS value"), + Arguments.of( + "interval expressions", + "SELECT INTERVAL 2 WEEK AS weeks, INTERVAL '3 months' AS months, INTERVAL nationkey DAY AS dynamic " + + "FROM tpch.tiny.nation WHERE nationkey = 2", + "SELECT 2 * INTERVAL '7' DAY AS weeks, 3 * INTERVAL '1' MONTH AS months, nationkey * INTERVAL '1' DAY AS dynamic " + + "FROM tpch.tiny.nation WHERE nationkey = 2")); } @Test From 35ca1550cd7f05bc1b7f4ed94f9d86c8eb66b4b2 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 16:38:29 -0400 Subject: [PATCH 066/136] feat(hogql): compile correlated in subqueries --- .../trino/hogql/compiler/TrinoAstFactory.java | 2 +- .../hogql/compiler/TestHogQlCompiler.java | 27 ++++++++ .../io/trino/hogql/parser/HogQlParser.java | 15 ++++- .../trino/hogql/parser/TestHogQlParser.java | 25 ++++++++ .../planner/TestHogQlCorrelatedSubquery.java | 63 +++++++++++++++++++ 5 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 72022a07ce57..34fc7b780cbf 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -555,7 +555,7 @@ private static Expression createInSubqueryExpression(InSubqueryExpression in, Ma new InPredicate( location, in.negated(), - new io.trino.sql.tree.SubqueryExpression(location, createQuery(in.query(), parameterIds)))); + new io.trino.sql.tree.SubqueryExpression(location(in.query().span()), createQuery(in.query(), parameterIds)))); } private static Expression createBinaryExpression(BinaryExpression binary, Map parameterIds) diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index 78230473667d..a3514aef0b2f 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -21,6 +21,7 @@ import io.trino.sql.tree.AliasedRelation; import io.trino.sql.tree.AllColumns; import io.trino.sql.tree.Identifier; +import io.trino.sql.tree.InPredicate; import io.trino.sql.tree.Join; import io.trino.sql.tree.JoinOn; import io.trino.sql.tree.Node; @@ -31,6 +32,7 @@ import io.trino.sql.tree.QuerySpecification; import io.trino.sql.tree.SingleColumn; import io.trino.sql.tree.Statement; +import io.trino.sql.tree.SubqueryExpression; import io.trino.sql.tree.SubscriptExpression; import io.trino.sql.tree.Table; import io.trino.sql.tree.TableSubquery; @@ -279,6 +281,31 @@ public void testLowersCtesAndDerivedTables(String hogql) assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); } + @Test + public void testLowersCorrelatedInSubqueryWithSourceLocations() + { + String hogql = + """ + SELECT o.orderkey + FROM orders o + WHERE o.custkey IN ( + SELECT c.custkey + FROM customer c + WHERE c.custkey = o.custkey + ) + """; + + Query query = (Query) compiler.compile(hogql); + Predicated where = (Predicated) ((QuerySpecification) query.getQueryBody()).getWhere().orElseThrow(); + InPredicate predicate = (InPredicate) where.getPredicate(); + SubqueryExpression subquery = (SubqueryExpression) predicate.getValueList(); + + assertThat(query).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(where.getLocation()).contains(new NodeLocation(3, 17)); + assertThat(subquery.getLocation()).contains(new NodeLocation(4, 5)); + assertThat(subquery.getQuery().getLocation()).contains(new NodeLocation(4, 5)); + } + @Test public void testBindsPlaceholdersAcrossCteAndDerivedQueryScopes() { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 596a79c692f9..720b1a659ed3 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -1173,6 +1173,17 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) } if (context instanceof HogQLParser.ColumnExprPrecedence3Context binary) { if (binary.IN() != null) { + if (binary.right instanceof HogQLParser.ColumnExprSubqueryContext subquery) { + if (binary.COHORT() != null) { + throw unsupported(binary, "IN COHORT subquery"); + } + return new InSubqueryExpression( + buildExpression(binary.left), + buildSetQuery(subquery.selectSetStmt()), + binary.NOT() != null, + sourceSpan(binary.NOT() == null ? binary.IN().getSymbol() : binary.NOT().getSymbol(), binary.getStop()), + sourceSpan(binary)); + } if (binary.COHORT() != null) { return new InCohortExpression( buildExpression(binary.left), @@ -1807,9 +1818,7 @@ private void validateExpressionScope(Expression expression, Set forbidde } case InSubqueryExpression in -> { validateExpressionScope(in.value(), forbiddenOuterRelations, localRelations); - Set nestedForbiddenRelations = new LinkedHashSet<>(forbiddenOuterRelations); - nestedForbiddenRelations.addAll(localRelations); - validateQueryScope(in.query(), Set.copyOf(nestedForbiddenRelations)); + validateQueryScope(in.query(), Set.of()); } case IntervalExpression interval -> validateExpressionScope(interval.value(), forbiddenOuterRelations, localRelations); case IsNullExpression isNull -> validateExpressionScope(isNull.value(), forbiddenOuterRelations, localRelations); diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index f1338b4137f2..ed12ebdb0ae4 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -23,6 +23,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; import io.trino.hogql.parser.tree.HogQlQuery.IntervalUnit; import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; @@ -331,6 +332,30 @@ public void testBuildsCteAndDerivedTableAstWithSourceSpans() assertThat(subquery.span()).isEqualTo(new HogQlQuery.SourceSpan(80, 120, 2, 18, 2, 58)); } + @Test + public void testBuildsCorrelatedInSubqueryWithSourceSpans() + { + HogQlQuery query = parser.parseStatement( + """ + SELECT o.orderkey + FROM orders o + WHERE o.custkey IN ( + SELECT c.custkey + FROM customer c + WHERE c.custkey = o.custkey + ) + """); + + InSubqueryExpression predicate = (InSubqueryExpression) query.where().orElseThrow(); + BinaryExpression correlation = (BinaryExpression) predicate.query().where().orElseThrow(); + ColumnReference outerReference = (ColumnReference) correlation.right(); + + assertThat(predicate.predicateSpan()).isEqualTo(new HogQlQuery.SourceSpan(48, 127, 3, 17, 7, 2)); + assertThat(predicate.query().span()).isEqualTo(new HogQlQuery.SourceSpan(57, 125, 4, 5, 6, 32)); + assertThat(outerReference.parts()).extracting(HogQlQuery.Identifier::value).containsExactly("o", "custkey"); + assertThat(outerReference.span()).isEqualTo(new HogQlQuery.SourceSpan(116, 125, 6, 23, 6, 32)); + } + @Test public void testBuildsSetOperationAstWithPrecedenceAndSourceSpans() { diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java new file mode 100644 index 000000000000..71f3fa590d3d --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.sql.planner; + +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.spi.TrinoException; +import io.trino.sql.SqlFormatter; +import io.trino.sql.planner.assertions.BasePlanTest; +import org.junit.jupiter.api.Test; + +import static io.trino.spi.StandardErrorCode.COLUMN_NOT_FOUND; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlCorrelatedSubquery + extends BasePlanTest +{ + private final HogQlCompiler compiler = new HogQlCompiler(); + + @Test + public void testAnalyzerBindsCorrelatedOuterColumn() + { + assertThatCode(() -> plan(compile( + "SELECT o.orderkey FROM orders o " + + "WHERE o.custkey IN (" + + "SELECT c.custkey FROM customer c WHERE c.custkey = o.custkey)"))) + .doesNotThrowAnyException(); + } + + @Test + public void testAnalyzerRejectsMissingCorrelatedOuterColumn() + { + AssertionError failure = catchThrowableOfType( + AssertionError.class, + () -> plan(compile( + "SELECT o.orderkey FROM orders o " + + "WHERE o.custkey IN (" + + "SELECT c.custkey FROM customer c WHERE c.custkey = missing.custkey)"))); + + assertThat(failure.getCause()) + .isInstanceOfSatisfying(TrinoException.class, cause -> { + assertThat(cause.getErrorCode()).isEqualTo(COLUMN_NOT_FOUND.toErrorCode()); + assertThat(cause).hasMessageContaining("Column 'missing.custkey' cannot be resolved"); + }); + } + + private String compile(String hogql) + { + return SqlFormatter.formatSql(compiler.compile(hogql)); + } +} From f962704a79a862a56928fd4e34972f31eadbde89 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 16:46:18 -0400 Subject: [PATCH 067/136] feat(hogql): compile scalar subqueries --- .../io/trino/hogql/compiler/HogQlCompiler.java | 3 +++ .../hogql/compiler/HogQlFunctionResolver.java | 2 ++ .../hogql/compiler/HogQlProjectionDemand.java | 2 ++ .../hogql/compiler/HogQlSemanticResolver.java | 4 ++++ .../trino/hogql/compiler/TrinoAstFactory.java | 4 ++++ .../trino/hogql/compiler/TestHogQlCompiler.java | 14 ++++++++++++++ .../java/io/trino/hogql/parser/HogQlParser.java | 5 +++++ .../io/trino/hogql/parser/tree/HogQlQuery.java | 11 +++++++++++ .../io/trino/hogql/parser/TestHogQlParser.java | 17 +++++++++++++++++ .../planner/TestHogQlCorrelatedSubquery.java | 10 ++++++++++ 10 files changed, 72 insertions(+) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index 26bcbdda7387..e55accf1f006 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -47,6 +47,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; @@ -345,6 +346,7 @@ private static boolean containsFunctionCall(Expression expression) case IntervalExpression interval -> containsFunctionCall(interval.value()); case IsNullExpression isNull -> containsFunctionCall(isNull.value()); case MemberAccessExpression memberAccess -> containsFunctionCall(memberAccess.base()); + case ScalarSubqueryExpression subquery -> containsSemanticCandidate(subquery.query()); case SubscriptExpression subscript -> containsFunctionCall(subscript.base()) || containsFunctionCall(subscript.index()); case TupleExpression tuple -> tuple.values().stream().anyMatch(HogQlCompiler::containsFunctionCall); case UnaryExpression unary -> containsFunctionCall(unary.operand()); @@ -453,6 +455,7 @@ private static void collectPlaceholders(Expression expression, List case Literal _ -> {} case MemberAccessExpression memberAccess -> collectPlaceholders(memberAccess.base(), placeholders); case Placeholder placeholder -> placeholders.add(placeholder); + case ScalarSubqueryExpression subquery -> collectPlaceholders(subquery.query(), placeholders); case SubscriptExpression subscript -> { collectPlaceholders(subscript.base(), placeholders); collectPlaceholders(subscript.index(), placeholders); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 1356301f5678..e81dd66a8453 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -50,6 +50,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; import io.trino.hogql.parser.tree.HogQlQuery.SortItem; @@ -242,6 +243,7 @@ private Expression resolveExpression(Expression expression) memberAccess.member(), memberAccess.span()); case Placeholder placeholder -> placeholder; + case ScalarSubqueryExpression subquery -> new ScalarSubqueryExpression(resolveQuery(subquery.query()), subquery.span()); case SubscriptExpression subscript -> new SubscriptExpression( resolveExpression(subscript.base()), resolveExpression(subscript.index()), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java index 594aa33c3b6b..7f6aba64b0de 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java @@ -39,6 +39,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; import io.trino.hogql.parser.tree.HogQlQuery.Star; import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; @@ -193,6 +194,7 @@ private static void collect(Expression expression, Builder builder) case IsNullExpression isNull -> collect(isNull.value(), builder); case Literal _, Placeholder _ -> {} case MemberAccessExpression member -> collect(member.base(), builder); + case ScalarSubqueryExpression _ -> builder.all = true; case SubscriptExpression subscript -> { collect(subscript.base(), builder); collect(subscript.index(), builder); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index ef738571c4e9..54c303b2e5be 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -79,6 +79,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; import io.trino.hogql.parser.tree.HogQlQuery.SortItem; @@ -868,6 +869,9 @@ private Expression resolveExpression(Expression expression) case Literal literal -> literal; case MemberAccessExpression memberAccess -> resolveMemberAccess(memberAccess); case Placeholder placeholder -> placeholder; + case ScalarSubqueryExpression subquery -> new ScalarSubqueryExpression( + new HogQlSemanticResolver(snapshot, expansionBudget).resolveNestedQuery(subquery.query()), + subquery.span()); case SubscriptExpression subscript -> resolveSubscript(subscript); case TupleExpression tuple -> new TupleExpression(tuple.values().stream().map(this::resolveExpression).toList(), tuple.span()); case UnaryExpression unary -> new UnaryExpression(unary.operator(), resolveExpression(unary.operand()), unary.span()); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 34fc7b780cbf..68d0e468696b 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -43,6 +43,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; @@ -351,6 +352,9 @@ private static Expression createExpression(HogQlQuery.Expression expression, Map createExpression(memberAccess.base(), parameterIds), createIdentifier(memberAccess.member())); case Placeholder placeholder -> new Parameter(location(placeholder.span()), parameterIds.get(placeholder.span())); + case ScalarSubqueryExpression subquery -> new io.trino.sql.tree.SubqueryExpression( + location(subquery.span()), + createQuery(subquery.query(), parameterIds)); case SubscriptExpression subscript -> new io.trino.sql.tree.SubscriptExpression( location(subscript.span()), createExpression(subscript.base(), parameterIds), diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index a3514aef0b2f..300990a0f42e 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -306,6 +306,20 @@ WHERE o.custkey IN ( assertThat(subquery.getQuery().getLocation()).contains(new NodeLocation(4, 5)); } + @Test + public void testLowersCorrelatedScalarSubqueryWithSourceLocations() + { + String hogql = "SELECT o.orderkey, (SELECT c.custkey FROM customer c WHERE c.custkey = o.custkey) AS matched FROM orders o"; + + Query query = (Query) compiler.compile(hogql); + SingleColumn projection = (SingleColumn) ((QuerySpecification) query.getQueryBody()).getSelect().getSelectItems().get(1); + SubqueryExpression subquery = (SubqueryExpression) projection.getExpression(); + + assertThat(query).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(subquery.getLocation()).contains(new NodeLocation(1, 20)); + assertThat(subquery.getQuery().getLocation()).contains(new NodeLocation(1, 21)); + } + @Test public void testBindsPlaceholdersAcrossCteAndDerivedQueryScopes() { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 720b1a659ed3..0df316641d60 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -55,6 +55,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; import io.trino.hogql.parser.tree.HogQlQuery.SetOperationType; import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; @@ -1117,6 +1118,9 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) if (context instanceof HogQLParser.ColumnExprIdentifierContext identifier) { return buildColumnReference(identifier.columnIdentifier()); } + if (context instanceof HogQLParser.ColumnExprSubqueryContext subquery) { + return new ScalarSubqueryExpression(buildSetQuery(subquery.selectSetStmt()), sourceSpan(subquery)); + } if (context instanceof HogQLParser.ColumnExprParensContext parens) { return buildExpression(parens.columnExpr()); } @@ -1824,6 +1828,7 @@ private void validateExpressionScope(Expression expression, Set forbidde case IsNullExpression isNull -> validateExpressionScope(isNull.value(), forbiddenOuterRelations, localRelations); case Literal _, Placeholder _ -> {} case MemberAccessExpression memberAccess -> validateExpressionScope(memberAccess.base(), forbiddenOuterRelations, localRelations); + case ScalarSubqueryExpression subquery -> validateQueryScope(subquery.query(), Set.of()); case SubscriptExpression subscript -> { validateExpressionScope(subscript.base(), forbiddenOuterRelations, localRelations); validateExpressionScope(subscript.index(), forbiddenOuterRelations, localRelations); diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index bf82d3b3d541..e531da048bf9 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -310,6 +310,7 @@ public sealed interface Expression Literal, MemberAccessExpression, Placeholder, + ScalarSubqueryExpression, SubscriptExpression, TupleExpression, UnaryExpression @@ -485,6 +486,16 @@ public record InSubqueryExpression(Expression value, HogQlQuery query, boolean n } } + public record ScalarSubqueryExpression(HogQlQuery query, SourceSpan span) + implements Expression + { + public ScalarSubqueryExpression + { + query = requireNonNull(query, "query is null"); + span = requireNonNull(span, "span is null"); + } + } + public record IsNullExpression(Expression value, boolean negated, SourceSpan predicateSpan, SourceSpan span) implements Expression { diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index ed12ebdb0ae4..698b8fbfd195 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -33,6 +33,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; import io.trino.hogql.parser.tree.HogQlQuery.SetOperationType; import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; @@ -356,6 +357,22 @@ WHERE o.custkey IN ( assertThat(outerReference.span()).isEqualTo(new HogQlQuery.SourceSpan(116, 125, 6, 23, 6, 32)); } + @Test + public void testBuildsCorrelatedScalarSubqueryWithSourceSpans() + { + HogQlQuery query = parser.parseStatement( + "SELECT o.orderkey, (SELECT c.custkey FROM customer c WHERE c.custkey = o.custkey) AS matched FROM orders o"); + + ScalarSubqueryExpression subquery = (ScalarSubqueryExpression) ((ExpressionProjection) query.projections().get(1)).expression(); + BinaryExpression correlation = (BinaryExpression) subquery.query().where().orElseThrow(); + ColumnReference outerReference = (ColumnReference) correlation.right(); + + assertThat(subquery.span()).isEqualTo(new HogQlQuery.SourceSpan(19, 81, 1, 20, 1, 82)); + assertThat(subquery.query().span()).isEqualTo(new HogQlQuery.SourceSpan(20, 80, 1, 21, 1, 81)); + assertThat(outerReference.parts()).extracting(HogQlQuery.Identifier::value).containsExactly("o", "custkey"); + assertThat(outerReference.span()).isEqualTo(new HogQlQuery.SourceSpan(71, 80, 1, 72, 1, 81)); + } + @Test public void testBuildsSetOperationAstWithPrecedenceAndSourceSpans() { diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java index 71f3fa590d3d..dcb9390315eb 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java @@ -39,6 +39,16 @@ public void testAnalyzerBindsCorrelatedOuterColumn() .doesNotThrowAnyException(); } + @Test + public void testAnalyzerBindsCorrelatedOuterColumnInScalarSubquery() + { + assertThatCode(() -> plan(compile( + "SELECT o.orderkey, (" + + "SELECT c.custkey FROM customer c WHERE c.custkey = o.custkey) " + + "FROM orders o"))) + .doesNotThrowAnyException(); + } + @Test public void testAnalyzerRejectsMissingCorrelatedOuterColumn() { From 7d16279cb537a999f0a9b42ceb1e57729502e5b9 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 16:55:55 -0400 Subject: [PATCH 068/136] feat(hogql): resolve semantic correlations --- .../hogql/compiler/HogQlSemanticResolver.java | 150 ++++++++++++++---- .../compiler/TestHogQlSemanticResolution.java | 58 +++++++ .../planner/TestHogQlCorrelatedSubquery.java | 40 +++++ .../planner/TestHogQlProjectionPruning.java | 4 +- 4 files changed, 222 insertions(+), 30 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index 54c303b2e5be..e6f8fa135e77 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -125,8 +125,10 @@ final class HogQlSemanticResolver private final PinnedSnapshot snapshot; private final ExpansionBudget expansionBudget; private final Optional requiredOutputs; + private final List outerBindingScopes; private final Map relationshipPaths = new LinkedHashMap<>(); private List bindings = List.of(); + private Set localRelationQualifiers = Set.of(); private boolean allRelationsLogical; private Optional expandedRelation = Optional.empty(); private int generatedRelationId; @@ -142,10 +144,20 @@ private HogQlSemanticResolver(PinnedSnapshot snapshot, ExpansionBudget expansion } private HogQlSemanticResolver(PinnedSnapshot snapshot, ExpansionBudget expansionBudget, Optional requiredOutputs) + { + this(snapshot, expansionBudget, requiredOutputs, List.of()); + } + + private HogQlSemanticResolver( + PinnedSnapshot snapshot, + ExpansionBudget expansionBudget, + Optional requiredOutputs, + List outerBindingScopes) { this.snapshot = requireNonNull(snapshot, "snapshot is null"); this.expansionBudget = requireNonNull(expansionBudget, "expansionBudget is null"); this.requiredOutputs = requireNonNull(requiredOutputs, "requiredOutputs is null"); + this.outerBindingScopes = List.copyOf(requireNonNull(outerBindingScopes, "outerBindingScopes is null")); } public static Optional resolve(PinnedSnapshot snapshot, HogQlQuery query) @@ -170,8 +182,8 @@ private HogQlQuery resolveNestedQuery(HogQlQuery query) SetOperation resolved = new SetOperation( setOperation.type(), setOperation.distinct(), - new HogQlSemanticResolver(snapshot, expansionBudget).resolveNestedQuery(setOperation.left()), - new HogQlSemanticResolver(snapshot, expansionBudget).resolveNestedQuery(setOperation.right()), + new HogQlSemanticResolver(snapshot, expansionBudget, Optional.empty(), outerBindingScopes).resolveNestedQuery(setOperation.left()), + new HogQlSemanticResolver(snapshot, expansionBudget, Optional.empty(), outerBindingScopes).resolveNestedQuery(setOperation.right()), setOperation.leftParenthesized(), setOperation.rightParenthesized(), setOperation.operatorSpan(), @@ -179,12 +191,13 @@ private HogQlQuery resolveNestedQuery(HogQlQuery query) return new HogQlQuery(commonTables, resolved, query.orderBy(), query.limit(), query.offset(), query.span()); } SelectQueryBody select = (SelectQueryBody) query.body(); + localRelationQualifiers = select.from().map(HogQlSemanticResolver::relationQualifiers).orElse(Set.of()); HogQlProjectionDemand projectionDemand = HogQlProjectionDemand.collect(query); Optional relation = select.from().map(value -> resolveRelation(value, projectionDemand)); bindings = relation.map(ResolvedRelation::bindings).orElse(List.of()); allRelationsLogical = relation.map(ResolvedRelation::allLogical).orElse(false); expandedRelation = relation.map(ResolvedRelation::relation); - if (bindings.isEmpty()) { + if (bindings.isEmpty() && outerBindingScopes.isEmpty()) { return new HogQlQuery( commonTables, select.distinct(), @@ -528,6 +541,23 @@ private static String identifierPath(List identifiers) return String.join(".", identifiers.stream().map(Identifier::value).toList()); } + private static Set relationQualifiers(Relation relation) + { + return switch (relation) { + case AliasedRelation alias -> Set.of(canonical(alias.alias().value())); + case CommonTableReference commonTable -> Set.of(canonical(commonTable.name().value())); + case JoinRelation join -> { + Set qualifiers = new HashSet<>(relationQualifiers(join.left())); + qualifiers.addAll(relationQualifiers(join.right())); + yield Set.copyOf(qualifiers); + } + case SubqueryRelation _ -> Set.of(); + case TablePlaceholder _ -> Set.of(); + case TableReference table -> Set.of(canonical(table.parts().getLast().value())); + case ValuesRelation _ -> Set.of(); + }; + } + private ResolvedRelation resolveRelation(Relation relation) { return resolveRelation(relation, HogQlProjectionDemand.preserveAll()); @@ -821,6 +851,30 @@ private List resolveSortItems(List sortItems) .toList(); } + private HogQlSemanticResolver correlatedSubqueryResolver() + { + List visibleOuterScopes = new ArrayList<>(); + currentBindingScope().ifPresent(visibleOuterScopes::add); + visibleOuterScopes.addAll(outerBindingScopes); + return new HogQlSemanticResolver(snapshot, expansionBudget, Optional.empty(), visibleOuterScopes); + } + + private Optional currentBindingScope() + { + if (bindings.isEmpty() && localRelationQualifiers.isEmpty()) { + return Optional.empty(); + } + return Optional.of(new BindingScope(bindings, localRelationQualifiers, allRelationsLogical)); + } + + private List visibleBindingScopes() + { + List scopes = new ArrayList<>(); + currentBindingScope().ifPresent(scopes::add); + scopes.addAll(outerBindingScopes); + return List.copyOf(scopes); + } + private Expression resolveExpression(Expression expression) { return switch (expression) { @@ -856,7 +910,7 @@ private Expression resolveExpression(Expression expression) in.span()); case InSubqueryExpression in -> new InSubqueryExpression( resolveExpression(in.value()), - new HogQlSemanticResolver(snapshot, expansionBudget).resolveNestedQuery(in.query()), + correlatedSubqueryResolver().resolveNestedQuery(in.query()), in.negated(), in.predicateSpan(), in.span()); @@ -870,7 +924,7 @@ private Expression resolveExpression(Expression expression) case MemberAccessExpression memberAccess -> resolveMemberAccess(memberAccess); case Placeholder placeholder -> placeholder; case ScalarSubqueryExpression subquery -> new ScalarSubqueryExpression( - new HogQlSemanticResolver(snapshot, expansionBudget).resolveNestedQuery(subquery.query()), + correlatedSubqueryResolver().resolveNestedQuery(subquery.query()), subquery.span()); case SubscriptExpression subscript -> resolveSubscript(subscript); case TupleExpression tuple -> new TupleExpression(tuple.values().stream().map(this::resolveExpression).toList(), tuple.span()); @@ -1103,46 +1157,77 @@ private Expression resolveColumn(ColumnReference reference) return propertyAccess.orElseThrow(); } if (parts.size() == 2) { - Optional binding = bindings.stream() - .filter(candidate -> candidate.qualifier().equals(canonical(parts.getFirst().value()))) - .findFirst(); - if (binding.isEmpty()) { - if (allRelationsLogical) { - throw resolutionError(reference, parts.getLast().value()); + String qualifier = canonical(parts.getFirst().value()); + for (BindingScope scope : visibleBindingScopes()) { + List qualifiedBindings = scope.bindings().stream() + .filter(candidate -> candidate.qualifier().equals(qualifier)) + .toList(); + if (!qualifiedBindings.isEmpty()) { + TableBinding binding = qualifiedBindings.getFirst(); + BoundField field = binding.fields().get(canonical(parts.getLast().value())); + if (field == null) { + throw resolutionError(reference, parts.getLast().value()); + } + return resolveBoundField(binding, field, Optional.of(binding.outputQualifier()), reference.span(), expansionBudget); + } + if (scope.relationQualifiers().contains(qualifier)) { + return reference; } - return reference; } - BoundField field = binding.orElseThrow().fields().get(canonical(parts.getLast().value())); - if (field == null) { + if (innermostScopeIsFullyLogical()) { throw resolutionError(reference, parts.getLast().value()); } - return resolveBoundField(binding.orElseThrow(), field, Optional.of(binding.orElseThrow().outputQualifier()), reference.span(), expansionBudget); + return reference; } if (parts.size() > 2) { - if (allRelationsLogical) { + String qualifier = canonical(parts.getFirst().value()); + if (visibleBindingScopes().stream().anyMatch(scope -> scope.bindings().stream().anyMatch(binding -> binding.qualifier().equals(qualifier))) || + innermostScopeIsFullyLogical()) { throw resolutionError(reference, parts.getLast().value()); } return reference; } String logicalName = parts.getLast().value(); - List matches = bindings.stream() - .map(binding -> new FieldMatch(binding, binding.fields().get(canonical(logicalName)))) - .filter(match -> match.field() != null) - .toList(); - if (matches.size() == 1 && (allRelationsLogical || bindings.size() == 1)) { - FieldMatch match = matches.getFirst(); - Optional qualifier = bindings.size() > 1 ? Optional.of(match.binding().outputQualifier()) : Optional.empty(); - return resolveBoundField(match.binding(), match.field(), qualifier, reference.span(), expansionBudget); - } - if (matches.size() > 1) { - throw ambiguousResolutionError(reference, logicalName); + Optional currentScope = currentBindingScope(); + List scopes = visibleBindingScopes(); + for (int scopeIndex = 0; scopeIndex < scopes.size(); scopeIndex++) { + BindingScope scope = scopes.get(scopeIndex); + List matches = scope.bindings().stream() + .map(binding -> new FieldMatch(binding, binding.fields().get(canonical(logicalName)))) + .filter(match -> match.field() != null) + .toList(); + if (matches.size() > 1) { + throw ambiguousResolutionError(reference, logicalName); + } + if (matches.size() == 1) { + if (!scope.allRelationsLogical() && scope.bindings().size() > 1) { + return reference; + } + FieldMatch match = matches.getFirst(); + boolean outer = currentScope.isEmpty() || scopeIndex > 0; + Optional qualifier = outer || scope.bindings().size() > 1 + ? Optional.of(match.binding().outputQualifier()) + : Optional.empty(); + return resolveBoundField(match.binding(), match.field(), qualifier, reference.span(), expansionBudget); + } + if (!scope.allRelationsLogical() && !scope.relationQualifiers().isEmpty()) { + return reference; + } } - if (allRelationsLogical) { + if (scopes.stream().anyMatch(BindingScope::allRelationsLogical)) { throw resolutionError(reference, logicalName); } return reference; } + private boolean innermostScopeIsFullyLogical() + { + return visibleBindingScopes().stream() + .findFirst() + .map(BindingScope::allRelationsLogical) + .orElse(false); + } + private Optional resolveSemanticPath(ColumnReference reference) { List parts = reference.parts(); @@ -1855,6 +1940,15 @@ private record ResolvedRelation(Relation relation, List bindings, } } + private record BindingScope(List bindings, Set relationQualifiers, boolean allRelationsLogical) + { + private BindingScope + { + bindings = List.copyOf(requireNonNull(bindings, "bindings is null")); + relationQualifiers = Set.copyOf(requireNonNull(relationQualifiers, "relationQualifiers is null")); + } + } + private record TableBinding( String relationName, String qualifier, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java index 6a3b2a8c389d..7f06a345b176 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -454,6 +454,64 @@ public void testResolvesAliasedLogicalTablesAndJoinCriteria() "JOIN analytics.\"Hog Data\".\"raw-persons\" p ON e.\"Person ID\" = p.person_id")); } + @Test + public void testResolvesCorrelatedLogicalFieldsInExpressionSubqueries() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult inSubquery = compiler.compile( + envelope( + "SELECT e.event FROM events e WHERE e.personId IN " + + "(SELECT p.personId FROM persons p WHERE p.personId = e.personId)", + OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult scalarSubquery = compiler.compile( + envelope( + "SELECT e.event, (SELECT p.name FROM persons p WHERE p.personId = e.personId) FROM events e", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(inSubquery.statement()).isEqualTo(sqlParser.createStatement( + "SELECT e.event_name AS event FROM analytics.\"Hog Data\".\"raw-events\" e " + + "WHERE e.\"Person ID\" IN (" + + "SELECT p.person_id AS personId FROM analytics.\"Hog Data\".\"raw-persons\" p " + + "WHERE p.person_id = e.\"Person ID\")")); + assertThat(scalarSubquery.statement()).isEqualTo(sqlParser.createStatement( + "SELECT e.event_name AS event, (" + + "SELECT p.full_name AS name FROM analytics.\"Hog Data\".\"raw-persons\" p " + + "WHERE p.person_id = e.\"Person ID\") FROM analytics.\"Hog Data\".\"raw-events\" e")); + } + + @Test + public void testInnerLogicalQualifierShadowsCorrelatedOuterQualifier() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compile( + envelope("SELECT (SELECT e.name FROM persons e) FROM events e", OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT (SELECT e.full_name AS name FROM analytics.\"Hog Data\".\"raw-persons\" e) " + + "FROM analytics.\"Hog Data\".\"raw-events\" e")); + } + + @Test + public void testRejectsUnknownQualifierInCorrelatedLogicalSubquery() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + String hogql = "SELECT e.event FROM events e WHERE e.personId IN " + + "(SELECT p.personId FROM persons p WHERE p.personId = missing.personId)"; + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope(hogql, OptionalLong.of(7)), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, hogql.indexOf("missing.personId") + 1)); + assertThat(exception).hasMessageContaining("Unknown HogQL field: personId"); + } + @Test public void testRejectsLogicalUsingWhenPhysicalColumnNamesDiffer() { diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java index dcb9390315eb..a42c71089b8d 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java @@ -13,13 +13,24 @@ */ package io.trino.sql.planner; +import io.trino.hogql.compiler.HogQlCompilationResult; +import io.trino.hogql.compiler.HogQlCompileEnvelope; import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.HogQlSemanticCatalogContext; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; import io.trino.spi.TrinoException; import io.trino.sql.SqlFormatter; import io.trino.sql.planner.assertions.BasePlanTest; import org.junit.jupiter.api.Test; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; + import static io.trino.spi.StandardErrorCode.COLUMN_NOT_FOUND; +import static io.trino.testing.TestingHandles.TEST_CATALOG_NAME; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.catchThrowableOfType; @@ -49,6 +60,15 @@ public void testAnalyzerBindsCorrelatedOuterColumnInScalarSubquery() .doesNotThrowAnyException(); } + @Test + public void testAnalyzerBindsResolvedCorrelatedLogicalField() + { + assertThatCode(() -> plan(compileSemantic( + "SELECT e.event FROM events e WHERE e.personId IN (" + + "SELECT p.personId FROM persons p WHERE p.personId = e.personId)"))) + .doesNotThrowAnyException(); + } + @Test public void testAnalyzerRejectsMissingCorrelatedOuterColumn() { @@ -70,4 +90,24 @@ private String compile(String hogql) { return SqlFormatter.formatSql(compiler.compile(hogql)); } + + private String compileSemantic(String hogql) + { + PhysicalIdentifier catalog = new PhysicalIdentifier(TEST_CATALOG_NAME, false); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext( + catalog, + _ -> new PinnedSnapshot(TestHogQlProjectionPruning.testingSnapshot())); + HogQlCompilationResult result = compiler.compile( + new HogQlCompileEnvelope( + hogql, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(1)), + Optional.of(context)); + return SqlFormatter.formatSql(result.statement()); + } } diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java index b58bb7583218..3dfb96f6216a 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java @@ -56,7 +56,7 @@ public class TestHogQlProjectionPruning extends BasePlanTest { private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier(TEST_CATALOG_NAME, false); - private static final HogQlSemanticCatalogSnapshot SNAPSHOT = snapshot(); + private static final HogQlSemanticCatalogSnapshot SNAPSHOT = testingSnapshot(); private final HogQlCompiler compiler = new HogQlCompiler(); @@ -117,7 +117,7 @@ private static List nodes(Plan plan, Class type) .toList(); } - private static HogQlSemanticCatalogSnapshot snapshot() + static HogQlSemanticCatalogSnapshot testingSnapshot() { LogicalTableDefinition events = new LogicalTableDefinition( "events", From 6b5128e70d8d10502ff35938ba1450c2758f8e9f Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 17:08:39 -0400 Subject: [PATCH 069/136] feat(hogql): propagate cte projection demand --- .../hogql/compiler/HogQlProjectionDemand.java | 61 +++ .../hogql/compiler/HogQlSemanticResolver.java | 352 +++++++++++++++++- .../compiler/TestHogQlSemanticResolution.java | 51 +++ .../planner/TestHogQlProjectionPruning.java | 24 ++ 4 files changed, 476 insertions(+), 12 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java index 7f6aba64b0de..b8d2cf6803eb 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java @@ -92,11 +92,46 @@ public static HogQlProjectionDemand collect(HogQlQuery query) return builder.build(); } + public static HogQlProjectionDemand collectNonProjection(HogQlQuery query) + { + Builder builder = new Builder(); + if (query.body() instanceof SelectQueryBody select) { + select.where().ifPresent(expression -> collect(expression, builder)); + select.groupBy().forEach(expression -> collect(expression, builder)); + select.having().ifPresent(expression -> collect(expression, builder)); + select.windows().forEach(window -> collect(window.specification(), builder)); + select.from().ifPresent(relation -> collectJoinCriteria(relation, builder)); + } + query.orderBy().forEach(sortItem -> collect(sortItem.expression(), builder)); + query.limit().ifPresent(expression -> collect(expression, builder)); + query.offset().ifPresent(expression -> collect(expression, builder)); + return builder.build(); + } + + public static HogQlProjectionDemand collect(Expression expression) + { + Builder builder = new Builder(); + collect(expression, builder); + return builder.build(); + } + + public static HogQlProjectionDemand column(String qualifier, String name) + { + Builder builder = new Builder(); + builder.addQualified(qualifier, name); + return builder.build(); + } + public static HogQlProjectionDemand preserveAll() { return new HogQlProjectionDemand(true, Set.of(), Set.of(), Map.of()); } + public static HogQlProjectionDemand preserveNone() + { + return new HogQlProjectionDemand(false, Set.of(), Set.of(), Map.of()); + } + public RequiredOutputs forAlias(Identifier alias) { String qualifier = canonical(alias.value()); @@ -116,6 +151,19 @@ public RequiredOutputs unqualified() return new RequiredOutputs(false, unqualified); } + public HogQlProjectionDemand merge(HogQlProjectionDemand other) + { + requireNonNull(other, "other is null"); + Map> mergedQualified = new HashMap<>(); + qualified.forEach((key, value) -> mergedQualified.put(key, new HashSet<>(value))); + other.qualified.forEach((key, value) -> mergedQualified.computeIfAbsent(key, _ -> new HashSet<>()).addAll(value)); + Set mergedUnqualified = new HashSet<>(unqualified); + mergedUnqualified.addAll(other.unqualified); + Set mergedAllQualifiers = new HashSet<>(allQualifiers); + mergedAllQualifiers.addAll(other.allQualifiers); + return new HogQlProjectionDemand(all || other.all, mergedUnqualified, mergedAllQualifiers, mergedQualified); + } + private static void collect(Projection projection, Builder builder) { switch (projection) { @@ -243,6 +291,14 @@ public boolean includes(String name) { return all || names.contains(canonical(name)); } + + public RequiredOutputs merge(RequiredOutputs other) + { + requireNonNull(other, "other is null"); + Set merged = new HashSet<>(names); + merged.addAll(other.names); + return new RequiredOutputs(all || other.all, merged); + } } private static final class Builder @@ -268,6 +324,11 @@ public void addUnqualified(Identifier identifier) unqualified.add(canonical(identifier.value())); } + public void addQualified(String qualifier, String name) + { + qualified.computeIfAbsent(canonical(qualifier), _ -> new HashSet<>()).add(canonical(name)); + } + public HogQlProjectionDemand build() { return new HogQlProjectionDemand(all, unqualified, allQualifiers, qualified); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index e6f8fa135e77..fd062b78eb17 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -121,11 +121,13 @@ final class HogQlSemanticResolver { private static final String MATCHES_ACTION = "matchesaction"; + private static final String CTE_RELATION_PREFIX = "\0cte:"; private final PinnedSnapshot snapshot; private final ExpansionBudget expansionBudget; private final Optional requiredOutputs; private final List outerBindingScopes; + private final Map commonTableBindings; private final Map relationshipPaths = new LinkedHashMap<>(); private List bindings = List.of(); private Set localRelationQualifiers = Set.of(); @@ -153,11 +155,22 @@ private HogQlSemanticResolver( ExpansionBudget expansionBudget, Optional requiredOutputs, List outerBindingScopes) + { + this(snapshot, expansionBudget, requiredOutputs, outerBindingScopes, Map.of()); + } + + private HogQlSemanticResolver( + PinnedSnapshot snapshot, + ExpansionBudget expansionBudget, + Optional requiredOutputs, + List outerBindingScopes, + Map commonTableBindings) { this.snapshot = requireNonNull(snapshot, "snapshot is null"); this.expansionBudget = requireNonNull(expansionBudget, "expansionBudget is null"); this.requiredOutputs = requireNonNull(requiredOutputs, "requiredOutputs is null"); this.outerBindingScopes = List.copyOf(requireNonNull(outerBindingScopes, "outerBindingScopes is null")); + this.commonTableBindings = Map.copyOf(requireNonNull(commonTableBindings, "commonTableBindings is null")); } public static Optional resolve(PinnedSnapshot snapshot, HogQlQuery query) @@ -171,19 +184,16 @@ public static Optional resolve(PinnedSnapshot snapshot, HogQlQuer private HogQlQuery resolveNestedQuery(HogQlQuery query) { - List commonTables = query.with().stream() - .map(commonTable -> new CommonTableExpression( - commonTable.name(), - commonTable.columnAliases(), - new HogQlSemanticResolver(snapshot, expansionBudget).resolveNestedQuery(commonTable.query()), - commonTable.span())) - .toList(); + if (!query.with().isEmpty()) { + return resolveWithQuery(query); + } + List commonTables = List.of(); if (query.body() instanceof SetOperation setOperation) { SetOperation resolved = new SetOperation( setOperation.type(), setOperation.distinct(), - new HogQlSemanticResolver(snapshot, expansionBudget, Optional.empty(), outerBindingScopes).resolveNestedQuery(setOperation.left()), - new HogQlSemanticResolver(snapshot, expansionBudget, Optional.empty(), outerBindingScopes).resolveNestedQuery(setOperation.right()), + new HogQlSemanticResolver(snapshot, expansionBudget, Optional.empty(), outerBindingScopes, commonTableBindings).resolveNestedQuery(setOperation.left()), + new HogQlSemanticResolver(snapshot, expansionBudget, Optional.empty(), outerBindingScopes, commonTableBindings).resolveNestedQuery(setOperation.right()), setOperation.leftParenthesized(), setOperation.rightParenthesized(), setOperation.operatorSpan(), @@ -215,6 +225,290 @@ private HogQlQuery resolveNestedQuery(HogQlQuery query) return resolveQuery(query, commonTables); } + private HogQlQuery resolveWithQuery(HogQlQuery query) + { + Map inferredBindings = new LinkedHashMap<>(commonTableBindings); + List analyses = new ArrayList<>(); + for (CommonTableExpression commonTable : query.with()) { + CteAnalysis analysis = inferCommonTable(commonTable, inferredBindings); + analyses.add(analysis); + inferredBindings.put(canonical(commonTable.name().value()), analysis.binding()); + } + + HogQlQuery body = new HogQlQuery(List.of(), query.body(), query.orderBy(), query.limit(), query.offset(), query.span()); + Map demands = new HashMap<>(); + collectCommonTableDemands(body, HogQlProjectionDemand.collect(body), inferredBindings, demands); + for (int index = analyses.size() - 1; index >= 0; index--) { + CteAnalysis analysis = analyses.get(index); + String name = canonical(analysis.commonTable().name().value()); + RequiredOutputs demand = demands.getOrDefault(name, new RequiredOutputs(false, Set.of())); + HogQlProjectionDemand sourceDemand = HogQlProjectionDemand.collectNonProjection(analysis.commonTable().query()); + for (CteOutput output : analysis.outputs()) { + if (demand.includes(output.name())) { + sourceDemand = sourceDemand.merge(output.sourceDemand()); + } + } + collectCommonTableDemands(analysis.commonTable().query(), sourceDemand, inferredBindings, demands); + } + + Map resolvedBindings = new LinkedHashMap<>(commonTableBindings); + List resolvedCommonTables = new ArrayList<>(); + for (CteAnalysis analysis : analyses) { + CommonTableExpression commonTable = analysis.commonTable(); + RequiredOutputs demand = commonTable.columnAliases().isEmpty() + ? demands.getOrDefault(canonical(commonTable.name().value()), new RequiredOutputs(false, Set.of())) + : RequiredOutputs.allOutputs(); + HogQlQuery resolved = new HogQlSemanticResolver( + snapshot, + expansionBudget, + Optional.of(demand), + List.of(), + resolvedBindings) + .resolveNestedQuery(commonTable.query()); + resolvedCommonTables.add(new CommonTableExpression(commonTable.name(), commonTable.columnAliases(), resolved, commonTable.span())); + resolvedBindings.put(canonical(commonTable.name().value()), analysis.binding()); + } + + HogQlQuery resolvedBody = new HogQlSemanticResolver( + snapshot, + expansionBudget, + requiredOutputs, + outerBindingScopes, + resolvedBindings) + .resolveNestedQuery(body); + return new HogQlQuery( + resolvedCommonTables, + resolvedBody.body(), + resolvedBody.orderBy(), + resolvedBody.limit(), + resolvedBody.offset(), + query.span()); + } + + private CteAnalysis inferCommonTable(CommonTableExpression commonTable, Map availableBindings) + { + if (!(commonTable.query().body() instanceof SelectQueryBody select)) { + throw cteInferenceError(commonTable.span(), "HogQL CTE output inference does not support set operations"); + } + List sourceBindings = select.from() + .map(relation -> inferRelationBindings(relation, availableBindings)) + .orElse(List.of()); + List outputs = new ArrayList<>(); + for (Projection projection : select.projections()) { + outputs.addAll(inferProjectionOutputs(projection, sourceBindings, !commonTable.columnAliases().isEmpty())); + } + if (!commonTable.columnAliases().isEmpty()) { + if (commonTable.columnAliases().size() != outputs.size()) { + throw cteInferenceError(commonTable.span(), "HogQL CTE column alias count does not match its output count"); + } + for (int index = 0; index < outputs.size(); index++) { + outputs.set(index, new CteOutput(commonTable.columnAliases().get(index).value(), outputs.get(index).sourceDemand())); + } + } + Set names = new HashSet<>(); + for (CteOutput output : outputs) { + if (!names.add(canonical(output.name()))) { + throw cteInferenceError(commonTable.span(), "HogQL CTE output names must be unique: " + output.name()); + } + } + List fields = outputs.stream() + .map(output -> new BoundField(output.name(), new PhysicalIdentifier(output.name(), true), true, Optional.empty())) + .toList(); + String name = commonTable.name().value(); + TableBinding binding = new TableBinding( + CTE_RELATION_PREFIX + canonical(name), + canonical(name), + new PhysicalIdentifier(name, commonTable.name().delimited()), + List.of(new PhysicalIdentifier(name, commonTable.name().delimited())), + false, + fields, + TableBinding.fieldMap(fields)); + return new CteAnalysis(commonTable, binding, outputs); + } + + private List inferRelationBindings(Relation relation, Map availableBindings) + { + return switch (relation) { + case AliasedRelation alias -> inferRelationBindings(alias.relation(), availableBindings).stream() + .map(binding -> binding.withAlias(alias.alias())) + .toList(); + case CommonTableReference commonTable -> Optional.ofNullable(availableBindings.get(canonical(commonTable.name().value()))) + .map(List::of) + .orElse(List.of()); + case JoinRelation join -> { + List bindings = new ArrayList<>(inferRelationBindings(join.left(), availableBindings)); + bindings.addAll(inferRelationBindings(join.right(), availableBindings)); + yield List.copyOf(bindings); + } + case TableReference table -> table.parts().size() == 1 + ? snapshot.logicalTable(table.parts().getFirst().value()) + .map(definition -> resolveLogicalTable(definition, table.span()).bindings()) + .orElse(List.of()) + : List.of(); + case SubqueryRelation _, TablePlaceholder _, ValuesRelation _ -> List.of(); + }; + } + + private List inferProjectionOutputs(Projection projection, List sourceBindings, boolean hasColumnAliases) + { + return switch (projection) { + case ExpressionProjection expression -> { + Optional name = expression.alias().map(Identifier::value); + if (name.isEmpty() && expression.expression() instanceof ColumnReference reference) { + name = Optional.of(reference.parts().getLast().value()); + } + if (name.isEmpty() && !hasColumnAliases) { + throw cteInferenceError(expression.span(), "Cannot infer HogQL CTE output name; add a projection alias or CTE column alias list"); + } + yield List.of(new CteOutput(name.orElse("__hogql_cte_output"), HogQlProjectionDemand.collect(expression.expression()))); + } + case ColumnsList columns -> columns.expressions().stream() + .map(expression -> { + if (!(expression instanceof ColumnReference reference)) { + throw cteInferenceError(expression.span(), "Cannot infer HogQL CTE COLUMNS output name"); + } + return new CteOutput(reference.parts().getLast().value(), HogQlProjectionDemand.collect(expression)); + }) + .toList(); + case ColumnsRegex columns -> inferRegexOutputs(columns, sourceBindings); + case Star star -> inferStarOutputs(star, sourceBindings); + }; + } + + private List inferRegexOutputs(ColumnsRegex columns, List sourceBindings) + { + Pattern pattern; + try { + pattern = Pattern.compile(columns.pattern()); + } + catch (PatternSyntaxException _) { + throw semanticEntityError(HOGQL_RESOLUTION_ERROR, columns.patternSpan(), "Invalid HogQL COLUMNS regex: " + columns.pattern()); + } + List outputs = sourceBindings.stream() + .flatMap(binding -> binding.orderedFields().stream() + .filter(BoundField::starVisible) + .filter(field -> pattern.find(utf8Slice(field.name()))) + .map(field -> new CteOutput(field.name(), HogQlProjectionDemand.column(binding.qualifier(), field.name())))) + .toList(); + if (outputs.isEmpty()) { + throw cteInferenceError(columns.patternSpan(), "No HogQL CTE fields matched COLUMNS regex: " + columns.pattern()); + } + return outputs; + } + + private List inferStarOutputs(Star star, List sourceBindings) + { + Optional lazyStar = inferLazyStar(star, sourceBindings); + if (lazyStar.isPresent()) { + return lazyStar.orElseThrow().definition().projections().stream() + .filter(LazyProjectionDefinition::starVisible) + .filter(projection -> star.exclusions().stream().noneMatch(exclusion -> matchesIdentifier(exclusion.parts().getLast(), projection.name()))) + .map(projection -> new CteOutput( + projection.name(), + star.replacements().stream() + .filter(replacement -> matchesIdentifier(replacement.target(), projection.name())) + .findFirst() + .map(replacement -> HogQlProjectionDemand.collect(replacement.expression())) + .orElseGet(HogQlProjectionDemand::preserveNone))) + .toList(); + } + List starBindings = star.qualifier().isEmpty() + ? sourceBindings + : sourceBindings.stream().filter(binding -> matchesStarQualifier(binding, star.qualifier())).toList(); + if (starBindings.isEmpty()) { + throw cteInferenceError(star.span(), "Cannot infer HogQL CTE star outputs from an unknown relation schema"); + } + Set exclusions = resolveStarExclusions(star, starBindings); + Map replacements = resolveStarReplacements(star, starBindings, exclusions); + return starBindings.stream() + .flatMap(binding -> binding.orderedFields().stream() + .filter(BoundField::starVisible) + .filter(field -> !exclusions.contains(new StarField(binding, field))) + .map(field -> { + StarReplacement replacement = replacements.get(new StarField(binding, field)); + HogQlProjectionDemand demand = replacement == null + ? HogQlProjectionDemand.column(binding.qualifier(), field.name()) + : HogQlProjectionDemand.collect(replacement.expression()); + return new CteOutput(field.name(), demand); + })) + .toList(); + } + + private Optional inferLazyStar(Star star, List sourceBindings) + { + if (star.qualifier().isEmpty() || star.qualifier().size() > 2) { + return Optional.empty(); + } + List matches; + if (star.qualifier().size() == 1) { + String lazyName = star.qualifier().getFirst().value(); + matches = sourceBindings.stream() + .flatMap(binding -> lazyTable(binding, lazyName).stream().map(definition -> new LazyStar(binding, definition))) + .toList(); + } + else { + Identifier owner = star.qualifier().getFirst(); + String lazyName = star.qualifier().getLast().value(); + matches = sourceBindings.stream() + .filter(binding -> matchesStarQualifier(binding, List.of(owner))) + .flatMap(binding -> lazyTable(binding, lazyName).stream().map(definition -> new LazyStar(binding, definition))) + .toList(); + } + if (matches.size() > 1) { + throw cteInferenceError(star.span(), "Ambiguous HogQL CTE lazy star qualifier: " + starQualifier(star)); + } + return matches.stream().findFirst(); + } + + private void collectCommonTableDemands( + HogQlQuery query, + HogQlProjectionDemand demand, + Map availableBindings, + Map demands) + { + if (query.body() instanceof SelectQueryBody select) { + select.from().ifPresent(relation -> collectCommonTableDemands(relation, demand, availableBindings, demands)); + } + } + + private void collectCommonTableDemands( + Relation relation, + HogQlProjectionDemand demand, + Map availableBindings, + Map demands) + { + switch (relation) { + case AliasedRelation alias -> { + if (alias.relation() instanceof CommonTableReference commonTable) { + addCommonTableDemand(commonTable, demand.forAlias(alias.alias()), availableBindings, demands); + } + else { + collectCommonTableDemands(alias.relation(), demand, availableBindings, demands); + } + } + case CommonTableReference commonTable -> addCommonTableDemand(commonTable, demand.forAlias(commonTable.name()), availableBindings, demands); + case JoinRelation join -> { + collectCommonTableDemands(join.left(), demand, availableBindings, demands); + collectCommonTableDemands(join.right(), demand, availableBindings, demands); + } + case SubqueryRelation _, TablePlaceholder _, TableReference _, ValuesRelation _ -> {} + } + } + + private static void addCommonTableDemand( + CommonTableReference commonTable, + RequiredOutputs demand, + Map availableBindings, + Map demands) + { + String name = canonical(commonTable.name().value()); + TableBinding binding = availableBindings.get(name); + if (binding == null || !binding.relationName().startsWith(CTE_RELATION_PREFIX)) { + return; + } + demands.merge(name, demand, RequiredOutputs::merge); + } + private HogQlQuery resolveQuery(HogQlQuery query, List commonTables) { List projections = new ArrayList<>(); @@ -253,6 +547,13 @@ private List resolveProjection(Projection projection) case ColumnsRegex columns -> resolveColumnsRegex(columns); case Star star -> resolveStar(star); case ExpressionProjection expressionProjection -> { + Optional outputName = expressionProjection.alias().map(Identifier::value); + if (outputName.isEmpty() && expressionProjection.expression() instanceof ColumnReference reference) { + outputName = Optional.of(reference.parts().getLast().value()); + } + if (outputName.isPresent() && !projectionDemanded(outputName.orElseThrow())) { + yield List.of(); + } Expression resolved = resolveExpression(expressionProjection.expression()); Optional alias = expressionProjection.alias(); if (alias.isEmpty() && expressionProjection.expression() instanceof ColumnReference reference) { @@ -286,6 +587,7 @@ private List resolveColumnsRegex(ColumnsRegex columns) .flatMap(binding -> binding.orderedFields().stream() .filter(BoundField::starVisible) .filter(field -> pattern.find(utf8Slice(field.name()))) + .filter(field -> projectionDemanded(field.name())) .map(field -> new ExpressionProjection( resolveBoundField(binding, field, binding.starQualifier(bindings.size()), columns.span(), expansionBudget), Optional.of(new Identifier(field.name(), true, columns.span()))))) @@ -582,7 +884,9 @@ yield new ResolvedRelation( aliasedBindings, child.allLogical()); } - case CommonTableReference commonTable -> new ResolvedRelation(commonTable, List.of(), false); + case CommonTableReference commonTable -> Optional.ofNullable(commonTableBindings.get(canonical(commonTable.name().value()))) + .map(binding -> new ResolvedRelation(commonTable, List.of(binding), true)) + .orElseGet(() -> new ResolvedRelation(commonTable, List.of(), false)); case JoinRelation join -> { ResolvedRelation left = resolveRelation(join.left(), projectionDemand); ResolvedRelation right = resolveRelation(join.right(), projectionDemand); @@ -614,7 +918,7 @@ private ResolvedRelation resolveSubquery(SubqueryRelation subquery, RequiredOutp { return new ResolvedRelation( new SubqueryRelation( - new HogQlSemanticResolver(snapshot, expansionBudget, Optional.of(requiredOutputs)).resolveNestedQuery(subquery.query()), + new HogQlSemanticResolver(snapshot, expansionBudget, Optional.of(requiredOutputs), List.of(), commonTableBindings).resolveNestedQuery(subquery.query()), subquery.span()), List.of(), false); @@ -856,7 +1160,7 @@ private HogQlSemanticResolver correlatedSubqueryResolver() List visibleOuterScopes = new ArrayList<>(); currentBindingScope().ifPresent(visibleOuterScopes::add); visibleOuterScopes.addAll(outerBindingScopes); - return new HogQlSemanticResolver(snapshot, expansionBudget, Optional.empty(), visibleOuterScopes); + return new HogQlSemanticResolver(snapshot, expansionBudget, Optional.empty(), visibleOuterScopes, commonTableBindings); } private Optional currentBindingScope() @@ -1831,6 +2135,11 @@ private static TrinoException semanticEntityError(HogQlErrorCode errorCode, HogQ null); } + private static TrinoException cteInferenceError(HogQlQuery.SourceSpan span, String message) + { + return semanticEntityError(HOGQL_RESOLUTION_ERROR, span, message); + } + private static TrinoException unsupportedColumns(HogQlQuery.SourceSpan span) { return semanticEntityError( @@ -1949,6 +2258,25 @@ private record BindingScope(List bindings, Set relationQua } } + private record CteAnalysis(CommonTableExpression commonTable, TableBinding binding, List outputs) + { + private CteAnalysis + { + commonTable = requireNonNull(commonTable, "commonTable is null"); + binding = requireNonNull(binding, "binding is null"); + outputs = List.copyOf(requireNonNull(outputs, "outputs is null")); + } + } + + private record CteOutput(String name, HogQlProjectionDemand sourceDemand) + { + private CteOutput + { + name = requireNonNull(name, "name is null"); + sourceDemand = requireNonNull(sourceDemand, "sourceDemand is null"); + } + } + private record TableBinding( String relationName, String qualifier, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java index 7f06a345b176..261b10834698 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -424,6 +424,57 @@ public void testCteScopeShadowsLogicalCatalogNamesWithoutFetchingMetadata() "WITH events AS (SELECT 1 AS event), next AS (SELECT event FROM events) SELECT event FROM next")); } + @Test + public void testInfersCteOutputsForColumnsAndStarModifiers() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult columns = compiler.compile( + envelope( + "WITH source AS (SELECT event AS eventName, personId AS person FROM events) " + + "SELECT COLUMNS('eventName|person') FROM source", + OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult modifiedStar = compiler.compile( + envelope( + "WITH source AS (SELECT event AS eventName, personId AS person FROM events) " + + "SELECT COLUMNS(source.* EXCLUDE (person) REPLACE (person AS eventName)) FROM source", + OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult columnAliases = compiler.compile( + envelope( + "WITH source(renamedEvent, renamedPerson) AS (SELECT event, personId FROM events) " + + "SELECT renamedEvent FROM source", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(columns.statement()).isEqualTo(sqlParser.createStatement( + "WITH source AS (SELECT event_name AS eventName, \"Person ID\" AS person FROM analytics.\"Hog Data\".\"raw-events\") " + + "SELECT \"eventName\" AS \"eventName\", \"person\" AS \"person\" FROM source")); + assertThat(modifiedStar.statement()).isEqualTo(sqlParser.createStatement( + "WITH source AS (SELECT event_name AS eventName, \"Person ID\" AS person FROM analytics.\"Hog Data\".\"raw-events\") " + + "SELECT \"person\" AS \"eventName\" FROM source")); + assertThat(columnAliases.statement()).isEqualTo(sqlParser.createStatement( + "WITH source(renamedEvent, renamedPerson) AS (" + + "SELECT event_name AS event, \"Person ID\" AS personId FROM analytics.\"Hog Data\".\"raw-events\") " + + "SELECT \"renamedEvent\" AS renamedEvent FROM source")); + } + + @Test + public void testRejectsCteOutputWithoutAnInferableName() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + String hogql = "WITH source AS (SELECT event + personId FROM events) SELECT * FROM source"; + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope(hogql, OptionalLong.of(7)), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, hogql.indexOf("event + personId") + 1)); + assertThat(exception).hasMessageContaining("Cannot infer HogQL CTE output name"); + } + @Test public void testUnknownLogicalFieldFailsAtOriginalLocation() { diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java index 3dfb96f6216a..263ed931d777 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java @@ -93,6 +93,30 @@ public void testUsedLazyProjectionRemainsVisibleToStockPredicatePushdown() .findFirst()).isPresent(); } + @Test + public void testUnusedLazyProjectionIsPrunedThroughChainedCtes() + { + Plan plan = plan(compile( + "WITH first_source AS (SELECT e.personProfile.*, e.* FROM events e), " + + "second_source AS (SELECT * FROM first_source) " + + "SELECT event FROM second_source")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(1); + assertThat(nodes(plan, JoinNode.class)).isEmpty(); + } + + @Test + public void testDemandedLazyProjectionRemainsInPlanThroughChainedCtes() + { + Plan plan = plan(compile( + "WITH first_source AS (SELECT e.personProfile.*, e.* FROM events e), " + + "second_source AS (SELECT * FROM first_source) " + + "SELECT name FROM second_source WHERE name = 'Customer#000000001'")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(2); + assertThat(nodes(plan, JoinNode.class)).hasSize(1); + } + private String compile(String query) { HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); From 17081aa8bbdca54947c213b42da9ebcaa7b826f3 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 18:43:50 -0400 Subject: [PATCH 070/136] feat(hogql): infer derived query outputs --- .../hogql/compiler/HogQlProjectionDemand.java | 56 ++++++++++ .../hogql/compiler/HogQlSemanticResolver.java | 100 ++++++++++++++---- .../compiler/TestHogQlSemanticResolution.java | 50 +++++++++ .../planner/TestHogQlProjectionPruning.java | 24 +++++ 4 files changed, 208 insertions(+), 22 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java index b8d2cf6803eb..252fbdbbebc5 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java @@ -92,6 +92,47 @@ public static HogQlProjectionDemand collect(HogQlQuery query) return builder.build(); } + public static HogQlProjectionDemand collect(HogQlQuery query, RequiredOutputs requiredOutputs) + { + requireNonNull(requiredOutputs, "requiredOutputs is null"); + if (requiredOutputs.all()) { + return collect(query); + } + Builder builder = collectNonProjection(query).toBuilder(); + if (query.body() instanceof SelectQueryBody select) { + select.projections().forEach(projection -> collectRequiredProjection(projection, requiredOutputs, builder)); + } + return builder.build(); + } + + private static void collectRequiredProjection(Projection projection, RequiredOutputs requiredOutputs, Builder builder) + { + switch (projection) { + case ExpressionProjection expression -> { + String name = expression.alias() + .map(Identifier::value) + .orElseGet(() -> expression.expression() instanceof ColumnReference reference ? reference.parts().getLast().value() : ""); + if (requiredOutputs.includes(name)) { + collect(expression.expression(), builder); + } + } + case ColumnsList columns -> columns.expressions().stream() + .filter(ColumnReference.class::isInstance) + .map(ColumnReference.class::cast) + .filter(reference -> requiredOutputs.includes(reference.parts().getLast().value())) + .forEach(builder::add); + case ColumnsRegex _ -> requiredOutputs.names().forEach(builder::addUnqualified); + case Star star -> requiredOutputs.names().forEach(name -> { + if (star.qualifier().isEmpty()) { + builder.addUnqualified(name); + } + else { + builder.addQualified(star.qualifier().getFirst().value(), name); + } + }); + } + } + public static HogQlProjectionDemand collectNonProjection(HogQlQuery query) { Builder builder = new Builder(); @@ -164,6 +205,16 @@ public HogQlProjectionDemand merge(HogQlProjectionDemand other) return new HogQlProjectionDemand(all || other.all, mergedUnqualified, mergedAllQualifiers, mergedQualified); } + private Builder toBuilder() + { + Builder builder = new Builder(); + builder.all = all; + builder.unqualified.addAll(unqualified); + builder.allQualifiers.addAll(allQualifiers); + qualified.forEach((key, value) -> builder.qualified.put(key, new HashSet<>(value))); + return builder; + } + private static void collect(Projection projection, Builder builder) { switch (projection) { @@ -324,6 +375,11 @@ public void addUnqualified(Identifier identifier) unqualified.add(canonical(identifier.value())); } + public void addUnqualified(String name) + { + unqualified.add(canonical(name)); + } + public void addQualified(String qualifier, String name) { qualified.computeIfAbsent(canonical(qualifier), _ -> new HashSet<>()).add(canonical(name)); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index fd062b78eb17..77a390bebc63 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -122,6 +122,7 @@ final class HogQlSemanticResolver { private static final String MATCHES_ACTION = "matchesaction"; private static final String CTE_RELATION_PREFIX = "\0cte:"; + private static final String DERIVED_RELATION_PREFIX = "\0derived:"; private final PinnedSnapshot snapshot; private final ExpansionBudget expansionBudget; @@ -202,7 +203,9 @@ private HogQlQuery resolveNestedQuery(HogQlQuery query) } SelectQueryBody select = (SelectQueryBody) query.body(); localRelationQualifiers = select.from().map(HogQlSemanticResolver::relationQualifiers).orElse(Set.of()); - HogQlProjectionDemand projectionDemand = HogQlProjectionDemand.collect(query); + HogQlProjectionDemand projectionDemand = requiredOutputs + .map(outputs -> HogQlProjectionDemand.collect(query, outputs)) + .orElseGet(() -> HogQlProjectionDemand.collect(query)); Optional relation = select.from().map(value -> resolveRelation(value, projectionDemand)); bindings = relation.map(ResolvedRelation::bindings).orElse(List.of()); allRelationsLogical = relation.map(ResolvedRelation::allLogical).orElse(false); @@ -287,51 +290,71 @@ private HogQlQuery resolveWithQuery(HogQlQuery query) private CteAnalysis inferCommonTable(CommonTableExpression commonTable, Map availableBindings) { - if (!(commonTable.query().body() instanceof SelectQueryBody select)) { - throw cteInferenceError(commonTable.span(), "HogQL CTE output inference does not support set operations"); + List outputs = inferQueryOutputs(commonTable.query(), commonTable.columnAliases(), commonTable.span(), availableBindings); + TableBinding binding = inferredBinding(CTE_RELATION_PREFIX, commonTable.name(), outputs); + return new CteAnalysis(commonTable, binding, outputs); + } + + private List inferQueryOutputs( + HogQlQuery query, + List columnAliases, + HogQlQuery.SourceSpan span, + Map availableBindings) + { + if (!(query.body() instanceof SelectQueryBody select)) { + throw cteInferenceError(span, "HogQL inferred relation outputs do not support set operations"); } List sourceBindings = select.from() .map(relation -> inferRelationBindings(relation, availableBindings)) .orElse(List.of()); List outputs = new ArrayList<>(); for (Projection projection : select.projections()) { - outputs.addAll(inferProjectionOutputs(projection, sourceBindings, !commonTable.columnAliases().isEmpty())); + outputs.addAll(inferProjectionOutputs(projection, sourceBindings, !columnAliases.isEmpty())); } - if (!commonTable.columnAliases().isEmpty()) { - if (commonTable.columnAliases().size() != outputs.size()) { - throw cteInferenceError(commonTable.span(), "HogQL CTE column alias count does not match its output count"); + if (!columnAliases.isEmpty()) { + if (columnAliases.size() != outputs.size()) { + throw cteInferenceError(span, "HogQL inferred relation column alias count does not match its output count"); } for (int index = 0; index < outputs.size(); index++) { - outputs.set(index, new CteOutput(commonTable.columnAliases().get(index).value(), outputs.get(index).sourceDemand())); + outputs.set(index, new CteOutput(columnAliases.get(index).value(), outputs.get(index).sourceDemand())); } } Set names = new HashSet<>(); for (CteOutput output : outputs) { if (!names.add(canonical(output.name()))) { - throw cteInferenceError(commonTable.span(), "HogQL CTE output names must be unique: " + output.name()); + throw cteInferenceError(span, "HogQL inferred relation output names must be unique: " + output.name()); } } + return List.copyOf(outputs); + } + + private static TableBinding inferredBinding(String relationPrefix, Identifier name, List outputs) + { List fields = outputs.stream() .map(output -> new BoundField(output.name(), new PhysicalIdentifier(output.name(), true), true, Optional.empty())) .toList(); - String name = commonTable.name().value(); - TableBinding binding = new TableBinding( - CTE_RELATION_PREFIX + canonical(name), - canonical(name), - new PhysicalIdentifier(name, commonTable.name().delimited()), - List.of(new PhysicalIdentifier(name, commonTable.name().delimited())), + return new TableBinding( + relationPrefix + canonical(name.value()), + canonical(name.value()), + new PhysicalIdentifier(name.value(), name.delimited()), + List.of(new PhysicalIdentifier(name.value(), name.delimited())), false, fields, TableBinding.fieldMap(fields)); - return new CteAnalysis(commonTable, binding, outputs); } private List inferRelationBindings(Relation relation, Map availableBindings) { return switch (relation) { - case AliasedRelation alias -> inferRelationBindings(alias.relation(), availableBindings).stream() - .map(binding -> binding.withAlias(alias.alias())) - .toList(); + case AliasedRelation alias -> { + if (alias.relation() instanceof SubqueryRelation subquery) { + List outputs = inferQueryOutputs(subquery.query(), alias.columnAliases(), alias.span(), availableBindings); + yield List.of(inferredBinding(DERIVED_RELATION_PREFIX, alias.alias(), outputs)); + } + yield inferRelationBindings(alias.relation(), availableBindings).stream() + .map(binding -> binding.withAlias(alias.alias())) + .toList(); + } case CommonTableReference commonTable -> Optional.ofNullable(availableBindings.get(canonical(commonTable.name().value()))) .map(List::of) .orElse(List.of()); @@ -479,7 +502,18 @@ private void collectCommonTableDemands( { switch (relation) { case AliasedRelation alias -> { - if (alias.relation() instanceof CommonTableReference commonTable) { + if (alias.relation() instanceof SubqueryRelation subquery) { + List outputs = inferQueryOutputs(subquery.query(), alias.columnAliases(), alias.span(), availableBindings); + RequiredOutputs requiredOutputs = demand.forAlias(alias.alias()); + HogQlProjectionDemand sourceDemand = HogQlProjectionDemand.collectNonProjection(subquery.query()); + for (CteOutput output : outputs) { + if (requiredOutputs.includes(output.name())) { + sourceDemand = sourceDemand.merge(output.sourceDemand()); + } + } + collectCommonTableDemands(subquery.query(), sourceDemand, availableBindings, demands); + } + else if (alias.relation() instanceof CommonTableReference commonTable) { addCommonTableDemand(commonTable, demand.forAlias(alias.alias()), availableBindings, demands); } else { @@ -870,8 +904,16 @@ private ResolvedRelation resolveRelation(Relation relation, HogQlProjectionDeman return switch (relation) { case AliasedRelation alias -> { ResolvedRelation child; - if (alias.relation() instanceof SubqueryRelation subquery && alias.columnAliases().isEmpty()) { - child = resolveSubquery(subquery, projectionDemand.forAlias(alias.alias())); + if (alias.relation() instanceof SubqueryRelation subquery) { + List outputs = inferQueryOutputs(subquery.query(), alias.columnAliases(), alias.span(), commonTableBindings); + RequiredOutputs demand = alias.columnAliases().isEmpty() + ? projectionDemand.forAlias(alias.alias()) + : RequiredOutputs.allOutputs(); + child = resolveSubquery(subquery, demand); + yield new ResolvedRelation( + new AliasedRelation(child.relation(), alias.alias(), alias.columnAliases(), alias.span()), + List.of(inferredBinding(DERIVED_RELATION_PREFIX, alias.alias(), outputs)), + true); } else { child = resolveRelation(alias.relation(), HogQlProjectionDemand.preserveAll()); @@ -1722,6 +1764,9 @@ private Identifier nextGeneratedRelationAlias(HogQlQuery.SourceSpan span) private Optional relationship(TableBinding binding, String name) { + if (isInferredRelation(binding)) { + return Optional.empty(); + } return snapshot.logicalTable(binding.relationName()).flatMap(table -> relationship(table, name)); } @@ -1734,6 +1779,9 @@ private static Optional relationship(LogicalTableDefinit private Optional lazyTable(TableBinding binding, String name) { + if (isInferredRelation(binding)) { + return Optional.empty(); + } return snapshot.snapshot().lazyTables().stream() .filter(candidate -> canonical(candidate.table()).equals(canonical(binding.relationName()))) .filter(candidate -> canonical(candidate.name()).equals(canonical(name))) @@ -1843,11 +1891,19 @@ else if (parts.size() == 2) { private List properties(TableBinding binding) { + if (isInferredRelation(binding)) { + return List.of(); + } return snapshot.logicalTable(binding.relationName()) .map(LogicalTableDefinition::properties) .orElse(List.of()); } + private static boolean isInferredRelation(TableBinding binding) + { + return binding.relationName().startsWith(CTE_RELATION_PREFIX) || binding.relationName().startsWith(DERIVED_RELATION_PREFIX); + } + private Expression resolveBoundField( TableBinding binding, BoundField field, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java index 261b10834698..6000f434af5d 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -475,6 +475,56 @@ public void testRejectsCteOutputWithoutAnInferableName() assertThat(exception).hasMessageContaining("Cannot infer HogQL CTE output name"); } + @Test + public void testInfersDerivedOutputsForColumnsAndStarModifiers() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult columns = compiler.compile( + envelope( + "SELECT COLUMNS('eventName|person') FROM " + + "(SELECT event AS eventName, personId AS person FROM events) source", + OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult modifiedStar = compiler.compile( + envelope( + "SELECT COLUMNS(source.* EXCLUDE (person) REPLACE (person AS eventName)) FROM " + + "(SELECT event AS eventName, personId AS person FROM events) source", + OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult columnAliases = compiler.compile( + envelope( + "SELECT renamedEvent FROM (SELECT event, personId FROM events) source(renamedEvent, renamedPerson)", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(columns.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"eventName\" AS \"eventName\", \"person\" AS \"person\" FROM (" + + "SELECT event_name AS eventName, \"Person ID\" AS person FROM analytics.\"Hog Data\".\"raw-events\") source")); + assertThat(modifiedStar.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"person\" AS \"eventName\" FROM (" + + "SELECT event_name AS eventName, \"Person ID\" AS person FROM analytics.\"Hog Data\".\"raw-events\") source")); + assertThat(columnAliases.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"renamedEvent\" AS renamedEvent FROM (" + + "SELECT event_name AS event, \"Person ID\" AS personId FROM analytics.\"Hog Data\".\"raw-events\") " + + "source(renamedEvent, renamedPerson)")); + } + + @Test + public void testRejectsDerivedOutputWithoutAnInferableName() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + String hogql = "SELECT * FROM (SELECT event + personId FROM events) source"; + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope(hogql, OptionalLong.of(7)), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, hogql.indexOf("event + personId") + 1)); + assertThat(exception).hasMessageContaining("Cannot infer HogQL CTE output name"); + } + @Test public void testUnknownLogicalFieldFailsAtOriginalLocation() { diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java index 263ed931d777..faa8a1595a49 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java @@ -117,6 +117,30 @@ public void testDemandedLazyProjectionRemainsInPlanThroughChainedCtes() assertThat(nodes(plan, JoinNode.class)).hasSize(1); } + @Test + public void testUnusedLazyProjectionIsPrunedThroughDerivedCteChain() + { + Plan plan = plan(compile( + "WITH first_source AS (SELECT e.personProfile.*, e.* FROM events e), " + + "second_source AS (SELECT derived.* FROM (SELECT * FROM first_source) derived) " + + "SELECT event FROM second_source")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(1); + assertThat(nodes(plan, JoinNode.class)).isEmpty(); + } + + @Test + public void testDemandedLazyProjectionRemainsThroughDerivedCteChain() + { + Plan plan = plan(compile( + "WITH first_source AS (SELECT e.personProfile.*, e.* FROM events e), " + + "second_source AS (SELECT derived.* FROM (SELECT * FROM first_source) derived) " + + "SELECT name FROM second_source WHERE name = 'Customer#000000001'")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(2); + assertThat(nodes(plan, JoinNode.class)).hasSize(1); + } + private String compile(String query) { HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); From c342bed7c9c3da3da925b10b6721eaccfc0257f8 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 18:44:31 -0400 Subject: [PATCH 071/136] feat(hogql): remap aliased projection demand --- .../hogql/compiler/HogQlSemanticResolver.java | 75 +++++++++++++++---- .../compiler/TestHogQlSemanticResolution.java | 16 ++-- .../planner/TestHogQlProjectionPruning.java | 18 +++-- 3 files changed, 77 insertions(+), 32 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index 77a390bebc63..80d03a10af62 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -258,17 +258,16 @@ private HogQlQuery resolveWithQuery(HogQlQuery query) List resolvedCommonTables = new ArrayList<>(); for (CteAnalysis analysis : analyses) { CommonTableExpression commonTable = analysis.commonTable(); - RequiredOutputs demand = commonTable.columnAliases().isEmpty() - ? demands.getOrDefault(canonical(commonTable.name().value()), new RequiredOutputs(false, Set.of())) - : RequiredOutputs.allOutputs(); + RequiredOutputs externalDemand = demands.getOrDefault(canonical(commonTable.name().value()), new RequiredOutputs(false, Set.of())); + AliasedOutputDemand demand = remapAliasedOutputDemand(analysis.outputs(), commonTable.columnAliases(), externalDemand); HogQlQuery resolved = new HogQlSemanticResolver( snapshot, expansionBudget, - Optional.of(demand), + Optional.of(demand.sourceOutputs()), List.of(), resolvedBindings) .resolveNestedQuery(commonTable.query()); - resolvedCommonTables.add(new CommonTableExpression(commonTable.name(), commonTable.columnAliases(), resolved, commonTable.span())); + resolvedCommonTables.add(new CommonTableExpression(commonTable.name(), demand.outputAliases(), resolved, commonTable.span())); resolvedBindings.put(canonical(commonTable.name().value()), analysis.binding()); } @@ -316,7 +315,8 @@ private List inferQueryOutputs( throw cteInferenceError(span, "HogQL inferred relation column alias count does not match its output count"); } for (int index = 0; index < outputs.size(); index++) { - outputs.set(index, new CteOutput(columnAliases.get(index).value(), outputs.get(index).sourceDemand())); + CteOutput output = outputs.get(index); + outputs.set(index, new CteOutput(columnAliases.get(index).value(), output.sourceName(), output.sourceDemand())); } } Set names = new HashSet<>(); @@ -328,6 +328,39 @@ private List inferQueryOutputs( return List.copyOf(outputs); } + private static AliasedOutputDemand remapAliasedOutputDemand( + List outputs, + List columnAliases, + RequiredOutputs demand) + { + if (columnAliases.isEmpty() || demand.all()) { + return new AliasedOutputDemand(demand, columnAliases); + } + List selectedIndexes = java.util.stream.IntStream.range(0, outputs.size()) + .filter(index -> demand.includes(outputs.get(index).name())) + .boxed() + .toList(); + if (selectedIndexes.stream().anyMatch(index -> outputs.get(index).sourceName().isEmpty())) { + return new AliasedOutputDemand(RequiredOutputs.allOutputs(), columnAliases); + } + Set sourceNames = selectedIndexes.stream() + .map(index -> canonical(outputs.get(index).sourceName().orElseThrow())) + .collect(java.util.stream.Collectors.toSet()); + boolean overlapsUnselectedOutput = java.util.stream.IntStream.range(0, outputs.size()) + .filter(index -> !selectedIndexes.contains(index)) + .mapToObj(index -> outputs.get(index).sourceName()) + .flatMap(Optional::stream) + .map(HogQlSemanticResolver::canonical) + .anyMatch(sourceNames::contains); + if (overlapsUnselectedOutput) { + return new AliasedOutputDemand(RequiredOutputs.allOutputs(), columnAliases); + } + List selectedAliases = selectedIndexes.stream() + .map(columnAliases::get) + .toList(); + return new AliasedOutputDemand(new RequiredOutputs(false, sourceNames), selectedAliases); + } + private static TableBinding inferredBinding(String relationPrefix, Identifier name, List outputs) { List fields = outputs.stream() @@ -383,14 +416,15 @@ private List inferProjectionOutputs(Projection projection, List
columns.expressions().stream() .map(expression -> { if (!(expression instanceof ColumnReference reference)) { throw cteInferenceError(expression.span(), "Cannot infer HogQL CTE COLUMNS output name"); } - return new CteOutput(reference.parts().getLast().value(), HogQlProjectionDemand.collect(expression)); + String name = reference.parts().getLast().value(); + return new CteOutput(name, Optional.of(name), HogQlProjectionDemand.collect(expression)); }) .toList(); case ColumnsRegex columns -> inferRegexOutputs(columns, sourceBindings); @@ -411,7 +445,7 @@ private List inferRegexOutputs(ColumnsRegex columns, List binding.orderedFields().stream() .filter(BoundField::starVisible) .filter(field -> pattern.find(utf8Slice(field.name()))) - .map(field -> new CteOutput(field.name(), HogQlProjectionDemand.column(binding.qualifier(), field.name())))) + .map(field -> new CteOutput(field.name(), Optional.of(field.name()), HogQlProjectionDemand.column(binding.qualifier(), field.name())))) .toList(); if (outputs.isEmpty()) { throw cteInferenceError(columns.patternSpan(), "No HogQL CTE fields matched COLUMNS regex: " + columns.pattern()); @@ -428,6 +462,7 @@ private List inferStarOutputs(Star star, List sourceBin .filter(projection -> star.exclusions().stream().noneMatch(exclusion -> matchesIdentifier(exclusion.parts().getLast(), projection.name()))) .map(projection -> new CteOutput( projection.name(), + Optional.of(projection.name()), star.replacements().stream() .filter(replacement -> matchesIdentifier(replacement.target(), projection.name())) .findFirst() @@ -452,7 +487,7 @@ private List inferStarOutputs(Star star, List sourceBin HogQlProjectionDemand demand = replacement == null ? HogQlProjectionDemand.column(binding.qualifier(), field.name()) : HogQlProjectionDemand.collect(replacement.expression()); - return new CteOutput(field.name(), demand); + return new CteOutput(field.name(), Optional.of(field.name()), demand); })) .toList(); } @@ -906,12 +941,10 @@ private ResolvedRelation resolveRelation(Relation relation, HogQlProjectionDeman ResolvedRelation child; if (alias.relation() instanceof SubqueryRelation subquery) { List outputs = inferQueryOutputs(subquery.query(), alias.columnAliases(), alias.span(), commonTableBindings); - RequiredOutputs demand = alias.columnAliases().isEmpty() - ? projectionDemand.forAlias(alias.alias()) - : RequiredOutputs.allOutputs(); - child = resolveSubquery(subquery, demand); + AliasedOutputDemand demand = remapAliasedOutputDemand(outputs, alias.columnAliases(), projectionDemand.forAlias(alias.alias())); + child = resolveSubquery(subquery, demand.sourceOutputs()); yield new ResolvedRelation( - new AliasedRelation(child.relation(), alias.alias(), alias.columnAliases(), alias.span()), + new AliasedRelation(child.relation(), alias.alias(), demand.outputAliases(), alias.span()), List.of(inferredBinding(DERIVED_RELATION_PREFIX, alias.alias(), outputs)), true); } @@ -2324,15 +2357,25 @@ private record CteAnalysis(CommonTableExpression commonTable, TableBinding bindi } } - private record CteOutput(String name, HogQlProjectionDemand sourceDemand) + private record CteOutput(String name, Optional sourceName, HogQlProjectionDemand sourceDemand) { private CteOutput { name = requireNonNull(name, "name is null"); + sourceName = requireNonNull(sourceName, "sourceName is null"); sourceDemand = requireNonNull(sourceDemand, "sourceDemand is null"); } } + private record AliasedOutputDemand(RequiredOutputs sourceOutputs, List outputAliases) + { + private AliasedOutputDemand + { + sourceOutputs = requireNonNull(sourceOutputs, "sourceOutputs is null"); + outputAliases = List.copyOf(requireNonNull(outputAliases, "outputAliases is null")); + } + } + private record TableBinding( String relationName, String qualifier, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java index 6000f434af5d..10003bdcd42f 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -444,7 +444,7 @@ public void testInfersCteOutputsForColumnsAndStarModifiers() HogQlCompilationResult columnAliases = compiler.compile( envelope( "WITH source(renamedEvent, renamedPerson) AS (SELECT event, personId FROM events) " + - "SELECT renamedEvent FROM source", + "SELECT renamedPerson FROM source", OptionalLong.of(7)), Optional.of(context)); @@ -455,9 +455,9 @@ public void testInfersCteOutputsForColumnsAndStarModifiers() "WITH source AS (SELECT event_name AS eventName, \"Person ID\" AS person FROM analytics.\"Hog Data\".\"raw-events\") " + "SELECT \"person\" AS \"eventName\" FROM source")); assertThat(columnAliases.statement()).isEqualTo(sqlParser.createStatement( - "WITH source(renamedEvent, renamedPerson) AS (" + - "SELECT event_name AS event, \"Person ID\" AS personId FROM analytics.\"Hog Data\".\"raw-events\") " + - "SELECT \"renamedEvent\" AS renamedEvent FROM source")); + "WITH source(renamedPerson) AS (" + + "SELECT \"Person ID\" AS personId FROM analytics.\"Hog Data\".\"raw-events\") " + + "SELECT \"renamedPerson\" AS renamedPerson FROM source")); } @Test @@ -494,7 +494,7 @@ public void testInfersDerivedOutputsForColumnsAndStarModifiers() Optional.of(context)); HogQlCompilationResult columnAliases = compiler.compile( envelope( - "SELECT renamedEvent FROM (SELECT event, personId FROM events) source(renamedEvent, renamedPerson)", + "SELECT renamedPerson FROM (SELECT event, personId FROM events) source(renamedEvent, renamedPerson)", OptionalLong.of(7)), Optional.of(context)); @@ -505,9 +505,9 @@ public void testInfersDerivedOutputsForColumnsAndStarModifiers() "SELECT \"person\" AS \"eventName\" FROM (" + "SELECT event_name AS eventName, \"Person ID\" AS person FROM analytics.\"Hog Data\".\"raw-events\") source")); assertThat(columnAliases.statement()).isEqualTo(sqlParser.createStatement( - "SELECT \"renamedEvent\" AS renamedEvent FROM (" + - "SELECT event_name AS event, \"Person ID\" AS personId FROM analytics.\"Hog Data\".\"raw-events\") " + - "source(renamedEvent, renamedPerson)")); + "SELECT \"renamedPerson\" AS renamedPerson FROM (" + + "SELECT \"Person ID\" AS personId FROM analytics.\"Hog Data\".\"raw-events\") " + + "source(renamedPerson)")); } @Test diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java index faa8a1595a49..91b04debc408 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java @@ -97,9 +97,9 @@ public void testUsedLazyProjectionRemainsVisibleToStockPredicatePushdown() public void testUnusedLazyProjectionIsPrunedThroughChainedCtes() { Plan plan = plan(compile( - "WITH first_source AS (SELECT e.personProfile.*, e.* FROM events e), " + + "WITH first_source(aliasedName, aliasedEvent, aliasedPersonId) AS (SELECT e.personProfile.*, e.* FROM events e), " + "second_source AS (SELECT * FROM first_source) " + - "SELECT event FROM second_source")); + "SELECT aliasedEvent FROM second_source")); assertThat(nodes(plan, TableScanNode.class)).hasSize(1); assertThat(nodes(plan, JoinNode.class)).isEmpty(); @@ -109,9 +109,9 @@ public void testUnusedLazyProjectionIsPrunedThroughChainedCtes() public void testDemandedLazyProjectionRemainsInPlanThroughChainedCtes() { Plan plan = plan(compile( - "WITH first_source AS (SELECT e.personProfile.*, e.* FROM events e), " + + "WITH first_source(aliasedName, aliasedEvent, aliasedPersonId) AS (SELECT e.personProfile.*, e.* FROM events e), " + "second_source AS (SELECT * FROM first_source) " + - "SELECT name FROM second_source WHERE name = 'Customer#000000001'")); + "SELECT aliasedName FROM second_source WHERE aliasedName = 'Customer#000000001'")); assertThat(nodes(plan, TableScanNode.class)).hasSize(2); assertThat(nodes(plan, JoinNode.class)).hasSize(1); @@ -122,8 +122,9 @@ public void testUnusedLazyProjectionIsPrunedThroughDerivedCteChain() { Plan plan = plan(compile( "WITH first_source AS (SELECT e.personProfile.*, e.* FROM events e), " + - "second_source AS (SELECT derived.* FROM (SELECT * FROM first_source) derived) " + - "SELECT event FROM second_source")); + "second_source AS (SELECT derived.* FROM (SELECT * FROM first_source) " + + "derived(aliasedName, aliasedEvent, aliasedPersonId)) " + + "SELECT aliasedEvent FROM second_source")); assertThat(nodes(plan, TableScanNode.class)).hasSize(1); assertThat(nodes(plan, JoinNode.class)).isEmpty(); @@ -134,8 +135,9 @@ public void testDemandedLazyProjectionRemainsThroughDerivedCteChain() { Plan plan = plan(compile( "WITH first_source AS (SELECT e.personProfile.*, e.* FROM events e), " + - "second_source AS (SELECT derived.* FROM (SELECT * FROM first_source) derived) " + - "SELECT name FROM second_source WHERE name = 'Customer#000000001'")); + "second_source AS (SELECT derived.* FROM (SELECT * FROM first_source) " + + "derived(aliasedName, aliasedEvent, aliasedPersonId)) " + + "SELECT aliasedName FROM second_source WHERE aliasedName = 'Customer#000000001'")); assertThat(nodes(plan, TableScanNode.class)).hasSize(2); assertThat(nodes(plan, JoinNode.class)).hasSize(1); From afb87dd693bbd3d23e421dd76012657903865f58 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 18:52:27 -0400 Subject: [PATCH 072/136] feat(hogql): propagate set operation demand --- .../hogql/compiler/HogQlSemanticResolver.java | 149 ++++++++++++++---- .../compiler/TestHogQlSemanticResolution.java | 82 ++++++++++ .../planner/TestHogQlProjectionPruning.java | 33 +++- 3 files changed, 235 insertions(+), 29 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index 80d03a10af62..bdf26c6b1082 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -190,11 +190,16 @@ private HogQlQuery resolveNestedQuery(HogQlQuery query) } List commonTables = List.of(); if (query.body() instanceof SetOperation setOperation) { + Optional branchDemand = requiredOutputs.map(outputs -> { + SetOperationOutputs inferredOutputs = inferSetOperationOutputs(setOperation, commonTableBindings); + RequiredOutputs effectiveOutputs = query.orderBy().isEmpty() ? outputs : RequiredOutputs.allOutputs(); + return remapSetOperationDemand(inferredOutputs, effectiveOutputs); + }); SetOperation resolved = new SetOperation( setOperation.type(), setOperation.distinct(), - new HogQlSemanticResolver(snapshot, expansionBudget, Optional.empty(), outerBindingScopes, commonTableBindings).resolveNestedQuery(setOperation.left()), - new HogQlSemanticResolver(snapshot, expansionBudget, Optional.empty(), outerBindingScopes, commonTableBindings).resolveNestedQuery(setOperation.right()), + new HogQlSemanticResolver(snapshot, expansionBudget, branchDemand.map(SetBranchDemand::left), outerBindingScopes, commonTableBindings).resolveNestedQuery(setOperation.left()), + new HogQlSemanticResolver(snapshot, expansionBudget, branchDemand.map(SetBranchDemand::right), outerBindingScopes, commonTableBindings).resolveNestedQuery(setOperation.right()), setOperation.leftParenthesized(), setOperation.rightParenthesized(), setOperation.operatorSpan(), @@ -240,18 +245,21 @@ private HogQlQuery resolveWithQuery(HogQlQuery query) HogQlQuery body = new HogQlQuery(List.of(), query.body(), query.orderBy(), query.limit(), query.offset(), query.span()); Map demands = new HashMap<>(); - collectCommonTableDemands(body, HogQlProjectionDemand.collect(body), inferredBindings, demands); + if (body.body() instanceof SetOperation) { + collectQueryOutputDemands(body, requiredOutputs.orElseGet(RequiredOutputs::allOutputs), inferredBindings, demands); + } + else { + collectCommonTableDemands(body, HogQlProjectionDemand.collect(body), inferredBindings, demands); + } for (int index = analyses.size() - 1; index >= 0; index--) { CteAnalysis analysis = analyses.get(index); String name = canonical(analysis.commonTable().name().value()); - RequiredOutputs demand = demands.getOrDefault(name, new RequiredOutputs(false, Set.of())); - HogQlProjectionDemand sourceDemand = HogQlProjectionDemand.collectNonProjection(analysis.commonTable().query()); - for (CteOutput output : analysis.outputs()) { - if (demand.includes(output.name())) { - sourceDemand = sourceDemand.merge(output.sourceDemand()); - } - } - collectCommonTableDemands(analysis.commonTable().query(), sourceDemand, inferredBindings, demands); + RequiredOutputs externalDemand = demands.getOrDefault(name, new RequiredOutputs(false, Set.of())); + RequiredOutputs sourceDemand = remapAliasedOutputDemand( + analysis.outputs(), + analysis.commonTable().columnAliases(), + externalDemand).sourceOutputs(); + collectQueryOutputDemands(analysis.commonTable().query(), sourceDemand, inferredBindings, demands); } Map resolvedBindings = new LinkedHashMap<>(commonTableBindings); @@ -300,16 +308,14 @@ private List inferQueryOutputs( HogQlQuery.SourceSpan span, Map availableBindings) { - if (!(query.body() instanceof SelectQueryBody select)) { - throw cteInferenceError(span, "HogQL inferred relation outputs do not support set operations"); - } - List sourceBindings = select.from() - .map(relation -> inferRelationBindings(relation, availableBindings)) - .orElse(List.of()); - List outputs = new ArrayList<>(); - for (Projection projection : select.projections()) { - outputs.addAll(inferProjectionOutputs(projection, sourceBindings, !columnAliases.isEmpty())); + if (!query.with().isEmpty()) { + throw cteInferenceError(span, "HogQL inferred relation outputs do not support branch-local WITH clauses"); } + List outputs = switch (query.body()) { + case SelectQueryBody select -> inferSelectOutputs(select, availableBindings, !columnAliases.isEmpty()); + case SetOperation setOperation -> inferSetOperationOutputs(setOperation, availableBindings).outputs(); + }; + outputs = new ArrayList<>(outputs); if (!columnAliases.isEmpty()) { if (columnAliases.size() != outputs.size()) { throw cteInferenceError(span, "HogQL inferred relation column alias count does not match its output count"); @@ -328,6 +334,53 @@ private List inferQueryOutputs( return List.copyOf(outputs); } + private List inferSelectOutputs( + SelectQueryBody select, + Map availableBindings, + boolean hasColumnAliases) + { + List sourceBindings = select.from() + .map(relation -> inferRelationBindings(relation, availableBindings)) + .orElse(List.of()); + List outputs = new ArrayList<>(); + for (Projection projection : select.projections()) { + outputs.addAll(inferProjectionOutputs(projection, sourceBindings, hasColumnAliases)); + } + return List.copyOf(outputs); + } + + private SetOperationOutputs inferSetOperationOutputs(SetOperation setOperation, Map availableBindings) + { + List left = inferQueryOutputs(setOperation.left(), List.of(), setOperation.left().span(), availableBindings); + List right = inferQueryOutputs(setOperation.right(), List.of(), setOperation.right().span(), availableBindings); + if (left.size() != right.size()) { + throw cteInferenceError(setOperation.operatorSpan(), "HogQL set operation branches have incompatible output arity"); + } + List outputs = java.util.stream.IntStream.range(0, left.size()) + .mapToObj(index -> new CteOutput( + left.get(index).name(), + Optional.of(left.get(index).name()), + left.get(index).sourceDemand().merge(right.get(index).sourceDemand()))) + .toList(); + return new SetOperationOutputs(outputs, left, right); + } + + private static SetBranchDemand remapSetOperationDemand(SetOperationOutputs outputs, RequiredOutputs demand) + { + if (demand.all()) { + return new SetBranchDemand(RequiredOutputs.allOutputs(), RequiredOutputs.allOutputs()); + } + Set left = new HashSet<>(); + Set right = new HashSet<>(); + for (int index = 0; index < outputs.outputs().size(); index++) { + if (demand.includes(outputs.outputs().get(index).name())) { + left.add(canonical(outputs.left().get(index).name())); + right.add(canonical(outputs.right().get(index).name())); + } + } + return new SetBranchDemand(new RequiredOutputs(false, left), new RequiredOutputs(false, right)); + } + private static AliasedOutputDemand remapAliasedOutputDemand( List outputs, List columnAliases, @@ -529,6 +582,30 @@ private void collectCommonTableDemands( } } + private void collectQueryOutputDemands( + HogQlQuery query, + RequiredOutputs outputDemand, + Map availableBindings, + Map demands) + { + if (query.body() instanceof SetOperation setOperation) { + SetOperationOutputs outputs = inferSetOperationOutputs(setOperation, availableBindings); + RequiredOutputs effectiveDemand = query.orderBy().isEmpty() ? outputDemand : RequiredOutputs.allOutputs(); + SetBranchDemand branchDemand = remapSetOperationDemand(outputs, effectiveDemand); + collectQueryOutputDemands(setOperation.left(), branchDemand.left(), availableBindings, demands); + collectQueryOutputDemands(setOperation.right(), branchDemand.right(), availableBindings, demands); + return; + } + List outputs = inferQueryOutputs(query, List.of(), query.span(), availableBindings); + HogQlProjectionDemand sourceDemand = HogQlProjectionDemand.collectNonProjection(query); + for (CteOutput output : outputs) { + if (outputDemand.includes(output.name())) { + sourceDemand = sourceDemand.merge(output.sourceDemand()); + } + } + collectCommonTableDemands(query, sourceDemand, availableBindings, demands); + } + private void collectCommonTableDemands( Relation relation, HogQlProjectionDemand demand, @@ -539,14 +616,11 @@ private void collectCommonTableDemands( case AliasedRelation alias -> { if (alias.relation() instanceof SubqueryRelation subquery) { List outputs = inferQueryOutputs(subquery.query(), alias.columnAliases(), alias.span(), availableBindings); - RequiredOutputs requiredOutputs = demand.forAlias(alias.alias()); - HogQlProjectionDemand sourceDemand = HogQlProjectionDemand.collectNonProjection(subquery.query()); - for (CteOutput output : outputs) { - if (requiredOutputs.includes(output.name())) { - sourceDemand = sourceDemand.merge(output.sourceDemand()); - } - } - collectCommonTableDemands(subquery.query(), sourceDemand, availableBindings, demands); + RequiredOutputs sourceDemand = remapAliasedOutputDemand( + outputs, + alias.columnAliases(), + demand.forAlias(alias.alias())).sourceOutputs(); + collectQueryOutputDemands(subquery.query(), sourceDemand, availableBindings, demands); } else if (alias.relation() instanceof CommonTableReference commonTable) { addCommonTableDemand(commonTable, demand.forAlias(alias.alias()), availableBindings, demands); @@ -2376,6 +2450,25 @@ private record AliasedOutputDemand(RequiredOutputs sourceOutputs, List outputs, List left, List right) + { + private SetOperationOutputs + { + outputs = List.copyOf(requireNonNull(outputs, "outputs is null")); + left = List.copyOf(requireNonNull(left, "left is null")); + right = List.copyOf(requireNonNull(right, "right is null")); + } + } + + private record SetBranchDemand(RequiredOutputs left, RequiredOutputs right) + { + private SetBranchDemand + { + left = requireNonNull(left, "left is null"); + right = requireNonNull(right, "right is null"); + } + } + private record TableBinding( String relationName, String qualifier, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java index 10003bdcd42f..5bf81f12b0b2 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -525,6 +525,88 @@ public void testRejectsDerivedOutputWithoutAnInferableName() assertThat(exception).hasMessageContaining("Cannot infer HogQL CTE output name"); } + @Test + public void testInfersSetOperationOutputsAndPrunesBranchesByPosition() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult cte = compiler.compile( + envelope( + "WITH source AS (" + + "SELECT event AS chosen, personId AS unused FROM events " + + "UNION ALL " + + "SELECT personId AS rightChosen, event AS rightUnused FROM events) " + + "SELECT chosen FROM source", + OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult derived = compiler.compile( + envelope( + "SELECT COLUMNS('value') FROM (" + + "SELECT event AS value FROM events " + + "INTERSECT " + + "SELECT personId AS other FROM events) source", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(cte.statement()).isEqualTo(sqlParser.createStatement( + "WITH source AS (" + + "SELECT event_name AS chosen FROM analytics.\"Hog Data\".\"raw-events\" " + + "UNION ALL " + + "SELECT \"Person ID\" AS rightChosen FROM analytics.\"Hog Data\".\"raw-events\") " + + "SELECT \"chosen\" AS chosen FROM source")); + assertThat(derived.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"value\" AS \"value\" FROM (" + + "SELECT event_name AS value FROM analytics.\"Hog Data\".\"raw-events\" " + + "INTERSECT " + + "SELECT \"Person ID\" AS other FROM analytics.\"Hog Data\".\"raw-events\") source")); + } + + @Test + public void testRejectsUnsafeSetOperationOutputInference() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + TrinoException incompatibleArity = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile( + envelope( + "WITH source AS (SELECT event FROM events EXCEPT SELECT event, personId FROM events) SELECT event FROM source", + OptionalLong.of(7)), + Optional.of(context))); + TrinoException unnamed = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile( + envelope( + "WITH source AS (SELECT event FROM events UNION ALL SELECT event + personId FROM events) SELECT event FROM source", + OptionalLong.of(7)), + Optional.of(context))); + TrinoException ambiguous = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile( + envelope( + "WITH source AS (SELECT event AS value, personId AS value FROM events UNION ALL " + + "SELECT event AS firstValue, personId AS secondValue FROM events) SELECT value FROM source", + OptionalLong.of(7)), + Optional.of(context))); + TrinoException unsafeAliases = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile( + envelope( + "WITH source(onlyValue) AS (SELECT event AS value, personId AS other FROM events UNION ALL " + + "SELECT event AS rightValue, personId AS rightOther FROM events) SELECT onlyValue FROM source", + OptionalLong.of(7)), + Optional.of(context))); + + assertThat(incompatibleArity.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(incompatibleArity).hasMessageContaining("set operation branches have incompatible output arity"); + assertThat(unnamed.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(unnamed).hasMessageContaining("Cannot infer HogQL CTE output name"); + assertThat(ambiguous.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(ambiguous).hasMessageContaining("output names must be unique"); + assertThat(unsafeAliases.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(unsafeAliases).hasMessageContaining("column alias count does not match its output count"); + } + @Test public void testUnknownLogicalFieldFailsAtOriginalLocation() { diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java index 91b04debc408..0b641c1d20d0 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java @@ -38,8 +38,11 @@ import io.trino.sql.planner.plan.FilterNode; import io.trino.sql.planner.plan.JoinNode; import io.trino.sql.planner.plan.PlanNode; +import io.trino.sql.planner.plan.SetOperationNode; import io.trino.sql.planner.plan.TableScanNode; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.List; import java.util.Map; @@ -48,6 +51,7 @@ import static io.trino.SystemSessionProperties.ENABLE_DYNAMIC_FILTERING; import static io.trino.SystemSessionProperties.JOIN_REORDERING_STRATEGY; +import static io.trino.sql.planner.LogicalPlanner.Stage.CREATED; import static io.trino.sql.planner.plan.JoinType.INNER; import static io.trino.testing.TestingHandles.TEST_CATALOG_NAME; import static org.assertj.core.api.Assertions.assertThat; @@ -143,6 +147,28 @@ public void testDemandedLazyProjectionRemainsThroughDerivedCteChain() assertThat(nodes(plan, JoinNode.class)).hasSize(1); } + @ParameterizedTest + @ValueSource(strings = {"UNION ALL", "INTERSECT ALL", "EXCEPT ALL"}) + public void testSetOperationDemandIsMappedByBranchPosition(String operator) + { + Plan plan = plan(compile( + "WITH source AS (" + + "SELECT CAST(e.event AS String) AS selected, e.personProfile.name AS discarded FROM events e " + + operator + " " + + "SELECT e.personProfile.name AS rightSelected, CAST(e.event AS String) AS rightDiscarded FROM events e) " + + "SELECT selected FROM source"), CREATED); + + List setOperations = nodes(plan, SetOperationNode.class); + assertThat(setOperations).hasSize(1); + SetOperationNode setOperation = setOperations.getFirst(); + assertThat(nodes(setOperation.getSources().getFirst(), TableScanNode.class)).hasSize(1); + assertThat(nodes(setOperation.getSources().getFirst(), JoinNode.class)).isEmpty(); + assertThat(nodes(setOperation.getSources().getLast(), TableScanNode.class)).hasSize(2); + assertThat(nodes(setOperation.getSources().getLast(), JoinNode.class)).hasSize(1); + assertThat(nodes(plan, TableScanNode.class)).hasSize(3); + assertThat(nodes(plan, JoinNode.class)).hasSize(1); + } + private String compile(String query) { HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); @@ -160,7 +186,12 @@ private String compile(String query) private static List nodes(Plan plan, Class type) { - return PlanNodeSearcher.searchFrom(plan.getRoot()) + return nodes(plan.getRoot(), type); + } + + private static List nodes(PlanNode root, Class type) + { + return PlanNodeSearcher.searchFrom(root) .whereIsInstanceOfAny(type) .findAll().stream() .map(type::cast) From 2b4b76a373608fc2b9459bf47ff444bd37e299ff Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 19:11:32 -0400 Subject: [PATCH 073/136] feat(hogql): compile pivot relations --- .../trino/hogql/compiler/HogQlCompiler.java | 17 +++ .../hogql/compiler/HogQlFunctionResolver.java | 20 ++++ .../hogql/compiler/HogQlProjectionDemand.java | 7 ++ .../hogql/compiler/HogQlSemanticResolver.java | 41 +++++++ .../trino/hogql/compiler/TrinoAstFactory.java | 22 ++++ .../hogql/compiler/TestHogQlCompiler.java | 38 +++++++ .../TestHogQlRelationshipExpansion.java | 6 +- .../io/trino/hogql/parser/HogQlParser.java | 102 ++++++++++++++++++ .../trino/hogql/parser/tree/HogQlQuery.java | 53 +++++++++ .../trino/hogql/parser/TestHogQlParser.java | 56 ++++++++++ .../io/trino/sql/planner/TestHogQlPivot.java | 53 +++++++++ 11 files changed, 412 insertions(+), 3 deletions(-) create mode 100644 core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlPivot.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index e55accf1f006..437a71db2f2b 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -274,6 +274,8 @@ private static boolean containsSemanticCandidate(Relation relation) case AliasedRelation alias -> containsSemanticCandidate(alias.relation()); case CommonTableReference _ -> false; case JoinRelation join -> containsSemanticCandidate(join.left()) || containsSemanticCandidate(join.right()); + case HogQlQuery.PivotRelation pivot -> containsSemanticCandidate(pivot.input()) || + pivot.aggregations().stream().anyMatch(aggregation -> containsFunctionCall(aggregation.expression())); case SubqueryRelation subquery -> containsSemanticCandidate(subquery.query()); case TablePlaceholder _ -> false; case HogQlQuery.TableReference table -> table.parts().size() == 1; @@ -310,6 +312,13 @@ private static boolean containsFunctionCall(Relation relation) .map(JoinOn::expression) .map(HogQlCompiler::containsFunctionCall) .orElse(false); + case HogQlQuery.PivotRelation pivot -> containsFunctionCall(pivot.input()) || + pivot.aggregations().stream().anyMatch(aggregation -> containsFunctionCall(aggregation.expression())) || + pivot.pivotColumns().stream().anyMatch(HogQlCompiler::containsFunctionCall) || + pivot.valueGroups().stream() + .flatMap(group -> group.values().stream()) + .anyMatch(HogQlCompiler::containsFunctionCall) || + pivot.groupBy().stream().anyMatch(HogQlCompiler::containsFunctionCall); case SubqueryRelation subquery -> containsFunctionCall(subquery.query()); case TablePlaceholder _, HogQlQuery.TableReference _ -> false; case ValuesRelation values -> values.rows().stream() @@ -489,6 +498,7 @@ private static void validateRelation(Relation relation) validateRelation(join.left()); validateRelation(join.right()); } + case HogQlQuery.PivotRelation pivot -> validateRelation(pivot.input()); case SubqueryRelation subquery -> validateQuery(subquery.query()); case TablePlaceholder tablePlaceholder -> throw bindingError( tablePlaceholder.span(), @@ -512,6 +522,13 @@ private static void collectPlaceholders(Relation relation, List pla } }); } + case HogQlQuery.PivotRelation pivot -> { + collectPlaceholders(pivot.input(), placeholders); + pivot.aggregations().forEach(aggregation -> collectPlaceholders(aggregation.expression(), placeholders)); + pivot.pivotColumns().forEach(expression -> collectPlaceholders(expression, placeholders)); + pivot.valueGroups().forEach(group -> group.values().forEach(expression -> collectPlaceholders(expression, placeholders))); + pivot.groupBy().forEach(expression -> collectPlaceholders(expression, placeholders)); + } case SubqueryRelation subquery -> collectPlaceholders(subquery.query(), placeholders); case TablePlaceholder _ -> {} case HogQlQuery.TableReference _ -> {} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index e81dd66a8453..bf9b20f52430 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -47,6 +47,9 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.PivotAggregation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotValueGroup; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; @@ -169,6 +172,23 @@ private Relation resolveRelation(Relation relation) case JoinUsing using -> using; }), join.span()); + case PivotRelation pivot -> new PivotRelation( + resolveRelation(pivot.input()), + pivot.aggregations().stream() + .map(aggregation -> new PivotAggregation( + resolveExpression(aggregation.expression()), + aggregation.alias(), + aggregation.span())) + .toList(), + pivot.pivotColumns().stream().map(this::resolveExpression).toList(), + pivot.valueGroups().stream() + .map(group -> new PivotValueGroup( + group.values().stream().map(this::resolveExpression).toList(), + group.alias(), + group.span())) + .toList(), + pivot.groupBy().stream().map(this::resolveExpression).toList(), + pivot.span()); case SubqueryRelation subquery -> new SubqueryRelation(resolveQuery(subquery.query()), subquery.span()); case TablePlaceholder placeholder -> placeholder; case TableReference table -> table; diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java index 252fbdbbebc5..5cb2020327ef 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java @@ -248,6 +248,13 @@ private static void collectJoinCriteria(Relation relation, Builder builder) } }); } + case HogQlQuery.PivotRelation pivot -> { + collectJoinCriteria(pivot.input(), builder); + pivot.aggregations().forEach(aggregation -> collect(aggregation.expression(), builder)); + pivot.pivotColumns().forEach(column -> collect(column, builder)); + pivot.valueGroups().forEach(group -> group.values().forEach(value -> collect(value, builder))); + pivot.groupBy().forEach(expression -> collect(expression, builder)); + } } } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index bdf26c6b1082..d6529732f292 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -76,6 +76,9 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.PivotAggregation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotValueGroup; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; @@ -449,6 +452,7 @@ yield inferRelationBindings(alias.relation(), availableBindings).stream() bindings.addAll(inferRelationBindings(join.right(), availableBindings)); yield List.copyOf(bindings); } + case PivotRelation _ -> List.of(); case TableReference table -> table.parts().size() == 1 ? snapshot.logicalTable(table.parts().getFirst().value()) .map(definition -> resolveLogicalTable(definition, table.span()).bindings()) @@ -634,6 +638,11 @@ else if (alias.relation() instanceof CommonTableReference commonTable) { collectCommonTableDemands(join.left(), demand, availableBindings, demands); collectCommonTableDemands(join.right(), demand, availableBindings, demands); } + case PivotRelation pivot -> collectCommonTableDemands( + pivot.input(), + HogQlProjectionDemand.preserveAll(), + availableBindings, + demands); case SubqueryRelation _, TablePlaceholder _, TableReference _, ValuesRelation _ -> {} } } @@ -996,6 +1005,7 @@ private static Set relationQualifiers(Relation relation) qualifiers.addAll(relationQualifiers(join.right())); yield Set.copyOf(qualifiers); } + case PivotRelation _ -> Set.of(); case SubqueryRelation _ -> Set.of(); case TablePlaceholder _ -> Set.of(); case TableReference table -> Set.of(canonical(table.parts().getLast().value())); @@ -1056,6 +1066,37 @@ yield new ResolvedRelation( joinBindings, left.allLogical() && right.allLogical()); } + case PivotRelation pivot -> { + ResolvedRelation input = resolveRelation(pivot.input(), HogQlProjectionDemand.preserveAll()); + List previousBindings = bindings; + boolean previousAllRelationsLogical = allRelationsLogical; + bindings = input.bindings(); + allRelationsLogical = input.allLogical(); + List aggregations = pivot.aggregations().stream() + .map(aggregation -> new PivotAggregation( + resolveExpression(aggregation.expression()), + aggregation.alias(), + aggregation.span())) + .toList(); + List pivotColumns = pivot.pivotColumns().stream() + .map(this::resolveExpression) + .toList(); + List valueGroups = pivot.valueGroups().stream() + .map(group -> new PivotValueGroup( + group.values().stream().map(this::resolveExpression).toList(), + group.alias(), + group.span())) + .toList(); + List groupBy = pivot.groupBy().stream() + .map(this::resolveExpression) + .toList(); + bindings = previousBindings; + allRelationsLogical = previousAllRelationsLogical; + yield new ResolvedRelation( + new PivotRelation(input.relation(), aggregations, pivotColumns, valueGroups, groupBy, pivot.span()), + List.of(), + false); + } case SubqueryRelation subquery -> resolveSubquery(subquery, projectionDemand.unqualified()); case TablePlaceholder placeholder -> new ResolvedRelation(placeholder, List.of(), false); case TableReference table -> resolveTable(table); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 68d0e468696b..88b36631b5e3 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -40,6 +40,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; @@ -642,6 +643,27 @@ private static io.trino.sql.tree.Relation createRelation(Relation relation, Map< .map(TrinoAstFactory::createIdentifier) .toList()); })); + case PivotRelation pivot -> new io.trino.sql.tree.Pivot( + location(pivot.span()), + createRelation(pivot.input(), parameterIds), + pivot.aggregations().stream() + .map(aggregation -> new io.trino.sql.tree.PivotAggregation( + location(aggregation.span()), + createExpression(aggregation.expression(), parameterIds), + aggregation.alias().map(TrinoAstFactory::createIdentifier))) + .toList(), + pivot.pivotColumns().stream() + .map(column -> createExpression(column, parameterIds)) + .toList(), + pivot.valueGroups().stream() + .map(group -> new io.trino.sql.tree.PivotValueGroup( + location(group.span()), + group.values().stream() + .map(value -> createExpression(value, parameterIds)) + .toList(), + group.alias().map(TrinoAstFactory::createIdentifier))) + .toList(), + createGroupBy(pivot.groupBy(), parameterIds)); case SubqueryRelation subquery -> new TableSubquery(location(subquery.span()), createQuery(subquery.query(), parameterIds)); case TablePlaceholder _ -> throw new IllegalArgumentException("table placeholder was not validated"); case TableReference table -> createTable(table); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index 300990a0f42e..bbaf1008a9ef 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -27,6 +27,7 @@ import io.trino.sql.tree.Node; import io.trino.sql.tree.NodeLocation; import io.trino.sql.tree.Parameter; +import io.trino.sql.tree.Pivot; import io.trino.sql.tree.Predicated; import io.trino.sql.tree.Query; import io.trino.sql.tree.QuerySpecification; @@ -265,6 +266,43 @@ public void testLowersUnparenthesizedUsing() .isEqualTo(sqlParser.createStatement("SELECT * FROM events JOIN persons USING (id)")); } + @Test + public void testLowersPivotToEquivalentStockAstWithSourceLocations() + { + String hogql = "SELECT * FROM orders PIVOT (sum(totalprice) AS total FOR (orderstatus, custkey) " + + "IN (('F', 1) AS filled, ('O', 2) AS open) GROUP BY clerk)"; + + Query query = (Query) compiler.compile(hogql); + Pivot pivot = (Pivot) ((QuerySpecification) query.getQueryBody()).getFrom().orElseThrow(); + + assertThat(query).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(query))).isEqualTo(query); + assertThat(pivot.getLocation()).contains(new NodeLocation(1, hogql.indexOf("orders") + 1)); + assertThat(pivot.getAggregations()).singleElement().satisfies(aggregation -> { + assertThat(aggregation.getLocation()).contains(new NodeLocation(1, hogql.indexOf("sum(totalprice)") + 1)); + assertThat(aggregation.getAlias()).get().satisfies(alias -> + assertThat(alias.getLocation()).contains(new NodeLocation(1, hogql.indexOf("total FOR") + 1))); + }); + assertThat(pivot.getPivotColumns()).hasSize(2); + assertThat(pivot.getValueGroups()).hasSize(2); + assertThat(pivot.getValueGroups().getFirst().getLocation()) + .contains(new NodeLocation(1, hogql.indexOf("('F', 1)") + 1)); + assertThat(pivot.getValueGroups().getFirst().getAlias()).get().satisfies(alias -> + assertThat(alias.getLocation()).contains(new NodeLocation(1, hogql.indexOf("filled") + 1))); + assertThat(pivot.getGroupBy()).isPresent(); + } + + @Test + public void testBindsPlaceholdersInsidePivot() + { + HogQlCompilationResult result = compiler.compile( + "SELECT * FROM orders PIVOT (sum({amount}) FOR orderstatus IN ({status}))", + Map.of("amount", typedValue("amount"), "status", typedValue("status"))); + + assertThat(result.parameterNames()).containsExactly("amount", "status"); + assertThat(parameters(result.statement())).extracting(Parameter::getId).containsExactly(0, 1); + } + @ParameterizedTest @ValueSource(strings = { "WITH base AS (SELECT 1 AS id) SELECT id FROM base", diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java index b4fcfa817abe..b426246033b1 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java @@ -100,7 +100,7 @@ public void testPrunesUnusedLazyStarProjectionBeforeExpansion() HogQlCompilationResult result = compile("SELECT sub.event FROM (SELECT e.personProfile.*, e.* FROM events e) sub"); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( - "SELECT sub.event FROM (SELECT e.event_name AS event FROM analytics.data.raw_events e) sub")); + "SELECT sub.\"event\" AS event FROM (SELECT e.event_name AS \"event\" FROM analytics.data.raw_events e) sub")); } @Test @@ -109,8 +109,8 @@ public void testRetainsDemandedLazyStarProjectionAsStockJoin() HogQlCompilationResult result = compile("SELECT sub.name FROM (SELECT e.personProfile.*, e.* FROM events e) sub"); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( - "SELECT sub.name FROM (" + - "SELECT \"__hogql_lazy_1\".full_name AS name " + + "SELECT sub.\"name\" AS name FROM (" + + "SELECT \"__hogql_lazy_1\".full_name AS \"name\" " + "FROM analytics.data.raw_events e " + "LEFT JOIN analytics.data.raw_persons \"__hogql_lazy_1\" " + "ON e.person_id = \"__hogql_lazy_1\".person_id " + diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 0df316641d60..08920d6a4867 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -52,6 +52,9 @@ import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; import io.trino.hogql.parser.tree.HogQlQuery.NullTreatment; +import io.trino.hogql.parser.tree.HogQlQuery.PivotAggregation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotValueGroup; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; import io.trino.hogql.parser.tree.HogQlQuery.Relation; @@ -1600,6 +1603,14 @@ private Relation buildRelation(HogQLParser.JoinExprContext context) Optional.of(buildJoinCriteria(join.joinConstraintClause())), sourceSpan(context)); } + if (context instanceof HogQLParser.JoinExprPivotContext pivot) { + return buildPivot( + buildRelation(pivot.joinExpr()), + pivot.columnExprList(), + pivot.pivotColumnList(), + pivot.GROUP() != null, + context); + } throw unsupported(context, "HogQL-specific join variant"); } @@ -1639,6 +1650,14 @@ private Relation buildTableExpression(HogQLParser.TableExprContext context) } return new ValuesRelation(rows, sourceSpan(context)); } + if (context instanceof HogQLParser.TableExprPivotContext pivot) { + return buildPivot( + buildTableExpression(pivot.tableExpr()), + pivot.columnExprList(), + pivot.pivotColumnList(), + pivot.GROUP() != null, + context); + } if (context instanceof HogQLParser.TableExprAliasContext alias) { Relation relation = buildTableExpression(alias.tableExpr()); Identifier identifier = alias.identifier() != null @@ -1657,6 +1676,78 @@ private Relation buildTableExpression(HogQLParser.TableExprContext context) throw unsupported(context, "table expression"); } + private PivotRelation buildPivot( + Relation input, + List expressionLists, + HogQLParser.PivotColumnListContext pivotColumnList, + boolean hasGroupBy, + ParserRuleContext context) + { + List aggregations = expressionLists.getFirst().columnExpr().stream() + .map(this::buildPivotAggregation) + .toList(); + List pivotColumns = pivotColumnList.pivotColumn(); + if (pivotColumns.size() != 1) { + throw unsupported(pivotColumns.get(1), "multiple PIVOT column clauses"); + } + HogQLParser.PivotColumnContext pivotColumn = pivotColumns.getFirst(); + List keys = buildPivotKeys(pivotColumn.columnExprTupleOrSingle()); + List valueGroups = pivotColumn.columnExprList().columnExpr().stream() + .map(this::buildPivotValueGroup) + .toList(); + List groupBy = hasGroupBy + ? expressionLists.get(1).columnExpr().stream().map(this::buildExpression).toList() + : List.of(); + return new PivotRelation(input, aggregations, keys, valueGroups, groupBy, sourceSpan(context)); + } + + private PivotAggregation buildPivotAggregation(HogQLParser.ColumnExprContext context) + { + PivotExpression expression = buildPivotExpression(context); + return new PivotAggregation(expression.expression(), expression.alias(), sourceSpan(context)); + } + + private List buildPivotKeys(HogQLParser.ColumnExprTupleOrSingleContext context) + { + List expressions = context.columnExprList() == null + ? List.of(context.columnExpr()) + : context.columnExprList().columnExpr(); + return expressions.stream() + .map(expression -> { + Expression key = buildExpression(expression); + if (!(key instanceof ColumnReference)) { + throw unsupported(expression, "non-column PIVOT key"); + } + return key; + }) + .toList(); + } + + private PivotValueGroup buildPivotValueGroup(HogQLParser.ColumnExprContext context) + { + PivotExpression expression = buildPivotExpression(context); + List values = expression.expression() instanceof TupleExpression tuple + ? tuple.values() + : List.of(expression.expression()); + return new PivotValueGroup(values, expression.alias(), sourceSpan(context)); + } + + private PivotExpression buildPivotExpression(HogQLParser.ColumnExprContext context) + { + if (context instanceof HogQLParser.ColumnExprAliasContext alias) { + Identifier identifier = alias.identifier() != null + ? buildIdentifier(alias.identifier()) + : new Identifier( + decodeQuoted(alias.STRING_LITERAL().getText()), + true, + sourceSpan(alias.STRING_LITERAL().getSymbol(), alias.STRING_LITERAL().getSymbol())); + return new PivotExpression(buildExpression(alias.columnExpr()), Optional.of(identifier)); + } + return new PivotExpression(buildExpression(context), Optional.empty()); + } + + private record PivotExpression(Expression expression, Optional alias) {} + private CommonTableNameResolution resolveCommonTableName(String name) { Iterator> visibleScopes = commonTableScopes.iterator(); @@ -1735,6 +1826,15 @@ private void validateRelationExpressionScopes(Relation relation, Set for } }); } + case PivotRelation pivot -> { + validateRelationExpressionScopes(pivot.input(), forbiddenOuterRelations, localRelations); + pivot.aggregations().forEach(aggregation -> + validateExpressionScope(aggregation.expression(), forbiddenOuterRelations, localRelations)); + pivot.pivotColumns().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); + pivot.valueGroups().forEach(group -> group.values().forEach(expression -> + validateExpressionScope(expression, forbiddenOuterRelations, localRelations))); + pivot.groupBy().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); + } case CommonTableReference _, SubqueryRelation _, TablePlaceholder _, TableReference _, ValuesRelation _ -> {} } } @@ -1747,6 +1847,7 @@ private void validateNestedRelationScopes(Relation relation, Set forbidd validateNestedRelationScopes(join.left(), forbiddenOuterRelations); validateNestedRelationScopes(join.right(), forbiddenOuterRelations); } + case PivotRelation pivot -> validateNestedRelationScopes(pivot.input(), forbiddenOuterRelations); case SubqueryRelation subquery -> validateQueryScope(subquery.query(), forbiddenOuterRelations); case ValuesRelation values -> values.rows().forEach(row -> row.forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, Set.of()))); case CommonTableReference _, TablePlaceholder _, TableReference _ -> {} @@ -1769,6 +1870,7 @@ private void collectRelationNames(Relation relation, Set names) collectRelationNames(join.left(), names); collectRelationNames(join.right(), names); } + case PivotRelation pivot -> collectRelationNames(pivot.input(), names); case SubqueryRelation _, TablePlaceholder _, ValuesRelation _ -> {} case TableReference table -> names.add(canonicalName(table.parts().getLast())); } diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index e531da048bf9..5e7509d0c2e7 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -743,6 +743,7 @@ public sealed interface Relation permits AliasedRelation, CommonTableReference, JoinRelation, + PivotRelation, SubqueryRelation, TablePlaceholder, TableReference, @@ -866,6 +867,58 @@ public record JoinRelation(JoinType type, Relation left, Relation right, Optiona } } + public record PivotRelation( + Relation input, + List aggregations, + List pivotColumns, + List valueGroups, + List groupBy, + SourceSpan span) + implements Relation + { + public PivotRelation + { + input = requireNonNull(input, "input is null"); + aggregations = List.copyOf(requireNonNull(aggregations, "aggregations is null")); + pivotColumns = List.copyOf(requireNonNull(pivotColumns, "pivotColumns is null")); + valueGroups = List.copyOf(requireNonNull(valueGroups, "valueGroups is null")); + groupBy = List.copyOf(requireNonNull(groupBy, "groupBy is null")); + span = requireNonNull(span, "span is null"); + if (aggregations.isEmpty()) { + throw new IllegalArgumentException("aggregations is empty"); + } + if (pivotColumns.isEmpty()) { + throw new IllegalArgumentException("pivotColumns is empty"); + } + if (valueGroups.isEmpty()) { + throw new IllegalArgumentException("valueGroups is empty"); + } + } + } + + public record PivotAggregation(Expression expression, Optional alias, SourceSpan span) + { + public PivotAggregation + { + expression = requireNonNull(expression, "expression is null"); + alias = requireNonNull(alias, "alias is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record PivotValueGroup(List values, Optional alias, SourceSpan span) + { + public PivotValueGroup + { + values = List.copyOf(requireNonNull(values, "values is null")); + alias = requireNonNull(alias, "alias is null"); + span = requireNonNull(span, "span is null"); + if (values.isEmpty()) { + throw new IllegalArgumentException("values is empty"); + } + } + } + public record TableReference(List parts, SourceSpan span) implements Relation { diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 698b8fbfd195..6c88676e951f 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -32,6 +32,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; +import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; @@ -313,6 +314,55 @@ public void testBuildsAliasedJoinAstWithSourceSpans() assertThat(criteria.span()).isEqualTo(new HogQlQuery.SourceSpan(52, 73, 3, 24, 3, 45)); } + @Test + public void testBuildsPivotAstWithCompositeKeysAliasesAndSourceSpans() + { + String hogql = "SELECT * FROM orders PIVOT (sum(totalprice) AS total FOR (orderstatus, custkey) " + + "IN (('F', 1) AS filled, ('O', 2) AS open) GROUP BY clerk)"; + + PivotRelation pivot = (PivotRelation) parser.parseStatement(hogql).from().orElseThrow(); + + assertThat(pivot.input()).isInstanceOf(HogQlQuery.TableReference.class); + assertThat(pivot.span()).isEqualTo(span(hogql, hogql.substring(hogql.indexOf("orders")))); + assertThat(pivot.aggregations()).singleElement().satisfies(aggregation -> { + assertThat(aggregation.expression()).isInstanceOf(FunctionCall.class); + assertThat(aggregation.alias()).get().extracting(HogQlQuery.Identifier::value).isEqualTo("total"); + assertThat(aggregation.span()).isEqualTo(span(hogql, "sum(totalprice) AS total")); + }); + assertThat(pivot.pivotColumns()).extracting(expression -> ((ColumnReference) expression).parts().getLast().value()) + .containsExactly("orderstatus", "custkey"); + assertThat(pivot.pivotColumns().getFirst().span()).isEqualTo(span(hogql, "orderstatus")); + assertThat(pivot.valueGroups()).hasSize(2); + assertThat(pivot.valueGroups().getFirst().values()).hasSize(2); + assertThat(pivot.valueGroups().getFirst().alias()).get().extracting(HogQlQuery.Identifier::value).isEqualTo("filled"); + assertThat(pivot.valueGroups().getFirst().span()).isEqualTo(span(hogql, "('F', 1) AS filled")); + assertThat(pivot.groupBy()).singleElement().isInstanceOf(ColumnReference.class); + } + + @Test + public void testAttachesPivotToWholeJoin() + { + HogQlQuery query = parser.parseStatement( + "SELECT * FROM orders o JOIN customer c ON o.custkey = c.custkey " + + "PIVOT (count(*) FOR o.orderstatus IN ('F'))"); + + assertThat(query.from()).get().isInstanceOfSatisfying(PivotRelation.class, pivot -> + assertThat(pivot.input()).isInstanceOf(JoinRelation.class)); + } + + @Test + public void testRejectsPivotShapesWithoutAStockRepresentation() + { + assertThatThrownBy(() -> parser.parseStatement( + "SELECT * FROM orders PIVOT (sum(totalprice) FOR orderstatus IN ('F') custkey IN (1))")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("multiple PIVOT column clauses"); + assertThatThrownBy(() -> parser.parseStatement( + "SELECT * FROM orders PIVOT (sum(totalprice) FOR orderstatus + 1 IN ('F'))")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("non-column PIVOT key"); + } + @Test public void testBuildsCteAndDerivedTableAstWithSourceSpans() { @@ -472,4 +522,10 @@ private static Stream nodes(Element element) } return Stream.concat(Stream.of(node), node.children().stream().flatMap(TestHogQlParser::nodes)); } + + private static HogQlQuery.SourceSpan span(String source, String value) + { + int start = source.indexOf(value); + return new HogQlQuery.SourceSpan(start, start + value.length(), 1, start + 1, 1, start + value.length() + 1); + } } diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlPivot.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlPivot.java new file mode 100644 index 000000000000..9d0088cbe04e --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlPivot.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.sql.planner; + +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.sql.SqlFormatter; +import io.trino.sql.planner.assertions.BasePlanTest; +import io.trino.sql.planner.optimizations.PlanNodeSearcher; +import io.trino.sql.planner.plan.AggregationNode; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static io.trino.sql.planner.LogicalPlanner.Stage.CREATED; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlPivot + extends BasePlanTest +{ + private final HogQlCompiler compiler = new HogQlCompiler(); + + @Test + public void testLowersToStockFilteredAggregations() + { + String hogql = "SELECT custkey, filled_total, open_total FROM orders " + + "PIVOT (sum(totalprice) AS total FOR orderstatus IN ('F' AS filled, 'O' AS open) GROUP BY custkey)"; + + Plan plan = plan(SqlFormatter.formatSql(compiler.compile(hogql)), CREATED); + + List aggregations = PlanNodeSearcher.searchFrom(plan.getRoot()) + .whereIsInstanceOfAny(AggregationNode.class) + .findAll().stream() + .map(AggregationNode.class::cast) + .toList(); + assertThat(aggregations).singleElement().satisfies(aggregation -> { + assertThat(aggregation.getGroupingKeys()).hasSize(1); + assertThat(aggregation.getAggregations()).hasSize(2); + assertThat(aggregation.getAggregations().values()) + .allSatisfy(value -> assertThat(value.getFilter()).isPresent()); + }); + } +} From 8ef8d1d8f504c0ebfaba7799155fb5484252362f Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 21:24:07 -0400 Subject: [PATCH 074/136] feat(hogql): freeze the v0 compiler profile --- core/trino-hogql-compiler/README.md | 31 ++- .../trino/hogql/compiler/HogQlCompiler.java | 49 +++- .../hogql/compiler/HogQlFunctionResolver.java | 33 ++- .../hogql/compiler/HogQlSemanticResolver.java | 16 +- .../compiler/HogQlV0FunctionRegistry.java | 135 +++++++++++ .../compiler/HogQlV0ProfileValidator.java | 227 ++++++++++++++++++ .../trino/hogql/compiler/TrinoAstFactory.java | 33 ++- ...syncHogQlSemanticCatalogSnapshotCache.java | 202 +++++++++++++--- .../catalog/HogQlSemanticCatalogSnapshot.java | 2 +- .../HogQlSemanticCatalogSnapshotCache.java | 9 + .../HogQlSemanticCatalogSnapshotProvider.java | 2 +- .../compiler/TestHogQlFunctionResolver.java | 52 +++- .../TestHogQlRelationshipExpansion.java | 51 +++- ...syncHogQlSemanticCatalogSnapshotCache.java | 149 +++++++++++- ...tHogQlSemanticCatalogSnapshotProvider.java | 26 ++ .../io/trino/execution/QueryPreparer.java | 3 +- .../TestHogQlStatementResource.java | 58 ++++- ...TestHogQlQueryPreparerSemanticCatalog.java | 123 ++-------- .../planner/TestHogQlProjectionPruning.java | 32 ++- 19 files changed, 1027 insertions(+), 206 deletions(-) create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 0e10794d8b7a..e4278dfc6300 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -43,10 +43,37 @@ explicit error for valid constructs that do not yet have a stock Trino AST mapping. Adding syntax support therefore cannot silently claim semantic or execution parity. +## MVP executable contract + +The MVP lowers a frozen, fail-closed subset directly to stock Trino AST nodes: + +- logical `events` and `persons` tables supplied by a pinned Duckgres manifest; +- scalar VARCHAR JSON property reads and the `events.person` lazy relationship; +- parameters, CTEs, subqueries, joins, set operations, arrays, maps, rows, + intervals, grouping, ordering, limits, and windows that have exact stock + Trino representations; +- numeric/conditional/string functions (`abs`, `coalesce`, `if`, `lower`, `upper`, `length`, + `concat`, `replace`), collection functions (`map`, `arraySort`, + `arrayDistinct`, `arrayFlatten`, `arrayStringConcat`), date functions + (`dateAdd`, `dateDiff`, `dateTrunc`), aggregates (`count`, `sum`, `min`, + `max`, `avg`, `any`, `argMin`, `argMax`, `array_agg`), and windows + (`first_value`, `rank`, `row_number`). + +Function aliases are resolved before Trino analysis. Unknown functions fail as +HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, +HogQLX, PIVOT/UNPIVOT execution, ClickHouse-only clauses, advanced lambda and +table functions, and complete type/function parity remain outside the MVP. + Set `hogql.enabled=true` on the coordinator to register `POST /v1/hogql`. The -request body is raw HogQL text and the response uses Trino's existing statement -protocol. Follow-up result and cancellation URIs remain under `/v1/statement`. +request body is the versioned JSON envelope containing the raw HogQL query, +language/protocol versions, and optional typed bindings. The response uses +Trino's existing statement protocol. Follow-up result and cancellation URIs +remain under `/v1/statement`. The endpoint is absent when the property is false, which is the default. +Production compilation uses a dedicated fixed-size executor with a bounded +queue and the `hogql.compilation-timeout` wall-time limit. Saturation and +timeout return retryable insufficient-resource errors; standard SQL bypasses +the compiler executor. Development is based on the fork's `ducklake-connector` integration branch. At the start of this milestone its head was `posthog-484-ducklake.9`, commit diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index 437a71db2f2b..244ce689bcb9 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -114,14 +114,32 @@ public HogQlCompilationResult compile(HogQlCompileEnvelope envelope, Optional catalogContext) + { + requireNonNull(envelope, "envelope is null"); + requireNonNull(catalogContext, "catalogContext is null"); + if (!envelope.modifiers().isEmpty()) { + throw unsupportedError(parse(envelope.query()).span(), "Modifiers are outside the HogQL v0 profile"); + } + return compile( + parse(envelope.query()), + envelope.parameters(), + Map.of(), + catalogContext, + envelope.languageVersion(), + envelope.catalogGeneration(), + true); } public HogQlCompilationResult compile(String hogql, Map parameters) { requireNonNull(hogql, "hogql is null"); requireNonNull(parameters, "parameters is null"); - return compile(parse(hogql), parameters, Map.of(), Optional.empty(), HogQlLanguageContract.current().languageVersion(), OptionalLong.empty()); + return compile(parse(hogql), parameters, Map.of(), Optional.empty(), HogQlLanguageContract.current().languageVersion(), OptionalLong.empty(), false); } private HogQlQuery parse(String hogql) @@ -145,7 +163,8 @@ private static HogQlCompilationResult compile( Map modifiers, Optional catalogContext, HogQlLanguageVersion languageVersion, - OptionalLong expectedCatalogGeneration) + OptionalLong expectedCatalogGeneration, + boolean v0Profile) { validateParameters(query, parameters); validateQuery(query); @@ -183,7 +202,10 @@ private static HogQlCompilationResult compile( for (int index = 0; index < placeholders.size(); index++) { parameterIds.put(placeholders.get(index).span(), index); } - ResolvedQuery resolved = resolveQuery(query, catalogContext, languageVersion, expectedCatalogGeneration, !modifiers.isEmpty()); + if (v0Profile) { + HogQlV0ProfileValidator.validate(query, Optional.empty()); + } + ResolvedQuery resolved = resolveQuery(query, catalogContext, languageVersion, expectedCatalogGeneration, !modifiers.isEmpty(), v0Profile); Statement statement = TrinoAstFactory.createStatement(resolved.query(), parameterIds); List modifierBindings = resolved.pinnedSnapshot() .map(snapshot -> HogQlModifierResolver.resolve(snapshot, modifiers, query.span())) @@ -231,7 +253,8 @@ private static ResolvedQuery resolveQuery( Optional catalogContext, HogQlLanguageVersion languageVersion, OptionalLong expectedCatalogGeneration, - boolean modifiersRequireSnapshot) + boolean modifiersRequireSnapshot, + boolean v0Profile) { boolean semanticCandidate = containsSemanticCandidate(query); if (!semanticCandidate && !modifiersRequireSnapshot) { @@ -241,16 +264,19 @@ private static ResolvedQuery resolveQuery( if (modifiersRequireSnapshot) { throw new HogQlSemanticCatalogException(Failure.UNAVAILABLE, "HogQL semantic catalog snapshot is required for modifiers"); } - return new ResolvedQuery(query, Optional.empty()); + return new ResolvedQuery(containsFunctionCall(query) ? HogQlFunctionResolver.resolve(query) : query, Optional.empty()); } HogQlSemanticCatalogContext context = catalogContext.orElseThrow(); PinnedSnapshot pinned = context.snapshotProvider().pin(new PinRequest( context.catalog(), requireNonNull(languageVersion, "languageVersion is null"), expectedCatalogGeneration)); + if (v0Profile) { + HogQlV0ProfileValidator.validate(query, Optional.of(pinned.snapshot())); + } HogQlQuery resolved = query; if (semanticCandidate) { - HogQlQuery functionsResolved = HogQlFunctionResolver.resolve(pinned, query); + HogQlQuery functionsResolved = v0Profile ? HogQlFunctionResolver.resolveV0(query) : HogQlFunctionResolver.resolve(pinned, query); resolved = HogQlSemanticResolver.resolve(pinned, functionsResolved) .map(HogQlSemanticResolver.ResolvedQuery::query) .orElse(functionsResolved); @@ -544,4 +570,13 @@ private static TrinoException bindingError(SourceSpan span, String message) message, null); } + + private static TrinoException unsupportedError(SourceSpan span, String message) + { + return new TrinoException( + io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index bf9b20f52430..5a99507d7773 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -74,12 +74,11 @@ import io.trino.spi.Location; import io.trino.spi.TrinoException; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; @@ -90,18 +89,35 @@ final class HogQlFunctionResolver private static final String MATCHES_ACTION = "matchesaction"; private final Map functions; + private final boolean semanticCatalogAvailable; - private HogQlFunctionResolver(PinnedSnapshot snapshot) + private HogQlFunctionResolver(Optional snapshot) { - functions = snapshot.snapshot().functions().stream() - .collect(Collectors.toUnmodifiableMap(function -> canonical(function.name()), Function.identity())); + Map functions = new LinkedHashMap<>(HogQlV0FunctionRegistry.functions()); + snapshot.stream() + .flatMap(pinned -> pinned.snapshot().functions().stream()) + .forEach(function -> functions.putIfAbsent(canonical(function.name()), function)); + this.functions = Map.copyOf(functions); + semanticCatalogAvailable = snapshot.isPresent(); } public static HogQlQuery resolve(PinnedSnapshot snapshot, HogQlQuery query) { requireNonNull(snapshot, "snapshot is null"); requireNonNull(query, "query is null"); - return new HogQlFunctionResolver(snapshot).resolveQuery(query); + return new HogQlFunctionResolver(Optional.of(snapshot)).resolveQuery(query); + } + + public static HogQlQuery resolve(HogQlQuery query) + { + requireNonNull(query, "query is null"); + return new HogQlFunctionResolver(Optional.empty()).resolveQuery(query); + } + + public static HogQlQuery resolveV0(HogQlQuery query) + { + requireNonNull(query, "query is null"); + return new HogQlFunctionResolver(Optional.empty()).resolveQuery(query); } private HogQlQuery resolveQuery(HogQlQuery query) @@ -276,7 +292,10 @@ private Expression resolveExpression(Expression expression) private Expression resolveFunction(FunctionCall function, boolean windowInvocation) { String name = function.name().value(); - if (function.nameParts().size() == 1 && canonical(name).equals(MATCHES_ACTION)) { + if (function.nameParts().size() != 1) { + throw unsupportedError(function, "Qualified HogQL functions are not supported"); + } + if (semanticCatalogAvailable && function.nameParts().size() == 1 && canonical(name).equals(MATCHES_ACTION)) { return new FunctionCall( function.nameParts(), function.arguments().stream().map(this::resolveExpression).toList(), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index d6529732f292..af09f2cb5800 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -2206,6 +2206,20 @@ private Expression expandOperator( case IS_NULL -> new IsNullExpression(arguments.getFirst(), false, span, span); case IS_NOT_NULL -> new IsNullExpression(arguments.getFirst(), true, span, span); case SUBSCRIPT -> new SubscriptExpression(arguments.getFirst(), arguments.getLast(), span); + case JSON_OBJECT_LOOKUP -> new SubscriptExpression( + new CastExpression( + new FunctionCall( + new Identifier("json_parse", false, span), + List.of(arguments.getFirst()), + false, + List.of(), + Optional.empty(), + span), + new Identifier("map(varchar, json)", false, span), + false, + span), + arguments.getLast(), + span); default -> new BinaryExpression(binaryOperator(operator.operator()), arguments.getFirst(), arguments.getLast(), span); }; } @@ -2290,7 +2304,7 @@ private static HogQlQuery.BinaryOperator binaryOperator(SemanticOperator operato case GREATER_THAN_OR_EQUAL -> HogQlQuery.BinaryOperator.GREATER_THAN_OR_EQUAL; case AND -> HogQlQuery.BinaryOperator.AND; case OR -> HogQlQuery.BinaryOperator.OR; - case NOT, NEGATE, IS_NULL, IS_NOT_NULL, SUBSCRIPT -> throw new IllegalArgumentException("operator cannot be lowered as binary"); + case NOT, NEGATE, IS_NULL, IS_NOT_NULL, SUBSCRIPT, JSON_OBJECT_LOOKUP -> throw new IllegalArgumentException("operator cannot be lowered as binary"); }; } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java new file mode 100644 index 000000000000..ec0adbd6d6ab --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -0,0 +1,135 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +final class HogQlV0FunctionRegistry +{ + private static final Map FUNCTIONS = functions( + scalar("coalesce", "coalesce", variadicSignature(1)), + scalar("if", "if", signature(3)), + scalar("abs", "abs", signature(1)), + scalar("lower", "lower", signature(1)), + scalar("upper", "upper", signature(1)), + scalar("length", "length", signature(1)), + scalar("concat", "concat", variadicSignature(2)), + scalar("replace", "replace", signature(3)), + scalar("map", "map", signature(2)), + scalar("dateDiff", "date_diff", signature(3)), + scalar("dateAdd", "date_add", signature(3)), + scalar("dateTrunc", "date_trunc", signature(2)), + scalar("arraySort", "array_sort", signature(1)), + scalar("arrayDistinct", "array_distinct", signature(1)), + scalar("arrayFlatten", "flatten", signature(1)), + scalar("arrayStringConcat", "array_join", signature(1), signature(2)), + aggregate("count", "count", true, signature(0), signature(1)), + aggregate("sum", "sum", true, signature(1)), + aggregate("min", "min", true, signature(1)), + aggregate("max", "max", true, signature(1)), + aggregate("avg", "avg", true, signature(1)), + aggregateWithOrderBy("array_agg", "array_agg", true, signature(1)), + nondeterministicAggregate("any", "arbitrary", false, signature(1)), + nondeterministicAggregate("argMin", "min_by", false, signature(2)), + nondeterministicAggregate("argMax", "max_by", false, signature(2)), + window("rank", "rank", signature(0)), + window("first_value", "first_value", signature(1)), + window("row_number", "row_number", signature(0))); + + private HogQlV0FunctionRegistry() {} + + public static Map functions() + { + return FUNCTIONS; + } + + private static Map functions(FunctionCapabilityDefinition... functions) + { + Map result = new LinkedHashMap<>(); + for (FunctionCapabilityDefinition function : functions) { + FunctionCapabilityDefinition previous = result.put(function.name().toLowerCase(java.util.Locale.ENGLISH), function); + if (previous != null) { + throw new IllegalArgumentException("duplicate HogQL v0 function"); + } + } + return Map.copyOf(result); + } + + private static FunctionCapabilityDefinition scalar(String hogQlName, String trinoName, FunctionSignature... signatures) + { + return function(hogQlName, trinoName, FunctionKind.SCALAR, false, false, false, false, true, signatures); + } + + private static FunctionCapabilityDefinition aggregate(String hogQlName, String trinoName, boolean supportsDistinct, FunctionSignature... signatures) + { + return function(hogQlName, trinoName, FunctionKind.AGGREGATE, supportsDistinct, false, true, true, true, signatures); + } + + private static FunctionCapabilityDefinition aggregateWithOrderBy(String hogQlName, String trinoName, boolean supportsDistinct, FunctionSignature... signatures) + { + return function(hogQlName, trinoName, FunctionKind.AGGREGATE, supportsDistinct, true, true, true, true, signatures); + } + + private static FunctionCapabilityDefinition nondeterministicAggregate(String hogQlName, String trinoName, boolean supportsDistinct, FunctionSignature... signatures) + { + return function(hogQlName, trinoName, FunctionKind.AGGREGATE, supportsDistinct, false, true, true, false, signatures); + } + + private static FunctionCapabilityDefinition window(String hogQlName, String trinoName, FunctionSignature... signatures) + { + return function(hogQlName, trinoName, FunctionKind.WINDOW, false, false, false, true, true, signatures); + } + + private static FunctionCapabilityDefinition function( + String hogQlName, + String trinoName, + FunctionKind kind, + boolean supportsDistinct, + boolean supportsOrderBy, + boolean supportsFilter, + boolean supportsWindow, + boolean deterministic, + FunctionSignature... signatures) + { + return new FunctionCapabilityDefinition( + hogQlName, + kind, + FunctionImplementation.STOCK, + List.of(new PhysicalIdentifier(trinoName, false)), + List.of(signatures), + deterministic, + supportsDistinct, + supportsOrderBy, + supportsFilter, + supportsWindow); + } + + private static FunctionSignature signature(int arity) + { + return new FunctionSignature(java.util.stream.Stream.generate(() -> "any").limit(arity).toList(), "any", false); + } + + private static FunctionSignature variadicSignature(int minimumArity) + { + return new FunctionSignature(java.util.stream.Stream.generate(() -> "any").limit(minimumArity + 1L).toList(), "any", true); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java new file mode 100644 index 000000000000..603195a45b6e --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java @@ -0,0 +1,227 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; +import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; +import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; +import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; +import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; + +import java.util.Locale; +import java.util.Optional; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static java.util.Objects.requireNonNull; + +final class HogQlV0ProfileValidator +{ + private HogQlV0ProfileValidator() {} + + public static void validate(HogQlQuery query, Optional snapshot) + { + requireNonNull(query, "query is null"); + requireNonNull(snapshot, "snapshot is null"); + query.with().forEach(commonTable -> validate(commonTable.query(), snapshot)); + switch (query.body()) { + case SelectQueryBody select -> { + select.projections().forEach(projection -> validate(projection, snapshot)); + select.from().ifPresent(relation -> validate(relation, snapshot)); + select.where().ifPresent(expression -> validate(expression, snapshot)); + select.groupBy().forEach(expression -> validate(expression, snapshot)); + select.having().ifPresent(expression -> validate(expression, snapshot)); + select.windows().forEach(window -> validate(window.specification(), snapshot)); + } + case SetOperation setOperation -> { + validate(setOperation.left(), snapshot); + validate(setOperation.right(), snapshot); + } + } + query.orderBy().forEach(sortItem -> validate(sortItem.expression(), snapshot)); + query.limit().ifPresent(expression -> validate(expression, snapshot)); + query.offset().ifPresent(expression -> validate(expression, snapshot)); + } + + private static void validate(Projection projection, Optional snapshot) + { + switch (projection) { + case ColumnsList columns -> columns.expressions().forEach(expression -> validate(expression, snapshot)); + case ColumnsRegex _ -> {} + case ExpressionProjection expression -> validate(expression.expression(), snapshot); + case Star star -> star.replacements().forEach(replacement -> validate(replacement.expression(), snapshot)); + } + } + + private static void validate(Relation relation, Optional snapshot) + { + switch (relation) { + case AliasedRelation alias -> validate(alias.relation(), snapshot); + case CommonTableReference _, TablePlaceholder _ -> {} + case JoinRelation join -> { + validate(join.left(), snapshot); + validate(join.right(), snapshot); + join.criteria().filter(JoinOn.class::isInstance) + .map(JoinOn.class::cast) + .ifPresent(on -> validate(on.expression(), snapshot)); + } + case HogQlQuery.PivotRelation pivot -> throw unsupported(pivot.span(), "PIVOT is outside the HogQL v0 profile"); + case SubqueryRelation subquery -> validate(subquery.query(), snapshot); + case TableReference table -> validateTable(table, snapshot); + case ValuesRelation values -> values.rows().forEach(row -> row.forEach(expression -> validate(expression, snapshot))); + } + } + + private static void validateTable(TableReference table, Optional snapshot) + { + if (table.parts().size() != 1 || snapshot.isEmpty()) { + return; + } + String name = canonical(table.parts().getFirst().value()); + HogQlSemanticCatalogSnapshot catalog = snapshot.orElseThrow(); + if (catalog.logicalTable(name).isPresent() && !name.equals("events") && !name.equals("persons")) { + throw unsupported(table.span(), "Logical table " + table.parts().getFirst().value() + " is outside the HogQL v0 profile"); + } + boolean deferredRelation = catalog.virtualTables().stream().anyMatch(tableDefinition -> canonical(tableDefinition.name()).equals(name)) || + catalog.savedQueries().stream().anyMatch(query -> canonical(query.name()).equals(name)) || + catalog.materializedViews().stream().anyMatch(view -> canonical(view.name()).equals(name)); + if (deferredRelation) { + throw unsupported(table.span(), "Semantic relation " + table.parts().getFirst().value() + " is outside the HogQL v0 profile"); + } + } + + private static void validate(Expression expression, Optional snapshot) + { + switch (expression) { + case ArrayExpression array -> array.values().forEach(value -> validate(value, snapshot)); + case BetweenExpression between -> { + validate(between.value(), snapshot); + validate(between.min(), snapshot); + validate(between.max(), snapshot); + } + case BinaryExpression binary -> { + validate(binary.left(), snapshot); + validate(binary.right(), snapshot); + } + case CaseExpression caseExpression -> { + caseExpression.operand().ifPresent(value -> validate(value, snapshot)); + caseExpression.whenClauses().forEach(when -> { + validate(when.operand(), snapshot); + validate(when.result(), snapshot); + }); + caseExpression.defaultValue().ifPresent(value -> validate(value, snapshot)); + } + case CastExpression cast -> validate(cast.value(), snapshot); + case ColumnReference _, Literal _, Placeholder _ -> {} + case FunctionCall function -> { + if (function.nameParts().size() != 1) { + throw unsupported(function.span(), "Qualified functions are outside the HogQL v0 profile"); + } + if (canonical(function.name().value()).equals("matchesaction")) { + throw unsupported(function.span(), "Actions are outside the HogQL v0 profile"); + } + function.arguments().forEach(argument -> validate(argument, snapshot)); + function.orderBy().forEach(sortItem -> validate(sortItem.expression(), snapshot)); + function.filter().ifPresent(filter -> validate(filter, snapshot)); + function.window().ifPresent(window -> validate(window, snapshot)); + } + case InCohortExpression in -> throw unsupported(in.span(), "Cohorts are outside the HogQL v0 profile"); + case InExpression in -> { + validate(in.value(), snapshot); + in.values().forEach(value -> validate(value, snapshot)); + } + case InSubqueryExpression in -> { + validate(in.value(), snapshot); + validate(in.query(), snapshot); + } + case IntervalExpression interval -> validate(interval.value(), snapshot); + case IsNullExpression isNull -> validate(isNull.value(), snapshot); + case MemberAccessExpression memberAccess -> validate(memberAccess.base(), snapshot); + case ScalarSubqueryExpression subquery -> validate(subquery.query(), snapshot); + case SubscriptExpression subscript -> { + validate(subscript.base(), snapshot); + validate(subscript.index(), snapshot); + } + case TupleExpression tuple -> tuple.values().forEach(value -> validate(value, snapshot)); + case UnaryExpression unary -> validate(unary.operand(), snapshot); + } + } + + private static void validate(Window window, Optional snapshot) + { + switch (window) { + case WindowReference _ -> {} + case WindowSpecification specification -> { + specification.partitionBy().forEach(expression -> validate(expression, snapshot)); + specification.orderBy().forEach(sortItem -> validate(sortItem.expression(), snapshot)); + specification.frame().ifPresent(frame -> { + frame.start().value().ifPresent(value -> validate(value, snapshot)); + frame.end().flatMap(HogQlQuery.FrameBound::value).ifPresent(value -> validate(value, snapshot)); + }); + } + } + } + + private static String canonical(String value) + { + return value.toLowerCase(Locale.ENGLISH); + } + + private static TrinoException unsupported(SourceSpan span, String message) + { + return new TrinoException( + HOGQL_UNSUPPORTED_FEATURE, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 88b36631b5e3..5573515a604d 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -72,11 +72,13 @@ import io.trino.sql.tree.BooleanLiteral; import io.trino.sql.tree.CallArgument; import io.trino.sql.tree.Cast; +import io.trino.sql.tree.CoalesceExpression; import io.trino.sql.tree.ComparisonPredicate; import io.trino.sql.tree.DereferenceExpression; import io.trino.sql.tree.Except; import io.trino.sql.tree.Expression; import io.trino.sql.tree.GroupBy; +import io.trino.sql.tree.IfExpression; import io.trino.sql.tree.InListExpression; import io.trino.sql.tree.InPredicate; import io.trino.sql.tree.Intersect; @@ -117,6 +119,7 @@ import io.trino.sql.tree.With; import io.trino.sql.tree.WithQuery; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -421,6 +424,29 @@ private static Expression createFunctionCall(FunctionCall function, Map arguments = function.arguments().stream() + .map(argument -> createExpression(argument, parameterIds)) + .toList(); + if (function.nameParts().size() == 1 && function.name().value().equalsIgnoreCase("coalesce")) { + if (arguments.size() == 1) { + return arguments.getFirst(); + } + return new CoalesceExpression(location(function.span()), arguments); + } + if (function.nameParts().size() == 1 && function.name().value().equalsIgnoreCase("if")) { + return new IfExpression( + location(function.span()), + arguments.get(0), + arguments.get(1), + arguments.size() == 3 ? arguments.get(2) : null); + } + List callArguments = new ArrayList<>(); + for (int index = 0; index < arguments.size(); index++) { + callArguments.add(new CallArgument(location(function.arguments().get(index).span()), Optional.empty(), arguments.get(index))); + } + if (function.nameParts().size() == 1 && function.name().value().equalsIgnoreCase("array_join") && callArguments.size() == 1) { + callArguments.add(new CallArgument(location(function.span()), Optional.empty(), new StringLiteral(location(function.span()), ""))); + } return new io.trino.sql.tree.FunctionCall( location(function.span()), QualifiedName.of(function.nameParts().stream() @@ -432,12 +458,7 @@ private static Expression createFunctionCall(FunctionCall function, Map io.trino.sql.tree.FunctionCall.NullTreatment.IGNORE), Optional.empty(), - function.arguments().stream() - .map(argument -> new CallArgument( - location(argument.span()), - Optional.empty(), - createExpression(argument, parameterIds))) - .toList()); + callArguments); } private static io.trino.sql.tree.WindowDefinition createWindowDefinition( diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlSemanticCatalogSnapshotCache.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlSemanticCatalogSnapshotCache.java index 2a04d5b2691e..db1eb7dcd1b2 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlSemanticCatalogSnapshotCache.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlSemanticCatalogSnapshotCache.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; @@ -42,7 +43,8 @@ public final class BoundedAsyncHogQlSemanticCatalogSnapshotCache private final LongSupplier ticker; private final Executor executor; private final SnapshotLoader loader; - private final LinkedHashMap entries = new LinkedHashMap<>(16, 0.75f, true); + private final LinkedHashMap entries = new LinkedHashMap<>(16, 0.75f, true); + private final LinkedHashMap observedGenerations = new LinkedHashMap<>(16, 0.75f, true); public BoundedAsyncHogQlSemanticCatalogSnapshotCache( int maximumEntries, @@ -70,24 +72,38 @@ public BoundedAsyncHogQlSemanticCatalogSnapshotCache( @Override public Optional currentSnapshot(PhysicalIdentifier catalog) + { + return currentSnapshot(catalog, OptionalLong.empty()); + } + + @Override + public Optional currentSnapshot(PhysicalIdentifier catalog, OptionalLong expectedGeneration) { requireNonNull(catalog, "catalog is null"); + requireNonNull(expectedGeneration, "expectedGeneration is null"); + CacheKey cacheKey = new CacheKey(catalog, expectedGeneration); RefreshTask refreshTask; Optional result; - List> evictedRefreshes = new ArrayList<>(1); + List evictedRefreshes = new ArrayList<>(1); long now = ticker.getAsLong(); synchronized (lock) { - Entry entry = entries.get(catalog); + Entry entry = entries.get(cacheKey); if (entry == null) { - entry = new Entry(); - entries.put(catalog, entry); - evictEntries(evictedRefreshes); + entry = inheritedExactEntry(cacheKey, now).orElseGet(Entry::new); + entries.put(cacheKey, entry); } HogQlSemanticCatalogSnapshot snapshot = entry.snapshot; long age = snapshot == null ? Long.MAX_VALUE : elapsedNanos(entry.loadedAtNanos, now); result = snapshot != null && age < expireAfterNanos ? Optional.of(snapshot) : Optional.empty(); - refreshTask = (snapshot == null || age >= refreshAfterNanos) ? prepareRefresh(catalog, entry, now, false) : null; + boolean refreshRequired = snapshot == null || + (expectedGeneration.isEmpty() && age >= refreshAfterNanos) || + (expectedGeneration.isPresent() && age >= expireAfterNanos); + refreshTask = refreshRequired ? prepareRefresh(cacheKey, entry, now, false) : null; + evictEntries(evictedRefreshes); + if (!entries.containsKey(cacheKey)) { + refreshTask = null; + } } cancelRefreshes(evictedRefreshes, "HogQL semantic catalog cache entry was evicted"); dispatch(refreshTask); @@ -95,20 +111,38 @@ public Optional currentSnapshot(PhysicalIdentifier } public CompletionStage prewarm(PhysicalIdentifier catalog) + { + return prewarm(catalog, OptionalLong.empty()); + } + + public CompletionStage prewarm(PhysicalIdentifier catalog, OptionalLong expectedGeneration) { requireNonNull(catalog, "catalog is null"); + requireNonNull(expectedGeneration, "expectedGeneration is null"); + CacheKey cacheKey = new CacheKey(catalog, expectedGeneration); RefreshTask refreshTask; CompletableFuture future; - List> evictedRefreshes = new ArrayList<>(1); + List evictedRefreshes = new ArrayList<>(1); + long now = ticker.getAsLong(); synchronized (lock) { - Entry entry = entries.get(catalog); + Entry entry = entries.get(cacheKey); if (entry == null) { - entry = new Entry(); - entries.put(catalog, entry); - evictEntries(evictedRefreshes); + entry = inheritedExactEntry(cacheKey, now).orElseGet(Entry::new); + entries.put(cacheKey, entry); + } + boolean unexpired = entry.snapshot != null && elapsedNanos(entry.loadedAtNanos, now) < expireAfterNanos; + if (unexpired && expectedGeneration.isPresent()) { + refreshTask = null; + future = CompletableFuture.completedFuture(entry.snapshot); + } + else { + refreshTask = prepareRefresh(cacheKey, entry, now, true); + future = refreshTask == null ? entry.refresh : refreshTask.result(); + } + evictEntries(evictedRefreshes); + if (!entries.containsKey(cacheKey)) { + refreshTask = null; } - refreshTask = prepareRefresh(catalog, entry, ticker.getAsLong(), true); - future = refreshTask == null ? entry.refresh : refreshTask.result(); } cancelRefreshes(evictedRefreshes, "HogQL semantic catalog cache entry was evicted"); dispatch(refreshTask); @@ -118,18 +152,43 @@ public CompletionStage prewarm(PhysicalIdentifier public void invalidate(PhysicalIdentifier catalog) { requireNonNull(catalog, "catalog is null"); - CompletableFuture refresh = null; + List refreshes = new ArrayList<>(); synchronized (lock) { - Entry removed = entries.remove(catalog); - if (removed != null) { - refresh = removed.refresh; - removed.refresh = null; + var iterator = entries.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry cacheEntry = iterator.next(); + if (cacheEntry.getKey().catalog().equals(catalog)) { + iterator.remove(); + if (cacheEntry.getValue().refresh != null) { + refreshes.add(new RefreshCancellation(cacheEntry.getValue().refresh, cacheEntry.getValue().upstream)); + cacheEntry.getValue().refresh = null; + cacheEntry.getValue().upstream = null; + } + } } } - cancelRefresh(refresh, "HogQL semantic catalog cache entry was invalidated"); + cancelRefreshes(refreshes, "HogQL semantic catalog cache entry was invalidated"); } - private RefreshTask prepareRefresh(PhysicalIdentifier catalog, Entry entry, long now, boolean force) + private Optional inheritedExactEntry(CacheKey cacheKey, long now) + { + if (cacheKey.expectedGeneration().isEmpty()) { + return Optional.empty(); + } + Entry latest = entries.get(new CacheKey(cacheKey.catalog(), OptionalLong.empty())); + if (latest == null || latest.snapshot == null || latest.snapshot.generation() != cacheKey.expectedGeneration().orElseThrow()) { + return Optional.empty(); + } + if (elapsedNanos(latest.loadedAtNanos, now) >= expireAfterNanos) { + return Optional.empty(); + } + Entry inherited = new Entry(); + inherited.snapshot = latest.snapshot; + inherited.loadedAtNanos = latest.loadedAtNanos; + return Optional.of(inherited); + } + + private RefreshTask prepareRefresh(CacheKey cacheKey, Entry entry, long now, boolean force) { if (entry.refresh != null) { return null; @@ -139,7 +198,7 @@ private RefreshTask prepareRefresh(PhysicalIdentifier catalog, Entry entry, long } CompletableFuture result = new CompletableFuture<>(); entry.refresh = result; - return new RefreshTask(catalog, entry, result); + return new RefreshTask(cacheKey, entry, result); } private void dispatch(RefreshTask refreshTask) @@ -159,7 +218,9 @@ private void load(RefreshTask refreshTask) { CompletionStage loaded; try { - loaded = requireNonNull(loader.load(refreshTask.catalog()), "snapshot loader returned null"); + loaded = requireNonNull(loader.load( + refreshTask.cacheKey().catalog(), + refreshTask.cacheKey().expectedGeneration()), "snapshot loader returned null"); } catch (RuntimeException failure) { completeFailure(refreshTask, failure); @@ -169,6 +230,19 @@ private void load(RefreshTask refreshTask) completeFailure(refreshTask, failure); throw failure; } + CompletableFuture upstream = loaded.toCompletableFuture(); + boolean owned; + synchronized (lock) { + Entry entry = entries.get(refreshTask.cacheKey()); + owned = entry == refreshTask.entry() && entry.refresh == refreshTask.result(); + if (owned) { + entry.upstream = upstream; + } + } + if (!owned) { + upstream.cancel(true); + return; + } try { loaded.whenComplete((snapshot, failure) -> { if (failure != null) { @@ -201,10 +275,21 @@ private void completeSuccess(RefreshTask refreshTask, HogQlSemanticCatalogSnapsh boolean published = false; synchronized (lock) { - Entry entry = entries.get(refreshTask.catalog()); + Entry entry = entries.get(refreshTask.cacheKey()); if (entry == refreshTask.entry() && entry.refresh == refreshTask.result()) { - if (entry.snapshot != null && snapshot.generation() < entry.snapshot.generation()) { + HogQlSemanticCatalogSnapshot observed = observedGenerations.get(new GenerationKey(snapshot.catalog(), snapshot.generation())); + if (observed != null && !observed.equals(snapshot)) { + entry.refresh = null; + entry.upstream = null; + entry.lastRefreshFailureAtNanos = ticker.getAsLong(); + entry.refreshBackoffActive = true; + rejection = new HogQlSemanticCatalogException( + Failure.GENERATION_MISMATCH, + "HogQL semantic catalog generation content changed"); + } + else if (refreshTask.cacheKey().expectedGeneration().isEmpty() && entry.snapshot != null && snapshot.generation() < entry.snapshot.generation()) { entry.refresh = null; + entry.upstream = null; entry.lastRefreshFailureAtNanos = ticker.getAsLong(); entry.refreshBackoffActive = true; rejection = new HogQlSemanticCatalogException( @@ -212,10 +297,15 @@ private void completeSuccess(RefreshTask refreshTask, HogQlSemanticCatalogSnapsh format("HogQL semantic catalog generation regressed from %s to %s", entry.snapshot.generation(), snapshot.generation())); } else { + observedGenerations.put(new GenerationKey(snapshot.catalog(), snapshot.generation()), snapshot); + while (observedGenerations.size() > maximumEntries) { + observedGenerations.remove(observedGenerations.entrySet().iterator().next().getKey()); + } entry.snapshot = snapshot; entry.loadedAtNanos = ticker.getAsLong(); entry.refreshBackoffActive = false; entry.refresh = null; + entry.upstream = null; published = true; } } @@ -233,11 +323,17 @@ else if (rejection != null) { private Throwable validateLoadedSnapshot(RefreshTask refreshTask, HogQlSemanticCatalogSnapshot snapshot) { - if (!snapshot.catalog().equals(refreshTask.catalog())) { + if (!snapshot.catalog().equals(refreshTask.cacheKey().catalog())) { return new HogQlSemanticCatalogException( Failure.CATALOG_MISMATCH, "HogQL semantic catalog snapshot does not match the refreshed catalog"); } + if (refreshTask.cacheKey().expectedGeneration().isPresent() && + snapshot.generation() != refreshTask.cacheKey().expectedGeneration().orElseThrow()) { + return new HogQlSemanticCatalogException( + Failure.GENERATION_MISMATCH, + "HogQL semantic catalog snapshot generation does not match the refreshed generation"); + } return null; } @@ -246,10 +342,11 @@ private void completeFailure(RefreshTask refreshTask, Throwable failure) requireNonNull(failure, "failure is null"); boolean owned; synchronized (lock) { - Entry entry = entries.get(refreshTask.catalog()); + Entry entry = entries.get(refreshTask.cacheKey()); owned = entry == refreshTask.entry() && entry.refresh == refreshTask.result(); if (owned) { entry.refresh = null; + entry.upstream = null; entry.lastRefreshFailureAtNanos = ticker.getAsLong(); entry.refreshBackoffActive = true; } @@ -262,28 +359,33 @@ private void completeFailure(RefreshTask refreshTask, Throwable failure) } } - private void evictEntries(List> evictedRefreshes) + private void evictEntries(List evictedRefreshes) { while (entries.size() > maximumEntries) { - Map.Entry eldest = entries.entrySet().iterator().next(); - entries.remove(eldest.getKey()); - if (eldest.getValue().refresh != null) { - evictedRefreshes.add(eldest.getValue().refresh); - eldest.getValue().refresh = null; + Map.Entry victim = entries.entrySet().stream() + .filter(entry -> entry.getKey().expectedGeneration().isPresent()) + .findFirst() + .orElseGet(() -> entries.entrySet().iterator().next()); + entries.remove(victim.getKey()); + if (victim.getValue().refresh != null) { + evictedRefreshes.add(new RefreshCancellation(victim.getValue().refresh, victim.getValue().upstream)); + victim.getValue().refresh = null; + victim.getValue().upstream = null; } } } - private static void cancelRefreshes(List> refreshes, String message) + private static void cancelRefreshes(List refreshes, String message) { refreshes.forEach(refresh -> cancelRefresh(refresh, message)); } - private static void cancelRefresh(CompletableFuture refresh, String message) + private static void cancelRefresh(RefreshCancellation refresh, String message) { - if (refresh != null) { - refresh.completeExceptionally(new CancellationException(message)); + if (refresh.upstream() != null) { + refresh.upstream().cancel(true); } + refresh.result().completeExceptionally(new CancellationException(message)); } private static long elapsedNanos(long start, long end) @@ -312,7 +414,7 @@ private static long positiveNanos(Duration duration, String name) @FunctionalInterface public interface SnapshotLoader { - CompletionStage load(PhysicalIdentifier catalog); + CompletionStage load(PhysicalIdentifier catalog, OptionalLong expectedGeneration); } private static final class Entry @@ -322,10 +424,32 @@ private static final class Entry private long lastRefreshFailureAtNanos; private boolean refreshBackoffActive; private CompletableFuture refresh; + private CompletableFuture upstream; } private record RefreshTask( - PhysicalIdentifier catalog, + CacheKey cacheKey, Entry entry, CompletableFuture result) {} + + private record RefreshCancellation( + CompletableFuture result, + CompletableFuture upstream) {} + + private record CacheKey(PhysicalIdentifier catalog, OptionalLong expectedGeneration) + { + private CacheKey + { + catalog = requireNonNull(catalog, "catalog is null"); + expectedGeneration = requireNonNull(expectedGeneration, "expectedGeneration is null"); + } + } + + private record GenerationKey(PhysicalIdentifier catalog, long generation) + { + private GenerationKey + { + catalog = requireNonNull(catalog, "catalog is null"); + } + } } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 297c6db5a578..0fe583509f3d 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -1242,7 +1242,7 @@ public enum ExpressionRecipeKind public enum SemanticOperator { - ADD, SUBTRACT, MULTIPLY, DIVIDE, MODULUS, EQUAL, NOT_EQUAL, LESS_THAN, LESS_THAN_OR_EQUAL, GREATER_THAN, GREATER_THAN_OR_EQUAL, AND, OR, NOT, NEGATE, IS_NULL, IS_NOT_NULL, SUBSCRIPT + ADD, SUBTRACT, MULTIPLY, DIVIDE, MODULUS, EQUAL, NOT_EQUAL, LESS_THAN, LESS_THAN_OR_EQUAL, GREATER_THAN, GREATER_THAN_OR_EQUAL, AND, OR, NOT, NEGATE, IS_NULL, IS_NOT_NULL, SUBSCRIPT, JSON_OBJECT_LOOKUP } public enum LiteralEncoding diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotCache.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotCache.java index 632a5afc5cb1..fb50feaaf8d8 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotCache.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotCache.java @@ -16,9 +16,18 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import java.util.Optional; +import java.util.OptionalLong; + +import static java.util.Objects.requireNonNull; @FunctionalInterface public interface HogQlSemanticCatalogSnapshotCache { Optional currentSnapshot(PhysicalIdentifier catalog); + + default Optional currentSnapshot(PhysicalIdentifier catalog, OptionalLong expectedGeneration) + { + requireNonNull(expectedGeneration, "expectedGeneration is null"); + return currentSnapshot(catalog); + } } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotProvider.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotProvider.java index 936c3a0213e0..1bd5d8dd2f57 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotProvider.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotProvider.java @@ -32,7 +32,7 @@ static HogQlSemanticCatalogSnapshotProvider fromCache(HogQlSemanticCatalogSnapsh { requireNonNull(cache, "cache is null"); return request -> { - HogQlSemanticCatalogSnapshot snapshot = cache.currentSnapshot(request.catalog()) + HogQlSemanticCatalogSnapshot snapshot = cache.currentSnapshot(request.catalog(), request.expectedGeneration()) .orElseThrow(() -> new HogQlSemanticCatalogException(Failure.UNAVAILABLE, "HogQL semantic catalog snapshot is unavailable")); if (!snapshot.catalog().equals(request.catalog())) { throw new HogQlSemanticCatalogException(Failure.CATALOG_MISMATCH, "HogQL semantic catalog snapshot does not match the requested catalog"); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 675500a6250e..6ea8d04cf265 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -81,6 +81,21 @@ public void testCompilerPinsOnceAndResolvesUserFunctions() assertThat(result.statement()).isEqualTo(sqlParser.createStatement("SELECT ARRAY[system.builtin.upper('one')][1]")); } + @Test + public void testV0CatalogFunctionsCannotExpandFrozenRegistry() + { + HogQlSemanticCatalogSnapshot snapshot = snapshot(List.of( + function("manifestOnly", FunctionKind.SCALAR, FunctionImplementation.STOCK, List.of("upper"), signature(1), false, false, false, false))); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(snapshot)); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compileV0(envelope("SELECT manifestOnly('value')"), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception).hasMessage("line 1:8: Unknown HogQL function: manifestOnly"); + } + @Test public void testCompilerRewritesNullPredicatesAndPreservesPlaceholders() { @@ -102,12 +117,43 @@ public void testCompilerRewritesNullPredicatesAndPreservesPlaceholders() } @Test - public void testCompilerPreservesOrdinaryFunctionsWithoutCatalogContext() + public void testCompilerResolvesV0FunctionsWithoutCatalogContext() { - HogQlCompilationResult result = new HogQlCompiler().compile(envelope("SELECT ordinary(1)")); + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT coalesce(NULL, 'fallback'), any(value), argMin(value, timestamp), " + + "arrayDistinct([1, 1]), dateDiff('day', start_time, end_time), rank() OVER () FROM metrics")); assertThat(result.catalogGeneration()).isEmpty(); - assertThat(result.statement()).isEqualTo(sqlParser.createStatement("SELECT ordinary(1)")); + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT coalesce(NULL, 'fallback'), arbitrary(value), min_by(value, timestamp), " + + "array_distinct(ARRAY[1, 1]), date_diff('day', start_time, end_time), rank() OVER () FROM metrics")); + } + + @Test + public void testCompilerRejectsUnknownFunctionsWithoutCatalogContext() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope("SELECT ordinary(1)"))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception).hasMessage("line 1:8: Unknown HogQL function: ordinary"); + } + + @Test + public void testCompilerEnforcesV0FunctionArities() + { + assertThat(new HogQlCompiler().compile(envelope("SELECT coalesce('value')")).statement()) + .isEqualTo(sqlParser.createStatement("SELECT 'value'")); + + for (String query : List.of("SELECT if(true, 1)", "SELECT concat('value')")) { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope(query))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception).hasMessageContaining("does not accept"); + } } @ParameterizedTest diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java index b426246033b1..fee73f97a2cf 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java @@ -94,6 +94,16 @@ public void testExpandsOnlyTheDeclaredPathThroughRelationshipCycle() "ON \"__hogql_lazy_1\".event_id = \"__hogql_lazy_2\".event_id")); } + @Test + public void testExpandsJsonObjectPropertyLookup() + { + HogQlCompilationResult result = compile("SELECT jsonProperties.plan FROM persons"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT CAST(CAST(json_parse(properties_json) AS map(varchar, json))[CAST('plan' AS varchar)] AS varchar) AS plan " + + "FROM analytics.data.raw_persons")); + } + @Test public void testPrunesUnusedLazyStarProjectionBeforeExpansion() { @@ -193,20 +203,35 @@ private static HogQlSemanticCatalogSnapshot snapshot() field("workspaceId", "workspace_id"), field("eventId", "event_id"), field("name", "full_name"), + new LogicalFieldDefinition("propertiesJson", new PhysicalIdentifier("properties_json", false), "varchar", LogicalType.STRING, true, false), new LogicalFieldDefinition("propertiesMap", new PhysicalIdentifier("properties_map", false), "map(varchar, varchar)", LogicalType.MAP, true, false)), - List.of(new PropertyDefinition( - "properties", - "propertiesMap", - PropertyStorage.MAP, - LogicalType.STRING, - true, - Optional.of("varchar"), - Optional.of("varchar"), - Optional.of(new OperatorRecipe( - SemanticOperator.SUBSCRIPT, - List.of( - new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE), - new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_KEY)))))), + List.of( + new PropertyDefinition( + "properties", + "propertiesMap", + PropertyStorage.MAP, + LogicalType.STRING, + true, + Optional.of("varchar"), + Optional.of("varchar"), + Optional.of(new OperatorRecipe( + SemanticOperator.SUBSCRIPT, + List.of( + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE), + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_KEY))))), + new PropertyDefinition( + "jsonProperties", + "propertiesJson", + PropertyStorage.JSON_OBJECT, + LogicalType.STRING, + true, + Optional.of("varchar"), + Optional.of("varchar"), + Optional.of(new OperatorRecipe( + SemanticOperator.JSON_OBJECT_LOOKUP, + List.of( + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE), + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_KEY)))))), List.of(new RelationshipDefinition( "event", "events", diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java index 915dee91a580..b992c2d3f3e6 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java @@ -97,6 +97,13 @@ public void testRejectsInvalidRefreshWithoutReplacingLastKnownGood() assertRefreshFailure(mismatchedRefresh, Failure.CATALOG_MISMATCH); assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(2L); + + CompletableFuture mutatedLoad = loader.expect(CATALOG); + CompletionStage mutatedRefresh = cache.prewarm(CATALOG); + mutatedLoad.complete(snapshot(CATALOG, 2, "changed_id")); + + assertRefreshFailure(mutatedRefresh, Failure.GENERATION_MISMATCH); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(snapshot -> snapshot.logicalTables().getFirst().fields().getFirst().name()).isEqualTo("id"); } @Test @@ -243,6 +250,106 @@ public void testPinnedSnapshotRemainsStableAcrossRefreshPublication() assertThat(provider.pin(new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.of(2))).generation()).isEqualTo(2); } + @Test + public void testColdExactGenerationLoadsPinnedEntry() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + CompletableFuture exactLoad = loader.expect(CATALOG, OptionalLong.of(7)); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(3, ticker, loader); + + assertThat(cache.currentSnapshot(CATALOG, OptionalLong.of(7))).isEmpty(); + assertThat(loader.loadCount(CATALOG, OptionalLong.of(7))).isEqualTo(1); + assertThat(loader.loadCount(CATALOG)).isZero(); + + exactLoad.complete(snapshot(CATALOG, 7)); + assertThat(cache.currentSnapshot(CATALOG, OptionalLong.of(7))) + .get() + .extracting(HogQlSemanticCatalogSnapshot::generation) + .isEqualTo(7L); + } + + @Test + public void testLatestCanAdvancePastCachedHistoricalGeneration() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(3, ticker, loader); + completePrewarm(cache, loader, snapshot(CATALOG, 1)); + + assertThat(cache.currentSnapshot(CATALOG, OptionalLong.of(1))) + .get() + .extracting(HogQlSemanticCatalogSnapshot::generation) + .isEqualTo(1L); + assertThat(loader.loadCount(CATALOG, OptionalLong.of(1))).isZero(); + + CompletableFuture latestRefresh = loader.expect(CATALOG); + ticker.set(REFRESH_AFTER.toNanos()); + assertThat(cache.currentSnapshot(CATALOG)).isPresent(); + latestRefresh.complete(snapshot(CATALOG, 2)); + + assertThat(cache.currentSnapshot(CATALOG)) + .get() + .extracting(HogQlSemanticCatalogSnapshot::generation) + .isEqualTo(2L); + assertThat(cache.currentSnapshot(CATALOG, OptionalLong.of(1))) + .get() + .extracting(HogQlSemanticCatalogSnapshot::generation) + .isEqualTo(1L); + } + + @Test + public void testExactGenerationChurnPreservesLatestAndCancelsEvictedLoad() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, snapshot(CATALOG, 1)); + + CompletableFuture generationSevenLoad = loader.expect(CATALOG, OptionalLong.of(7)); + CompletionStage generationSeven = cache.prewarm(CATALOG, OptionalLong.of(7)); + + CompletableFuture generationEightLoad = loader.expect(CATALOG, OptionalLong.of(8)); + CompletionStage generationEight = cache.prewarm(CATALOG, OptionalLong.of(8)); + + assertThat(generationSevenLoad).isCancelled(); + assertThat(generationSeven.toCompletableFuture()).isCompletedExceptionally(); + assertThat(cache.currentSnapshot(CATALOG)) + .get() + .extracting(HogQlSemanticCatalogSnapshot::generation) + .isEqualTo(1L); + + generationEightLoad.complete(snapshot(CATALOG, 8)); + assertThat(generationEight.toCompletableFuture()).isCompletedWithValueMatching(snapshot -> snapshot.generation() == 8); + } + + @Test + public void testExactGenerationOutageFailsClosedWithoutLatestFallback() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(3, ticker, loader); + + CompletableFuture exactLoad = loader.expect(CATALOG, OptionalLong.of(7)); + assertThat(cache.currentSnapshot(CATALOG, OptionalLong.of(7))).isEmpty(); + exactLoad.complete(snapshot(CATALOG, 7)); + + ticker.set(11); + completePrewarm(cache, loader, snapshot(CATALOG, 8)); + + CompletableFuture failedReload = loader.expect(CATALOG, OptionalLong.of(7)); + ticker.set(EXPIRE_AFTER.toNanos()); + assertThat(cache.currentSnapshot(CATALOG, OptionalLong.of(7))).isEmpty(); + failedReload.completeExceptionally(new IllegalStateException("metadata unavailable")); + + assertThat(cache.currentSnapshot(CATALOG, OptionalLong.of(7))).isEmpty(); + assertThat(loader.loadCount(CATALOG, OptionalLong.of(7))).isEqualTo(2); + assertThat(cache.currentSnapshot(CATALOG)) + .get() + .extracting(HogQlSemanticCatalogSnapshot::generation) + .isEqualTo(8L); + } + @Test public void testOrdinaryAndDelimitedCatalogIdentifiersHaveIsolatedEntries() { @@ -272,7 +379,9 @@ public void testMalformedSchemaV2RefreshRetainsLastKnownGoodSnapshot() BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache( 2, ticker, - catalog -> jsonLoader.load(LoadRequest.latest(catalog, LANGUAGE_VERSION))); + (catalog, expectedGeneration) -> jsonLoader.load(expectedGeneration.isPresent() + ? LoadRequest.pinned(catalog, LANGUAGE_VERSION, expectedGeneration.orElseThrow()) + : LoadRequest.latest(catalog, LANGUAGE_VERSION))); CompletableFuture initialPayload = new CompletableFuture<>(); payloads.add(initialPayload); @@ -342,6 +451,11 @@ private static PhysicalIdentifier catalog(String name) } private static HogQlSemanticCatalogSnapshot snapshot(PhysicalIdentifier catalog, long generation) + { + return snapshot(catalog, generation, "id"); + } + + private static HogQlSemanticCatalogSnapshot snapshot(PhysicalIdentifier catalog, long generation, String fieldName) { return new HogQlSemanticCatalogSnapshot( 2, @@ -352,8 +466,8 @@ private static HogQlSemanticCatalogSnapshot snapshot(PhysicalIdentifier catalog, "events", new PhysicalQualifiedName(catalog, catalog("default"), catalog("events")), List.of(new LogicalFieldDefinition( - "id", - catalog("id"), + fieldName, + catalog(fieldName), "varchar", LogicalType.STRING, false, @@ -385,30 +499,43 @@ private static String snapshotJson(long generation) private static final class ControlledLoader implements SnapshotLoader { - private final Map>> loads = new HashMap<>(); - private final Map loadCounts = new HashMap<>(); + private final Map>> loads = new HashMap<>(); + private final Map loadCounts = new HashMap<>(); public CompletableFuture expect(PhysicalIdentifier catalog) + { + return expect(catalog, OptionalLong.empty()); + } + + public CompletableFuture expect(PhysicalIdentifier catalog, OptionalLong expectedGeneration) { CompletableFuture load = new CompletableFuture<>(); - loads.computeIfAbsent(catalog, _ -> new ArrayDeque<>()).add(load); + loads.computeIfAbsent(new LoadKey(catalog, expectedGeneration), _ -> new ArrayDeque<>()).add(load); return load; } @Override - public CompletionStage load(PhysicalIdentifier catalog) + public CompletionStage load(PhysicalIdentifier catalog, OptionalLong expectedGeneration) { - loadCounts.computeIfAbsent(catalog, _ -> new AtomicInteger()).incrementAndGet(); - return Optional.ofNullable(loads.get(catalog)) + LoadKey key = new LoadKey(catalog, expectedGeneration); + loadCounts.computeIfAbsent(key, _ -> new AtomicInteger()).incrementAndGet(); + return Optional.ofNullable(loads.get(key)) .map(ArrayDeque::poll) - .orElseThrow(() -> new IllegalStateException("unexpected load for catalog " + catalog.value())); + .orElseThrow(() -> new IllegalStateException("unexpected load for catalog " + catalog.value() + " and generation " + expectedGeneration)); } public int loadCount(PhysicalIdentifier catalog) { - return Optional.ofNullable(loadCounts.get(catalog)) + return loadCount(catalog, OptionalLong.empty()); + } + + public int loadCount(PhysicalIdentifier catalog, OptionalLong expectedGeneration) + { + return Optional.ofNullable(loadCounts.get(new LoadKey(catalog, expectedGeneration))) .map(AtomicInteger::get) .orElse(0); } } + + private record LoadKey(PhysicalIdentifier catalog, OptionalLong expectedGeneration) {} } diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java index e0ed504dfd23..2769cc7cc03a 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java @@ -31,6 +31,7 @@ import java.util.Optional; import java.util.OptionalLong; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; import static io.trino.spi.ErrorType.EXTERNAL; @@ -58,6 +59,31 @@ public void testPinsOneSnapshotWithoutCrossGenerationReads() assertThat(reads).hasValue(1); } + @Test + public void testExpectedGenerationIsPassedToExactCacheLookup() + { + AtomicReference requestedGeneration = new AtomicReference<>(); + HogQlSemanticCatalogSnapshotCache cache = new HogQlSemanticCatalogSnapshotCache() + { + @Override + public Optional currentSnapshot(PhysicalIdentifier catalog) + { + throw new AssertionError("expected exact cache lookup"); + } + + @Override + public Optional currentSnapshot(PhysicalIdentifier catalog, OptionalLong expectedGeneration) + { + requestedGeneration.set(expectedGeneration); + return Optional.of(snapshot(7)); + } + }; + HogQlSemanticCatalogSnapshotProvider provider = HogQlSemanticCatalogSnapshotProvider.fromCache(cache); + + assertThat(provider.pin(new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.of(7))).generation()).isEqualTo(7); + assertThat(requestedGeneration).hasValue(OptionalLong.of(7)); + } + @ParameterizedTest(name = "{0}") @MethodSource("failClosedCases") public void testFailsClosed(String name, HogQlSemanticCatalogSnapshotCache cache, PinRequest request, Failure failure) diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java index ab95bd84f245..089927cf9030 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java @@ -144,7 +144,8 @@ public PreparedQuery prepareQuery(Session session, QuerySubmission submission) try { HogQlCompilationResult result = tracker.observe(COMPILATION, () -> hogQlCompilationExecutor.execute(() -> hogQlCompiler .orElseThrow(() -> new TrinoException(NOT_SUPPORTED, "HogQL query submission is disabled")) - .compile(envelope, semanticCatalogContext(session)))); + .compileV0(envelope, semanticCatalogContext(session)))); + tracker.catalogGeneration(result.catalogGeneration()); Statement statement = explain(result.statement(), submission.hogQlExplain()); preparedQuery = tracker.observe(PARAMETER_BINDING, () -> prepareQuery( session, diff --git a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java index 6f0686a3a18f..d568fda9acda 100644 --- a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java +++ b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java @@ -214,9 +214,11 @@ private static Stream differentialQueries() "GROUP BY r.regionkey ORDER BY r.regionkey"), Arguments.of( "window frames", - "SELECT nationkey, sum(nationkey) OVER (ORDER BY nationkey ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS running " + + "SELECT nationkey, sum(nationkey) OVER (ORDER BY nationkey ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS running, " + + "rank() OVER (ORDER BY nationkey) AS ranked, row_number() OVER (ORDER BY nationkey) AS numbered " + "FROM tpch.tiny.nation WHERE nationkey < 5 ORDER BY nationkey", - "SELECT nationkey, sum(nationkey) OVER (ORDER BY nationkey ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS running " + + "SELECT nationkey, sum(nationkey) OVER (ORDER BY nationkey ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS running, " + + "rank() OVER (ORDER BY nationkey) AS ranked, row_number() OVER (ORDER BY nationkey) AS numbered " + "FROM tpch.tiny.nation WHERE nationkey < 5 ORDER BY nationkey"), Arguments.of( "values and set operations", @@ -235,7 +237,33 @@ private static Stream differentialQueries() "SELECT INTERVAL 2 WEEK AS weeks, INTERVAL '3 months' AS months, INTERVAL nationkey DAY AS dynamic " + "FROM tpch.tiny.nation WHERE nationkey = 2", "SELECT 2 * INTERVAL '7' DAY AS weeks, 3 * INTERVAL '1' MONTH AS months, nationkey * INTERVAL '1' DAY AS dynamic " + - "FROM tpch.tiny.nation WHERE nationkey = 2")); + "FROM tpch.tiny.nation WHERE nationkey = 2"), + Arguments.of( + "v0 scalar and collection functions", + "SELECT abs(-2), coalesce(NULL, 'fallback'), if(true, 1, 2), lower('ABC'), upper('abc'), length('abc'), " + + "concat('a', 'b'), replace('abc', 'b', 'x'), arrayDistinct([1, 1, 2]), arraySort([2, 1]), " + + "arrayFlatten([[1], [2]]), arrayStringConcat(['a', 'b']), " + + "dateAdd('day', 1, CAST('2024-01-01' AS Date)), " + + "dateDiff('day', CAST('2024-01-01' AS Date), CAST('2024-01-03' AS Date)), " + + "dateTrunc('day', CAST('2024-01-01 12:34:56' AS Timestamp(3)))", + "SELECT abs(-2), coalesce(NULL, 'fallback'), if(true, 1, 2), lower('ABC'), upper('abc'), length('abc'), " + + "concat('a', 'b'), replace('abc', 'b', 'x'), array_distinct(ARRAY[1, 1, 2]), array_sort(ARRAY[2, 1]), " + + "flatten(ARRAY[ARRAY[1], ARRAY[2]]), array_join(ARRAY['a', 'b'], ''), " + + "date_add('day', 1, CAST('2024-01-01' AS date)), " + + "date_diff('day', CAST('2024-01-01' AS date), CAST('2024-01-03' AS date)), " + + "date_trunc('day', CAST('2024-01-01 12:34:56' AS timestamp(3)))"), + Arguments.of( + "v0 aggregate functions", + "SELECT count(), sum(nationkey), min(nationkey), max(nationkey), avg(nationkey), any(nationkey), " + + "argMin(name, nationkey), argMax(name, nationkey), array_agg(nationkey ORDER BY nationkey) " + + "FROM tpch.tiny.nation WHERE nationkey < 3", + "SELECT count(), sum(nationkey), min(nationkey), max(nationkey), avg(nationkey), arbitrary(nationkey), " + + "min_by(name, nationkey), max_by(name, nationkey), array_agg(nationkey ORDER BY nationkey) " + + "FROM tpch.tiny.nation WHERE nationkey < 3"), + Arguments.of( + "v0 value window function", + "SELECT first_value(name) OVER (ORDER BY nationkey) FROM tpch.tiny.nation WHERE nationkey < 3 ORDER BY nationkey", + "SELECT first_value(name) OVER (ORDER BY nationkey) FROM tpch.tiny.nation WHERE nationkey < 3 ORDER BY nationkey")); } @Test @@ -353,6 +381,18 @@ public void testEmptyResultsAndCompilerDiagnostics() List invalidResults = runHogQlToCompletion(hogQlRequest("SELECT (")); assertThat(invalidResults.getLast().getError().getErrorName()).isEqualTo("HOGQL_SYNTAX_ERROR"); + + List unsupportedFunction = runHogQlToCompletion(hogQlRequest("SELECT uniq(1)")); + assertThat(unsupportedFunction.getLast().getError().getErrorName()).isEqualTo("HOGQL_RESOLUTION_ERROR"); + + List pivot = runHogQlToCompletion(hogQlRequest( + "SELECT * FROM tpch.tiny.nation PIVOT (sum(nationkey) FOR regionkey IN (0))")); + assertThat(pivot.getLast().getError().getErrorName()).isEqualTo("HOGQL_UNSUPPORTED_FEATURE"); + assertThat(pivot.getLast().getError().getMessage()).contains("PIVOT is outside the HogQL v0 profile"); + + List modifier = runHogQlToCompletion(hogQlRequestWithModifier("SELECT 1")); + assertThat(modifier.getLast().getError().getErrorName()).isEqualTo("HOGQL_UNSUPPORTED_FEATURE"); + assertThat(modifier.getLast().getError().getMessage()).contains("Modifiers are outside the HogQL v0 profile"); } @Test @@ -576,6 +616,18 @@ private static String hogQlRequestWithTypedValues(String query) """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); } + private static String hogQlRequestWithModifier(String query) + { + return """ + { + "query": %s, + "protocolVersion": 1, + "languageVersion": "%s", + "modifiers": {"sampling": {"type": "boolean", "value": true}} + } + """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); + } + private static String hogQlExplainRequest(String query, String type, String format) { return """ diff --git a/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerSemanticCatalog.java b/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerSemanticCatalog.java index 37b67fd3b2ab..beba8818f44c 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerSemanticCatalog.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerSemanticCatalog.java @@ -13,37 +13,28 @@ */ package io.trino.execution; -import io.trino.Session; import io.trino.execution.QueryPreparer.PreparedQuery; +import io.trino.hogql.HogQlCompilationEvent; import io.trino.hogql.compiler.HogQlCompileEnvelope; import io.trino.hogql.compiler.HogQlCompiler; import io.trino.hogql.compiler.HogQlErrorCode; import io.trino.hogql.compiler.HogQlTypedValue; import io.trino.hogql.compiler.HogQlTypedValue.BooleanValue; -import io.trino.hogql.compiler.HogQlTypedValue.StringValue; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; -import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; -import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ModifierBehavior; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; -import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticModifierDefault; -import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinRequest; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; import io.trino.hogql.parser.HogQlLanguageContract; -import io.trino.metadata.SessionPropertyManager; import io.trino.spi.TrinoException; -import io.trino.sql.SessionPropertyResolver; -import io.trino.sql.SqlEnvironmentConfig; import io.trino.sql.parser.SqlParser; import io.trino.sql.tree.Identifier; import io.trino.sql.tree.Query; import io.trino.sql.tree.QuerySpecification; import io.trino.sql.tree.Table; -import io.trino.testing.StandaloneQueryRunner; import org.junit.jupiter.api.Test; import java.util.List; @@ -53,10 +44,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import static io.trino.SessionTestUtils.TEST_SESSION; import static io.trino.execution.QuerySubmission.hogQl; import static io.trino.hogql.HogQlCompilationObserver.NOOP; -import static io.trino.spi.session.PropertyMetadata.booleanProperty; import static io.trino.testing.TestingSession.testSessionBuilder; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -75,40 +64,15 @@ public class TestHogQlQueryPreparerSemanticCatalog List.of(new LogicalFieldDefinition("event", new PhysicalIdentifier("event_name", false), "varchar", LogicalType.STRING, false, true)), List.of(), List.of()))); - private static final HogQlSemanticCatalogSnapshot MODIFIER_SNAPSHOT = new HogQlSemanticCatalogSnapshot( - HogQlSemanticCatalogSnapshot.PROTOCOL_VERSION, - HogQlSemanticCatalogSnapshot.SCHEMA_VERSION, - HogQlLanguageContract.current().languageVersion(), - CATALOG, - 7, - List.of(), - List.of(), - List.of(), - List.of(), - List.of(), - List.of(), - List.of(new SemanticModifierDefault( - "sampling", - ModifierBehavior.TRINO_SESSION_PROPERTY, - new TypedLiteral("boolean", LiteralEncoding.BOOLEAN, "false"), - List.of(new PhysicalIdentifier("hogql_sampling", false))), - new SemanticModifierDefault( - "legacyMode", - ModifierBehavior.SAFE_NOOP, - new TypedLiteral("boolean", LiteralEncoding.BOOLEAN, "false"), - List.of())), - List.of(), - List.of(), - List.of()); - @Test public void testSessionCatalogPinsSemanticSnapshotAndResolvesLogicalTable() { AtomicReference pinRequest = new AtomicReference<>(); + AtomicReference compilationEvent = new AtomicReference<>(); QueryPreparer queryPreparer = new QueryPreparer( new SqlParser(), new HogQlCompiler(), - NOOP, + compilationEvent::set, Optional.of(request -> { pinRequest.set(request); return new PinnedSnapshot(SNAPSHOT); @@ -119,6 +83,7 @@ public void testSessionCatalogPinsSemanticSnapshotAndResolvesLogicalTable() hogQl(envelope("SELECT event FROM events", OptionalLong.of(7)))); assertThat(pinRequest.get()).isEqualTo(new PinRequest(CATALOG, HogQlLanguageContract.current().languageVersion(), OptionalLong.of(7))); + assertThat(compilationEvent.get().dimensions().catalogGeneration()).hasValue(7); QuerySpecification query = (QuerySpecification) ((Query) preparedQuery.getStatement()).getQueryBody(); assertThat(((Table) query.getFrom().orElseThrow()).getName().getOriginalParts()) .extracting(Identifier::getValue) @@ -146,93 +111,33 @@ public void testMissingSessionCatalogOrProviderDoesNotFetchMetadata() } @Test - public void testModifierOverrideIsAppliedOutsideStatementAst() - { - try (StandaloneQueryRunner queryRunner = new StandaloneQueryRunner(TEST_SESSION)) { - SessionPropertyManager sessionPropertyManager = queryRunner.getSessionPropertyManager(); - sessionPropertyManager.addSystemSessionProperty(booleanProperty("hogql_sampling", "HogQL sampling", false, false)); - Session session = testSessionBuilder(sessionPropertyManager).setCatalog("analytics").build(); - QueryPreparer queryPreparer = new QueryPreparer( - new SqlParser(), - new HogQlCompiler(), - NOOP, - Optional.of(_ -> new PinnedSnapshot(MODIFIER_SNAPSHOT))); - - PreparedQuery preparedQuery = queryPreparer.prepareQuery( - session, - hogQl(envelope( - "SELECT 1", - OptionalLong.empty(), - Map.of("sampling", new HogQlTypedValue("boolean", new BooleanValue(true)))))); - - assertThat(((Query) preparedQuery.getStatement()).getSessionProperties()).isEmpty(); - assertThat(preparedQuery.getSessionPropertyOverrides()).singleElement() - .extracting(property -> property.getName().toString()) - .isEqualTo("hogql_sampling"); - - SessionPropertyResolver resolver = new SessionPropertyResolver( - new SessionPropertyEvaluator( - queryRunner.getPlannerContext(), - queryRunner.getAccessControl(), - sessionPropertyManager, - new SqlEnvironmentConfig()), - queryRunner.getAccessControl()); - Session overridden = resolver.getSessionPropertiesApplier(preparedQuery).apply(session); - - assertThat(overridden.getSystemProperty("hogql_sampling", Boolean.class)).isTrue(); - } - } - - @Test - public void testInvalidModifierValueFailsAsRedactedBindingError() - { - QueryPreparer queryPreparer = new QueryPreparer( - new SqlParser(), - new HogQlCompiler(), - NOOP, - Optional.of(_ -> new PinnedSnapshot(MODIFIER_SNAPSHOT))); - - assertThatThrownBy(() -> queryPreparer.prepareQuery( - testSessionBuilder().setCatalog("analytics").build(), - hogQl(envelope( - "SELECT 1", - OptionalLong.empty(), - Map.of("sampling", new HogQlTypedValue("boolean", new StringValue("secret-value"))))))) - .isInstanceOfSatisfying(TrinoException.class, exception -> { - assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_BINDING_ERROR.toErrorCode()); - assertThat(exception.getLocation()).hasValueSatisfying(location -> { - assertThat(location.lineNumber()).isEqualTo(1); - assertThat(location.columnNumber()).isEqualTo(1); - }); - assertThat(exception).hasMessageContaining("Invalid HogQL modifier binding: sampling"); - assertThat(exception).hasMessageNotContaining("secret-value"); - }); - } - - @Test - public void testInvalidSafeNoopValueFailsAsRedactedBindingError() + public void testV0RejectsModifiersBeforeCatalogResolution() { + AtomicInteger pins = new AtomicInteger(); QueryPreparer queryPreparer = new QueryPreparer( new SqlParser(), new HogQlCompiler(), NOOP, - Optional.of(_ -> new PinnedSnapshot(MODIFIER_SNAPSHOT))); + Optional.of(_ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + })); assertThatThrownBy(() -> queryPreparer.prepareQuery( testSessionBuilder().setCatalog("analytics").build(), hogQl(envelope( "SELECT 1", OptionalLong.empty(), - Map.of("legacyMode", new HogQlTypedValue("boolean", new StringValue("secret-value"))))))) + Map.of("sampling", new HogQlTypedValue("boolean", new BooleanValue(true))))))) .isInstanceOfSatisfying(TrinoException.class, exception -> { - assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_BINDING_ERROR.toErrorCode()); + assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); assertThat(exception.getLocation()).hasValueSatisfying(location -> { assertThat(location.lineNumber()).isEqualTo(1); assertThat(location.columnNumber()).isEqualTo(1); }); - assertThat(exception).hasMessageContaining("Invalid HogQL modifier binding: legacyMode"); - assertThat(exception).hasMessageNotContaining("secret-value"); + assertThat(exception).hasMessageContaining("Modifiers are outside the HogQL v0 profile"); }); + assertThat(pins).hasValue(0); } private static HogQlCompileEnvelope envelope(String query, OptionalLong generation) diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java index 0b641c1d20d0..c67c473c669f 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java @@ -19,6 +19,8 @@ import io.trino.hogql.compiler.HogQlCompiler; import io.trino.hogql.compiler.HogQlSemanticCatalogContext; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionArgument; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyProjectionDefinition; @@ -26,10 +28,14 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; import io.trino.hogql.parser.HogQlLanguageContract; import io.trino.sql.SqlFormatter; @@ -147,6 +153,15 @@ public void testDemandedLazyProjectionRemainsThroughDerivedCteChain() assertThat(nodes(plan, JoinNode.class)).hasSize(1); } + @Test + public void testVarcharJsonPropertyPredicateReachesStockPlanner() + { + Plan plan = plan(compile("SELECT properties.plan FROM persons WHERE properties.plan = 'pro'")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(1); + assertThat(nodes(plan, FilterNode.class)).hasSize(1); + } + @ParameterizedTest @ValueSource(strings = {"UNION ALL", "INTERSECT ALL", "EXCEPT ALL"}) public void testSetOperationDemandIsMappedByBranchPosition(String operator) @@ -217,8 +232,21 @@ static HogQlSemanticCatalogSnapshot testingSnapshot() physicalName("customer"), List.of( field("personId", "custkey", "bigint", LogicalType.INTEGER), - field("name", "name", "varchar", LogicalType.STRING)), - List.of(), + field("name", "name", "varchar", LogicalType.STRING), + field("properties", "comment", "varchar", LogicalType.STRING)), + List.of(new PropertyDefinition( + "properties", + "properties", + PropertyStorage.JSON_OBJECT, + LogicalType.STRING, + true, + Optional.of("varchar"), + Optional.of("varchar"), + Optional.of(new OperatorRecipe( + SemanticOperator.JSON_OBJECT_LOOKUP, + List.of( + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE), + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_KEY)))))), List.of()); LazyTableDefinition personProfile = new LazyTableDefinition( "events", From 130cf8b7ad95c352ea8fca57e5d52416b0ca5e38 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 21:24:24 -0400 Subject: [PATCH 075/136] feat(hogql): pin authenticated catalog generations --- .../execution/HogQlCompilationExecutor.java | 13 ++++ .../io/trino/hogql/HogQlCompilationEvent.java | 12 +++ .../io/trino/hogql/HogQlCompilationStats.java | 10 +++ .../trino/hogql/HogQlCompilationTracker.java | 11 ++- .../main/java/io/trino/hogql/HogQlConfig.java | 21 +++++ .../hogql/HogQlCoordinatorErrorCode.java | 1 + .../hogql/HogQlSemanticCatalogConfig.java | 21 +++++ .../HogQlSemanticCatalogHttpTransport.java | 62 +++++++++++++-- .../hogql/HogQlSemanticCatalogManager.java | 4 +- .../TestHogQlCompilationExecutor.java | 37 +++++++++ .../TestHogQlCompilationObservability.java | 21 +++-- .../java/io/trino/hogql/TestHogQlConfig.java | 11 ++- .../hogql/TestHogQlSemanticCatalogConfig.java | 15 ++++ .../TestHogQlSemanticCatalogGuiceWiring.java | 12 ++- ...TestHogQlSemanticCatalogHttpTransport.java | 76 ++++++++++++++++++- .../TestHogQlSemanticCatalogManager.java | 32 ++++++++ 16 files changed, 341 insertions(+), 18 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/execution/HogQlCompilationExecutor.java b/core/trino-main/src/main/java/io/trino/execution/HogQlCompilationExecutor.java index 7b10681922d2..6e3346c3634e 100644 --- a/core/trino-main/src/main/java/io/trino/execution/HogQlCompilationExecutor.java +++ b/core/trino-main/src/main/java/io/trino/execution/HogQlCompilationExecutor.java @@ -19,6 +19,7 @@ import jakarta.annotation.PreDestroy; import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -29,10 +30,12 @@ import java.util.concurrent.FutureTask; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.trino.hogql.HogQlCoordinatorErrorCode.HOGQL_COMPILATION_QUEUE_FULL; +import static io.trino.hogql.HogQlCoordinatorErrorCode.HOGQL_COMPILATION_TIMEOUT; import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -41,6 +44,7 @@ public final class HogQlCompilationExecutor { private final Executor executor; private final Optional ownedExecutor; + private final OptionalLong timeoutMillis; private final Set> tasks = ConcurrentHashMap.newKeySet(); @Inject @@ -60,12 +64,14 @@ public HogQlCompilationExecutor(HogQlConfig config) new ThreadPoolExecutor.AbortPolicy()); this.executor = executor; this.ownedExecutor = Optional.of(executor); + this.timeoutMillis = OptionalLong.of(config.getCompilationTimeout().toMillis()); } private HogQlCompilationExecutor(Executor executor) { this.executor = requireNonNull(executor, "executor is null"); ownedExecutor = Optional.empty(); + timeoutMillis = OptionalLong.empty(); } static HogQlCompilationExecutor directExecutor() @@ -87,6 +93,9 @@ public T execute(Supplier compilation) } try { + if (timeoutMillis.isPresent()) { + return task.get(timeoutMillis.orElseThrow(), MILLISECONDS); + } return task.get(); } catch (InterruptedException failure) { @@ -97,6 +106,10 @@ public T execute(Supplier compilation) catch (CancellationException failure) { throw new TrinoException(HOGQL_COMPILATION_QUEUE_FULL, "HogQL compilation capacity is unavailable; retry later", failure); } + catch (TimeoutException failure) { + task.cancel(true); + throw new TrinoException(HOGQL_COMPILATION_TIMEOUT, "HogQL compilation exceeded its time limit; retry later", failure); + } catch (ExecutionException failure) { Throwable cause = failure.getCause(); if (cause instanceof RuntimeException runtimeException) { diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationEvent.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationEvent.java index 50637da09cff..49bb6a6c0aa3 100644 --- a/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationEvent.java +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationEvent.java @@ -101,5 +101,17 @@ public static Dimensions fromEnvelope(HogQlCompileEnvelope envelope) envelope.modifiers().size(), envelope.catalogGeneration()); } + + public Dimensions withCatalogGeneration(OptionalLong catalogGeneration) + { + return new Dimensions( + protocolVersion, + languageVersion, + parameterCount, + variableCount, + filterCount, + modifierCount, + catalogGeneration); + } } } diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationStats.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationStats.java index b2e122b34eae..5e4d7f47c0db 100644 --- a/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationStats.java +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationStats.java @@ -19,6 +19,8 @@ import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; +import java.util.concurrent.atomic.AtomicLong; + import static java.util.concurrent.TimeUnit.NANOSECONDS; public final class HogQlCompilationStats @@ -36,12 +38,14 @@ public final class HogQlCompilationStats private final TimeStat bindTime = new TimeStat(NANOSECONDS); private final TimeStat lowerTime = new TimeStat(NANOSECONDS); private final TimeStat parameterBindingTime = new TimeStat(NANOSECONDS); + private final AtomicLong lastCatalogGeneration = new AtomicLong(-1); @Override public void compilationCompleted(HogQlCompilationEvent event) { completedCompilations.update(1); totalTime.addNanos(event.totalNanos()); + event.dimensions().catalogGeneration().ifPresent(lastCatalogGeneration::set); event.phaseNanos().forEach((phase, duration) -> phaseTime(phase).addNanos(duration)); switch (event.outcome()) { case SUCCESS -> successfulCompilations.update(1); @@ -105,6 +109,12 @@ public CounterStat getInsufficientResourcesFailures() return insufficientResourcesFailures; } + @Managed + public long getLastCatalogGeneration() + { + return lastCatalogGeneration.get(); + } + @Managed @Nested public TimeStat getTotalTime() diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationTracker.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationTracker.java index 1e20e3aee31c..0330694f96c3 100644 --- a/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationTracker.java +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationTracker.java @@ -22,6 +22,7 @@ import java.util.EnumMap; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; import java.util.function.LongSupplier; import java.util.function.Supplier; @@ -36,7 +37,7 @@ public final class HogQlCompilationTracker { private final HogQlCompilationObserver observer; - private final Dimensions dimensions; + private Dimensions dimensions; private final LongSupplier ticker; private final long startedNanos; private final EnumMap phaseNanos = new EnumMap<>(Phase.class); @@ -82,6 +83,14 @@ public void succeeded() complete(SUCCESS, Optional.empty()); } + public void catalogGeneration(OptionalLong catalogGeneration) + { + if (completed) { + throw new IllegalStateException("HogQL compilation observation is complete"); + } + dimensions = dimensions.withCatalogGeneration(requireNonNull(catalogGeneration, "catalogGeneration is null")); + } + public void failed(Throwable failure) { requireNonNull(failure, "failure is null"); diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlConfig.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlConfig.java index e5051febc03c..bba91b55c345 100644 --- a/core/trino-main/src/main/java/io/trino/hogql/HogQlConfig.java +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlConfig.java @@ -15,13 +15,19 @@ import io.airlift.configuration.Config; import io.airlift.configuration.ConfigDescription; +import io.airlift.units.Duration; +import io.airlift.units.MinDuration; import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; + +import static java.util.concurrent.TimeUnit.SECONDS; public class HogQlConfig { private boolean enabled; private int compilationThreads = 2; private int compilationQueueCapacity = 32; + private Duration compilationTimeout = new Duration(10, SECONDS); public boolean isEnabled() { @@ -63,4 +69,19 @@ public HogQlConfig setCompilationQueueCapacity(int compilationQueueCapacity) this.compilationQueueCapacity = compilationQueueCapacity; return this; } + + @NotNull + @MinDuration("1ms") + public Duration getCompilationTimeout() + { + return compilationTimeout; + } + + @Config("hogql.compilation-timeout") + @ConfigDescription("Maximum wall time for one HogQL compilation, including queue wait") + public HogQlConfig setCompilationTimeout(Duration compilationTimeout) + { + this.compilationTimeout = compilationTimeout; + return this; + } } diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCoordinatorErrorCode.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCoordinatorErrorCode.java index 612a412c1374..346a8e96b31a 100644 --- a/core/trino-main/src/main/java/io/trino/hogql/HogQlCoordinatorErrorCode.java +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCoordinatorErrorCode.java @@ -23,6 +23,7 @@ public enum HogQlCoordinatorErrorCode implements ErrorCodeSupplier { HOGQL_COMPILATION_QUEUE_FULL(0, INSUFFICIENT_RESOURCES), + HOGQL_COMPILATION_TIMEOUT(1, INSUFFICIENT_RESOURCES), ; private final ErrorCode errorCode; diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java index 29d342fd13d8..639db487dce4 100644 --- a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java @@ -20,6 +20,7 @@ import io.airlift.units.MaxDataSize; import io.airlift.units.MinDataSize; import io.airlift.units.MinDuration; +import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotNull; @@ -32,6 +33,7 @@ public class HogQlSemanticCatalogConfig { private URI uri; + private String authenticationTokenFile; private int maximumEntries = 100; private Duration refreshAfter = new Duration(1, MINUTES); private Duration expireAfter = new Duration(5, MINUTES); @@ -54,6 +56,25 @@ public HogQlSemanticCatalogConfig setUri(URI uri) return this; } + public String getAuthenticationTokenFile() + { + return authenticationTokenFile; + } + + @Config("hogql.semantic-catalog.authentication-token-file") + @ConfigDescription("File containing the Duckgres HogQL compatibility metadata authentication token") + public HogQlSemanticCatalogConfig setAuthenticationTokenFile(String authenticationTokenFile) + { + this.authenticationTokenFile = authenticationTokenFile; + return this; + } + + @AssertTrue(message = "hogql.semantic-catalog.authentication-token-file is required when hogql.semantic-catalog.uri is set") + public boolean isAuthenticationConfigured() + { + return uri == null || (authenticationTokenFile != null && !authenticationTokenFile.isBlank()); + } + @Min(1) public int getMaximumEntries() { diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogHttpTransport.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogHttpTransport.java index d3b988c8006f..8d30acfd34ce 100644 --- a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogHttpTransport.java +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogHttpTransport.java @@ -28,7 +28,12 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import java.util.function.Supplier; import static com.google.common.net.MediaType.JSON_UTF_8; import static io.airlift.concurrent.MoreFutures.toCompletableFuture; @@ -45,11 +50,14 @@ public final class HogQlSemanticCatalogHttpTransport implements JsonTransport { private static final String METADATA_PATH = "v1/hogql/compatibility/semantic-catalog"; + private static final String AUTHENTICATION_HEADER = "X-Duckgres-Internal-Secret"; + private static final int MAXIMUM_AUTHENTICATION_TOKEN_BYTES = 4096; private static final MediaType JSON = JSON_UTF_8.withoutParameters(); private final URI baseUri; private final HttpClient httpClient; private final int maximumResponseBytes; + private final Supplier authenticationTokenSupplier; @Inject public HogQlSemanticCatalogHttpTransport( @@ -58,15 +66,16 @@ public HogQlSemanticCatalogHttpTransport( { this(requireNonNull(config, "config is null").getUri(), httpClient, - toIntExact(config.getMaximumResponseSize().toBytes())); + toIntExact(config.getMaximumResponseSize().toBytes()), + tokenSupplier(config.getAuthenticationTokenFile())); } - HogQlSemanticCatalogHttpTransport(URI baseUri, HttpClient httpClient) + HogQlSemanticCatalogHttpTransport(URI baseUri, HttpClient httpClient, Supplier authenticationTokenSupplier) { - this(baseUri, httpClient, HogQlSemanticCatalogSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES); + this(baseUri, httpClient, HogQlSemanticCatalogSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES, authenticationTokenSupplier); } - private HogQlSemanticCatalogHttpTransport(URI baseUri, HttpClient httpClient, int maximumResponseBytes) + private HogQlSemanticCatalogHttpTransport(URI baseUri, HttpClient httpClient, int maximumResponseBytes, Supplier authenticationTokenSupplier) { this.baseUri = validateBaseUri(baseUri); this.httpClient = requireNonNull(httpClient, "httpClient is null"); @@ -74,19 +83,57 @@ private HogQlSemanticCatalogHttpTransport(URI baseUri, HttpClient httpClient, in throw new IllegalArgumentException("invalid HogQL semantic catalog response size limit"); } this.maximumResponseBytes = maximumResponseBytes; + this.authenticationTokenSupplier = requireNonNull(authenticationTokenSupplier, "authenticationTokenSupplier is null"); } @Override public CompletionStage load(LoadRequest request) { requireNonNull(request, "request is null"); + String authenticationToken; + try { + authenticationToken = validateAuthenticationToken(authenticationTokenSupplier.get()); + } + catch (RuntimeException _) { + return CompletableFuture.failedFuture(unavailable()); + } Request httpRequest = prepareGet() .setUri(buildUri(baseUri, request)) .setHeader(ACCEPT, JSON.toString()) + .setHeader(AUTHENTICATION_HEADER, authenticationToken) .build(); return toCompletableFuture(httpClient.executeAsync(httpRequest, new MetadataResponseHandler(request, maximumResponseBytes))); } + private static Supplier tokenSupplier(String authenticationTokenFile) + { + if (authenticationTokenFile == null || authenticationTokenFile.isBlank()) { + throw new IllegalArgumentException("HogQL semantic catalog authentication token file is not configured"); + } + Path path = Path.of(authenticationTokenFile); + return () -> { + try (InputStream input = Files.newInputStream(path)) { + byte[] token = input.readNBytes(MAXIMUM_AUTHENTICATION_TOKEN_BYTES + 1); + if (token.length > MAXIMUM_AUTHENTICATION_TOKEN_BYTES) { + throw new IllegalArgumentException("invalid HogQL semantic catalog authentication token"); + } + return new String(token, StandardCharsets.UTF_8).strip(); + } + catch (IOException e) { + throw new IllegalStateException("HogQL semantic catalog authentication token is unavailable", e); + } + }; + } + + private static String validateAuthenticationToken(String token) + { + requireNonNull(token, "authentication token is null"); + if (token.isBlank() || token.indexOf('\r') >= 0 || token.indexOf('\n') >= 0) { + throw new IllegalArgumentException("invalid HogQL semantic catalog authentication token"); + } + return token; + } + static URI buildUri(URI baseUri, LoadRequest request) { var uriBuilder = uriBuilderFrom(validateBaseUri(baseUri)) @@ -110,6 +157,11 @@ private static URI validateBaseUri(URI baseUri) return baseUri; } + private static HogQlSemanticCatalogException unavailable() + { + return new HogQlSemanticCatalogException(Failure.UNAVAILABLE, "HogQL semantic catalog metadata is unavailable"); + } + private static final class MetadataResponseHandler implements ResponseHandler { @@ -174,7 +226,7 @@ private static HogQlSemanticCatalogException statusFailure(int statusCode, LoadR private static HogQlSemanticCatalogException unavailable() { - return new HogQlSemanticCatalogException(Failure.UNAVAILABLE, "HogQL semantic catalog metadata is unavailable"); + return HogQlSemanticCatalogHttpTransport.unavailable(); } } } diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogManager.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogManager.java index b62c364e8f34..2bbdb36491b6 100644 --- a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogManager.java +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogManager.java @@ -91,7 +91,9 @@ private HogQlSemanticCatalogManager( config.getFailureBackoff().toJavaTime(), ticker, loaderExecutor, - catalog -> loader.load(LoadRequest.latest(catalog, languageVersion))); + (catalog, expectedGeneration) -> loader.load(expectedGeneration.isPresent() + ? LoadRequest.pinned(catalog, languageVersion, expectedGeneration.orElseThrow()) + : LoadRequest.latest(catalog, languageVersion))); } public HogQlSemanticCatalogSnapshotLoader loader() diff --git a/core/trino-main/src/test/java/io/trino/execution/TestHogQlCompilationExecutor.java b/core/trino-main/src/test/java/io/trino/execution/TestHogQlCompilationExecutor.java index e6bdf6e72ad9..1c07831d0a13 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestHogQlCompilationExecutor.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestHogQlCompilationExecutor.java @@ -13,6 +13,7 @@ */ package io.trino.execution; +import io.airlift.units.Duration; import io.trino.hogql.HogQlCompilationEvent; import io.trino.hogql.HogQlConfig; import io.trino.hogql.compiler.HogQlCompileEnvelope; @@ -36,8 +37,10 @@ import static io.trino.hogql.HogQlCompilationEvent.Outcome.INSUFFICIENT_RESOURCES; import static io.trino.hogql.HogQlCompilationEvent.Phase.COMPILATION; import static io.trino.hogql.HogQlCoordinatorErrorCode.HOGQL_COMPILATION_QUEUE_FULL; +import static io.trino.hogql.HogQlCoordinatorErrorCode.HOGQL_COMPILATION_TIMEOUT; import static io.trino.testing.TestingSession.testSessionBuilder; import static java.util.concurrent.Executors.newSingleThreadExecutor; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -153,6 +156,29 @@ public void testShutdownInterruptsWorkAndRejectsNewCompilations() } } + @Test + public void testCompilationTimeoutInterruptsWorker() + throws Exception + { + HogQlCompilationExecutor executor = new HogQlCompilationExecutor(config(1, 0) + .setCompilationTimeout(new Duration(10, MILLISECONDS))); + CountDownLatch workerInterrupted = new CountDownLatch(1); + try { + assertThatThrownBy(() -> executor.execute(() -> { + awaitInterruption(workerInterrupted); + return null; + })) + .isInstanceOfSatisfying(TrinoException.class, exception -> { + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_COMPILATION_TIMEOUT.toErrorCode()); + assertThat(exception).hasMessageContaining("time limit"); + }); + assertThat(workerInterrupted.await(10, SECONDS)).isTrue(); + } + finally { + executor.shutdown(); + } + } + private static HogQlConfig config(int threads, int queueCapacity) { return new HogQlConfig() @@ -183,4 +209,15 @@ private static void await(CountDownLatch latch) throw new RuntimeException(exception); } } + + private static void awaitInterruption(CountDownLatch interrupted) + { + try { + new CountDownLatch(1).await(); + } + catch (InterruptedException exception) { + interrupted.countDown(); + Thread.currentThread().interrupt(); + } + } } diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlCompilationObservability.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlCompilationObservability.java index cb38dee35c79..422777a044b2 100644 --- a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlCompilationObservability.java +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlCompilationObservability.java @@ -107,7 +107,21 @@ public void testFailureOutcomeAndPhaseAreStable(Supplier failu } @Test - public void testStatsRecordOutcomesAndPhaseDurations() + public void testResolvedCatalogGenerationReplacesRequestedGeneration() + { + List events = new ArrayList<>(); + HogQlCompilationTracker tracker = new HogQlCompilationTracker(events::add, Dimensions.fromEnvelope(envelope()), () -> 0); + + tracker.catalogGeneration(OptionalLong.of(43)); + tracker.succeeded(); + + assertThat(events).singleElement() + .extracting(event -> event.dimensions().catalogGeneration()) + .isEqualTo(OptionalLong.of(43)); + } + + @Test + public void testStatsRecordOutcomes() { HogQlCompilationStats stats = new HogQlCompilationStats(); Dimensions dimensions = Dimensions.fromEnvelope(envelope()); @@ -119,10 +133,7 @@ public void testStatsRecordOutcomesAndPhaseDurations() assertThat(stats.getSuccessfulCompilations().getTotalCount()).isEqualTo(1); assertThat(stats.getUserErrorFailures().getTotalCount()).isEqualTo(1); assertThat(stats.getInternalErrorFailures().getTotalCount()).isZero(); - assertThat(stats.getTotalTime().getAllTime().getCount()).isEqualTo(2); - assertThat(stats.getParseTime().getAllTime().getCount()).isEqualTo(2); - assertThat(stats.getBindTime().getAllTime().getCount()).isEqualTo(1); - assertThat(stats.getLowerTime().getAllTime().getCount()).isEqualTo(1); + assertThat(stats.getLastCatalogGeneration()).isEqualTo(41); } private static Stream failures() diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlConfig.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlConfig.java index f14202bafbb7..9d62fb36efd6 100644 --- a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlConfig.java +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlConfig.java @@ -13,6 +13,7 @@ */ package io.trino.hogql; +import io.airlift.units.Duration; import org.junit.jupiter.api.Test; import java.util.Map; @@ -20,6 +21,7 @@ import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static java.util.concurrent.TimeUnit.SECONDS; public class TestHogQlConfig { @@ -29,7 +31,8 @@ public void testDefaults() assertRecordedDefaults(recordDefaults(HogQlConfig.class) .setEnabled(false) .setCompilationThreads(2) - .setCompilationQueueCapacity(32)); + .setCompilationQueueCapacity(32) + .setCompilationTimeout(new Duration(10, SECONDS))); } @Test @@ -39,10 +42,12 @@ public void testExplicitPropertyMappings() Map.of( "hogql.enabled", "true", "hogql.compilation-threads", "4", - "hogql.compilation-queue-capacity", "128"), + "hogql.compilation-queue-capacity", "128", + "hogql.compilation-timeout", "3s"), new HogQlConfig() .setEnabled(true) .setCompilationThreads(4) - .setCompilationQueueCapacity(128)); + .setCompilationQueueCapacity(128) + .setCompilationTimeout(new Duration(3, SECONDS))); } } diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java index 487240c9ac3a..3d5e647394bf 100644 --- a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java @@ -15,6 +15,7 @@ import io.airlift.units.DataSize; import io.airlift.units.Duration; +import jakarta.validation.constraints.AssertTrue; import org.junit.jupiter.api.Test; import java.net.URI; @@ -23,6 +24,7 @@ import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static io.airlift.testing.ValidationAssertions.assertFailsValidation; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; @@ -34,6 +36,7 @@ public void testDefaults() { assertRecordedDefaults(recordDefaults(HogQlSemanticCatalogConfig.class) .setUri(null) + .setAuthenticationTokenFile(null) .setMaximumEntries(100) .setRefreshAfter(new Duration(1, MINUTES)) .setExpireAfter(new Duration(5, MINUTES)) @@ -49,6 +52,7 @@ public void testExplicitPropertyMappings() { Map properties = Map.of( "hogql.semantic-catalog.uri", "https://duckgres.example/metadata", + "hogql.semantic-catalog.authentication-token-file", "/run/secrets/hogql-catalog-token", "hogql.semantic-catalog.maximum-entries", "17", "hogql.semantic-catalog.refresh-after", "2m", "hogql.semantic-catalog.expire-after", "9m", @@ -60,6 +64,7 @@ public void testExplicitPropertyMappings() HogQlSemanticCatalogConfig expected = new HogQlSemanticCatalogConfig() .setUri(URI.create("https://duckgres.example/metadata")) + .setAuthenticationTokenFile("/run/secrets/hogql-catalog-token") .setMaximumEntries(17) .setRefreshAfter(new Duration(2, MINUTES)) .setExpireAfter(new Duration(9, MINUTES)) @@ -71,4 +76,14 @@ public void testExplicitPropertyMappings() assertFullMapping(properties, expected); } + + @Test + public void testUriRequiresAuthenticationTokenFile() + { + assertFailsValidation( + new HogQlSemanticCatalogConfig().setUri(URI.create("https://duckgres.example/metadata")), + "authenticationConfigured", + "hogql.semantic-catalog.authentication-token-file is required when hogql.semantic-catalog.uri is set", + AssertTrue.class); + } } diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogGuiceWiring.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogGuiceWiring.java index 72d995d534f8..54a780cd084b 100644 --- a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogGuiceWiring.java +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogGuiceWiring.java @@ -20,8 +20,11 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider; import io.trino.server.testing.TestingTrinoServer; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -35,10 +38,16 @@ public class TestHogQlSemanticCatalogGuiceWiring private static final Key> CATALOG_LISTENERS_KEY = Key.get(new TypeLiteral<>() {}); + @TempDir + public Path temporaryDirectory; + @Test public void testOptionalProviderFollowsHogQlAndCatalogConfiguration() throws IOException { + Path tokenFile = temporaryDirectory.resolve("catalog-token"); + Files.writeString(tokenFile, "test-token"); + try (TestingTrinoServer server = TestingTrinoServer.builder() .setProperties(Map.of("hogql.enabled", "false")) .build()) { @@ -49,7 +58,8 @@ public void testOptionalProviderFollowsHogQlAndCatalogConfiguration() try (TestingTrinoServer server = TestingTrinoServer.builder() .setProperties(Map.of( "hogql.enabled", "true", - "hogql.semantic-catalog.uri", "https://duckgres.example/metadata")) + "hogql.semantic-catalog.uri", "https://duckgres.example/metadata", + "hogql.semantic-catalog.authentication-token-file", tokenFile.toString())) .build()) { Optional provider = server.getInstance(OPTIONAL_PROVIDER_KEY); HogQlSemanticCatalogManager manager = server.getInstance(Key.get(HogQlSemanticCatalogManager.class)); diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogHttpTransport.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogHttpTransport.java index daf38445c3b7..56949a298e78 100644 --- a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogHttpTransport.java +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogHttpTransport.java @@ -24,12 +24,15 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; import io.trino.hogql.parser.HogQlLanguageVersion; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import java.net.URI; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; @@ -50,6 +53,7 @@ public class TestHogQlSemanticCatalogHttpTransport private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("Sales & Growth/2026", true); private static final URI BASE_URI = URI.create("https://duckgres.example/control-plane/"); private static final String SECRET = "secret-metadata-response"; + private static final String AUTHENTICATION_TOKEN = "test-authentication-token"; @Test public void testConstructsEncodedLatestAndPinnedUris() @@ -59,7 +63,7 @@ public void testConstructsEncodedLatestAndPinnedUris() requests.add(request); return response(OK, "{}"); }); - HogQlSemanticCatalogHttpTransport transport = new HogQlSemanticCatalogHttpTransport(BASE_URI, httpClient); + HogQlSemanticCatalogHttpTransport transport = new HogQlSemanticCatalogHttpTransport(BASE_URI, httpClient, () -> AUTHENTICATION_TOKEN); assertThat(transport.load(LoadRequest.latest(CATALOG, LANGUAGE_VERSION)).toCompletableFuture().join()) .isEqualTo("{}".getBytes(StandardCharsets.UTF_8)); @@ -72,15 +76,78 @@ public void testConstructsEncodedLatestAndPinnedUris() assertThat(requests).allSatisfy(request -> { assertThat(request.getMethod()).isEqualTo("GET"); assertThat(request.getHeader(ACCEPT)).isEqualTo(JSON_UTF_8.withoutParameters().toString()); + assertThat(request.getHeader("X-Duckgres-Internal-Secret")).isEqualTo(AUTHENTICATION_TOKEN); }); } + @Test + public void testReadsAuthenticationTokenForEveryRequest() + { + List requests = new ArrayList<>(); + List tokens = new ArrayList<>(List.of("first-token", "second-token")); + TestingHttpClient httpClient = new TestingHttpClient(request -> { + requests.add(request); + return response(OK, "{}"); + }); + HogQlSemanticCatalogHttpTransport transport = new HogQlSemanticCatalogHttpTransport(BASE_URI, httpClient, () -> tokens.removeFirst()); + + transport.load(LoadRequest.latest(CATALOG, LANGUAGE_VERSION)).toCompletableFuture().join(); + transport.load(LoadRequest.latest(CATALOG, LANGUAGE_VERSION)).toCompletableFuture().join(); + + assertThat(requests).extracting(request -> request.getHeader("X-Duckgres-Internal-Secret")) + .containsExactly("first-token", "second-token"); + } + + @Test + public void testReadsRotatingAuthenticationTokenFileForEveryRequest(@TempDir Path temporaryDirectory) + throws Exception + { + Path tokenFile = temporaryDirectory.resolve("hogql-catalog-token"); + Files.writeString(tokenFile, "first-token\n"); + List requests = new ArrayList<>(); + TestingHttpClient httpClient = new TestingHttpClient(request -> { + requests.add(request); + return response(OK, "{}"); + }); + HogQlSemanticCatalogHttpTransport transport = new HogQlSemanticCatalogHttpTransport( + new HogQlSemanticCatalogConfig() + .setUri(BASE_URI) + .setAuthenticationTokenFile(tokenFile.toString()), + httpClient); + + transport.load(LoadRequest.latest(CATALOG, LANGUAGE_VERSION)).toCompletableFuture().join(); + Files.writeString(tokenFile, "second-token"); + transport.load(LoadRequest.latest(CATALOG, LANGUAGE_VERSION)).toCompletableFuture().join(); + + assertThat(requests).extracting(request -> request.getHeader("X-Duckgres-Internal-Secret")) + .containsExactly("first-token", "second-token"); + } + + @ParameterizedTest + @MethodSource("invalidAuthenticationTokens") + public void testRejectsInvalidAuthenticationTokensWithoutDisclosure(String token) + { + HogQlSemanticCatalogHttpTransport transport = new HogQlSemanticCatalogHttpTransport( + BASE_URI, + new TestingHttpClient(_ -> response(OK, "{}")), + () -> token); + + assertThatThrownBy(() -> transport.load(LoadRequest.latest(CATALOG, LANGUAGE_VERSION)).toCompletableFuture().join()) + .cause() + .isInstanceOfSatisfying(HogQlSemanticCatalogException.class, exception -> { + assertThat(exception.failure()).isEqualTo(Failure.UNAVAILABLE); + if (!token.isEmpty()) { + assertThat(exception).hasMessageNotContaining(token); + } + }); + } + @ParameterizedTest(name = "{0}") @MethodSource("statusFailures") public void testMapsHttpStatusWithoutResponseDisclosure(String name, HttpStatus status, LoadRequest request, Failure failure) { TestingHttpClient httpClient = new TestingHttpClient(_ -> response(status, SECRET)); - HogQlSemanticCatalogHttpTransport transport = new HogQlSemanticCatalogHttpTransport(BASE_URI, httpClient); + HogQlSemanticCatalogHttpTransport transport = new HogQlSemanticCatalogHttpTransport(BASE_URI, httpClient, () -> AUTHENTICATION_TOKEN); assertThatThrownBy(() -> transport.load(request).toCompletableFuture().join()) .cause() @@ -100,6 +167,11 @@ private static Stream statusFailures() Arguments.of("server failure", INTERNAL_SERVER_ERROR, LoadRequest.latest(CATALOG, LANGUAGE_VERSION), Failure.UNAVAILABLE)); } + private static Stream invalidAuthenticationTokens() + { + return Stream.of("", " ", "line-one\nline-two"); + } + private static Response response(HttpStatus status, String body) { return new TestingResponse(status, contentType(JSON_UTF_8), body.getBytes(StandardCharsets.UTF_8)); diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java index 3a3dd480b2cc..49e97bdce043 100644 --- a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java @@ -17,18 +17,21 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; import io.trino.hogql.parser.HogQlLanguageVersion; import io.trino.spi.catalog.CatalogName; import org.junit.jupiter.api.Test; import java.util.List; import java.util.Optional; +import java.util.OptionalLong; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import static io.trino.testing.assertions.Assert.assertEventually; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -82,6 +85,35 @@ public void testColdAndExpiredReadsStayLocalWhileRefreshIsAsynchronous() } } + @Test + public void testColdExactGenerationUsesPinnedLoadRequest() + { + CompletableFuture requestedLoad = new CompletableFuture<>(); + CompletableFuture metadata = new CompletableFuture<>(); + HogQlSemanticCatalogManager manager = new HogQlSemanticCatalogManager( + config(), + request -> { + requestedLoad.complete(request); + return metadata; + }, + LANGUAGE_VERSION, + System::nanoTime); + try { + assertThat(manager.cache().currentSnapshot(CATALOG, OptionalLong.of(7))).isEmpty(); + assertThat(requestedLoad.join()).isEqualTo(LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 7)); + + metadata.complete(snapshot(7)); + assertEventually(() -> assertThat(manager.cache().currentSnapshot(CATALOG, OptionalLong.of(7))) + .get() + .extracting(HogQlSemanticCatalogSnapshot::generation) + .isEqualTo(7L)); + } + finally { + metadata.completeExceptionally(new IllegalStateException("test shutdown")); + manager.shutdown(); + } + } + @Test public void testShutdownTerminatesDedicatedLoaderExecutor() { From 5dc283d1027d9983c2c7eaca8959ef2ba8933b8f Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 21:24:36 -0400 Subject: [PATCH 076/136] chore(hogql): identify release artifacts --- core/docker/Dockerfile | 14 ++++++ core/docker/build.sh | 31 ++++++++++++- docs/src/main/sphinx/admin/hogql.md | 67 ++++++++++++++++++++++++++--- 3 files changed, 105 insertions(+), 7 deletions(-) diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile index 232856796106..22a9d0cee9c0 100644 --- a/core/docker/Dockerfile +++ b/core/docker/Dockerfile @@ -50,6 +50,20 @@ RUN \ FROM registry.access.redhat.com/hi/core-runtime:latest ARG JDK_VERSION ARG ARCH +ARG TRINO_SOURCE_REVISION +ARG HOGQL_LANGUAGE_VERSION +ARG HOGQL_CATALOG_PROTOCOL_VERSION +ARG HOGQL_CATALOG_SCHEMA_VERSION +ARG TRINO_SERVER_SHA256 +ARG TRINO_CLI_SHA256 +LABEL org.opencontainers.image.revision="${TRINO_SOURCE_REVISION}" \ + io.posthog.trino.ducklake.revision="${TRINO_SOURCE_REVISION}" \ + io.posthog.trino.hogql.compiler-build="${TRINO_SOURCE_REVISION}" \ + io.posthog.trino.hogql.language-version="${HOGQL_LANGUAGE_VERSION}" \ + io.posthog.trino.hogql.catalog-protocol-version="${HOGQL_CATALOG_PROTOCOL_VERSION}" \ + io.posthog.trino.hogql.catalog-schema-version="${HOGQL_CATALOG_SCHEMA_VERSION}" \ + io.posthog.trino.server-sha256="${TRINO_SERVER_SHA256}" \ + io.posthog.trino.cli-sha256="${TRINO_CLI_SHA256}" ENV JAVA_HOME="/usr/lib/jvm/${JDK_VERSION}" ENV PATH=$PATH:$JAVA_HOME/bin ENV LANG=C.UTF-8 diff --git a/core/docker/build.sh b/core/docker/build.sh index 695b28867249..99f7d3961d97 100755 --- a/core/docker/build.sh +++ b/core/docker/build.sh @@ -32,6 +32,15 @@ TEMURIN_RELEASE=$("${SOURCE_DIR}/mvnw" -f "${SOURCE_DIR}/pom.xml" --quiet help:e TEMURIN_DOWNLOAD_URL="https://api.adoptium.net/v3/binary/version/{release_name}/linux/{arch}/jdk/hotspot/normal/eclipse?project=jdk" SKIP_TESTS=false +EXPLICIT_TRINO_SOURCE_REVISION="${TRINO_SOURCE_REVISION:-}" +TRINO_SOURCE_REVISION="${EXPLICIT_TRINO_SOURCE_REVISION}" +if [ -z "${TRINO_SOURCE_REVISION}" ] && git -C "${SOURCE_DIR}" rev-parse HEAD >/dev/null 2>&1; then + TRINO_SOURCE_REVISION="$(git -C "${SOURCE_DIR}" rev-parse HEAD)" +fi +TRINO_SOURCE_REVISION="${TRINO_SOURCE_REVISION:-unknown}" +HOGQL_LANGUAGE_VERSION="${HOGQL_LANGUAGE_VERSION:-1.0.0}" +HOGQL_CATALOG_PROTOCOL_VERSION="${HOGQL_CATALOG_PROTOCOL_VERSION:-1}" +HOGQL_CATALOG_SCHEMA_VERSION="${HOGQL_CATALOG_SCHEMA_VERSION:-2}" while getopts ":a:h:r:p:t:j:x" o; do case "${o}" in @@ -72,6 +81,11 @@ while getopts ":a:h:r:p:t:j:x" o; do done shift $((OPTIND - 1)) +if [ -n "${TRINO_VERSION}" ] && [ -z "${EXPLICIT_TRINO_SOURCE_REVISION}" ]; then + echo >&2 "TRINO_SOURCE_REVISION must be set when building downloaded release artifacts" + exit 1 +fi + function check_environment() { if ! command -v jq &> /dev/null; then echo >&2 "Please install jq" @@ -79,6 +93,14 @@ function check_environment() { fi } +function sha256_file() { + if command -v sha256sum &> /dev/null; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + function temurin_download_uri() { local RELEASE_NAME="${1}" local ARCH="${2}" @@ -112,6 +134,8 @@ else trino_server="${SOURCE_DIR}/core/${SERVER_ARTIFACT}/target/${SERVER_ARTIFACT}-${TRINO_VERSION}.tar.gz" trino_client="${SOURCE_DIR}/client/trino-cli/target/trino-cli-${TRINO_VERSION}-executable.jar" fi +TRINO_SERVER_SHA256="$(sha256_file "${trino_server}")" +TRINO_CLI_SHA256="$(sha256_file "${trino_client}")" echo "🧱 Preparing the image build context directory" WORK_DIR="$(mktemp -d)" @@ -138,6 +162,12 @@ for arch in "${ARCHITECTURES[@]}"; do --build-arg ARCH="${arch}" \ --build-arg JDK_VERSION="${TEMURIN_RELEASE}" \ --build-arg JDK_DOWNLOAD_LINK="${JDK_DOWNLOAD_LINK}" \ + --build-arg TRINO_SOURCE_REVISION="${TRINO_SOURCE_REVISION}" \ + --build-arg HOGQL_LANGUAGE_VERSION="${HOGQL_LANGUAGE_VERSION}" \ + --build-arg HOGQL_CATALOG_PROTOCOL_VERSION="${HOGQL_CATALOG_PROTOCOL_VERSION}" \ + --build-arg HOGQL_CATALOG_SCHEMA_VERSION="${HOGQL_CATALOG_SCHEMA_VERSION}" \ + --build-arg TRINO_SERVER_SHA256="${TRINO_SERVER_SHA256}" \ + --build-arg TRINO_CLI_SHA256="${TRINO_CLI_SHA256}" \ --platform "linux/$arch" \ -f Dockerfile \ -t "${TAG}-$arch" @@ -157,4 +187,3 @@ else docker image inspect -f '🚀 Built {{.RepoTags}} {{.Id}}' "${TAG}-$arch" done fi - diff --git a/docs/src/main/sphinx/admin/hogql.md b/docs/src/main/sphinx/admin/hogql.md index 696de638b2fb..20c33693394b 100644 --- a/docs/src/main/sphinx/admin/hogql.md +++ b/docs/src/main/sphinx/admin/hogql.md @@ -11,6 +11,17 @@ Set the following coordinator property to enable the endpoint: hogql.enabled=true ``` +Compilation runs on a dedicated fixed-size executor. Configure +`hogql.compilation-threads`, `hogql.compilation-queue-capacity`, and +`hogql.compilation-timeout`; saturation and timeout fail with retryable +insufficient-resource errors without affecting standard SQL submission. + +The MVP profile supports read-only queries over the logical `events` and +`persons` tables, scalar reads from their VARCHAR JSON `properties` columns, +and the lazy `events.person` relationship. It also supports the frozen +fail-closed function set documented below. Physical tables remain available +through normal Trino names. + The endpoint accepts a JSON request envelope. Identity, catalog, schema, time zone, source, tags, and allowed session properties continue to use the standard Trino request headers. @@ -111,13 +122,41 @@ Trino `relation.*`. `EXCLUDE` needs the manifest's ordered, star-visible field list, and fails with a source-located compatibility error when that schema is unavailable. Duckgres should publish physical tables as physical-derived semantic definitions from the physical catalog inventory when those tables -need HogQL star exclusion. Schema inference for CTE and subquery outputs is not -currently supported. +need HogQL star exclusion. + +Set `hogql.semantic-catalog.authentication-token-file` to a file containing a +Duckgres read-only or admin token. Trino reads the file for every manifest +request so token rotation does not require a coordinator restart. Surrounding +whitespace is stripped; blank tokens, oversized tokens, and embedded CR or LF +are rejected. The token is sent as `X-Duckgres-Internal-Secret` and is never +included in error messages. + +Clients can set `catalogGeneration` in the request envelope to require an exact +immutable generation. Exact-generation cache entries never fall back to the +latest manifest. A missing, expired, or mismatched generation fails closed. + +## MVP function profile -The Trino transport does not currently send an application authentication -credential. Duckgres requires `X-Duckgres-Internal-Secret` by default, so the -production connection remains incomplete until the coordinator credential -mechanism is configured. +The compiler accepts only the following unqualified function names. Unknown +names fail before Trino analysis. + +| Family | HogQL names | +| --- | --- | +| Numeric, conditional, and strings | `abs`, `coalesce`, `if`, `lower`, `upper`, `length`, `concat`, `replace` | +| Collections | `map`, `arraySort`, `arrayDistinct`, `arrayFlatten`, `arrayStringConcat` | +| Date and time | `dateAdd`, `dateDiff`, `dateTrunc` | +| Aggregates | `count`, `sum`, `min`, `max`, `avg`, `any`, `argMin`, `argMax`, `array_agg` | +| Windows | `first_value`, `rank`, `row_number` | + +The MVP intentionally excludes actions, cohorts, saved queries, explicit +modifiers, HogQLX, PIVOT/UNPIVOT execution, ClickHouse-only clauses, advanced +lambda and table functions, and complete HogQL type/function parity. These +constructs return typed compatibility errors; they do not fall through as +Trino functions. + +`any`, `argMin`, and `argMax` are nondeterministic when more than one input can +satisfy the selection rule. Differential validation uses unique selection keys +or invariant-based comparisons for those calls. ## Physical catalog inventory @@ -154,6 +193,7 @@ accept a separate credential or an existing transaction ID. | Property | Default | Description | | --- | --- | --- | | `hogql.semantic-catalog.maximum-entries` | `100` | Maximum catalog snapshots retained by one coordinator. | +| `hogql.semantic-catalog.authentication-token-file` | none | Required when the semantic catalog URI is set. File containing a rotating Duckgres read-only or admin token. | | `hogql.semantic-catalog.refresh-after` | `1m` | Age at which a cached manifest is refreshed in the background. | | `hogql.semantic-catalog.expire-after` | `5m` | Age after which a cached manifest cannot be used. | | `hogql.semantic-catalog.failure-backoff` | `10s` | Minimum delay before retrying a failed refresh. | @@ -164,3 +204,18 @@ accept a separate credential or an existing transaction ID. Keep `refresh-after` below `expire-after`. Size the entry count, loader threads, and queue capacity for the number of catalogs assigned to one coordinator. + +## Image identity and rollback + +Release images use tags of the form +`-ducklake.-hogql.`. Inspect the immutable digest and +the `io.posthog.trino.*` OCI labels to verify the source/DuckLake revision, +compiler build, server and CLI artifact digests, language version, and semantic +catalog protocol/schema. + +To roll back, first stop routing clients to `/v1/hogql`. Set +`hogql.enabled=false` and roll the coordinators; standard `/v1/statement` +queries remain on the unchanged Trino path. Restore the previous immutable +image digest if needed. Semantic catalog content is immutable, so restore prior +content by publishing it as a new higher generation rather than mutating an +existing generation. From 099c57ef187ec8a7047f5c07d98a028704341e88 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 21:32:14 -0400 Subject: [PATCH 077/136] fix(hogql): verify release image provenance --- core/docker/build.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/core/docker/build.sh b/core/docker/build.sh index 99f7d3961d97..be61d29d3da2 100755 --- a/core/docker/build.sh +++ b/core/docker/build.sh @@ -173,6 +173,28 @@ for arch in "${ARCHITECTURES[@]}"; do -t "${TAG}-$arch" done +function require_image_label() { + local image="$1" + local name="$2" + local expected="$3" + local actual + actual="$(docker image inspect --format "{{index .Config.Labels \"${name}\"}}" "${image}")" + if [ "${actual}" != "${expected}" ]; then + echo >&2 "Image label ${name} is ${actual}, expected ${expected}" + exit 1 + fi +} + +for arch in "${ARCHITECTURES[@]}"; do + image="${TAG}-${arch}" + require_image_label "${image}" "org.opencontainers.image.revision" "${TRINO_SOURCE_REVISION}" + require_image_label "${image}" "io.posthog.trino.hogql.language-version" "${HOGQL_LANGUAGE_VERSION}" + require_image_label "${image}" "io.posthog.trino.hogql.catalog-protocol-version" "${HOGQL_CATALOG_PROTOCOL_VERSION}" + require_image_label "${image}" "io.posthog.trino.hogql.catalog-schema-version" "${HOGQL_CATALOG_SCHEMA_VERSION}" + require_image_label "${image}" "io.posthog.trino.server-sha256" "${TRINO_SERVER_SHA256}" + require_image_label "${image}" "io.posthog.trino.cli-sha256" "${TRINO_CLI_SHA256}" +done + echo "🧹 Cleaning up the build context directory" rm -r "${WORK_DIR}" From ad1477bf9648e6ffca0d9103c3540e4ab56d10e5 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 22:48:24 -0400 Subject: [PATCH 078/136] feat(hogql): lower additional query syntax --- .../trino/hogql/compiler/HogQlCompiler.java | 8 ++ .../hogql/compiler/HogQlFunctionResolver.java | 8 ++ .../hogql/compiler/HogQlProjectionDemand.java | 28 +++++- .../hogql/compiler/HogQlSemanticResolver.java | 51 ++++++++++- .../compiler/HogQlV0ProfileValidator.java | 4 + .../trino/hogql/compiler/TrinoAstFactory.java | 52 +++++++++++ .../hogql/compiler/TestHogQlCompiler.java | 32 +++++++ .../io/trino/hogql/parser/HogQlParser.java | 91 ++++++++++++++++++- .../trino/hogql/parser/tree/HogQlQuery.java | 37 ++++++++ 9 files changed, 302 insertions(+), 9 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index 244ce689bcb9..c10bd5a2f960 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -43,6 +43,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; @@ -57,6 +58,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; import io.trino.hogql.parser.tree.HogQlQuery.Window; import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; @@ -305,6 +307,7 @@ private static boolean containsSemanticCandidate(Relation relation) case SubqueryRelation subquery -> containsSemanticCandidate(subquery.query()); case TablePlaceholder _ -> false; case HogQlQuery.TableReference table -> table.parts().size() == 1; + case UnnestRelation unnest -> unnest.expressions().stream().anyMatch(HogQlCompiler::containsFunctionCall); case ValuesRelation _ -> false; }; } @@ -347,6 +350,7 @@ private static boolean containsFunctionCall(Relation relation) pivot.groupBy().stream().anyMatch(HogQlCompiler::containsFunctionCall); case SubqueryRelation subquery -> containsFunctionCall(subquery.query()); case TablePlaceholder _, HogQlQuery.TableReference _ -> false; + case UnnestRelation unnest -> unnest.expressions().stream().anyMatch(HogQlCompiler::containsFunctionCall); case ValuesRelation values -> values.rows().stream() .flatMap(List::stream) .anyMatch(HogQlCompiler::containsFunctionCall); @@ -380,6 +384,7 @@ private static boolean containsFunctionCall(Expression expression) case InSubqueryExpression in -> containsFunctionCall(in.value()) || containsFunctionCall(in.query()); case IntervalExpression interval -> containsFunctionCall(interval.value()); case IsNullExpression isNull -> containsFunctionCall(isNull.value()); + case LambdaExpression lambda -> containsFunctionCall(lambda.body()); case MemberAccessExpression memberAccess -> containsFunctionCall(memberAccess.base()); case ScalarSubqueryExpression subquery -> containsSemanticCandidate(subquery.query()); case SubscriptExpression subscript -> containsFunctionCall(subscript.base()) || containsFunctionCall(subscript.index()); @@ -487,6 +492,7 @@ private static void collectPlaceholders(Expression expression, List } case IntervalExpression interval -> collectPlaceholders(interval.value(), placeholders); case IsNullExpression isNull -> collectPlaceholders(isNull.value(), placeholders); + case LambdaExpression lambda -> collectPlaceholders(lambda.body(), placeholders); case Literal _ -> {} case MemberAccessExpression memberAccess -> collectPlaceholders(memberAccess.base(), placeholders); case Placeholder placeholder -> placeholders.add(placeholder); @@ -530,6 +536,7 @@ private static void validateRelation(Relation relation) tablePlaceholder.span(), "HogQL parameter placeholders are not supported in table positions: " + tablePlaceholder.placeholder().name()); case HogQlQuery.TableReference _ -> {} + case UnnestRelation _ -> {} case ValuesRelation _ -> {} } } @@ -558,6 +565,7 @@ private static void collectPlaceholders(Relation relation, List pla case SubqueryRelation subquery -> collectPlaceholders(subquery.query(), placeholders); case TablePlaceholder _ -> {} case HogQlQuery.TableReference _ -> {} + case UnnestRelation unnest -> unnest.expressions().forEach(expression -> collectPlaceholders(expression, placeholders)); case ValuesRelation values -> values.rows().forEach(row -> row.forEach(expression -> collectPlaceholders(expression, placeholders))); } } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 5a99507d7773..e49a17bb24d5 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -46,6 +46,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.PivotAggregation; import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; @@ -65,6 +66,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; import io.trino.hogql.parser.tree.HogQlQuery.Window; import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; @@ -208,6 +210,11 @@ private Relation resolveRelation(Relation relation) case SubqueryRelation subquery -> new SubqueryRelation(resolveQuery(subquery.query()), subquery.span()); case TablePlaceholder placeholder -> placeholder; case TableReference table -> table; + case UnnestRelation unnest -> new UnnestRelation( + unnest.expressions().stream().map(this::resolveExpression).toList(), + unnest.alias(), + unnest.columnAliases(), + unnest.span()); case ValuesRelation values -> new ValuesRelation( values.rows().stream() .map(row -> row.stream().map(this::resolveExpression).toList()) @@ -273,6 +280,7 @@ private Expression resolveExpression(Expression expression) isNull.negated(), isNull.predicateSpan(), isNull.span()); + case LambdaExpression lambda -> new LambdaExpression(lambda.arguments(), resolveExpression(lambda.body()), lambda.span()); case Literal literal -> literal; case MemberAccessExpression memberAccess -> new MemberAccessExpression( resolveExpression(memberAccess.base()), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java index 5cb2020327ef..3e0af14fe7c0 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java @@ -35,6 +35,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; @@ -45,6 +46,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; import io.trino.hogql.parser.tree.HogQlQuery.Window; import io.trino.hogql.parser.tree.HogQlQuery.WindowFrame; import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; @@ -255,6 +257,7 @@ private static void collectJoinCriteria(Relation relation, Builder builder) pivot.valueGroups().forEach(group -> group.values().forEach(value -> collect(value, builder))); pivot.groupBy().forEach(expression -> collect(expression, builder)); } + case UnnestRelation unnest -> unnest.expressions().forEach(expression -> collect(expression, builder)); } } @@ -298,6 +301,9 @@ private static void collect(Expression expression, Builder builder) } case IntervalExpression interval -> collect(interval.value(), builder); case IsNullExpression isNull -> collect(isNull.value(), builder); + case LambdaExpression lambda -> builder.withSuppressed( + lambda.arguments().stream().map(Identifier::value).map(HogQlProjectionDemand::canonical).collect(java.util.stream.Collectors.toSet()), + () -> collect(lambda.body(), builder)); case Literal _, Placeholder _ -> {} case MemberAccessExpression member -> collect(member.base(), builder); case ScalarSubqueryExpression _ -> builder.all = true; @@ -365,18 +371,38 @@ private static final class Builder private final Set unqualified = new HashSet<>(); private final Set allQualifiers = new HashSet<>(); private final Map> qualified = new HashMap<>(); + private final Set suppressed = new HashSet<>(); public void add(ColumnReference reference) { if (reference.parts().size() == 1) { - unqualified.add(canonical(reference.parts().getFirst().value())); + String name = canonical(reference.parts().getFirst().value()); + if (!suppressed.contains(name)) { + unqualified.add(name); + } return; } String qualifier = canonical(reference.parts().getFirst().value()); + if (suppressed.contains(qualifier)) { + return; + } String field = canonical(reference.parts().get(1).value()); qualified.computeIfAbsent(qualifier, _ -> new HashSet<>()).add(field); } + public void withSuppressed(Set names, Runnable action) + { + Set previous = Set.copyOf(suppressed); + suppressed.addAll(names); + try { + action.run(); + } + finally { + suppressed.clear(); + suppressed.addAll(previous); + } + } + public void addUnqualified(Identifier identifier) { unqualified.add(canonical(identifier.value())); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index af09f2cb5800..2a0fd7dd1bbb 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -75,6 +75,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.PivotAggregation; import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; @@ -94,6 +95,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; import io.trino.hogql.parser.tree.HogQlQuery.Window; import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; @@ -137,6 +139,7 @@ final class HogQlSemanticResolver private Set localRelationQualifiers = Set.of(); private boolean allRelationsLogical; private Optional expandedRelation = Optional.empty(); + private final Set lambdaArguments = new HashSet<>(); private int generatedRelationId; private HogQlSemanticResolver(PinnedSnapshot snapshot) @@ -458,7 +461,7 @@ yield inferRelationBindings(alias.relation(), availableBindings).stream() .map(definition -> resolveLogicalTable(definition, table.span()).bindings()) .orElse(List.of()) : List.of(); - case SubqueryRelation _, TablePlaceholder _, ValuesRelation _ -> List.of(); + case SubqueryRelation _, TablePlaceholder _, UnnestRelation _, ValuesRelation _ -> List.of(); }; } @@ -643,7 +646,7 @@ else if (alias.relation() instanceof CommonTableReference commonTable) { HogQlProjectionDemand.preserveAll(), availableBindings, demands); - case SubqueryRelation _, TablePlaceholder _, TableReference _, ValuesRelation _ -> {} + case SubqueryRelation _, TablePlaceholder _, TableReference _, UnnestRelation _, ValuesRelation _ -> {} } } @@ -1009,6 +1012,7 @@ private static Set relationQualifiers(Relation relation) case SubqueryRelation _ -> Set.of(); case TablePlaceholder _ -> Set.of(); case TableReference table -> Set.of(canonical(table.parts().getLast().value())); + case UnnestRelation unnest -> Set.of(canonical(unnest.alias().value())); case ValuesRelation _ -> Set.of(); }; } @@ -1048,7 +1052,20 @@ yield new ResolvedRelation( .orElseGet(() -> new ResolvedRelation(commonTable, List.of(), false)); case JoinRelation join -> { ResolvedRelation left = resolveRelation(join.left(), projectionDemand); - ResolvedRelation right = resolveRelation(join.right(), projectionDemand); + List bindingsBeforeRight = bindings; + boolean allRelationsLogicalBeforeRight = allRelationsLogical; + if (join.right() instanceof UnnestRelation) { + bindings = left.bindings(); + allRelationsLogical = left.allLogical(); + } + ResolvedRelation right; + try { + right = resolveRelation(join.right(), projectionDemand); + } + finally { + bindings = bindingsBeforeRight; + allRelationsLogical = allRelationsLogicalBeforeRight; + } List joinBindings = new ArrayList<>(left.bindings()); joinBindings.addAll(right.bindings()); List previousBindings = bindings; @@ -1100,6 +1117,14 @@ yield new ResolvedRelation( case SubqueryRelation subquery -> resolveSubquery(subquery, projectionDemand.unqualified()); case TablePlaceholder placeholder -> new ResolvedRelation(placeholder, List.of(), false); case TableReference table -> resolveTable(table); + case UnnestRelation unnest -> new ResolvedRelation( + new UnnestRelation( + unnest.expressions().stream().map(this::resolveExpression).toList(), + unnest.alias(), + unnest.columnAliases(), + unnest.span()), + List.of(), + false); case ValuesRelation values -> new ResolvedRelation(values, List.of(), false); }; } @@ -1414,6 +1439,7 @@ private Expression resolveExpression(Expression expression) isNull.negated(), isNull.predicateSpan(), isNull.span()); + case LambdaExpression lambda -> resolveLambda(lambda); case Literal literal -> literal; case MemberAccessExpression memberAccess -> resolveMemberAccess(memberAccess); case Placeholder placeholder -> placeholder; @@ -1426,6 +1452,22 @@ private Expression resolveExpression(Expression expression) }; } + private LambdaExpression resolveLambda(LambdaExpression lambda) + { + Set previous = Set.copyOf(lambdaArguments); + lambda.arguments().stream() + .map(Identifier::value) + .map(HogQlSemanticResolver::canonical) + .forEach(lambdaArguments::add); + try { + return new LambdaExpression(lambda.arguments(), resolveExpression(lambda.body()), lambda.span()); + } + finally { + lambdaArguments.clear(); + lambdaArguments.addAll(previous); + } + } + private Expression resolveFunctionExpression(FunctionCall function) { if (function.nameParts().size() == 1 && canonical(function.name().value()).equals(MATCHES_ACTION)) { @@ -1642,6 +1684,9 @@ private WindowFrame resolveWindowFrame(WindowFrame frame) private Expression resolveColumn(ColumnReference reference) { List parts = reference.parts(); + if (lambdaArguments.contains(canonical(parts.getFirst().value()))) { + return reference; + } Optional semanticPath = resolveSemanticPath(reference); if (semanticPath.isPresent()) { return semanticPath.orElseThrow(); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java index 603195a45b6e..c72ea52879a2 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java @@ -36,6 +36,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; import io.trino.hogql.parser.tree.HogQlQuery.Projection; @@ -51,6 +52,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; import io.trino.hogql.parser.tree.HogQlQuery.Window; import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; @@ -117,6 +119,7 @@ private static void validate(Relation relation, Optional throw unsupported(pivot.span(), "PIVOT is outside the HogQL v0 profile"); case SubqueryRelation subquery -> validate(subquery.query(), snapshot); case TableReference table -> validateTable(table, snapshot); + case UnnestRelation unnest -> unnest.expressions().forEach(expression -> validate(expression, snapshot)); case ValuesRelation values -> values.rows().forEach(row -> row.forEach(expression -> validate(expression, snapshot))); } } @@ -185,6 +188,7 @@ private static void validate(Expression expression, Optional validate(interval.value(), snapshot); case IsNullExpression isNull -> validate(isNull.value(), snapshot); + case LambdaExpression lambda -> validate(lambda.body(), snapshot); case MemberAccessExpression memberAccess -> validate(memberAccess.base(), snapshot); case ScalarSubqueryExpression subquery -> validate(subquery.query(), snapshot); case SubscriptExpression subscript -> { diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 5573515a604d..30c835911ec3 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -55,6 +55,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; import io.trino.hogql.parser.tree.HogQlQuery.Window; import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; @@ -75,6 +76,7 @@ import io.trino.sql.tree.CoalesceExpression; import io.trino.sql.tree.ComparisonPredicate; import io.trino.sql.tree.DereferenceExpression; +import io.trino.sql.tree.DoubleLiteral; import io.trino.sql.tree.Except; import io.trino.sql.tree.Expression; import io.trino.sql.tree.GroupBy; @@ -86,6 +88,9 @@ import io.trino.sql.tree.IntervalLiteral; import io.trino.sql.tree.IsNullPredicate; import io.trino.sql.tree.Limit; +import io.trino.sql.tree.LambdaArgumentDeclaration; +import io.trino.sql.tree.LambdaExpression; +import io.trino.sql.tree.LikePredicate; import io.trino.sql.tree.LogicalExpression; import io.trino.sql.tree.LongLiteral; import io.trino.sql.tree.NodeLocation; @@ -114,6 +119,7 @@ import io.trino.sql.tree.Table; import io.trino.sql.tree.TableSubquery; import io.trino.sql.tree.Union; +import io.trino.sql.tree.Unnest; import io.trino.sql.tree.Values; import io.trino.sql.tree.WhenClause; import io.trino.sql.tree.With; @@ -350,6 +356,12 @@ private static Expression createExpression(HogQlQuery.Expression expression, Map location(isNull.predicateSpan()), createExpression(isNull.value(), parameterIds), new IsNullPredicate(location(isNull.predicateSpan()), isNull.negated())); + case HogQlQuery.LambdaExpression lambda -> new LambdaExpression( + location(lambda.span()), + lambda.arguments().stream() + .map(argument -> new LambdaArgumentDeclaration(location(argument.span()), createIdentifier(argument))) + .toList(), + createExpression(lambda.body(), parameterIds)); case Literal literal -> createLiteral(literal); case MemberAccessExpression memberAccess -> new DereferenceExpression( location(memberAccess.span()), @@ -596,6 +608,10 @@ private static Expression createBinaryExpression(BinaryExpression binary, Map new ArithmeticBinaryExpression(location, ArithmeticBinaryExpression.Operator.DIVIDE, left, right); case MODULO -> new ArithmeticBinaryExpression(location, ArithmeticBinaryExpression.Operator.MODULO, left, right); case AND -> new LogicalExpression(location, LogicalExpression.Operator.AND, List.of(left, right)); + case CONCAT -> new io.trino.sql.tree.FunctionCall( + location, + QualifiedName.of("concat"), + List.of(left, right)); case OR -> new LogicalExpression(location, LogicalExpression.Operator.OR, List.of(left, right)); case EQUAL -> comparison(location, ComparisonPredicate.Operator.EQUAL, left, right); case NOT_EQUAL -> comparison(location, ComparisonPredicate.Operator.NOT_EQUAL, left, right); @@ -603,9 +619,36 @@ private static Expression createBinaryExpression(BinaryExpression binary, Map comparison(location, ComparisonPredicate.Operator.LESS_THAN_OR_EQUAL, left, right); case GREATER_THAN -> comparison(location, ComparisonPredicate.Operator.GREATER_THAN, left, right); case GREATER_THAN_OR_EQUAL -> comparison(location, ComparisonPredicate.Operator.GREATER_THAN_OR_EQUAL, left, right); + case LIKE -> like(location, left, right, false, false); + case NOT_LIKE -> like(location, left, right, true, false); + case ILIKE -> like(location, left, right, false, true); + case NOT_ILIKE -> like(location, left, right, true, true); }; } + private static Expression like(NodeLocation location, Expression value, Expression pattern, boolean negated, boolean caseInsensitive) + { + if (caseInsensitive) { + value = lower(location, value); + pattern = lower(location, pattern); + } + return new Predicated(location, value, new LikePredicate(location, negated, pattern, Optional.empty())); + } + + private static Expression lower(NodeLocation location, Expression value) + { + return new io.trino.sql.tree.FunctionCall( + location, + QualifiedName.of("lower"), + Optional.empty(), + Optional.empty(), + Optional.empty(), + false, + Optional.empty(), + Optional.empty(), + List.of(new CallArgument(location, Optional.empty(), value))); + } + private static Expression comparison(NodeLocation location, ComparisonPredicate.Operator operator, Expression left, Expression right) { return new Predicated(location, left, new ComparisonPredicate(location, operator, right)); @@ -616,6 +659,7 @@ private static Expression createLiteral(Literal literal) NodeLocation location = location(literal.span()); return switch (literal.kind()) { case BOOLEAN -> new BooleanLiteral(location, literal.value()); + case FLOAT -> new DoubleLiteral(location, literal.value()); case INTEGER -> new LongLiteral(location, literal.value()); case NULL -> new NullLiteral(location); case STRING -> new StringLiteral(location, literal.value()); @@ -688,6 +732,14 @@ private static io.trino.sql.tree.Relation createRelation(Relation relation, Map< case SubqueryRelation subquery -> new TableSubquery(location(subquery.span()), createQuery(subquery.query(), parameterIds)); case TablePlaceholder _ -> throw new IllegalArgumentException("table placeholder was not validated"); case TableReference table -> createTable(table); + case UnnestRelation unnest -> new io.trino.sql.tree.AliasedRelation( + location(unnest.span()), + new Unnest( + location(unnest.span()), + unnest.expressions().stream().map(expression -> createExpression(expression, parameterIds)).toList(), + false), + createIdentifier(unnest.alias()), + unnest.columnAliases().stream().map(TrinoAstFactory::createIdentifier).toList()); case ValuesRelation values -> createValuesRelation(values, parameterIds); }; } diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index bbaf1008a9ef..733b416e2a14 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -113,12 +113,44 @@ public void testLowersDistinctOrderingAndClickHousePaginationOrder() SELECT value IN (1, 2, 3) | SELECT value IN (1, 2, 3) SELECT value IN (1) | SELECT value IN (1) SELECT value NOT IN (1, 2, 3) | SELECT value NOT IN (1, 2, 3) + SELECT value LIKE 'pro%' | SELECT value LIKE 'pro%' + SELECT value NOT LIKE 'pro%' | SELECT value NOT LIKE 'pro%' + SELECT value ILIKE 'Pro%' | SELECT lower(value) LIKE lower('Pro%') + SELECT value NOT ILIKE 'Pro%' | SELECT lower(value) NOT LIKE lower('Pro%') + SELECT 1.5 | SELECT 1.5E0 + SELECT .5 | SELECT .5E0 + SELECT 1. | SELECT 1E0 + SELECT -1.5 | SELECT -1.5E0 + SELECT +1.5 | SELECT +1.5E0 + SELECT 1e3 | SELECT 1E3 + SELECT map(x -> x + 1, [1]) | SELECT map(x -> x + 1, ARRAY[1]) + SELECT map((x, y) -> x + y, [1]) | SELECT map((x, y) -> x + y, ARRAY[1]) + SELECT map(lambda x: x + 1, [1]) | SELECT map(x -> x + 1, ARRAY[1]) """) public void testLowersCollectionAndPredicateExpressions(String hogql, String trinoSql) { assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(trinoSql)); } + @Test + public void testLowersConcatenationOperator() + { + assertThat(compiler.compile("SELECT first || second")) + .isEqualTo(sqlParser.createStatement("SELECT first || second")); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', textBlock = """ + SELECT item FROM events ARRAY JOIN attributes AS item | SELECT item FROM events CROSS JOIN UNNEST(attributes) AS __hogql_array_join(item) + SELECT item FROM events LEFT ARRAY JOIN attributes AS item | SELECT item FROM events LEFT JOIN UNNEST(attributes) AS __hogql_array_join(item) ON TRUE + SELECT item, key FROM events ARRAY JOIN attributes AS item, keys AS key | SELECT item, key FROM events CROSS JOIN UNNEST(attributes, keys) AS __hogql_array_join(item, key) + SELECT item ARRAY JOIN [1] AS item | SELECT item FROM UNNEST(ARRAY[1]) AS __hogql_array_join(item) + """) + public void testLowersArrayJoin(String hogql, String trinoSql) + { + assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(trinoSql)); + } + @ParameterizedTest @CsvSource(delimiter = '|', textBlock = """ SELECT INTERVAL 2 SECOND | SELECT 2 * INTERVAL '1' SECOND diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 08920d6a4867..d631f8e16cc1 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -48,6 +48,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; import io.trino.hogql.parser.tree.HogQlQuery.JoinType; import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; +import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; @@ -72,6 +73,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.TableReference; import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; import io.trino.hogql.parser.tree.HogQlQuery.Window; import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; @@ -111,14 +113,19 @@ import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.ADD; import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.AND; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.CONCAT; import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.DIVIDE; import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.EQUAL; import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.GREATER_THAN; import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.GREATER_THAN_OR_EQUAL; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.ILIKE; import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.LESS_THAN; import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.LESS_THAN_OR_EQUAL; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.LIKE; import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.MODULO; import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.MULTIPLY; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.NOT_ILIKE; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.NOT_LIKE; import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.NOT_EQUAL; import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.OR; import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.SUBTRACT; @@ -740,6 +747,9 @@ private HogQlQuery buildQuery(SelectedQuery selectedQuery) Optional from = Optional.ofNullable(select.fromClause()) .map(HogQLParser.FromClauseContext::joinExpr) .map(this::buildRelation); + if (select.arrayJoinClause() != null) { + from = Optional.of(buildArrayJoin(from, select.arrayJoinClause())); + } Optional where = Optional.ofNullable(select.whereClause()) .map(HogQLParser.WhereClauseContext::columnExpr) .map(this::buildExpression); @@ -838,7 +848,6 @@ private void rejectUnsupportedClauses(HogQLParser.SelectStmtContext context) { List clauses = new ArrayList<>(); clauses.add(context.topClause()); - clauses.add(context.arrayJoinClause()); clauses.add(context.prewhereClause()); clauses.addAll(context.sampleClause()); clauses.add(context.qualifyClause()); @@ -852,6 +861,32 @@ private void rejectUnsupportedClauses(HogQLParser.SelectStmtContext context) } } + private Relation buildArrayJoin(Optional from, HogQLParser.ArrayJoinClauseContext context) + { + List expressions = new ArrayList<>(); + List columnAliases = new ArrayList<>(); + for (HogQLParser.ColumnExprContext expression : context.columnExprList().columnExpr()) { + if (!(expression instanceof HogQLParser.ColumnExprAliasContext alias) || alias.identifier() == null) { + throw unsupported(expression, "ARRAY JOIN expression without an identifier alias"); + } + expressions.add(buildExpression(alias.columnExpr())); + columnAliases.add(buildIdentifier(alias.identifier())); + } + Identifier relationAlias = buildIdentifier("__hogql_array_join", context); + UnnestRelation unnest = new UnnestRelation(expressions, relationAlias, columnAliases, sourceSpan(context)); + if (from.isEmpty()) { + if (context.LEFT() != null) { + throw unsupported(context, "LEFT ARRAY JOIN without FROM"); + } + return unnest; + } + JoinType type = context.LEFT() == null ? JoinType.CROSS : JoinType.LEFT; + Optional criteria = context.LEFT() == null + ? Optional.empty() + : Optional.of(new JoinOn(new Literal(HogQlQuery.LiteralKind.BOOLEAN, "true", sourceSpan(context)), sourceSpan(context))); + return new JoinRelation(type, from.orElseThrow(), unnest, criteria, sourceSpan(context)); + } + private List buildOrderBy(HogQLParser.OrderByClauseContext context) { if (context == null) { @@ -1174,6 +1209,7 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) BinaryOperator operator = switch (binary.operator.getType()) { case HogQLParser.PLUS -> ADD; case HogQLParser.DASH -> SUBTRACT; + case HogQLParser.CONCAT -> CONCAT; default -> throw unsupported(binary, "additive operator"); }; return binary(operator, binary.left, binary.right, binary); @@ -1206,6 +1242,12 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) sourceSpan(binary.NOT() == null ? binary.IN().getSymbol() : binary.NOT().getSymbol(), binary.getStop()), sourceSpan(binary)); } + if (binary.LIKE() != null) { + return binary(binary.NOT() == null ? LIKE : NOT_LIKE, binary.left, binary.right, binary); + } + if (binary.ILIKE() != null) { + return binary(binary.NOT() == null ? ILIKE : NOT_ILIKE, binary.left, binary.right, binary); + } if (binary.operator == null) { throw unsupported(binary, "comparison operator"); } @@ -1275,6 +1317,31 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) function.filterExpr, new WindowReference(buildIdentifier(function.identifier(1)), sourceSpan(function.identifier(1)))); } + if (context instanceof HogQLParser.ColumnExprLambdaContext lambda) { + HogQLParser.ColumnLambdaExprContext lambdaExpression = lambda.columnLambdaExpr(); + if (lambdaExpression instanceof HogQLParser.ArrowLambdaContext arrow) { + if (arrow.block() != null) { + throw unsupported(arrow.block(), "lambda block"); + } + return new LambdaExpression( + arrow.identifier().stream().map(this::buildIdentifier).toList(), + buildExpression(arrow.columnExpr()), + sourceSpan(arrow)); + } + if (lambdaExpression instanceof HogQLParser.ColonLambdaContext colon) { + return new LambdaExpression( + colon.identifier().stream().map(this::buildIdentifier).toList(), + buildExpression(colon.columnExpr()), + sourceSpan(colon)); + } + throw unsupported(lambdaExpression, "lambda expression"); + } + if (context instanceof HogQLParser.ColumnExprColonLambdaContext lambda) { + return new LambdaExpression( + lambda.identifier().stream().map(this::buildIdentifier).toList(), + buildExpression(lambda.columnExpr()), + sourceSpan(lambda)); + } throw unsupported(context, "expression " + context.getClass().getSimpleName()); } @@ -1524,17 +1591,27 @@ private Expression buildLiteral(HogQLParser.LiteralContext context) } String value = context.numberLiteral().getText(); - if (!value.matches("[+-]?[0-9]+")) { + if (value.matches("[+-]?[0-9]+")) { + if (value.startsWith("-")) { + return new Literal(INTEGER, "-" + normalizeInteger(value.substring(1)), sourceSpan(context)); + } + if (value.startsWith("+")) { + Literal magnitude = new Literal(INTEGER, normalizeInteger(value.substring(1)), sourceSpan(context)); + return new UnaryExpression(POSITIVE, magnitude, sourceSpan(context)); + } + return new Literal(INTEGER, normalizeInteger(value), sourceSpan(context)); + } + if (!value.matches("[+-]?(?:[0-9]+(?:\\.[0-9]*)?|\\.[0-9]+)(?:[eE][+-]?[0-9]+)?")) { throw unsupported(context, "numeric literal"); } if (value.startsWith("-")) { - return new Literal(INTEGER, "-" + normalizeInteger(value.substring(1)), sourceSpan(context)); + return new Literal(HogQlQuery.LiteralKind.FLOAT, value, sourceSpan(context)); } if (value.startsWith("+")) { - Literal magnitude = new Literal(INTEGER, normalizeInteger(value.substring(1)), sourceSpan(context)); + Literal magnitude = new Literal(HogQlQuery.LiteralKind.FLOAT, value.substring(1), sourceSpan(context)); return new UnaryExpression(POSITIVE, magnitude, sourceSpan(context)); } - return new Literal(INTEGER, normalizeInteger(value), sourceSpan(context)); + return new Literal(HogQlQuery.LiteralKind.FLOAT, value, sourceSpan(context)); } private Expression buildColumnReference(HogQLParser.ColumnIdentifierContext context) @@ -1835,6 +1912,7 @@ private void validateRelationExpressionScopes(Relation relation, Set for validateExpressionScope(expression, forbiddenOuterRelations, localRelations))); pivot.groupBy().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); } + case UnnestRelation unnest -> unnest.expressions().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); case CommonTableReference _, SubqueryRelation _, TablePlaceholder _, TableReference _, ValuesRelation _ -> {} } } @@ -1849,6 +1927,7 @@ private void validateNestedRelationScopes(Relation relation, Set forbidd } case PivotRelation pivot -> validateNestedRelationScopes(pivot.input(), forbiddenOuterRelations); case SubqueryRelation subquery -> validateQueryScope(subquery.query(), forbiddenOuterRelations); + case UnnestRelation _ -> {} case ValuesRelation values -> values.rows().forEach(row -> row.forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, Set.of()))); case CommonTableReference _, TablePlaceholder _, TableReference _ -> {} } @@ -1871,6 +1950,7 @@ private void collectRelationNames(Relation relation, Set names) collectRelationNames(join.right(), names); } case PivotRelation pivot -> collectRelationNames(pivot.input(), names); + case UnnestRelation unnest -> names.add(canonicalName(unnest.alias())); case SubqueryRelation _, TablePlaceholder _, ValuesRelation _ -> {} case TableReference table -> names.add(canonicalName(table.parts().getLast())); } @@ -1928,6 +2008,7 @@ private void validateExpressionScope(Expression expression, Set forbidde } case IntervalExpression interval -> validateExpressionScope(interval.value(), forbiddenOuterRelations, localRelations); case IsNullExpression isNull -> validateExpressionScope(isNull.value(), forbiddenOuterRelations, localRelations); + case LambdaExpression lambda -> validateExpressionScope(lambda.body(), forbiddenOuterRelations, localRelations); case Literal _, Placeholder _ -> {} case MemberAccessExpression memberAccess -> validateExpressionScope(memberAccess.base(), forbiddenOuterRelations, localRelations); case ScalarSubqueryExpression subquery -> validateQueryScope(subquery.query(), Set.of()); diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index 5e7509d0c2e7..d9963fb2e759 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -307,6 +307,7 @@ public sealed interface Expression InSubqueryExpression, IntervalExpression, IsNullExpression, + LambdaExpression, Literal, MemberAccessExpression, Placeholder, @@ -389,6 +390,17 @@ public record IntervalExpression(Expression value, IntervalUnit unit, SourceSpan } } + public record LambdaExpression(List arguments, Expression body, SourceSpan span) + implements Expression + { + public LambdaExpression + { + arguments = List.copyOf(requireNonNull(arguments, "arguments is null")); + body = requireNonNull(body, "body is null"); + span = requireNonNull(span, "span is null"); + } + } + public record ArrayExpression(List values, SourceSpan span) implements Expression { @@ -681,14 +693,19 @@ public enum BinaryOperator { ADD, AND, + CONCAT, DIVIDE, EQUAL, GREATER_THAN, GREATER_THAN_OR_EQUAL, + ILIKE, LESS_THAN, LESS_THAN_OR_EQUAL, + LIKE, MODULO, MULTIPLY, + NOT_ILIKE, + NOT_LIKE, NOT_EQUAL, OR, SUBTRACT, @@ -734,6 +751,7 @@ public record Literal(LiteralKind kind, String value, SourceSpan span) public enum LiteralKind { BOOLEAN, + FLOAT, INTEGER, NULL, STRING, @@ -747,6 +765,7 @@ public sealed interface Relation SubqueryRelation, TablePlaceholder, TableReference, + UnnestRelation, ValuesRelation { SourceSpan span(); @@ -769,6 +788,24 @@ public AliasedRelation(Relation relation, Identifier alias, SourceSpan span) } } + public record UnnestRelation(List expressions, Identifier alias, List columnAliases, SourceSpan span) + implements Relation + { + public UnnestRelation + { + expressions = List.copyOf(requireNonNull(expressions, "expressions is null")); + if (expressions.isEmpty()) { + throw new IllegalArgumentException("expressions is empty"); + } + alias = requireNonNull(alias, "alias is null"); + columnAliases = List.copyOf(requireNonNull(columnAliases, "columnAliases is null")); + if (expressions.size() != columnAliases.size()) { + throw new IllegalArgumentException("expressions and columnAliases sizes differ"); + } + span = requireNonNull(span, "span is null"); + } + } + public record CommonTableReference(Identifier name, SourceSpan span) implements Relation { From 3eba4f9f5da9222c08f1c3b2a13d1b2f95d6f7ff Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 22:55:41 -0400 Subject: [PATCH 079/136] feat(hogql): add common function rewrites --- .../hogql/compiler/HogQlFunctionResolver.java | 40 +++++++++++++++--- .../compiler/HogQlV0FunctionRegistry.java | 42 +++++++++++++++++++ .../trino/hogql/compiler/TrinoAstFactory.java | 4 ++ .../catalog/HogQlSemanticCatalogSnapshot.java | 17 ++++++-- .../compiler/TestHogQlFunctionResolver.java | 12 +++++- 5 files changed, 105 insertions(+), 10 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index e49a17bb24d5..66a308f3bdc0 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -346,11 +346,7 @@ private Expression resolveFunction(FunctionCall function, boolean windowInvocati if (capability.implementation() == FunctionImplementation.REWRITE) { FunctionRewrite rewrite = capability.rewrite() .orElseThrow(() -> unsupportedError(function, "HogQL function " + name + " has no compiler rewrite")); - boolean negated = switch (rewrite) { - case IS_NULL -> false; - case IS_NOT_NULL -> true; - }; - return new IsNullExpression(arguments.getFirst(), negated, function.span(), function.span()); + return rewrite(function, rewrite, arguments); } return new FunctionCall( capability.trinoName().stream() @@ -365,6 +361,40 @@ private Expression resolveFunction(FunctionCall function, boolean windowInvocati function.span()); } + private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List arguments) + { + HogQlQuery.SourceSpan span = function.span(); + return switch (rewrite) { + case IS_NULL -> new IsNullExpression(arguments.getFirst(), false, span, span); + case IS_NOT_NULL -> new IsNullExpression(arguments.getFirst(), true, span, span); + case CAST_DATE -> cast(arguments.getFirst(), "date", span); + case CAST_DOUBLE -> cast(arguments.getFirst(), "double", span); + case CAST_BIGINT -> cast(arguments.getFirst(), "bigint", span); + case CAST_TIMESTAMP -> cast(arguments.getFirst(), "timestamp(0)", span); + case CAST_VARCHAR -> cast(arguments.getFirst(), "varchar", span); + case DATE_TRUNC_DAY -> dateTrunc("day", arguments.getFirst(), span); + case DATE_TRUNC_HOUR -> dateTrunc("hour", arguments.getFirst(), span); + case DATE_TRUNC_MONTH -> dateTrunc("month", arguments.getFirst(), span); + case DATE_TRUNC_WEEK -> dateTrunc("week", arguments.getFirst(), span); + }; + } + + private static CastExpression cast(Expression value, String type, HogQlQuery.SourceSpan span) + { + return new CastExpression(value, new Identifier(type, false, span), false, span); + } + + private static FunctionCall dateTrunc(String unit, Expression value, HogQlQuery.SourceSpan span) + { + return new FunctionCall( + new Identifier("date_trunc", false, span), + List.of(new Literal(HogQlQuery.LiteralKind.STRING, unit, span), value), + false, + List.of(), + Optional.empty(), + span); + } + private WindowDefinition resolveWindowDefinition(WindowDefinition definition) { return new WindowDefinition(definition.name(), (WindowSpecification) resolveWindow(definition.specification()), definition.span()); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index ec0adbd6d6ab..cac4ff95de79 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -16,6 +16,7 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionRewrite; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; @@ -34,6 +35,22 @@ final class HogQlV0FunctionRegistry scalar("length", "length", signature(1)), scalar("concat", "concat", variadicSignature(2)), scalar("replace", "replace", signature(3)), + scalar("nullIf", "nullif", signature(2)), + scalar("ifNull", "coalesce", signature(2)), + scalar("trim", "trim", signature(1)), + scalar("round", "round", signature(1), signature(2)), + nondeterministicScalar("now", "now", signature(0)), + nondeterministicScalar("current_timestamp", "now", signature(0)), + rewrite("isNotNull", FunctionRewrite.IS_NOT_NULL, "boolean", signature(1)), + rewrite("toInt", FunctionRewrite.CAST_BIGINT, "bigint", signature(1)), + rewrite("toFloat", FunctionRewrite.CAST_DOUBLE, "double", signature(1)), + rewrite("toString", FunctionRewrite.CAST_VARCHAR, "varchar", signature(1)), + rewrite("toDate", FunctionRewrite.CAST_DATE, "date", signature(1)), + rewrite("toDateTime", FunctionRewrite.CAST_TIMESTAMP, "timestamp(0)", signature(1)), + rewrite("toStartOfMonth", FunctionRewrite.DATE_TRUNC_MONTH, "timestamp", signature(1)), + rewrite("toStartOfDay", FunctionRewrite.DATE_TRUNC_DAY, "timestamp", signature(1)), + rewrite("toStartOfHour", FunctionRewrite.DATE_TRUNC_HOUR, "timestamp", signature(1)), + rewrite("toMonday", FunctionRewrite.DATE_TRUNC_WEEK, "timestamp", signature(1)), scalar("map", "map", signature(2)), scalar("dateDiff", "date_diff", signature(3)), scalar("dateAdd", "date_add", signature(3)), @@ -48,6 +65,8 @@ final class HogQlV0FunctionRegistry aggregate("max", "max", true, signature(1)), aggregate("avg", "avg", true, signature(1)), aggregateWithOrderBy("array_agg", "array_agg", true, signature(1)), + aggregateWithOrderBy("groupArray", "array_agg", false, signature(1)), + nondeterministicAggregate("uniq", "approx_distinct", false, signature(1)), nondeterministicAggregate("any", "arbitrary", false, signature(1)), nondeterministicAggregate("argMin", "min_by", false, signature(2)), nondeterministicAggregate("argMax", "max_by", false, signature(2)), @@ -79,6 +98,29 @@ private static FunctionCapabilityDefinition scalar(String hogQlName, String trin return function(hogQlName, trinoName, FunctionKind.SCALAR, false, false, false, false, true, signatures); } + private static FunctionCapabilityDefinition nondeterministicScalar(String hogQlName, String trinoName, FunctionSignature... signatures) + { + return function(hogQlName, trinoName, FunctionKind.SCALAR, false, false, false, false, false, signatures); + } + + private static FunctionCapabilityDefinition rewrite(String hogQlName, FunctionRewrite rewrite, String returnType, FunctionSignature... signatures) + { + return new FunctionCapabilityDefinition( + hogQlName, + FunctionKind.SCALAR, + FunctionImplementation.REWRITE, + List.of(), + java.util.Optional.of(rewrite), + java.util.Arrays.stream(signatures) + .map(signature -> new FunctionSignature(signature.argumentTypes(), returnType, signature.variadic())) + .toList(), + true, + false, + false, + false, + false); + } + private static FunctionCapabilityDefinition aggregate(String hogQlName, String trinoName, boolean supportsDistinct, FunctionSignature... signatures) { return function(hogQlName, trinoName, FunctionKind.AGGREGATE, supportsDistinct, false, true, true, true, signatures); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 30c835911ec3..6b51a6b1bf45 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -96,6 +96,7 @@ import io.trino.sql.tree.NodeLocation; import io.trino.sql.tree.NotExpression; import io.trino.sql.tree.NullLiteral; +import io.trino.sql.tree.NullIfExpression; import io.trino.sql.tree.Offset; import io.trino.sql.tree.OrderBy; import io.trino.sql.tree.Parameter; @@ -452,6 +453,9 @@ private static Expression createFunctionCall(FunctionCall function, Map callArguments = new ArrayList<>(); for (int index = 0; index < arguments.size(); index++) { callArguments.add(new CallArgument(location(function.arguments().get(index).span()), Optional.empty(), arguments.get(index))); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 0fe583509f3d..9399a3ddb4b9 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -269,8 +269,9 @@ private static Map validateFunctions(List< if (function.signatures().stream().anyMatch(signature -> signature.variadic() || signature.argumentTypes().size() != 1)) { throw new IllegalArgumentException("rewrite function must declare unary signatures"); } - if (function.signatures().stream().anyMatch(signature -> !signature.returnType().equalsIgnoreCase("boolean"))) { - throw new IllegalArgumentException("rewrite function signatures must return boolean"); + if ((function.rewrite().orElseThrow() == FunctionRewrite.IS_NULL || function.rewrite().orElseThrow() == FunctionRewrite.IS_NOT_NULL) && + function.signatures().stream().anyMatch(signature -> !signature.returnType().equalsIgnoreCase("boolean"))) { + throw new IllegalArgumentException("null predicate rewrite function signatures must return boolean"); } } else { @@ -1267,7 +1268,17 @@ public enum FunctionImplementation public enum FunctionRewrite { - IS_NULL, IS_NOT_NULL + CAST_DATE, + CAST_DOUBLE, + CAST_BIGINT, + CAST_TIMESTAMP, + CAST_VARCHAR, + DATE_TRUNC_DAY, + DATE_TRUNC_HOUR, + DATE_TRUNC_MONTH, + DATE_TRUNC_WEEK, + IS_NULL, + IS_NOT_NULL } public enum ModifierBehavior diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 6ea8d04cf265..de4cd82e27df 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -121,12 +121,20 @@ public void testCompilerResolvesV0FunctionsWithoutCatalogContext() { HogQlCompilationResult result = new HogQlCompiler().compile(envelope( "SELECT coalesce(NULL, 'fallback'), any(value), argMin(value, timestamp), " + - "arrayDistinct([1, 1]), dateDiff('day', start_time, end_time), rank() OVER () FROM metrics")); + "arrayDistinct([1, 1]), dateDiff('day', start_time, end_time), rank() OVER (), " + + "nullIf(value, ''), ifNull(value, 'fallback'), trim(value), round(score, 2), " + + "now(), current_timestamp(), isNotNull(value), groupArray(value), uniq(value), " + + "toInt(value), toFloat(value), toString(value), toDate(timestamp), toDateTime(timestamp), " + + "toStartOfMonth(timestamp), toStartOfDay(timestamp), toStartOfHour(timestamp), toMonday(timestamp) FROM metrics")); assertThat(result.catalogGeneration()).isEmpty(); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( "SELECT coalesce(NULL, 'fallback'), arbitrary(value), min_by(value, timestamp), " + - "array_distinct(ARRAY[1, 1]), date_diff('day', start_time, end_time), rank() OVER () FROM metrics")); + "array_distinct(ARRAY[1, 1]), date_diff('day', start_time, end_time), rank() OVER (), " + + "nullif(value, ''), coalesce(value, 'fallback'), \"trim\"(value), round(score, 2), " + + "now(), now(), value IS NOT NULL, array_agg(value), approx_distinct(value), " + + "CAST(value AS bigint), CAST(value AS double), CAST(value AS varchar), CAST(timestamp AS date), CAST(timestamp AS timestamp(0)), " + + "date_trunc('month', timestamp), date_trunc('day', timestamp), date_trunc('hour', timestamp), date_trunc('week', timestamp) FROM metrics")); } @Test From d23432752b7f5b36853a3a76c4c62cc2bbcfeda2 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 22:57:37 -0400 Subject: [PATCH 080/136] fix(hogql): pass through physical CTE schemas --- .../trino/hogql/compiler/HogQlCompiler.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index c10bd5a2f960..578a7b2fc1f6 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -279,13 +279,29 @@ private static ResolvedQuery resolveQuery( HogQlQuery resolved = query; if (semanticCandidate) { HogQlQuery functionsResolved = v0Profile ? HogQlFunctionResolver.resolveV0(query) : HogQlFunctionResolver.resolve(pinned, query); - resolved = HogQlSemanticResolver.resolve(pinned, functionsResolved) - .map(HogQlSemanticResolver.ResolvedQuery::query) - .orElse(functionsResolved); + if (hasSemanticDefinitions(pinned.snapshot())) { + resolved = HogQlSemanticResolver.resolve(pinned, functionsResolved) + .map(HogQlSemanticResolver.ResolvedQuery::query) + .orElse(functionsResolved); + } + else { + resolved = functionsResolved; + } } return new ResolvedQuery(resolved, Optional.of(pinned)); } + private static boolean hasSemanticDefinitions(io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot snapshot) + { + return !snapshot.logicalTables().isEmpty() || + !snapshot.virtualTables().isEmpty() || + !snapshot.savedQueries().isEmpty() || + !snapshot.materializedViews().isEmpty() || + !snapshot.expressionFields().isEmpty() || + !snapshot.actions().isEmpty() || + !snapshot.cohorts().isEmpty(); + } + private static boolean containsSemanticCandidate(HogQlQuery query) { return containsFunctionCall(query) || From f9390430b1e7a3edbe6db4ba94f0157dc07a0b77 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 23:07:56 -0400 Subject: [PATCH 081/136] feat(hogql): lower conditional functions --- core/trino-hogql-compiler/README.md | 5 +++ .../hogql/compiler/HogQlFunctionResolver.java | 31 ++++++++++++++ .../compiler/HogQlV0FunctionRegistry.java | 25 ++++++++++++ .../catalog/HogQlSemanticCatalogSnapshot.java | 40 ++++++++++++++++--- .../compiler/TestHogQlFunctionResolver.java | 16 +++++++- .../TestHogQlSemanticCatalogSnapshot.java | 20 ++++++++-- 6 files changed, 126 insertions(+), 11 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index e4278dfc6300..2751320d322e 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -59,6 +59,11 @@ The MVP lowers a frozen, fail-closed subset directly to stock Trino AST nodes: `max`, `avg`, `any`, `argMin`, `argMax`, `array_agg`), and windows (`first_value`, `rank`, `row_number`). +The compatibility extension also lowers conditional aggregates (`countIf`, +`sumIf`, `maxIf`, and `uniqIf`), distinct aggregates (`uniqExact` and +`groupUniqArray`), and `multiIf` directly to stock Trino aggregate filters, +DISTINCT aggregates, and searched CASE expressions. + Function aliases are resolved before Trino analysis. Unknown functions fail as HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, HogQLX, PIVOT/UNPIVOT execution, ClickHouse-only clauses, advanced lambda and diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 66a308f3bdc0..6f2dffb834b3 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -76,6 +76,7 @@ import io.trino.spi.Location; import io.trino.spi.TrinoException; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -376,9 +377,39 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite case DATE_TRUNC_HOUR -> dateTrunc("hour", arguments.getFirst(), span); case DATE_TRUNC_MONTH -> dateTrunc("month", arguments.getFirst(), span); case DATE_TRUNC_WEEK -> dateTrunc("week", arguments.getFirst(), span); + case COUNT_IF -> aggregate("count", List.of(), false, arguments.getFirst(), span); + case SUM_IF -> aggregate("sum", List.of(arguments.getFirst()), false, arguments.get(1), span); + case MAX_IF -> aggregate("max", List.of(arguments.getFirst()), false, arguments.get(1), span); + case UNIQ_IF -> aggregate("approx_distinct", List.of(arguments.getFirst()), false, arguments.get(1), span); + case UNIQ_EXACT -> aggregate("count", List.of(arguments.getFirst()), true, null, span); + case GROUP_UNIQ_ARRAY -> aggregate("array_agg", List.of(arguments.getFirst()), true, null, span); + case MULTI_IF -> multiIf(function, arguments); }; } + private static FunctionCall aggregate(String name, List arguments, boolean distinct, Expression filter, HogQlQuery.SourceSpan span) + { + return new FunctionCall( + new Identifier(name, false, span), + arguments, + distinct, + List.of(), + Optional.ofNullable(filter), + span); + } + + private static CaseExpression multiIf(FunctionCall function, List arguments) + { + if (arguments.size() % 2 == 0) { + throw resolutionError(function, "HogQL function " + function.name().value() + " requires condition/result pairs followed by a default value"); + } + List clauses = new ArrayList<>((arguments.size() - 1) / 2); + for (int index = 0; index < arguments.size() - 1; index += 2) { + clauses.add(new CaseWhen(arguments.get(index), arguments.get(index + 1), function.span())); + } + return new CaseExpression(Optional.empty(), clauses, Optional.of(arguments.getLast()), function.span()); + } + private static CastExpression cast(Expression value, String type, HogQlQuery.SourceSpan span) { return new CastExpression(value, new Identifier(type, false, span), false, span); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index cac4ff95de79..8e7a25a63eaf 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -51,6 +51,7 @@ final class HogQlV0FunctionRegistry rewrite("toStartOfDay", FunctionRewrite.DATE_TRUNC_DAY, "timestamp", signature(1)), rewrite("toStartOfHour", FunctionRewrite.DATE_TRUNC_HOUR, "timestamp", signature(1)), rewrite("toMonday", FunctionRewrite.DATE_TRUNC_WEEK, "timestamp", signature(1)), + rewrite("multiIf", FunctionRewrite.MULTI_IF, "any", variadicSignature(3)), scalar("map", "map", signature(2)), scalar("dateDiff", "date_diff", signature(3)), scalar("dateAdd", "date_add", signature(3)), @@ -66,6 +67,12 @@ final class HogQlV0FunctionRegistry aggregate("avg", "avg", true, signature(1)), aggregateWithOrderBy("array_agg", "array_agg", true, signature(1)), aggregateWithOrderBy("groupArray", "array_agg", false, signature(1)), + aggregateRewrite("countIf", FunctionRewrite.COUNT_IF, "bigint", signature(1)), + aggregateRewrite("sumIf", FunctionRewrite.SUM_IF, "any", signature(2)), + aggregateRewrite("maxIf", FunctionRewrite.MAX_IF, "any", signature(2)), + aggregateRewrite("uniqIf", FunctionRewrite.UNIQ_IF, "bigint", signature(2)), + aggregateRewrite("uniqExact", FunctionRewrite.UNIQ_EXACT, "bigint", signature(1)), + aggregateRewrite("groupUniqArray", FunctionRewrite.GROUP_UNIQ_ARRAY, "array(any)", signature(1)), nondeterministicAggregate("uniq", "approx_distinct", false, signature(1)), nondeterministicAggregate("any", "arbitrary", false, signature(1)), nondeterministicAggregate("argMin", "min_by", false, signature(2)), @@ -126,6 +133,24 @@ private static FunctionCapabilityDefinition aggregate(String hogQlName, String t return function(hogQlName, trinoName, FunctionKind.AGGREGATE, supportsDistinct, false, true, true, true, signatures); } + private static FunctionCapabilityDefinition aggregateRewrite(String hogQlName, FunctionRewrite rewrite, String returnType, FunctionSignature... signatures) + { + return new FunctionCapabilityDefinition( + hogQlName, + FunctionKind.AGGREGATE, + FunctionImplementation.REWRITE, + List.of(), + java.util.Optional.of(rewrite), + java.util.Arrays.stream(signatures) + .map(signature -> new FunctionSignature(signature.argumentTypes(), returnType, signature.variadic())) + .toList(), + true, + false, + false, + false, + false); + } + private static FunctionCapabilityDefinition aggregateWithOrderBy(String hogQlName, String trinoName, boolean supportsDistinct, FunctionSignature... signatures) { return function(hogQlName, trinoName, FunctionKind.AGGREGATE, supportsDistinct, true, true, true, true, signatures); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 9399a3ddb4b9..e59014331661 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -248,8 +248,9 @@ private static Map validateFunctions(List< if (function.rewrite().isEmpty()) { throw new IllegalArgumentException("rewrite function must declare a rewrite"); } - if (function.kind() != FunctionKind.SCALAR) { - throw new IllegalArgumentException("rewrite function must be scalar"); + FunctionRewrite rewrite = function.rewrite().orElseThrow(); + if (rewriteFunctionKind(rewrite) != function.kind()) { + throw new IllegalArgumentException("rewrite function kind must be " + rewriteFunctionKind(rewrite).name().toLowerCase(Locale.ENGLISH)); } if (!function.deterministic()) { throw new IllegalArgumentException("rewrite function must be deterministic"); @@ -266,10 +267,10 @@ private static Map validateFunctions(List< if (function.supportsWindow()) { throw new IllegalArgumentException("rewrite function cannot support window invocation"); } - if (function.signatures().stream().anyMatch(signature -> signature.variadic() || signature.argumentTypes().size() != 1)) { - throw new IllegalArgumentException("rewrite function must declare unary signatures"); + if (function.signatures().stream().anyMatch(signature -> !validRewriteSignature(rewrite, signature))) { + throw new IllegalArgumentException("rewrite function declares an invalid signature"); } - if ((function.rewrite().orElseThrow() == FunctionRewrite.IS_NULL || function.rewrite().orElseThrow() == FunctionRewrite.IS_NOT_NULL) && + if ((rewrite == FunctionRewrite.IS_NULL || rewrite == FunctionRewrite.IS_NOT_NULL) && function.signatures().stream().anyMatch(signature -> !signature.returnType().equalsIgnoreCase("boolean"))) { throw new IllegalArgumentException("null predicate rewrite function signatures must return boolean"); } @@ -286,6 +287,26 @@ private static Map validateFunctions(List< return Map.copyOf(functions); } + private static FunctionKind rewriteFunctionKind(FunctionRewrite rewrite) + { + return switch (rewrite) { + case COUNT_IF, GROUP_UNIQ_ARRAY, MAX_IF, SUM_IF, UNIQ_EXACT, UNIQ_IF -> FunctionKind.AGGREGATE; + default -> FunctionKind.SCALAR; + }; + } + + private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSignature signature) + { + return switch (rewrite) { + case CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_TIMESTAMP, CAST_VARCHAR, + DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, + GROUP_UNIQ_ARRAY, IS_NOT_NULL, IS_NULL, COUNT_IF, UNIQ_EXACT -> + !signature.variadic() && signature.argumentTypes().size() == 1; + case MAX_IF, SUM_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; + case MULTI_IF -> signature.variadic() && signature.argumentTypes().size() == 4; + }; + } + private static Map indexExpressionFields(List definitions, Map tables) { Map fields = new HashMap<>(); @@ -1277,8 +1298,15 @@ public enum FunctionRewrite DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, + COUNT_IF, + GROUP_UNIQ_ARRAY, IS_NULL, - IS_NOT_NULL + IS_NOT_NULL, + MAX_IF, + MULTI_IF, + SUM_IF, + UNIQ_EXACT, + UNIQ_IF } public enum ModifierBehavior diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index de4cd82e27df..44fd93ce6aaf 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -125,7 +125,9 @@ public void testCompilerResolvesV0FunctionsWithoutCatalogContext() "nullIf(value, ''), ifNull(value, 'fallback'), trim(value), round(score, 2), " + "now(), current_timestamp(), isNotNull(value), groupArray(value), uniq(value), " + "toInt(value), toFloat(value), toString(value), toDate(timestamp), toDateTime(timestamp), " + - "toStartOfMonth(timestamp), toStartOfDay(timestamp), toStartOfHour(timestamp), toMonday(timestamp) FROM metrics")); + "toStartOfMonth(timestamp), toStartOfDay(timestamp), toStartOfHour(timestamp), toMonday(timestamp), " + + "countIf(active), sumIf(value, active), maxIf(value, active), uniqIf(value, active), " + + "uniqExact(value), groupUniqArray(value), multiIf(first, 'one', second, 'two', 'other') FROM metrics")); assertThat(result.catalogGeneration()).isEmpty(); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( @@ -134,7 +136,10 @@ public void testCompilerResolvesV0FunctionsWithoutCatalogContext() "nullif(value, ''), coalesce(value, 'fallback'), \"trim\"(value), round(score, 2), " + "now(), now(), value IS NOT NULL, array_agg(value), approx_distinct(value), " + "CAST(value AS bigint), CAST(value AS double), CAST(value AS varchar), CAST(timestamp AS date), CAST(timestamp AS timestamp(0)), " + - "date_trunc('month', timestamp), date_trunc('day', timestamp), date_trunc('hour', timestamp), date_trunc('week', timestamp) FROM metrics")); + "date_trunc('month', timestamp), date_trunc('day', timestamp), date_trunc('hour', timestamp), date_trunc('week', timestamp), " + + "count(*) FILTER (WHERE active), sum(value) FILTER (WHERE active), max(value) FILTER (WHERE active), " + + "approx_distinct(value) FILTER (WHERE active), count(DISTINCT value), array_agg(DISTINCT value), " + + "CASE WHEN first THEN 'one' WHEN second THEN 'two' ELSE 'other' END FROM metrics")); } @Test @@ -162,6 +167,13 @@ public void testCompilerEnforcesV0FunctionArities() assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); assertThat(exception).hasMessageContaining("does not accept"); } + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope("SELECT multiIf(first, 1, second, 2)"))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception).hasMessageContaining("requires condition/result pairs followed by a default value"); } @ParameterizedTest diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java index e757decbccb4..c9c370a4cb19 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java @@ -168,11 +168,17 @@ public void testValidatesFunctionRewriteContract() { FunctionCapabilityDefinition isNull = rewriteFunction("isNull", FunctionKind.SCALAR, List.of(), Optional.of(FunctionRewrite.IS_NULL), List.of(new FunctionSignature(List.of("varchar"), "boolean", false))); FunctionCapabilityDefinition isNotNull = rewriteFunction("isNotNull", FunctionKind.SCALAR, List.of(), Optional.of(FunctionRewrite.IS_NOT_NULL), List.of(new FunctionSignature(List.of("varchar"), "boolean", false))); + FunctionCapabilityDefinition countIf = rewriteFunction("countIf", FunctionKind.AGGREGATE, List.of(), Optional.of(FunctionRewrite.COUNT_IF), List.of(new FunctionSignature(List.of("boolean"), "bigint", false))); + FunctionCapabilityDefinition multiIf = rewriteFunction("multiIf", FunctionKind.SCALAR, List.of(), Optional.of(FunctionRewrite.MULTI_IF), List.of(new FunctionSignature(List.of("boolean", "any", "boolean", "any"), "any", true))); - HogQlSemanticCatalogSnapshot snapshot = semanticSnapshot(List.of(), List.of(isNull, isNotNull)); + HogQlSemanticCatalogSnapshot snapshot = semanticSnapshot(List.of(), List.of(isNull, isNotNull, countIf, multiIf)); assertThat(snapshot.functions()).extracting(FunctionCapabilityDefinition::rewrite) - .containsExactly(Optional.of(FunctionRewrite.IS_NULL), Optional.of(FunctionRewrite.IS_NOT_NULL)); + .containsExactly( + Optional.of(FunctionRewrite.IS_NULL), + Optional.of(FunctionRewrite.IS_NOT_NULL), + Optional.of(FunctionRewrite.COUNT_IF), + Optional.of(FunctionRewrite.MULTI_IF)); assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( rewriteFunction("missing", FunctionKind.SCALAR, List.of(), Optional.empty(), List.of(new FunctionSignature(List.of("varchar"), "boolean", false)))))) .isInstanceOf(IllegalArgumentException.class) @@ -203,7 +209,15 @@ public void testValidatesFunctionRewriteContract() assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( rewriteFunction("binary", FunctionKind.SCALAR, List.of(), Optional.of(FunctionRewrite.IS_NULL), List.of(new FunctionSignature(List.of("varchar", "varchar"), "boolean", false)))))) .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("unary signatures"); + .hasMessageContaining("invalid signature"); + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( + rewriteFunction("scalarCountIf", FunctionKind.SCALAR, List.of(), Optional.of(FunctionRewrite.COUNT_IF), List.of(new FunctionSignature(List.of("boolean"), "bigint", false)))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("kind must be aggregate"); + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( + rewriteFunction("fixedMultiIf", FunctionKind.SCALAR, List.of(), Optional.of(FunctionRewrite.MULTI_IF), List.of(new FunctionSignature(List.of("boolean", "any", "any"), "any", false)))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("invalid signature"); assertInvalidRewrite(rewriteFunction(false, false, false, false, false, "boolean"), "must be deterministic"); assertInvalidRewrite(rewriteFunction(true, true, false, false, false, "boolean"), "cannot support DISTINCT"); assertInvalidRewrite(rewriteFunction(true, false, true, false, false, "boolean"), "cannot support ORDER BY"); From 38aefcbefd28c646a83b8659528b4b88e6affa3f Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 23:20:55 -0400 Subject: [PATCH 082/136] feat(hogql): lower json extraction functions --- core/trino-hogql-compiler/README.md | 3 + .../hogql/compiler/HogQlFunctionResolver.java | 116 ++++++++++++++++++ .../compiler/HogQlV0FunctionRegistry.java | 8 ++ .../catalog/HogQlSemanticCatalogSnapshot.java | 17 ++- .../compiler/TestHogQlFunctionResolver.java | 29 +++++ 5 files changed, 171 insertions(+), 2 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 2751320d322e..3ad349aa6c4c 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -63,6 +63,9 @@ The compatibility extension also lowers conditional aggregates (`countIf`, `sumIf`, `maxIf`, and `uniqIf`), distinct aggregates (`uniqExact` and `groupUniqArray`), and `multiIf` directly to stock Trino aggregate filters, DISTINCT aggregates, and searched CASE expressions. +Literal-path JSON extraction lowers to Trino JSON paths with HogQL-compatible +defaults. The compatibility profile also supports the corpus's typed map and +key/value extraction forms through checked JSON-to-map casts. Function aliases are resolved before Trino analysis. Unknown functions fail as HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 6f2dffb834b3..511ce47218ad 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -384,6 +384,20 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite case UNIQ_EXACT -> aggregate("count", List.of(arguments.getFirst()), true, null, span); case GROUP_UNIQ_ARRAY -> aggregate("array_agg", List.of(arguments.getFirst()), true, null, span); case MULTI_IF -> multiIf(function, arguments); + case JSON_EXTRACT_STRING -> jsonExtractScalar(function, arguments, "varchar", new Literal(HogQlQuery.LiteralKind.STRING, "", span)); + case JSON_EXTRACT_INT -> jsonExtractScalar(function, arguments, "bigint", new Literal(HogQlQuery.LiteralKind.INTEGER, "0", span)); + case JSON_EXTRACT_FLOAT -> jsonExtractScalar(function, arguments, "double", new Literal(HogQlQuery.LiteralKind.FLOAT, "0.0", span)); + case JSON_EXTRACT_RAW -> coalesce( + call("json_format", List.of(call("json_extract", List.of(arguments.getFirst(), jsonPath(function, arguments.subList(1, arguments.size()))), span)), span), + new Literal(HogQlQuery.LiteralKind.STRING, "", span), + span); + case JSON_EXTRACT_TYPED -> jsonExtractTyped(function, arguments); + case JSON_KEYS_AND_VALUES -> jsonExtractKeysAndValues(function, arguments); + case JSON_KEYS_AND_VALUES_RAW -> jsonExtractKeysAndValuesRaw(function, arguments.getFirst()); + case JSON_LENGTH -> coalesce( + call("json_size", List.of(arguments.getFirst(), jsonPath(function, arguments.subList(1, arguments.size()))), span), + new Literal(HogQlQuery.LiteralKind.INTEGER, "0", span), + span); }; } @@ -410,6 +424,108 @@ private static CaseExpression multiIf(FunctionCall function, List ar return new CaseExpression(Optional.empty(), clauses, Optional.of(arguments.getLast()), function.span()); } + private static Expression jsonExtractScalar(FunctionCall function, List arguments, String type, Literal defaultValue) + { + HogQlQuery.SourceSpan span = function.span(); + Expression extracted = call( + "json_extract_scalar", + List.of(arguments.getFirst(), jsonPath(function, arguments.subList(1, arguments.size()))), + span); + if (!type.equals("varchar")) { + extracted = new CastExpression(extracted, new Identifier(type, false, span), true, span); + } + return coalesce(extracted, defaultValue, span); + } + + private static Expression jsonExtractTyped(FunctionCall function, List arguments) + { + String type = jsonType(function, arguments.get(1)); + return switch (type) { + case "Map(String, Float64)" -> typedJsonMap(arguments.getFirst(), "double", function.span()); + default -> throw unsupportedError(function, "Unsupported HogQL JSON extraction type: " + type); + }; + } + + private static Expression jsonExtractKeysAndValues(FunctionCall function, List arguments) + { + String type = jsonType(function, arguments.get(1)); + if (!type.equals("Float64")) { + throw unsupportedError(function, "Unsupported HogQL JSON key/value type: " + type); + } + return call("map_entries", List.of(typedJsonMap(arguments.getFirst(), "double", function.span())), function.span()); + } + + private static Expression jsonExtractKeysAndValuesRaw(FunctionCall function, Expression value) + { + HogQlQuery.SourceSpan span = function.span(); + Identifier key = new Identifier("key", false, span); + Identifier item = new Identifier("value", false, span); + LambdaExpression formatValue = new LambdaExpression( + List.of(key, item), + call("json_format", List.of(new ColumnReference(List.of(item), span)), span), + span); + Expression formatted = call( + "transform_values", + List.of(typedJsonMap(value, "json", span), formatValue), + span); + return call("map_entries", List.of(formatted), span); + } + + private static Expression typedJsonMap(Expression value, String valueType, HogQlQuery.SourceSpan span) + { + String type = "map(varchar," + valueType + ")"; + Expression parsed = call("json_parse", List.of(value), span); + Expression converted = new CastExpression(parsed, new Identifier(type, false, span), true, span); + Expression empty = new CastExpression( + call("map", List.of(new ArrayExpression(List.of(), span), new ArrayExpression(List.of(), span)), span), + new Identifier(type, false, span), + false, + span); + return coalesce(converted, empty, span); + } + + private static String jsonType(FunctionCall function, Expression expression) + { + if (expression instanceof Literal literal && literal.kind() == HogQlQuery.LiteralKind.STRING) { + return literal.value(); + } + throw unsupportedError(function, "HogQL JSON extraction type must be a string literal"); + } + + private static Literal jsonPath(FunctionCall function, List segments) + { + StringBuilder path = new StringBuilder("$"); + for (Expression segment : segments) { + if (!(segment instanceof Literal literal)) { + throw unsupportedError(function, "HogQL JSON path segments must be string or integer literals"); + } + switch (literal.kind()) { + case STRING -> path.append("[\"") + .append(literal.value().replace("\\", "\\\\").replace("\"", "\\\"")) + .append("\"]"); + case INTEGER -> path.append('[').append(literal.value()).append(']'); + default -> throw unsupportedError(function, "HogQL JSON path segments must be string or integer literals"); + } + } + return new Literal(HogQlQuery.LiteralKind.STRING, path.toString(), function.span()); + } + + private static FunctionCall coalesce(Expression value, Expression defaultValue, HogQlQuery.SourceSpan span) + { + return call("coalesce", List.of(value, defaultValue), span); + } + + private static FunctionCall call(String name, List arguments, HogQlQuery.SourceSpan span) + { + return new FunctionCall( + new Identifier(name, false, span), + arguments, + false, + List.of(), + Optional.empty(), + span); + } + private static CastExpression cast(Expression value, String type, HogQlQuery.SourceSpan span) { return new CastExpression(value, new Identifier(type, false, span), false, span); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 8e7a25a63eaf..4ad8e34cfb6b 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -52,6 +52,14 @@ final class HogQlV0FunctionRegistry rewrite("toStartOfHour", FunctionRewrite.DATE_TRUNC_HOUR, "timestamp", signature(1)), rewrite("toMonday", FunctionRewrite.DATE_TRUNC_WEEK, "timestamp", signature(1)), rewrite("multiIf", FunctionRewrite.MULTI_IF, "any", variadicSignature(3)), + rewrite("JSONExtractString", FunctionRewrite.JSON_EXTRACT_STRING, "varchar", variadicSignature(2)), + rewrite("JSONExtractInt", FunctionRewrite.JSON_EXTRACT_INT, "bigint", variadicSignature(2)), + rewrite("JSONExtractFloat", FunctionRewrite.JSON_EXTRACT_FLOAT, "double", variadicSignature(2)), + rewrite("JSONExtractRaw", FunctionRewrite.JSON_EXTRACT_RAW, "varchar", variadicSignature(2)), + rewrite("JSONLength", FunctionRewrite.JSON_LENGTH, "bigint", signature(1), variadicSignature(2)), + rewrite("JSONExtract", FunctionRewrite.JSON_EXTRACT_TYPED, "any", signature(2)), + rewrite("JSONExtractKeysAndValues", FunctionRewrite.JSON_KEYS_AND_VALUES, "array(row(varchar,any))", signature(2)), + rewrite("JSONExtractKeysAndValuesRaw", FunctionRewrite.JSON_KEYS_AND_VALUES_RAW, "array(row(varchar,varchar))", signature(1)), scalar("map", "map", signature(2)), scalar("dateDiff", "date_diff", signature(3)), scalar("dateAdd", "date_add", signature(3)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index e59014331661..b6856a9fb553 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -300,9 +300,14 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi return switch (rewrite) { case CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_TIMESTAMP, CAST_VARCHAR, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, - GROUP_UNIQ_ARRAY, IS_NOT_NULL, IS_NULL, COUNT_IF, UNIQ_EXACT -> + GROUP_UNIQ_ARRAY, IS_NOT_NULL, IS_NULL, COUNT_IF, JSON_KEYS_AND_VALUES_RAW, UNIQ_EXACT -> !signature.variadic() && signature.argumentTypes().size() == 1; - case MAX_IF, SUM_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; + case JSON_EXTRACT_TYPED, JSON_KEYS_AND_VALUES, MAX_IF, SUM_IF, UNIQ_IF -> + !signature.variadic() && signature.argumentTypes().size() == 2; + case JSON_EXTRACT_FLOAT, JSON_EXTRACT_INT, JSON_EXTRACT_RAW, JSON_EXTRACT_STRING -> + signature.variadic() && signature.argumentTypes().size() == 3; + case JSON_LENGTH -> (!signature.variadic() && signature.argumentTypes().size() == 1) || + (signature.variadic() && signature.argumentTypes().size() == 3); case MULTI_IF -> signature.variadic() && signature.argumentTypes().size() == 4; }; } @@ -1302,6 +1307,14 @@ public enum FunctionRewrite GROUP_UNIQ_ARRAY, IS_NULL, IS_NOT_NULL, + JSON_EXTRACT_FLOAT, + JSON_EXTRACT_INT, + JSON_EXTRACT_RAW, + JSON_EXTRACT_STRING, + JSON_EXTRACT_TYPED, + JSON_KEYS_AND_VALUES, + JSON_KEYS_AND_VALUES_RAW, + JSON_LENGTH, MAX_IF, MULTI_IF, SUM_IF, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 44fd93ce6aaf..7bbabb3b4f75 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -153,6 +153,35 @@ public void testCompilerRejectsUnknownFunctionsWithoutCatalogContext() assertThat(exception).hasMessage("line 1:8: Unknown HogQL function: ordinary"); } + @Test + public void testCompilerLowersJsonExtractionFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT JSONExtractString(payload, 'name'), JSONExtractInt(payload, 'items', 0), " + + "JSONExtractFloat(payload, 'score'), JSONExtractRaw(payload, 'object'), " + + "JSONLength(payload), JSONLength(payload, 'items'), " + + "JSONExtract(payload_text, 'Map(String, Float64)'), " + + "JSONExtractKeysAndValues(payload_text, 'Float64'), JSONExtractKeysAndValuesRaw(payload_text) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT coalesce(json_extract_scalar(payload, '$[\"name\"]'), ''), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"items\"][0]') AS bigint), 0), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"score\"]') AS double), 0E0), " + + "coalesce(json_format(json_extract(payload, '$[\"object\"]')), ''), " + + "coalesce(json_size(payload, '$'), 0), coalesce(json_size(payload, '$[\"items\"]'), 0), " + + "coalesce(TRY_CAST(json_parse(payload_text) AS map(varchar, double)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, double))), " + + "map_entries(coalesce(TRY_CAST(json_parse(payload_text) AS map(varchar, double)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, double)))), " + + "map_entries(transform_values(coalesce(TRY_CAST(json_parse(payload_text) AS map(varchar, json)), " + + "CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json))), (key, value) -> json_format(value))) FROM records")); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope("SELECT JSONExtractString(payload, dynamic_key)"))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception).hasMessageContaining("JSON path segments must be string or integer literals"); + } + @Test public void testCompilerEnforcesV0FunctionArities() { From 19ac03cdd50e9f0526005dff35de60a7ddb9ab46 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 23:25:27 -0400 Subject: [PATCH 083/136] feat(hogql): lower date time compatibility --- core/trino-hogql-compiler/README.md | 3 +++ .../hogql/compiler/HogQlFunctionResolver.java | 21 ++++++++++++++++++- .../compiler/HogQlV0FunctionRegistry.java | 13 ++++++++++-- .../catalog/HogQlSemanticCatalogSnapshot.java | 19 ++++++++++++++--- .../compiler/TestHogQlFunctionResolver.java | 17 +++++++++++++++ 5 files changed, 67 insertions(+), 6 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 3ad349aa6c4c..aecda2cffe7b 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -66,6 +66,9 @@ DISTINCT aggregates, and searched CASE expressions. Literal-path JSON extraction lowers to Trino JSON paths with HogQL-compatible defaults. The compatibility profile also supports the corpus's typed map and key/value extraction forms through checked JSON-to-map casts. +Date/time compatibility includes calendar arithmetic, interval-day values, +epoch conversion, timezone attachment/conversion, formatting, and checked +best-effort timestamp parsing. Function aliases are resolved before Trino analysis. Unknown functions fail as HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 511ce47218ad..58e8b2b4a1f8 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -371,7 +371,9 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite case CAST_DATE -> cast(arguments.getFirst(), "date", span); case CAST_DOUBLE -> cast(arguments.getFirst(), "double", span); case CAST_BIGINT -> cast(arguments.getFirst(), "bigint", span); - case CAST_TIMESTAMP -> cast(arguments.getFirst(), "timestamp(0)", span); + case CAST_TIMESTAMP -> arguments.size() == 1 + ? cast(arguments.getFirst(), "timestamp(0)", span) + : call("with_timezone", List.of(cast(arguments.getFirst(), "timestamp(0)", span), arguments.get(1)), span); case CAST_VARCHAR -> cast(arguments.getFirst(), "varchar", span); case DATE_TRUNC_DAY -> dateTrunc("day", arguments.getFirst(), span); case DATE_TRUNC_HOUR -> dateTrunc("hour", arguments.getFirst(), span); @@ -398,9 +400,26 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite call("json_size", List.of(arguments.getFirst(), jsonPath(function, arguments.subList(1, arguments.size()))), span), new Literal(HogQlQuery.LiteralKind.INTEGER, "0", span), span); + case TODAY -> cast(call("now", List.of(), span), "date", span); + case INTERVAL_DAY -> new IntervalExpression(arguments.getFirst(), HogQlQuery.IntervalUnit.DAY, span); + case ADD_DAYS -> dateAdd("day", arguments, span); + case ADD_MONTHS -> dateAdd("month", arguments, span); + case DATE_ADD -> arguments.size() == 2 + ? new BinaryExpression(HogQlQuery.BinaryOperator.ADD, arguments.getFirst(), arguments.get(1), span) + : call("date_add", arguments, span); + case TO_UNIX_TIMESTAMP -> cast(call("to_unixtime", arguments, span), "bigint", span); + case PARSE_TIMESTAMP -> new CastExpression(arguments.getFirst(), new Identifier("timestamp(3)", false, span), true, span); }; } + private static FunctionCall dateAdd(String unit, List arguments, HogQlQuery.SourceSpan span) + { + return call( + "date_add", + List.of(new Literal(HogQlQuery.LiteralKind.STRING, unit, span), arguments.get(1), arguments.getFirst()), + span); + } + private static FunctionCall aggregate(String name, List arguments, boolean distinct, Expression filter, HogQlQuery.SourceSpan span) { return new FunctionCall( diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 4ad8e34cfb6b..11f620e51ef7 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -46,7 +46,7 @@ final class HogQlV0FunctionRegistry rewrite("toFloat", FunctionRewrite.CAST_DOUBLE, "double", signature(1)), rewrite("toString", FunctionRewrite.CAST_VARCHAR, "varchar", signature(1)), rewrite("toDate", FunctionRewrite.CAST_DATE, "date", signature(1)), - rewrite("toDateTime", FunctionRewrite.CAST_TIMESTAMP, "timestamp(0)", signature(1)), + rewrite("toDateTime", FunctionRewrite.CAST_TIMESTAMP, "timestamp(0)", signature(1), signature(2)), rewrite("toStartOfMonth", FunctionRewrite.DATE_TRUNC_MONTH, "timestamp", signature(1)), rewrite("toStartOfDay", FunctionRewrite.DATE_TRUNC_DAY, "timestamp", signature(1)), rewrite("toStartOfHour", FunctionRewrite.DATE_TRUNC_HOUR, "timestamp", signature(1)), @@ -60,9 +60,18 @@ final class HogQlV0FunctionRegistry rewrite("JSONExtract", FunctionRewrite.JSON_EXTRACT_TYPED, "any", signature(2)), rewrite("JSONExtractKeysAndValues", FunctionRewrite.JSON_KEYS_AND_VALUES, "array(row(varchar,any))", signature(2)), rewrite("JSONExtractKeysAndValuesRaw", FunctionRewrite.JSON_KEYS_AND_VALUES_RAW, "array(row(varchar,varchar))", signature(1)), + rewrite("today", FunctionRewrite.TODAY, "date", signature(0)), + rewrite("toIntervalDay", FunctionRewrite.INTERVAL_DAY, "interval day to second", signature(1)), + rewrite("addDays", FunctionRewrite.ADD_DAYS, "any", signature(2)), + rewrite("addMonths", FunctionRewrite.ADD_MONTHS, "any", signature(2)), + rewrite("dateAdd", FunctionRewrite.DATE_ADD, "any", signature(2), signature(3)), + rewrite("toUnixTimestamp", FunctionRewrite.TO_UNIX_TIMESTAMP, "bigint", signature(1)), + rewrite("parseDateTimeBestEffort", FunctionRewrite.PARSE_TIMESTAMP, "timestamp(3)", signature(1)), + scalar("fromUnixTimestamp", "from_unixtime", signature(1)), + scalar("formatDateTime", "date_format", signature(2)), + scalar("toTimeZone", "at_timezone", signature(2)), scalar("map", "map", signature(2)), scalar("dateDiff", "date_diff", signature(3)), - scalar("dateAdd", "date_add", signature(3)), scalar("dateTrunc", "date_trunc", signature(2)), scalar("arraySort", "array_sort", signature(1)), scalar("arrayDistinct", "array_distinct", signature(1)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index b6856a9fb553..bc0e5aecab89 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -298,17 +298,23 @@ private static FunctionKind rewriteFunctionKind(FunctionRewrite rewrite) private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSignature signature) { return switch (rewrite) { - case CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_TIMESTAMP, CAST_VARCHAR, + case CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_VARCHAR, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, - GROUP_UNIQ_ARRAY, IS_NOT_NULL, IS_NULL, COUNT_IF, JSON_KEYS_AND_VALUES_RAW, UNIQ_EXACT -> + GROUP_UNIQ_ARRAY, INTERVAL_DAY, IS_NOT_NULL, IS_NULL, COUNT_IF, + JSON_KEYS_AND_VALUES_RAW, PARSE_TIMESTAMP, TO_UNIX_TIMESTAMP, UNIQ_EXACT -> !signature.variadic() && signature.argumentTypes().size() == 1; - case JSON_EXTRACT_TYPED, JSON_KEYS_AND_VALUES, MAX_IF, SUM_IF, UNIQ_IF -> + case ADD_DAYS, ADD_MONTHS, JSON_EXTRACT_TYPED, JSON_KEYS_AND_VALUES, MAX_IF, SUM_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; + case CAST_TIMESTAMP -> !signature.variadic() && + (signature.argumentTypes().size() == 1 || signature.argumentTypes().size() == 2); + case DATE_ADD -> !signature.variadic() && + (signature.argumentTypes().size() == 2 || signature.argumentTypes().size() == 3); case JSON_EXTRACT_FLOAT, JSON_EXTRACT_INT, JSON_EXTRACT_RAW, JSON_EXTRACT_STRING -> signature.variadic() && signature.argumentTypes().size() == 3; case JSON_LENGTH -> (!signature.variadic() && signature.argumentTypes().size() == 1) || (signature.variadic() && signature.argumentTypes().size() == 3); case MULTI_IF -> signature.variadic() && signature.argumentTypes().size() == 4; + case TODAY -> !signature.variadic() && signature.argumentTypes().isEmpty(); }; } @@ -1299,12 +1305,16 @@ public enum FunctionRewrite CAST_BIGINT, CAST_TIMESTAMP, CAST_VARCHAR, + ADD_DAYS, + ADD_MONTHS, + DATE_ADD, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, COUNT_IF, GROUP_UNIQ_ARRAY, + INTERVAL_DAY, IS_NULL, IS_NOT_NULL, JSON_EXTRACT_FLOAT, @@ -1317,7 +1327,10 @@ public enum FunctionRewrite JSON_LENGTH, MAX_IF, MULTI_IF, + PARSE_TIMESTAMP, SUM_IF, + TODAY, + TO_UNIX_TIMESTAMP, UNIQ_EXACT, UNIQ_IF } diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 7bbabb3b4f75..c5b41a0ba135 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -182,6 +182,23 @@ public void testCompilerLowersJsonExtractionFunctions() assertThat(exception).hasMessageContaining("JSON path segments must be string or integer literals"); } + @Test + public void testCompilerLowersDateTimeCompatibilityFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT today(), toIntervalDay(2), addDays(timestamp, 3), addMonths(timestamp, -2), " + + "dateAdd(timestamp, INTERVAL 4 DAY), fromUnixTimestamp(epoch), toUnixTimestamp(timestamp), " + + "toDateTime(timestamp, 'UTC'), formatDateTime(timestamp, '%Y-%m-%d'), " + + "toTimeZone(timestamp, 'UTC'), parseDateTimeBestEffort(text) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT CAST(now() AS date), 2 * INTERVAL '1' DAY, date_add('day', 3, timestamp), " + + "date_add('month', -2, timestamp), timestamp + 4 * INTERVAL '1' DAY, " + + "from_unixtime(epoch), CAST(to_unixtime(timestamp) AS bigint), " + + "with_timezone(CAST(timestamp AS timestamp(0)), 'UTC'), date_format(timestamp, '%Y-%m-%d'), " + + "at_timezone(timestamp, 'UTC'), TRY_CAST(text AS timestamp(3)) FROM records")); + } + @Test public void testCompilerEnforcesV0FunctionArities() { From e52d7c0db8c09c3a6dae0f028571954e5ae8f018 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 23:29:40 -0400 Subject: [PATCH 084/136] feat(hogql): lower scalar syntax sugar --- core/trino-hogql-compiler/README.md | 2 ++ .../io/trino/hogql/compiler/TrinoAstFactory.java | 10 +++++++++- .../io/trino/hogql/compiler/TestHogQlCompiler.java | 6 ++++++ .../java/io/trino/hogql/parser/HogQlParser.java | 13 +++++++++++++ .../java/io/trino/hogql/parser/tree/HogQlQuery.java | 3 --- 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index aecda2cffe7b..31994f56e644 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -69,6 +69,8 @@ key/value extraction forms through checked JSON-to-map casts. Date/time compatibility includes calendar arithmetic, interval-day values, epoch conversion, timezone attachment/conversion, formatting, and checked best-effort timestamp parsing. +Array-form membership and the nullish operator lower to stock `IN`, `IF`, and +`COALESCE` nodes, including nullable empty-membership behavior. Function aliases are resolved before Trino analysis. Unknown functions fail as HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 6b51a6b1bf45..150b12368e13 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -575,9 +575,17 @@ private static Expression createBetweenExpression(BetweenExpression between, Map private static Expression createInExpression(InExpression in, Map parameterIds) { NodeLocation location = location(in.predicateSpan()); + Expression testedValue = createExpression(in.value(), parameterIds); + if (in.values().isEmpty()) { + return new IfExpression( + location, + new Predicated(location, testedValue, new IsNullPredicate(location, false)), + new NullLiteral(location), + new BooleanLiteral(location, Boolean.toString(in.negated()))); + } return new Predicated( location, - createExpression(in.value(), parameterIds), + testedValue, new InPredicate( location, in.negated(), diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index 733b416e2a14..7be9851a9035 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -113,6 +113,12 @@ public void testLowersDistinctOrderingAndClickHousePaginationOrder() SELECT value IN (1, 2, 3) | SELECT value IN (1, 2, 3) SELECT value IN (1) | SELECT value IN (1) SELECT value NOT IN (1, 2, 3) | SELECT value NOT IN (1, 2, 3) + SELECT value IN [1, 2, 3] | SELECT value IN (1, 2, 3) + SELECT value NOT IN [1] | SELECT value NOT IN (1) + SELECT value IN [] | SELECT IF(value IS NULL, NULL, false) + SELECT value NOT IN [] | SELECT IF(value IS NULL, NULL, true) + SELECT value ?? fallback | SELECT coalesce(value, fallback) + SELECT first ?? second ?? third | SELECT coalesce(coalesce(first, second), third) SELECT value LIKE 'pro%' | SELECT value LIKE 'pro%' SELECT value NOT LIKE 'pro%' | SELECT value NOT LIKE 'pro%' SELECT value ILIKE 'Pro%' | SELECT lower(value) LIKE lower('Pro%') diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index d631f8e16cc1..6e2fed4f46cb 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -1269,6 +1269,16 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) sourceSpan(isNull.IS().getSymbol(), isNull.getStop()), sourceSpan(isNull)); } + if (context instanceof HogQLParser.ColumnExprNullishContext nullish) { + Token operator = nullish.NULLISH().getSymbol(); + return new FunctionCall( + new Identifier("ifNull", false, sourceSpan(operator, operator)), + List.of(buildExpression(nullish.columnExprValue(0)), buildExpression(nullish.columnExprValue(1))), + false, + List.of(), + Optional.empty(), + sourceSpan(nullish)); + } if (context instanceof HogQLParser.ColumnExprBetweenContext between) { return new BetweenExpression( buildExpression(between.columnExprValue(0)), @@ -1432,6 +1442,9 @@ private List buildExpressions(HogQLParser.ColumnExprListContext cont private List buildInValues(HogQLParser.ColumnExprValueContext context) { + if (context instanceof HogQLParser.ColumnExprArrayContext array) { + return array.columnExprList() == null ? List.of() : buildExpressions(array.columnExprList()); + } if (context instanceof HogQLParser.ColumnExprTupleContext tuple) { return buildExpressions(tuple.columnExprList()); } diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index d9963fb2e759..7f45a87fa7ca 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -466,9 +466,6 @@ public record InExpression(Expression value, List values, boolean ne { value = requireNonNull(value, "value is null"); values = List.copyOf(requireNonNull(values, "values is null")); - if (values.isEmpty()) { - throw new IllegalArgumentException("values is empty"); - } predicateSpan = requireNonNull(predicateSpan, "predicateSpan is null"); span = requireNonNull(span, "span is null"); } From c94d373471926dfa157f2085445b6b6575f02156 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 23:32:03 -0400 Subject: [PATCH 085/136] feat(hogql): lower scalar compatibility functions --- core/trino-hogql-compiler/README.md | 2 ++ .../hogql/compiler/HogQlFunctionResolver.java | 4 ++++ .../hogql/compiler/HogQlV0FunctionRegistry.java | 10 ++++++++++ .../catalog/HogQlSemanticCatalogSnapshot.java | 9 +++++++-- .../hogql/compiler/TestHogQlFunctionResolver.java | 14 ++++++++++++++ 5 files changed, 37 insertions(+), 2 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 31994f56e644..4ac6d39c8811 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -71,6 +71,8 @@ epoch conversion, timezone attachment/conversion, formatting, and checked best-effort timestamp parsing. Array-form membership and the nullish operator lower to stock `IN`, `IF`, and `COALESCE` nodes, including nullable empty-membership behavior. +Function-form boolean/comparison operators and common string/math aliases +lower to their equivalent stock operators and functions. Function aliases are resolved before Trino analysis. Unknown functions fail as HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 58e8b2b4a1f8..7db84806b5b1 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -409,6 +409,10 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite : call("date_add", arguments, span); case TO_UNIX_TIMESTAMP -> cast(call("to_unixtime", arguments, span), "bigint", span); case PARSE_TIMESTAMP -> new CastExpression(arguments.getFirst(), new Identifier("timestamp(3)", false, span), true, span); + case NOT -> new UnaryExpression(HogQlQuery.UnaryOperator.NOT, arguments.getFirst(), span); + case AND -> new BinaryExpression(HogQlQuery.BinaryOperator.AND, arguments.getFirst(), arguments.get(1), span); + case GREATER -> new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN, arguments.getFirst(), arguments.get(1), span); + case LIKE -> new BinaryExpression(HogQlQuery.BinaryOperator.LIKE, arguments.getFirst(), arguments.get(1), span); }; } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 11f620e51ef7..e8c5ecf65908 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -67,9 +67,19 @@ final class HogQlV0FunctionRegistry rewrite("dateAdd", FunctionRewrite.DATE_ADD, "any", signature(2), signature(3)), rewrite("toUnixTimestamp", FunctionRewrite.TO_UNIX_TIMESTAMP, "bigint", signature(1)), rewrite("parseDateTimeBestEffort", FunctionRewrite.PARSE_TIMESTAMP, "timestamp(3)", signature(1)), + rewrite("not", FunctionRewrite.NOT, "boolean", signature(1)), + rewrite("and", FunctionRewrite.AND, "boolean", signature(2)), + rewrite("greater", FunctionRewrite.GREATER, "boolean", signature(2)), + rewrite("like", FunctionRewrite.LIKE, "boolean", signature(2)), scalar("fromUnixTimestamp", "from_unixtime", signature(1)), scalar("formatDateTime", "date_format", signature(2)), scalar("toTimeZone", "at_timezone", signature(2)), + scalar("least", "least", variadicSignature(1)), + scalar("greatest", "greatest", variadicSignature(1)), + scalar("position", "strpos", signature(2)), + scalar("startsWith", "starts_with", signature(2)), + scalar("substring", "substring", signature(2), signature(3)), + scalar("log10", "log10", signature(1)), scalar("map", "map", signature(2)), scalar("dateDiff", "date_diff", signature(3)), scalar("dateTrunc", "date_trunc", signature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index bc0e5aecab89..eee9b3962c34 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -301,9 +301,10 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi case CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_VARCHAR, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, GROUP_UNIQ_ARRAY, INTERVAL_DAY, IS_NOT_NULL, IS_NULL, COUNT_IF, - JSON_KEYS_AND_VALUES_RAW, PARSE_TIMESTAMP, TO_UNIX_TIMESTAMP, UNIQ_EXACT -> + JSON_KEYS_AND_VALUES_RAW, NOT, PARSE_TIMESTAMP, TO_UNIX_TIMESTAMP, UNIQ_EXACT -> !signature.variadic() && signature.argumentTypes().size() == 1; - case ADD_DAYS, ADD_MONTHS, JSON_EXTRACT_TYPED, JSON_KEYS_AND_VALUES, MAX_IF, SUM_IF, UNIQ_IF -> + case ADD_DAYS, ADD_MONTHS, AND, GREATER, JSON_EXTRACT_TYPED, JSON_KEYS_AND_VALUES, + LIKE, MAX_IF, SUM_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; case CAST_TIMESTAMP -> !signature.variadic() && (signature.argumentTypes().size() == 1 || signature.argumentTypes().size() == 2); @@ -1307,6 +1308,7 @@ public enum FunctionRewrite CAST_VARCHAR, ADD_DAYS, ADD_MONTHS, + AND, DATE_ADD, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, @@ -1314,6 +1316,7 @@ public enum FunctionRewrite DATE_TRUNC_WEEK, COUNT_IF, GROUP_UNIQ_ARRAY, + GREATER, INTERVAL_DAY, IS_NULL, IS_NOT_NULL, @@ -1325,8 +1328,10 @@ public enum FunctionRewrite JSON_KEYS_AND_VALUES, JSON_KEYS_AND_VALUES_RAW, JSON_LENGTH, + LIKE, MAX_IF, MULTI_IF, + NOT, PARSE_TIMESTAMP, SUM_IF, TODAY, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index c5b41a0ba135..6e6309e2f0dd 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -199,6 +199,20 @@ public void testCompilerLowersDateTimeCompatibilityFunctions() "at_timezone(timestamp, 'UTC'), TRY_CAST(text AS timestamp(3)) FROM records")); } + @Test + public void testCompilerLowersScalarCompatibilityFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT not(active), and(active, ready), greater(score, 10), like(name, 'pro%'), " + + "least(first, second), greatest(first, second), position(name, 'needle'), " + + "startsWith(name, 'prefix'), substring(name, 2, 4), log10(score) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT NOT active, active AND ready, score > 10, name LIKE 'pro%', " + + "least(first, second), greatest(first, second), strpos(name, 'needle'), " + + "starts_with(name, 'prefix'), substring(name, 2, 4), log10(score) FROM records")); + } + @Test public void testCompilerEnforcesV0FunctionArities() { From 5f6a9f4f7b6a19948223a464c72592eb238153aa Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 23:35:30 -0400 Subject: [PATCH 086/136] feat(hogql): lower regex compatibility functions --- core/trino-hogql-compiler/README.md | 2 + .../hogql/compiler/HogQlFunctionResolver.java | 83 +++++++++++++++++++ .../compiler/HogQlV0FunctionRegistry.java | 3 + .../catalog/HogQlSemanticCatalogSnapshot.java | 5 +- .../compiler/TestHogQlFunctionResolver.java | 13 +++ 5 files changed, 105 insertions(+), 1 deletion(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 4ac6d39c8811..baf17c787173 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -73,6 +73,8 @@ Array-form membership and the nullish operator lower to stock `IN`, `IF`, and `COALESCE` nodes, including nullable empty-membership behavior. Function-form boolean/comparison operators and common string/math aliases lower to their equivalent stock operators and functions. +Literal regex extraction preserves HogQL's capture-group and empty-default +behavior; matching and replace-all lower to Trino regex functions. Function aliases are resolved before Trino analysis. Unknown functions fail as HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 7db84806b5b1..56729bef9a3c 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -413,9 +413,92 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite case AND -> new BinaryExpression(HogQlQuery.BinaryOperator.AND, arguments.getFirst(), arguments.get(1), span); case GREATER -> new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN, arguments.getFirst(), arguments.get(1), span); case LIKE -> new BinaryExpression(HogQlQuery.BinaryOperator.LIKE, arguments.getFirst(), arguments.get(1), span); + case REGEX_EXTRACT -> regexExtract(function, arguments); + case REGEX_REPLACE_ALL -> regexReplaceAll(function, arguments); }; } + private static Expression regexExtract(FunctionCall function, List arguments) + { + Literal pattern = stringLiteral(function, arguments.get(1), "regular expression"); + Literal group = new Literal( + HogQlQuery.LiteralKind.INTEGER, + hasCapturingGroup(pattern.value()) ? "1" : "0", + function.span()); + Expression extracted = call("regexp_extract", List.of(arguments.getFirst(), pattern, group), function.span()); + return coalesce(extracted, new Literal(HogQlQuery.LiteralKind.STRING, "", function.span()), function.span()); + } + + private static Expression regexReplaceAll(FunctionCall function, List arguments) + { + Literal pattern = stringLiteral(function, arguments.get(1), "regular expression"); + Literal replacement = stringLiteral(function, arguments.get(2), "regular expression replacement"); + String trinoReplacement = regexReplacement(replacement.value()); + return call( + "regexp_replace", + List.of(arguments.getFirst(), pattern, new Literal(HogQlQuery.LiteralKind.STRING, trinoReplacement, replacement.span())), + function.span()); + } + + private static String regexReplacement(String replacement) + { + StringBuilder result = new StringBuilder(replacement.length()); + for (int index = 0; index < replacement.length(); index++) { + char current = replacement.charAt(index); + if (current == '\\' && index + 1 < replacement.length() && Character.isDigit(replacement.charAt(index + 1))) { + result.append('$').append(replacement.charAt(++index)); + } + else { + result.append(current); + } + } + return result.toString(); + } + + private static Literal stringLiteral(FunctionCall function, Expression expression, String description) + { + if (expression instanceof Literal literal && literal.kind() == HogQlQuery.LiteralKind.STRING) { + return literal; + } + throw unsupportedError(function, "HogQL " + description + " must be a string literal"); + } + + private static boolean hasCapturingGroup(String pattern) + { + boolean escaped = false; + boolean characterClass = false; + for (int index = 0; index < pattern.length(); index++) { + char current = pattern.charAt(index); + if (escaped) { + escaped = false; + continue; + } + if (current == '\\') { + escaped = true; + continue; + } + if (current == '[') { + characterClass = true; + continue; + } + if (current == ']' && characterClass) { + characterClass = false; + continue; + } + if (current != '(' || characterClass) { + continue; + } + if (index + 1 >= pattern.length() || pattern.charAt(index + 1) != '?') { + return true; + } + if (index + 2 < pattern.length() && pattern.charAt(index + 2) == '<' && + (index + 3 >= pattern.length() || (pattern.charAt(index + 3) != '=' && pattern.charAt(index + 3) != '!'))) { + return true; + } + } + return false; + } + private static FunctionCall dateAdd(String unit, List arguments, HogQlQuery.SourceSpan span) { return call( diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index e8c5ecf65908..5dc1fb7c73c0 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -71,6 +71,8 @@ final class HogQlV0FunctionRegistry rewrite("and", FunctionRewrite.AND, "boolean", signature(2)), rewrite("greater", FunctionRewrite.GREATER, "boolean", signature(2)), rewrite("like", FunctionRewrite.LIKE, "boolean", signature(2)), + rewrite("extract", FunctionRewrite.REGEX_EXTRACT, "varchar", signature(2)), + rewrite("replaceRegexpAll", FunctionRewrite.REGEX_REPLACE_ALL, "varchar", signature(3)), scalar("fromUnixTimestamp", "from_unixtime", signature(1)), scalar("formatDateTime", "date_format", signature(2)), scalar("toTimeZone", "at_timezone", signature(2)), @@ -80,6 +82,7 @@ final class HogQlV0FunctionRegistry scalar("startsWith", "starts_with", signature(2)), scalar("substring", "substring", signature(2), signature(3)), scalar("log10", "log10", signature(1)), + scalar("match", "regexp_like", signature(2)), scalar("map", "map", signature(2)), scalar("dateDiff", "date_diff", signature(3)), scalar("dateTrunc", "date_trunc", signature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index eee9b3962c34..1ee9ebde738b 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -304,8 +304,9 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi JSON_KEYS_AND_VALUES_RAW, NOT, PARSE_TIMESTAMP, TO_UNIX_TIMESTAMP, UNIQ_EXACT -> !signature.variadic() && signature.argumentTypes().size() == 1; case ADD_DAYS, ADD_MONTHS, AND, GREATER, JSON_EXTRACT_TYPED, JSON_KEYS_AND_VALUES, - LIKE, MAX_IF, SUM_IF, UNIQ_IF -> + LIKE, MAX_IF, REGEX_EXTRACT, SUM_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; + case REGEX_REPLACE_ALL -> !signature.variadic() && signature.argumentTypes().size() == 3; case CAST_TIMESTAMP -> !signature.variadic() && (signature.argumentTypes().size() == 1 || signature.argumentTypes().size() == 2); case DATE_ADD -> !signature.variadic() && @@ -1333,6 +1334,8 @@ public enum FunctionRewrite MULTI_IF, NOT, PARSE_TIMESTAMP, + REGEX_EXTRACT, + REGEX_REPLACE_ALL, SUM_IF, TODAY, TO_UNIX_TIMESTAMP, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 6e6309e2f0dd..0f278eb698a8 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -213,6 +213,19 @@ public void testCompilerLowersScalarCompatibilityFunctions() "starts_with(name, 'prefix'), substring(name, 2, 4), log10(score) FROM records")); } + @Test + public void testCompilerLowersRegexCompatibilityFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT extract(name, '([a-z]+)'), extract(name, '[a-z]+'), match(name, '^prefix'), " + + "replaceRegexpAll(name, '([a-z])', 'x_') FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT coalesce(regexp_extract(name, '([a-z]+)', 1), ''), " + + "coalesce(regexp_extract(name, '[a-z]+', 0), ''), regexp_like(name, '^prefix'), " + + "regexp_replace(name, '([a-z])', 'x_') FROM records")); + } + @Test public void testCompilerEnforcesV0FunctionArities() { From 2b71c9ad1e7c6253525a97a8f43cba507349ea84 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 23:39:34 -0400 Subject: [PATCH 087/136] feat(hogql): lower aggregate combinators --- core/trino-hogql-compiler/README.md | 8 ++++--- .../hogql/compiler/HogQlFunctionResolver.java | 8 +++++++ .../compiler/HogQlV0FunctionRegistry.java | 8 +++++++ .../catalog/HogQlSemanticCatalogSnapshot.java | 22 ++++++++++++++----- .../compiler/TestHogQlFunctionResolver.java | 15 +++++++++++++ 5 files changed, 53 insertions(+), 8 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index baf17c787173..a4d6c7a8297e 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -60,9 +60,11 @@ The MVP lowers a frozen, fail-closed subset directly to stock Trino AST nodes: (`first_value`, `rank`, `row_number`). The compatibility extension also lowers conditional aggregates (`countIf`, -`sumIf`, `maxIf`, and `uniqIf`), distinct aggregates (`uniqExact` and -`groupUniqArray`), and `multiIf` directly to stock Trino aggregate filters, -DISTINCT aggregates, and searched CASE expressions. +`sumIf`, `minIf`, `maxIf`, `avgIf`, `anyIf`, `argMaxIf`, `groupArrayIf`, +`uniqIf`, `uniqExactIf`, and `groupUniqArrayIf`), distinct aggregates +(`uniqExact`, `countDistinct`, and `groupUniqArray`), and `multiIf` directly to +stock Trino aggregate filters, DISTINCT aggregates, and searched CASE +expressions. Literal-path JSON extraction lowers to Trino JSON paths with HogQL-compatible defaults. The compatibility profile also supports the corpus's typed map and key/value extraction forms through checked JSON-to-map casts. diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 56729bef9a3c..3fa9fd1900db 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -385,6 +385,14 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite case UNIQ_IF -> aggregate("approx_distinct", List.of(arguments.getFirst()), false, arguments.get(1), span); case UNIQ_EXACT -> aggregate("count", List.of(arguments.getFirst()), true, null, span); case GROUP_UNIQ_ARRAY -> aggregate("array_agg", List.of(arguments.getFirst()), true, null, span); + case ARG_MAX_IF -> aggregate("max_by", List.of(arguments.getFirst(), arguments.get(1)), false, arguments.get(2), span); + case ANY_IF -> aggregate("arbitrary", List.of(arguments.getFirst()), false, arguments.get(1), span); + case MIN_IF -> aggregate("min", List.of(arguments.getFirst()), false, arguments.get(1), span); + case AVG_IF -> aggregate("avg", List.of(arguments.getFirst()), false, arguments.get(1), span); + case GROUP_ARRAY_IF -> aggregate("array_agg", List.of(arguments.getFirst()), false, arguments.get(1), span); + case UNIQ_EXACT_IF -> aggregate("count", List.of(arguments.getFirst()), true, arguments.get(1), span); + case GROUP_UNIQ_ARRAY_IF -> aggregate("array_agg", List.of(arguments.getFirst()), true, arguments.get(1), span); + case COUNT_DISTINCT -> aggregate("count", List.of(arguments.getFirst()), true, null, span); case MULTI_IF -> multiIf(function, arguments); case JSON_EXTRACT_STRING -> jsonExtractScalar(function, arguments, "varchar", new Literal(HogQlQuery.LiteralKind.STRING, "", span)); case JSON_EXTRACT_INT -> jsonExtractScalar(function, arguments, "bigint", new Literal(HogQlQuery.LiteralKind.INTEGER, "0", span)); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 5dc1fb7c73c0..7ef1f7844719 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -103,6 +103,14 @@ final class HogQlV0FunctionRegistry aggregateRewrite("uniqIf", FunctionRewrite.UNIQ_IF, "bigint", signature(2)), aggregateRewrite("uniqExact", FunctionRewrite.UNIQ_EXACT, "bigint", signature(1)), aggregateRewrite("groupUniqArray", FunctionRewrite.GROUP_UNIQ_ARRAY, "array(any)", signature(1)), + aggregateRewrite("argMaxIf", FunctionRewrite.ARG_MAX_IF, "any", signature(3)), + aggregateRewrite("anyIf", FunctionRewrite.ANY_IF, "any", signature(2)), + aggregateRewrite("minIf", FunctionRewrite.MIN_IF, "any", signature(2)), + aggregateRewrite("avgIf", FunctionRewrite.AVG_IF, "double", signature(2)), + aggregateRewrite("groupArrayIf", FunctionRewrite.GROUP_ARRAY_IF, "array(any)", signature(2)), + aggregateRewrite("uniqExactIf", FunctionRewrite.UNIQ_EXACT_IF, "bigint", signature(2)), + aggregateRewrite("groupUniqArrayIf", FunctionRewrite.GROUP_UNIQ_ARRAY_IF, "array(any)", signature(2)), + aggregateRewrite("countDistinct", FunctionRewrite.COUNT_DISTINCT, "bigint", signature(1)), nondeterministicAggregate("uniq", "approx_distinct", false, signature(1)), nondeterministicAggregate("any", "arbitrary", false, signature(1)), nondeterministicAggregate("argMin", "min_by", false, signature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 1ee9ebde738b..df49159437ca 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -290,7 +290,9 @@ private static Map validateFunctions(List< private static FunctionKind rewriteFunctionKind(FunctionRewrite rewrite) { return switch (rewrite) { - case COUNT_IF, GROUP_UNIQ_ARRAY, MAX_IF, SUM_IF, UNIQ_EXACT, UNIQ_IF -> FunctionKind.AGGREGATE; + case ANY_IF, ARG_MAX_IF, AVG_IF, COUNT_DISTINCT, COUNT_IF, GROUP_ARRAY_IF, + GROUP_UNIQ_ARRAY, GROUP_UNIQ_ARRAY_IF, MAX_IF, MIN_IF, SUM_IF, + UNIQ_EXACT, UNIQ_EXACT_IF, UNIQ_IF -> FunctionKind.AGGREGATE; default -> FunctionKind.SCALAR; }; } @@ -301,12 +303,14 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi case CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_VARCHAR, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, GROUP_UNIQ_ARRAY, INTERVAL_DAY, IS_NOT_NULL, IS_NULL, COUNT_IF, - JSON_KEYS_AND_VALUES_RAW, NOT, PARSE_TIMESTAMP, TO_UNIX_TIMESTAMP, UNIQ_EXACT -> + JSON_KEYS_AND_VALUES_RAW, NOT, PARSE_TIMESTAMP, TO_UNIX_TIMESTAMP, + COUNT_DISTINCT, UNIQ_EXACT -> !signature.variadic() && signature.argumentTypes().size() == 1; - case ADD_DAYS, ADD_MONTHS, AND, GREATER, JSON_EXTRACT_TYPED, JSON_KEYS_AND_VALUES, - LIKE, MAX_IF, REGEX_EXTRACT, SUM_IF, UNIQ_IF -> + case ADD_DAYS, ADD_MONTHS, AND, ANY_IF, AVG_IF, GREATER, GROUP_ARRAY_IF, + GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_KEYS_AND_VALUES, + LIKE, MAX_IF, MIN_IF, REGEX_EXTRACT, SUM_IF, UNIQ_EXACT_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; - case REGEX_REPLACE_ALL -> !signature.variadic() && signature.argumentTypes().size() == 3; + case ARG_MAX_IF, REGEX_REPLACE_ALL -> !signature.variadic() && signature.argumentTypes().size() == 3; case CAST_TIMESTAMP -> !signature.variadic() && (signature.argumentTypes().size() == 1 || signature.argumentTypes().size() == 2); case DATE_ADD -> !signature.variadic() && @@ -1310,14 +1314,20 @@ public enum FunctionRewrite ADD_DAYS, ADD_MONTHS, AND, + ANY_IF, + ARG_MAX_IF, + AVG_IF, DATE_ADD, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, COUNT_IF, + COUNT_DISTINCT, GROUP_UNIQ_ARRAY, GREATER, + GROUP_ARRAY_IF, + GROUP_UNIQ_ARRAY_IF, INTERVAL_DAY, IS_NULL, IS_NOT_NULL, @@ -1331,6 +1341,7 @@ public enum FunctionRewrite JSON_LENGTH, LIKE, MAX_IF, + MIN_IF, MULTI_IF, NOT, PARSE_TIMESTAMP, @@ -1340,6 +1351,7 @@ public enum FunctionRewrite TODAY, TO_UNIX_TIMESTAMP, UNIQ_EXACT, + UNIQ_EXACT_IF, UNIQ_IF } diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 0f278eb698a8..590f73e25bfe 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -226,6 +226,21 @@ public void testCompilerLowersRegexCompatibilityFunctions() "regexp_replace(name, '([a-z])', 'x_') FROM records")); } + @Test + public void testCompilerLowersAggregateCombinators() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT argMaxIf(value, timestamp, active), anyIf(value, active), minIf(value, active), " + + "avgIf(value, active), groupArrayIf(value, active), uniqExactIf(value, active), " + + "groupUniqArrayIf(value, active), countDistinct(value) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT max_by(value, timestamp) FILTER (WHERE active), arbitrary(value) FILTER (WHERE active), " + + "min(value) FILTER (WHERE active), avg(value) FILTER (WHERE active), " + + "array_agg(value) FILTER (WHERE active), count(DISTINCT value) FILTER (WHERE active), " + + "array_agg(DISTINCT value) FILTER (WHERE active), count(DISTINCT value) FROM records")); + } + @Test public void testCompilerEnforcesV0FunctionArities() { From 68cfc5da455f1149be140e0278774a0a119f3012 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 23:45:07 -0400 Subject: [PATCH 088/136] feat(hogql): lower numeric conversions --- core/trino-hogql-compiler/README.md | 4 +- .../hogql/compiler/HogQlFunctionResolver.java | 62 +++++++++++++++++++ .../compiler/HogQlV0FunctionRegistry.java | 4 ++ .../catalog/HogQlSemanticCatalogSnapshot.java | 10 ++- .../compiler/TestHogQlFunctionResolver.java | 23 +++++++ 5 files changed, 99 insertions(+), 4 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index a4d6c7a8297e..bfc3e9eb1a91 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -74,7 +74,9 @@ best-effort timestamp parsing. Array-form membership and the nullish operator lower to stock `IN`, `IF`, and `COALESCE` nodes, including nullable empty-membership behavior. Function-form boolean/comparison operators and common string/math aliases -lower to their equivalent stock operators and functions. +lower to their equivalent stock operators and functions. Checked/defaulting +float conversions, Decimal64 casts, and integer division lower to stock casts +and arithmetic. Literal regex extraction preserves HogQL's capture-group and empty-default behavior; matching and replace-all lower to Trino regex functions. diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 3fa9fd1900db..9461a873f8f1 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -370,6 +370,10 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite case IS_NOT_NULL -> new IsNullExpression(arguments.getFirst(), true, span, span); case CAST_DATE -> cast(arguments.getFirst(), "date", span); case CAST_DOUBLE -> cast(arguments.getFirst(), "double", span); + case FLOAT_OR_ZERO -> coalesce(tryCast(arguments.getFirst(), "double", span), new Literal(HogQlQuery.LiteralKind.FLOAT, "0.0", span), span); + case FLOAT_OR_DEFAULT -> coalesce(tryCast(arguments.getFirst(), "double", span), cast(arguments.get(1), "double", span), span); + case DECIMAL_CAST -> tryCast(arguments.getFirst(), decimalType(function, arguments.get(1)), span); + case INT_DIV -> intDiv(arguments, span); case CAST_BIGINT -> cast(arguments.getFirst(), "bigint", span); case CAST_TIMESTAMP -> arguments.size() == 1 ? cast(arguments.getFirst(), "timestamp(0)", span) @@ -645,6 +649,64 @@ private static CastExpression cast(Expression value, String type, HogQlQuery.Sou return new CastExpression(value, new Identifier(type, false, span), false, span); } + private static CastExpression tryCast(Expression value, String type, HogQlQuery.SourceSpan span) + { + return new CastExpression(value, new Identifier(type, false, span), true, span); + } + + private static String decimalType(FunctionCall function, Expression scaleExpression) + { + if (!(scaleExpression instanceof Literal literal) || literal.kind() != HogQlQuery.LiteralKind.INTEGER) { + throw unsupportedError(function, "HogQL decimal scale must be an integer literal"); + } + int scale; + try { + scale = Integer.parseInt(literal.value()); + } + catch (NumberFormatException _) { + throw resolutionError(function, "HogQL decimal scale is outside the supported range"); + } + if (scale < 0 || scale > 18) { + throw resolutionError(function, "HogQL Decimal64 scale must be between 0 and 18"); + } + return "decimal(18," + scale + ")"; + } + + private static Expression intDiv(List arguments, HogQlQuery.SourceSpan span) + { + Expression dividend = cast(arguments.getFirst(), "bigint", span); + Expression divisor = cast(arguments.get(1), "bigint", span); + Literal zero = new Literal(HogQlQuery.LiteralKind.INTEGER, "0", span); + Literal one = new Literal(HogQlQuery.LiteralKind.INTEGER, "1", span); + Expression hasRemainder = new BinaryExpression( + HogQlQuery.BinaryOperator.NOT_EQUAL, + new BinaryExpression(HogQlQuery.BinaryOperator.MODULO, dividend, divisor, span), + zero, + span); + Expression signsDiffer = new BinaryExpression( + HogQlQuery.BinaryOperator.OR, + new BinaryExpression( + HogQlQuery.BinaryOperator.AND, + new BinaryExpression(HogQlQuery.BinaryOperator.LESS_THAN, dividend, zero, span), + new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN, divisor, zero, span), + span), + new BinaryExpression( + HogQlQuery.BinaryOperator.AND, + new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN, dividend, zero, span), + new BinaryExpression(HogQlQuery.BinaryOperator.LESS_THAN, divisor, zero, span), + span), + span); + Expression adjustment = call( + "if", + List.of(new BinaryExpression(HogQlQuery.BinaryOperator.AND, hasRemainder, signsDiffer, span), one, zero), + span); + return new BinaryExpression( + HogQlQuery.BinaryOperator.SUBTRACT, + new BinaryExpression(HogQlQuery.BinaryOperator.DIVIDE, dividend, divisor, span), + adjustment, + span); + } + private static FunctionCall dateTrunc(String unit, Expression value, HogQlQuery.SourceSpan span) { return new FunctionCall( diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 7ef1f7844719..0b79d4299555 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -44,6 +44,10 @@ final class HogQlV0FunctionRegistry rewrite("isNotNull", FunctionRewrite.IS_NOT_NULL, "boolean", signature(1)), rewrite("toInt", FunctionRewrite.CAST_BIGINT, "bigint", signature(1)), rewrite("toFloat", FunctionRewrite.CAST_DOUBLE, "double", signature(1)), + rewrite("toFloatOrZero", FunctionRewrite.FLOAT_OR_ZERO, "double", signature(1)), + rewrite("toFloatOrDefault", FunctionRewrite.FLOAT_OR_DEFAULT, "double", signature(2)), + rewrite("toDecimal", FunctionRewrite.DECIMAL_CAST, "decimal", signature(2)), + rewrite("intDiv", FunctionRewrite.INT_DIV, "bigint", signature(2)), rewrite("toString", FunctionRewrite.CAST_VARCHAR, "varchar", signature(1)), rewrite("toDate", FunctionRewrite.CAST_DATE, "date", signature(1)), rewrite("toDateTime", FunctionRewrite.CAST_TIMESTAMP, "timestamp(0)", signature(1), signature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index df49159437ca..5a4a3ef291d4 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -300,15 +300,15 @@ private static FunctionKind rewriteFunctionKind(FunctionRewrite rewrite) private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSignature signature) { return switch (rewrite) { - case CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_VARCHAR, + case CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_VARCHAR, FLOAT_OR_ZERO, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, GROUP_UNIQ_ARRAY, INTERVAL_DAY, IS_NOT_NULL, IS_NULL, COUNT_IF, JSON_KEYS_AND_VALUES_RAW, NOT, PARSE_TIMESTAMP, TO_UNIX_TIMESTAMP, COUNT_DISTINCT, UNIQ_EXACT -> !signature.variadic() && signature.argumentTypes().size() == 1; - case ADD_DAYS, ADD_MONTHS, AND, ANY_IF, AVG_IF, GREATER, GROUP_ARRAY_IF, + case ADD_DAYS, ADD_MONTHS, AND, ANY_IF, AVG_IF, DECIMAL_CAST, FLOAT_OR_DEFAULT, GREATER, GROUP_ARRAY_IF, GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_KEYS_AND_VALUES, - LIKE, MAX_IF, MIN_IF, REGEX_EXTRACT, SUM_IF, UNIQ_EXACT_IF, UNIQ_IF -> + INT_DIV, LIKE, MAX_IF, MIN_IF, REGEX_EXTRACT, SUM_IF, UNIQ_EXACT_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; case ARG_MAX_IF, REGEX_REPLACE_ALL -> !signature.variadic() && signature.argumentTypes().size() == 3; case CAST_TIMESTAMP -> !signature.variadic() && @@ -1324,11 +1324,15 @@ public enum FunctionRewrite DATE_TRUNC_WEEK, COUNT_IF, COUNT_DISTINCT, + DECIMAL_CAST, + FLOAT_OR_DEFAULT, + FLOAT_OR_ZERO, GROUP_UNIQ_ARRAY, GREATER, GROUP_ARRAY_IF, GROUP_UNIQ_ARRAY_IF, INTERVAL_DAY, + INT_DIV, IS_NULL, IS_NOT_NULL, JSON_EXTRACT_FLOAT, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 590f73e25bfe..9c6dfc4c7b19 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -241,6 +241,29 @@ public void testCompilerLowersAggregateCombinators() "array_agg(DISTINCT value) FILTER (WHERE active), count(DISTINCT value) FROM records")); } + @Test + public void testCompilerLowersNumericConversions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT toFloatOrZero(text), toFloatOrDefault(text, 1), toDecimal(text, 4), intDiv(total, 1000), intDiv(-5, 2) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT coalesce(TRY_CAST(text AS double), 0E0), " + + "coalesce(TRY_CAST(text AS double), CAST(1 AS double)), " + + "TRY_CAST(text AS decimal(18, 4)), " + + "CAST(total AS bigint) / CAST(1000 AS bigint) - if(CAST(total AS bigint) % CAST(1000 AS bigint) <> 0 AND " + + "(CAST(total AS bigint) < 0 AND CAST(1000 AS bigint) > 0 OR CAST(total AS bigint) > 0 AND CAST(1000 AS bigint) < 0), 1, 0), " + + "CAST(-5 AS bigint) / CAST(2 AS bigint) - if(CAST(-5 AS bigint) % CAST(2 AS bigint) <> 0 AND " + + "(CAST(-5 AS bigint) < 0 AND CAST(2 AS bigint) > 0 OR CAST(-5 AS bigint) > 0 AND CAST(2 AS bigint) < 0), 1, 0) FROM records")); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope("SELECT toDecimal(value, scale)"))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception).hasMessageContaining("decimal scale must be an integer literal"); + } + @Test public void testCompilerEnforcesV0FunctionArities() { From 9224cf12c1e350cc201814df9b9a6f2895b391a7 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 23:51:06 -0400 Subject: [PATCH 089/136] feat(hogql): lower array compatibility functions --- core/trino-hogql-compiler/README.md | 2 + .../hogql/compiler/HogQlFunctionResolver.java | 39 +++++++++++++++++++ .../compiler/HogQlV0FunctionRegistry.java | 9 +++++ .../catalog/HogQlSemanticCatalogSnapshot.java | 17 ++++++-- .../compiler/TestHogQlFunctionResolver.java | 14 +++++++ 5 files changed, 78 insertions(+), 3 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index bfc3e9eb1a91..0c8c749d7a38 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -79,6 +79,8 @@ float conversions, Decimal64 casts, and integer division lower to stock casts and arithmetic. Literal regex extraction preserves HogQL's capture-group and empty-default behavior; matching and replace-all lower to Trino regex functions. +Common array transforms, predicates, indexing, summation, ranges, tuple +indexing, splitting, and membership lower to Trino higher-order functions. Function aliases are resolved before Trino analysis. Unknown functions fail as HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 9461a873f8f1..0bc506db1b15 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -374,6 +374,23 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite case FLOAT_OR_DEFAULT -> coalesce(tryCast(arguments.getFirst(), "double", span), cast(arguments.get(1), "double", span), span); case DECIMAL_CAST -> tryCast(arguments.getFirst(), decimalType(function, arguments.get(1)), span); case INT_DIV -> intDiv(arguments, span); + case ARRAY_ELEMENT -> call("element_at", arguments, span); + case ARRAY_FILTER -> call("filter", List.of(arguments.get(1), arguments.getFirst()), span); + case ARRAY_FIRST -> call( + "element_at", + List.of(call("filter", List.of(arguments.get(1), arguments.getFirst()), span), integerLiteral("1", span)), + span); + case ARRAY_MAP -> call("transform", List.of(arguments.get(1), arguments.getFirst()), span); + case ARRAY_SUM -> arraySum(arguments.getFirst(), span); + case RANGE -> call( + "sequence", + List.of( + integerLiteral("0", span), + new BinaryExpression(HogQlQuery.BinaryOperator.SUBTRACT, arguments.getFirst(), integerLiteral("1", span), span)), + span); + case TUPLE_ELEMENT -> new SubscriptExpression(arguments.getFirst(), arguments.get(1), span); + case SPLIT_CHAR -> call("split", List.of(arguments.get(1), arguments.getFirst()), span); + case HAS -> call("contains", arguments, span); case CAST_BIGINT -> cast(arguments.getFirst(), "bigint", span); case CAST_TIMESTAMP -> arguments.size() == 1 ? cast(arguments.getFirst(), "timestamp(0)", span) @@ -707,6 +724,28 @@ private static Expression intDiv(List arguments, HogQlQuery.SourceSp span); } + private static Expression arraySum(Expression array, HogQlQuery.SourceSpan span) + { + Identifier sum = new Identifier("_hogql_sum", false, span); + Identifier item = new Identifier("_hogql_item", false, span); + Expression sumReference = new ColumnReference(List.of(sum), span); + LambdaExpression add = new LambdaExpression( + List.of(sum, item), + new BinaryExpression( + HogQlQuery.BinaryOperator.ADD, + sumReference, + new ColumnReference(List.of(item), span), + span), + span); + LambdaExpression finish = new LambdaExpression(List.of(sum), sumReference, span); + return call("reduce", List.of(array, integerLiteral("0", span), add, finish), span); + } + + private static Literal integerLiteral(String value, HogQlQuery.SourceSpan span) + { + return new Literal(HogQlQuery.LiteralKind.INTEGER, value, span); + } + private static FunctionCall dateTrunc(String unit, Expression value, HogQlQuery.SourceSpan span) { return new FunctionCall( diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 0b79d4299555..44e6f9ac518b 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -48,6 +48,15 @@ final class HogQlV0FunctionRegistry rewrite("toFloatOrDefault", FunctionRewrite.FLOAT_OR_DEFAULT, "double", signature(2)), rewrite("toDecimal", FunctionRewrite.DECIMAL_CAST, "decimal", signature(2)), rewrite("intDiv", FunctionRewrite.INT_DIV, "bigint", signature(2)), + rewrite("arrayElement", FunctionRewrite.ARRAY_ELEMENT, "any", signature(2)), + rewrite("arrayFilter", FunctionRewrite.ARRAY_FILTER, "array(any)", signature(2)), + rewrite("arrayFirst", FunctionRewrite.ARRAY_FIRST, "any", signature(2)), + rewrite("arrayMap", FunctionRewrite.ARRAY_MAP, "array(any)", signature(2)), + rewrite("arraySum", FunctionRewrite.ARRAY_SUM, "any", signature(1)), + rewrite("range", FunctionRewrite.RANGE, "array(bigint)", signature(1)), + rewrite("tupleElement", FunctionRewrite.TUPLE_ELEMENT, "any", signature(2)), + rewrite("splitByChar", FunctionRewrite.SPLIT_CHAR, "array(varchar)", signature(2)), + rewrite("has", FunctionRewrite.HAS, "boolean", signature(2)), rewrite("toString", FunctionRewrite.CAST_VARCHAR, "varchar", signature(1)), rewrite("toDate", FunctionRewrite.CAST_DATE, "date", signature(1)), rewrite("toDateTime", FunctionRewrite.CAST_TIMESTAMP, "timestamp(0)", signature(1), signature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 5a4a3ef291d4..3511b08d6447 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -300,15 +300,17 @@ private static FunctionKind rewriteFunctionKind(FunctionRewrite rewrite) private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSignature signature) { return switch (rewrite) { - case CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_VARCHAR, FLOAT_OR_ZERO, + case ARRAY_SUM, CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_VARCHAR, FLOAT_OR_ZERO, RANGE, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, GROUP_UNIQ_ARRAY, INTERVAL_DAY, IS_NOT_NULL, IS_NULL, COUNT_IF, JSON_KEYS_AND_VALUES_RAW, NOT, PARSE_TIMESTAMP, TO_UNIX_TIMESTAMP, COUNT_DISTINCT, UNIQ_EXACT -> !signature.variadic() && signature.argumentTypes().size() == 1; - case ADD_DAYS, ADD_MONTHS, AND, ANY_IF, AVG_IF, DECIMAL_CAST, FLOAT_OR_DEFAULT, GREATER, GROUP_ARRAY_IF, + case ADD_DAYS, ADD_MONTHS, AND, ANY_IF, ARRAY_ELEMENT, ARRAY_FILTER, ARRAY_FIRST, ARRAY_MAP, + AVG_IF, DECIMAL_CAST, FLOAT_OR_DEFAULT, GREATER, GROUP_ARRAY_IF, HAS, GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_KEYS_AND_VALUES, - INT_DIV, LIKE, MAX_IF, MIN_IF, REGEX_EXTRACT, SUM_IF, UNIQ_EXACT_IF, UNIQ_IF -> + INT_DIV, LIKE, MAX_IF, MIN_IF, REGEX_EXTRACT, SPLIT_CHAR, SUM_IF, + TUPLE_ELEMENT, UNIQ_EXACT_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; case ARG_MAX_IF, REGEX_REPLACE_ALL -> !signature.variadic() && signature.argumentTypes().size() == 3; case CAST_TIMESTAMP -> !signature.variadic() && @@ -1316,6 +1318,11 @@ public enum FunctionRewrite AND, ANY_IF, ARG_MAX_IF, + ARRAY_ELEMENT, + ARRAY_FILTER, + ARRAY_FIRST, + ARRAY_MAP, + ARRAY_SUM, AVG_IF, DATE_ADD, DATE_TRUNC_DAY, @@ -1331,6 +1338,7 @@ public enum FunctionRewrite GREATER, GROUP_ARRAY_IF, GROUP_UNIQ_ARRAY_IF, + HAS, INTERVAL_DAY, INT_DIV, IS_NULL, @@ -1351,9 +1359,12 @@ public enum FunctionRewrite PARSE_TIMESTAMP, REGEX_EXTRACT, REGEX_REPLACE_ALL, + RANGE, + SPLIT_CHAR, SUM_IF, TODAY, TO_UNIX_TIMESTAMP, + TUPLE_ELEMENT, UNIQ_EXACT, UNIQ_EXACT_IF, UNIQ_IF diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 9c6dfc4c7b19..4e54cb82d47f 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -264,6 +264,20 @@ public void testCompilerLowersNumericConversions() assertThat(exception).hasMessageContaining("decimal scale must be an integer literal"); } + @Test + public void testCompilerLowersArrayCompatibilityFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT arrayElement(items, -1), arrayFilter(x -> x > 0, items), arrayFirst(x -> x > 0, items), " + + "arrayMap(x -> x + 1, items), arraySum(items), range(3), tupleElement(item, 2), " + + "splitByChar(',', text), has(items, 3) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT element_at(items, -1), filter(items, x -> x > 0), element_at(filter(items, x -> x > 0), 1), " + + "transform(items, x -> x + 1), reduce(items, 0, (_hogql_sum, _hogql_item) -> _hogql_sum + _hogql_item, _hogql_sum -> _hogql_sum), " + + "sequence(0, 3 - 1), item[2], split(text, ','), contains(items, 3) FROM records")); + } + @Test public void testCompilerEnforcesV0FunctionArities() { From e547eae2cb6296f10dadb9ff88a0150ad7e8d224 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 23:55:43 -0400 Subject: [PATCH 090/136] fix(hogql): preserve array edge semantics --- core/trino-hogql-compiler/README.md | 3 ++ .../hogql/compiler/HogQlFunctionResolver.java | 41 +++++++++++++++---- .../compiler/TestHogQlFunctionResolver.java | 9 ++-- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 0c8c749d7a38..35a85bcc019c 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -81,6 +81,9 @@ Literal regex extraction preserves HogQL's capture-group and empty-default behavior; matching and replace-all lower to Trino regex functions. Common array transforms, predicates, indexing, summation, ranges, tuple indexing, splitting, and membership lower to Trino higher-order functions. +Range and membership rewrites preserve empty-range and null-search behavior. +Typed defaults for out-of-range and no-match element lookups require resolved +element types and remain outside the frozen compatibility subset. Function aliases are resolved before Trino analysis. Unknown functions fail as HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 0bc506db1b15..3720c85e6ff3 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -382,15 +382,10 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite span); case ARRAY_MAP -> call("transform", List.of(arguments.get(1), arguments.getFirst()), span); case ARRAY_SUM -> arraySum(arguments.getFirst(), span); - case RANGE -> call( - "sequence", - List.of( - integerLiteral("0", span), - new BinaryExpression(HogQlQuery.BinaryOperator.SUBTRACT, arguments.getFirst(), integerLiteral("1", span), span)), - span); + case RANGE -> range(arguments.getFirst(), span); case TUPLE_ELEMENT -> new SubscriptExpression(arguments.getFirst(), arguments.get(1), span); case SPLIT_CHAR -> call("split", List.of(arguments.get(1), arguments.getFirst()), span); - case HAS -> call("contains", arguments, span); + case HAS -> has(arguments, span); case CAST_BIGINT -> cast(arguments.getFirst(), "bigint", span); case CAST_TIMESTAMP -> arguments.size() == 1 ? cast(arguments.getFirst(), "timestamp(0)", span) @@ -746,6 +741,38 @@ private static Literal integerLiteral(String value, HogQlQuery.SourceSpan span) return new Literal(HogQlQuery.LiteralKind.INTEGER, value, span); } + private static Expression range(Expression end, HogQlQuery.SourceSpan span) + { + Literal zero = integerLiteral("0", span); + Expression isEmpty = new BinaryExpression(HogQlQuery.BinaryOperator.LESS_THAN_OR_EQUAL, end, zero, span); + Expression empty = cast(new ArrayExpression(List.of(), span), "array(bigint)", span); + Expression sequence = call( + "sequence", + List.of(zero, new BinaryExpression(HogQlQuery.BinaryOperator.SUBTRACT, end, integerLiteral("1", span), span)), + span); + return call("if", List.of(isEmpty, empty, sequence), span); + } + + private static Expression has(List arguments, HogQlQuery.SourceSpan span) + { + Expression array = arguments.getFirst(); + Expression value = arguments.get(1); + Identifier item = new Identifier("_hogql_item", false, span); + LambdaExpression isNull = new LambdaExpression( + List.of(item), + new IsNullExpression(new ColumnReference(List.of(item), span), false, span, span), + span); + Expression findNull = call("any_match", List.of(array, isNull), span); + Expression contains = coalesce( + call("contains", List.of(array, value), span), + new Literal(HogQlQuery.LiteralKind.BOOLEAN, "false", span), + span); + return call( + "if", + List.of(new IsNullExpression(value, false, span, span), findNull, contains), + span); + } + private static FunctionCall dateTrunc(String unit, Expression value, HogQlQuery.SourceSpan span) { return new FunctionCall( diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 4e54cb82d47f..e2daaad1cb6c 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -269,13 +269,16 @@ public void testCompilerLowersArrayCompatibilityFunctions() { HogQlCompilationResult result = new HogQlCompiler().compile(envelope( "SELECT arrayElement(items, -1), arrayFilter(x -> x > 0, items), arrayFirst(x -> x > 0, items), " + - "arrayMap(x -> x + 1, items), arraySum(items), range(3), tupleElement(item, 2), " + - "splitByChar(',', text), has(items, 3) FROM records")); + "arrayMap(x -> x + 1, items), arraySum(items), range(3), range(0), tupleElement(item, 2), " + + "splitByChar(',', text), has(items, 3), has(items, NULL) FROM records")); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( "SELECT element_at(items, -1), filter(items, x -> x > 0), element_at(filter(items, x -> x > 0), 1), " + "transform(items, x -> x + 1), reduce(items, 0, (_hogql_sum, _hogql_item) -> _hogql_sum + _hogql_item, _hogql_sum -> _hogql_sum), " + - "sequence(0, 3 - 1), item[2], split(text, ','), contains(items, 3) FROM records")); + "if(3 <= 0, CAST(ARRAY[] AS array(bigint)), sequence(0, 3 - 1)), " + + "if(0 <= 0, CAST(ARRAY[] AS array(bigint)), sequence(0, 0 - 1)), item[2], split(text, ','), " + + "if(3 IS NULL, any_match(items, _hogql_item -> _hogql_item IS NULL), coalesce(contains(items, 3), false)), " + + "if(NULL IS NULL, any_match(items, _hogql_item -> _hogql_item IS NULL), coalesce(contains(items, NULL), false)) FROM records")); } @Test From c5e9383d5c8f0904dd8425aa3890553bde5cf7e8 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Thu, 27 Aug 2026 23:59:34 -0400 Subject: [PATCH 091/136] feat(hogql): lower expression and date aliases --- core/trino-hogql-compiler/README.md | 3 ++ .../hogql/compiler/HogQlFunctionResolver.java | 41 +++++++++++++++++++ .../compiler/HogQlV0FunctionRegistry.java | 19 +++++++++ .../catalog/HogQlSemanticCatalogSnapshot.java | 22 ++++++++-- .../compiler/TestHogQlFunctionResolver.java | 19 +++++++++ 5 files changed, 101 insertions(+), 3 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 35a85bcc019c..7c124080a7b9 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -84,6 +84,9 @@ indexing, splitting, and membership lower to Trino higher-order functions. Range and membership rewrites preserve empty-range and null-search behavior. Typed defaults for out-of-range and no-match element lookups require resolved element types and remain outside the frozen compatibility subset. +Expression-form comparison, arithmetic, membership, tuples, nullability +assertions, string splitting, month arithmetic, and common window aliases also +lower to stock Trino nodes. Function aliases are resolved before Trino analysis. Unknown functions fail as HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 3720c85e6ff3..12f400db4b71 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -386,6 +386,21 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite case TUPLE_ELEMENT -> new SubscriptExpression(arguments.getFirst(), arguments.get(1), span); case SPLIT_CHAR -> call("split", List.of(arguments.get(1), arguments.getFirst()), span); case HAS -> has(arguments, span); + case ASSUME_NOT_NULL -> arguments.getFirst(); + case EMPTY -> empty(arguments.getFirst(), true, span); + case NOT_EMPTY -> empty(arguments.getFirst(), false, span); + case EQUALS -> new BinaryExpression(HogQlQuery.BinaryOperator.EQUAL, arguments.getFirst(), arguments.get(1), span); + case MULTIPLY, MULTIPLY_DECIMAL -> new BinaryExpression(HogQlQuery.BinaryOperator.MULTIPLY, arguments.getFirst(), arguments.get(1), span); + case DIVIDE_DECIMAL -> new BinaryExpression(HogQlQuery.BinaryOperator.DIVIDE, arguments.getFirst(), arguments.get(1), span); + case IN_ARRAY -> call("contains", List.of(arguments.get(1), arguments.getFirst()), span); + case TUPLE -> new TupleExpression(arguments, span); + case SUBTRACT_MONTHS -> dateAdd( + "month", + List.of(arguments.getFirst(), new UnaryExpression(HogQlQuery.UnaryOperator.NEGATE, arguments.get(1), span)), + span); + case INTERVAL_MONTH -> new IntervalExpression(arguments.getFirst(), HogQlQuery.IntervalUnit.MONTH, span); + case START_WEEK -> startOfWeek(arguments.getFirst(), span); + case SPLIT_STRING -> call("split", List.of(arguments.get(1), arguments.getFirst()), span); case CAST_BIGINT -> cast(arguments.getFirst(), "bigint", span); case CAST_TIMESTAMP -> arguments.size() == 1 ? cast(arguments.getFirst(), "timestamp(0)", span) @@ -773,6 +788,32 @@ private static Expression has(List arguments, HogQlQuery.SourceSpan span); } + private static Expression empty(Expression value, boolean expectedEmpty, HogQlQuery.SourceSpan span) + { + Expression length = coalesce( + call("length", List.of(cast(value, "varchar", span)), span), + integerLiteral("0", span), + span); + return new BinaryExpression( + expectedEmpty ? HogQlQuery.BinaryOperator.EQUAL : HogQlQuery.BinaryOperator.GREATER_THAN, + length, + integerLiteral("0", span), + span); + } + + private static Expression startOfWeek(Expression value, HogQlQuery.SourceSpan span) + { + Expression shifted = call("date_add", List.of( + new Literal(HogQlQuery.LiteralKind.STRING, "day", span), + integerLiteral("1", span), + value), span); + Expression monday = dateTrunc("week", shifted, span); + return call("date_add", List.of( + new Literal(HogQlQuery.LiteralKind.STRING, "day", span), + integerLiteral("-1", span), + monday), span); + } + private static FunctionCall dateTrunc(String unit, Expression value, HogQlQuery.SourceSpan span) { return new FunctionCall( diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 44e6f9ac518b..f4f0b7d595df 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -57,6 +57,19 @@ final class HogQlV0FunctionRegistry rewrite("tupleElement", FunctionRewrite.TUPLE_ELEMENT, "any", signature(2)), rewrite("splitByChar", FunctionRewrite.SPLIT_CHAR, "array(varchar)", signature(2)), rewrite("has", FunctionRewrite.HAS, "boolean", signature(2)), + rewrite("assumeNotNull", FunctionRewrite.ASSUME_NOT_NULL, "any", signature(1)), + rewrite("empty", FunctionRewrite.EMPTY, "boolean", signature(1)), + rewrite("notEmpty", FunctionRewrite.NOT_EMPTY, "boolean", signature(1)), + rewrite("equals", FunctionRewrite.EQUALS, "boolean", signature(2)), + rewrite("multiply", FunctionRewrite.MULTIPLY, "any", signature(2)), + rewrite("multiplyDecimal", FunctionRewrite.MULTIPLY_DECIMAL, "decimal", signature(2)), + rewrite("divideDecimal", FunctionRewrite.DIVIDE_DECIMAL, "decimal", signature(2)), + rewrite("in", FunctionRewrite.IN_ARRAY, "boolean", signature(2)), + rewrite("tuple", FunctionRewrite.TUPLE, "row", variadicSignature(1)), + rewrite("subtractMonths", FunctionRewrite.SUBTRACT_MONTHS, "any", signature(2)), + rewrite("toIntervalMonth", FunctionRewrite.INTERVAL_MONTH, "interval year to month", signature(1)), + rewrite("toStartOfWeek", FunctionRewrite.START_WEEK, "timestamp", signature(1)), + rewrite("splitByString", FunctionRewrite.SPLIT_STRING, "array(varchar)", signature(2)), rewrite("toString", FunctionRewrite.CAST_VARCHAR, "varchar", signature(1)), rewrite("toDate", FunctionRewrite.CAST_DATE, "date", signature(1)), rewrite("toDateTime", FunctionRewrite.CAST_TIMESTAMP, "timestamp(0)", signature(1), signature(2)), @@ -96,6 +109,9 @@ final class HogQlV0FunctionRegistry scalar("substring", "substring", signature(2), signature(3)), scalar("log10", "log10", signature(1)), scalar("match", "regexp_like", signature(2)), + scalar("hasAny", "arrays_overlap", signature(2)), + scalar("parseDateTime", "date_parse", signature(2)), + scalar("toLastDayOfMonth", "last_day_of_month", signature(1)), scalar("map", "map", signature(2)), scalar("dateDiff", "date_diff", signature(3)), scalar("dateTrunc", "date_trunc", signature(2)), @@ -128,8 +144,11 @@ final class HogQlV0FunctionRegistry nondeterministicAggregate("any", "arbitrary", false, signature(1)), nondeterministicAggregate("argMin", "min_by", false, signature(2)), nondeterministicAggregate("argMax", "max_by", false, signature(2)), + nondeterministicAggregate("anyLast", "arbitrary", false, signature(1)), window("rank", "rank", signature(0)), window("first_value", "first_value", signature(1)), + window("lag", "lag", signature(1), signature(2), signature(3)), + window("lagInFrame", "lag", signature(1), signature(2), signature(3)), window("row_number", "row_number", signature(0))); private HogQlV0FunctionRegistry() {} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 3511b08d6447..da67e32d58e9 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -300,16 +300,18 @@ private static FunctionKind rewriteFunctionKind(FunctionRewrite rewrite) private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSignature signature) { return switch (rewrite) { - case ARRAY_SUM, CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_VARCHAR, FLOAT_OR_ZERO, RANGE, + case ARRAY_SUM, ASSUME_NOT_NULL, CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_VARCHAR, + EMPTY, FLOAT_OR_ZERO, INTERVAL_MONTH, NOT_EMPTY, RANGE, START_WEEK, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, GROUP_UNIQ_ARRAY, INTERVAL_DAY, IS_NOT_NULL, IS_NULL, COUNT_IF, JSON_KEYS_AND_VALUES_RAW, NOT, PARSE_TIMESTAMP, TO_UNIX_TIMESTAMP, COUNT_DISTINCT, UNIQ_EXACT -> !signature.variadic() && signature.argumentTypes().size() == 1; case ADD_DAYS, ADD_MONTHS, AND, ANY_IF, ARRAY_ELEMENT, ARRAY_FILTER, ARRAY_FIRST, ARRAY_MAP, - AVG_IF, DECIMAL_CAST, FLOAT_OR_DEFAULT, GREATER, GROUP_ARRAY_IF, HAS, + AVG_IF, DECIMAL_CAST, DIVIDE_DECIMAL, EQUALS, FLOAT_OR_DEFAULT, GREATER, GROUP_ARRAY_IF, HAS, GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_KEYS_AND_VALUES, - INT_DIV, LIKE, MAX_IF, MIN_IF, REGEX_EXTRACT, SPLIT_CHAR, SUM_IF, + IN_ARRAY, INT_DIV, LIKE, MAX_IF, MIN_IF, MULTIPLY, MULTIPLY_DECIMAL, + REGEX_EXTRACT, SPLIT_CHAR, SPLIT_STRING, SUBTRACT_MONTHS, SUM_IF, TUPLE_ELEMENT, UNIQ_EXACT_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; case ARG_MAX_IF, REGEX_REPLACE_ALL -> !signature.variadic() && signature.argumentTypes().size() == 3; @@ -322,6 +324,7 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi case JSON_LENGTH -> (!signature.variadic() && signature.argumentTypes().size() == 1) || (signature.variadic() && signature.argumentTypes().size() == 3); case MULTI_IF -> signature.variadic() && signature.argumentTypes().size() == 4; + case TUPLE -> signature.variadic() && signature.argumentTypes().size() == 2; case TODAY -> !signature.variadic() && signature.argumentTypes().isEmpty(); }; } @@ -1323,6 +1326,7 @@ public enum FunctionRewrite ARRAY_FIRST, ARRAY_MAP, ARRAY_SUM, + ASSUME_NOT_NULL, AVG_IF, DATE_ADD, DATE_TRUNC_DAY, @@ -1332,6 +1336,9 @@ public enum FunctionRewrite COUNT_IF, COUNT_DISTINCT, DECIMAL_CAST, + DIVIDE_DECIMAL, + EMPTY, + EQUALS, FLOAT_OR_DEFAULT, FLOAT_OR_ZERO, GROUP_UNIQ_ARRAY, @@ -1339,8 +1346,10 @@ public enum FunctionRewrite GROUP_ARRAY_IF, GROUP_UNIQ_ARRAY_IF, HAS, + IN_ARRAY, INTERVAL_DAY, INT_DIV, + INTERVAL_MONTH, IS_NULL, IS_NOT_NULL, JSON_EXTRACT_FLOAT, @@ -1354,16 +1363,23 @@ public enum FunctionRewrite LIKE, MAX_IF, MIN_IF, + MULTIPLY, + MULTIPLY_DECIMAL, MULTI_IF, NOT, + NOT_EMPTY, PARSE_TIMESTAMP, REGEX_EXTRACT, REGEX_REPLACE_ALL, RANGE, SPLIT_CHAR, + SPLIT_STRING, + START_WEEK, + SUBTRACT_MONTHS, SUM_IF, TODAY, TO_UNIX_TIMESTAMP, + TUPLE, TUPLE_ELEMENT, UNIQ_EXACT, UNIQ_EXACT_IF, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index e2daaad1cb6c..af89db8fbbd7 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -281,6 +281,25 @@ public void testCompilerLowersArrayCompatibilityFunctions() "if(NULL IS NULL, any_match(items, _hogql_item -> _hogql_item IS NULL), coalesce(contains(items, NULL), false)) FROM records")); } + @Test + public void testCompilerLowersExpressionAndDateAliases() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT assumeNotNull(name), empty(name), notEmpty(name), equals(first, second), multiply(amount, 2), " + + "multiplyDecimal(amount, rate), divideDecimal(amount, rate), in(kind, ['a', 'b']), tuple(name, amount), " + + "subtractMonths(timestamp, 2), toIntervalMonth(3), toStartOfWeek(timestamp), " + + "splitByString('::', name), hasAny(first_array, second_array), parseDateTime(text, '%Y-%m-%d'), " + + "toLastDayOfMonth(timestamp), anyLast(name), lag(name, 2) OVER (), lagInFrame(name) OVER () FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT name, coalesce(length(CAST(name AS varchar)), 0) = 0, coalesce(length(CAST(name AS varchar)), 0) > 0, " + + "first = second, amount * 2, amount * rate, amount / rate, contains(ARRAY['a', 'b'], kind), ROW(name, amount), " + + "date_add('month', -(2), timestamp), 3 * INTERVAL '1' MONTH, " + + "date_add('day', -1, date_trunc('week', date_add('day', 1, timestamp))), split(name, '::'), " + + "arrays_overlap(first_array, second_array), date_parse(text, '%Y-%m-%d'), last_day_of_month(timestamp), " + + "arbitrary(name), lag(name, 2) OVER (), lag(name) OVER () FROM records")); + } + @Test public void testCompilerEnforcesV0FunctionArities() { From 5a7cf9de678219481afeb13bb69d95e65f41e2c8 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 00:05:01 -0400 Subject: [PATCH 092/136] feat(hogql): lower extended json and regex functions --- core/trino-hogql-compiler/README.md | 2 + .../hogql/compiler/HogQlFunctionResolver.java | 71 +++++++++++++++++-- .../compiler/HogQlV0FunctionRegistry.java | 7 +- .../catalog/HogQlSemanticCatalogSnapshot.java | 17 +++-- .../compiler/TestHogQlFunctionResolver.java | 31 ++++++++ 5 files changed, 115 insertions(+), 13 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 7c124080a7b9..243457862ba2 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -79,6 +79,8 @@ float conversions, Decimal64 casts, and integer division lower to stock casts and arithmetic. Literal regex extraction preserves HogQL's capture-group and empty-default behavior; matching and replace-all lower to Trino regex functions. +The extended profile also lowers extract-all and first-match replacement, plus +boolean, unsigned-integer, raw-array, and nested raw-map JSON extraction. Common array transforms, predicates, indexing, summation, ranges, tuple indexing, splitting, and membership lower to Trino higher-order functions. Range and membership rewrites preserve empty-range and null-search behavior. diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 12f400db4b71..0372cdadb420 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -428,13 +428,16 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite case JSON_EXTRACT_STRING -> jsonExtractScalar(function, arguments, "varchar", new Literal(HogQlQuery.LiteralKind.STRING, "", span)); case JSON_EXTRACT_INT -> jsonExtractScalar(function, arguments, "bigint", new Literal(HogQlQuery.LiteralKind.INTEGER, "0", span)); case JSON_EXTRACT_FLOAT -> jsonExtractScalar(function, arguments, "double", new Literal(HogQlQuery.LiteralKind.FLOAT, "0.0", span)); + case JSON_EXTRACT_BOOL -> jsonExtractScalar(function, arguments, "boolean", new Literal(HogQlQuery.LiteralKind.BOOLEAN, "false", span)); + case JSON_EXTRACT_UINT -> jsonExtractScalar(function, arguments, "bigint", integerLiteral("0", span)); + case JSON_EXTRACT_ARRAY_RAW -> jsonExtractArrayRaw(function, arguments); case JSON_EXTRACT_RAW -> coalesce( call("json_format", List.of(call("json_extract", List.of(arguments.getFirst(), jsonPath(function, arguments.subList(1, arguments.size()))), span)), span), new Literal(HogQlQuery.LiteralKind.STRING, "", span), span); case JSON_EXTRACT_TYPED -> jsonExtractTyped(function, arguments); case JSON_KEYS_AND_VALUES -> jsonExtractKeysAndValues(function, arguments); - case JSON_KEYS_AND_VALUES_RAW -> jsonExtractKeysAndValuesRaw(function, arguments.getFirst()); + case JSON_KEYS_AND_VALUES_RAW -> jsonExtractKeysAndValuesRaw(function, arguments); case JSON_LENGTH -> coalesce( call("json_size", List.of(arguments.getFirst(), jsonPath(function, arguments.subList(1, arguments.size()))), span), new Literal(HogQlQuery.LiteralKind.INTEGER, "0", span), @@ -453,7 +456,9 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite case GREATER -> new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN, arguments.getFirst(), arguments.get(1), span); case LIKE -> new BinaryExpression(HogQlQuery.BinaryOperator.LIKE, arguments.getFirst(), arguments.get(1), span); case REGEX_EXTRACT -> regexExtract(function, arguments); + case REGEX_EXTRACT_ALL -> regexExtractAll(function, arguments); case REGEX_REPLACE_ALL -> regexReplaceAll(function, arguments); + case REGEX_REPLACE_ONE -> regexReplaceOne(function, arguments); }; } @@ -468,24 +473,46 @@ private static Expression regexExtract(FunctionCall function, List a return coalesce(extracted, new Literal(HogQlQuery.LiteralKind.STRING, "", function.span()), function.span()); } + private static Expression regexExtractAll(FunctionCall function, List arguments) + { + Literal pattern = stringLiteral(function, arguments.get(1), "regular expression"); + Literal group = integerLiteral(hasCapturingGroup(pattern.value()) ? "1" : "0", function.span()); + return call("regexp_extract_all", List.of(arguments.getFirst(), pattern, group), function.span()); + } + private static Expression regexReplaceAll(FunctionCall function, List arguments) { Literal pattern = stringLiteral(function, arguments.get(1), "regular expression"); Literal replacement = stringLiteral(function, arguments.get(2), "regular expression replacement"); - String trinoReplacement = regexReplacement(replacement.value()); + String trinoReplacement = regexReplacement(replacement.value(), 0); return call( "regexp_replace", List.of(arguments.getFirst(), pattern, new Literal(HogQlQuery.LiteralKind.STRING, trinoReplacement, replacement.span())), function.span()); } - private static String regexReplacement(String replacement) + private static Expression regexReplaceOne(FunctionCall function, List arguments) + { + Literal pattern = stringLiteral(function, arguments.get(1), "regular expression"); + Literal replacement = stringLiteral(function, arguments.get(2), "regular expression replacement"); + Literal firstPattern = new Literal( + HogQlQuery.LiteralKind.STRING, + "(?s)^(.*?)(" + pattern.value() + ")", + pattern.span()); + Literal firstReplacement = new Literal( + HogQlQuery.LiteralKind.STRING, + "$1" + regexReplacement(replacement.value(), 2), + replacement.span()); + return call("regexp_replace", List.of(arguments.getFirst(), firstPattern, firstReplacement), function.span()); + } + + private static String regexReplacement(String replacement, int groupOffset) { StringBuilder result = new StringBuilder(replacement.length()); for (int index = 0; index < replacement.length(); index++) { char current = replacement.charAt(index); if (current == '\\' && index + 1 < replacement.length() && Character.isDigit(replacement.charAt(index + 1))) { - result.append('$').append(replacement.charAt(++index)); + result.append('$').append(Character.digit(replacement.charAt(++index), 10) + groupOffset); } else { result.append(current); @@ -600,7 +627,7 @@ private static Expression jsonExtractKeysAndValues(FunctionCall function, List arguments) { HogQlQuery.SourceSpan span = function.span(); Identifier key = new Identifier("key", false, span); @@ -611,15 +638,34 @@ private static Expression jsonExtractKeysAndValuesRaw(FunctionCall function, Exp span); Expression formatted = call( "transform_values", - List.of(typedJsonMap(value, "json", span), formatValue), + List.of(typedJsonMapFromJson(jsonValue(function, arguments), "json", span), formatValue), span); return call("map_entries", List.of(formatted), span); } + private static Expression jsonExtractArrayRaw(FunctionCall function, List arguments) + { + HogQlQuery.SourceSpan span = function.span(); + String type = "array(json)"; + Expression converted = tryCast(jsonValue(function, arguments), type, span); + Expression empty = cast(new ArrayExpression(List.of(), span), type, span); + Expression values = coalesce(converted, empty, span); + Identifier item = new Identifier("_hogql_json_item", false, span); + LambdaExpression format = new LambdaExpression( + List.of(item), + call("json_format", List.of(new ColumnReference(List.of(item), span)), span), + span); + return call("transform", List.of(values, format), span); + } + private static Expression typedJsonMap(Expression value, String valueType, HogQlQuery.SourceSpan span) + { + return typedJsonMapFromJson(call("json_parse", List.of(value), span), valueType, span); + } + + private static Expression typedJsonMapFromJson(Expression parsed, String valueType, HogQlQuery.SourceSpan span) { String type = "map(varchar," + valueType + ")"; - Expression parsed = call("json_parse", List.of(value), span); Expression converted = new CastExpression(parsed, new Identifier(type, false, span), true, span); Expression empty = new CastExpression( call("map", List.of(new ArrayExpression(List.of(), span), new ArrayExpression(List.of(), span)), span), @@ -629,6 +675,17 @@ private static Expression typedJsonMap(Expression value, String valueType, HogQl return coalesce(converted, empty, span); } + private static Expression jsonValue(FunctionCall function, List arguments) + { + if (arguments.size() == 1) { + return call("json_parse", List.of(arguments.getFirst()), function.span()); + } + return call( + "json_extract", + List.of(arguments.getFirst(), jsonPath(function, arguments.subList(1, arguments.size()))), + function.span()); + } + private static String jsonType(FunctionCall function, Expression expression) { if (expression instanceof Literal literal && literal.kind() == HogQlQuery.LiteralKind.STRING) { diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index f4f0b7d595df..4719a0c7fdd3 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -81,11 +81,14 @@ final class HogQlV0FunctionRegistry rewrite("JSONExtractString", FunctionRewrite.JSON_EXTRACT_STRING, "varchar", variadicSignature(2)), rewrite("JSONExtractInt", FunctionRewrite.JSON_EXTRACT_INT, "bigint", variadicSignature(2)), rewrite("JSONExtractFloat", FunctionRewrite.JSON_EXTRACT_FLOAT, "double", variadicSignature(2)), + rewrite("JSONExtractBool", FunctionRewrite.JSON_EXTRACT_BOOL, "boolean", variadicSignature(2)), + rewrite("JSONExtractUInt", FunctionRewrite.JSON_EXTRACT_UINT, "bigint", variadicSignature(2)), + rewrite("JSONExtractArrayRaw", FunctionRewrite.JSON_EXTRACT_ARRAY_RAW, "array(varchar)", signature(1), variadicSignature(2)), rewrite("JSONExtractRaw", FunctionRewrite.JSON_EXTRACT_RAW, "varchar", variadicSignature(2)), rewrite("JSONLength", FunctionRewrite.JSON_LENGTH, "bigint", signature(1), variadicSignature(2)), rewrite("JSONExtract", FunctionRewrite.JSON_EXTRACT_TYPED, "any", signature(2)), rewrite("JSONExtractKeysAndValues", FunctionRewrite.JSON_KEYS_AND_VALUES, "array(row(varchar,any))", signature(2)), - rewrite("JSONExtractKeysAndValuesRaw", FunctionRewrite.JSON_KEYS_AND_VALUES_RAW, "array(row(varchar,varchar))", signature(1)), + rewrite("JSONExtractKeysAndValuesRaw", FunctionRewrite.JSON_KEYS_AND_VALUES_RAW, "array(row(varchar,varchar))", signature(1), variadicSignature(2)), rewrite("today", FunctionRewrite.TODAY, "date", signature(0)), rewrite("toIntervalDay", FunctionRewrite.INTERVAL_DAY, "interval day to second", signature(1)), rewrite("addDays", FunctionRewrite.ADD_DAYS, "any", signature(2)), @@ -99,6 +102,8 @@ final class HogQlV0FunctionRegistry rewrite("like", FunctionRewrite.LIKE, "boolean", signature(2)), rewrite("extract", FunctionRewrite.REGEX_EXTRACT, "varchar", signature(2)), rewrite("replaceRegexpAll", FunctionRewrite.REGEX_REPLACE_ALL, "varchar", signature(3)), + rewrite("replaceRegexpOne", FunctionRewrite.REGEX_REPLACE_ONE, "varchar", signature(3)), + rewrite("extractAll", FunctionRewrite.REGEX_EXTRACT_ALL, "array(varchar)", signature(2)), scalar("fromUnixTimestamp", "from_unixtime", signature(1)), scalar("formatDateTime", "date_format", signature(2)), scalar("toTimeZone", "at_timezone", signature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index da67e32d58e9..35028d940238 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -304,24 +304,26 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi EMPTY, FLOAT_OR_ZERO, INTERVAL_MONTH, NOT_EMPTY, RANGE, START_WEEK, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, GROUP_UNIQ_ARRAY, INTERVAL_DAY, IS_NOT_NULL, IS_NULL, COUNT_IF, - JSON_KEYS_AND_VALUES_RAW, NOT, PARSE_TIMESTAMP, TO_UNIX_TIMESTAMP, + NOT, PARSE_TIMESTAMP, TO_UNIX_TIMESTAMP, COUNT_DISTINCT, UNIQ_EXACT -> !signature.variadic() && signature.argumentTypes().size() == 1; case ADD_DAYS, ADD_MONTHS, AND, ANY_IF, ARRAY_ELEMENT, ARRAY_FILTER, ARRAY_FIRST, ARRAY_MAP, AVG_IF, DECIMAL_CAST, DIVIDE_DECIMAL, EQUALS, FLOAT_OR_DEFAULT, GREATER, GROUP_ARRAY_IF, HAS, GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_KEYS_AND_VALUES, IN_ARRAY, INT_DIV, LIKE, MAX_IF, MIN_IF, MULTIPLY, MULTIPLY_DECIMAL, - REGEX_EXTRACT, SPLIT_CHAR, SPLIT_STRING, SUBTRACT_MONTHS, SUM_IF, + REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, SUBTRACT_MONTHS, SUM_IF, TUPLE_ELEMENT, UNIQ_EXACT_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; - case ARG_MAX_IF, REGEX_REPLACE_ALL -> !signature.variadic() && signature.argumentTypes().size() == 3; + case ARG_MAX_IF, REGEX_REPLACE_ALL, REGEX_REPLACE_ONE -> !signature.variadic() && signature.argumentTypes().size() == 3; case CAST_TIMESTAMP -> !signature.variadic() && (signature.argumentTypes().size() == 1 || signature.argumentTypes().size() == 2); case DATE_ADD -> !signature.variadic() && (signature.argumentTypes().size() == 2 || signature.argumentTypes().size() == 3); - case JSON_EXTRACT_FLOAT, JSON_EXTRACT_INT, JSON_EXTRACT_RAW, JSON_EXTRACT_STRING -> + case JSON_EXTRACT_BOOL, JSON_EXTRACT_FLOAT, JSON_EXTRACT_INT, + JSON_EXTRACT_RAW, JSON_EXTRACT_STRING, JSON_EXTRACT_UINT -> signature.variadic() && signature.argumentTypes().size() == 3; - case JSON_LENGTH -> (!signature.variadic() && signature.argumentTypes().size() == 1) || + case JSON_EXTRACT_ARRAY_RAW, JSON_KEYS_AND_VALUES_RAW, JSON_LENGTH -> + (!signature.variadic() && signature.argumentTypes().size() == 1) || (signature.variadic() && signature.argumentTypes().size() == 3); case MULTI_IF -> signature.variadic() && signature.argumentTypes().size() == 4; case TUPLE -> signature.variadic() && signature.argumentTypes().size() == 2; @@ -1352,11 +1354,14 @@ public enum FunctionRewrite INTERVAL_MONTH, IS_NULL, IS_NOT_NULL, + JSON_EXTRACT_ARRAY_RAW, + JSON_EXTRACT_BOOL, JSON_EXTRACT_FLOAT, JSON_EXTRACT_INT, JSON_EXTRACT_RAW, JSON_EXTRACT_STRING, JSON_EXTRACT_TYPED, + JSON_EXTRACT_UINT, JSON_KEYS_AND_VALUES, JSON_KEYS_AND_VALUES_RAW, JSON_LENGTH, @@ -1370,7 +1375,9 @@ public enum FunctionRewrite NOT_EMPTY, PARSE_TIMESTAMP, REGEX_EXTRACT, + REGEX_EXTRACT_ALL, REGEX_REPLACE_ALL, + REGEX_REPLACE_ONE, RANGE, SPLIT_CHAR, SPLIT_STRING, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index af89db8fbbd7..670641860ddf 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -226,6 +226,37 @@ public void testCompilerLowersRegexCompatibilityFunctions() "regexp_replace(name, '([a-z])', 'x_') FROM records")); } + @Test + public void testCompilerLowersExtendedRegexCompatibilityFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT extractAll(name, '([a-z]+)'), extractAll(name, '[a-z]+'), " + + "replaceRegexpOne(name, '([a-z])', 'x_') FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT regexp_extract_all(name, '([a-z]+)', 1), regexp_extract_all(name, '[a-z]+', 0), " + + "regexp_replace(name, '(?s)^(.*?)(([a-z]))', '$1x_') FROM records")); + } + + @Test + public void testCompilerLowersExtendedJsonCompatibilityFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT JSONExtractBool(payload, 'active'), JSONExtractUInt(payload, 'count'), " + + "JSONExtractArrayRaw(payload_text), JSONExtractArrayRaw(payload, 'items'), " + + "JSONExtractKeysAndValuesRaw(payload, 'object') FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"active\"]') AS boolean), false), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"count\"]') AS bigint), 0), " + + "transform(coalesce(TRY_CAST(json_parse(payload_text) AS array(json)), CAST(ARRAY[] AS array(json))), " + + "_hogql_json_item -> json_format(_hogql_json_item)), " + + "transform(coalesce(TRY_CAST(json_extract(payload, '$[\"items\"]') AS array(json)), CAST(ARRAY[] AS array(json))), " + + "_hogql_json_item -> json_format(_hogql_json_item)), " + + "map_entries(transform_values(coalesce(TRY_CAST(json_extract(payload, '$[\"object\"]') AS map(varchar, json)), " + + "CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json))), (key, value) -> json_format(value))) FROM records")); + } + @Test public void testCompilerLowersAggregateCombinators() { From e52bab0fab6a2b81d20192ae04cb18ac9e489203 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 00:09:08 -0400 Subject: [PATCH 093/136] feat(hogql): lower remaining scalar aliases --- core/trino-hogql-compiler/README.md | 3 ++ .../hogql/compiler/HogQlFunctionResolver.java | 47 ++++++++++++++++++- .../compiler/HogQlV0FunctionRegistry.java | 18 ++++++- .../catalog/HogQlSemanticCatalogSnapshot.java | 34 ++++++++++---- .../compiler/TestHogQlFunctionResolver.java | 20 ++++++++ 5 files changed, 109 insertions(+), 13 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 243457862ba2..f0108f9f6f0c 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -89,6 +89,9 @@ element types and remain outside the frozen compatibility subset. Expression-form comparison, arithmetic, membership, tuples, nullability assertions, string splitting, month arithmetic, and common window aliases also lower to stock Trino nodes. +Additional scalar coverage includes array slicing/enumeration, UUID and integer +conversion, JSON existence/value/serialization, hashing, powers, UTF-8 +substrings, banker-rounding aliases, and conditional median aggregation. Function aliases are resolved before Trino analysis. Unknown functions fail as HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 0372cdadb420..5bc8bf30ec99 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -452,16 +452,47 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite case TO_UNIX_TIMESTAMP -> cast(call("to_unixtime", arguments, span), "bigint", span); case PARSE_TIMESTAMP -> new CastExpression(arguments.getFirst(), new Identifier("timestamp(3)", false, span), true, span); case NOT -> new UnaryExpression(HogQlQuery.UnaryOperator.NOT, arguments.getFirst(), span); - case AND -> new BinaryExpression(HogQlQuery.BinaryOperator.AND, arguments.getFirst(), arguments.get(1), span); + case AND -> and(arguments, span); case GREATER -> new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN, arguments.getFirst(), arguments.get(1), span); case LIKE -> new BinaryExpression(HogQlQuery.BinaryOperator.LIKE, arguments.getFirst(), arguments.get(1), span); case REGEX_EXTRACT -> regexExtract(function, arguments); case REGEX_EXTRACT_ALL -> regexExtractAll(function, arguments); case REGEX_REPLACE_ALL -> regexReplaceAll(function, arguments); case REGEX_REPLACE_ONE -> regexReplaceOne(function, arguments); + case ARRAY_SLICE -> call("slice", arguments, span); + case ARRAY_ENUMERATE -> arrayEnumerate(arguments.getFirst(), span); + case SUBTRACT_YEARS -> dateAdd( + "year", + List.of(arguments.getFirst(), new UnaryExpression(HogQlQuery.UnaryOperator.NEGATE, arguments.get(1), span)), + span); + case INT_OR_ZERO -> coalesce(tryCast(arguments.getFirst(), "bigint", span), integerLiteral("0", span), span); + case CAST_UUID -> tryCast(arguments.getFirst(), "uuid", span); + case TO_JSON_STRING -> call("json_format", List.of(cast(arguments.getFirst(), "json", span)), span); + case JSON_HAS -> new IsNullExpression( + call("json_extract", List.of(arguments.getFirst(), jsonPath(function, arguments.subList(1, arguments.size()))), span), + true, + span, + span); + case JSON_VALUE -> call("json_extract_scalar", arguments, span); + case MD5 -> call("md5", List.of(call("to_utf8", List.of(cast(arguments.getFirst(), "varchar", span)), span)), span); + case MEDIAN_IF -> aggregate( + "approx_percentile", + List.of(arguments.getFirst(), new Literal(HogQlQuery.LiteralKind.FLOAT, "0.5", span)), + false, + arguments.get(1), + span); }; } + private static Expression and(List arguments, HogQlQuery.SourceSpan span) + { + Expression result = arguments.getFirst(); + for (int index = 1; index < arguments.size(); index++) { + result = new BinaryExpression(HogQlQuery.BinaryOperator.AND, result, arguments.get(index), span); + } + return result; + } + private static Expression regexExtract(FunctionCall function, List arguments) { Literal pattern = stringLiteral(function, arguments.get(1), "regular expression"); @@ -825,6 +856,20 @@ private static Expression range(Expression end, HogQlQuery.SourceSpan span) return call("if", List.of(isEmpty, empty, sequence), span); } + private static Expression arrayEnumerate(Expression array, HogQlQuery.SourceSpan span) + { + Expression size = call("cardinality", List.of(array), span); + Expression empty = cast(new ArrayExpression(List.of(), span), "array(bigint)", span); + Expression sequence = call("sequence", List.of(integerLiteral("1", span), size), span); + return call( + "if", + List.of( + new BinaryExpression(HogQlQuery.BinaryOperator.EQUAL, size, integerLiteral("0", span), span), + empty, + sequence), + span); + } + private static Expression has(List arguments, HogQlQuery.SourceSpan span) { Expression array = arguments.getFirst(); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 4719a0c7fdd3..62fcbb84c402 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -53,6 +53,8 @@ final class HogQlV0FunctionRegistry rewrite("arrayFirst", FunctionRewrite.ARRAY_FIRST, "any", signature(2)), rewrite("arrayMap", FunctionRewrite.ARRAY_MAP, "array(any)", signature(2)), rewrite("arraySum", FunctionRewrite.ARRAY_SUM, "any", signature(1)), + rewrite("arraySlice", FunctionRewrite.ARRAY_SLICE, "array(any)", signature(3)), + rewrite("arrayEnumerate", FunctionRewrite.ARRAY_ENUMERATE, "array(bigint)", signature(1)), rewrite("range", FunctionRewrite.RANGE, "array(bigint)", signature(1)), rewrite("tupleElement", FunctionRewrite.TUPLE_ELEMENT, "any", signature(2)), rewrite("splitByChar", FunctionRewrite.SPLIT_CHAR, "array(varchar)", signature(2)), @@ -69,6 +71,13 @@ final class HogQlV0FunctionRegistry rewrite("subtractMonths", FunctionRewrite.SUBTRACT_MONTHS, "any", signature(2)), rewrite("toIntervalMonth", FunctionRewrite.INTERVAL_MONTH, "interval year to month", signature(1)), rewrite("toStartOfWeek", FunctionRewrite.START_WEEK, "timestamp", signature(1)), + rewrite("subtractYears", FunctionRewrite.SUBTRACT_YEARS, "any", signature(2)), + rewrite("toIntOrZero", FunctionRewrite.INT_OR_ZERO, "bigint", signature(1)), + rewrite("toUUID", FunctionRewrite.CAST_UUID, "uuid", signature(1)), + rewrite("toJSONString", FunctionRewrite.TO_JSON_STRING, "varchar", signature(1)), + rewrite("JSONHas", FunctionRewrite.JSON_HAS, "boolean", signature(2)), + rewrite("JSON_VALUE", FunctionRewrite.JSON_VALUE, "varchar", signature(2)), + rewrite("md5", FunctionRewrite.MD5, "varbinary", signature(1)), rewrite("splitByString", FunctionRewrite.SPLIT_STRING, "array(varchar)", signature(2)), rewrite("toString", FunctionRewrite.CAST_VARCHAR, "varchar", signature(1)), rewrite("toDate", FunctionRewrite.CAST_DATE, "date", signature(1)), @@ -78,7 +87,7 @@ final class HogQlV0FunctionRegistry rewrite("toStartOfHour", FunctionRewrite.DATE_TRUNC_HOUR, "timestamp", signature(1)), rewrite("toMonday", FunctionRewrite.DATE_TRUNC_WEEK, "timestamp", signature(1)), rewrite("multiIf", FunctionRewrite.MULTI_IF, "any", variadicSignature(3)), - rewrite("JSONExtractString", FunctionRewrite.JSON_EXTRACT_STRING, "varchar", variadicSignature(2)), + rewrite("JSONExtractString", FunctionRewrite.JSON_EXTRACT_STRING, "varchar", signature(1), variadicSignature(2)), rewrite("JSONExtractInt", FunctionRewrite.JSON_EXTRACT_INT, "bigint", variadicSignature(2)), rewrite("JSONExtractFloat", FunctionRewrite.JSON_EXTRACT_FLOAT, "double", variadicSignature(2)), rewrite("JSONExtractBool", FunctionRewrite.JSON_EXTRACT_BOOL, "boolean", variadicSignature(2)), @@ -97,7 +106,7 @@ final class HogQlV0FunctionRegistry rewrite("toUnixTimestamp", FunctionRewrite.TO_UNIX_TIMESTAMP, "bigint", signature(1)), rewrite("parseDateTimeBestEffort", FunctionRewrite.PARSE_TIMESTAMP, "timestamp(3)", signature(1)), rewrite("not", FunctionRewrite.NOT, "boolean", signature(1)), - rewrite("and", FunctionRewrite.AND, "boolean", signature(2)), + rewrite("and", FunctionRewrite.AND, "boolean", signature(2), variadicSignature(2)), rewrite("greater", FunctionRewrite.GREATER, "boolean", signature(2)), rewrite("like", FunctionRewrite.LIKE, "boolean", signature(2)), rewrite("extract", FunctionRewrite.REGEX_EXTRACT, "varchar", signature(2)), @@ -114,6 +123,10 @@ final class HogQlV0FunctionRegistry scalar("substring", "substring", signature(2), signature(3)), scalar("log10", "log10", signature(1)), scalar("match", "regexp_like", signature(2)), + scalar("pow", "power", signature(2)), + scalar("substringUTF8", "substring", signature(2), signature(3)), + scalar("arrayConcat", "concat", variadicSignature(2)), + scalar("roundBankers", "round", signature(1), signature(2)), scalar("hasAny", "arrays_overlap", signature(2)), scalar("parseDateTime", "date_parse", signature(2)), scalar("toLastDayOfMonth", "last_day_of_month", signature(1)), @@ -145,6 +158,7 @@ final class HogQlV0FunctionRegistry aggregateRewrite("uniqExactIf", FunctionRewrite.UNIQ_EXACT_IF, "bigint", signature(2)), aggregateRewrite("groupUniqArrayIf", FunctionRewrite.GROUP_UNIQ_ARRAY_IF, "array(any)", signature(2)), aggregateRewrite("countDistinct", FunctionRewrite.COUNT_DISTINCT, "bigint", signature(1)), + aggregateRewrite("medianIf", FunctionRewrite.MEDIAN_IF, "double", signature(2)), nondeterministicAggregate("uniq", "approx_distinct", false, signature(1)), nondeterministicAggregate("any", "arbitrary", false, signature(1)), nondeterministicAggregate("argMin", "min_by", false, signature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 35028d940238..d3e9dc913c92 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -291,7 +291,7 @@ private static FunctionKind rewriteFunctionKind(FunctionRewrite rewrite) { return switch (rewrite) { case ANY_IF, ARG_MAX_IF, AVG_IF, COUNT_DISTINCT, COUNT_IF, GROUP_ARRAY_IF, - GROUP_UNIQ_ARRAY, GROUP_UNIQ_ARRAY_IF, MAX_IF, MIN_IF, SUM_IF, + GROUP_UNIQ_ARRAY, GROUP_UNIQ_ARRAY_IF, MAX_IF, MEDIAN_IF, MIN_IF, SUM_IF, UNIQ_EXACT, UNIQ_EXACT_IF, UNIQ_IF -> FunctionKind.AGGREGATE; default -> FunctionKind.SCALAR; }; @@ -300,31 +300,35 @@ private static FunctionKind rewriteFunctionKind(FunctionRewrite rewrite) private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSignature signature) { return switch (rewrite) { - case ARRAY_SUM, ASSUME_NOT_NULL, CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_VARCHAR, - EMPTY, FLOAT_OR_ZERO, INTERVAL_MONTH, NOT_EMPTY, RANGE, START_WEEK, + case ARRAY_ENUMERATE, ARRAY_SUM, ASSUME_NOT_NULL, CAST_BIGINT, CAST_DATE, CAST_DOUBLE, + CAST_UUID, CAST_VARCHAR, EMPTY, FLOAT_OR_ZERO, INTERVAL_MONTH, INT_OR_ZERO, + MD5, NOT_EMPTY, RANGE, START_WEEK, TO_JSON_STRING, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, GROUP_UNIQ_ARRAY, INTERVAL_DAY, IS_NOT_NULL, IS_NULL, COUNT_IF, NOT, PARSE_TIMESTAMP, TO_UNIX_TIMESTAMP, COUNT_DISTINCT, UNIQ_EXACT -> !signature.variadic() && signature.argumentTypes().size() == 1; - case ADD_DAYS, ADD_MONTHS, AND, ANY_IF, ARRAY_ELEMENT, ARRAY_FILTER, ARRAY_FIRST, ARRAY_MAP, + case ADD_DAYS, ADD_MONTHS, ANY_IF, ARRAY_ELEMENT, ARRAY_FILTER, ARRAY_FIRST, ARRAY_MAP, AVG_IF, DECIMAL_CAST, DIVIDE_DECIMAL, EQUALS, FLOAT_OR_DEFAULT, GREATER, GROUP_ARRAY_IF, HAS, - GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_KEYS_AND_VALUES, + GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_HAS, JSON_KEYS_AND_VALUES, JSON_VALUE, IN_ARRAY, INT_DIV, LIKE, MAX_IF, MIN_IF, MULTIPLY, MULTIPLY_DECIMAL, - REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, SUBTRACT_MONTHS, SUM_IF, + MEDIAN_IF, REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, + SUBTRACT_MONTHS, SUBTRACT_YEARS, SUM_IF, TUPLE_ELEMENT, UNIQ_EXACT_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; - case ARG_MAX_IF, REGEX_REPLACE_ALL, REGEX_REPLACE_ONE -> !signature.variadic() && signature.argumentTypes().size() == 3; + case ARG_MAX_IF, ARRAY_SLICE, REGEX_REPLACE_ALL, REGEX_REPLACE_ONE -> + !signature.variadic() && signature.argumentTypes().size() == 3; case CAST_TIMESTAMP -> !signature.variadic() && (signature.argumentTypes().size() == 1 || signature.argumentTypes().size() == 2); case DATE_ADD -> !signature.variadic() && (signature.argumentTypes().size() == 2 || signature.argumentTypes().size() == 3); - case JSON_EXTRACT_BOOL, JSON_EXTRACT_FLOAT, JSON_EXTRACT_INT, - JSON_EXTRACT_RAW, JSON_EXTRACT_STRING, JSON_EXTRACT_UINT -> + case JSON_EXTRACT_BOOL, JSON_EXTRACT_FLOAT, JSON_EXTRACT_INT, JSON_EXTRACT_RAW, JSON_EXTRACT_UINT -> signature.variadic() && signature.argumentTypes().size() == 3; - case JSON_EXTRACT_ARRAY_RAW, JSON_KEYS_AND_VALUES_RAW, JSON_LENGTH -> + case JSON_EXTRACT_ARRAY_RAW, JSON_EXTRACT_STRING, JSON_KEYS_AND_VALUES_RAW, JSON_LENGTH -> (!signature.variadic() && signature.argumentTypes().size() == 1) || (signature.variadic() && signature.argumentTypes().size() == 3); + case AND -> (!signature.variadic() && signature.argumentTypes().size() == 2) || + (signature.variadic() && signature.argumentTypes().size() == 3); case MULTI_IF -> signature.variadic() && signature.argumentTypes().size() == 4; case TUPLE -> signature.variadic() && signature.argumentTypes().size() == 2; case TODAY -> !signature.variadic() && signature.argumentTypes().isEmpty(); @@ -1317,6 +1321,7 @@ public enum FunctionRewrite CAST_DOUBLE, CAST_BIGINT, CAST_TIMESTAMP, + CAST_UUID, CAST_VARCHAR, ADD_DAYS, ADD_MONTHS, @@ -1324,9 +1329,11 @@ public enum FunctionRewrite ANY_IF, ARG_MAX_IF, ARRAY_ELEMENT, + ARRAY_ENUMERATE, ARRAY_FILTER, ARRAY_FIRST, ARRAY_MAP, + ARRAY_SLICE, ARRAY_SUM, ASSUME_NOT_NULL, AVG_IF, @@ -1351,6 +1358,7 @@ public enum FunctionRewrite IN_ARRAY, INTERVAL_DAY, INT_DIV, + INT_OR_ZERO, INTERVAL_MONTH, IS_NULL, IS_NOT_NULL, @@ -1362,11 +1370,15 @@ public enum FunctionRewrite JSON_EXTRACT_STRING, JSON_EXTRACT_TYPED, JSON_EXTRACT_UINT, + JSON_HAS, JSON_KEYS_AND_VALUES, JSON_KEYS_AND_VALUES_RAW, JSON_LENGTH, + JSON_VALUE, LIKE, MAX_IF, + MD5, + MEDIAN_IF, MIN_IF, MULTIPLY, MULTIPLY_DECIMAL, @@ -1383,8 +1395,10 @@ public enum FunctionRewrite SPLIT_STRING, START_WEEK, SUBTRACT_MONTHS, + SUBTRACT_YEARS, SUM_IF, TODAY, + TO_JSON_STRING, TO_UNIX_TIMESTAMP, TUPLE, TUPLE_ELEMENT, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 670641860ddf..6605b5d8f125 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -331,6 +331,26 @@ public void testCompilerLowersExpressionAndDateAliases() "arbitrary(name), lag(name, 2) OVER (), lag(name) OVER () FROM records")); } + @Test + public void testCompilerLowersRemainingScalarAliases() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT arraySlice(items, 2, 3), arrayEnumerate(items), pow(2, exponent), substringUTF8(name, 2, 3), " + + "arrayConcat(first_array, second_array), subtractYears(timestamp, 2), toIntOrZero(text), toUUID(uuid_text), " + + "toJSONString(payload), JSONHas(payload, 'key'), JSON_VALUE(payload, '$.key'), md5(name), roundBankers(score, 2), " + + "and(first, second, third), JSONExtractString(payload), medianIf(score, active) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT slice(items, 2, 3), " + + "if(cardinality(items) = 0, CAST(ARRAY[] AS array(bigint)), sequence(1, cardinality(items))), " + + "power(2, exponent), substring(name, 2, 3), concat(first_array, second_array), " + + "date_add('year', -(2), timestamp), coalesce(TRY_CAST(text AS bigint), 0), TRY_CAST(uuid_text AS uuid), " + + "json_format(CAST(payload AS json)), json_extract(payload, '$[\"key\"]') IS NOT NULL, " + + "json_extract_scalar(payload, '$.key'), md5(to_utf8(CAST(name AS varchar))), round(score, 2), " + + "(first AND second) AND third, coalesce(json_extract_scalar(payload, '$'), ''), " + + "approx_percentile(score, 5E-1) FILTER (WHERE active) FROM records")); + } + @Test public void testCompilerEnforcesV0FunctionArities() { From 8c4b623c94f93936ad30dbff111361238a02689e Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 00:15:02 -0400 Subject: [PATCH 094/136] feat(hogql): map final stock function aliases --- core/trino-hogql-compiler/README.md | 2 ++ .../hogql/compiler/HogQlFunctionResolver.java | 20 +++++++++++++++++++ .../compiler/HogQlV0FunctionRegistry.java | 5 +++++ .../catalog/HogQlSemanticCatalogSnapshot.java | 3 ++- .../compiler/TestHogQlFunctionResolver.java | 11 ++++++++++ 5 files changed, 40 insertions(+), 1 deletion(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index f0108f9f6f0c..80d6278011af 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -92,6 +92,8 @@ lower to stock Trino nodes. Additional scalar coverage includes array slicing/enumeration, UUID and integer conversion, JSON existence/value/serialization, hashing, powers, UTF-8 substrings, banker-rounding aliases, and conditional median aggregation. +Simple floor, calendar-part, map construction, and null-predicate aliases are +mapped to their stock equivalents. Function aliases are resolved before Trino analysis. Unknown functions fail as HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 5bc8bf30ec99..d0f572d2c0db 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -481,9 +481,29 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite false, arguments.get(1), span); + case DATE_PART -> datePart(function, arguments); }; } + private static Expression datePart(FunctionCall function, List arguments) + { + Literal unit = stringLiteral(function, arguments.getFirst(), "date part"); + String trinoFunction = switch (unit.value().toLowerCase(Locale.ENGLISH)) { + case "year" -> "year"; + case "quarter" -> "quarter"; + case "month" -> "month"; + case "week" -> "week"; + case "day" -> "day"; + case "dow", "dayofweek" -> "day_of_week"; + case "doy", "dayofyear" -> "day_of_year"; + case "hour" -> "hour"; + case "minute" -> "minute"; + case "second" -> "second"; + default -> throw unsupportedError(function, "Unsupported HogQL date part: " + unit.value()); + }; + return call(trinoFunction, List.of(arguments.get(1)), function.span()); + } + private static Expression and(List arguments, HogQlQuery.SourceSpan span) { Expression result = arguments.getFirst(); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 62fcbb84c402..5a4c24d40dab 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -42,6 +42,7 @@ final class HogQlV0FunctionRegistry nondeterministicScalar("now", "now", signature(0)), nondeterministicScalar("current_timestamp", "now", signature(0)), rewrite("isNotNull", FunctionRewrite.IS_NOT_NULL, "boolean", signature(1)), + rewrite("isNull", FunctionRewrite.IS_NULL, "boolean", signature(1)), rewrite("toInt", FunctionRewrite.CAST_BIGINT, "bigint", signature(1)), rewrite("toFloat", FunctionRewrite.CAST_DOUBLE, "double", signature(1)), rewrite("toFloatOrZero", FunctionRewrite.FLOAT_OR_ZERO, "double", signature(1)), @@ -78,6 +79,7 @@ final class HogQlV0FunctionRegistry rewrite("JSONHas", FunctionRewrite.JSON_HAS, "boolean", signature(2)), rewrite("JSON_VALUE", FunctionRewrite.JSON_VALUE, "varchar", signature(2)), rewrite("md5", FunctionRewrite.MD5, "varbinary", signature(1)), + rewrite("date_part", FunctionRewrite.DATE_PART, "bigint", signature(2)), rewrite("splitByString", FunctionRewrite.SPLIT_STRING, "array(varchar)", signature(2)), rewrite("toString", FunctionRewrite.CAST_VARCHAR, "varchar", signature(1)), rewrite("toDate", FunctionRewrite.CAST_DATE, "date", signature(1)), @@ -123,6 +125,9 @@ final class HogQlV0FunctionRegistry scalar("substring", "substring", signature(2), signature(3)), scalar("log10", "log10", signature(1)), scalar("match", "regexp_like", signature(2)), + scalar("floor", "floor", signature(1)), + scalar("toDayOfMonth", "day", signature(1)), + scalar("mapFromArrays", "map", signature(2)), scalar("pow", "power", signature(2)), scalar("substringUTF8", "substring", signature(2), signature(3)), scalar("arrayConcat", "concat", variadicSignature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index d3e9dc913c92..5d360740c0e8 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -309,7 +309,7 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi COUNT_DISTINCT, UNIQ_EXACT -> !signature.variadic() && signature.argumentTypes().size() == 1; case ADD_DAYS, ADD_MONTHS, ANY_IF, ARRAY_ELEMENT, ARRAY_FILTER, ARRAY_FIRST, ARRAY_MAP, - AVG_IF, DECIMAL_CAST, DIVIDE_DECIMAL, EQUALS, FLOAT_OR_DEFAULT, GREATER, GROUP_ARRAY_IF, HAS, + AVG_IF, DATE_PART, DECIMAL_CAST, DIVIDE_DECIMAL, EQUALS, FLOAT_OR_DEFAULT, GREATER, GROUP_ARRAY_IF, HAS, GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_HAS, JSON_KEYS_AND_VALUES, JSON_VALUE, IN_ARRAY, INT_DIV, LIKE, MAX_IF, MIN_IF, MULTIPLY, MULTIPLY_DECIMAL, MEDIAN_IF, REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, @@ -1338,6 +1338,7 @@ public enum FunctionRewrite ASSUME_NOT_NULL, AVG_IF, DATE_ADD, + DATE_PART, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 6605b5d8f125..e89a6e2caf3f 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -351,6 +351,17 @@ public void testCompilerLowersRemainingScalarAliases() "approx_percentile(score, 5E-1) FILTER (WHERE active) FROM records")); } + @Test + public void testCompilerLowersFinalStockAliases() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT floor(score), toDayOfMonth(timestamp), mapFromArrays(keys, values_array), " + + "date_part('hour', timestamp), isNull(value) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT floor(score), day(timestamp), map(keys, values_array), hour(timestamp), value IS NULL FROM records")); + } + @Test public void testCompilerEnforcesV0FunctionArities() { From a9a8b48d05037b09070a4d215e1edcbce2a29b44 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 00:18:07 -0400 Subject: [PATCH 095/136] feat(hogql): preserve windows on aggregate rewrites --- core/trino-hogql-compiler/README.md | 2 + .../hogql/compiler/HogQlFunctionResolver.java | 37 ++++++++++--------- .../compiler/HogQlV0FunctionRegistry.java | 2 +- .../catalog/HogQlSemanticCatalogSnapshot.java | 4 +- .../compiler/TestHogQlFunctionResolver.java | 12 ++++++ 5 files changed, 37 insertions(+), 20 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 80d6278011af..a311a880f7bc 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -65,6 +65,8 @@ The compatibility extension also lowers conditional aggregates (`countIf`, (`uniqExact`, `countDistinct`, and `groupUniqArray`), and `multiIf` directly to stock Trino aggregate filters, DISTINCT aggregates, and searched CASE expressions. +Aggregate rewrites preserve `OVER` specifications when the target Trino +aggregate is used as a window function. Literal-path JSON extraction lowers to Trino JSON paths with HogQL-compatible defaults. The compatibility profile also supports the corpus's typed map and key/value extraction forms through checked JSON-to-map casts. diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index d0f572d2c0db..933e0b22ba57 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -362,7 +362,7 @@ private Expression resolveFunction(FunctionCall function, boolean windowInvocati function.span()); } - private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List arguments) + private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List arguments) { HogQlQuery.SourceSpan span = function.span(); return switch (rewrite) { @@ -410,20 +410,20 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite case DATE_TRUNC_HOUR -> dateTrunc("hour", arguments.getFirst(), span); case DATE_TRUNC_MONTH -> dateTrunc("month", arguments.getFirst(), span); case DATE_TRUNC_WEEK -> dateTrunc("week", arguments.getFirst(), span); - case COUNT_IF -> aggregate("count", List.of(), false, arguments.getFirst(), span); - case SUM_IF -> aggregate("sum", List.of(arguments.getFirst()), false, arguments.get(1), span); - case MAX_IF -> aggregate("max", List.of(arguments.getFirst()), false, arguments.get(1), span); - case UNIQ_IF -> aggregate("approx_distinct", List.of(arguments.getFirst()), false, arguments.get(1), span); - case UNIQ_EXACT -> aggregate("count", List.of(arguments.getFirst()), true, null, span); - case GROUP_UNIQ_ARRAY -> aggregate("array_agg", List.of(arguments.getFirst()), true, null, span); - case ARG_MAX_IF -> aggregate("max_by", List.of(arguments.getFirst(), arguments.get(1)), false, arguments.get(2), span); - case ANY_IF -> aggregate("arbitrary", List.of(arguments.getFirst()), false, arguments.get(1), span); - case MIN_IF -> aggregate("min", List.of(arguments.getFirst()), false, arguments.get(1), span); - case AVG_IF -> aggregate("avg", List.of(arguments.getFirst()), false, arguments.get(1), span); - case GROUP_ARRAY_IF -> aggregate("array_agg", List.of(arguments.getFirst()), false, arguments.get(1), span); - case UNIQ_EXACT_IF -> aggregate("count", List.of(arguments.getFirst()), true, arguments.get(1), span); - case GROUP_UNIQ_ARRAY_IF -> aggregate("array_agg", List.of(arguments.getFirst()), true, arguments.get(1), span); - case COUNT_DISTINCT -> aggregate("count", List.of(arguments.getFirst()), true, null, span); + case COUNT_IF -> aggregate(function, "count", List.of(), false, arguments.getFirst(), span); + case SUM_IF -> aggregate(function, "sum", List.of(arguments.getFirst()), false, arguments.get(1), span); + case MAX_IF -> aggregate(function, "max", List.of(arguments.getFirst()), false, arguments.get(1), span); + case UNIQ_IF -> aggregate(function, "approx_distinct", List.of(arguments.getFirst()), false, arguments.get(1), span); + case UNIQ_EXACT -> aggregate(function, "count", List.of(arguments.getFirst()), true, null, span); + case GROUP_UNIQ_ARRAY -> aggregate(function, "array_agg", List.of(arguments.getFirst()), true, null, span); + case ARG_MAX_IF -> aggregate(function, "max_by", List.of(arguments.getFirst(), arguments.get(1)), false, arguments.get(2), span); + case ANY_IF -> aggregate(function, "arbitrary", List.of(arguments.getFirst()), false, arguments.get(1), span); + case MIN_IF -> aggregate(function, "min", List.of(arguments.getFirst()), false, arguments.get(1), span); + case AVG_IF -> aggregate(function, "avg", List.of(arguments.getFirst()), false, arguments.get(1), span); + case GROUP_ARRAY_IF -> aggregate(function, "array_agg", List.of(arguments.getFirst()), false, arguments.get(1), span); + case UNIQ_EXACT_IF -> aggregate(function, "count", List.of(arguments.getFirst()), true, arguments.get(1), span); + case GROUP_UNIQ_ARRAY_IF -> aggregate(function, "array_agg", List.of(arguments.getFirst()), true, arguments.get(1), span); + case COUNT_DISTINCT -> aggregate(function, "count", List.of(arguments.getFirst()), true, null, span); case MULTI_IF -> multiIf(function, arguments); case JSON_EXTRACT_STRING -> jsonExtractScalar(function, arguments, "varchar", new Literal(HogQlQuery.LiteralKind.STRING, "", span)); case JSON_EXTRACT_INT -> jsonExtractScalar(function, arguments, "bigint", new Literal(HogQlQuery.LiteralKind.INTEGER, "0", span)); @@ -476,6 +476,7 @@ private static Expression rewrite(FunctionCall function, FunctionRewrite rewrite case JSON_VALUE -> call("json_extract_scalar", arguments, span); case MD5 -> call("md5", List.of(call("to_utf8", List.of(cast(arguments.getFirst(), "varchar", span)), span)), span); case MEDIAN_IF -> aggregate( + function, "approx_percentile", List.of(arguments.getFirst(), new Literal(HogQlQuery.LiteralKind.FLOAT, "0.5", span)), false, @@ -624,14 +625,16 @@ private static FunctionCall dateAdd(String unit, List arguments, Hog span); } - private static FunctionCall aggregate(String name, List arguments, boolean distinct, Expression filter, HogQlQuery.SourceSpan span) + private FunctionCall aggregate(FunctionCall source, String name, List arguments, boolean distinct, Expression filter, HogQlQuery.SourceSpan span) { return new FunctionCall( - new Identifier(name, false, span), + List.of(new Identifier(name, false, span)), arguments, distinct, List.of(), Optional.ofNullable(filter), + Optional.empty(), + source.window().map(this::resolveWindow), span); } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 5a4c24d40dab..5c65deb47b91 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -242,7 +242,7 @@ private static FunctionCapabilityDefinition aggregateRewrite(String hogQlName, F false, false, false, - false); + true); } private static FunctionCapabilityDefinition aggregateWithOrderBy(String hogQlName, String trinoName, boolean supportsDistinct, FunctionSignature... signatures) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 5d360740c0e8..771559c7e446 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -264,8 +264,8 @@ private static Map validateFunctions(List< if (function.supportsFilter()) { throw new IllegalArgumentException("rewrite function cannot support FILTER"); } - if (function.supportsWindow()) { - throw new IllegalArgumentException("rewrite function cannot support window invocation"); + if (function.supportsWindow() && rewriteFunctionKind(rewrite) != FunctionKind.AGGREGATE) { + throw new IllegalArgumentException("scalar rewrite function cannot support window invocation"); } if (function.signatures().stream().anyMatch(signature -> !validRewriteSignature(rewrite, signature))) { throw new IllegalArgumentException("rewrite function declares an invalid signature"); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index e89a6e2caf3f..b9882ccd054a 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -272,6 +272,18 @@ public void testCompilerLowersAggregateCombinators() "array_agg(DISTINCT value) FILTER (WHERE active), count(DISTINCT value) FROM records")); } + @Test + public void testCompilerPreservesWindowsOnAggregateRewrites() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT countIf(active) OVER (PARTITION BY account_id), " + + "countDistinct(value) OVER (PARTITION BY account_id) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT count(*) FILTER (WHERE active) OVER (PARTITION BY account_id), " + + "count(DISTINCT value) OVER (PARTITION BY account_id) FROM records")); + } + @Test public void testCompilerLowersNumericConversions() { From e5f4d43a73e6b7bd970885d6664db7321aadcc2e Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 00:22:13 -0400 Subject: [PATCH 096/136] feat(hogql): lower array join expressions --- core/trino-hogql-compiler/README.md | 2 + .../hogql/compiler/HogQlFunctionResolver.java | 83 +++++++++++++++---- .../compiler/TestHogQlFunctionResolver.java | 23 +++++ 3 files changed, 92 insertions(+), 16 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index a311a880f7bc..310946acffdd 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -85,6 +85,8 @@ The extended profile also lowers extract-all and first-match replacement, plus boolean, unsigned-integer, raw-array, and nested raw-map JSON extraction. Common array transforms, predicates, indexing, summation, ranges, tuple indexing, splitting, and membership lower to Trino higher-order functions. +The row-expanding `arrayJoin` function is hoisted to a generated lateral +`CROSS JOIN UNNEST`; repeated calls remain independent row expansions. Range and membership rewrites preserve empty-range and null-search behavior. Typed defaults for out-of-range and no-match element lookups require resolved element types and remain outside the frozen compatibility subset. diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 933e0b22ba57..35fb929b1cec 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -93,6 +93,7 @@ final class HogQlFunctionResolver private final Map functions; private final boolean semanticCatalogAvailable; + private ArrayList activeArrayJoins; private HogQlFunctionResolver(Optional snapshot) { @@ -132,17 +133,13 @@ private HogQlQuery resolveQuery(HogQlQuery query) resolveQuery(commonTable.query()), commonTable.span())) .toList(); - HogQlQuery.QueryBody body = switch (query.body()) { - case SelectQueryBody select -> new SelectQueryBody( - select.distinct(), - select.projections().stream().map(this::resolveProjection).toList(), - select.from().map(this::resolveRelation), - select.where().map(this::resolveExpression), - select.groupBy().stream().map(this::resolveExpression).toList(), - select.having().map(this::resolveExpression), - select.windows().stream().map(this::resolveWindowDefinition).toList(), - select.span()); - case SetOperation set -> new SetOperation( + if (query.body() instanceof SelectQueryBody select) { + return resolveSelectQuery(query, commonTables, select); + } + SetOperation set = (SetOperation) query.body(); + return new HogQlQuery( + commonTables, + new SetOperation( set.type(), set.distinct(), resolveQuery(set.left()), @@ -150,17 +147,45 @@ private HogQlQuery resolveQuery(HogQlQuery query) set.leftParenthesized(), set.rightParenthesized(), set.operatorSpan(), - set.span()); - }; - return new HogQlQuery( - commonTables, - body, + set.span()), resolveSortItems(query.orderBy()), query.limit().map(this::resolveExpression), query.offset().map(this::resolveExpression), query.span()); } + private HogQlQuery resolveSelectQuery(HogQlQuery query, List commonTables, SelectQueryBody select) + { + ArrayList parentArrayJoins = activeArrayJoins; + activeArrayJoins = new ArrayList<>(); + try { + Optional from = select.from().map(this::resolveRelation); + List projections = select.projections().stream().map(this::resolveProjection).toList(); + Optional where = select.where().map(this::resolveExpression); + List groupBy = select.groupBy().stream().map(this::resolveExpression).toList(); + Optional having = select.having().map(this::resolveExpression); + List windows = select.windows().stream().map(this::resolveWindowDefinition).toList(); + List orderBy = resolveSortItems(query.orderBy()); + Optional limit = query.limit().map(this::resolveExpression); + Optional offset = query.offset().map(this::resolveExpression); + for (UnnestRelation unnest : activeArrayJoins) { + from = Optional.of(from + .map(left -> new JoinRelation(HogQlQuery.JoinType.CROSS, left, unnest, Optional.empty(), unnest.span())) + .orElse(unnest)); + } + return new HogQlQuery( + commonTables, + new SelectQueryBody(select.distinct(), projections, from, where, groupBy, having, windows, select.span()), + orderBy, + limit, + offset, + query.span()); + } + finally { + activeArrayJoins = parentArrayJoins; + } + } + private Projection resolveProjection(Projection projection) { return switch (projection) { @@ -304,6 +329,9 @@ private Expression resolveFunction(FunctionCall function, boolean windowInvocati if (function.nameParts().size() != 1) { throw unsupportedError(function, "Qualified HogQL functions are not supported"); } + if (canonical(name).equals("arrayjoin")) { + return resolveArrayJoin(function); + } if (semanticCatalogAvailable && function.nameParts().size() == 1 && canonical(name).equals(MATCHES_ACTION)) { return new FunctionCall( function.nameParts(), @@ -362,6 +390,29 @@ private Expression resolveFunction(FunctionCall function, boolean windowInvocati function.span()); } + private Expression resolveArrayJoin(FunctionCall function) + { + if (function.arguments().size() != 1) { + throw resolutionError(function, "HogQL function " + function.name().value() + " does not accept " + function.arguments().size() + " arguments"); + } + if (function.distinct() || !function.orderBy().isEmpty() || function.filter().isPresent() || + function.nullTreatment().isPresent() || function.window().isPresent()) { + throw unsupportedError(function, "HogQL arrayJoin does not accept function modifiers"); + } + if (activeArrayJoins == null) { + throw unsupportedError(function, "HogQL arrayJoin requires a SELECT query context"); + } + int index = activeArrayJoins.size(); + Identifier alias = new Identifier("__hogql_array_join_" + index, false, function.span()); + Identifier column = new Identifier("__hogql_value_" + index, false, function.span()); + activeArrayJoins.add(new UnnestRelation( + List.of(resolveExpression(function.arguments().getFirst())), + alias, + List.of(column), + function.span())); + return new ColumnReference(List.of(alias, column), function.span()); + } + private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List arguments) { HogQlQuery.SourceSpan span = function.span(); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index b9882ccd054a..0a8ecaee0570 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -284,6 +284,29 @@ public void testCompilerPreservesWindowsOnAggregateRewrites() "count(DISTINCT value) OVER (PARTITION BY account_id) FROM records")); } + @Test + public void testCompilerHoistsArrayJoinToCrossJoinUnnest() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT arrayJoin(items) AS item, upper(arrayJoin(names)) FROM records " + + "WHERE arrayJoin(flags) ORDER BY arrayJoin(scores)")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT __hogql_array_join_0.__hogql_value_0 AS item, upper(__hogql_array_join_1.__hogql_value_1) " + + "FROM records " + + "CROSS JOIN UNNEST(items) AS __hogql_array_join_0 (__hogql_value_0) " + + "CROSS JOIN UNNEST(names) AS __hogql_array_join_1 (__hogql_value_1) " + + "CROSS JOIN UNNEST(flags) AS __hogql_array_join_2 (__hogql_value_2) " + + "CROSS JOIN UNNEST(scores) AS __hogql_array_join_3 (__hogql_value_3) " + + "WHERE __hogql_array_join_2.__hogql_value_2 " + + "ORDER BY __hogql_array_join_3.__hogql_value_3")); + + assertThat(new HogQlCompiler().compile(envelope("SELECT arrayJoin([1, 2])")).statement()) + .isEqualTo(sqlParser.createStatement( + "SELECT __hogql_array_join_0.__hogql_value_0 " + + "FROM UNNEST(ARRAY[1, 2]) AS __hogql_array_join_0 (__hogql_value_0)")); + } + @Test public void testCompilerLowersNumericConversions() { From ed2bb842d91877333a7fa0ad64f03bcc1d494210 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 00:27:41 -0400 Subject: [PATCH 097/136] feat(hogql): lower final stock compatibility aliases --- core/trino-hogql-compiler/README.md | 4 +++- .../hogql/compiler/HogQlFunctionResolver.java | 16 +++++++++++----- .../compiler/HogQlV0FunctionRegistry.java | 6 +++++- .../catalog/HogQlSemanticCatalogSnapshot.java | 10 +++++++--- .../compiler/TestHogQlFunctionResolver.java | 19 +++++++++++++++++-- 5 files changed, 43 insertions(+), 12 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 310946acffdd..b25883e737d1 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -97,7 +97,9 @@ Additional scalar coverage includes array slicing/enumeration, UUID and integer conversion, JSON existence/value/serialization, hashing, powers, UTF-8 substrings, banker-rounding aliases, and conditional median aggregation. Simple floor, calendar-part, map construction, and null-predicate aliases are -mapped to their stock equivalents. +mapped to their stock equivalents. Two-bound ranges, subtraction aliases, +calendar-month extraction, ceiling aliases, and JSON object-key extraction +also lower to stock expressions. Function aliases are resolved before Trino analysis. Unknown functions fail as HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 35fb929b1cec..74f3d61a199f 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -433,7 +433,7 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< span); case ARRAY_MAP -> call("transform", List.of(arguments.get(1), arguments.getFirst()), span); case ARRAY_SUM -> arraySum(arguments.getFirst(), span); - case RANGE -> range(arguments.getFirst(), span); + case RANGE -> range(arguments, span); case TUPLE_ELEMENT -> new SubscriptExpression(arguments.getFirst(), arguments.get(1), span); case SPLIT_CHAR -> call("split", List.of(arguments.get(1), arguments.getFirst()), span); case HAS -> has(arguments, span); @@ -441,6 +441,7 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< case EMPTY -> empty(arguments.getFirst(), true, span); case NOT_EMPTY -> empty(arguments.getFirst(), false, span); case EQUALS -> new BinaryExpression(HogQlQuery.BinaryOperator.EQUAL, arguments.getFirst(), arguments.get(1), span); + case MINUS -> new BinaryExpression(HogQlQuery.BinaryOperator.SUBTRACT, arguments.getFirst(), arguments.get(1), span); case MULTIPLY, MULTIPLY_DECIMAL -> new BinaryExpression(HogQlQuery.BinaryOperator.MULTIPLY, arguments.getFirst(), arguments.get(1), span); case DIVIDE_DECIMAL -> new BinaryExpression(HogQlQuery.BinaryOperator.DIVIDE, arguments.getFirst(), arguments.get(1), span); case IN_ARRAY -> call("contains", List.of(arguments.get(1), arguments.getFirst()), span); @@ -482,6 +483,10 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< case JSON_EXTRACT_BOOL -> jsonExtractScalar(function, arguments, "boolean", new Literal(HogQlQuery.LiteralKind.BOOLEAN, "false", span)); case JSON_EXTRACT_UINT -> jsonExtractScalar(function, arguments, "bigint", integerLiteral("0", span)); case JSON_EXTRACT_ARRAY_RAW -> jsonExtractArrayRaw(function, arguments); + case JSON_EXTRACT_KEYS -> call( + "map_keys", + List.of(typedJsonMapFromJson(jsonValue(function, arguments), "json", span)), + span); case JSON_EXTRACT_RAW -> coalesce( call("json_format", List.of(call("json_extract", List.of(arguments.getFirst(), jsonPath(function, arguments.subList(1, arguments.size()))), span)), span), new Literal(HogQlQuery.LiteralKind.STRING, "", span), @@ -918,14 +923,15 @@ private static Literal integerLiteral(String value, HogQlQuery.SourceSpan span) return new Literal(HogQlQuery.LiteralKind.INTEGER, value, span); } - private static Expression range(Expression end, HogQlQuery.SourceSpan span) + private static Expression range(List arguments, HogQlQuery.SourceSpan span) { - Literal zero = integerLiteral("0", span); - Expression isEmpty = new BinaryExpression(HogQlQuery.BinaryOperator.LESS_THAN_OR_EQUAL, end, zero, span); + Expression start = arguments.size() == 1 ? integerLiteral("0", span) : arguments.getFirst(); + Expression end = arguments.getLast(); + Expression isEmpty = new BinaryExpression(HogQlQuery.BinaryOperator.LESS_THAN_OR_EQUAL, end, start, span); Expression empty = cast(new ArrayExpression(List.of(), span), "array(bigint)", span); Expression sequence = call( "sequence", - List.of(zero, new BinaryExpression(HogQlQuery.BinaryOperator.SUBTRACT, end, integerLiteral("1", span), span)), + List.of(start, new BinaryExpression(HogQlQuery.BinaryOperator.SUBTRACT, end, integerLiteral("1", span), span)), span); return call("if", List.of(isEmpty, empty, sequence), span); } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 5c65deb47b91..161190d072b6 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -56,7 +56,7 @@ final class HogQlV0FunctionRegistry rewrite("arraySum", FunctionRewrite.ARRAY_SUM, "any", signature(1)), rewrite("arraySlice", FunctionRewrite.ARRAY_SLICE, "array(any)", signature(3)), rewrite("arrayEnumerate", FunctionRewrite.ARRAY_ENUMERATE, "array(bigint)", signature(1)), - rewrite("range", FunctionRewrite.RANGE, "array(bigint)", signature(1)), + rewrite("range", FunctionRewrite.RANGE, "array(bigint)", signature(1), signature(2)), rewrite("tupleElement", FunctionRewrite.TUPLE_ELEMENT, "any", signature(2)), rewrite("splitByChar", FunctionRewrite.SPLIT_CHAR, "array(varchar)", signature(2)), rewrite("has", FunctionRewrite.HAS, "boolean", signature(2)), @@ -77,9 +77,11 @@ final class HogQlV0FunctionRegistry rewrite("toUUID", FunctionRewrite.CAST_UUID, "uuid", signature(1)), rewrite("toJSONString", FunctionRewrite.TO_JSON_STRING, "varchar", signature(1)), rewrite("JSONHas", FunctionRewrite.JSON_HAS, "boolean", signature(2)), + rewrite("JSONExtractKeys", FunctionRewrite.JSON_EXTRACT_KEYS, "array(varchar)", signature(1), variadicSignature(2)), rewrite("JSON_VALUE", FunctionRewrite.JSON_VALUE, "varchar", signature(2)), rewrite("md5", FunctionRewrite.MD5, "varbinary", signature(1)), rewrite("date_part", FunctionRewrite.DATE_PART, "bigint", signature(2)), + rewrite("minus", FunctionRewrite.MINUS, "any", signature(2)), rewrite("splitByString", FunctionRewrite.SPLIT_STRING, "array(varchar)", signature(2)), rewrite("toString", FunctionRewrite.CAST_VARCHAR, "varchar", signature(1)), rewrite("toDate", FunctionRewrite.CAST_DATE, "date", signature(1)), @@ -128,6 +130,8 @@ final class HogQlV0FunctionRegistry scalar("floor", "floor", signature(1)), scalar("toDayOfMonth", "day", signature(1)), scalar("mapFromArrays", "map", signature(2)), + scalar("toMonth", "month", signature(1)), + scalar("ceil", "ceiling", signature(1)), scalar("pow", "power", signature(2)), scalar("substringUTF8", "substring", signature(2), signature(3)), scalar("arrayConcat", "concat", variadicSignature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 771559c7e446..e86c76f542e2 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -302,7 +302,7 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi return switch (rewrite) { case ARRAY_ENUMERATE, ARRAY_SUM, ASSUME_NOT_NULL, CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_UUID, CAST_VARCHAR, EMPTY, FLOAT_OR_ZERO, INTERVAL_MONTH, INT_OR_ZERO, - MD5, NOT_EMPTY, RANGE, START_WEEK, TO_JSON_STRING, + MD5, NOT_EMPTY, START_WEEK, TO_JSON_STRING, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, GROUP_UNIQ_ARRAY, INTERVAL_DAY, IS_NOT_NULL, IS_NULL, COUNT_IF, NOT, PARSE_TIMESTAMP, TO_UNIX_TIMESTAMP, @@ -312,7 +312,7 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi AVG_IF, DATE_PART, DECIMAL_CAST, DIVIDE_DECIMAL, EQUALS, FLOAT_OR_DEFAULT, GREATER, GROUP_ARRAY_IF, HAS, GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_HAS, JSON_KEYS_AND_VALUES, JSON_VALUE, IN_ARRAY, INT_DIV, LIKE, MAX_IF, MIN_IF, MULTIPLY, MULTIPLY_DECIMAL, - MEDIAN_IF, REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, + MEDIAN_IF, MINUS, REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, SUBTRACT_MONTHS, SUBTRACT_YEARS, SUM_IF, TUPLE_ELEMENT, UNIQ_EXACT_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; @@ -324,11 +324,13 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi (signature.argumentTypes().size() == 2 || signature.argumentTypes().size() == 3); case JSON_EXTRACT_BOOL, JSON_EXTRACT_FLOAT, JSON_EXTRACT_INT, JSON_EXTRACT_RAW, JSON_EXTRACT_UINT -> signature.variadic() && signature.argumentTypes().size() == 3; - case JSON_EXTRACT_ARRAY_RAW, JSON_EXTRACT_STRING, JSON_KEYS_AND_VALUES_RAW, JSON_LENGTH -> + case JSON_EXTRACT_ARRAY_RAW, JSON_EXTRACT_KEYS, JSON_EXTRACT_STRING, JSON_KEYS_AND_VALUES_RAW, JSON_LENGTH -> (!signature.variadic() && signature.argumentTypes().size() == 1) || (signature.variadic() && signature.argumentTypes().size() == 3); case AND -> (!signature.variadic() && signature.argumentTypes().size() == 2) || (signature.variadic() && signature.argumentTypes().size() == 3); + case RANGE -> !signature.variadic() && + (signature.argumentTypes().size() == 1 || signature.argumentTypes().size() == 2); case MULTI_IF -> signature.variadic() && signature.argumentTypes().size() == 4; case TUPLE -> signature.variadic() && signature.argumentTypes().size() == 2; case TODAY -> !signature.variadic() && signature.argumentTypes().isEmpty(); @@ -1367,6 +1369,7 @@ public enum FunctionRewrite JSON_EXTRACT_BOOL, JSON_EXTRACT_FLOAT, JSON_EXTRACT_INT, + JSON_EXTRACT_KEYS, JSON_EXTRACT_RAW, JSON_EXTRACT_STRING, JSON_EXTRACT_TYPED, @@ -1381,6 +1384,7 @@ public enum FunctionRewrite MD5, MEDIAN_IF, MIN_IF, + MINUS, MULTIPLY, MULTIPLY_DECIMAL, MULTI_IF, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 0a8ecaee0570..511dba2c165d 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -335,18 +335,33 @@ public void testCompilerLowersArrayCompatibilityFunctions() { HogQlCompilationResult result = new HogQlCompiler().compile(envelope( "SELECT arrayElement(items, -1), arrayFilter(x -> x > 0, items), arrayFirst(x -> x > 0, items), " + - "arrayMap(x -> x + 1, items), arraySum(items), range(3), range(0), tupleElement(item, 2), " + + "arrayMap(x -> x + 1, items), arraySum(items), range(3), range(0), range(2, 5), tupleElement(item, 2), " + "splitByChar(',', text), has(items, 3), has(items, NULL) FROM records")); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( "SELECT element_at(items, -1), filter(items, x -> x > 0), element_at(filter(items, x -> x > 0), 1), " + "transform(items, x -> x + 1), reduce(items, 0, (_hogql_sum, _hogql_item) -> _hogql_sum + _hogql_item, _hogql_sum -> _hogql_sum), " + "if(3 <= 0, CAST(ARRAY[] AS array(bigint)), sequence(0, 3 - 1)), " + - "if(0 <= 0, CAST(ARRAY[] AS array(bigint)), sequence(0, 0 - 1)), item[2], split(text, ','), " + + "if(0 <= 0, CAST(ARRAY[] AS array(bigint)), sequence(0, 0 - 1)), " + + "if(5 <= 2, CAST(ARRAY[] AS array(bigint)), sequence(2, 5 - 1)), item[2], split(text, ','), " + "if(3 IS NULL, any_match(items, _hogql_item -> _hogql_item IS NULL), coalesce(contains(items, 3), false)), " + "if(NULL IS NULL, any_match(items, _hogql_item -> _hogql_item IS NULL), coalesce(contains(items, NULL), false)) FROM records")); } + @Test + public void testCompilerLowersFinalStockCompatibilityFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT minus(total, discount), toMonth(timestamp), ceil(score), " + + "JSONExtractKeys(payload), JSONExtractKeys(payload, 'nested') FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT total - discount, month(timestamp), ceiling(score), " + + "map_keys(coalesce(TRY_CAST(json_parse(payload) AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))), " + + "map_keys(coalesce(TRY_CAST(json_extract(payload, '$[\"nested\"]') AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))) " + + "FROM records")); + } + @Test public void testCompilerLowersExpressionAndDateAliases() { From 80ec328a8edc9a2de1eba99b77ca9fe3f8cfc9f0 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 00:29:55 -0400 Subject: [PATCH 098/136] feat(hogql): lower exposed scalar aliases --- core/trino-hogql-compiler/README.md | 6 +++--- .../java/io/trino/hogql/compiler/HogQlFunctionResolver.java | 2 ++ .../io/trino/hogql/compiler/HogQlV0FunctionRegistry.java | 3 +++ .../compiler/catalog/HogQlSemanticCatalogSnapshot.java | 6 ++++-- .../io/trino/hogql/compiler/TestHogQlFunctionResolver.java | 4 ++-- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index b25883e737d1..25527ed879bc 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -97,9 +97,9 @@ Additional scalar coverage includes array slicing/enumeration, UUID and integer conversion, JSON existence/value/serialization, hashing, powers, UTF-8 substrings, banker-rounding aliases, and conditional median aggregation. Simple floor, calendar-part, map construction, and null-predicate aliases are -mapped to their stock equivalents. Two-bound ranges, subtraction aliases, -calendar-month extraction, ceiling aliases, and JSON object-key extraction -also lower to stock expressions. +mapped to their stock equivalents. Two-bound ranges, comparison and +subtraction aliases, calendar month/year extraction, ceiling aliases, narrow +integer casts, and JSON object-key extraction also lower to stock expressions. Function aliases are resolved before Trino analysis. Unknown functions fail as HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 74f3d61a199f..26d48a9aa000 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -421,6 +421,7 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< case IS_NOT_NULL -> new IsNullExpression(arguments.getFirst(), true, span, span); case CAST_DATE -> cast(arguments.getFirst(), "date", span); case CAST_DOUBLE -> cast(arguments.getFirst(), "double", span); + case CAST_SMALLINT -> cast(arguments.getFirst(), "smallint", span); case FLOAT_OR_ZERO -> coalesce(tryCast(arguments.getFirst(), "double", span), new Literal(HogQlQuery.LiteralKind.FLOAT, "0.0", span), span); case FLOAT_OR_DEFAULT -> coalesce(tryCast(arguments.getFirst(), "double", span), cast(arguments.get(1), "double", span), span); case DECIMAL_CAST -> tryCast(arguments.getFirst(), decimalType(function, arguments.get(1)), span); @@ -442,6 +443,7 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< case NOT_EMPTY -> empty(arguments.getFirst(), false, span); case EQUALS -> new BinaryExpression(HogQlQuery.BinaryOperator.EQUAL, arguments.getFirst(), arguments.get(1), span); case MINUS -> new BinaryExpression(HogQlQuery.BinaryOperator.SUBTRACT, arguments.getFirst(), arguments.get(1), span); + case NOT_EQUALS -> new BinaryExpression(HogQlQuery.BinaryOperator.NOT_EQUAL, arguments.getFirst(), arguments.get(1), span); case MULTIPLY, MULTIPLY_DECIMAL -> new BinaryExpression(HogQlQuery.BinaryOperator.MULTIPLY, arguments.getFirst(), arguments.get(1), span); case DIVIDE_DECIMAL -> new BinaryExpression(HogQlQuery.BinaryOperator.DIVIDE, arguments.getFirst(), arguments.get(1), span); case IN_ARRAY -> call("contains", List.of(arguments.get(1), arguments.getFirst()), span); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 161190d072b6..2060877f110c 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -74,6 +74,7 @@ final class HogQlV0FunctionRegistry rewrite("toStartOfWeek", FunctionRewrite.START_WEEK, "timestamp", signature(1)), rewrite("subtractYears", FunctionRewrite.SUBTRACT_YEARS, "any", signature(2)), rewrite("toIntOrZero", FunctionRewrite.INT_OR_ZERO, "bigint", signature(1)), + rewrite("_toInt16", FunctionRewrite.CAST_SMALLINT, "smallint", signature(1)), rewrite("toUUID", FunctionRewrite.CAST_UUID, "uuid", signature(1)), rewrite("toJSONString", FunctionRewrite.TO_JSON_STRING, "varchar", signature(1)), rewrite("JSONHas", FunctionRewrite.JSON_HAS, "boolean", signature(2)), @@ -82,6 +83,7 @@ final class HogQlV0FunctionRegistry rewrite("md5", FunctionRewrite.MD5, "varbinary", signature(1)), rewrite("date_part", FunctionRewrite.DATE_PART, "bigint", signature(2)), rewrite("minus", FunctionRewrite.MINUS, "any", signature(2)), + rewrite("notEquals", FunctionRewrite.NOT_EQUALS, "boolean", signature(2)), rewrite("splitByString", FunctionRewrite.SPLIT_STRING, "array(varchar)", signature(2)), rewrite("toString", FunctionRewrite.CAST_VARCHAR, "varchar", signature(1)), rewrite("toDate", FunctionRewrite.CAST_DATE, "date", signature(1)), @@ -131,6 +133,7 @@ final class HogQlV0FunctionRegistry scalar("toDayOfMonth", "day", signature(1)), scalar("mapFromArrays", "map", signature(2)), scalar("toMonth", "month", signature(1)), + scalar("toYear", "year", signature(1)), scalar("ceil", "ceiling", signature(1)), scalar("pow", "power", signature(2)), scalar("substringUTF8", "substring", signature(2), signature(3)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index e86c76f542e2..96bf3b38da73 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -300,7 +300,7 @@ private static FunctionKind rewriteFunctionKind(FunctionRewrite rewrite) private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSignature signature) { return switch (rewrite) { - case ARRAY_ENUMERATE, ARRAY_SUM, ASSUME_NOT_NULL, CAST_BIGINT, CAST_DATE, CAST_DOUBLE, + case ARRAY_ENUMERATE, ARRAY_SUM, ASSUME_NOT_NULL, CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_SMALLINT, CAST_UUID, CAST_VARCHAR, EMPTY, FLOAT_OR_ZERO, INTERVAL_MONTH, INT_OR_ZERO, MD5, NOT_EMPTY, START_WEEK, TO_JSON_STRING, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, @@ -312,7 +312,7 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi AVG_IF, DATE_PART, DECIMAL_CAST, DIVIDE_DECIMAL, EQUALS, FLOAT_OR_DEFAULT, GREATER, GROUP_ARRAY_IF, HAS, GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_HAS, JSON_KEYS_AND_VALUES, JSON_VALUE, IN_ARRAY, INT_DIV, LIKE, MAX_IF, MIN_IF, MULTIPLY, MULTIPLY_DECIMAL, - MEDIAN_IF, MINUS, REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, + MEDIAN_IF, MINUS, NOT_EQUALS, REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, SUBTRACT_MONTHS, SUBTRACT_YEARS, SUM_IF, TUPLE_ELEMENT, UNIQ_EXACT_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; @@ -1321,6 +1321,7 @@ public enum FunctionRewrite { CAST_DATE, CAST_DOUBLE, + CAST_SMALLINT, CAST_BIGINT, CAST_TIMESTAMP, CAST_UUID, @@ -1389,6 +1390,7 @@ public enum FunctionRewrite MULTIPLY_DECIMAL, MULTI_IF, NOT, + NOT_EQUALS, NOT_EMPTY, PARSE_TIMESTAMP, REGEX_EXTRACT, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 511dba2c165d..2edc58720330 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -352,11 +352,11 @@ public void testCompilerLowersArrayCompatibilityFunctions() public void testCompilerLowersFinalStockCompatibilityFunctions() { HogQlCompilationResult result = new HogQlCompiler().compile(envelope( - "SELECT minus(total, discount), toMonth(timestamp), ceil(score), " + + "SELECT minus(total, discount), notEquals(status, 'ignored'), toMonth(timestamp), toYear(timestamp), ceil(score), _toInt16(value), " + "JSONExtractKeys(payload), JSONExtractKeys(payload, 'nested') FROM records")); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( - "SELECT total - discount, month(timestamp), ceiling(score), " + + "SELECT total - discount, status <> 'ignored', month(timestamp), year(timestamp), ceiling(score), CAST(value AS smallint), " + "map_keys(coalesce(TRY_CAST(json_parse(payload) AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))), " + "map_keys(coalesce(TRY_CAST(json_extract(payload, '$[\"nested\"]') AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))) " + "FROM records")); From ff567cd54b5a8ba3ce82304b04072d8eeb5bcf8a Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 00:31:33 -0400 Subject: [PATCH 099/136] feat(hogql): lower division alias --- .../java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java | 1 + .../io/trino/hogql/compiler/TestHogQlFunctionResolver.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 2060877f110c..8bacd00a6f74 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -67,6 +67,7 @@ final class HogQlV0FunctionRegistry rewrite("multiply", FunctionRewrite.MULTIPLY, "any", signature(2)), rewrite("multiplyDecimal", FunctionRewrite.MULTIPLY_DECIMAL, "decimal", signature(2)), rewrite("divideDecimal", FunctionRewrite.DIVIDE_DECIMAL, "decimal", signature(2)), + rewrite("divide", FunctionRewrite.DIVIDE_DECIMAL, "any", signature(2)), rewrite("in", FunctionRewrite.IN_ARRAY, "boolean", signature(2)), rewrite("tuple", FunctionRewrite.TUPLE, "row", variadicSignature(1)), rewrite("subtractMonths", FunctionRewrite.SUBTRACT_MONTHS, "any", signature(2)), diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 2edc58720330..0108ec9045b3 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -352,11 +352,11 @@ public void testCompilerLowersArrayCompatibilityFunctions() public void testCompilerLowersFinalStockCompatibilityFunctions() { HogQlCompilationResult result = new HogQlCompiler().compile(envelope( - "SELECT minus(total, discount), notEquals(status, 'ignored'), toMonth(timestamp), toYear(timestamp), ceil(score), _toInt16(value), " + + "SELECT minus(total, discount), divide(total, count), notEquals(status, 'ignored'), toMonth(timestamp), toYear(timestamp), ceil(score), _toInt16(value), " + "JSONExtractKeys(payload), JSONExtractKeys(payload, 'nested') FROM records")); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( - "SELECT total - discount, status <> 'ignored', month(timestamp), year(timestamp), ceiling(score), CAST(value AS smallint), " + + "SELECT total - discount, total / count, status <> 'ignored', month(timestamp), year(timestamp), ceiling(score), CAST(value AS smallint), " + "map_keys(coalesce(TRY_CAST(json_parse(payload) AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))), " + "map_keys(coalesce(TRY_CAST(json_extract(payload, '$[\"nested\"]') AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))) " + "FROM records")); From 32210f43d5fcee82728f156620fd5a065506ba9d Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 00:38:21 -0400 Subject: [PATCH 100/136] feat(hogql): lower static survey responses --- core/trino-hogql-compiler/README.md | 2 + .../hogql/compiler/HogQlFunctionResolver.java | 41 +++++++++++++++++++ .../compiler/HogQlV0FunctionRegistry.java | 1 + .../catalog/HogQlSemanticCatalogSnapshot.java | 3 +- .../compiler/TestHogQlFunctionResolver.java | 23 +++++++++++ 5 files changed, 69 insertions(+), 1 deletion(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 25527ed879bc..f862df27c70f 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -100,6 +100,8 @@ Simple floor, calendar-part, map construction, and null-predicate aliases are mapped to their stock equivalents. Two-bound ranges, comparison and subtraction aliases, calendar month/year extraction, ceiling aliases, narrow integer casts, and JSON object-key extraction also lower to stock expressions. +The static-ID, single-choice `getSurveyResponse(index, question_id)` macro +prefers the modern response property and falls back to the legacy index key. Function aliases are resolved before Trino analysis. Unknown functions fail as HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 26d48a9aa000..5532f1882965 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -532,6 +532,7 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< span, span); case JSON_VALUE -> call("json_extract_scalar", arguments, span); + case SURVEY_RESPONSE -> surveyResponse(function, arguments); case MD5 -> call("md5", List.of(call("to_utf8", List.of(cast(arguments.getFirst(), "varchar", span)), span)), span); case MEDIAN_IF -> aggregate( function, @@ -721,6 +722,41 @@ private static Expression jsonExtractScalar(FunctionCall function, List arguments) + { + HogQlQuery.SourceSpan span = function.span(); + int questionIndex = surveyQuestionIndex(function, arguments.getFirst()); + if (!(arguments.get(1) instanceof Literal questionId) || questionId.kind() != HogQlQuery.LiteralKind.STRING || questionId.value().isEmpty()) { + throw unsupportedError(function, "HogQL getSurveyResponse question ID must be a non-empty string literal"); + } + Expression properties = new ColumnReference(List.of(new Identifier("properties", false, span)), span); + Literal empty = new Literal(HogQlQuery.LiteralKind.STRING, "", span); + Expression idResponse = call( + "nullif", + List.of(jsonExtractScalar(function, List.of(properties, stringLiteral("$survey_response_" + questionId.value(), span)), "varchar", empty), empty), + span); + String indexKey = questionIndex == 0 ? "$survey_response" : "$survey_response_" + questionIndex; + Expression indexResponse = call( + "nullif", + List.of(jsonExtractScalar(function, List.of(properties, stringLiteral(indexKey, span)), "varchar", empty), empty), + span); + return call("coalesce", List.of(idResponse, indexResponse), span); + } + + private static int surveyQuestionIndex(FunctionCall function, Expression expression) + { + if (!(expression instanceof Literal literal) || + (literal.kind() != HogQlQuery.LiteralKind.INTEGER && literal.kind() != HogQlQuery.LiteralKind.STRING)) { + throw unsupportedError(function, "HogQL getSurveyResponse question index must be an integer literal"); + } + try { + return Integer.parseInt(literal.value()); + } + catch (NumberFormatException _) { + throw resolutionError(function, "HogQL getSurveyResponse question index is outside the supported range"); + } + } + private static Expression jsonExtractTyped(FunctionCall function, List arguments) { String type = jsonType(function, arguments.get(1)); @@ -925,6 +961,11 @@ private static Literal integerLiteral(String value, HogQlQuery.SourceSpan span) return new Literal(HogQlQuery.LiteralKind.INTEGER, value, span); } + private static Literal stringLiteral(String value, HogQlQuery.SourceSpan span) + { + return new Literal(HogQlQuery.LiteralKind.STRING, value, span); + } + private static Expression range(List arguments, HogQlQuery.SourceSpan span) { Expression start = arguments.size() == 1 ? integerLiteral("0", span) : arguments.getFirst(); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 8bacd00a6f74..ba34c71f897a 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -81,6 +81,7 @@ final class HogQlV0FunctionRegistry rewrite("JSONHas", FunctionRewrite.JSON_HAS, "boolean", signature(2)), rewrite("JSONExtractKeys", FunctionRewrite.JSON_EXTRACT_KEYS, "array(varchar)", signature(1), variadicSignature(2)), rewrite("JSON_VALUE", FunctionRewrite.JSON_VALUE, "varchar", signature(2)), + rewrite("getSurveyResponse", FunctionRewrite.SURVEY_RESPONSE, "varchar", signature(2)), rewrite("md5", FunctionRewrite.MD5, "varbinary", signature(1)), rewrite("date_part", FunctionRewrite.DATE_PART, "bigint", signature(2)), rewrite("minus", FunctionRewrite.MINUS, "any", signature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 96bf3b38da73..83ee1ce92ff5 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -312,7 +312,7 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi AVG_IF, DATE_PART, DECIMAL_CAST, DIVIDE_DECIMAL, EQUALS, FLOAT_OR_DEFAULT, GREATER, GROUP_ARRAY_IF, HAS, GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_HAS, JSON_KEYS_AND_VALUES, JSON_VALUE, IN_ARRAY, INT_DIV, LIKE, MAX_IF, MIN_IF, MULTIPLY, MULTIPLY_DECIMAL, - MEDIAN_IF, MINUS, NOT_EQUALS, REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, + MEDIAN_IF, MINUS, NOT_EQUALS, REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, SURVEY_RESPONSE, SUBTRACT_MONTHS, SUBTRACT_YEARS, SUM_IF, TUPLE_ELEMENT, UNIQ_EXACT_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; @@ -1404,6 +1404,7 @@ public enum FunctionRewrite SUBTRACT_MONTHS, SUBTRACT_YEARS, SUM_IF, + SURVEY_RESPONSE, TODAY, TO_JSON_STRING, TO_UNIX_TIMESTAMP, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 0108ec9045b3..26555d202115 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -362,6 +362,29 @@ public void testCompilerLowersFinalStockCompatibilityFunctions() "FROM records")); } + @Test + public void testCompilerLowersStaticSurveyResponse() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT getSurveyResponse(0, 'question-id'), getSurveyResponse(2, 'another-id') FROM events")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT coalesce(" + + "nullif(coalesce(json_extract_scalar(properties, '$[\"$survey_response_question-id\"]'), ''), ''), " + + "nullif(coalesce(json_extract_scalar(properties, '$[\"$survey_response\"]'), ''), '')), " + + "coalesce(" + + "nullif(coalesce(json_extract_scalar(properties, '$[\"$survey_response_another-id\"]'), ''), ''), " + + "nullif(coalesce(json_extract_scalar(properties, '$[\"$survey_response_2\"]'), ''), '')) " + + "FROM events")); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope("SELECT getSurveyResponse(question_index, 'question-id') FROM events"))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception).hasMessageContaining("question index must be an integer literal"); + } + @Test public void testCompilerLowersExpressionAndDateAliases() { From 6044ffd482cf9d3a66d0feb04e34b0c5265d152d Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 00:44:19 -0400 Subject: [PATCH 101/136] fix(hogql): delimit property identifiers --- .../trino/hogql/compiler/TrinoAstFactory.java | 26 ++++++++++++++++++- .../hogql/compiler/TestHogQlCompiler.java | 7 +++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 150b12368e13..e418765cd32c 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -798,7 +798,31 @@ private static Table createTable(TableReference table) private static io.trino.sql.tree.Identifier createIdentifier(Identifier identifier) { - return new io.trino.sql.tree.Identifier(location(identifier.span()), identifier.value(), identifier.delimited()); + return new io.trino.sql.tree.Identifier( + location(identifier.span()), + identifier.value(), + identifier.delimited() || !isValidTrinoIdentifier(identifier.value())); + } + + private static boolean isValidTrinoIdentifier(String value) + { + if (value.isEmpty() || !isValidTrinoIdentifierFirstCharacter(value.charAt(0))) { + return false; + } + for (int index = 1; index < value.length(); index++) { + char character = value.charAt(index); + if (!isValidTrinoIdentifierFirstCharacter(character) && (character < '0' || character > '9')) { + return false; + } + } + return true; + } + + private static boolean isValidTrinoIdentifierFirstCharacter(char character) + { + return (character >= 'a' && character <= 'z') || + (character >= 'A' && character <= 'Z') || + character == '_'; } private static NodeLocation location(SourceSpan span) diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index 7be9851a9035..4945e12b894f 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -87,6 +87,13 @@ public void testLowersToEquivalentStockTrinoAst(String hogql) assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); } + @Test + public void testQuotesHogQlPropertyIdentifiersForTrino() + { + assertThat(compiler.compile("SELECT properties.$user_id, properties.$host FROM events")) + .isEqualTo(sqlParser.createStatement("SELECT properties.\"$user_id\", properties.\"$host\" FROM events")); + } + @Test public void testLowersDistinctOrderingAndClickHousePaginationOrder() { From 7b8e76b9e2c235545a396e382238cc79955421e4 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 00:46:24 -0400 Subject: [PATCH 102/136] feat(hogql): preserve local set operand clauses --- .../java/io/trino/hogql/compiler/TestHogQlCompiler.java | 8 ++++++++ .../src/main/java/io/trino/hogql/parser/HogQlParser.java | 5 ----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index 4945e12b894f..3f6726aeb012 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -102,6 +102,14 @@ public void testLowersDistinctOrderingAndClickHousePaginationOrder() assertThat(statement).isEqualTo(sqlParser.createStatement("SELECT DISTINCT event FROM events ORDER BY event DESC NULLS FIRST OFFSET 2 LIMIT 10")); } + @Test + public void testWrapsSetOperandsWithLocalClauses() + { + Statement statement = compiler.compile("SELECT 1 ORDER BY 1 LIMIT 1 UNION ALL SELECT 2"); + + assertThat(statement).isEqualTo(sqlParser.createStatement("(SELECT 1 ORDER BY 1 LIMIT 1) UNION ALL SELECT 2")); + } + @ParameterizedTest @CsvSource(delimiter = '|', textBlock = """ SELECT [1, 2, 3] | SELECT ARRAY[1, 2, 3] diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 6e2fed4f46cb..345a348e6438 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -542,11 +542,6 @@ private HogQlQuery buildSetQuery(HogQLParser.SelectSetStmtContext context) operands.add(operand); } - for (QueryOperand operand : operands.subList(0, operands.size() - 1)) { - if (!operand.parenthesized() && hasQueryClauses(operand.query())) { - throw unsupported(context, "unparenthesized set operand ORDER BY or pagination"); - } - } List orderBy = buildOrderBy(context.orderByClause()); Pagination pagination = context.limitAndOffsetClauseOptional() == null ? new Pagination(Optional.empty(), Optional.empty()) From 2ca7c8786a571175e128511279b81a46bbf0d797 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 00:49:55 -0400 Subject: [PATCH 103/136] fix(hogql): resolve derived set output scope --- .../io/trino/hogql/compiler/TestHogQlCompiler.java | 10 ++++++++++ .../main/java/io/trino/hogql/parser/HogQlParser.java | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index 3f6726aeb012..db56b483dc11 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -110,6 +110,16 @@ public void testWrapsSetOperandsWithLocalClauses() assertThat(statement).isEqualTo(sqlParser.createStatement("(SELECT 1 ORDER BY 1 LIMIT 1) UNION ALL SELECT 2")); } + @Test + public void testPreservesUnqualifiedOutputReferencesInDerivedSetQueries() + { + Statement statement = compiler.compile( + "SELECT * FROM (SELECT 1 AS value UNION ALL SELECT 2 ORDER BY value) AS nested"); + + assertThat(statement).isEqualTo(sqlParser.createStatement( + "SELECT * FROM (SELECT 1 AS value UNION ALL SELECT 2 ORDER BY value) AS nested")); + } + @ParameterizedTest @CsvSource(delimiter = '|', textBlock = """ SELECT [1, 2, 3] | SELECT ARRAY[1, 2, 3] diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 345a348e6438..4db4b0b763ea 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -1871,7 +1871,7 @@ private void validateQueryScope(HogQlQuery query, Set forbiddenOuterRela { query.with().forEach(commonTable -> validateQueryScope(commonTable.query(), forbiddenOuterRelations)); if (query.body() instanceof SetOperation setOperation) { - query.orderBy().forEach(item -> validateExpressionScope(item.expression(), forbiddenOuterRelations, Set.of())); + query.orderBy().forEach(item -> validateExpressionScope(item.expression(), forbiddenOuterRelations, Set.of("__hogql_set_output"))); query.limit().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, Set.of())); query.offset().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, Set.of())); validateQueryScope(setOperation.left(), forbiddenOuterRelations); From 8878d05d486e585f3b97ac335f2021d1ba07c8b5 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 00:51:41 -0400 Subject: [PATCH 104/136] feat(hogql): lower comparison aliases --- .../io/trino/hogql/compiler/HogQlFunctionResolver.java | 2 ++ .../io/trino/hogql/compiler/HogQlV0FunctionRegistry.java | 2 ++ .../compiler/catalog/HogQlSemanticCatalogSnapshot.java | 6 ++++-- .../io/trino/hogql/compiler/TestHogQlFunctionResolver.java | 7 +++++-- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 5532f1882965..c1d3cddb254d 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -512,6 +512,8 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< case NOT -> new UnaryExpression(HogQlQuery.UnaryOperator.NOT, arguments.getFirst(), span); case AND -> and(arguments, span); case GREATER -> new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN, arguments.getFirst(), arguments.get(1), span); + case GREATER_OR_EQUAL -> new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN_OR_EQUAL, arguments.getFirst(), arguments.get(1), span); + case LESS_OR_EQUAL -> new BinaryExpression(HogQlQuery.BinaryOperator.LESS_THAN_OR_EQUAL, arguments.getFirst(), arguments.get(1), span); case LIKE -> new BinaryExpression(HogQlQuery.BinaryOperator.LIKE, arguments.getFirst(), arguments.get(1), span); case REGEX_EXTRACT -> regexExtract(function, arguments); case REGEX_EXTRACT_ALL -> regexExtractAll(function, arguments); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index ba34c71f897a..53710faeab60 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -116,6 +116,8 @@ final class HogQlV0FunctionRegistry rewrite("not", FunctionRewrite.NOT, "boolean", signature(1)), rewrite("and", FunctionRewrite.AND, "boolean", signature(2), variadicSignature(2)), rewrite("greater", FunctionRewrite.GREATER, "boolean", signature(2)), + rewrite("greaterOrEquals", FunctionRewrite.GREATER_OR_EQUAL, "boolean", signature(2)), + rewrite("lessOrEquals", FunctionRewrite.LESS_OR_EQUAL, "boolean", signature(2)), rewrite("like", FunctionRewrite.LIKE, "boolean", signature(2)), rewrite("extract", FunctionRewrite.REGEX_EXTRACT, "varchar", signature(2)), rewrite("replaceRegexpAll", FunctionRewrite.REGEX_REPLACE_ALL, "varchar", signature(3)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 83ee1ce92ff5..370ef5acb50d 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -309,9 +309,9 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi COUNT_DISTINCT, UNIQ_EXACT -> !signature.variadic() && signature.argumentTypes().size() == 1; case ADD_DAYS, ADD_MONTHS, ANY_IF, ARRAY_ELEMENT, ARRAY_FILTER, ARRAY_FIRST, ARRAY_MAP, - AVG_IF, DATE_PART, DECIMAL_CAST, DIVIDE_DECIMAL, EQUALS, FLOAT_OR_DEFAULT, GREATER, GROUP_ARRAY_IF, HAS, + AVG_IF, DATE_PART, DECIMAL_CAST, DIVIDE_DECIMAL, EQUALS, FLOAT_OR_DEFAULT, GREATER, GREATER_OR_EQUAL, GROUP_ARRAY_IF, HAS, GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_HAS, JSON_KEYS_AND_VALUES, JSON_VALUE, - IN_ARRAY, INT_DIV, LIKE, MAX_IF, MIN_IF, MULTIPLY, MULTIPLY_DECIMAL, + IN_ARRAY, INT_DIV, LESS_OR_EQUAL, LIKE, MAX_IF, MIN_IF, MULTIPLY, MULTIPLY_DECIMAL, MEDIAN_IF, MINUS, NOT_EQUALS, REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, SURVEY_RESPONSE, SUBTRACT_MONTHS, SUBTRACT_YEARS, SUM_IF, TUPLE_ELEMENT, UNIQ_EXACT_IF, UNIQ_IF -> @@ -1356,6 +1356,7 @@ public enum FunctionRewrite FLOAT_OR_ZERO, GROUP_UNIQ_ARRAY, GREATER, + GREATER_OR_EQUAL, GROUP_ARRAY_IF, GROUP_UNIQ_ARRAY_IF, HAS, @@ -1381,6 +1382,7 @@ public enum FunctionRewrite JSON_LENGTH, JSON_VALUE, LIKE, + LESS_OR_EQUAL, MAX_IF, MD5, MEDIAN_IF, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 26555d202115..c2ecc9cf0e9b 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -352,11 +352,14 @@ public void testCompilerLowersArrayCompatibilityFunctions() public void testCompilerLowersFinalStockCompatibilityFunctions() { HogQlCompilationResult result = new HogQlCompiler().compile(envelope( - "SELECT minus(total, discount), divide(total, count), notEquals(status, 'ignored'), toMonth(timestamp), toYear(timestamp), ceil(score), _toInt16(value), " + + "SELECT minus(total, discount), divide(total, count), notEquals(status, 'ignored'), " + + "greaterOrEquals(total, minimum), lessOrEquals(total, maximum), " + + "toMonth(timestamp), toYear(timestamp), ceil(score), _toInt16(value), " + "JSONExtractKeys(payload), JSONExtractKeys(payload, 'nested') FROM records")); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( - "SELECT total - discount, total / count, status <> 'ignored', month(timestamp), year(timestamp), ceiling(score), CAST(value AS smallint), " + + "SELECT total - discount, total / count, status <> 'ignored', total >= minimum, total <= maximum, " + + "month(timestamp), year(timestamp), ceiling(score), CAST(value AS smallint), " + "map_keys(coalesce(TRY_CAST(json_parse(payload) AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))), " + "map_keys(coalesce(TRY_CAST(json_extract(payload, '$[\"nested\"]') AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))) " + "FROM records")); From 924c42e57d4feee583e49f470b52ef28cd6537e7 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 00:57:12 -0400 Subject: [PATCH 105/136] feat(hogql): lower parametric aggregates --- core/trino-hogql-compiler/README.md | 4 ++++ .../hogql/compiler/HogQlFunctionResolver.java | 18 +++++++++++++++++- .../compiler/HogQlV0FunctionRegistry.java | 5 ++++- .../catalog/HogQlSemanticCatalogSnapshot.java | 13 +++++++++---- .../compiler/TestHogQlFunctionResolver.java | 13 +++++++++++++ .../io/trino/hogql/parser/HogQlParser.java | 12 ++++++++---- 6 files changed, 55 insertions(+), 10 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index f862df27c70f..9362ed8c870d 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -102,6 +102,10 @@ subtraction aliases, calendar month/year extraction, ceiling aliases, narrow integer casts, and JSON object-key extraction also lower to stock expressions. The static-ID, single-choice `getSurveyResponse(index, question_id)` macro prefers the modern response property and falls back to the legacy index key. +Parametric quantiles lower to Trino percentile aggregates, and limited +conditional group arrays lower to filtered `array_agg` plus `slice`. +Algorithm-level parity for `quantileExact` remains a compatibility-validation +item even though its query shape has a stock AST lowering. Function aliases are resolved before Trino analysis. Unknown functions fail as HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index c1d3cddb254d..69d228a5289a 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -474,7 +474,15 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< case ANY_IF -> aggregate(function, "arbitrary", List.of(arguments.getFirst()), false, arguments.get(1), span); case MIN_IF -> aggregate(function, "min", List.of(arguments.getFirst()), false, arguments.get(1), span); case AVG_IF -> aggregate(function, "avg", List.of(arguments.getFirst()), false, arguments.get(1), span); - case GROUP_ARRAY_IF -> aggregate(function, "array_agg", List.of(arguments.getFirst()), false, arguments.get(1), span); + case GROUP_ARRAY_IF -> arguments.size() == 2 + ? aggregate(function, "array_agg", List.of(arguments.getFirst()), false, arguments.get(1), span) + : call( + "slice", + List.of( + aggregate(function, "array_agg", List.of(arguments.getFirst()), false, arguments.get(1), span), + integerLiteral("1", span), + arguments.get(2)), + span); case UNIQ_EXACT_IF -> aggregate(function, "count", List.of(arguments.getFirst()), true, arguments.get(1), span); case GROUP_UNIQ_ARRAY_IF -> aggregate(function, "array_agg", List.of(arguments.getFirst()), true, arguments.get(1), span); case COUNT_DISTINCT -> aggregate(function, "count", List.of(arguments.getFirst()), true, null, span); @@ -543,6 +551,14 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< false, arguments.get(1), span); + case QUANTILE, QUANTILE_EXACT -> aggregate(function, "approx_percentile", arguments, false, null, span); + case QUANTILE_IF -> aggregate( + function, + "approx_percentile", + List.of(arguments.getFirst(), arguments.get(2)), + false, + arguments.get(1), + span); case DATE_PART -> datePart(function, arguments); }; } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 53710faeab60..73f96543560c 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -170,7 +170,10 @@ final class HogQlV0FunctionRegistry aggregateRewrite("anyIf", FunctionRewrite.ANY_IF, "any", signature(2)), aggregateRewrite("minIf", FunctionRewrite.MIN_IF, "any", signature(2)), aggregateRewrite("avgIf", FunctionRewrite.AVG_IF, "double", signature(2)), - aggregateRewrite("groupArrayIf", FunctionRewrite.GROUP_ARRAY_IF, "array(any)", signature(2)), + aggregateRewrite("groupArrayIf", FunctionRewrite.GROUP_ARRAY_IF, "array(any)", signature(2), signature(3)), + aggregateRewrite("quantile", FunctionRewrite.QUANTILE, "any", signature(2)), + aggregateRewrite("quantileExact", FunctionRewrite.QUANTILE_EXACT, "any", signature(2)), + aggregateRewrite("quantileIf", FunctionRewrite.QUANTILE_IF, "any", signature(3)), aggregateRewrite("uniqExactIf", FunctionRewrite.UNIQ_EXACT_IF, "bigint", signature(2)), aggregateRewrite("groupUniqArrayIf", FunctionRewrite.GROUP_UNIQ_ARRAY_IF, "array(any)", signature(2)), aggregateRewrite("countDistinct", FunctionRewrite.COUNT_DISTINCT, "bigint", signature(1)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 370ef5acb50d..23b5c8785cca 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -291,7 +291,7 @@ private static FunctionKind rewriteFunctionKind(FunctionRewrite rewrite) { return switch (rewrite) { case ANY_IF, ARG_MAX_IF, AVG_IF, COUNT_DISTINCT, COUNT_IF, GROUP_ARRAY_IF, - GROUP_UNIQ_ARRAY, GROUP_UNIQ_ARRAY_IF, MAX_IF, MEDIAN_IF, MIN_IF, SUM_IF, + GROUP_UNIQ_ARRAY, GROUP_UNIQ_ARRAY_IF, MAX_IF, MEDIAN_IF, MIN_IF, QUANTILE, QUANTILE_EXACT, QUANTILE_IF, SUM_IF, UNIQ_EXACT, UNIQ_EXACT_IF, UNIQ_IF -> FunctionKind.AGGREGATE; default -> FunctionKind.SCALAR; }; @@ -309,15 +309,17 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi COUNT_DISTINCT, UNIQ_EXACT -> !signature.variadic() && signature.argumentTypes().size() == 1; case ADD_DAYS, ADD_MONTHS, ANY_IF, ARRAY_ELEMENT, ARRAY_FILTER, ARRAY_FIRST, ARRAY_MAP, - AVG_IF, DATE_PART, DECIMAL_CAST, DIVIDE_DECIMAL, EQUALS, FLOAT_OR_DEFAULT, GREATER, GREATER_OR_EQUAL, GROUP_ARRAY_IF, HAS, + AVG_IF, DATE_PART, DECIMAL_CAST, DIVIDE_DECIMAL, EQUALS, FLOAT_OR_DEFAULT, GREATER, GREATER_OR_EQUAL, HAS, GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_HAS, JSON_KEYS_AND_VALUES, JSON_VALUE, IN_ARRAY, INT_DIV, LESS_OR_EQUAL, LIKE, MAX_IF, MIN_IF, MULTIPLY, MULTIPLY_DECIMAL, - MEDIAN_IF, MINUS, NOT_EQUALS, REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, SURVEY_RESPONSE, + MEDIAN_IF, MINUS, NOT_EQUALS, QUANTILE, QUANTILE_EXACT, REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, SURVEY_RESPONSE, SUBTRACT_MONTHS, SUBTRACT_YEARS, SUM_IF, TUPLE_ELEMENT, UNIQ_EXACT_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; - case ARG_MAX_IF, ARRAY_SLICE, REGEX_REPLACE_ALL, REGEX_REPLACE_ONE -> + case ARG_MAX_IF, ARRAY_SLICE, QUANTILE_IF, REGEX_REPLACE_ALL, REGEX_REPLACE_ONE -> !signature.variadic() && signature.argumentTypes().size() == 3; + case GROUP_ARRAY_IF -> !signature.variadic() && + (signature.argumentTypes().size() == 2 || signature.argumentTypes().size() == 3); case CAST_TIMESTAMP -> !signature.variadic() && (signature.argumentTypes().size() == 1 || signature.argumentTypes().size() == 2); case DATE_ADD -> !signature.variadic() && @@ -1395,6 +1397,9 @@ public enum FunctionRewrite NOT_EQUALS, NOT_EMPTY, PARSE_TIMESTAMP, + QUANTILE, + QUANTILE_EXACT, + QUANTILE_IF, REGEX_EXTRACT, REGEX_EXTRACT_ALL, REGEX_REPLACE_ALL, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index c2ecc9cf0e9b..80466f9c05b4 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -388,6 +388,19 @@ public void testCompilerLowersStaticSurveyResponse() assertThat(exception).hasMessageContaining("question index must be an integer literal"); } + @Test + public void testCompilerLowersParametricAggregates() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT quantile(0.25)(value), quantileExact(0.5)(value), " + + "quantileIf(0.75)(value, active), groupArrayIf(20)(value, active) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT approx_percentile(value, 2.5E-1), approx_percentile(value, 5E-1), " + + "approx_percentile(value, 7.5E-1) FILTER (WHERE active), " + + "slice(array_agg(value) FILTER (WHERE active), 1, 20) FROM records")); + } + @Test public void testCompilerLowersExpressionAndDateAliases() { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 4db4b0b763ea..b48fe92d4fce 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -1461,15 +1461,19 @@ private BinaryExpression binary(BinaryOperator operator, HogQLParser.ColumnExprV private FunctionCall buildFunction(HogQLParser.ColumnExprFunctionContext context) { - if (context.columnExprs != null) { + Identifier name = buildIdentifier(context.identifier()); + if (context.columnExprs != null && !Set.of("quantile", "quantileexact", "quantileif", "grouparrayif").contains(canonicalName(name))) { throw unsupported(context, "parametric function"); } - List arguments = buildFunctionArguments(context); + List arguments = new ArrayList<>(buildFunctionArguments(context)); + if (context.columnExprs != null) { + arguments.addAll(buildExpressions(context.columnExprs)); + } List orderBy = context.orderExprList() == null ? List.of() : buildSortItems(context.orderExprList()); Optional filter = Optional.ofNullable(context.filterExpr).map(this::buildExpression); return new FunctionCall( - buildIdentifier(context.identifier()), - arguments, + name, + List.copyOf(arguments), context.DISTINCT() != null, orderBy, filter, From ea77ff99b02609d721997b4a832594849876bc27 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 00:59:25 -0400 Subject: [PATCH 106/136] feat(hogql): lower exposed aggregate aliases --- .../java/io/trino/hogql/compiler/HogQlFunctionResolver.java | 1 + .../io/trino/hogql/compiler/HogQlV0FunctionRegistry.java | 2 ++ .../compiler/catalog/HogQlSemanticCatalogSnapshot.java | 5 +++-- .../io/trino/hogql/compiler/TestHogQlFunctionResolver.java | 6 ++++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 69d228a5289a..77d3bbdb516e 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -471,6 +471,7 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< case UNIQ_EXACT -> aggregate(function, "count", List.of(arguments.getFirst()), true, null, span); case GROUP_UNIQ_ARRAY -> aggregate(function, "array_agg", List.of(arguments.getFirst()), true, null, span); case ARG_MAX_IF -> aggregate(function, "max_by", List.of(arguments.getFirst(), arguments.get(1)), false, arguments.get(2), span); + case ARG_MIN_IF -> aggregate(function, "min_by", List.of(arguments.getFirst(), arguments.get(1)), false, arguments.get(2), span); case ANY_IF -> aggregate(function, "arbitrary", List.of(arguments.getFirst()), false, arguments.get(1), span); case MIN_IF -> aggregate(function, "min", List.of(arguments.getFirst()), false, arguments.get(1), span); case AVG_IF -> aggregate(function, "avg", List.of(arguments.getFirst()), false, arguments.get(1), span); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 73f96543560c..623efe50656d 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -153,6 +153,7 @@ final class HogQlV0FunctionRegistry scalar("arrayDistinct", "array_distinct", signature(1)), scalar("arrayFlatten", "flatten", signature(1)), scalar("arrayStringConcat", "array_join", signature(1), signature(2)), + scalar("replaceAll", "replace", signature(3)), aggregate("count", "count", true, signature(0), signature(1)), aggregate("sum", "sum", true, signature(1)), aggregate("min", "min", true, signature(1)), @@ -167,6 +168,7 @@ final class HogQlV0FunctionRegistry aggregateRewrite("uniqExact", FunctionRewrite.UNIQ_EXACT, "bigint", signature(1)), aggregateRewrite("groupUniqArray", FunctionRewrite.GROUP_UNIQ_ARRAY, "array(any)", signature(1)), aggregateRewrite("argMaxIf", FunctionRewrite.ARG_MAX_IF, "any", signature(3)), + aggregateRewrite("argMinIf", FunctionRewrite.ARG_MIN_IF, "any", signature(3)), aggregateRewrite("anyIf", FunctionRewrite.ANY_IF, "any", signature(2)), aggregateRewrite("minIf", FunctionRewrite.MIN_IF, "any", signature(2)), aggregateRewrite("avgIf", FunctionRewrite.AVG_IF, "double", signature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 23b5c8785cca..0fbc906d9f36 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -290,7 +290,7 @@ private static Map validateFunctions(List< private static FunctionKind rewriteFunctionKind(FunctionRewrite rewrite) { return switch (rewrite) { - case ANY_IF, ARG_MAX_IF, AVG_IF, COUNT_DISTINCT, COUNT_IF, GROUP_ARRAY_IF, + case ANY_IF, ARG_MAX_IF, ARG_MIN_IF, AVG_IF, COUNT_DISTINCT, COUNT_IF, GROUP_ARRAY_IF, GROUP_UNIQ_ARRAY, GROUP_UNIQ_ARRAY_IF, MAX_IF, MEDIAN_IF, MIN_IF, QUANTILE, QUANTILE_EXACT, QUANTILE_IF, SUM_IF, UNIQ_EXACT, UNIQ_EXACT_IF, UNIQ_IF -> FunctionKind.AGGREGATE; default -> FunctionKind.SCALAR; @@ -316,7 +316,7 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi SUBTRACT_MONTHS, SUBTRACT_YEARS, SUM_IF, TUPLE_ELEMENT, UNIQ_EXACT_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; - case ARG_MAX_IF, ARRAY_SLICE, QUANTILE_IF, REGEX_REPLACE_ALL, REGEX_REPLACE_ONE -> + case ARG_MAX_IF, ARG_MIN_IF, ARRAY_SLICE, QUANTILE_IF, REGEX_REPLACE_ALL, REGEX_REPLACE_ONE -> !signature.variadic() && signature.argumentTypes().size() == 3; case GROUP_ARRAY_IF -> !signature.variadic() && (signature.argumentTypes().size() == 2 || signature.argumentTypes().size() == 3); @@ -1333,6 +1333,7 @@ public enum FunctionRewrite AND, ANY_IF, ARG_MAX_IF, + ARG_MIN_IF, ARRAY_ELEMENT, ARRAY_ENUMERATE, ARRAY_FILTER, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 80466f9c05b4..c3ca7e1082b6 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -393,12 +393,14 @@ public void testCompilerLowersParametricAggregates() { HogQlCompilationResult result = new HogQlCompiler().compile(envelope( "SELECT quantile(0.25)(value), quantileExact(0.5)(value), " + - "quantileIf(0.75)(value, active), groupArrayIf(20)(value, active) FROM records")); + "quantileIf(0.75)(value, active), groupArrayIf(20)(value, active), " + + "argMinIf(value, timestamp, active), replaceAll(name, 'old', 'new') FROM records")); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( "SELECT approx_percentile(value, 2.5E-1), approx_percentile(value, 5E-1), " + "approx_percentile(value, 7.5E-1) FILTER (WHERE active), " + - "slice(array_agg(value) FILTER (WHERE active), 1, 20) FROM records")); + "slice(array_agg(value) FILTER (WHERE active), 1, 20), " + + "min_by(value, timestamp) FILTER (WHERE active), replace(name, 'old', 'new') FROM records")); } @Test From d162483b86e0a77f4f4611a0040361e85445658e Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 01:02:41 -0400 Subject: [PATCH 107/136] feat(hogql): lower array key sorting --- .../hogql/compiler/HogQlFunctionResolver.java | 70 +++++++++++++++++++ .../compiler/HogQlV0FunctionRegistry.java | 3 +- .../catalog/HogQlSemanticCatalogSnapshot.java | 3 +- .../compiler/TestHogQlFunctionResolver.java | 12 ++++ 4 files changed, 86 insertions(+), 2 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 77d3bbdb516e..eeac8006c3c6 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -529,6 +529,7 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< case REGEX_REPLACE_ALL -> regexReplaceAll(function, arguments); case REGEX_REPLACE_ONE -> regexReplaceOne(function, arguments); case ARRAY_SLICE -> call("slice", arguments, span); + case ARRAY_SORT -> arraySort(function, arguments); case ARRAY_ENUMERATE -> arrayEnumerate(arguments.getFirst(), span); case SUBTRACT_YEARS -> dateAdd( "year", @@ -1012,6 +1013,75 @@ private static Expression arrayEnumerate(Expression array, HogQlQuery.SourceSpan span); } + private static Expression arraySort(FunctionCall function, List arguments) + { + if (arguments.size() == 1) { + return call("array_sort", arguments, function.span()); + } + if (!(arguments.getFirst() instanceof LambdaExpression lambda) || lambda.arguments().size() != 1) { + throw unsupportedError(function, "HogQL arraySort key must be a single-argument lambda"); + } + HogQlQuery.SourceSpan span = function.span(); + Identifier left = new Identifier("__hogql_array_sort_left", false, span); + Identifier right = new Identifier("__hogql_array_sort_right", false, span); + String parameter = canonical(lambda.arguments().getFirst().value()); + Expression leftKey = substituteArraySortParameter(function, lambda.body(), parameter, new ColumnReference(List.of(left), span)); + Expression rightKey = substituteArraySortParameter(function, lambda.body(), parameter, new ColumnReference(List.of(right), span)); + Expression comparator = new CaseExpression( + Optional.empty(), + List.of( + new CaseWhen( + new BinaryExpression(HogQlQuery.BinaryOperator.LESS_THAN, leftKey, rightKey, span), + new UnaryExpression(HogQlQuery.UnaryOperator.NEGATE, integerLiteral("1", span), span), + span), + new CaseWhen( + new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN, leftKey, rightKey, span), + integerLiteral("1", span), + span)), + Optional.of(integerLiteral("0", span)), + span); + return call( + "array_sort", + List.of( + arguments.get(1), + new LambdaExpression(List.of(left, right), comparator, span)), + span); + } + + private static Expression substituteArraySortParameter(FunctionCall function, Expression expression, String parameter, Expression replacement) + { + return switch (expression) { + case ColumnReference reference -> reference.parts().size() == 1 && canonical(reference.parts().getFirst().value()).equals(parameter) + ? replacement + : reference; + case SubscriptExpression subscript -> new SubscriptExpression( + substituteArraySortParameter(function, subscript.base(), parameter, replacement), + substituteArraySortParameter(function, subscript.index(), parameter, replacement), + subscript.span()); + case MemberAccessExpression member -> new MemberAccessExpression( + substituteArraySortParameter(function, member.base(), parameter, replacement), + member.member(), + member.span()); + case CastExpression cast -> new CastExpression( + substituteArraySortParameter(function, cast.value(), parameter, replacement), + cast.type(), + cast.safe(), + cast.typeDialect(), + cast.span()); + case UnaryExpression unary -> new UnaryExpression( + unary.operator(), + substituteArraySortParameter(function, unary.operand(), parameter, replacement), + unary.span()); + case BinaryExpression binary -> new BinaryExpression( + binary.operator(), + substituteArraySortParameter(function, binary.left(), parameter, replacement), + substituteArraySortParameter(function, binary.right(), parameter, replacement), + binary.span()); + case Literal literal -> literal; + default -> throw unsupportedError(function, "HogQL arraySort key expression is outside the supported subset"); + }; + } + private static Expression has(List arguments, HogQlQuery.SourceSpan span) { Expression array = arguments.getFirst(); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 623efe50656d..2e48301106fb 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -149,7 +149,8 @@ final class HogQlV0FunctionRegistry scalar("map", "map", signature(2)), scalar("dateDiff", "date_diff", signature(3)), scalar("dateTrunc", "date_trunc", signature(2)), - scalar("arraySort", "array_sort", signature(1)), + rewrite("arraySort", FunctionRewrite.ARRAY_SORT, "array(any)", signature(1), signature(2)), + scalar("arrayMin", "array_min", signature(1)), scalar("arrayDistinct", "array_distinct", signature(1)), scalar("arrayFlatten", "flatten", signature(1)), scalar("arrayStringConcat", "array_join", signature(1), signature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 0fbc906d9f36..7430abc0c01b 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -331,7 +331,7 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi (signature.variadic() && signature.argumentTypes().size() == 3); case AND -> (!signature.variadic() && signature.argumentTypes().size() == 2) || (signature.variadic() && signature.argumentTypes().size() == 3); - case RANGE -> !signature.variadic() && + case ARRAY_SORT, RANGE -> !signature.variadic() && (signature.argumentTypes().size() == 1 || signature.argumentTypes().size() == 2); case MULTI_IF -> signature.variadic() && signature.argumentTypes().size() == 4; case TUPLE -> signature.variadic() && signature.argumentTypes().size() == 2; @@ -1340,6 +1340,7 @@ public enum FunctionRewrite ARRAY_FIRST, ARRAY_MAP, ARRAY_SLICE, + ARRAY_SORT, ARRAY_SUM, ASSUME_NOT_NULL, AVG_IF, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index c3ca7e1082b6..8ce810b64a02 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -348,6 +348,18 @@ public void testCompilerLowersArrayCompatibilityFunctions() "if(NULL IS NULL, any_match(items, _hogql_item -> _hogql_item IS NULL), coalesce(contains(items, NULL), false)) FROM records")); } + @Test + public void testCompilerLowersArrayExtremaAndKeySorting() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT arrayMin(items), arraySort(item -> item.1, tuples) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT array_min(items), array_sort(tuples, (__hogql_array_sort_left, __hogql_array_sort_right) -> " + + "CASE WHEN __hogql_array_sort_left[1] < __hogql_array_sort_right[1] THEN -(1) " + + "WHEN __hogql_array_sort_left[1] > __hogql_array_sort_right[1] THEN 1 ELSE 0 END) FROM records")); + } + @Test public void testCompilerLowersFinalStockCompatibilityFunctions() { From e8b74412522d920106075cd1bd52083f0aba641b Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 01:04:37 -0400 Subject: [PATCH 108/136] feat(hogql): lower iso day of week --- .../java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java | 1 + .../io/trino/hogql/compiler/TestHogQlFunctionResolver.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 2e48301106fb..c4ccfe45d84e 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -135,6 +135,7 @@ final class HogQlV0FunctionRegistry scalar("match", "regexp_like", signature(2)), scalar("floor", "floor", signature(1)), scalar("toDayOfMonth", "day", signature(1)), + scalar("toDayOfWeek", "day_of_week", signature(1)), scalar("mapFromArrays", "map", signature(2)), scalar("toMonth", "month", signature(1)), scalar("toYear", "year", signature(1)), diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 8ce810b64a02..73937f0d2d9a 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -366,12 +366,12 @@ public void testCompilerLowersFinalStockCompatibilityFunctions() HogQlCompilationResult result = new HogQlCompiler().compile(envelope( "SELECT minus(total, discount), divide(total, count), notEquals(status, 'ignored'), " + "greaterOrEquals(total, minimum), lessOrEquals(total, maximum), " + - "toMonth(timestamp), toYear(timestamp), ceil(score), _toInt16(value), " + + "toMonth(timestamp), toYear(timestamp), toDayOfWeek(timestamp), ceil(score), _toInt16(value), " + "JSONExtractKeys(payload), JSONExtractKeys(payload, 'nested') FROM records")); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( "SELECT total - discount, total / count, status <> 'ignored', total >= minimum, total <= maximum, " + - "month(timestamp), year(timestamp), ceiling(score), CAST(value AS smallint), " + + "month(timestamp), year(timestamp), day_of_week(timestamp), ceiling(score), CAST(value AS smallint), " + "map_keys(coalesce(TRY_CAST(json_parse(payload) AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))), " + "map_keys(coalesce(TRY_CAST(json_extract(payload, '$[\"nested\"]') AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))) " + "FROM records")); From f8812beb9b62e4bce74f2da08ea49a570e0803b4 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 01:07:50 -0400 Subject: [PATCH 109/136] feat(hogql): lower numbers table function --- .../hogql/compiler/TestHogQlCompiler.java | 9 +++++++ .../io/trino/hogql/parser/HogQlParser.java | 25 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index db56b483dc11..e53f540a447a 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -120,6 +120,15 @@ public void testPreservesUnqualifiedOutputReferencesInDerivedSetQueries() "SELECT * FROM (SELECT 1 AS value UNION ALL SELECT 2 ORDER BY value) AS nested")); } + @Test + public void testLowersNumbersTableFunction() + { + Statement statement = compiler.compile("SELECT number FROM numbers(5)"); + + assertThat(statement).isEqualTo(sqlParser.createStatement( + "SELECT number FROM UNNEST(if(5 <= 0, CAST(ARRAY[] AS array(bigint)), sequence(0, 5 - 1))) AS numbers(number)")); + } + @ParameterizedTest @CsvSource(delimiter = '|', textBlock = """ SELECT [1, 2, 3] | SELECT ARRAY[1, 2, 3] diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index b48fe92d4fce..c0991596658b 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -1719,6 +1719,31 @@ private Relation buildTableExpression(HogQLParser.TableExprContext context) } return new TableReference(parts, sourceSpan(context)); } + if (context instanceof HogQLParser.TableExprFunctionContext function) { + Identifier name = buildIdentifier(function.tableFunctionExpr().identifier()); + if (!canonicalName(name).equals("numbers")) { + throw unsupported(context, "table expression"); + } + List arguments = function.tableFunctionExpr().tableArgList() == null + ? List.of() + : function.tableFunctionExpr().tableArgList().columnExpr(); + if (arguments.size() != 1) { + throw unsupported(context, "numbers table function arity"); + } + SourceSpan span = sourceSpan(context); + FunctionCall range = new FunctionCall( + new Identifier("range", false, span), + List.of(buildExpression(arguments.getFirst())), + false, + List.of(), + Optional.empty(), + span); + return new UnnestRelation( + List.of(range), + new Identifier("numbers", false, span), + List.of(new Identifier("number", false, span)), + span); + } if (context instanceof HogQLParser.TableExprPlaceholderContext placeholder) { return new TablePlaceholder(buildPlaceholder(placeholder.placeholder())); } From 80f51338b6c2387fa76f2cd50c5bed96283fffbb Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 01:13:57 -0400 Subject: [PATCH 110/136] feat(hogql): lower dynamic json paths --- core/trino-hogql-compiler/README.md | 7 ++++--- .../hogql/compiler/HogQlFunctionResolver.java | 18 +++++++++++++++--- .../compiler/TestHogQlFunctionResolver.java | 9 ++++++++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 9362ed8c870d..85e72245c235 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -67,9 +67,10 @@ stock Trino aggregate filters, DISTINCT aggregates, and searched CASE expressions. Aggregate rewrites preserve `OVER` specifications when the target Trino aggregate is used as a window function. -Literal-path JSON extraction lowers to Trino JSON paths with HogQL-compatible -defaults. The compatibility profile also supports the corpus's typed map and -key/value extraction forms through checked JSON-to-map casts. +JSON extraction lowers literal and runtime string/integer segments to Trino +JSON paths with HogQL-compatible defaults. The compatibility profile also +supports the corpus's typed map and key/value extraction forms through checked +JSON-to-map casts. Date/time compatibility includes calendar arithmetic, interval-day values, epoch conversion, timezone attachment/conversion, formatting, and checked best-effort timestamp parsing. diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index eeac8006c3c6..e5e96c77f553 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -862,12 +862,18 @@ private static String jsonType(FunctionCall function, Expression expression) throw unsupportedError(function, "HogQL JSON extraction type must be a string literal"); } - private static Literal jsonPath(FunctionCall function, List segments) + private static Expression jsonPath(FunctionCall function, List segments) { StringBuilder path = new StringBuilder("$"); + List pathParts = new ArrayList<>(); for (Expression segment : segments) { if (!(segment instanceof Literal literal)) { - throw unsupportedError(function, "HogQL JSON path segments must be string or integer literals"); + pathParts.add(stringLiteral(path.toString(), function.span())); + path.setLength(0); + pathParts.add(stringLiteral("[", function.span())); + pathParts.add(call("json_format", List.of(cast(segment, "json", function.span())), function.span())); + pathParts.add(stringLiteral("]", function.span())); + continue; } switch (literal.kind()) { case STRING -> path.append("[\"") @@ -877,7 +883,13 @@ private static Literal jsonPath(FunctionCall function, List segments default -> throw unsupportedError(function, "HogQL JSON path segments must be string or integer literals"); } } - return new Literal(HogQlQuery.LiteralKind.STRING, path.toString(), function.span()); + if (pathParts.isEmpty()) { + return stringLiteral(path.toString(), function.span()); + } + if (!path.isEmpty()) { + pathParts.add(stringLiteral(path.toString(), function.span())); + } + return call("concat", pathParts, function.span()); } private static FunctionCall coalesce(Expression value, Expression defaultValue, HogQlQuery.SourceSpan span) diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 73937f0d2d9a..e4bc27ddc4d9 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -174,9 +174,16 @@ public void testCompilerLowersJsonExtractionFunctions() "map_entries(transform_values(coalesce(TRY_CAST(json_parse(payload_text) AS map(varchar, json)), " + "CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json))), (key, value) -> json_format(value))) FROM records")); + HogQlCompilationResult dynamicPathResult = new HogQlCompiler().compile(envelope( + "SELECT JSONExtractString(payload, dynamic_key), JSONExtractInt(payload, 'items', dynamic_index) FROM records")); + + assertThat(dynamicPathResult.statement()).isEqualTo(sqlParser.createStatement( + "SELECT coalesce(json_extract_scalar(payload, concat('$', '[', json_format(CAST(dynamic_key AS json)), ']')), ''), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, concat('$[\"items\"]', '[', json_format(CAST(dynamic_index AS json)), ']')) AS bigint), 0) FROM records")); + TrinoException exception = catchThrowableOfType( TrinoException.class, - () -> new HogQlCompiler().compile(envelope("SELECT JSONExtractString(payload, dynamic_key)"))); + () -> new HogQlCompiler().compile(envelope("SELECT JSONExtractString(payload, true)"))); assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); assertThat(exception).hasMessageContaining("JSON path segments must be string or integer literals"); From 9d568131976a4fd10d941b46f52fb3448420aa99 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 01:16:40 -0400 Subject: [PATCH 111/136] feat(hogql): lower addition alias --- .../java/io/trino/hogql/compiler/HogQlFunctionResolver.java | 1 + .../java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java | 1 + .../hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java | 3 ++- .../io/trino/hogql/compiler/TestHogQlFunctionResolver.java | 4 ++-- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index e5e96c77f553..32018481752f 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -442,6 +442,7 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< case EMPTY -> empty(arguments.getFirst(), true, span); case NOT_EMPTY -> empty(arguments.getFirst(), false, span); case EQUALS -> new BinaryExpression(HogQlQuery.BinaryOperator.EQUAL, arguments.getFirst(), arguments.get(1), span); + case PLUS -> new BinaryExpression(HogQlQuery.BinaryOperator.ADD, arguments.getFirst(), arguments.get(1), span); case MINUS -> new BinaryExpression(HogQlQuery.BinaryOperator.SUBTRACT, arguments.getFirst(), arguments.get(1), span); case NOT_EQUALS -> new BinaryExpression(HogQlQuery.BinaryOperator.NOT_EQUAL, arguments.getFirst(), arguments.get(1), span); case MULTIPLY, MULTIPLY_DECIMAL -> new BinaryExpression(HogQlQuery.BinaryOperator.MULTIPLY, arguments.getFirst(), arguments.get(1), span); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index c4ccfe45d84e..568ea8efae5e 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -64,6 +64,7 @@ final class HogQlV0FunctionRegistry rewrite("empty", FunctionRewrite.EMPTY, "boolean", signature(1)), rewrite("notEmpty", FunctionRewrite.NOT_EMPTY, "boolean", signature(1)), rewrite("equals", FunctionRewrite.EQUALS, "boolean", signature(2)), + rewrite("plus", FunctionRewrite.PLUS, "any", signature(2)), rewrite("multiply", FunctionRewrite.MULTIPLY, "any", signature(2)), rewrite("multiplyDecimal", FunctionRewrite.MULTIPLY_DECIMAL, "decimal", signature(2)), rewrite("divideDecimal", FunctionRewrite.DIVIDE_DECIMAL, "decimal", signature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 7430abc0c01b..c4b9b3b48c71 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -312,7 +312,7 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi AVG_IF, DATE_PART, DECIMAL_CAST, DIVIDE_DECIMAL, EQUALS, FLOAT_OR_DEFAULT, GREATER, GREATER_OR_EQUAL, HAS, GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_HAS, JSON_KEYS_AND_VALUES, JSON_VALUE, IN_ARRAY, INT_DIV, LESS_OR_EQUAL, LIKE, MAX_IF, MIN_IF, MULTIPLY, MULTIPLY_DECIMAL, - MEDIAN_IF, MINUS, NOT_EQUALS, QUANTILE, QUANTILE_EXACT, REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, SURVEY_RESPONSE, + MEDIAN_IF, MINUS, NOT_EQUALS, PLUS, QUANTILE, QUANTILE_EXACT, REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, SURVEY_RESPONSE, SUBTRACT_MONTHS, SUBTRACT_YEARS, SUM_IF, TUPLE_ELEMENT, UNIQ_EXACT_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; @@ -1399,6 +1399,7 @@ public enum FunctionRewrite NOT_EQUALS, NOT_EMPTY, PARSE_TIMESTAMP, + PLUS, QUANTILE, QUANTILE_EXACT, QUANTILE_IF, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index e4bc27ddc4d9..775e8febd712 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -371,13 +371,13 @@ public void testCompilerLowersArrayExtremaAndKeySorting() public void testCompilerLowersFinalStockCompatibilityFunctions() { HogQlCompilationResult result = new HogQlCompiler().compile(envelope( - "SELECT minus(total, discount), divide(total, count), notEquals(status, 'ignored'), " + + "SELECT plus(total, surcharge), minus(total, discount), divide(total, count), notEquals(status, 'ignored'), " + "greaterOrEquals(total, minimum), lessOrEquals(total, maximum), " + "toMonth(timestamp), toYear(timestamp), toDayOfWeek(timestamp), ceil(score), _toInt16(value), " + "JSONExtractKeys(payload), JSONExtractKeys(payload, 'nested') FROM records")); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( - "SELECT total - discount, total / count, status <> 'ignored', total >= minimum, total <= maximum, " + + "SELECT total + surcharge, total - discount, total / count, status <> 'ignored', total >= minimum, total <= maximum, " + "month(timestamp), year(timestamp), day_of_week(timestamp), ceiling(score), CAST(value AS smallint), " + "map_keys(coalesce(TRY_CAST(json_parse(payload) AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))), " + "map_keys(coalesce(TRY_CAST(json_extract(payload, '$[\"nested\"]') AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))) " + From efad11d62272ea2cd7e69f856dc4acc2deaaaf54 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 01:28:43 -0400 Subject: [PATCH 112/136] feat(ducklake): add clickhouse cityhash64 --- .../trino/plugin/ducklake/DuckLakePlugin.java | 10 + .../ducklake/function/CityHash64Function.java | 35 ++++ .../function/ClickHouseCityHash64.java | 178 ++++++++++++++++++ .../function/TestCityHash64Function.java | 89 +++++++++ 4 files changed, 312 insertions(+) create mode 100644 plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/CityHash64Function.java create mode 100644 plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/ClickHouseCityHash64.java create mode 100644 plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/TestCityHash64Function.java diff --git a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/DuckLakePlugin.java b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/DuckLakePlugin.java index 92a423ce7df6..9a081de6f259 100644 --- a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/DuckLakePlugin.java +++ b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/DuckLakePlugin.java @@ -14,9 +14,13 @@ package io.trino.plugin.ducklake; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import io.trino.plugin.ducklake.function.CityHash64Function; import io.trino.spi.Plugin; import io.trino.spi.connector.ConnectorFactory; +import java.util.Set; + public class DuckLakePlugin implements Plugin { @@ -25,4 +29,10 @@ public Iterable getConnectorFactories() { return ImmutableList.of(new DuckLakeConnectorFactory()); } + + @Override + public Set> getFunctions() + { + return ImmutableSet.of(CityHash64Function.class); + } } diff --git a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/CityHash64Function.java b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/CityHash64Function.java new file mode 100644 index 000000000000..49c13c671f6f --- /dev/null +++ b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/CityHash64Function.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ducklake.function; + +import io.airlift.slice.Slice; +import io.trino.spi.function.Description; +import io.trino.spi.function.LiteralParameters; +import io.trino.spi.function.ScalarFunction; +import io.trino.spi.function.SqlType; +import io.trino.spi.type.Int128; + +public final class CityHash64Function +{ + private CityHash64Function() {} + + @Description("Returns the ClickHouse CityHash 1.0.2 hash of a string") + @ScalarFunction("cityhash64") + @LiteralParameters("x") + @SqlType("decimal(20,0)") + public static Int128 cityHash64(@SqlType("varchar(x)") Slice value) + { + return Int128.valueOf(0, ClickHouseCityHash64.hash(value)); + } +} diff --git a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/ClickHouseCityHash64.java b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/ClickHouseCityHash64.java new file mode 100644 index 000000000000..0ded7eebd9df --- /dev/null +++ b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/ClickHouseCityHash64.java @@ -0,0 +1,178 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ducklake.function; + +import io.airlift.slice.Slice; + +final class ClickHouseCityHash64 +{ + private static final long K0 = 0xC3A5C85C97CB3127L; + private static final long K1 = 0xB492B66FBE98F273L; + private static final long K2 = 0x9AE16A3B2F90404FL; + private static final long K3 = 0xC949D7C7509E6557L; + private static final long HASH_128_TO_64_MULTIPLIER = 0x9DDFEA08EB382D69L; + + private ClickHouseCityHash64() {} + + public static long hash(Slice value) + { + int length = value.length(); + if (length <= 16) { + return hashLength0To16(value, length); + } + if (length <= 32) { + return hashLength17To32(value, length); + } + if (length <= 64) { + return hashLength33To64(value, length); + } + + long x = fetch64(value, 0); + long y = fetch64(value, length - 16) ^ K1; + long z = fetch64(value, length - 56) ^ K0; + LongPair v = weakHashLength32WithSeeds(value, length - 64, length, y); + LongPair w = weakHashLength32WithSeeds(value, length - 32, length * K1, K0); + z += shiftMix(v.high()) * K1; + x = rotateRight(z + x, 39) * K1; + y = rotateRight(y, 33) * K1; + + int offset = 0; + int remaining = (length - 1) & ~63; + while (remaining > 0) { + x = rotateRight(x + y + v.low() + fetch64(value, offset + 16), 37) * K1; + y = rotateRight(y + v.high() + fetch64(value, offset + 48), 42) * K1; + x ^= w.high(); + y ^= v.low(); + z = rotateRight(z ^ w.low(), 33); + v = weakHashLength32WithSeeds(value, offset, v.high() * K1, x + w.low()); + w = weakHashLength32WithSeeds(value, offset + 32, z + w.high(), y); + long previousX = x; + x = z; + z = previousX; + offset += 64; + remaining -= 64; + } + + return hash16( + hash16(v.low(), w.low()) + shiftMix(y) * K1 + z, + hash16(v.high(), w.high()) + x); + } + + private static long hashLength0To16(Slice value, int length) + { + if (length > 8) { + long a = fetch64(value, 0); + long b = fetch64(value, length - 8); + return hash16(a, rotateRight(b + length, length)) ^ b; + } + if (length >= 4) { + long a = fetch32(value, 0); + return hash16(length + (a << 3), fetch32(value, length - 4)); + } + if (length > 0) { + int a = value.getUnsignedByte(0); + int b = value.getUnsignedByte(length >>> 1); + int c = value.getUnsignedByte(length - 1); + int y = a + (b << 8); + int z = length + (c << 2); + return shiftMix((Integer.toUnsignedLong(y) * K2) ^ (Integer.toUnsignedLong(z) * K3)) * K2; + } + return K2; + } + + private static long hashLength17To32(Slice value, int length) + { + long a = fetch64(value, 0) * K1; + long b = fetch64(value, 8); + long c = fetch64(value, length - 8) * K2; + long d = fetch64(value, length - 16) * K0; + return hash16( + rotateRight(a - b, 43) + rotateRight(c, 30) + d, + a + rotateRight(b ^ K3, 20) - c + length); + } + + private static long hashLength33To64(Slice value, int length) + { + long z = fetch64(value, 24); + long a = fetch64(value, 0) + (length + fetch64(value, length - 16)) * K0; + long b = rotateRight(a + z, 52); + long c = rotateRight(a, 37); + a += fetch64(value, 8); + c += rotateRight(a, 7); + a += fetch64(value, 16); + + long vLow = a + z; + long vHigh = b + rotateRight(a, 31) + c; + a = fetch64(value, 16) + fetch64(value, length - 32); + z = fetch64(value, length - 8); + b = rotateRight(a + z, 52); + c = rotateRight(a, 37); + a += fetch64(value, length - 24); + c += rotateRight(a, 7); + a += fetch64(value, length - 16); + + long wLow = a + z; + long wHigh = b + rotateRight(a, 31) + c; + long r = shiftMix((vLow + wHigh) * K2 + (wLow + vHigh) * K0); + return shiftMix(r * K0 + vHigh) * K2; + } + + private static LongPair weakHashLength32WithSeeds(Slice value, int offset, long a, long b) + { + long w = fetch64(value, offset); + long x = fetch64(value, offset + 8); + long y = fetch64(value, offset + 16); + long z = fetch64(value, offset + 24); + a += w; + b = rotateRight(b + a + z, 21); + long c = a; + a += x + y; + b += rotateRight(a, 44); + return new LongPair(a + z, b + c); + } + + private static long hash16(long low, long high) + { + long a = (low ^ high) * HASH_128_TO_64_MULTIPLIER; + a ^= a >>> 47; + long b = (high ^ a) * HASH_128_TO_64_MULTIPLIER; + b ^= b >>> 47; + return b * HASH_128_TO_64_MULTIPLIER; + } + + private static long shiftMix(long value) + { + return value ^ (value >>> 47); + } + + private static long rotateRight(long value, int shift) + { + return Long.rotateRight(value, shift); + } + + private static long fetch32(Slice value, int offset) + { + return value.getUnsignedByte(offset) + | ((long) value.getUnsignedByte(offset + 1) << 8) + | ((long) value.getUnsignedByte(offset + 2) << 16) + | ((long) value.getUnsignedByte(offset + 3) << 24); + } + + private static long fetch64(Slice value, int offset) + { + return fetch32(value, offset) | (fetch32(value, offset + 4) << 32); + } + + private record LongPair(long low, long high) {} +} diff --git a/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/TestCityHash64Function.java b/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/TestCityHash64Function.java new file mode 100644 index 000000000000..7561fc237ef7 --- /dev/null +++ b/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/TestCityHash64Function.java @@ -0,0 +1,89 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ducklake.function; + +import io.airlift.slice.Slices; +import io.trino.plugin.ducklake.DuckLakePlugin; +import io.trino.spi.type.DecimalType; +import io.trino.sql.query.QueryAssertions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import java.util.Map; + +import static io.trino.spi.type.DecimalType.createDecimalType; +import static io.trino.spi.type.SqlDecimal.decimal; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; + +@TestInstance(PER_CLASS) +final class TestCityHash64Function +{ + private QueryAssertions assertions; + + @BeforeAll + void init() + { + assertions = new QueryAssertions(); + assertions.addPlugin(new DuckLakePlugin()); + } + + @AfterAll + void teardown() + { + assertions.close(); + assertions = null; + } + + @Test + void testClickHouseCompatibility() + { + Map expectedHashes = Map.ofEntries( + Map.entry("", "11160318154034397263"), + Map.entry("a", "2603192927274642682"), + Map.entry("abc", "4220206313085259313"), + Map.entry("abcd", "17823623939509273229"), + Map.entry("12345678", "7177601938557627951"), + Map.entry("123456789", "12390271160407166709"), + Map.entry("1234567890abcdef", "10283158570132023530"), + Map.entry("1234567890abcdefg", "4637769396780685212"), + Map.entry("1234567890abcdef1234567890abcdef", "5983709264297605835"), + Map.entry("1234567890abcdef1234567890abcdefg", "1434880753424560409"), + Map.entry("x".repeat(64), "6437053381938498259"), + Map.entry("x".repeat(65), "5260653789997849295"), + Map.entry("x".repeat(128), "1142585757146322653"), + Map.entry("y".repeat(129), "15472941566376925510"), + Map.entry("z".repeat(200), "14427141409027721128"), + Map.entry("q".repeat(257), "14786733977854578775"), + Map.entry("Moscow", "12507901496292878638"), + Map.entry("Grüße", "17364701079286602353")); + + expectedHashes.forEach((input, expected) -> assertThat(ClickHouseCityHash64.hash(Slices.utf8Slice(input))) + .isEqualTo(Long.parseUnsignedLong(expected))); + } + + @Test + void testFunctionRegistrationAndUnsignedResult() + { + DecimalType resultType = createDecimalType(20, 0); + assertThat(assertions.function("cityHash64", "'Moscow'")) + .hasType(resultType) + .isEqualTo(decimal("12507901496292878638", resultType)); + + assertThat(assertions.function("cityHash64", "CAST(NULL AS varchar)")) + .isNull(resultType); + } +} From 65d737ce4e3e3cff90d760e39d7af3f35e59fae8 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 01:30:50 -0400 Subject: [PATCH 113/136] feat(hogql): route cityhash64 compatibility --- .../java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java | 1 + .../io/trino/hogql/compiler/TestHogQlFunctionResolver.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 568ea8efae5e..9703860dc8a7 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -145,6 +145,7 @@ final class HogQlV0FunctionRegistry scalar("substringUTF8", "substring", signature(2), signature(3)), scalar("arrayConcat", "concat", variadicSignature(2)), scalar("roundBankers", "round", signature(1), signature(2)), + scalar("cityHash64", "cityhash64", signature(1)), scalar("hasAny", "arrays_overlap", signature(2)), scalar("parseDateTime", "date_parse", signature(2)), scalar("toLastDayOfMonth", "last_day_of_month", signature(1)), diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 775e8febd712..88d166348c5c 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -373,12 +373,12 @@ public void testCompilerLowersFinalStockCompatibilityFunctions() HogQlCompilationResult result = new HogQlCompiler().compile(envelope( "SELECT plus(total, surcharge), minus(total, discount), divide(total, count), notEquals(status, 'ignored'), " + "greaterOrEquals(total, minimum), lessOrEquals(total, maximum), " + - "toMonth(timestamp), toYear(timestamp), toDayOfWeek(timestamp), ceil(score), _toInt16(value), " + + "toMonth(timestamp), toYear(timestamp), toDayOfWeek(timestamp), ceil(score), _toInt16(value), cityHash64(value), " + "JSONExtractKeys(payload), JSONExtractKeys(payload, 'nested') FROM records")); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( "SELECT total + surcharge, total - discount, total / count, status <> 'ignored', total >= minimum, total <= maximum, " + - "month(timestamp), year(timestamp), day_of_week(timestamp), ceiling(score), CAST(value AS smallint), " + + "month(timestamp), year(timestamp), day_of_week(timestamp), ceiling(score), CAST(value AS smallint), cityhash64(value), " + "map_keys(coalesce(TRY_CAST(json_parse(payload) AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))), " + "map_keys(coalesce(TRY_CAST(json_extract(payload, '$[\"nested\"]') AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))) " + "FROM records")); From 061f3402b2dedd6111b57f1b11ffe6ca9aa2a049 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 01:31:41 -0400 Subject: [PATCH 114/136] feat(hogql): lower limit by queries --- core/trino-hogql-compiler/README.md | 2 + .../trino/hogql/compiler/HogQlCompiler.java | 11 +- .../hogql/compiler/HogQlFunctionResolver.java | 8 +- .../hogql/compiler/HogQlLimitByRewriter.java | 355 ++++++++++++++++++ .../hogql/compiler/HogQlProjectionDemand.java | 10 + .../hogql/compiler/HogQlSemanticResolver.java | 40 +- .../compiler/HogQlV0ProfileValidator.java | 5 + .../trino/hogql/compiler/TrinoAstFactory.java | 1 + .../hogql/compiler/TestHogQlCompiler.java | 22 ++ .../io/trino/hogql/parser/HogQlParser.java | 36 +- .../trino/hogql/parser/tree/HogQlQuery.java | 36 +- .../trino/hogql/parser/TestHogQlParser.java | 19 + 12 files changed, 520 insertions(+), 25 deletions(-) create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlLimitByRewriter.java diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 85e72245c235..5baf3347219b 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -52,6 +52,8 @@ The MVP lowers a frozen, fail-closed subset directly to stock Trino AST nodes: - parameters, CTEs, subqueries, joins, set operations, arrays, maps, rows, intervals, grouping, ordering, limits, and windows that have exact stock Trino representations; +- `LIMIT BY` over explicit projected outputs, lowered through a partitioned + `row_number` query while preserving global ordering and pagination; - numeric/conditional/string functions (`abs`, `coalesce`, `if`, `lower`, `upper`, `length`, `concat`, `replace`), collection functions (`map`, `arraySort`, `arrayDistinct`, `arrayFlatten`, `arrayStringConcat`), date functions diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index 578a7b2fc1f6..51aa0781853a 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -239,6 +239,11 @@ private static void collectPlaceholders(HogQlQuery query, List plac select.groupBy().forEach(expression -> collectPlaceholders(expression, placeholders)); select.having().ifPresent(expression -> collectPlaceholders(expression, placeholders)); select.windows().forEach(window -> collectPlaceholders(window.specification(), placeholders)); + select.limitBy().ifPresent(limitBy -> { + collectPlaceholders(limitBy.limit(), placeholders); + limitBy.offset().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + limitBy.partitionBy().forEach(expression -> collectPlaceholders(expression, placeholders)); + }); } case SetOperation setOperation -> { collectPlaceholders(setOperation.left(), placeholders); @@ -337,7 +342,11 @@ private static boolean containsFunctionCall(HogQlQuery query) select.where().map(HogQlCompiler::containsFunctionCall).orElse(false) || select.groupBy().stream().anyMatch(HogQlCompiler::containsFunctionCall) || select.having().map(HogQlCompiler::containsFunctionCall).orElse(false) || - select.windows().stream().map(WindowDefinition::specification).anyMatch(HogQlCompiler::containsFunctionCall); + select.windows().stream().map(WindowDefinition::specification).anyMatch(HogQlCompiler::containsFunctionCall) || + select.limitBy().map(limitBy -> + containsFunctionCall(limitBy.limit()) || + containsFunctionCall(limitBy.offset()) || + limitBy.partitionBy().stream().anyMatch(HogQlCompiler::containsFunctionCall)).orElse(false); case SetOperation set -> containsFunctionCall(set.left()) || containsFunctionCall(set.right()); } || query.orderBy().stream().anyMatch(sortItem -> containsFunctionCall(sortItem.expression())) || diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 32018481752f..909190212339 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -47,6 +47,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; +import io.trino.hogql.parser.tree.HogQlQuery.LimitBy; import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.PivotAggregation; import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; @@ -165,6 +166,11 @@ private HogQlQuery resolveSelectQuery(HogQlQuery query, List groupBy = select.groupBy().stream().map(this::resolveExpression).toList(); Optional having = select.having().map(this::resolveExpression); List windows = select.windows().stream().map(this::resolveWindowDefinition).toList(); + Optional limitBy = select.limitBy().map(clause -> new LimitBy( + resolveExpression(clause.limit()), + clause.offset().map(this::resolveExpression), + clause.partitionBy().stream().map(this::resolveExpression).toList(), + clause.span())); List orderBy = resolveSortItems(query.orderBy()); Optional limit = query.limit().map(this::resolveExpression); Optional offset = query.offset().map(this::resolveExpression); @@ -175,7 +181,7 @@ private HogQlQuery resolveSelectQuery(HogQlQuery query, List outputs = outputs(select.projections(), limitBy.span()); + Map outputsByName = outputsByName(outputs); + + List baseProjections = outputs.stream() + .map(output -> new ExpressionProjection(output.expression(), Optional.of(output.innerName()))) + .map(Projection.class::cast) + .toList(); + HogQlQuery baseQuery = new HogQlQuery( + List.of(), + new SelectQueryBody( + select.distinct(), + baseProjections, + select.from(), + select.where(), + select.groupBy(), + select.having(), + select.windows(), + Optional.empty(), + select.span()), + List.of(), + Optional.empty(), + Optional.empty(), + query.span()); + + Identifier baseAlias = identifier(BASE_ALIAS, limitBy.span()); + List rankedProjections = new ArrayList<>(); + outputs.forEach(output -> rankedProjections.add(new ExpressionProjection( + column(baseAlias, output.innerName(), output.expression().span()), + Optional.of(output.innerName())))); + WindowSpecification rankingWindow = new WindowSpecification( + limitBy.partitionBy().stream() + .map(expression -> remap(expression, outputsByName, baseAlias)) + .toList(), + query.orderBy().stream() + .map(sortItem -> remap(sortItem, outputsByName, baseAlias)) + .toList(), + Optional.empty(), + limitBy.span()); + FunctionCall rowNumber = new FunctionCall( + List.of(identifier("row_number", limitBy.span())), + List.of(), + false, + List.of(), + Optional.empty(), + Optional.empty(), + Optional.of(rankingWindow), + limitBy.span()); + rankedProjections.add(new ExpressionProjection(rowNumber, Optional.of(identifier(ROW_NUMBER, limitBy.span())))); + HogQlQuery rankedQuery = new HogQlQuery( + List.of(), + new SelectQueryBody( + false, + rankedProjections, + Optional.of(new HogQlQuery.AliasedRelation(new SubqueryRelation(baseQuery, query.span()), baseAlias, query.span())), + Optional.empty(), + List.of(), + Optional.empty(), + List.of(), + Optional.empty(), + select.span()), + List.of(), + Optional.empty(), + Optional.empty(), + query.span()); + + Identifier rankedAlias = identifier(RANKED_ALIAS, limitBy.span()); + List outerProjections = outputs.stream() + .map(output -> new ExpressionProjection( + column(rankedAlias, output.innerName(), output.expression().span()), + Optional.of(output.outputName()))) + .map(Projection.class::cast) + .toList(); + Expression rowNumberReference = column(rankedAlias, identifier(ROW_NUMBER, limitBy.span()), limitBy.span()); + Expression upperBound = limitBy.offset() + .map(offset -> new BinaryExpression(BinaryOperator.ADD, offset, limitBy.limit(), limitBy.span())) + .orElse(limitBy.limit()); + Expression filter = new BinaryExpression(BinaryOperator.LESS_THAN_OR_EQUAL, rowNumberReference, upperBound, limitBy.span()); + if (limitBy.offset().isPresent()) { + filter = new BinaryExpression( + BinaryOperator.AND, + new BinaryExpression(BinaryOperator.GREATER_THAN, rowNumberReference, limitBy.offset().orElseThrow(), limitBy.span()), + filter, + limitBy.span()); + } + List outerOrderBy = query.orderBy().stream() + .map(sortItem -> remap(sortItem, outputsByName, rankedAlias)) + .toList(); + return new HogQlQuery( + query.with(), + new SelectQueryBody( + false, + outerProjections, + Optional.of(new HogQlQuery.AliasedRelation(new SubqueryRelation(rankedQuery, query.span()), rankedAlias, query.span())), + Optional.of(filter), + List.of(), + Optional.empty(), + List.of(), + Optional.empty(), + select.span()), + outerOrderBy, + query.limit(), + query.offset(), + query.span()); + } + + private static List outputs(List projections, SourceSpan span) + { + List outputs = new ArrayList<>(); + for (Projection projection : projections) { + if (!(projection instanceof ExpressionProjection expression)) { + throw unsupported(span, "HogQL LIMIT BY requires explicit output columns"); + } + Identifier outputName = expression.alias() + .orElseGet(() -> { + if (expression.expression() instanceof ColumnReference reference) { + return reference.parts().getLast(); + } + throw unsupported(expression.span(), "HogQL LIMIT BY expressions require output aliases"); + }); + outputs.add(new OutputColumn( + expression.expression(), + outputName, + identifier("__hogql_limit_by_column_" + outputs.size(), expression.span()))); + } + return List.copyOf(outputs); + } + + private static Map outputsByName(List outputs) + { + Map byName = new HashMap<>(); + Set duplicates = new HashSet<>(); + for (OutputColumn output : outputs) { + String name = canonical(output.outputName().value()); + if (byName.putIfAbsent(name, output) != null) { + duplicates.add(name); + } + } + duplicates.forEach(byName::remove); + return Map.copyOf(byName); + } + + private static SortItem remap(SortItem sortItem, Map outputs, Identifier relationAlias) + { + return new SortItem( + remap(sortItem.expression(), outputs, relationAlias), + sortItem.direction(), + sortItem.nullPlacement(), + sortItem.span()); + } + + private static Expression remap(Expression expression, Map outputs, Identifier relationAlias) + { + return switch (expression) { + case ArrayExpression array -> new ArrayExpression(array.values().stream().map(value -> remap(value, outputs, relationAlias)).toList(), array.span()); + case BetweenExpression between -> new BetweenExpression( + remap(between.value(), outputs, relationAlias), + remap(between.min(), outputs, relationAlias), + remap(between.max(), outputs, relationAlias), + between.negated(), + between.predicateSpan(), + between.span()); + case BinaryExpression binary -> new BinaryExpression( + binary.operator(), + remap(binary.left(), outputs, relationAlias), + remap(binary.right(), outputs, relationAlias), + binary.span()); + case CaseExpression caseExpression -> new CaseExpression( + caseExpression.operand().map(value -> remap(value, outputs, relationAlias)), + caseExpression.whenClauses().stream() + .map(when -> new CaseWhen( + remap(when.operand(), outputs, relationAlias), + remap(when.result(), outputs, relationAlias), + when.span())) + .toList(), + caseExpression.defaultValue().map(value -> remap(value, outputs, relationAlias)), + caseExpression.span()); + case CastExpression cast -> new CastExpression(remap(cast.value(), outputs, relationAlias), cast.type(), cast.safe(), cast.typeDialect(), cast.span()); + case ColumnReference reference -> remap(reference, outputs, relationAlias); + case FunctionCall function -> new FunctionCall( + function.nameParts(), + function.arguments().stream().map(argument -> remap(argument, outputs, relationAlias)).toList(), + function.distinct(), + function.orderBy().stream().map(item -> remap(item, outputs, relationAlias)).toList(), + function.filter().map(filter -> remap(filter, outputs, relationAlias)), + function.nullTreatment(), + function.window().map(_ -> { + throw unsupported(function.span(), "HogQL LIMIT BY does not support window expressions in its keys or ordering"); + }), + function.span()); + case InCohortExpression in -> new InCohortExpression( + remap(in.value(), outputs, relationAlias), + remap(in.cohort(), outputs, relationAlias), + in.negated(), + in.predicateSpan(), + in.span()); + case InExpression in -> new InExpression( + remap(in.value(), outputs, relationAlias), + in.values().stream().map(value -> remap(value, outputs, relationAlias)).toList(), + in.negated(), + in.predicateSpan(), + in.span()); + case HogQlQuery.InSubqueryExpression in -> throw unsupported(in.span(), "HogQL LIMIT BY does not support subqueries in its keys or ordering"); + case IntervalExpression interval -> new IntervalExpression(remap(interval.value(), outputs, relationAlias), interval.unit(), interval.span()); + case IsNullExpression isNull -> new IsNullExpression( + remap(isNull.value(), outputs, relationAlias), + isNull.negated(), + isNull.predicateSpan(), + isNull.span()); + case HogQlQuery.LambdaExpression lambda -> throw unsupported(lambda.span(), "HogQL LIMIT BY does not support lambdas in its keys or ordering"); + case Literal literal -> literal; + case MemberAccessExpression member -> new MemberAccessExpression(remap(member.base(), outputs, relationAlias), member.member(), member.span()); + case Placeholder placeholder -> placeholder; + case ScalarSubqueryExpression subquery -> throw unsupported(subquery.span(), "HogQL LIMIT BY does not support subqueries in its keys or ordering"); + case SubscriptExpression subscript -> new SubscriptExpression( + remap(subscript.base(), outputs, relationAlias), + remap(subscript.index(), outputs, relationAlias), + subscript.span()); + case TupleExpression tuple -> new TupleExpression(tuple.values().stream().map(value -> remap(value, outputs, relationAlias)).toList(), tuple.span()); + case UnaryExpression unary -> new UnaryExpression(unary.operator(), remap(unary.operand(), outputs, relationAlias), unary.span()); + }; + } + + private static Expression remap(ColumnReference reference, Map outputs, Identifier relationAlias) + { + if (reference.parts().size() == 1) { + OutputColumn output = outputs.get(canonical(reference.parts().getFirst().value())); + if (output != null) { + return column(relationAlias, output.innerName(), reference.span()); + } + } + Optional exact = outputs.values().stream() + .filter(output -> output.expression() instanceof ColumnReference) + .filter(output -> sameReference(reference, (ColumnReference) output.expression())) + .findFirst(); + if (exact.isPresent()) { + return column(relationAlias, exact.orElseThrow().innerName(), reference.span()); + } + throw unsupported(reference.span(), "HogQL LIMIT BY keys and ordering must reference projected outputs"); + } + + private static boolean sameReference(ColumnReference left, ColumnReference right) + { + if (left.parts().size() != right.parts().size()) { + return false; + } + for (int index = 0; index < left.parts().size(); index++) { + Identifier leftPart = left.parts().get(index); + Identifier rightPart = right.parts().get(index); + if (leftPart.delimited() != rightPart.delimited() || !canonical(leftPart.value()).equals(canonical(rightPart.value()))) { + return false; + } + } + return true; + } + + private static ColumnReference column(Identifier relation, Identifier column, SourceSpan span) + { + return new ColumnReference(List.of(relation, column), span); + } + + private static Identifier identifier(String value, SourceSpan span) + { + return new Identifier(value, false, span); + } + + private static String canonical(String value) + { + return value.toLowerCase(Locale.ENGLISH); + } + + private static TrinoException unsupported(SourceSpan span, String message) + { + return new TrinoException( + HOGQL_UNSUPPORTED_FEATURE, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } + + private record OutputColumn(Expression expression, Identifier outputName, Identifier innerName) {} +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java index 3e0af14fe7c0..abfa5091285d 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java @@ -86,6 +86,11 @@ public static HogQlProjectionDemand collect(HogQlQuery query) select.groupBy().forEach(expression -> collect(expression, builder)); select.having().ifPresent(expression -> collect(expression, builder)); select.windows().forEach(window -> collect(window.specification(), builder)); + select.limitBy().ifPresent(limitBy -> { + collect(limitBy.limit(), builder); + limitBy.offset().ifPresent(expression -> collect(expression, builder)); + limitBy.partitionBy().forEach(expression -> collect(expression, builder)); + }); select.from().ifPresent(relation -> collectJoinCriteria(relation, builder)); } query.orderBy().forEach(sortItem -> collect(sortItem.expression(), builder)); @@ -143,6 +148,11 @@ public static HogQlProjectionDemand collectNonProjection(HogQlQuery query) select.groupBy().forEach(expression -> collect(expression, builder)); select.having().ifPresent(expression -> collect(expression, builder)); select.windows().forEach(window -> collect(window.specification(), builder)); + select.limitBy().ifPresent(limitBy -> { + collect(limitBy.limit(), builder); + limitBy.offset().ifPresent(expression -> collect(expression, builder)); + limitBy.partitionBy().forEach(expression -> collect(expression, builder)); + }); select.from().ifPresent(relation -> collectJoinCriteria(relation, builder)); } query.orderBy().forEach(sortItem -> collect(sortItem.expression(), builder)); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index 2a0fd7dd1bbb..7c1c591b5c36 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -76,6 +76,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; +import io.trino.hogql.parser.tree.HogQlQuery.LimitBy; import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.PivotAggregation; import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; @@ -224,13 +225,16 @@ private HogQlQuery resolveNestedQuery(HogQlQuery query) if (bindings.isEmpty() && outerBindingScopes.isEmpty()) { return new HogQlQuery( commonTables, - select.distinct(), - select.projections(), - relation.map(ResolvedRelation::relation), - select.where(), - select.groupBy(), - select.having(), - select.windows(), + new SelectQueryBody( + select.distinct(), + select.projections(), + relation.map(ResolvedRelation::relation), + select.where(), + select.groupBy(), + select.having(), + select.windows(), + select.limitBy(), + select.span()), query.orderBy(), query.limit(), query.offset(), @@ -677,18 +681,26 @@ private HogQlQuery resolveQuery(HogQlQuery query, List co List groupBy = query.groupBy().stream().map(this::resolveExpression).toList(); Optional having = query.having().map(this::resolveExpression); List windows = query.windows().stream().map(this::resolveWindowDefinition).toList(); + Optional limitBy = query.limitBy().map(clause -> new LimitBy( + resolveExpression(clause.limit()), + clause.offset().map(this::resolveExpression), + clause.partitionBy().stream().map(this::resolveExpression).toList(), + clause.span())); List orderBy = resolveSortItems(query.orderBy()); Optional limit = query.limit().map(this::resolveExpression); Optional offset = query.offset().map(this::resolveExpression); return new HogQlQuery( commonTables, - query.distinct(), - projections, - expandedRelation, - where, - groupBy, - having, - windows, + new SelectQueryBody( + query.distinct(), + projections, + expandedRelation, + where, + groupBy, + having, + windows, + limitBy, + query.body().span()), orderBy, limit, offset, diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java index c72ea52879a2..6d248417f721 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java @@ -83,6 +83,11 @@ public static void validate(HogQlQuery query, Optional validate(expression, snapshot)); select.having().ifPresent(expression -> validate(expression, snapshot)); select.windows().forEach(window -> validate(window.specification(), snapshot)); + select.limitBy().ifPresent(limitBy -> { + validate(limitBy.limit(), snapshot); + limitBy.offset().ifPresent(expression -> validate(expression, snapshot)); + limitBy.partitionBy().forEach(expression -> validate(expression, snapshot)); + }); } case SetOperation setOperation -> { validate(setOperation.left(), snapshot); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index e418765cd32c..2ddb67ec72f6 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -147,6 +147,7 @@ public static Statement createStatement(HogQlQuery query, Map parameterIds) { + query = HogQlLimitByRewriter.rewrite(query); NodeLocation location = location(query.span()); QueryBody queryBody = createQueryBody(query, parameterIds); boolean setOperation = query.body() instanceof SetOperation; diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index e53f540a447a..668a65a2a777 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -102,6 +102,28 @@ public void testLowersDistinctOrderingAndClickHousePaginationOrder() assertThat(statement).isEqualTo(sqlParser.createStatement("SELECT DISTINCT event FROM events ORDER BY event DESC NULLS FIRST OFFSET 2 LIMIT 10")); } + @Test + public void testLowersLimitByToPartitionedRowNumber() + { + Statement statement = compiler.compile( + "SELECT event, created_at AS ts FROM events ORDER BY ts DESC LIMIT 1, 2 BY event LIMIT 10"); + + assertThat(statement).isEqualTo(sqlParser.createStatement( + "SELECT __hogql_limit_by_ranked.__hogql_limit_by_column_0 AS event, " + + "__hogql_limit_by_ranked.__hogql_limit_by_column_1 AS ts " + + "FROM (" + + "SELECT __hogql_limit_by_base.__hogql_limit_by_column_0 AS __hogql_limit_by_column_0, " + + "__hogql_limit_by_base.__hogql_limit_by_column_1 AS __hogql_limit_by_column_1, " + + "row_number() OVER (PARTITION BY __hogql_limit_by_base.__hogql_limit_by_column_0 " + + "ORDER BY __hogql_limit_by_base.__hogql_limit_by_column_1 DESC) AS __hogql_limit_by_row_number " + + "FROM (SELECT event AS __hogql_limit_by_column_0, created_at AS __hogql_limit_by_column_1 FROM events) " + + "AS __hogql_limit_by_base" + + ") AS __hogql_limit_by_ranked " + + "WHERE __hogql_limit_by_ranked.__hogql_limit_by_row_number > 1 " + + "AND __hogql_limit_by_ranked.__hogql_limit_by_row_number <= 1 + 2 " + + "ORDER BY __hogql_limit_by_ranked.__hogql_limit_by_column_1 DESC LIMIT 10")); + } + @Test public void testWrapsSetOperandsWithLocalClauses() { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index c0991596658b..4f83d0dc337b 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -49,6 +49,7 @@ import io.trino.hogql.parser.tree.HogQlQuery.JoinType; import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; +import io.trino.hogql.parser.tree.HogQlQuery.LimitBy; import io.trino.hogql.parser.tree.HogQlQuery.Literal; import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; @@ -754,19 +755,23 @@ private HogQlQuery buildQuery(SelectedQuery selectedQuery) .map(this::buildExpression); List windows = buildWindowDefinitions(select.windowClause()); List orderBy = buildOrderBy(selectedQuery.orderBy() != null ? selectedQuery.orderBy() : select.orderByClause()); + Optional limitBy = Optional.ofNullable(select.limitByClause()).map(this::buildLimitBy); Pagination pagination = mergePagination( buildPagination(select.limitAndOffsetClause(), select.offsetOnlyClause()), selectedQuery.pagination() == null ? new Pagination(Optional.empty(), Optional.empty()) : buildPagination(selectedQuery.pagination()), select); return new HogQlQuery( with, - select.DISTINCT() != null, - projections, - from, - where, - groupBy, - having, - windows, + new HogQlQuery.SelectQueryBody( + select.DISTINCT() != null, + projections, + from, + where, + groupBy, + having, + windows, + limitBy, + sourceSpan(select)), orderBy, pagination.limit(), pagination.offset(), @@ -846,7 +851,6 @@ private void rejectUnsupportedClauses(HogQLParser.SelectStmtContext context) clauses.add(context.prewhereClause()); clauses.addAll(context.sampleClause()); clauses.add(context.qualifyClause()); - clauses.add(context.limitByClause()); clauses.add(context.settingsClause()); Optional firstClause = clauses.stream() .filter(requireNonNullClause -> requireNonNullClause != null) @@ -856,6 +860,22 @@ private void rejectUnsupportedClauses(HogQLParser.SelectStmtContext context) } } + private LimitBy buildLimitBy(HogQLParser.LimitByClauseContext context) + { + List limitExpressions = context.limitExpr().columnExpr(); + Expression limit = buildPaginationExpression( + limitExpressions.size() == 2 && context.limitExpr().COMMA() != null + ? limitExpressions.getLast() + : limitExpressions.getFirst()); + Optional offset = limitExpressions.size() == 2 + ? Optional.of(buildPaginationExpression( + context.limitExpr().COMMA() != null + ? limitExpressions.getFirst() + : limitExpressions.getLast())) + : Optional.empty(); + return new LimitBy(limit, offset, buildExpressions(context.columnExprList()), sourceSpan(context)); + } + private Relation buildArrayJoin(Optional from, HogQLParser.ArrayJoinClauseContext context) { List expressions = new ArrayList<>(); diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index 7f45a87fa7ca..6554c62e71ff 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -99,6 +99,11 @@ public Optional having() return selectBody().having(); } + public Optional limitBy() + { + return selectBody().limitBy(); + } + public List windows() { return selectBody().windows(); @@ -126,6 +131,7 @@ public record SelectQueryBody( List groupBy, Optional having, List windows, + Optional limitBy, SourceSpan span) implements QueryBody { @@ -137,9 +143,23 @@ public record SelectQueryBody( groupBy = List.copyOf(requireNonNull(groupBy, "groupBy is null")); having = requireNonNull(having, "having is null"); windows = List.copyOf(requireNonNull(windows, "windows is null")); + limitBy = requireNonNull(limitBy, "limitBy is null"); span = requireNonNull(span, "span is null"); } + public SelectQueryBody( + boolean distinct, + List projections, + Optional from, + Optional where, + List groupBy, + Optional having, + List windows, + SourceSpan span) + { + this(distinct, projections, from, where, groupBy, having, windows, Optional.empty(), span); + } + public SelectQueryBody( boolean distinct, List projections, @@ -149,7 +169,21 @@ public SelectQueryBody( Optional having, SourceSpan span) { - this(distinct, projections, from, where, groupBy, having, List.of(), span); + this(distinct, projections, from, where, groupBy, having, List.of(), Optional.empty(), span); + } + } + + public record LimitBy(Expression limit, Optional offset, List partitionBy, SourceSpan span) + { + public LimitBy + { + limit = requireNonNull(limit, "limit is null"); + offset = requireNonNull(offset, "offset is null"); + partitionBy = List.copyOf(requireNonNull(partitionBy, "partitionBy is null")); + if (partitionBy.isEmpty()) { + throw new IllegalArgumentException("partitionBy is empty"); + } + span = requireNonNull(span, "span is null"); } } diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 6c88676e951f..7440aca9034c 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -248,6 +248,25 @@ public void testBuildsDistinctOrderingAndPaginationAst() assertThat(query.offset()).get().extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("2"); } + @Test + public void testBuildsLimitByAst() + { + HogQlQuery query = parser.parseStatement("SELECT event FROM events LIMIT 2, 3 BY event, properties.plan LIMIT 10"); + + assertThat(query.limitBy()).get().satisfies(limitBy -> { + assertThat(limitBy.limit()).extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("3"); + assertThat(limitBy.offset()).get().extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("2"); + assertThat(limitBy.partitionBy()).hasSize(2); + }); + assertThat(query.limit()).get().extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("10"); + + HogQlQuery offsetQuery = parser.parseStatement("SELECT event FROM events LIMIT 3 OFFSET 2 BY event"); + assertThat(offsetQuery.limitBy()).get().satisfies(limitBy -> { + assertThat(limitBy.limit()).extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("3"); + assertThat(limitBy.offset()).get().extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("2"); + }); + } + @Test public void testBuildsCollectionAccessWithSourceSpans() { From 8872c8faea580d4c25ae2615d3232b5b1053c33c Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 01:39:27 -0400 Subject: [PATCH 115/136] feat(hogql): lower map constructors and updates --- core/trino-hogql-compiler/README.md | 2 ++ .../hogql/compiler/HogQlFunctionResolver.java | 24 +++++++++++++++++++ .../compiler/HogQlV0FunctionRegistry.java | 4 +++- .../catalog/HogQlSemanticCatalogSnapshot.java | 3 +++ .../compiler/TestHogQlFunctionResolver.java | 18 +++++++++++--- 5 files changed, 47 insertions(+), 4 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 5baf3347219b..1955e486fbcf 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -88,6 +88,8 @@ The extended profile also lowers extract-all and first-match replacement, plus boolean, unsigned-integer, raw-array, and nested raw-map JSON extraction. Common array transforms, predicates, indexing, summation, ranges, tuple indexing, splitting, and membership lower to Trino higher-order functions. +Map key/value constructors and `mapUpdate` lower to stock map construction and +last-map-wins concatenation. The row-expanding `arrayJoin` function is hoisted to a generated lateral `CROSS JOIN UNNEST`; repeated calls remain independent row expansions. Range and membership rewrites preserve empty-range and null-search behavior. diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 909190212339..f8c062f36fe9 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -454,6 +454,7 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< case MULTIPLY, MULTIPLY_DECIMAL -> new BinaryExpression(HogQlQuery.BinaryOperator.MULTIPLY, arguments.getFirst(), arguments.get(1), span); case DIVIDE_DECIMAL -> new BinaryExpression(HogQlQuery.BinaryOperator.DIVIDE, arguments.getFirst(), arguments.get(1), span); case IN_ARRAY -> call("contains", List.of(arguments.get(1), arguments.getFirst()), span); + case MAP_CONSTRUCTOR -> map(function, arguments); case TUPLE -> new TupleExpression(arguments, span); case SUBTRACT_MONTHS -> dateAdd( "month", @@ -899,6 +900,29 @@ private static Expression jsonPath(FunctionCall function, List segme return call("concat", pathParts, function.span()); } + private static Expression map(FunctionCall function, List arguments) + { + if (arguments.isEmpty()) { + return call("map", List.of(), function.span()); + } + if (arguments.getFirst() instanceof LambdaExpression) { + return call("map", arguments, function.span()); + } + if (arguments.size() % 2 != 0) { + throw resolutionError(function, "HogQL function map requires key/value argument pairs"); + } + List keys = new ArrayList<>(arguments.size() / 2); + List values = new ArrayList<>(arguments.size() / 2); + for (int index = 0; index < arguments.size(); index += 2) { + keys.add(arguments.get(index)); + values.add(arguments.get(index + 1)); + } + return call( + "map", + List.of(new ArrayExpression(keys, function.span()), new ArrayExpression(values, function.span())), + function.span()); + } + private static FunctionCall coalesce(Expression value, Expression defaultValue, HogQlQuery.SourceSpan span) { return call("coalesce", List.of(value, defaultValue), span); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 9703860dc8a7..796eb268edd0 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -138,6 +138,7 @@ final class HogQlV0FunctionRegistry scalar("toDayOfMonth", "day", signature(1)), scalar("toDayOfWeek", "day_of_week", signature(1)), scalar("mapFromArrays", "map", signature(2)), + scalar("mapUpdate", "map_concat", signature(2)), scalar("toMonth", "month", signature(1)), scalar("toYear", "year", signature(1)), scalar("ceil", "ceiling", signature(1)), @@ -146,10 +147,11 @@ final class HogQlV0FunctionRegistry scalar("arrayConcat", "concat", variadicSignature(2)), scalar("roundBankers", "round", signature(1), signature(2)), scalar("cityHash64", "cityhash64", signature(1)), + scalar("formatReadableTimeDelta", "format_readable_time_delta", signature(1), signature(2), signature(3)), scalar("hasAny", "arrays_overlap", signature(2)), scalar("parseDateTime", "date_parse", signature(2)), scalar("toLastDayOfMonth", "last_day_of_month", signature(1)), - scalar("map", "map", signature(2)), + rewrite("map", FunctionRewrite.MAP_CONSTRUCTOR, "map(any,any)", signature(0), variadicSignature(2)), scalar("dateDiff", "date_diff", signature(3)), scalar("dateTrunc", "date_trunc", signature(2)), rewrite("arraySort", FunctionRewrite.ARRAY_SORT, "array(any)", signature(1), signature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index c4b9b3b48c71..e5726d898abd 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -333,6 +333,8 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi (signature.variadic() && signature.argumentTypes().size() == 3); case ARRAY_SORT, RANGE -> !signature.variadic() && (signature.argumentTypes().size() == 1 || signature.argumentTypes().size() == 2); + case MAP_CONSTRUCTOR -> (!signature.variadic() && signature.argumentTypes().isEmpty()) || + (signature.variadic() && signature.argumentTypes().size() == 3); case MULTI_IF -> signature.variadic() && signature.argumentTypes().size() == 4; case TUPLE -> signature.variadic() && signature.argumentTypes().size() == 2; case TODAY -> !signature.variadic() && signature.argumentTypes().isEmpty(); @@ -1388,6 +1390,7 @@ public enum FunctionRewrite LIKE, LESS_OR_EQUAL, MAX_IF, + MAP_CONSTRUCTOR, MD5, MEDIAN_IF, MIN_IF, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 88d166348c5c..1dabdb954d23 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -374,11 +374,15 @@ public void testCompilerLowersFinalStockCompatibilityFunctions() "SELECT plus(total, surcharge), minus(total, discount), divide(total, count), notEquals(status, 'ignored'), " + "greaterOrEquals(total, minimum), lessOrEquals(total, maximum), " + "toMonth(timestamp), toYear(timestamp), toDayOfWeek(timestamp), ceil(score), _toInt16(value), cityHash64(value), " + + "formatReadableTimeDelta(duration), formatReadableTimeDelta(duration, 'days'), " + + "formatReadableTimeDelta(duration, 'hours', 'minutes'), " + "JSONExtractKeys(payload), JSONExtractKeys(payload, 'nested') FROM records")); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( "SELECT total + surcharge, total - discount, total / count, status <> 'ignored', total >= minimum, total <= maximum, " + "month(timestamp), year(timestamp), day_of_week(timestamp), ceiling(score), CAST(value AS smallint), cityhash64(value), " + + "format_readable_time_delta(duration), format_readable_time_delta(duration, 'days'), " + + "format_readable_time_delta(duration, 'hours', 'minutes'), " + "map_keys(coalesce(TRY_CAST(json_parse(payload) AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))), " + "map_keys(coalesce(TRY_CAST(json_extract(payload, '$[\"nested\"]') AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))) " + "FROM records")); @@ -465,11 +469,19 @@ public void testCompilerLowersRemainingScalarAliases() public void testCompilerLowersFinalStockAliases() { HogQlCompilationResult result = new HogQlCompiler().compile(envelope( - "SELECT floor(score), toDayOfMonth(timestamp), mapFromArrays(keys, values_array), " + - "date_part('hour', timestamp), isNull(value) FROM records")); + "SELECT floor(score), toDayOfMonth(timestamp), mapFromArrays(keys, values_array), mapUpdate(existing, updates), " + + "map(), map('a', 1, 'b', 2), date_part('hour', timestamp), isNull(value) FROM records")); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( - "SELECT floor(score), day(timestamp), map(keys, values_array), hour(timestamp), value IS NULL FROM records")); + "SELECT floor(score), day(timestamp), map(keys, values_array), map_concat(existing, updates), " + + "map(), map(ARRAY['a', 'b'], ARRAY[1, 2]), hour(timestamp), value IS NULL FROM records")); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope("SELECT map('a', 1, 'b')"))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception).hasMessageContaining("requires key/value argument pairs"); } @Test From d661b5502c0fca96db535c83db79bbfbd1eaec17 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 01:47:02 -0400 Subject: [PATCH 116/136] fix(hogql): bound large set query lowering --- .../trino/hogql/compiler/TrinoAstFactory.java | 54 +++++++++++++++++-- .../hogql/compiler/TestHogQlCompiler.java | 24 +++++++++ .../trino/hogql/parser/HogQlParserLimits.java | 2 +- .../hogql/parser/TestHogQlParserLimits.java | 27 ++++++++++ 4 files changed, 102 insertions(+), 5 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 2ddb67ec72f6..246cbeb652e3 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -126,7 +126,10 @@ import io.trino.sql.tree.With; import io.trino.sql.tree.WithQuery; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Deque; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -195,10 +198,38 @@ private static QuerySpecification createQuerySpecification( } private static QueryBody createSetOperation(SetOperation setOperation, Map parameterIds) + { + Deque pending = new ArrayDeque<>(); + Map lowered = new IdentityHashMap<>(); + pending.push(new SetOperationFrame(setOperation, false)); + + while (!pending.isEmpty()) { + SetOperationFrame frame = pending.pop(); + if (!frame.operandsLowered()) { + pending.push(new SetOperationFrame(frame.operation(), true)); + addInlineSetOperand(pending, frame.operation().right(), frame.operation().rightParenthesized()); + addInlineSetOperand(pending, frame.operation().left(), frame.operation().leftParenthesized()); + continue; + } + + SetOperation operation = frame.operation(); + QueryBody left = createSetOperand(operation.left(), operation.leftParenthesized(), parameterIds, lowered); + QueryBody right = createSetOperand(operation.right(), operation.rightParenthesized(), parameterIds, lowered); + lowered.put(operation, createSetOperation(operation, left, right)); + } + return lowered.get(setOperation); + } + + private static void addInlineSetOperand(Deque pending, HogQlQuery query, boolean parenthesized) + { + if (!requiresQueryWrapper(query, parenthesized) && query.body() instanceof SetOperation operation) { + pending.push(new SetOperationFrame(operation, false)); + } + } + + private static QueryBody createSetOperation(SetOperation setOperation, QueryBody left, QueryBody right) { NodeLocation location = location(setOperation.operatorSpan()); - QueryBody left = createSetOperand(setOperation.left(), setOperation.leftParenthesized(), parameterIds); - QueryBody right = createSetOperand(setOperation.right(), setOperation.rightParenthesized(), parameterIds); return switch (setOperation.type()) { case EXCEPT -> new Except(location, left, right, setOperation.distinct(), Optional.empty()); case INTERSECT -> new Intersect(location, List.of(left, right), setOperation.distinct(), Optional.empty()); @@ -209,14 +240,29 @@ private static QueryBody createSetOperation(SetOperation setOperation, Map parameterIds) + Map parameterIds, + Map lowered) { - if (parenthesized || !query.with().isEmpty() || !query.orderBy().isEmpty() || query.limit().isPresent() || query.offset().isPresent()) { + if (requiresQueryWrapper(query, parenthesized)) { return new TableSubquery(location(query.span()), createQuery(query, parameterIds)); } + if (query.body() instanceof SetOperation setOperation) { + QueryBody result = lowered.get(setOperation); + if (result == null) { + throw new IllegalStateException("set operation operand has not been lowered"); + } + return result; + } return createQueryBody(query, parameterIds); } + private static boolean requiresQueryWrapper(HogQlQuery query, boolean parenthesized) + { + return parenthesized || !query.with().isEmpty() || !query.orderBy().isEmpty() || query.limit().isPresent() || query.offset().isPresent(); + } + + private record SetOperationFrame(SetOperation operation, boolean operandsLowered) {} + private static Optional createWith(HogQlQuery query, Map parameterIds) { if (query.with().isEmpty()) { diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index 668a65a2a777..332042d31910 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -520,6 +520,30 @@ public void testLowersSetOperationOrderLimitAndOffset() assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); } + @Test + public void testLowersLargeSetOperation() + { + int operandCount = 3_000; + StringBuilder hogql = new StringBuilder("SELECT 0"); + for (int operand = 1; operand < operandCount; operand++) { + hogql.append(" UNION ALL SELECT ").append(operand); + } + + Query query = (Query) compiler.compile(hogql.toString()); + Deque pending = new ArrayDeque<>(); + pending.add(query); + int querySpecificationCount = 0; + while (!pending.isEmpty()) { + Node node = pending.removeFirst(); + if (node instanceof QuerySpecification) { + querySpecificationCount++; + } + pending.addAll(node.getChildren()); + } + + assertThat(querySpecificationCount).isEqualTo(operandCount); + } + @Test public void testBindsPlaceholdersAcrossSetBranchesAndValuesRows() { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParserLimits.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParserLimits.java index 5aca7dd88308..738ee134d160 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParserLimits.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParserLimits.java @@ -15,7 +15,7 @@ public record HogQlParserLimits(int maxTokens, int maxParseDepth, int maxParseTreeNodes) { - private static final HogQlParserLimits DEFAULTS = new HogQlParserLimits(100_000, 256, 500_000); + private static final HogQlParserLimits DEFAULTS = new HogQlParserLimits(200_000, 256, 500_000); public HogQlParserLimits { diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserLimits.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserLimits.java index a93bf654a92f..2db41271590a 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserLimits.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserLimits.java @@ -20,6 +20,8 @@ public class TestHogQlParserLimits { + private static final int LARGE_VALID_QUERY_TOKEN_COUNT = 160_000; + @Test public void testRejectsTokenBomb() { @@ -59,4 +61,29 @@ public void testAcceptsInputWithinInjectedLimits() assertThatCode(() -> parser.parseSyntax("SELECT event FROM events WHERE event = 'signup'")) .doesNotThrowAnyException(); } + + @Test + public void testDefaultLimitsAcceptLargeValidQuery() + { + assertThatCode(() -> new HogQlParser().parseSyntax(queryWithTokenCount(LARGE_VALID_QUERY_TOKEN_COUNT))) + .doesNotThrowAnyException(); + } + + @Test + public void testDefaultTokenLimitRemainsBounded() + { + int tokenCount = HogQlParserLimits.defaults().maxTokens() + 1; + + assertThatThrownBy(() -> new HogQlParser().parseSyntax(queryWithTokenCount(tokenCount))) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("token limit exceeded"); + } + + private static String queryWithTokenCount(int tokenCount) + { + if (tokenCount < 3) { + throw new IllegalArgumentException("tokenCount must be at least 3"); + } + return "SELECT" + " /**/".repeat(tokenCount - 3) + " 1"; + } } From 31a10c7cbbbf057744dbb68bfbe2d8023b5e9515 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 02:07:19 -0400 Subject: [PATCH 117/136] feat(hogql): bind scoped placeholders --- .../hogql/compiler/HogQlCompileEnvelope.java | 17 ++++ .../trino/hogql/compiler/HogQlCompiler.java | 36 +++++--- .../compiler/TestHogQlScopedPlaceholders.java | 84 +++++++++++++++++++ .../io/trino/hogql/parser/HogQlParser.java | 14 +++- .../execution/HogQlParameterDecoder.java | 7 +- .../io/trino/execution/QueryPreparer.java | 2 +- .../TestHogQlStatementResource.java | 24 ++++++ 7 files changed, 168 insertions(+), 16 deletions(-) create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlScopedPlaceholders.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompileEnvelope.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompileEnvelope.java index c2412f112d78..233e2a24b14a 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompileEnvelope.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompileEnvelope.java @@ -17,6 +17,7 @@ import io.trino.hogql.parser.HogQlLanguageVersion; import java.util.Map; +import java.util.Optional; import java.util.OptionalLong; import java.util.Set; @@ -83,6 +84,22 @@ public static HogQlCompileEnvelope fromSemanticFields( catalogGeneration); } + public Optional bindingForPlaceholder(String placeholder) + { + requireNonNull(placeholder, "placeholder is null"); + int separator = placeholder.indexOf('.'); + if (separator < 0) { + return Optional.ofNullable(parameters.get(placeholder)); + } + String namespace = placeholder.substring(0, separator); + String name = placeholder.substring(separator + 1); + return switch (namespace) { + case "variables" -> Optional.ofNullable(variables.get(name)); + case "filters" -> Optional.ofNullable(filters.get(name)); + default -> Optional.empty(); + }; + } + private static Map immutableValues(String field, Map values) { requireNonNull(values, field + " is null"); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index 51aa0781853a..c5ae8c3c226d 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -112,7 +112,7 @@ public HogQlCompilationResult compile(HogQlCompileEnvelope envelope, Optional { requireNonNull(hogql, "hogql is null"); requireNonNull(parameters, "parameters is null"); - return compile(parse(hogql), parameters, Map.of(), Optional.empty(), HogQlLanguageContract.current().languageVersion(), OptionalLong.empty(), false); + return compile( + parse(hogql), + new HogQlCompileEnvelope( + hogql, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + parameters, + Map.of(), + Map.of(), + Map.of(), + OptionalLong.empty()), + Map.of(), + Optional.empty(), + HogQlLanguageContract.current().languageVersion(), + OptionalLong.empty(), + false); } private HogQlQuery parse(String hogql) @@ -161,14 +176,14 @@ private HogQlQuery parse(String hogql) private static HogQlCompilationResult compile( HogQlQuery query, - Map parameters, + HogQlCompileEnvelope envelope, Map modifiers, Optional catalogContext, HogQlLanguageVersion languageVersion, OptionalLong expectedCatalogGeneration, boolean v0Profile) { - validateParameters(query, parameters); + validateParameters(query, envelope.parameters()); validateQuery(query); List placeholders = new ArrayList<>(); @@ -177,7 +192,7 @@ private static HogQlCompilationResult compile( List missing = placeholders.stream() .map(Placeholder::name) - .filter(name -> !parameters.containsKey(name)) + .filter(name -> envelope.bindingForPlaceholder(name).isEmpty()) .distinct() .toList(); if (!missing.isEmpty()) { @@ -188,12 +203,13 @@ private static HogQlCompilationResult compile( throw bindingError(firstMissing.span(), "Missing HogQL parameter bindings: " + String.join(", ", missing)); } - Set placeholderNames = new HashSet<>(); + Set parameterPlaceholderNames = new HashSet<>(); placeholders.stream() .map(Placeholder::name) - .forEach(placeholderNames::add); - List extra = parameters.keySet().stream() - .filter(name -> !placeholderNames.contains(name)) + .filter(name -> !name.contains(".")) + .forEach(parameterPlaceholderNames::add); + List extra = envelope.parameters().keySet().stream() + .filter(name -> !parameterPlaceholderNames.contains(name)) .sorted() .toList(); if (!extra.isEmpty()) { diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlScopedPlaceholders.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlScopedPlaceholders.java new file mode 100644 index 000000000000..54d14930f6c8 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlScopedPlaceholders.java @@ -0,0 +1,84 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.HogQlTypedValue.NumberValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.TrinoException; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.OptionalLong; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlScopedPlaceholders +{ + private final HogQlCompiler compiler = new HogQlCompiler(); + + @Test + public void testBindsVariablesAndFiltersWithoutInterpolation() + { + HogQlCompilationResult result = compiler.compile(envelope( + "SELECT {variables.organization_id}, {filters.date_from}, {variables.organization_id}", + Map.of("organization_id", new HogQlTypedValue("bigint", new NumberValue("42"))), + Map.of("date_from", new HogQlTypedValue("varchar", new StringValue("2026-01-01"))))); + + assertThat(result.parameterNames()).containsExactly( + "variables.organization_id", + "filters.date_from", + "variables.organization_id"); + assertThat(result.statement().toString()).doesNotContain("42", "2026-01-01"); + } + + @Test + public void testReportsMissingScopedBinding() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope("SELECT {variables.organization_id}", Map.of(), Map.of()))); + + assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_BINDING_ERROR.toErrorCode()); + assertThat(exception).hasMessageContaining("Missing HogQL parameter bindings: variables.organization_id"); + } + + @Test + public void testLeavesOtherDottedPlaceholderExpressionsUnsupported() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope("SELECT {payload.organization_id}", Map.of(), Map.of()))); + + assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_SYNTAX_ERROR.toErrorCode()); + assertThat(exception).hasMessageContaining("non-name placeholder"); + } + + private static HogQlCompileEnvelope envelope( + String query, + Map variables, + Map filters) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + variables, + filters, + Map.of(), + OptionalLong.empty()); + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 4f83d0dc337b..0f23974051cc 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -1670,10 +1670,20 @@ private Expression buildColumnReference(HogQLParser.ColumnIdentifierContext cont private Placeholder buildPlaceholder(HogQLParser.PlaceholderContext context) { Expression expression = buildExpression(context.columnExpr()); - if (!(expression instanceof ColumnReference reference) || reference.parts().size() != 1) { + if (!(expression instanceof ColumnReference reference)) { throw unsupported(context, "non-name placeholder"); } - return new Placeholder(reference.parts().getFirst().value(), sourceSpan(context)); + if (reference.parts().size() == 1) { + return new Placeholder(reference.parts().getFirst().value(), sourceSpan(context)); + } + if (reference.parts().size() == 2 && + (reference.parts().getFirst().value().equalsIgnoreCase("variables") || + reference.parts().getFirst().value().equalsIgnoreCase("filters"))) { + return new Placeholder( + (reference.parts().getFirst().value().equalsIgnoreCase("variables") ? "variables." : "filters.") + reference.parts().getLast().value(), + sourceSpan(context)); + } + throw unsupported(context, "non-name placeholder"); } private Relation buildRelation(HogQLParser.JoinExprContext context) diff --git a/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java b/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java index f156d20bd0e3..01e6f9c8ba09 100644 --- a/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java +++ b/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.collect.ImmutableList; +import io.trino.hogql.compiler.HogQlCompileEnvelope; import io.trino.hogql.compiler.HogQlCompilationResult; import io.trino.hogql.compiler.HogQlModifierBinding; import io.trino.hogql.compiler.HogQlTypedValue; @@ -98,10 +99,10 @@ final class HogQlParameterDecoder this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); } - List decode(HogQlCompilationResult result, Map bindings) + List decode(HogQlCompilationResult result, HogQlCompileEnvelope envelope) { requireNonNull(result, "result is null"); - requireNonNull(bindings, "bindings is null"); + requireNonNull(envelope, "envelope is null"); List parameters = ParameterExtractor.extractParameters(result.statement()); if (parameters.size() != result.parameterNames().size()) { throw new IllegalStateException("HogQL compiler returned inconsistent parameter metadata"); @@ -112,7 +113,7 @@ List decode(HogQlCompilationResult result, Map new NullPointerException("binding is null")); values.add(decode(binding, location)); } catch (RuntimeException _) { diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java index 089927cf9030..bcde05a520d5 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java @@ -150,7 +150,7 @@ public PreparedQuery prepareQuery(Session session, QuerySubmission submission) preparedQuery = tracker.observe(PARAMETER_BINDING, () -> prepareQuery( session, statement, - Optional.of(hogQlParameterDecoder.decode(result, envelope.parameters())), + Optional.of(hogQlParameterDecoder.decode(result, envelope)), sessionPropertyOverrides(result, statement))); } catch (RuntimeException | Error failure) { diff --git a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java index d568fda9acda..3d4fdc15dc70 100644 --- a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java +++ b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java @@ -183,6 +183,17 @@ public void testEnabledEndpointUsesStatementProtocol() assertThat(rows(sqlResults)).containsExactly(ImmutableList.of(2)); } + @Test + public void testBindsScopedVariablesAndFilters() + throws Exception + { + List results = runHogQlToCompletion(hogQlRequestWithScopedBindings( + "SELECT {variables.organization_id}, {filters.date_from}")); + + assertThat(results.getLast().getError()).isNull(); + assertThat(rows(results)).containsExactly(ImmutableList.of(42L, "2026-01-01")); + } + @ParameterizedTest(name = "{0}") @MethodSource("differentialQueries") public void testHogQlExecutionMatchesTrinoSql(String name, String hogql, String trinoSql) @@ -616,6 +627,19 @@ private static String hogQlRequestWithTypedValues(String query) """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); } + private static String hogQlRequestWithScopedBindings(String query) + { + return """ + { + "query": %s, + "protocolVersion": 1, + "languageVersion": "%s", + "variables": {"organization_id": {"type": "bigint", "value": 42}}, + "filters": {"date_from": {"type": "varchar", "value": "2026-01-01"}} + } + """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); + } + private static String hogQlRequestWithModifier(String query) { return """ From 5fb929f3513c88b27aeef191213973bd8f26fe44 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 02:08:03 -0400 Subject: [PATCH 118/136] feat(ducklake): add readable time delta compatibility --- .../trino/plugin/ducklake/DuckLakePlugin.java | 3 +- .../FormatReadableTimeDeltaFunction.java | 230 ++++++++++++++++++ .../TestFormatReadableTimeDeltaFunction.java | 116 +++++++++ 3 files changed, 348 insertions(+), 1 deletion(-) create mode 100644 plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/FormatReadableTimeDeltaFunction.java create mode 100644 plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/TestFormatReadableTimeDeltaFunction.java diff --git a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/DuckLakePlugin.java b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/DuckLakePlugin.java index 9a081de6f259..9e19f8b03fd1 100644 --- a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/DuckLakePlugin.java +++ b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/DuckLakePlugin.java @@ -16,6 +16,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import io.trino.plugin.ducklake.function.CityHash64Function; +import io.trino.plugin.ducklake.function.FormatReadableTimeDeltaFunction; import io.trino.spi.Plugin; import io.trino.spi.connector.ConnectorFactory; @@ -33,6 +34,6 @@ public Iterable getConnectorFactories() @Override public Set> getFunctions() { - return ImmutableSet.of(CityHash64Function.class); + return ImmutableSet.of(CityHash64Function.class, FormatReadableTimeDeltaFunction.class); } } diff --git a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/FormatReadableTimeDeltaFunction.java b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/FormatReadableTimeDeltaFunction.java new file mode 100644 index 000000000000..b11af437efd5 --- /dev/null +++ b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/FormatReadableTimeDeltaFunction.java @@ -0,0 +1,230 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ducklake.function; + +import io.airlift.slice.Slice; +import io.trino.spi.TrinoException; +import io.trino.spi.function.Description; +import io.trino.spi.function.ScalarFunction; +import io.trino.spi.function.SqlType; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import static io.airlift.slice.Slices.utf8Slice; +import static io.trino.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static io.trino.spi.type.StandardTypes.DOUBLE; +import static io.trino.spi.type.StandardTypes.VARCHAR; + +public final class FormatReadableTimeDeltaFunction +{ + private FormatReadableTimeDeltaFunction() {} + + @Description("Formats a numeric duration in seconds using ClickHouse time delta wording") + @ScalarFunction("format_readable_time_delta") + @SqlType(VARCHAR) + public static Slice formatReadableTimeDelta(@SqlType(DOUBLE) double value) + { + return utf8Slice(format(value, Unit.YEARS, Unit.SECONDS)); + } + + @Description("Formats a numeric duration in seconds using ClickHouse time delta wording") + @ScalarFunction("format_readable_time_delta") + @SqlType(VARCHAR) + public static Slice formatReadableTimeDelta( + @SqlType(DOUBLE) double value, + @SqlType(VARCHAR) Slice maximumUnit) + { + Unit maximum = parseUnit(maximumUnit, "maximum"); + Unit minimum = maximum.ordinal() < Unit.SECONDS.ordinal() ? Unit.NANOSECONDS : Unit.SECONDS; + return utf8Slice(format(value, maximum, minimum)); + } + + @Description("Formats a numeric duration in seconds using ClickHouse time delta wording") + @ScalarFunction("format_readable_time_delta") + @SqlType(VARCHAR) + public static Slice formatReadableTimeDelta( + @SqlType(DOUBLE) double value, + @SqlType(VARCHAR) Slice maximumUnit, + @SqlType(VARCHAR) Slice minimumUnit) + { + Unit maximum = parseUnit(maximumUnit, "maximum"); + Unit minimum = parseUnit(minimumUnit, "minimum"); + if (minimum.ordinal() > maximum.ordinal()) { + throw new TrinoException( + INVALID_FUNCTION_ARGUMENT, + "Minimum unit (%s) must not be greater than maximum unit (%s)" + .formatted(minimum.name().toLowerCase(Locale.ENGLISH), maximum.name().toLowerCase(Locale.ENGLISH))); + } + return utf8Slice(format(value, maximum, minimum)); + } + + private static String format(double value, Unit maximum, Unit minimum) + { + if (Double.isNaN(value)) { + return "nan"; + } + if (value == Double.POSITIVE_INFINITY) { + return "inf"; + } + if (value == Double.NEGATIVE_INFINITY) { + return "-inf"; + } + + boolean negative = value < 0; + value = Math.abs(value); + double wholeSeconds = Math.floor(value); + long fractionalNanoseconds = roundedFractionalNanoseconds(value - wholeSeconds); + List parts = new ArrayList<>(); + + for (int index = maximum.ordinal(); index >= minimum.ordinal(); index--) { + Unit unit = Unit.values()[index]; + if (wholeSeconds + 1.0 == wholeSeconds) { + double units = Math.floor(wholeSeconds * unit.scaleMultiplier() / unit.secondsMultiplier()); + parts.add(formatDouble(units) + " " + unit.singular() + "s"); + break; + } + + long units; + if (unit.scale() == 0) { + units = (long) Math.floor(wholeSeconds / unit.secondsMultiplier()); + if (units == 0 && (unit != Unit.SECONDS || !parts.isEmpty())) { + continue; + } + wholeSeconds -= units * unit.secondsMultiplier(); + } + else { + long scaleMultiplier = unit.scaleMultiplier(); + units = (long) wholeSeconds * scaleMultiplier; + wholeSeconds = 0; + long divisor = pow10(9 - unit.scale()); + units += fractionalNanoseconds / divisor; + fractionalNanoseconds %= divisor; + if (units == 0 && (unit != minimum || !parts.isEmpty())) { + continue; + } + } + parts.add(units + " " + unit.singular() + (units == 1 ? "" : "s")); + } + + String result = joinParts(parts); + return negative ? "-" + result : result; + } + + private static long roundedFractionalNanoseconds(double fractional) + { + return Math.round(fractional * 1_000_000_000) % 1_000_000_000; + } + + private static String joinParts(List parts) + { + if (parts.isEmpty()) { + return ""; + } + if (parts.size() == 1) { + return parts.getFirst(); + } + if (parts.size() == 2) { + return parts.getFirst() + " and " + parts.getLast(); + } + return String.join(", ", parts.subList(0, parts.size() - 1)) + " and " + parts.getLast(); + } + + private static String formatDouble(double value) + { + if (Double.isInfinite(value)) { + return "inf"; + } + if (value < 1e21) { + return BigDecimal.valueOf(value).stripTrailingZeros().toPlainString(); + } + String formatted = Double.toString(value).replace("E+", "e").replace('E', 'e'); + int exponent = formatted.indexOf('e'); + if (exponent > 1 && formatted.substring(0, exponent).endsWith(".0")) { + return formatted.substring(0, exponent - 2) + formatted.substring(exponent); + } + return formatted; + } + + private static long pow10(int exponent) + { + return switch (exponent) { + case 0 -> 1; + case 3 -> 1_000; + case 6 -> 1_000_000; + case 9 -> 1_000_000_000; + default -> throw new IllegalArgumentException("unsupported decimal exponent: " + exponent); + }; + } + + private static Unit parseUnit(Slice unit, String bound) + { + String value = unit.toStringUtf8(); + for (Unit candidate : Unit.values()) { + if (candidate.name().toLowerCase(Locale.ENGLISH).equals(value)) { + return candidate; + } + } + throw new TrinoException( + INVALID_FUNCTION_ARGUMENT, + "Unexpected value of %s unit argument (%s); expected nanoseconds, microseconds, milliseconds, seconds, minutes, hours, days, months, or years" + .formatted(bound, value)); + } + + private enum Unit + { + NANOSECONDS(1, 9, "nanosecond"), + MICROSECONDS(1, 6, "microsecond"), + MILLISECONDS(1, 3, "millisecond"), + SECONDS(1, 0, "second"), + MINUTES(60, 0, "minute"), + HOURS(3_600, 0, "hour"), + DAYS(86_400, 0, "day"), + MONTHS(2_635_200, 0, "month"), + YEARS(31_536_000, 0, "year"); + + private final long secondsMultiplier; + private final int scale; + private final String singular; + + Unit(long secondsMultiplier, int scale, String singular) + { + this.secondsMultiplier = secondsMultiplier; + this.scale = scale; + this.singular = singular; + } + + public long secondsMultiplier() + { + return secondsMultiplier; + } + + public int scale() + { + return scale; + } + + public long scaleMultiplier() + { + return pow10(scale); + } + + public String singular() + { + return singular; + } + } +} diff --git a/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/TestFormatReadableTimeDeltaFunction.java b/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/TestFormatReadableTimeDeltaFunction.java new file mode 100644 index 000000000000..4ee7ce956aa8 --- /dev/null +++ b/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/TestFormatReadableTimeDeltaFunction.java @@ -0,0 +1,116 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ducklake.function; + +import io.airlift.slice.Slices; +import io.trino.plugin.ducklake.DuckLakePlugin; +import io.trino.spi.TrinoException; +import io.trino.sql.query.QueryAssertions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import java.util.Map; + +import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; + +@TestInstance(PER_CLASS) +final class TestFormatReadableTimeDeltaFunction +{ + private QueryAssertions assertions; + + @BeforeAll + void init() + { + assertions = new QueryAssertions(); + assertions.addPlugin(new DuckLakePlugin()); + } + + @AfterAll + void teardown() + { + assertions.close(); + assertions = null; + } + + @Test + void testClickHouseCompatibility() + { + Map expectedResults = Map.ofEntries( + Map.entry("5.5", "5 seconds"), + Map.entry("330.0", "5 minutes and 30 seconds"), + Map.entry("475200.0", "5 days and 12 hours"), + Map.entry("CAST(14256000 AS double)", "5 months, 12 days and 12 hours"), + Map.entry("CAST(173448000 AS double)", "5 years, 5 months and 30 days"), + Map.entry("-34261261.0", "-1 year, 1 month, 1 day, 1 hour, 1 minute and 1 second"), + Map.entry("CAST(0 AS double)", "0 seconds"), + Map.entry("CAST('Infinity' AS double)", "inf"), + Map.entry("CAST('-Infinity' AS double)", "-inf"), + Map.entry("nan()", "nan"), + Map.entry("1e100", "3.1709791983764585e92 years"), + Map.entry("CAST(1152921504606846976 AS double)", "36558901084 years")); + + expectedResults.forEach((input, expected) -> assertThat(assertions.function("format_readable_time_delta", input)) + .hasType(VARCHAR) + .isEqualTo(expected)); + } + + @Test + void testMaximumAndMinimumUnits() + { + assertThat(assertions.function("format_readable_time_delta", "475200.0", "'minutes'")) + .isEqualTo("7920 minutes"); + assertThat(assertions.function("format_readable_time_delta", "67.79797979", "'milliseconds'")) + .isEqualTo("67797 milliseconds, 979 microseconds and 790 nanoseconds"); + assertThat(assertions.function("format_readable_time_delta", "3601.000000003", "'hours'", "'microseconds'")) + .isEqualTo("1 hour and 1 second"); + assertThat(assertions.function("format_readable_time_delta", "1.0005", "'milliseconds'")) + .isEqualTo("1000 milliseconds and 500 microseconds"); + assertThat(assertions.function("format_readable_time_delta", "CAST(0 AS double)", "'milliseconds'")) + .isEqualTo("0 nanoseconds"); + assertThat(assertions.function("format_readable_time_delta", "CAST(0 AS double)", "'years'", "'years'")) + .isEqualTo(""); + } + + @Test + void testInvalidUnits() + { + assertThatThrownBy(() -> FormatReadableTimeDeltaFunction.formatReadableTimeDelta( + 1.0, + Slices.utf8Slice("second"))) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("Unexpected value of maximum unit argument"); + assertThatThrownBy(() -> FormatReadableTimeDeltaFunction.formatReadableTimeDelta( + 1.0, + Slices.utf8Slice("seconds"), + Slices.utf8Slice("hours"))) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("must not be greater than maximum unit"); + } + + @Test + void testNullPropagation() + { + assertThat(assertions.function("format_readable_time_delta", "CAST(NULL AS double)")) + .isNull(VARCHAR); + assertThat(assertions.function("format_readable_time_delta", "1.0", "CAST(NULL AS varchar)")) + .isNull(VARCHAR); + assertThat(assertions.function("format_readable_time_delta", "1.0", "'hours'", "CAST(NULL AS varchar)")) + .isNull(VARCHAR); + } +} From c899d7a6690db44688da28f31cf7ec9656a8c0f1 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 02:24:53 -0400 Subject: [PATCH 119/136] feat(hogql): lower template strings --- .../compiler/HogQlV0FunctionRegistry.java | 1 + .../compiler/TestHogQlFunctionResolver.java | 4 +- .../TestHogQlTemplateStringCompiler.java | 35 +++++++++++ .../io/trino/hogql/parser/HogQlParser.java | 44 +++++++++++++ .../parser/TestHogQlTemplateStringParser.java | 61 +++++++++++++++++++ 5 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlTemplateStringCompiler.java create mode 100644 core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlTemplateStringParser.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 796eb268edd0..c4c85469197e 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -153,6 +153,7 @@ final class HogQlV0FunctionRegistry scalar("toLastDayOfMonth", "last_day_of_month", signature(1)), rewrite("map", FunctionRewrite.MAP_CONSTRUCTOR, "map(any,any)", signature(0), variadicSignature(2)), scalar("dateDiff", "date_diff", signature(3)), + scalar("date_diff", "date_diff", signature(3)), scalar("dateTrunc", "date_trunc", signature(2)), rewrite("arraySort", FunctionRewrite.ARRAY_SORT, "array(any)", signature(1), signature(2)), scalar("arrayMin", "array_min", signature(1)), diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 1dabdb954d23..f5ffcf9b7d1e 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -121,7 +121,7 @@ public void testCompilerResolvesV0FunctionsWithoutCatalogContext() { HogQlCompilationResult result = new HogQlCompiler().compile(envelope( "SELECT coalesce(NULL, 'fallback'), any(value), argMin(value, timestamp), " + - "arrayDistinct([1, 1]), dateDiff('day', start_time, end_time), rank() OVER (), " + + "arrayDistinct([1, 1]), dateDiff('day', start_time, end_time), date_diff('second', start_time, end_time), rank() OVER (), " + "nullIf(value, ''), ifNull(value, 'fallback'), trim(value), round(score, 2), " + "now(), current_timestamp(), isNotNull(value), groupArray(value), uniq(value), " + "toInt(value), toFloat(value), toString(value), toDate(timestamp), toDateTime(timestamp), " + @@ -132,7 +132,7 @@ public void testCompilerResolvesV0FunctionsWithoutCatalogContext() assertThat(result.catalogGeneration()).isEmpty(); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( "SELECT coalesce(NULL, 'fallback'), arbitrary(value), min_by(value, timestamp), " + - "array_distinct(ARRAY[1, 1]), date_diff('day', start_time, end_time), rank() OVER (), " + + "array_distinct(ARRAY[1, 1]), date_diff('day', start_time, end_time), date_diff('second', start_time, end_time), rank() OVER (), " + "nullif(value, ''), coalesce(value, 'fallback'), \"trim\"(value), round(score, 2), " + "now(), now(), value IS NOT NULL, array_agg(value), approx_distinct(value), " + "CAST(value AS bigint), CAST(value AS double), CAST(value AS varchar), CAST(timestamp AS date), CAST(timestamp AS timestamp(0)), " + diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlTemplateStringCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlTemplateStringCompiler.java new file mode 100644 index 000000000000..9f8df8e8ff1b --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlTemplateStringCompiler.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.sql.SqlFormatter; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlTemplateStringCompiler +{ + @Test + public void testLowersTemplateStringToStockTrinoAst() + { + String sql = SqlFormatter.formatSql(new HogQlCompiler().compile( + "SELECT f'{year}-{month}-{day}' FROM calendar")); + + assertThat(sql) + .contains("concat(") + .contains("CAST(year AS varchar)") + .contains("CAST(month AS varchar)") + .contains("CAST(day AS varchar)"); + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 0f23974051cc..ae63b53603f4 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -138,6 +138,7 @@ import static io.trino.hogql.parser.tree.HogQlQuery.UnaryOperator.POSITIVE; import static java.lang.Character.digit; import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.joining; public final class HogQlParser { @@ -1181,6 +1182,9 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) List values = array.columnExprList() == null ? List.of() : buildExpressions(array.columnExprList()); return new ArrayExpression(values, sourceSpan(array)); } + if (context instanceof HogQLParser.ColumnExprTemplateStringContext template) { + return buildTemplateString(template.templateString()); + } if (context instanceof HogQLParser.ColumnExprTupleContext tuple) { return new TupleExpression(buildExpressions(tuple.columnExprList()), sourceSpan(tuple)); } @@ -1370,6 +1374,46 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) throw unsupported(context, "expression " + context.getClass().getSimpleName()); } + private Expression buildTemplateString(HogQLParser.TemplateStringContext context) + { + List parts = new ArrayList<>(); + boolean interpolated = false; + for (HogQLParser.StringContentsContext contents : context.stringContents()) { + if (contents.STRING_TEXT() != null) { + Token token = contents.STRING_TEXT().getSymbol(); + parts.add(new Literal(STRING, decodeQuoted("'" + token.getText() + "'"), sourceSpan(token, token))); + } + else { + interpolated = true; + Expression value = buildExpression(contents.columnExpr()); + parts.add(new FunctionCall( + new Identifier("toString", false, value.span()), + List.of(value), + false, + List.of(), + Optional.empty(), + value.span())); + } + } + if (!interpolated) { + String value = parts.stream() + .map(Literal.class::cast) + .map(Literal::value) + .collect(joining()); + return new Literal(STRING, value, sourceSpan(context)); + } + if (parts.size() == 1) { + return parts.getFirst(); + } + return new FunctionCall( + new Identifier("concat", false, sourceSpan(context)), + parts, + false, + List.of(), + Optional.empty(), + sourceSpan(context)); + } + private IntervalExpression buildStringInterval(HogQLParser.ColumnExprIntervalStringContext context) { String value = decodeQuoted(context.STRING_LITERAL().getText()); diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlTemplateStringParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlTemplateStringParser.java new file mode 100644 index 000000000000..d6921dac1d47 --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlTemplateStringParser.java @@ -0,0 +1,61 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlTemplateStringParser +{ + private final HogQlParser parser = new HogQlParser(); + + @Test + public void testDesugarsInterpolatedTemplateToTypedConcatenation() + { + HogQlQuery query = parser.parseStatement("SELECT f'{year}-{month}-{day}'"); + FunctionCall concat = (FunctionCall) ((ExpressionProjection) query.projections().getFirst()).expression(); + + assertThat(concat.name().value()).isEqualTo("concat"); + assertThat(concat.arguments()).hasSize(5); + assertThat(concat.arguments().get(0)).isInstanceOfSatisfying( + FunctionCall.class, + function -> assertThat(function.name().value()).isEqualTo("toString")); + assertThat(concat.arguments().get(1)).isInstanceOfSatisfying( + Literal.class, + literal -> assertThat(literal.value()).isEqualTo("-")); + } + + @Test + public void testKeepsLiteralOnlyTemplateAsLiteral() + { + HogQlQuery query = parser.parseStatement("SELECT f'hello\\nworld'"); + Literal literal = (Literal) ((ExpressionProjection) query.projections().getFirst()).expression(); + + assertThat(literal.value()).isEqualTo("hello\nworld"); + } + + @Test + public void testKeepsEmptyTemplateAsEmptyLiteral() + { + HogQlQuery query = parser.parseStatement("SELECT f''"); + Literal literal = (Literal) ((ExpressionProjection) query.projections().getFirst()).expression(); + + assertThat(literal.value()).isEmpty(); + } +} From 70f14467ca5c519d2af6616da169a8e08d825be0 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 02:25:21 -0400 Subject: [PATCH 120/136] feat(hogql): lower left any joins --- .../trino/hogql/compiler/TrinoAstFactory.java | 149 ++++++++++++++++-- .../hogql/compiler/TestHogQlCompiler.java | 24 +++ .../io/trino/hogql/parser/HogQlParser.java | 8 +- .../trino/hogql/parser/tree/HogQlQuery.java | 1 + .../trino/hogql/parser/TestHogQlParser.java | 12 ++ .../planner/TestHogQlCorrelatedSubquery.java | 9 ++ 6 files changed, 185 insertions(+), 18 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index 246cbeb652e3..ede64303b6e9 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -87,6 +87,7 @@ import io.trino.sql.tree.IntervalField; import io.trino.sql.tree.IntervalLiteral; import io.trino.sql.tree.IsNullPredicate; +import io.trino.sql.tree.Lateral; import io.trino.sql.tree.Limit; import io.trino.sql.tree.LambdaArgumentDeclaration; import io.trino.sql.tree.LambdaExpression; @@ -750,23 +751,7 @@ private static io.trino.sql.tree.Relation createRelation(Relation relation, Map< case CommonTableReference commonTable -> new Table( location(commonTable.span()), QualifiedName.of(List.of(createIdentifier(commonTable.name())))); - case JoinRelation join -> new io.trino.sql.tree.Join( - location(join.span()), - switch (join.type()) { - case CROSS -> io.trino.sql.tree.Join.Type.CROSS; - case INNER -> io.trino.sql.tree.Join.Type.INNER; - case LEFT -> io.trino.sql.tree.Join.Type.LEFT; - case RIGHT -> io.trino.sql.tree.Join.Type.RIGHT; - case FULL -> io.trino.sql.tree.Join.Type.FULL; - }, - createRelation(join.left(), parameterIds), - createRelation(join.right(), parameterIds), - join.criteria().map(criteria -> switch (criteria) { - case JoinOn on -> new io.trino.sql.tree.JoinOn(createExpression(on.expression(), parameterIds)); - case JoinUsing using -> new io.trino.sql.tree.JoinUsing(using.columns().stream() - .map(TrinoAstFactory::createIdentifier) - .toList()); - })); + case JoinRelation join -> createJoin(join, parameterIds); case PivotRelation pivot -> new io.trino.sql.tree.Pivot( location(pivot.span()), createRelation(pivot.input(), parameterIds), @@ -803,6 +788,136 @@ private static io.trino.sql.tree.Relation createRelation(Relation relation, Map< }; } + private static io.trino.sql.tree.Relation createJoin(JoinRelation join, Map parameterIds) + { + if (join.type() == HogQlQuery.JoinType.LEFT_ANY) { + return createLeftAnyJoin(join, parameterIds); + } + return new io.trino.sql.tree.Join( + location(join.span()), + switch (join.type()) { + case CROSS -> io.trino.sql.tree.Join.Type.CROSS; + case INNER -> io.trino.sql.tree.Join.Type.INNER; + case LEFT -> io.trino.sql.tree.Join.Type.LEFT; + case LEFT_ANY -> throw new IllegalStateException("LEFT ANY JOIN must be lowered as a lateral join"); + case RIGHT -> io.trino.sql.tree.Join.Type.RIGHT; + case FULL -> io.trino.sql.tree.Join.Type.FULL; + }, + createRelation(join.left(), parameterIds), + createRelation(join.right(), parameterIds), + join.criteria().map(criteria -> switch (criteria) { + case JoinOn on -> new io.trino.sql.tree.JoinOn(createExpression(on.expression(), parameterIds)); + case JoinUsing using -> new io.trino.sql.tree.JoinUsing(using.columns().stream() + .map(TrinoAstFactory::createIdentifier) + .toList()); + })); + } + + private static io.trino.sql.tree.Relation createLeftAnyJoin(JoinRelation join, Map parameterIds) + { + HogQlQuery.JoinCriteria criteria = join.criteria().orElseThrow(); + if (!(criteria instanceof JoinOn on)) { + throw unsupportedSemanticExpression(criteria.span(), "HogQL LEFT ANY JOIN with USING cannot preserve merged-column semantics in Trino"); + } + + Identifier alias = leftAnyJoinAlias(join.right()) + .orElseThrow(() -> unsupportedSemanticExpression( + join.right().span(), + "HogQL LEFT ANY JOIN requires a right relation alias for Trino correlation")); + HogQlQuery.Expression orderingKey = leftAnyJoinOrderingKey(on.expression(), alias) + .orElseThrow(() -> unsupportedSemanticExpression( + on.span(), + "HogQL LEFT ANY JOIN requires an equality predicate on a qualified right column for Trino decorrelation")); + NodeLocation rightLocation = location(join.right().span()); + QuerySpecification matches = new QuerySpecification( + rightLocation, + new Select(rightLocation, false, List.of(new AllColumns(rightLocation))), + Optional.of(createRelation(join.right(), parameterIds)), + Optional.of(createExpression(on.expression(), parameterIds)), + Optional.empty(), + Optional.empty(), + List.of(), + Optional.of(new OrderBy( + location(orderingKey.span()), + List.of(new io.trino.sql.tree.SortItem( + location(orderingKey.span()), + createExpression(orderingKey, parameterIds), + Ordering.ASCENDING, + NullOrdering.UNDEFINED)))), + Optional.empty(), + Optional.of(new Limit(rightLocation, new LongLiteral(rightLocation, "1")))); + Query query = new Query( + rightLocation, + List.of(), + List.of(), + Optional.empty(), + matches, + Optional.empty(), + Optional.empty(), + Optional.empty()); + io.trino.sql.tree.Relation lateral = new Lateral(rightLocation, query); + lateral = new io.trino.sql.tree.AliasedRelation( + rightLocation, + lateral, + createIdentifier(alias), + null); + return new io.trino.sql.tree.Join( + location(join.span()), + io.trino.sql.tree.Join.Type.LEFT, + createRelation(join.left(), parameterIds), + lateral, + Optional.of(new io.trino.sql.tree.JoinOn(new BooleanLiteral(location(join.span()), "true")))); + } + + private static Optional leftAnyJoinOrderingKey(HogQlQuery.Expression expression, Identifier rightAlias) + { + if (!(expression instanceof BinaryExpression binary)) { + return Optional.empty(); + } + if (binary.operator() == HogQlQuery.BinaryOperator.AND) { + return leftAnyJoinOrderingKey(binary.left(), rightAlias) + .or(() -> leftAnyJoinOrderingKey(binary.right(), rightAlias)); + } + if (binary.operator() != HogQlQuery.BinaryOperator.EQUAL) { + return Optional.empty(); + } + if (isQualifiedBy(binary.left(), rightAlias)) { + return Optional.of(binary.left()); + } + if (isQualifiedBy(binary.right(), rightAlias)) { + return Optional.of(binary.right()); + } + return Optional.empty(); + } + + private static boolean isQualifiedBy(HogQlQuery.Expression expression, Identifier qualifier) + { + return expression instanceof ColumnReference reference && + reference.parts().size() > 1 && + reference.parts().getFirst().value().equalsIgnoreCase(qualifier.value()); + } + + private static Optional leftAnyJoinAlias(Relation relation) + { + return switch (relation) { + case AliasedRelation alias -> Optional.of(alias.alias()); + case CommonTableReference commonTable -> Optional.of(commonTable.name()); + case JoinRelation join -> throw unsupportedSemanticExpression( + join.span(), + "HogQL LEFT ANY JOIN with a compound right relation cannot preserve relation qualifiers in Trino"); + case TableReference table -> { + if (table.parts().size() > 1) { + throw unsupportedSemanticExpression( + table.span(), + "HogQL LEFT ANY JOIN with an unaliased qualified table cannot preserve its qualifier in Trino"); + } + yield Optional.of(table.parts().getLast()); + } + case UnnestRelation unnest -> Optional.of(unnest.alias()); + case PivotRelation _, SubqueryRelation _, TablePlaceholder _, ValuesRelation _ -> Optional.empty(); + }; + } + private static TableSubquery createValuesRelation(ValuesRelation values, Map parameterIds) { NodeLocation valuesLocation = new NodeLocation(values.span().startLine(), values.span().startColumn() + 1); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index 332042d31910..ab333b7df726 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -577,6 +577,30 @@ public void testPreservesSetOperationAndValuesSourceLocations() assertThat(values.getLocation()).contains(new NodeLocation(3, 16)); } + @Test + public void testLowersLeftAnyJoinToCorrelatedLateralLimit() + { + Statement statement = compiler.compile( + "SELECT e.id, p.name FROM events e LEFT ANY JOIN persons p ON e.person_id = p.id"); + Statement expected = sqlParser.createStatement( + "SELECT e.id, p.name FROM events e " + + "LEFT JOIN LATERAL (SELECT * FROM persons p WHERE e.person_id = p.id ORDER BY p.id LIMIT 1) p ON true"); + + assertThat(statement).isEqualTo(expected); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @Test + public void testRejectsLeftAnyJoinUsing() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile("SELECT * FROM events e LEFT ANY JOIN persons p USING (person_id)")); + + assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception).hasMessageContaining("cannot preserve merged-column semantics"); + } + @Test public void testPreservesAliasedJoinSourceLocations() { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index ae63b53603f4..5e635f9ad16b 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -2144,9 +2144,15 @@ private JoinType buildJoinType(HogQLParser.JoinOpContext context) return JoinType.INNER; } if (context instanceof HogQLParser.JoinOpLeftRightContext outer) { - if (outer.ANTI() != null || outer.SEMI() != null || outer.ASOF() != null || outer.ALL() != null || outer.ANY() != null) { + if (outer.ANTI() != null || outer.SEMI() != null || outer.ASOF() != null || outer.ALL() != null) { throw unsupported(context, "HogQL-specific outer join modifier"); } + if (outer.ANY() != null) { + if (outer.LEFT() == null) { + throw unsupported(context, "RIGHT ANY JOIN"); + } + return JoinType.LEFT_ANY; + } return outer.LEFT() != null ? JoinType.LEFT : JoinType.RIGHT; } if (context instanceof HogQLParser.JoinOpFullContext full) { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index 6554c62e71ff..32b9a86ad0b6 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -886,6 +886,7 @@ public enum JoinType CROSS, INNER, LEFT, + LEFT_ANY, RIGHT, FULL, } diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 7440aca9034c..180d6e813048 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -333,6 +333,18 @@ public void testBuildsAliasedJoinAstWithSourceSpans() assertThat(criteria.span()).isEqualTo(new HogQlQuery.SourceSpan(52, 73, 3, 24, 3, 45)); } + @ParameterizedTest + @ValueSource(strings = {"LEFT ANY JOIN", "ANY LEFT JOIN"}) + public void testBuildsLeftAnyJoinAst(String joinOperator) + { + HogQlQuery query = parser.parseStatement( + "SELECT e.id FROM events e " + joinOperator + " persons p ON e.person_id = p.id"); + + JoinRelation join = (JoinRelation) query.from().orElseThrow(); + assertThat(join.type()).isEqualTo(JoinType.LEFT_ANY); + assertThat(join.criteria()).hasValueSatisfying(JoinOn.class::isInstance); + } + @Test public void testBuildsPivotAstWithCompositeKeysAliasesAndSourceSpans() { diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java index a42c71089b8d..62efdb631bb7 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java @@ -69,6 +69,15 @@ public void testAnalyzerBindsResolvedCorrelatedLogicalField() .doesNotThrowAnyException(); } + @Test + public void testAnalyzerPlansLeftAnyJoinLateralLimit() + { + assertThatCode(() -> plan(compile( + "SELECT r.regionkey, n.nationkey FROM region r " + + "LEFT ANY JOIN nation n ON r.regionkey = n.regionkey"))) + .doesNotThrowAnyException(); + } + @Test public void testAnalyzerRejectsMissingCorrelatedOuterColumn() { From 629ef0af5e04623e48b665699ba42315e9edfcc5 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 02:30:58 -0400 Subject: [PATCH 121/136] feat(hogql): support remaining week functions --- .../hogql/compiler/HogQlFunctionResolver.java | 20 ++++++++++++++++-- .../compiler/HogQlV0FunctionRegistry.java | 3 ++- .../catalog/HogQlSemanticCatalogSnapshot.java | 4 +++- .../compiler/TestHogQlFunctionResolver.java | 21 +++++++++++++++++++ 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index f8c062f36fe9..ab5e4b1c3236 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -461,7 +461,7 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< List.of(arguments.getFirst(), new UnaryExpression(HogQlQuery.UnaryOperator.NEGATE, arguments.get(1), span)), span); case INTERVAL_MONTH -> new IntervalExpression(arguments.getFirst(), HogQlQuery.IntervalUnit.MONTH, span); - case START_WEEK -> startOfWeek(arguments.getFirst(), span); + case START_WEEK -> startOfWeek(function, arguments); case SPLIT_STRING -> call("split", List.of(arguments.get(1), arguments.getFirst()), span); case CAST_BIGINT -> cast(arguments.getFirst(), "bigint", span); case CAST_TIMESTAMP -> arguments.size() == 1 @@ -1158,8 +1158,24 @@ private static Expression empty(Expression value, boolean expectedEmpty, HogQlQu span); } - private static Expression startOfWeek(Expression value, HogQlQuery.SourceSpan span) + private static Expression startOfWeek(FunctionCall function, List arguments) { + HogQlQuery.SourceSpan span = function.span(); + String mode = "0"; + if (arguments.size() == 2) { + Expression modeExpression = arguments.get(1); + if (!(modeExpression instanceof Literal literal) || literal.kind() != HogQlQuery.LiteralKind.INTEGER) { + throw unsupportedError(function, "HogQL toStartOfWeek mode must be an integer literal"); + } + mode = literal.value(); + } + if (mode.equals("1") || mode.equals("3")) { + return dateTrunc("week", arguments.getFirst(), span); + } + if (!mode.equals("0")) { + throw unsupportedError(function, "HogQL toStartOfWeek mode must be 0, 1, or 3"); + } + Expression value = arguments.getFirst(); Expression shifted = call("date_add", List.of( new Literal(HogQlQuery.LiteralKind.STRING, "day", span), integerLiteral("1", span), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index c4c85469197e..6ae8e95588df 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -73,7 +73,7 @@ final class HogQlV0FunctionRegistry rewrite("tuple", FunctionRewrite.TUPLE, "row", variadicSignature(1)), rewrite("subtractMonths", FunctionRewrite.SUBTRACT_MONTHS, "any", signature(2)), rewrite("toIntervalMonth", FunctionRewrite.INTERVAL_MONTH, "interval year to month", signature(1)), - rewrite("toStartOfWeek", FunctionRewrite.START_WEEK, "timestamp", signature(1)), + rewrite("toStartOfWeek", FunctionRewrite.START_WEEK, "timestamp", signature(1), signature(2)), rewrite("subtractYears", FunctionRewrite.SUBTRACT_YEARS, "any", signature(2)), rewrite("toIntOrZero", FunctionRewrite.INT_OR_ZERO, "bigint", signature(1)), rewrite("_toInt16", FunctionRewrite.CAST_SMALLINT, "smallint", signature(1)), @@ -133,6 +133,7 @@ final class HogQlV0FunctionRegistry scalar("startsWith", "starts_with", signature(2)), scalar("substring", "substring", signature(2), signature(3)), scalar("log10", "log10", signature(1)), + scalar("exp", "exp", signature(1)), scalar("match", "regexp_like", signature(2)), scalar("floor", "floor", signature(1)), scalar("toDayOfMonth", "day", signature(1)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index e5726d898abd..a430f5e578f1 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -302,12 +302,14 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi return switch (rewrite) { case ARRAY_ENUMERATE, ARRAY_SUM, ASSUME_NOT_NULL, CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_SMALLINT, CAST_UUID, CAST_VARCHAR, EMPTY, FLOAT_OR_ZERO, INTERVAL_MONTH, INT_OR_ZERO, - MD5, NOT_EMPTY, START_WEEK, TO_JSON_STRING, + MD5, NOT_EMPTY, TO_JSON_STRING, DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, GROUP_UNIQ_ARRAY, INTERVAL_DAY, IS_NOT_NULL, IS_NULL, COUNT_IF, NOT, PARSE_TIMESTAMP, TO_UNIX_TIMESTAMP, COUNT_DISTINCT, UNIQ_EXACT -> !signature.variadic() && signature.argumentTypes().size() == 1; + case START_WEEK -> + !signature.variadic() && (signature.argumentTypes().size() == 1 || signature.argumentTypes().size() == 2); case ADD_DAYS, ADD_MONTHS, ANY_IF, ARRAY_ELEMENT, ARRAY_FILTER, ARRAY_FIRST, ARRAY_MAP, AVG_IF, DATE_PART, DECIMAL_CAST, DIVIDE_DECIMAL, EQUALS, FLOAT_OR_DEFAULT, GREATER, GREATER_OR_EQUAL, HAS, GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_HAS, JSON_KEYS_AND_VALUES, JSON_VALUE, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index f5ffcf9b7d1e..f6ba629c5736 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -142,6 +142,27 @@ public void testCompilerResolvesV0FunctionsWithoutCatalogContext() "CASE WHEN first THEN 'one' WHEN second THEN 'two' ELSE 'other' END FROM metrics")); } + @Test + public void testCompilerResolvesExponentialAndWeekStartModes() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT exp(value), toStartOfWeek(timestamp, 1), toStartOfWeek(timestamp, 3) FROM metrics")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT exp(value), date_trunc('week', timestamp), date_trunc('week', timestamp) FROM metrics")); + } + + @Test + public void testCompilerRejectsUnsupportedWeekStartMode() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope("SELECT toStartOfWeek(timestamp, 2) FROM metrics"))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception).hasMessageContaining("mode must be 0, 1, or 3"); + } + @Test public void testCompilerRejectsUnknownFunctionsWithoutCatalogContext() { From 0bacabb15c64c58b9ec0d8a97241c89af0971ae2 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 02:35:14 -0400 Subject: [PATCH 122/136] feat(hogql): lower hogqlx tags --- .../hogql/compiler/TestHogQlXCompiler.java | 34 +++++++++ .../io/trino/hogql/parser/HogQlParser.java | 74 +++++++++++++++++++ .../trino/hogql/parser/TestHogQlXParser.java | 53 +++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlXCompiler.java create mode 100644 core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlXParser.java diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlXCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlXCompiler.java new file mode 100644 index 000000000000..019ccef32cb0 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlXCompiler.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlXCompiler +{ + private final SqlParser sqlParser = new SqlParser(); + + @Test + public void testLowersNestedTagToStockTrinoRows() + { + assertThat(new HogQlCompiler().compile( + "SELECT {event}Bold! FROM events")) + .isEqualTo(sqlParser.createStatement( + "SELECT ROW('__hx_tag', 'a', 'href', 'https://example.com', 'target', true, " + + "'children', ROW(event, ROW('__hx_tag', 'strong', 'children', ROW('Bold!')))) FROM events")); + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 5e635f9ad16b..10c53076c24a 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -1185,6 +1185,9 @@ private Expression buildExpression(HogQLParser.ColumnExprValueContext context) if (context instanceof HogQLParser.ColumnExprTemplateStringContext template) { return buildTemplateString(template.templateString()); } + if (context instanceof HogQLParser.ColumnExprTagElementContext tag) { + return buildHogQlXTag(tag.hogqlxTagElement()); + } if (context instanceof HogQLParser.ColumnExprTupleContext tuple) { return new TupleExpression(buildExpressions(tuple.columnExprList()), sourceSpan(tuple)); } @@ -1414,6 +1417,77 @@ private Expression buildTemplateString(HogQLParser.TemplateStringContext context sourceSpan(context)); } + private TupleExpression buildHogQlXTag(HogQLParser.HogqlxTagElementContext context) + { + HogQLParser.IdentifierContext openingIdentifier; + List attributes; + List children = new ArrayList<>(); + if (context instanceof HogQLParser.HogqlxTagElementClosedContext closed) { + openingIdentifier = closed.identifier(); + attributes = closed.hogqlxTagAttribute(); + } + else if (context instanceof HogQLParser.HogqlxTagElementNestedContext nested) { + openingIdentifier = nested.identifier(0); + Identifier opening = buildIdentifier(openingIdentifier); + Identifier closing = buildIdentifier(nested.identifier(1)); + if (!opening.value().equals(closing.value())) { + throw unsupported(nested.identifier(1), "mismatched HogQLX closing tag"); + } + attributes = nested.hogqlxTagAttribute(); + for (HogQLParser.HogqlxChildElementContext child : nested.hogqlxChildElement()) { + if (child.hogqlxTagElement() != null) { + children.add(buildHogQlXTag(child.hogqlxTagElement())); + } + else if (child.hogqlxText() != null) { + String text = child.hogqlxText().getText(); + if (!((text.indexOf('\n') >= 0 || text.indexOf('\r') >= 0) && text.isBlank())) { + children.add(new Literal(STRING, text, sourceSpan(child.hogqlxText()))); + } + } + else { + children.add(buildExpression(child.columnExpr())); + } + } + } + else { + throw unsupported(context, "HogQLX tag"); + } + + List tupleValues = new ArrayList<>(); + tupleValues.add(new Literal(STRING, "__hx_tag", sourceSpan(context))); + tupleValues.add(new Literal(STRING, buildIdentifier(openingIdentifier).value(), sourceSpan(openingIdentifier))); + boolean hasChildrenAttribute = false; + for (HogQLParser.HogqlxTagAttributeContext attribute : attributes) { + Identifier name = buildIdentifier(attribute.identifier()); + hasChildrenAttribute |= name.value().equals("children"); + tupleValues.add(new Literal(STRING, name.value(), sourceSpan(attribute.identifier()))); + tupleValues.add(buildHogQlXAttributeValue(attribute)); + } + if (!children.isEmpty()) { + if (hasChildrenAttribute) { + throw unsupported(context, "HogQLX tag with both nested children and a children attribute"); + } + tupleValues.add(new Literal(STRING, "children", sourceSpan(context))); + tupleValues.add(new TupleExpression(children, sourceSpan(context))); + } + return new TupleExpression(tupleValues, sourceSpan(context)); + } + + private Expression buildHogQlXAttributeValue(HogQLParser.HogqlxTagAttributeContext attribute) + { + if (attribute.string() != null) { + HogQLParser.StringContext value = attribute.string(); + if (value.STRING_LITERAL() != null) { + return new Literal(STRING, decodeQuoted(value.STRING_LITERAL().getText()), sourceSpan(value)); + } + return buildTemplateString(value.templateString()); + } + if (attribute.columnExpr() != null) { + return buildExpression(attribute.columnExpr()); + } + return new Literal(HogQlQuery.LiteralKind.BOOLEAN, "true", sourceSpan(attribute)); + } + private IntervalExpression buildStringInterval(HogQLParser.ColumnExprIntervalStringContext context) { String value = decodeQuoted(context.STRING_LITERAL().getText()); diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlXParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlXParser.java new file mode 100644 index 000000000000..c57032d29393 --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlXParser.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlXParser +{ + private final HogQlParser parser = new HogQlParser(); + + @Test + public void testLowersNestedTagToHxTuple() + { + HogQlQuery query = parser.parseStatement( + "SELECT {event}Bold! FROM events"); + TupleExpression tag = (TupleExpression) ((ExpressionProjection) query.projections().getFirst()).expression(); + + assertThat(tag.values()).hasSize(8); + assertThat(tag.values().get(0)).isEqualTo(new Literal(HogQlQuery.LiteralKind.STRING, "__hx_tag", tag.values().get(0).span())); + assertThat(tag.values().get(1)).isInstanceOfSatisfying( + Literal.class, + literal -> assertThat(literal.value()).isEqualTo("a")); + assertThat(tag.values().get(7)).isInstanceOfSatisfying( + TupleExpression.class, + children -> assertThat(children.values()).hasSize(2)); + } + + @Test + public void testRejectsMismatchedClosingTag() + { + assertThatThrownBy(() -> parser.parseStatement("SELECT text")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("mismatched HogQLX closing tag"); + } +} From 9d42e45420679576975d7c53a13cd13ce57b7c40 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 02:39:55 -0400 Subject: [PATCH 123/136] feat(hogql): lower nested expression aliases --- .../main/java/io/trino/hogql/parser/HogQlParser.java | 3 +++ .../java/io/trino/hogql/parser/TestHogQlParser.java | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 10c53076c24a..7a9902d12e08 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -1134,6 +1134,9 @@ private List buildStarReplacements(HogQLParser.ColumnsReplaceLi private Expression buildExpression(HogQLParser.ColumnExprContext context) { + if (context instanceof HogQLParser.ColumnExprAliasContext alias) { + return buildExpression(alias.columnExpr()); + } if (context instanceof HogQLParser.ColumnExprValuePassthroughContext passthrough) { return buildExpression(passthrough.columnExprValue()); } diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 180d6e813048..2f29527783c2 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -345,6 +345,16 @@ public void testBuildsLeftAnyJoinAst(String joinOperator) assertThat(join.criteria()).hasValueSatisfying(JoinOn.class::isInstance); } + @Test + public void testUnwrapsNestedExpressionAlias() + { + HogQlQuery query = parser.parseStatement("SELECT md5((first || second) AS TEXT)"); + FunctionCall md5 = (FunctionCall) ((ExpressionProjection) query.projections().getFirst()).expression(); + + assertThat(md5.arguments()).hasSize(1); + assertThat(md5.arguments().getFirst()).isInstanceOf(BinaryExpression.class); + } + @Test public void testBuildsPivotAstWithCompositeKeysAliasesAndSourceSpans() { From 70bf7dfa49550ea4001341099cc6628dd93633d4 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 02:42:56 -0400 Subject: [PATCH 124/136] feat(hogql): expand scalar with aliases --- .../io/trino/hogql/parser/HogQlParser.java | 25 +++++++++++++++++-- .../trino/hogql/parser/TestHogQlParser.java | 16 +++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 7a9902d12e08..48a01a9b8a67 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -106,9 +106,11 @@ import java.util.ArrayList; import java.util.Deque; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Optional; import java.util.Set; @@ -488,6 +490,7 @@ private static final class AstBuilder private final SourcePositions sourcePositions; private final Deque> commonTableScopes = new ArrayDeque<>(); private final Deque> prohibitedCommonTableScopes = new ArrayDeque<>(); + private final Deque> withExpressionScopes = new ArrayDeque<>(); private AstBuilder(String source) { @@ -735,6 +738,7 @@ private HogQlQuery buildQuery(SelectedQuery selectedQuery) rejectUnsupportedClauses(select); commonTableScopes.push(new LinkedHashSet<>()); prohibitedCommonTableScopes.push(new LinkedHashSet<>()); + withExpressionScopes.push(new LinkedHashMap<>()); try { List with = buildCommonTables(select.withClause()); @@ -779,6 +783,7 @@ private HogQlQuery buildQuery(SelectedQuery selectedQuery) sourceSpan(select)); } finally { + withExpressionScopes.pop(); prohibitedCommonTableScopes.pop(); commonTableScopes.pop(); } @@ -795,9 +800,16 @@ private List buildCommonTables(HogQLParser.WithClauseCont List commonTables = new ArrayList<>(); Set localNames = commonTableScopes.getFirst(); for (HogQLParser.WithExprContext expression : context.withExprList().withExpr()) { - if (!(expression instanceof HogQLParser.WithExprSubqueryContext subquery)) { - throw unsupported(expression, "non-query WITH expression"); + if (expression instanceof HogQLParser.WithExprColumnContext column) { + Identifier name = buildIdentifier(column.identifier()); + String canonicalName = canonicalName(name); + if (!localNames.add(canonicalName)) { + throw unsupported(column, "duplicate WITH name"); + } + withExpressionScopes.getFirst().put(canonicalName, buildExpression(column.columnExpr())); + continue; } + HogQLParser.WithExprSubqueryContext subquery = (HogQLParser.WithExprSubqueryContext) expression; if (subquery.USING() != null) { throw unsupported(subquery, "CTE USING KEY"); } @@ -1785,6 +1797,15 @@ private Expression buildColumnReference(HogQLParser.ColumnIdentifierContext cont return new Literal(HogQlQuery.LiteralKind.BOOLEAN, "false", sourceSpan(context)); } } + if (parts.size() == 1) { + String name = canonicalName(parts.getFirst()); + for (Map scope : withExpressionScopes) { + Expression expression = scope.get(name); + if (expression != null) { + return expression; + } + } + } return new ColumnReference(parts, sourceSpan(context)); } diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 2f29527783c2..c4a2680b2628 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -84,7 +84,6 @@ public class TestHogQlParser "WITH x AS MATERIALIZED (SELECT 1) SELECT * FROM x", "WITH x AS NOT MATERIALIZED (SELECT 1) SELECT * FROM x", "WITH x USING KEY (id) AS (SELECT 1) SELECT * FROM x", - "WITH 1 AS x SELECT x", "WITH x AS (SELECT * FROM x) SELECT * FROM x", "WITH x AS (SELECT 1), y AS (WITH x AS (SELECT * FROM x) SELECT * FROM x) SELECT * FROM y", "WITH x AS (SELECT 1), x AS (SELECT 2) SELECT * FROM x", @@ -355,6 +354,21 @@ public void testUnwrapsNestedExpressionAlias() assertThat(md5.arguments().getFirst()).isInstanceOf(BinaryExpression.class); } + @Test + public void testExpandsWithExpressionAliases() + { + HogQlQuery query = parser.parseStatement( + "WITH (SELECT max(date) FROM daily) AS last_day SELECT last_day, last_day + 1"); + + assertThat(query.with()).isEmpty(); + assertThat(((ExpressionProjection) query.projections().get(0)).expression()) + .isInstanceOf(ScalarSubqueryExpression.class); + assertThat(((ExpressionProjection) query.projections().get(1)).expression()) + .isInstanceOfSatisfying( + BinaryExpression.class, + addition -> assertThat(addition.left()).isInstanceOf(ScalarSubqueryExpression.class)); + } + @Test public void testBuildsPivotAstWithCompositeKeysAliasesAndSourceSpans() { From 901c2dca86cd08f89d09969508589a12ab93c801 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 02:46:56 -0400 Subject: [PATCH 125/136] feat(hogql): lower inner any joins --- .../trino/hogql/compiler/TrinoAstFactory.java | 60 +++++++++++++------ .../hogql/compiler/TestHogQlCompiler.java | 25 ++++++-- .../io/trino/hogql/parser/HogQlParser.java | 4 +- .../trino/hogql/parser/tree/HogQlQuery.java | 1 + .../trino/hogql/parser/TestHogQlParser.java | 11 +++- 5 files changed, 73 insertions(+), 28 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java index ede64303b6e9..b62f0ef6d64e 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -790,14 +790,15 @@ private static io.trino.sql.tree.Relation createRelation(Relation relation, Map< private static io.trino.sql.tree.Relation createJoin(JoinRelation join, Map parameterIds) { - if (join.type() == HogQlQuery.JoinType.LEFT_ANY) { - return createLeftAnyJoin(join, parameterIds); + if (join.type() == HogQlQuery.JoinType.LEFT_ANY || join.type() == HogQlQuery.JoinType.INNER_ANY) { + return createAnyJoin(join, parameterIds); } return new io.trino.sql.tree.Join( location(join.span()), switch (join.type()) { case CROSS -> io.trino.sql.tree.Join.Type.CROSS; case INNER -> io.trino.sql.tree.Join.Type.INNER; + case INNER_ANY -> throw new IllegalStateException("INNER ANY JOIN must be lowered as a lateral join"); case LEFT -> io.trino.sql.tree.Join.Type.LEFT; case LEFT_ANY -> throw new IllegalStateException("LEFT ANY JOIN must be lowered as a lateral join"); case RIGHT -> io.trino.sql.tree.Join.Type.RIGHT; @@ -813,27 +814,27 @@ private static io.trino.sql.tree.Relation createJoin(JoinRelation join, Map parameterIds) + private static io.trino.sql.tree.Relation createAnyJoin(JoinRelation join, Map parameterIds) { HogQlQuery.JoinCriteria criteria = join.criteria().orElseThrow(); - if (!(criteria instanceof JoinOn on)) { - throw unsupportedSemanticExpression(criteria.span(), "HogQL LEFT ANY JOIN with USING cannot preserve merged-column semantics in Trino"); - } - - Identifier alias = leftAnyJoinAlias(join.right()) + Identifier rightAlias = anyJoinAlias(join.right()) .orElseThrow(() -> unsupportedSemanticExpression( join.right().span(), - "HogQL LEFT ANY JOIN requires a right relation alias for Trino correlation")); - HogQlQuery.Expression orderingKey = leftAnyJoinOrderingKey(on.expression(), alias) + "HogQL ANY JOIN requires a right relation alias for Trino correlation")); + HogQlQuery.Expression predicate = switch (criteria) { + case JoinOn on -> on.expression(); + case JoinUsing using -> anyJoinUsingPredicate(join.left(), rightAlias, using); + }; + HogQlQuery.Expression orderingKey = anyJoinOrderingKey(predicate, rightAlias) .orElseThrow(() -> unsupportedSemanticExpression( - on.span(), - "HogQL LEFT ANY JOIN requires an equality predicate on a qualified right column for Trino decorrelation")); + criteria.span(), + "HogQL ANY JOIN requires an equality predicate on a qualified right column for Trino decorrelation")); NodeLocation rightLocation = location(join.right().span()); QuerySpecification matches = new QuerySpecification( rightLocation, new Select(rightLocation, false, List.of(new AllColumns(rightLocation))), Optional.of(createRelation(join.right(), parameterIds)), - Optional.of(createExpression(on.expression(), parameterIds)), + Optional.of(createExpression(predicate, parameterIds)), Optional.empty(), Optional.empty(), List.of(), @@ -859,24 +860,45 @@ private static io.trino.sql.tree.Relation createLeftAnyJoin(JoinRelation join, M lateral = new io.trino.sql.tree.AliasedRelation( rightLocation, lateral, - createIdentifier(alias), + createIdentifier(rightAlias), null); return new io.trino.sql.tree.Join( location(join.span()), - io.trino.sql.tree.Join.Type.LEFT, + join.type() == HogQlQuery.JoinType.LEFT_ANY ? io.trino.sql.tree.Join.Type.LEFT : io.trino.sql.tree.Join.Type.INNER, createRelation(join.left(), parameterIds), lateral, Optional.of(new io.trino.sql.tree.JoinOn(new BooleanLiteral(location(join.span()), "true")))); } - private static Optional leftAnyJoinOrderingKey(HogQlQuery.Expression expression, Identifier rightAlias) + private static HogQlQuery.Expression anyJoinUsingPredicate(Relation left, Identifier rightAlias, JoinUsing using) + { + Identifier leftAlias = anyJoinAlias(left) + .orElseThrow(() -> unsupportedSemanticExpression( + left.span(), + "HogQL ANY JOIN with USING requires a left relation alias for Trino correlation")); + HogQlQuery.Expression predicate = null; + for (Identifier column : using.columns()) { + ColumnReference leftColumn = new ColumnReference(List.of(leftAlias, column), using.span()); + ColumnReference rightColumn = new ColumnReference(List.of(rightAlias, column), using.span()); + HogQlQuery.Expression equality = new BinaryExpression(HogQlQuery.BinaryOperator.EQUAL, leftColumn, rightColumn, using.span()); + predicate = predicate == null + ? equality + : new BinaryExpression(HogQlQuery.BinaryOperator.AND, predicate, equality, using.span()); + } + if (predicate == null) { + throw unsupportedSemanticExpression(using.span(), "HogQL ANY JOIN USING requires at least one column"); + } + return predicate; + } + + private static Optional anyJoinOrderingKey(HogQlQuery.Expression expression, Identifier rightAlias) { if (!(expression instanceof BinaryExpression binary)) { return Optional.empty(); } if (binary.operator() == HogQlQuery.BinaryOperator.AND) { - return leftAnyJoinOrderingKey(binary.left(), rightAlias) - .or(() -> leftAnyJoinOrderingKey(binary.right(), rightAlias)); + return anyJoinOrderingKey(binary.left(), rightAlias) + .or(() -> anyJoinOrderingKey(binary.right(), rightAlias)); } if (binary.operator() != HogQlQuery.BinaryOperator.EQUAL) { return Optional.empty(); @@ -897,7 +919,7 @@ private static boolean isQualifiedBy(HogQlQuery.Expression expression, Identifie reference.parts().getFirst().value().equalsIgnoreCase(qualifier.value()); } - private static Optional leftAnyJoinAlias(Relation relation) + private static Optional anyJoinAlias(Relation relation) { return switch (relation) { case AliasedRelation alias -> Optional.of(alias.alias()); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index ab333b7df726..f93a4893c676 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -591,14 +591,27 @@ public void testLowersLeftAnyJoinToCorrelatedLateralLimit() } @Test - public void testRejectsLeftAnyJoinUsing() + public void testLowersInnerAnyJoinUsingToCorrelatedLateralLimit() { - TrinoException exception = catchThrowableOfType( - TrinoException.class, - () -> compiler.compile("SELECT * FROM events e LEFT ANY JOIN persons p USING (person_id)")); + Statement statement = compiler.compile( + "SELECT e.id, p.name FROM events e ANY INNER JOIN persons p USING (person_id)"); + Statement expected = sqlParser.createStatement( + "SELECT e.id, p.name FROM events e " + + "INNER JOIN LATERAL (SELECT * FROM persons p WHERE e.person_id = p.person_id ORDER BY p.person_id LIMIT 1) p ON true"); - assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); - assertThat(exception).hasMessageContaining("cannot preserve merged-column semantics"); + assertThat(statement).isEqualTo(expected); + } + + @Test + public void testLowersLeftAnyJoinUsingToCorrelatedLateralLimit() + { + Statement statement = compiler.compile( + "SELECT e.id, p.name FROM events e LEFT ANY JOIN persons p USING (person_id)"); + Statement expected = sqlParser.createStatement( + "SELECT e.id, p.name FROM events e " + + "LEFT JOIN LATERAL (SELECT * FROM persons p WHERE e.person_id = p.person_id ORDER BY p.person_id LIMIT 1) p ON true"); + + assertThat(statement).isEqualTo(expected); } @Test diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 48a01a9b8a67..8f354f5fd044 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -2236,10 +2236,10 @@ private JoinType buildJoinType(HogQLParser.JoinOpContext context) return JoinType.INNER; } if (context instanceof HogQLParser.JoinOpInnerContext inner) { - if (inner.ANTI() != null || inner.SEMI() != null || inner.ASOF() != null || inner.ALL() != null || inner.ANY() != null) { + if (inner.ANTI() != null || inner.SEMI() != null || inner.ASOF() != null || inner.ALL() != null) { throw unsupported(context, "HogQL-specific inner join modifier"); } - return JoinType.INNER; + return inner.ANY() == null ? JoinType.INNER : JoinType.INNER_ANY; } if (context instanceof HogQLParser.JoinOpLeftRightContext outer) { if (outer.ANTI() != null || outer.SEMI() != null || outer.ASOF() != null || outer.ALL() != null) { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java index 32b9a86ad0b6..e640e11944f8 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -885,6 +885,7 @@ public enum JoinType { CROSS, INNER, + INNER_ANY, LEFT, LEFT_ANY, RIGHT, diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index c4a2680b2628..3833846b01a6 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -65,7 +65,6 @@ public class TestHogQlParser "SELECT", "SELECT * FROM first JOIN second", "SELECT * FROM first NATURAL JOIN second", - "SELECT * FROM first ANY INNER JOIN second ON first.id = second.id", "SELECT * FROM first LEFT SEMI JOIN second ON first.id = second.id", "SELECT * FROM first ASOF JOIN second ON first.id = second.id", "SELECT * FROM first POSITIONAL JOIN second", @@ -344,6 +343,16 @@ public void testBuildsLeftAnyJoinAst(String joinOperator) assertThat(join.criteria()).hasValueSatisfying(JoinOn.class::isInstance); } + @Test + public void testBuildsInnerAnyJoinAst() + { + HogQlQuery query = parser.parseStatement( + "SELECT e.id FROM events e ANY INNER JOIN persons p USING (person_id)"); + + JoinRelation join = (JoinRelation) query.from().orElseThrow(); + assertThat(join.type()).isEqualTo(JoinType.INNER_ANY); + } + @Test public void testUnwrapsNestedExpressionAlias() { From 05f39a74e5771478a353279b973e7492380096e7 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 02:49:31 -0400 Subject: [PATCH 126/136] feat(hogql): lower subtract days --- .../java/io/trino/hogql/compiler/HogQlFunctionResolver.java | 4 ++++ .../java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java | 1 + .../hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java | 3 ++- .../io/trino/hogql/compiler/TestHogQlFunctionResolver.java | 4 ++-- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index ab5e4b1c3236..9ac8bccd0363 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -460,6 +460,10 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< "month", List.of(arguments.getFirst(), new UnaryExpression(HogQlQuery.UnaryOperator.NEGATE, arguments.get(1), span)), span); + case SUBTRACT_DAYS -> dateAdd( + "day", + List.of(arguments.getFirst(), new UnaryExpression(HogQlQuery.UnaryOperator.NEGATE, arguments.get(1), span)), + span); case INTERVAL_MONTH -> new IntervalExpression(arguments.getFirst(), HogQlQuery.IntervalUnit.MONTH, span); case START_WEEK -> startOfWeek(function, arguments); case SPLIT_STRING -> call("split", List.of(arguments.get(1), arguments.getFirst()), span); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index 6ae8e95588df..d77693088d16 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -72,6 +72,7 @@ final class HogQlV0FunctionRegistry rewrite("in", FunctionRewrite.IN_ARRAY, "boolean", signature(2)), rewrite("tuple", FunctionRewrite.TUPLE, "row", variadicSignature(1)), rewrite("subtractMonths", FunctionRewrite.SUBTRACT_MONTHS, "any", signature(2)), + rewrite("subtractDays", FunctionRewrite.SUBTRACT_DAYS, "any", signature(2)), rewrite("toIntervalMonth", FunctionRewrite.INTERVAL_MONTH, "interval year to month", signature(1)), rewrite("toStartOfWeek", FunctionRewrite.START_WEEK, "timestamp", signature(1), signature(2)), rewrite("subtractYears", FunctionRewrite.SUBTRACT_YEARS, "any", signature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index a430f5e578f1..3e29c0abf791 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -315,7 +315,7 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_HAS, JSON_KEYS_AND_VALUES, JSON_VALUE, IN_ARRAY, INT_DIV, LESS_OR_EQUAL, LIKE, MAX_IF, MIN_IF, MULTIPLY, MULTIPLY_DECIMAL, MEDIAN_IF, MINUS, NOT_EQUALS, PLUS, QUANTILE, QUANTILE_EXACT, REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, SURVEY_RESPONSE, - SUBTRACT_MONTHS, SUBTRACT_YEARS, SUM_IF, + SUBTRACT_DAYS, SUBTRACT_MONTHS, SUBTRACT_YEARS, SUM_IF, TUPLE_ELEMENT, UNIQ_EXACT_IF, UNIQ_IF -> !signature.variadic() && signature.argumentTypes().size() == 2; case ARG_MAX_IF, ARG_MIN_IF, ARRAY_SLICE, QUANTILE_IF, REGEX_REPLACE_ALL, REGEX_REPLACE_ONE -> @@ -1416,6 +1416,7 @@ public enum FunctionRewrite SPLIT_CHAR, SPLIT_STRING, START_WEEK, + SUBTRACT_DAYS, SUBTRACT_MONTHS, SUBTRACT_YEARS, SUM_IF, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index f6ba629c5736..164cda105231 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -214,14 +214,14 @@ public void testCompilerLowersJsonExtractionFunctions() public void testCompilerLowersDateTimeCompatibilityFunctions() { HogQlCompilationResult result = new HogQlCompiler().compile(envelope( - "SELECT today(), toIntervalDay(2), addDays(timestamp, 3), addMonths(timestamp, -2), " + + "SELECT today(), toIntervalDay(2), addDays(timestamp, 3), subtractDays(timestamp, 3), addMonths(timestamp, -2), " + "dateAdd(timestamp, INTERVAL 4 DAY), fromUnixTimestamp(epoch), toUnixTimestamp(timestamp), " + "toDateTime(timestamp, 'UTC'), formatDateTime(timestamp, '%Y-%m-%d'), " + "toTimeZone(timestamp, 'UTC'), parseDateTimeBestEffort(text) FROM records")); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( "SELECT CAST(now() AS date), 2 * INTERVAL '1' DAY, date_add('day', 3, timestamp), " + - "date_add('month', -2, timestamp), timestamp + 4 * INTERVAL '1' DAY, " + + "date_add('day', -(3), timestamp), date_add('month', -2, timestamp), timestamp + 4 * INTERVAL '1' DAY, " + "from_unixtime(epoch), CAST(to_unixtime(timestamp) AS bigint), " + "with_timezone(CAST(timestamp AS timestamp(0)), 'UTC'), date_format(timestamp, '%Y-%m-%d'), " + "at_timezone(timestamp, 'UTC'), TRY_CAST(text AS timestamp(3)) FROM records")); From 748431e74c38af6669ea248e9a7baa6530bf956c Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 02:52:13 -0400 Subject: [PATCH 127/136] feat(hogql): load exchange rate snapshots --- ...edAsyncHogQlExchangeRateSnapshotCache.java | 396 ++++++++++++++++++ .../catalog/HogQlExchangeRateException.java | 52 +++ .../catalog/HogQlExchangeRateSnapshot.java | 119 ++++++ .../HogQlExchangeRateSnapshotCache.java | 44 ++ .../HogQlExchangeRateSnapshotJsonDecoder.java | 199 +++++++++ .../HogQlExchangeRateSnapshotLoader.java | 63 +++ .../HogQlExchangeRateSnapshotProvider.java | 53 +++ ...edAsyncHogQlExchangeRateSnapshotCache.java | 220 ++++++++++ ...tHogQlExchangeRateSnapshotJsonDecoder.java | 106 +++++ .../TestHogQlExchangeRateSnapshotLoader.java | 61 +++ ...TestHogQlExchangeRateSnapshotProvider.java | 69 +++ .../hogql/HogQlExchangeRateHttpTransport.java | 186 ++++++++ .../TestHogQlExchangeRateHttpTransport.java | 165 ++++++++ 13 files changed, 1733 insertions(+) create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlExchangeRateSnapshotCache.java create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateException.java create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshot.java create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotCache.java create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotJsonDecoder.java create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotLoader.java create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotProvider.java create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlExchangeRateSnapshotCache.java create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotJsonDecoder.java create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotLoader.java create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotProvider.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateHttpTransport.java create mode 100644 core/trino-main/src/test/java/io/trino/hogql/TestHogQlExchangeRateHttpTransport.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlExchangeRateSnapshotCache.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlExchangeRateSnapshotCache.java new file mode 100644 index 000000000000..cca111bb1d6a --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlExchangeRateSnapshotCache.java @@ -0,0 +1,396 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException.Failure; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.function.LongSupplier; + +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public final class BoundedAsyncHogQlExchangeRateSnapshotCache + implements HogQlExchangeRateSnapshotCache +{ + private final Object lock = new Object(); + private final int maximumEntries; + private final long refreshAfterNanos; + private final long expireAfterNanos; + private final long failureBackoffNanos; + private final LongSupplier ticker; + private final Executor executor; + private final SnapshotLoader loader; + private final LinkedHashMap entries = new LinkedHashMap<>(16, 0.75f, true); + private final LinkedHashMap observedGenerations = new LinkedHashMap<>(16, 0.75f, true); + + public BoundedAsyncHogQlExchangeRateSnapshotCache( + int maximumEntries, + Duration refreshAfter, + Duration expireAfter, + Duration failureBackoff, + LongSupplier ticker, + Executor executor, + SnapshotLoader loader) + { + if (maximumEntries <= 0) { + throw new IllegalArgumentException("maximumEntries must be positive"); + } + this.maximumEntries = maximumEntries; + refreshAfterNanos = nonNegativeNanos(refreshAfter, "refreshAfter"); + expireAfterNanos = positiveNanos(expireAfter, "expireAfter"); + failureBackoffNanos = nonNegativeNanos(failureBackoff, "failureBackoff"); + if (refreshAfterNanos > expireAfterNanos) { + throw new IllegalArgumentException("refreshAfter must not exceed expireAfter"); + } + this.ticker = requireNonNull(ticker, "ticker is null"); + this.executor = requireNonNull(executor, "executor is null"); + this.loader = requireNonNull(loader, "loader is null"); + } + + @Override + public Optional currentSnapshot(OptionalLong expectedGeneration) + { + expectedGeneration = HogQlExchangeRateSnapshotCache.validateExpectedGeneration(expectedGeneration); + RefreshTask refreshTask; + Optional result; + List evictedRefreshes = new ArrayList<>(1); + long now = ticker.getAsLong(); + synchronized (lock) { + Entry entry = entries.get(expectedGeneration); + if (entry == null) { + entry = inheritedExactEntry(expectedGeneration, now).orElseGet(Entry::new); + entries.put(expectedGeneration, entry); + } + HogQlExchangeRateSnapshot snapshot = entry.snapshot; + long age = snapshot == null ? Long.MAX_VALUE : elapsedNanos(entry.loadedAtNanos, now); + result = snapshot != null && age < expireAfterNanos ? Optional.of(snapshot) : Optional.empty(); + boolean refreshRequired = snapshot == null || + (expectedGeneration.isEmpty() && age >= refreshAfterNanos) || + (expectedGeneration.isPresent() && age >= expireAfterNanos); + refreshTask = refreshRequired ? prepareRefresh(expectedGeneration, entry, now, false) : null; + evictEntries(evictedRefreshes); + if (!entries.containsKey(expectedGeneration)) { + refreshTask = null; + } + } + cancelRefreshes(evictedRefreshes, "HogQL exchange-rate cache entry was evicted"); + dispatch(refreshTask); + return result; + } + + public CompletionStage prewarm(OptionalLong expectedGeneration) + { + expectedGeneration = HogQlExchangeRateSnapshotCache.validateExpectedGeneration(expectedGeneration); + RefreshTask refreshTask; + CompletableFuture future; + List evictedRefreshes = new ArrayList<>(1); + long now = ticker.getAsLong(); + synchronized (lock) { + Entry entry = entries.get(expectedGeneration); + if (entry == null) { + entry = inheritedExactEntry(expectedGeneration, now).orElseGet(Entry::new); + entries.put(expectedGeneration, entry); + } + boolean unexpired = entry.snapshot != null && elapsedNanos(entry.loadedAtNanos, now) < expireAfterNanos; + if (unexpired && expectedGeneration.isPresent()) { + refreshTask = null; + future = CompletableFuture.completedFuture(entry.snapshot); + } + else { + refreshTask = prepareRefresh(expectedGeneration, entry, now, true); + future = refreshTask == null ? entry.refresh : refreshTask.result(); + } + evictEntries(evictedRefreshes); + if (!entries.containsKey(expectedGeneration)) { + refreshTask = null; + } + } + cancelRefreshes(evictedRefreshes, "HogQL exchange-rate cache entry was evicted"); + dispatch(refreshTask); + return future.minimalCompletionStage(); + } + + public CompletionStage prewarm() + { + return prewarm(OptionalLong.empty()); + } + + public void invalidate() + { + List refreshes = new ArrayList<>(); + synchronized (lock) { + for (Entry entry : entries.values()) { + if (entry.refresh != null) { + refreshes.add(new RefreshCancellation(entry.refresh, entry.upstream)); + entry.refresh = null; + entry.upstream = null; + } + } + entries.clear(); + } + cancelRefreshes(refreshes, "HogQL exchange-rate cache was invalidated"); + } + + private Optional inheritedExactEntry(OptionalLong expectedGeneration, long now) + { + if (expectedGeneration.isEmpty()) { + return Optional.empty(); + } + Entry latest = entries.get(OptionalLong.empty()); + if (latest == null || latest.snapshot == null || latest.snapshot.generation() != expectedGeneration.orElseThrow()) { + return Optional.empty(); + } + if (elapsedNanos(latest.loadedAtNanos, now) >= expireAfterNanos) { + return Optional.empty(); + } + Entry inherited = new Entry(); + inherited.snapshot = latest.snapshot; + inherited.loadedAtNanos = latest.loadedAtNanos; + return Optional.of(inherited); + } + + private RefreshTask prepareRefresh(OptionalLong expectedGeneration, Entry entry, long now, boolean force) + { + if (entry.refresh != null) { + return null; + } + if (!force && entry.refreshBackoffActive && elapsedNanos(entry.lastRefreshFailureAtNanos, now) < failureBackoffNanos) { + return null; + } + CompletableFuture result = new CompletableFuture<>(); + entry.refresh = result; + return new RefreshTask(expectedGeneration, entry, result); + } + + private void dispatch(RefreshTask refreshTask) + { + if (refreshTask == null) { + return; + } + try { + executor.execute(() -> load(refreshTask)); + } + catch (RuntimeException e) { + completeFailure(refreshTask, e); + } + } + + private void load(RefreshTask refreshTask) + { + CompletionStage loaded; + try { + loaded = requireNonNull(loader.load(refreshTask.expectedGeneration()), "snapshot loader returned null"); + } + catch (RuntimeException failure) { + completeFailure(refreshTask, failure); + return; + } + catch (Error failure) { + completeFailure(refreshTask, failure); + throw failure; + } + CompletableFuture upstream = loaded.toCompletableFuture(); + boolean owned; + synchronized (lock) { + Entry entry = entries.get(refreshTask.expectedGeneration()); + owned = entry == refreshTask.entry() && entry.refresh == refreshTask.result(); + if (owned) { + entry.upstream = upstream; + } + } + if (!owned) { + upstream.cancel(true); + return; + } + loaded.whenComplete((snapshot, failure) -> { + if (failure != null) { + completeFailure(refreshTask, failure); + } + else if (snapshot == null) { + completeFailure(refreshTask, new NullPointerException("snapshot loader completed with null")); + } + else { + completeSuccess(refreshTask, snapshot); + } + }); + } + + private void completeSuccess(RefreshTask refreshTask, HogQlExchangeRateSnapshot snapshot) + { + Throwable rejection = validateLoadedSnapshot(refreshTask, snapshot); + if (rejection != null) { + completeFailure(refreshTask, rejection); + return; + } + + boolean published = false; + synchronized (lock) { + Entry entry = entries.get(refreshTask.expectedGeneration()); + if (entry == refreshTask.entry() && entry.refresh == refreshTask.result()) { + HogQlExchangeRateSnapshot observed = observedGenerations.get(snapshot.generation()); + if (observed != null && !observed.equals(snapshot)) { + rejection = reject(entry, "HogQL exchange-rate generation content changed"); + } + else if (refreshTask.expectedGeneration().isEmpty() && entry.snapshot != null && snapshot.generation() < entry.snapshot.generation()) { + rejection = reject(entry, format("HogQL exchange-rate generation regressed from %s to %s", entry.snapshot.generation(), snapshot.generation())); + } + else { + observedGenerations.put(snapshot.generation(), snapshot); + while (observedGenerations.size() > maximumEntries) { + observedGenerations.remove(observedGenerations.entrySet().iterator().next().getKey()); + } + entry.snapshot = snapshot; + entry.loadedAtNanos = ticker.getAsLong(); + entry.refreshBackoffActive = false; + entry.refresh = null; + entry.upstream = null; + published = true; + } + } + } + if (published) { + refreshTask.result().complete(snapshot); + } + else if (rejection != null) { + refreshTask.result().completeExceptionally(rejection); + } + else { + refreshTask.result().completeExceptionally(new CancellationException("HogQL exchange-rate refresh no longer owns its cache entry")); + } + } + + private Throwable reject(Entry entry, String message) + { + entry.refresh = null; + entry.upstream = null; + entry.lastRefreshFailureAtNanos = ticker.getAsLong(); + entry.refreshBackoffActive = true; + return new HogQlExchangeRateException(Failure.GENERATION_MISMATCH, message); + } + + private static Throwable validateLoadedSnapshot(RefreshTask refreshTask, HogQlExchangeRateSnapshot snapshot) + { + if (refreshTask.expectedGeneration().isPresent() && snapshot.generation() != refreshTask.expectedGeneration().orElseThrow()) { + return new HogQlExchangeRateException(Failure.GENERATION_MISMATCH, "HogQL exchange-rate snapshot generation does not match the refreshed generation"); + } + return null; + } + + private void completeFailure(RefreshTask refreshTask, Throwable failure) + { + requireNonNull(failure, "failure is null"); + boolean owned; + synchronized (lock) { + Entry entry = entries.get(refreshTask.expectedGeneration()); + owned = entry == refreshTask.entry() && entry.refresh == refreshTask.result(); + if (owned) { + entry.refresh = null; + entry.upstream = null; + entry.lastRefreshFailureAtNanos = ticker.getAsLong(); + entry.refreshBackoffActive = true; + } + } + if (owned) { + refreshTask.result().completeExceptionally(failure); + } + else { + refreshTask.result().completeExceptionally(new CancellationException("HogQL exchange-rate refresh no longer owns its cache entry")); + } + } + + private void evictEntries(List evictedRefreshes) + { + while (entries.size() > maximumEntries) { + Map.Entry victim = entries.entrySet().stream() + .filter(entry -> entry.getKey().isPresent()) + .findFirst() + .orElseGet(() -> entries.entrySet().iterator().next()); + entries.remove(victim.getKey()); + if (victim.getValue().refresh != null) { + evictedRefreshes.add(new RefreshCancellation(victim.getValue().refresh, victim.getValue().upstream)); + victim.getValue().refresh = null; + victim.getValue().upstream = null; + } + } + } + + private static void cancelRefreshes(List refreshes, String message) + { + refreshes.forEach(refresh -> { + if (refresh.upstream() != null) { + refresh.upstream().cancel(true); + } + refresh.result().completeExceptionally(new CancellationException(message)); + }); + } + + private static long elapsedNanos(long start, long end) + { + return Math.max(0, end - start); + } + + private static long nonNegativeNanos(Duration duration, String name) + { + requireNonNull(duration, name + " is null"); + if (duration.isNegative()) { + throw new IllegalArgumentException(name + " must not be negative"); + } + return duration.toNanos(); + } + + private static long positiveNanos(Duration duration, String name) + { + long nanos = nonNegativeNanos(duration, name); + if (nanos == 0) { + throw new IllegalArgumentException(name + " must be positive"); + } + return nanos; + } + + @FunctionalInterface + public interface SnapshotLoader + { + CompletionStage load(OptionalLong expectedGeneration); + } + + private static final class Entry + { + private HogQlExchangeRateSnapshot snapshot; + private long loadedAtNanos; + private long lastRefreshFailureAtNanos; + private boolean refreshBackoffActive; + private CompletableFuture refresh; + private CompletableFuture upstream; + } + + private record RefreshTask( + OptionalLong expectedGeneration, + Entry entry, + CompletableFuture result) {} + + private record RefreshCancellation( + CompletableFuture result, + CompletableFuture upstream) {} +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateException.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateException.java new file mode 100644 index 000000000000..4da3aa52047d --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateException.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.HogQlErrorCode; +import io.trino.spi.TrinoException; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_CATALOG_GENERATION_MISMATCH; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_CATALOG_NOT_READY; +import static java.util.Objects.requireNonNull; + +public final class HogQlExchangeRateException + extends TrinoException +{ + private final Failure failure; + + public HogQlExchangeRateException(Failure failure, String message) + { + super(errorCode(requireNonNull(failure, "failure is null")), message); + this.failure = failure; + } + + public Failure failure() + { + return failure; + } + + private static HogQlErrorCode errorCode(Failure failure) + { + return switch (failure) { + case UNAVAILABLE -> HOGQL_CATALOG_NOT_READY; + case GENERATION_MISMATCH -> HOGQL_CATALOG_GENERATION_MISMATCH; + }; + } + + public enum Failure + { + UNAVAILABLE, + GENERATION_MISMATCH, + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshot.java new file mode 100644 index 000000000000..0df0092cd9fe --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshot.java @@ -0,0 +1,119 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import java.time.LocalDate; +import java.time.format.DateTimeParseException; +import java.util.List; +import java.util.regex.Pattern; + +import static java.util.Objects.requireNonNull; + +public record HogQlExchangeRateSnapshot( + int protocolVersion, + int schemaVersion, + long generation, + String baseCurrency, + int decimalScale, + List rates) +{ + public static final int PROTOCOL_VERSION = 1; + public static final int SCHEMA_VERSION = 1; + public static final String BASE_CURRENCY = "USD"; + public static final int DECIMAL_SCALE = 10; + public static final int MAXIMUM_RATES = 1_000_000; + + private static final Pattern CURRENCY_PATTERN = Pattern.compile("[A-Z]{3}"); + private static final Pattern UNSCALED_RATE_PATTERN = Pattern.compile("0|[1-9][0-9]{0,17}"); + private static final String UNSCALED_ONE = "10000000000"; + + public HogQlExchangeRateSnapshot + { + if (protocolVersion != PROTOCOL_VERSION) { + throw new IllegalArgumentException("unsupported HogQL exchange-rate protocol"); + } + if (schemaVersion != SCHEMA_VERSION) { + throw new IllegalArgumentException("unsupported HogQL exchange-rate schema"); + } + if (generation <= 0) { + throw new IllegalArgumentException("HogQL exchange-rate generation must be positive"); + } + baseCurrency = requireNonNull(baseCurrency, "baseCurrency is null"); + if (!baseCurrency.equals(BASE_CURRENCY)) { + throw new IllegalArgumentException("unsupported HogQL exchange-rate base currency"); + } + if (decimalScale != DECIMAL_SCALE) { + throw new IllegalArgumentException("unsupported HogQL exchange-rate decimal scale"); + } + rates = List.copyOf(requireNonNull(rates, "rates is null")); + if (rates.isEmpty()) { + throw new IllegalArgumentException("HogQL exchange-rate snapshot has no rates"); + } + if (rates.size() > MAXIMUM_RATES) { + throw new IllegalArgumentException("HogQL exchange-rate snapshot exceeds rate limit"); + } + + boolean baseCurrencyPresent = false; + ExchangeRate previous = null; + for (ExchangeRate rate : rates) { + requireNonNull(rate, "rate is null"); + if (rate.currency().equals(BASE_CURRENCY)) { + baseCurrencyPresent = true; + if (!rate.unscaledRate().equals(UNSCALED_ONE)) { + throw new IllegalArgumentException("HogQL base-currency rate must equal one"); + } + } + if (previous != null && compare(previous, rate) >= 0) { + throw new IllegalArgumentException("HogQL exchange rates are not strictly sorted"); + } + previous = rate; + } + if (!baseCurrencyPresent) { + throw new IllegalArgumentException("HogQL exchange-rate snapshot has no base-currency rate"); + } + } + + private static int compare(ExchangeRate left, ExchangeRate right) + { + int currencyComparison = left.currency().compareTo(right.currency()); + if (currencyComparison != 0) { + return currencyComparison; + } + return left.effectiveDate().compareTo(right.effectiveDate()); + } + + public record ExchangeRate(String currency, String effectiveDate, String unscaledRate) + { + public ExchangeRate + { + currency = requireNonNull(currency, "currency is null"); + effectiveDate = requireNonNull(effectiveDate, "effectiveDate is null"); + unscaledRate = requireNonNull(unscaledRate, "unscaledRate is null"); + if (!CURRENCY_PATTERN.matcher(currency).matches()) { + throw new IllegalArgumentException("invalid HogQL exchange-rate currency"); + } + try { + if (!LocalDate.parse(effectiveDate).toString().equals(effectiveDate)) { + throw new IllegalArgumentException("noncanonical HogQL exchange-rate date"); + } + } + catch (DateTimeParseException e) { + throw new IllegalArgumentException("invalid HogQL exchange-rate date", e); + } + if (!UNSCALED_RATE_PATTERN.matcher(unscaledRate).matches()) { + throw new IllegalArgumentException("invalid HogQL exchange-rate value"); + } + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotCache.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotCache.java new file mode 100644 index 000000000000..56643f6997dc --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotCache.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import java.util.Optional; +import java.util.OptionalLong; + +import static java.util.Objects.requireNonNull; + +@FunctionalInterface +public interface HogQlExchangeRateSnapshotCache +{ + Optional currentSnapshot(OptionalLong expectedGeneration); + + default Optional currentSnapshot() + { + return currentSnapshot(OptionalLong.empty()); + } + + default Optional currentSnapshot(long expectedGeneration) + { + return currentSnapshot(OptionalLong.of(expectedGeneration)); + } + + static OptionalLong validateExpectedGeneration(OptionalLong expectedGeneration) + { + requireNonNull(expectedGeneration, "expectedGeneration is null"); + if (expectedGeneration.isPresent() && expectedGeneration.orElseThrow() <= 0) { + throw new IllegalArgumentException("expected exchange-rate generation must be positive"); + } + return expectedGeneration; + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotJsonDecoder.java new file mode 100644 index 000000000000..dcfad08ca4ed --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotJsonDecoder.java @@ -0,0 +1,199 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.StreamReadFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot.ExchangeRate; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.LoadRequest; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public final class HogQlExchangeRateSnapshotJsonDecoder +{ + public static final int MAXIMUM_PAYLOAD_BYTES = 32 * 1024 * 1024; + + private static final Set SNAPSHOT_FIELDS = Set.of( + "protocolVersion", + "schemaVersion", + "generation", + "baseCurrency", + "decimalScale", + "rates"); + private static final Set RATE_FIELDS = Set.of("currency", "effectiveDate", "unscaledRate"); + + private final ObjectMapper objectMapper; + + public HogQlExchangeRateSnapshotJsonDecoder() + { + JsonFactory jsonFactory = JsonFactory.builder() + .streamReadConstraints(StreamReadConstraints.builder() + .maxNestingDepth(16) + .maxStringLength(MAXIMUM_PAYLOAD_BYTES) + .maxNumberLength(64) + .build()) + .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) + .build(); + objectMapper = new ObjectMapper(jsonFactory); + } + + public HogQlExchangeRateSnapshot decode(byte[] payload, LoadRequest request) + { + requireNonNull(request, "request is null"); + if (payload == null) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + if (payload.length > MAXIMUM_PAYLOAD_BYTES) { + throw failure(DecodeFailure.LIMIT_EXCEEDED); + } + + try (JsonParser parser = objectMapper.createParser(payload)) { + JsonNode document = objectMapper.readTree(parser); + if (!(document instanceof ObjectNode root) || parser.nextToken() != null) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + requireFields(root, SNAPSHOT_FIELDS); + int protocolVersion = integer(root, "protocolVersion"); + if (protocolVersion != HogQlExchangeRateSnapshot.PROTOCOL_VERSION) { + throw failure(DecodeFailure.UNSUPPORTED_PROTOCOL); + } + int schemaVersion = integer(root, "schemaVersion"); + if (schemaVersion != HogQlExchangeRateSnapshot.SCHEMA_VERSION) { + throw failure(DecodeFailure.UNSUPPORTED_SCHEMA); + } + long generation = longInteger(root, "generation"); + if (request.expectedGeneration().isPresent() && generation != request.expectedGeneration().orElseThrow()) { + throw failure(DecodeFailure.GENERATION_MISMATCH); + } + String baseCurrency = text(root, "baseCurrency"); + int decimalScale = integer(root, "decimalScale"); + ArrayNode rateNodes = array(root, "rates"); + if (rateNodes.size() > HogQlExchangeRateSnapshot.MAXIMUM_RATES) { + throw failure(DecodeFailure.LIMIT_EXCEEDED); + } + List rates = new ArrayList<>(rateNodes.size()); + for (JsonNode node : rateNodes) { + if (!(node instanceof ObjectNode rate)) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + requireFields(rate, RATE_FIELDS); + rates.add(new ExchangeRate( + text(rate, "currency"), + text(rate, "effectiveDate"), + text(rate, "unscaledRate"))); + } + return new HogQlExchangeRateSnapshot(protocolVersion, schemaVersion, generation, baseCurrency, decimalScale, rates); + } + catch (DecodeException e) { + throw e; + } + catch (IOException | RuntimeException e) { + throw new DecodeException(DecodeFailure.INVALID_PAYLOAD, e); + } + } + + private static void requireFields(ObjectNode object, Set expected) + { + Set actual = new HashSet<>(); + object.fieldNames().forEachRemaining(actual::add); + if (!actual.equals(expected)) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + } + + private static int integer(ObjectNode object, String field) + { + JsonNode value = object.get(field); + if (value == null || !value.isIntegralNumber() || !value.canConvertToInt()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return value.intValue(); + } + + private static long longInteger(ObjectNode object, String field) + { + JsonNode value = object.get(field); + if (value == null || !value.isIntegralNumber() || !value.canConvertToLong()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return value.longValue(); + } + + private static String text(ObjectNode object, String field) + { + JsonNode value = object.get(field); + if (value == null || !value.isTextual()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return value.textValue(); + } + + private static ArrayNode array(ObjectNode object, String field) + { + JsonNode value = object.get(field); + if (!(value instanceof ArrayNode array)) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return array; + } + + private static DecodeException failure(DecodeFailure failure) + { + return new DecodeException(failure); + } + + public enum DecodeFailure + { + INVALID_PAYLOAD, + LIMIT_EXCEEDED, + UNSUPPORTED_PROTOCOL, + UNSUPPORTED_SCHEMA, + GENERATION_MISMATCH, + } + + public static final class DecodeException + extends IllegalArgumentException + { + private final DecodeFailure failure; + + private DecodeException(DecodeFailure failure) + { + super("invalid HogQL exchange-rate snapshot"); + this.failure = requireNonNull(failure, "failure is null"); + } + + private DecodeException(DecodeFailure failure, Throwable cause) + { + super("invalid HogQL exchange-rate snapshot", cause); + this.failure = requireNonNull(failure, "failure is null"); + } + + public DecodeFailure failure() + { + return failure; + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotLoader.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotLoader.java new file mode 100644 index 000000000000..03a7b0eb1c78 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotLoader.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import java.util.OptionalLong; +import java.util.concurrent.CompletionStage; + +import static java.util.Objects.requireNonNull; + +@FunctionalInterface +public interface HogQlExchangeRateSnapshotLoader +{ + CompletionStage load(LoadRequest request); + + static HogQlExchangeRateSnapshotLoader fromJsonTransport(JsonTransport transport, HogQlExchangeRateSnapshotJsonDecoder decoder) + { + requireNonNull(transport, "transport is null"); + requireNonNull(decoder, "decoder is null"); + return request -> { + requireNonNull(request, "request is null"); + CompletionStage response = requireNonNull(transport.load(request), "transport returned null"); + return response.thenApply(payload -> decoder.decode(payload, request)); + }; + } + + @FunctionalInterface + interface JsonTransport + { + CompletionStage load(LoadRequest request); + } + + record LoadRequest(OptionalLong expectedGeneration) + { + public LoadRequest + { + expectedGeneration = requireNonNull(expectedGeneration, "expectedGeneration is null"); + if (expectedGeneration.isPresent() && expectedGeneration.orElseThrow() <= 0) { + throw new IllegalArgumentException("expected exchange-rate generation must be positive"); + } + } + + public static LoadRequest latest() + { + return new LoadRequest(OptionalLong.empty()); + } + + public static LoadRequest pinned(long generation) + { + return new LoadRequest(OptionalLong.of(generation)); + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotProvider.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotProvider.java new file mode 100644 index 000000000000..7c089abf0f8d --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotProvider.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException.Failure; + +import java.util.OptionalLong; + +import static java.util.Objects.requireNonNull; + +@FunctionalInterface +public interface HogQlExchangeRateSnapshotProvider +{ + PinnedSnapshot pin(OptionalLong expectedGeneration); + + static HogQlExchangeRateSnapshotProvider fromCache(HogQlExchangeRateSnapshotCache cache) + { + requireNonNull(cache, "cache is null"); + return expectedGeneration -> { + HogQlExchangeRateSnapshotCache.validateExpectedGeneration(expectedGeneration); + HogQlExchangeRateSnapshot snapshot = cache.currentSnapshot(expectedGeneration) + .orElseThrow(() -> new HogQlExchangeRateException(Failure.UNAVAILABLE, "HogQL exchange-rate snapshot is unavailable")); + if (expectedGeneration.isPresent() && snapshot.generation() != expectedGeneration.orElseThrow()) { + throw new HogQlExchangeRateException(Failure.GENERATION_MISMATCH, "HogQL exchange-rate snapshot generation does not match the request"); + } + return new PinnedSnapshot(snapshot); + }; + } + + record PinnedSnapshot(HogQlExchangeRateSnapshot snapshot) + { + public PinnedSnapshot + { + snapshot = requireNonNull(snapshot, "snapshot is null"); + } + + public long generation() + { + return snapshot.generation(); + } + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlExchangeRateSnapshotCache.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlExchangeRateSnapshotCache.java new file mode 100644 index 000000000000..3c2dc2a94173 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlExchangeRateSnapshotCache.java @@ -0,0 +1,220 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.BoundedAsyncHogQlExchangeRateSnapshotCache.SnapshotLoader; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException.Failure; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot.ExchangeRate; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicLong; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestBoundedAsyncHogQlExchangeRateSnapshotCache +{ + private static final Duration REFRESH_AFTER = Duration.ofNanos(10); + private static final Duration EXPIRE_AFTER = Duration.ofNanos(20); + private static final Duration FAILURE_BACKOFF = Duration.ofNanos(100); + + @Test + public void testLatestAndExactReadsShareLoadsAndRemainGenerationPinned() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlExchangeRateSnapshotCache cache = cache(3, ticker, loader); + + CompletableFuture initialLoad = loader.expect(OptionalLong.empty()); + CompletionStage first = cache.prewarm(); + CompletionStage shared = cache.prewarm(); + assertThat(loader.loadCount(OptionalLong.empty())).isEqualTo(1); + initialLoad.complete(snapshot(1)); + assertThat(first.toCompletableFuture()).isCompletedWithValue(snapshot(1)); + assertThat(shared.toCompletableFuture()).isCompletedWithValue(snapshot(1)); + + assertThat(cache.currentSnapshot(OptionalLong.of(1))).contains(snapshot(1)); + assertThat(loader.loadCount(OptionalLong.of(1))).isZero(); + + CompletableFuture latestRefresh = loader.expect(OptionalLong.empty()); + ticker.set(REFRESH_AFTER.toNanos()); + assertThat(cache.currentSnapshot()).contains(snapshot(1)); + latestRefresh.complete(snapshot(2)); + + assertThat(cache.currentSnapshot()).contains(snapshot(2)); + assertThat(cache.currentSnapshot(OptionalLong.of(1))).contains(snapshot(1)); + } + + @Test + public void testRejectsGenerationRegressionAndChangedSameGeneration() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlExchangeRateSnapshotCache cache = cache(3, ticker, loader); + completePrewarm(cache, loader, OptionalLong.empty(), snapshot(2)); + + CompletableFuture regressedLoad = loader.expect(OptionalLong.empty()); + CompletionStage regressed = cache.prewarm(); + regressedLoad.complete(snapshot(1)); + assertFailure(regressed, Failure.GENERATION_MISMATCH); + assertThat(cache.currentSnapshot()).contains(snapshot(2)); + + CompletableFuture changedLoad = loader.expect(OptionalLong.empty()); + CompletionStage changed = cache.prewarm(); + changedLoad.complete(snapshot(2, "9100000000")); + assertFailure(changed, Failure.GENERATION_MISMATCH); + assertThat(cache.currentSnapshot()).contains(snapshot(2)); + } + + @Test + public void testExactGenerationEvictionCancelsLoadAndNeverFallsBackToLatest() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlExchangeRateSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, OptionalLong.empty(), snapshot(1)); + + CompletableFuture generationSevenLoad = loader.expect(OptionalLong.of(7)); + CompletionStage generationSeven = cache.prewarm(OptionalLong.of(7)); + CompletableFuture generationEightLoad = loader.expect(OptionalLong.of(8)); + CompletionStage generationEight = cache.prewarm(OptionalLong.of(8)); + + assertThat(generationSevenLoad).isCancelled(); + assertThat(generationSeven.toCompletableFuture()).isCompletedExceptionally(); + assertThat(cache.currentSnapshot()).contains(snapshot(1)); + generationEightLoad.complete(snapshot(8)); + assertThat(generationEight.toCompletableFuture()).isCompletedWithValue(snapshot(8)); + + cache.invalidate(); + CompletableFuture latestLoad = loader.expect(OptionalLong.empty()); + assertThat(cache.currentSnapshot()).isEmpty(); + latestLoad.complete(snapshot(9)); + CompletableFuture missingExact = loader.expect(OptionalLong.of(7)); + assertThat(cache.currentSnapshot(OptionalLong.of(7))).isEmpty(); + missingExact.completeExceptionally(new IllegalStateException("metadata unavailable")); + assertThat(cache.currentSnapshot(OptionalLong.of(7))).isEmpty(); + assertThat(cache.currentSnapshot()).contains(snapshot(9)); + } + + @Test + public void testRefreshFailureServesLastGoodOnlyUntilExpiryAndBacksOff() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlExchangeRateSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, OptionalLong.empty(), snapshot(1)); + + CompletableFuture failedLoad = loader.expect(OptionalLong.empty()); + ticker.set(REFRESH_AFTER.toNanos()); + assertThat(cache.currentSnapshot()).contains(snapshot(1)); + failedLoad.completeExceptionally(new IllegalStateException("metadata unavailable")); + + ticker.set(EXPIRE_AFTER.toNanos()); + assertThat(cache.currentSnapshot()).isEmpty(); + assertThat(loader.loadCount(OptionalLong.empty())).isEqualTo(2); + + CompletableFuture recoveredLoad = loader.expect(OptionalLong.empty()); + ticker.set(REFRESH_AFTER.plus(FAILURE_BACKOFF).toNanos()); + assertThat(cache.currentSnapshot()).isEmpty(); + recoveredLoad.complete(snapshot(2)); + assertThat(cache.currentSnapshot()).contains(snapshot(2)); + } + + private static BoundedAsyncHogQlExchangeRateSnapshotCache cache(int maximumEntries, AtomicLong ticker, SnapshotLoader loader) + { + return new BoundedAsyncHogQlExchangeRateSnapshotCache( + maximumEntries, + REFRESH_AFTER, + EXPIRE_AFTER, + FAILURE_BACKOFF, + ticker::get, + Runnable::run, + loader); + } + + private static void completePrewarm( + BoundedAsyncHogQlExchangeRateSnapshotCache cache, + ControlledLoader loader, + OptionalLong expectedGeneration, + HogQlExchangeRateSnapshot snapshot) + { + CompletableFuture load = loader.expect(expectedGeneration); + CompletionStage refresh = cache.prewarm(expectedGeneration); + load.complete(snapshot); + assertThat(refresh.toCompletableFuture()).isCompletedWithValue(snapshot); + } + + private static void assertFailure(CompletionStage refresh, Failure failure) + { + assertThatThrownBy(refresh.toCompletableFuture()::join) + .cause() + .isInstanceOfSatisfying(HogQlExchangeRateException.class, exception -> assertThat(exception.failure()).isEqualTo(failure)); + } + + private static HogQlExchangeRateSnapshot snapshot(long generation) + { + return snapshot(generation, "9049000000"); + } + + private static HogQlExchangeRateSnapshot snapshot(long generation, String eurRate) + { + return new HogQlExchangeRateSnapshot( + 1, + 1, + generation, + "USD", + 10, + List.of( + new ExchangeRate("EUR", "2024-01-01", eurRate), + new ExchangeRate("USD", "1970-01-01", "10000000000"))); + } + + private static final class ControlledLoader + implements SnapshotLoader + { + private final Map>> expected = new HashMap<>(); + private final Map loadCounts = new HashMap<>(); + + public CompletableFuture expect(OptionalLong generation) + { + CompletableFuture future = new CompletableFuture<>(); + expected.computeIfAbsent(generation, _ -> new ArrayDeque<>()).add(future); + return future; + } + + public int loadCount(OptionalLong generation) + { + return loadCounts.getOrDefault(generation, 0); + } + + @Override + public CompletionStage load(OptionalLong expectedGeneration) + { + loadCounts.merge(expectedGeneration, 1, Integer::sum); + ArrayDeque> queue = expected.get(expectedGeneration); + if (queue == null || queue.isEmpty()) { + throw new AssertionError("unexpected exchange-rate load: " + expectedGeneration); + } + return queue.remove(); + } + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotJsonDecoder.java new file mode 100644 index 000000000000..a64dc486acf1 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotJsonDecoder.java @@ -0,0 +1,106 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotJsonDecoder.DecodeException; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotJsonDecoder.DecodeFailure; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.LoadRequest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlExchangeRateSnapshotJsonDecoder +{ + private final HogQlExchangeRateSnapshotJsonDecoder decoder = new HogQlExchangeRateSnapshotJsonDecoder(); + + @Test + public void testDecodesCanonicalLatestAndPinnedSnapshots() + { + HogQlExchangeRateSnapshot latest = decoder.decode(snapshotJson(7).getBytes(StandardCharsets.UTF_8), LoadRequest.latest()); + HogQlExchangeRateSnapshot pinned = decoder.decode(snapshotJson(7).getBytes(StandardCharsets.UTF_8), LoadRequest.pinned(7)); + + assertThat(latest).isEqualTo(pinned); + assertThat(latest.generation()).isEqualTo(7); + assertThat(latest.rates()).containsExactly( + new HogQlExchangeRateSnapshot.ExchangeRate("EUR", "2024-01-01", "9049000000"), + new HogQlExchangeRateSnapshot.ExchangeRate("USD", "1970-01-01", "10000000000")); + assertThatThrownBy(() -> latest.rates().clear()).isInstanceOf(UnsupportedOperationException.class); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("invalidDocuments") + public void testRejectsNoncanonicalDocuments(String name, String document, LoadRequest request, DecodeFailure failure) + { + assertThatThrownBy(() -> decoder.decode(document.getBytes(StandardCharsets.UTF_8), request)) + .isInstanceOfSatisfying(DecodeException.class, exception -> { + assertThat(exception.failure()).isEqualTo(failure); + assertThat(exception.getMessage()).hasSizeLessThan(128); + }); + } + + @Test + public void testRejectsPayloadOver32MiB() + { + byte[] payload = new byte[HogQlExchangeRateSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES + 1]; + + assertThatThrownBy(() -> decoder.decode(payload, LoadRequest.latest())) + .isInstanceOfSatisfying(DecodeException.class, exception -> assertThat(exception.failure()).isEqualTo(DecodeFailure.LIMIT_EXCEEDED)); + } + + private static Stream invalidDocuments() + { + String valid = snapshotJson(7); + return Stream.of( + Arguments.of("unknown field", valid.replace("\"generation\": 7", "\"generation\": 7, \"unknown\": true"), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("missing field", valid.replace("\"decimalScale\": 10,", ""), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("duplicate field", valid.replace("\"generation\": 7", "\"generation\": 7, \"generation\": 7"), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("trailing document", valid + "{}", LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("null document", "null", LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("unsupported protocol", valid.replace("\"protocolVersion\": 1", "\"protocolVersion\": 2"), LoadRequest.latest(), DecodeFailure.UNSUPPORTED_PROTOCOL), + Arguments.of("unsupported schema", valid.replace("\"schemaVersion\": 1", "\"schemaVersion\": 2"), LoadRequest.latest(), DecodeFailure.UNSUPPORTED_SCHEMA), + Arguments.of("generation mismatch", valid, LoadRequest.pinned(8), DecodeFailure.GENERATION_MISMATCH), + Arguments.of("nonpositive generation", valid.replace("\"generation\": 7", "\"generation\": 0"), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("wrong base", valid.replace("\"baseCurrency\": \"USD\"", "\"baseCurrency\": \"EUR\""), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("wrong scale", valid.replace("\"decimalScale\": 10", "\"decimalScale\": 9"), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("lowercase currency", valid.replace("\"EUR\"", "\"eur\""), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("invalid date", valid.replace("2024-01-01", "2024-02-30"), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("leading zero", valid.replace("9049000000", "09049000000"), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("base rate differs", valid.replace("10000000000", "9999999999"), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("unsorted rates", valid.replace("\"EUR\", \"effectiveDate\": \"2024-01-01\", \"unscaledRate\": \"9049000000\"", "\"ZZZ\", \"effectiveDate\": \"2024-01-01\", \"unscaledRate\": \"9049000000\""), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD)); + } + + private static String snapshotJson(long generation) + { + return """ + { + "protocolVersion": 1, + "schemaVersion": 1, + "generation": %s, + "baseCurrency": "USD", + "decimalScale": 10, + "rates": [ + {"currency": "EUR", "effectiveDate": "2024-01-01", "unscaledRate": "9049000000"}, + {"currency": "USD", "effectiveDate": "1970-01-01", "unscaledRate": "10000000000"} + ] + } + """.formatted(generation); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotLoader.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotLoader.java new file mode 100644 index 000000000000..3e1b5f90825d --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotLoader.java @@ -0,0 +1,61 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.JsonTransport; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.LoadRequest; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlExchangeRateSnapshotLoader +{ + @Test + public void testPreservesLatestAndPinnedSemanticsThroughJsonTransport() + { + List requests = new ArrayList<>(); + JsonTransport transport = request -> { + requests.add(request); + long generation = request.expectedGeneration().orElse(9); + return CompletableFuture.completedFuture(snapshotJson(generation).getBytes(StandardCharsets.UTF_8)); + }; + HogQlExchangeRateSnapshotLoader loader = HogQlExchangeRateSnapshotLoader.fromJsonTransport( + transport, + new HogQlExchangeRateSnapshotJsonDecoder()); + + assertThat(loader.load(LoadRequest.latest()).toCompletableFuture().join().generation()).isEqualTo(9); + assertThat(loader.load(LoadRequest.pinned(7)).toCompletableFuture().join().generation()).isEqualTo(7); + assertThat(requests).containsExactly(LoadRequest.latest(), LoadRequest.pinned(7)); + } + + @Test + public void testRejectsInvalidPinnedGenerationBeforeTransport() + { + assertThatThrownBy(() -> LoadRequest.pinned(0)).isInstanceOf(IllegalArgumentException.class); + } + + private static String snapshotJson(long generation) + { + return """ + {"protocolVersion":1,"schemaVersion":1,"generation":%s,"baseCurrency":"USD","decimalScale":10, + "rates":[{"currency":"USD","effectiveDate":"1970-01-01","unscaledRate":"10000000000"}]} + """.formatted(generation); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotProvider.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotProvider.java new file mode 100644 index 000000000000..126b2ee7af9a --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotProvider.java @@ -0,0 +1,69 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException.Failure; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot.ExchangeRate; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlExchangeRateSnapshotProvider +{ + @Test + public void testPinsOneExactGeneration() + { + AtomicReference requestedGeneration = new AtomicReference<>(); + HogQlExchangeRateSnapshotCache cache = expectedGeneration -> { + requestedGeneration.set(expectedGeneration); + return Optional.of(snapshot(7)); + }; + HogQlExchangeRateSnapshotProvider provider = HogQlExchangeRateSnapshotProvider.fromCache(cache); + + HogQlExchangeRateSnapshotProvider.PinnedSnapshot pinned = provider.pin(OptionalLong.of(7)); + + assertThat(requestedGeneration).hasValue(OptionalLong.of(7)); + assertThat(pinned.generation()).isEqualTo(7); + assertThat(pinned.snapshot()).isEqualTo(snapshot(7)); + } + + @Test + public void testFailsClosedForUnavailableAndMismatchedGeneration() + { + HogQlExchangeRateSnapshotProvider unavailable = HogQlExchangeRateSnapshotProvider.fromCache(_ -> Optional.empty()); + assertThatThrownBy(() -> unavailable.pin(OptionalLong.empty())) + .isInstanceOfSatisfying(HogQlExchangeRateException.class, exception -> assertThat(exception.failure()).isEqualTo(Failure.UNAVAILABLE)); + + HogQlExchangeRateSnapshotProvider mismatched = HogQlExchangeRateSnapshotProvider.fromCache(_ -> Optional.of(snapshot(2))); + assertThatThrownBy(() -> mismatched.pin(OptionalLong.of(1))) + .isInstanceOfSatisfying(HogQlExchangeRateException.class, exception -> assertThat(exception.failure()).isEqualTo(Failure.GENERATION_MISMATCH)); + } + + private static HogQlExchangeRateSnapshot snapshot(long generation) + { + return new HogQlExchangeRateSnapshot( + 1, + 1, + generation, + "USD", + 10, + List.of(new ExchangeRate("USD", "1970-01-01", "10000000000"))); + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateHttpTransport.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateHttpTransport.java new file mode 100644 index 000000000000..34aae458533c --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateHttpTransport.java @@ -0,0 +1,186 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.common.net.MediaType; +import io.airlift.http.client.HttpClient; +import io.airlift.http.client.Request; +import io.airlift.http.client.Response; +import io.airlift.http.client.ResponseHandler; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException.Failure; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotJsonDecoder; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.JsonTransport; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.LoadRequest; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.Supplier; + +import static com.google.common.net.MediaType.JSON_UTF_8; +import static io.airlift.concurrent.MoreFutures.toCompletableFuture; +import static io.airlift.http.client.HeaderNames.ACCEPT; +import static io.airlift.http.client.HeaderNames.CONTENT_TYPE; +import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static io.airlift.http.client.Request.Builder.prepareGet; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +public final class HogQlExchangeRateHttpTransport + implements JsonTransport +{ + private static final String METADATA_PATH = "v1/hogql/compatibility/exchange-rates"; + private static final String AUTHENTICATION_HEADER = "X-Duckgres-Internal-Secret"; + private static final MediaType JSON = JSON_UTF_8.withoutParameters(); + + private final URI baseUri; + private final HttpClient httpClient; + private final int maximumResponseBytes; + private final Supplier authenticationTokenSupplier; + + HogQlExchangeRateHttpTransport(URI baseUri, HttpClient httpClient, Supplier authenticationTokenSupplier) + { + this(baseUri, httpClient, HogQlExchangeRateSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES, authenticationTokenSupplier); + } + + HogQlExchangeRateHttpTransport(URI baseUri, HttpClient httpClient, int maximumResponseBytes, Supplier authenticationTokenSupplier) + { + this.baseUri = validateBaseUri(baseUri); + this.httpClient = requireNonNull(httpClient, "httpClient is null"); + if (maximumResponseBytes <= 0 || maximumResponseBytes > HogQlExchangeRateSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES) { + throw new IllegalArgumentException("invalid HogQL exchange-rate response size limit"); + } + this.maximumResponseBytes = maximumResponseBytes; + this.authenticationTokenSupplier = requireNonNull(authenticationTokenSupplier, "authenticationTokenSupplier is null"); + } + + @Override + public CompletionStage load(LoadRequest request) + { + requireNonNull(request, "request is null"); + String authenticationToken; + try { + authenticationToken = validateAuthenticationToken(authenticationTokenSupplier.get()); + } + catch (RuntimeException _) { + return CompletableFuture.failedFuture(unavailable()); + } + Request httpRequest = prepareGet() + .setUri(buildUri(baseUri, request)) + .setHeader(ACCEPT, JSON.toString()) + .setHeader(AUTHENTICATION_HEADER, authenticationToken) + .build(); + return toCompletableFuture(httpClient.executeAsync(httpRequest, new MetadataResponseHandler(request, maximumResponseBytes))); + } + + static URI buildUri(URI baseUri, LoadRequest request) + { + var uriBuilder = uriBuilderFrom(validateBaseUri(baseUri)) + .appendPath(METADATA_PATH) + .addParameter("protocolVersion", Integer.toString(1)); + request.expectedGeneration().ifPresent(generation -> uriBuilder.addParameter("generation", Long.toString(generation))); + return uriBuilder.build(); + } + + private static String validateAuthenticationToken(String token) + { + requireNonNull(token, "authentication token is null"); + if (token.isBlank() || token.indexOf('\r') >= 0 || token.indexOf('\n') >= 0) { + throw new IllegalArgumentException("invalid HogQL exchange-rate authentication token"); + } + return token; + } + + private static URI validateBaseUri(URI baseUri) + { + requireNonNull(baseUri, "HogQL exchange-rate URI is null"); + String scheme = baseUri.getScheme(); + if (scheme == null || !(scheme.toLowerCase(ENGLISH).equals("http") || scheme.toLowerCase(ENGLISH).equals("https")) || + baseUri.getHost() == null || baseUri.getUserInfo() != null || baseUri.getQuery() != null || baseUri.getFragment() != null) { + throw new IllegalArgumentException("invalid HogQL exchange-rate URI"); + } + return baseUri; + } + + private static HogQlExchangeRateException unavailable() + { + return new HogQlExchangeRateException(Failure.UNAVAILABLE, "HogQL exchange-rate metadata is unavailable"); + } + + private static final class MetadataResponseHandler + implements ResponseHandler + { + private final LoadRequest loadRequest; + private final int maximumResponseBytes; + + private MetadataResponseHandler(LoadRequest loadRequest, int maximumResponseBytes) + { + this.loadRequest = loadRequest; + this.maximumResponseBytes = maximumResponseBytes; + } + + @Override + public byte[] handleException(Request request, Exception exception) + { + throw unavailable(); + } + + @Override + public byte[] handle(Request request, Response response) + { + int statusCode = response.getStatusCode(); + if (statusCode < 200 || statusCode >= 300) { + throw statusFailure(statusCode, loadRequest); + } + if (!isJson(response)) { + throw unavailable(); + } + try (InputStream input = response.getInputStream()) { + byte[] payload = input.readNBytes(maximumResponseBytes + 1); + if (payload.length > maximumResponseBytes) { + throw unavailable(); + } + return payload; + } + catch (IOException e) { + throw unavailable(); + } + } + + private static boolean isJson(Response response) + { + return response.getHeader(CONTENT_TYPE) + .map(value -> { + try { + return MediaType.parse(value).withoutParameters().equals(JSON); + } + catch (IllegalArgumentException _) { + return false; + } + }) + .orElse(false); + } + + private static HogQlExchangeRateException statusFailure(int statusCode, LoadRequest loadRequest) + { + if (statusCode == 409 || (statusCode == 404 && loadRequest.expectedGeneration().isPresent())) { + return new HogQlExchangeRateException(Failure.GENERATION_MISMATCH, "HogQL exchange-rate generation is unavailable"); + } + return unavailable(); + } + } +} diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlExchangeRateHttpTransport.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlExchangeRateHttpTransport.java new file mode 100644 index 000000000000..e8507c837cc4 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlExchangeRateHttpTransport.java @@ -0,0 +1,165 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.common.collect.ImmutableListMultimap; +import io.airlift.http.client.HttpStatus; +import io.airlift.http.client.Request; +import io.airlift.http.client.Response; +import io.airlift.http.client.testing.TestingHttpClient; +import io.airlift.http.client.testing.TestingResponse; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException.Failure; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.LoadRequest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import static com.google.common.net.MediaType.JSON_UTF_8; +import static io.airlift.http.client.HeaderNames.ACCEPT; +import static io.airlift.http.client.HttpStatus.CONFLICT; +import static io.airlift.http.client.HttpStatus.INTERNAL_SERVER_ERROR; +import static io.airlift.http.client.HttpStatus.NOT_FOUND; +import static io.airlift.http.client.HttpStatus.OK; +import static io.airlift.http.client.testing.TestingResponse.contentType; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlExchangeRateHttpTransport +{ + private static final URI BASE_URI = URI.create("https://duckgres.example/control-plane/"); + private static final String SECRET = "secret-metadata-response"; + private static final String AUTHENTICATION_TOKEN = "test-authentication-token"; + + @Test + public void testConstructsAuthenticatedLatestAndExactRequests() + { + List requests = new ArrayList<>(); + TestingHttpClient httpClient = new TestingHttpClient(request -> { + requests.add(request); + return response(OK, "{}"); + }); + HogQlExchangeRateHttpTransport transport = new HogQlExchangeRateHttpTransport(BASE_URI, httpClient, () -> AUTHENTICATION_TOKEN); + + assertThat(transport.load(LoadRequest.latest()).toCompletableFuture().join()).isEqualTo("{}".getBytes(StandardCharsets.UTF_8)); + assertThat(transport.load(LoadRequest.pinned(42)).toCompletableFuture().join()).isEqualTo("{}".getBytes(StandardCharsets.UTF_8)); + + assertThat(requests).extracting(request -> request.getUri().toASCIIString()).containsExactly( + "https://duckgres.example/control-plane/v1/hogql/compatibility/exchange-rates?protocolVersion=1", + "https://duckgres.example/control-plane/v1/hogql/compatibility/exchange-rates?protocolVersion=1&generation=42"); + assertThat(requests).allSatisfy(request -> { + assertThat(request.getMethod()).isEqualTo("GET"); + assertThat(request.getHeader(ACCEPT)).isEqualTo(JSON_UTF_8.withoutParameters().toString()); + assertThat(request.getHeader("X-Duckgres-Internal-Secret")).isEqualTo(AUTHENTICATION_TOKEN); + }); + } + + @Test + public void testReadsAuthenticationTokenForEveryRequest() + { + List requests = new ArrayList<>(); + List tokens = new ArrayList<>(List.of("first-token", "second-token")); + TestingHttpClient httpClient = new TestingHttpClient(request -> { + requests.add(request); + return response(OK, "{}"); + }); + HogQlExchangeRateHttpTransport transport = new HogQlExchangeRateHttpTransport(BASE_URI, httpClient, tokens::removeFirst); + + transport.load(LoadRequest.latest()).toCompletableFuture().join(); + transport.load(LoadRequest.latest()).toCompletableFuture().join(); + + assertThat(requests).extracting(request -> request.getHeader("X-Duckgres-Internal-Secret")) + .containsExactly("first-token", "second-token"); + } + + @Test + public void testRejectsOversizedOrNonJsonResponses() + { + HogQlExchangeRateHttpTransport oversized = new HogQlExchangeRateHttpTransport( + BASE_URI, + new TestingHttpClient(_ -> response(OK, "12345")), + 4, + () -> AUTHENTICATION_TOKEN); + assertUnavailable(oversized, LoadRequest.latest()); + + TestingResponse plainText = new TestingResponse(OK, ImmutableListMultimap.of(), "plain text".getBytes(StandardCharsets.UTF_8)); + HogQlExchangeRateHttpTransport nonJson = new HogQlExchangeRateHttpTransport( + BASE_URI, + new TestingHttpClient(_ -> plainText), + () -> AUTHENTICATION_TOKEN); + assertUnavailable(nonJson, LoadRequest.latest()); + } + + @Test + public void testRejectsInvalidAuthenticationWithoutDisclosure() + { + HogQlExchangeRateHttpTransport transport = new HogQlExchangeRateHttpTransport( + BASE_URI, + new TestingHttpClient(_ -> response(OK, "{}")), + () -> "line-one\nline-two"); + + assertThatThrownBy(() -> transport.load(LoadRequest.latest()).toCompletableFuture().join()) + .cause() + .isInstanceOfSatisfying(HogQlExchangeRateException.class, exception -> { + assertThat(exception.failure()).isEqualTo(Failure.UNAVAILABLE); + assertThat(exception).hasMessageNotContaining("line-one"); + }); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("statusFailures") + public void testMapsHttpStatusWithoutResponseDisclosure(String name, HttpStatus status, LoadRequest request, Failure failure) + { + HogQlExchangeRateHttpTransport transport = new HogQlExchangeRateHttpTransport( + BASE_URI, + new TestingHttpClient(_ -> response(status, SECRET)), + () -> AUTHENTICATION_TOKEN); + + assertThatThrownBy(() -> transport.load(request).toCompletableFuture().join()) + .cause() + .isInstanceOfSatisfying(HogQlExchangeRateException.class, exception -> { + assertThat(exception.failure()).isEqualTo(failure); + assertThat(exception).hasMessageNotContaining(SECRET); + assertThat(exception.getMessage()).hasSizeLessThan(128); + }); + } + + private static Stream statusFailures() + { + return Stream.of( + Arguments.of("latest missing", NOT_FOUND, LoadRequest.latest(), Failure.UNAVAILABLE), + Arguments.of("exact missing", NOT_FOUND, LoadRequest.pinned(7), Failure.GENERATION_MISMATCH), + Arguments.of("generation conflict", CONFLICT, LoadRequest.pinned(7), Failure.GENERATION_MISMATCH), + Arguments.of("server failure", INTERNAL_SERVER_ERROR, LoadRequest.latest(), Failure.UNAVAILABLE)); + } + + private static void assertUnavailable(HogQlExchangeRateHttpTransport transport, LoadRequest request) + { + assertThatThrownBy(() -> transport.load(request).toCompletableFuture().join()) + .cause() + .isInstanceOfSatisfying(HogQlExchangeRateException.class, exception -> assertThat(exception.failure()).isEqualTo(Failure.UNAVAILABLE)); + } + + private static Response response(HttpStatus status, String body) + { + return new TestingResponse(status, contentType(JSON_UTF_8), body.getBytes(StandardCharsets.UTF_8)); + } +} From 494bade31a868d9215b16aaa5ab13d513d571e1c Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 02:52:35 -0400 Subject: [PATCH 128/136] feat(ducklake): add exchange rate conversion engine --- plugin/trino-ducklake/pom.xml | 10 + .../ExchangeRateConversionEngine.java | 104 ++++++++ .../currency/ExchangeRateSnapshot.java | 119 +++++++++ .../ExchangeRateSnapshotJsonDecoder.java | 234 ++++++++++++++++++ .../TestExchangeRateConversionEngine.java | 108 ++++++++ .../TestExchangeRateSnapshotJsonDecoder.java | 143 +++++++++++ 6 files changed, 718 insertions(+) create mode 100644 plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateConversionEngine.java create mode 100644 plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshot.java create mode 100644 plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshotJsonDecoder.java create mode 100644 plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateConversionEngine.java create mode 100644 plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateSnapshotJsonDecoder.java diff --git a/plugin/trino-ducklake/pom.xml b/plugin/trino-ducklake/pom.xml index bf4f9c2fc627..6217acfd0f20 100644 --- a/plugin/trino-ducklake/pom.xml +++ b/plugin/trino-ducklake/pom.xml @@ -19,6 +19,16 @@ + + com.fasterxml.jackson.core + jackson-core + + + + com.fasterxml.jackson.core + jackson-databind + + com.google.errorprone error_prone_annotations diff --git a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateConversionEngine.java b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateConversionEngine.java new file mode 100644 index 000000000000..b9395054f5d9 --- /dev/null +++ b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateConversionEngine.java @@ -0,0 +1,104 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ducklake.function.currency; + +import com.google.common.collect.ImmutableMap; +import io.trino.plugin.ducklake.function.currency.ExchangeRateSnapshot.ExchangeRate; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.time.LocalDate; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Optional; +import java.util.TreeMap; + +import static io.trino.plugin.ducklake.function.currency.ExchangeRateSnapshot.DECIMAL_SCALE; +import static java.util.Objects.requireNonNull; + +public final class ExchangeRateConversionEngine +{ + private static final BigDecimal ZERO = BigDecimal.ZERO.setScale(DECIMAL_SCALE); + + private final long generation; + private final Map> ratesByCurrency; + + public ExchangeRateConversionEngine(ExchangeRateSnapshot snapshot) + { + requireNonNull(snapshot, "snapshot is null"); + generation = snapshot.generation(); + + Map> mutableRates = new HashMap<>(); + for (ExchangeRate rate : snapshot.rates()) { + mutableRates.computeIfAbsent(rate.currency(), _ -> new TreeMap<>()) + .put(LocalDate.parse(rate.effectiveDate()), new BigDecimal(new BigInteger(rate.unscaledRate()), DECIMAL_SCALE)); + } + ImmutableMap.Builder> immutableRates = ImmutableMap.builder(); + mutableRates.forEach((currency, rates) -> immutableRates.put(currency, Collections.unmodifiableNavigableMap(rates))); + ratesByCurrency = immutableRates.buildOrThrow(); + } + + public long generation() + { + return generation; + } + + public Optional rate(long expectedGeneration, String currency, LocalDate date) + { + verifyGeneration(expectedGeneration); + requireNonNull(currency, "currency is null"); + requireNonNull(date, "date is null"); + NavigableMap currencyRates = ratesByCurrency.get(currency); + if (currencyRates == null) { + return Optional.empty(); + } + Map.Entry rate = currencyRates.floorEntry(date); + return rate == null ? Optional.empty() : Optional.of(rate.getValue()); + } + + public BigDecimal convert(long expectedGeneration, String sourceCurrency, String targetCurrency, BigDecimal amount, LocalDate date) + { + verifyGeneration(expectedGeneration); + requireNonNull(sourceCurrency, "sourceCurrency is null"); + requireNonNull(targetCurrency, "targetCurrency is null"); + requireNonNull(amount, "amount is null"); + requireNonNull(date, "date is null"); + + BigDecimal decimalAmount = amount.setScale(DECIMAL_SCALE, RoundingMode.DOWN); + if (sourceCurrency.equals(targetCurrency)) { + return decimalAmount; + } + + BigDecimal sourceRate = rate(expectedGeneration, sourceCurrency, date).orElse(ZERO); + BigDecimal targetRate = rate(expectedGeneration, targetCurrency, date).orElse(ZERO); + if (sourceRate.signum() == 0 || targetRate.signum() == 0) { + return ZERO; + } + + return decimalAmount + .divide(sourceRate, DECIMAL_SCALE, RoundingMode.DOWN) + .multiply(targetRate) + .setScale(DECIMAL_SCALE, RoundingMode.DOWN); + } + + private void verifyGeneration(long expectedGeneration) + { + if (expectedGeneration != generation) { + throw new IllegalArgumentException("exchange-rate snapshot generation mismatch"); + } + } +} diff --git a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshot.java b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshot.java new file mode 100644 index 000000000000..7979e742558f --- /dev/null +++ b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshot.java @@ -0,0 +1,119 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ducklake.function.currency; + +import java.time.LocalDate; +import java.time.format.DateTimeParseException; +import java.util.List; +import java.util.regex.Pattern; + +import static java.util.Objects.requireNonNull; + +public record ExchangeRateSnapshot( + int protocolVersion, + int schemaVersion, + long generation, + String baseCurrency, + int decimalScale, + List rates) +{ + public static final int PROTOCOL_VERSION = 1; + public static final int SCHEMA_VERSION = 1; + public static final String BASE_CURRENCY = "USD"; + public static final int DECIMAL_SCALE = 10; + public static final int MAXIMUM_RATES = 1_000_000; + + private static final Pattern CURRENCY_PATTERN = Pattern.compile("[A-Z]{3}"); + private static final Pattern UNSCALED_RATE_PATTERN = Pattern.compile("0|[1-9][0-9]{0,17}"); + private static final String UNSCALED_ONE = "10000000000"; + + public ExchangeRateSnapshot + { + if (protocolVersion != PROTOCOL_VERSION) { + throw new IllegalArgumentException("unsupported exchange-rate snapshot protocol"); + } + if (schemaVersion != SCHEMA_VERSION) { + throw new IllegalArgumentException("unsupported exchange-rate snapshot schema"); + } + if (generation <= 0) { + throw new IllegalArgumentException("exchange-rate snapshot generation must be positive"); + } + baseCurrency = requireNonNull(baseCurrency, "baseCurrency is null"); + if (!baseCurrency.equals(BASE_CURRENCY)) { + throw new IllegalArgumentException("unsupported exchange-rate base currency"); + } + if (decimalScale != DECIMAL_SCALE) { + throw new IllegalArgumentException("unsupported exchange-rate decimal scale"); + } + rates = List.copyOf(requireNonNull(rates, "rates is null")); + if (rates.isEmpty()) { + throw new IllegalArgumentException("exchange-rate snapshot has no rates"); + } + if (rates.size() > MAXIMUM_RATES) { + throw new IllegalArgumentException("exchange-rate snapshot exceeds rate limit"); + } + + boolean baseCurrencyPresent = false; + ExchangeRate previous = null; + for (ExchangeRate rate : rates) { + requireNonNull(rate, "rate is null"); + if (rate.currency().equals(BASE_CURRENCY)) { + baseCurrencyPresent = true; + if (!rate.unscaledRate().equals(UNSCALED_ONE)) { + throw new IllegalArgumentException("base-currency rate must equal one"); + } + } + if (previous != null && compare(previous, rate) >= 0) { + throw new IllegalArgumentException("exchange rates are not strictly sorted"); + } + previous = rate; + } + if (!baseCurrencyPresent) { + throw new IllegalArgumentException("exchange-rate snapshot has no base-currency rate"); + } + } + + private static int compare(ExchangeRate left, ExchangeRate right) + { + int currencyComparison = left.currency().compareTo(right.currency()); + if (currencyComparison != 0) { + return currencyComparison; + } + return left.effectiveDate().compareTo(right.effectiveDate()); + } + + public record ExchangeRate(String currency, String effectiveDate, String unscaledRate) + { + public ExchangeRate + { + currency = requireNonNull(currency, "currency is null"); + effectiveDate = requireNonNull(effectiveDate, "effectiveDate is null"); + unscaledRate = requireNonNull(unscaledRate, "unscaledRate is null"); + if (!CURRENCY_PATTERN.matcher(currency).matches()) { + throw new IllegalArgumentException("invalid exchange-rate currency"); + } + try { + if (!LocalDate.parse(effectiveDate).toString().equals(effectiveDate)) { + throw new IllegalArgumentException("noncanonical exchange-rate date"); + } + } + catch (DateTimeParseException e) { + throw new IllegalArgumentException("invalid exchange-rate date", e); + } + if (!UNSCALED_RATE_PATTERN.matcher(unscaledRate).matches()) { + throw new IllegalArgumentException("invalid exchange-rate value"); + } + } + } +} diff --git a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshotJsonDecoder.java b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshotJsonDecoder.java new file mode 100644 index 000000000000..4d738bfcfe36 --- /dev/null +++ b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshotJsonDecoder.java @@ -0,0 +1,234 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ducklake.function.currency; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.StreamReadFeature; +import com.fasterxml.jackson.core.exc.StreamConstraintsException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.trino.plugin.ducklake.function.currency.ExchangeRateSnapshot.ExchangeRate; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public final class ExchangeRateSnapshotJsonDecoder +{ + public static final int MAXIMUM_PAYLOAD_BYTES = 32 * 1024 * 1024; + + private static final int MAXIMUM_NESTING_DEPTH = 16; + private static final Set SNAPSHOT_FIELDS = Set.of("protocolVersion", "schemaVersion", "generation", "baseCurrency", "decimalScale", "rates"); + private static final Set RATE_FIELDS = Set.of("currency", "effectiveDate", "unscaledRate"); + + private final ObjectMapper objectMapper; + + public ExchangeRateSnapshotJsonDecoder() + { + JsonFactory jsonFactory = JsonFactory.builder() + .streamReadConstraints(StreamReadConstraints.builder() + .maxNestingDepth(MAXIMUM_NESTING_DEPTH) + .maxStringLength(64) + .maxNumberLength(19) + .build()) + .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) + .build(); + objectMapper = new ObjectMapper(jsonFactory); + } + + public ExchangeRateSnapshot decode(byte[] payload, long expectedGeneration) + { + if (payload == null) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + if (payload.length > MAXIMUM_PAYLOAD_BYTES) { + throw failure(DecodeFailure.LIMIT_EXCEEDED); + } + if (expectedGeneration <= 0) { + throw failure(DecodeFailure.GENERATION_MISMATCH); + } + + try { + ObjectNode root = parse(payload); + validateFields(root, SNAPSHOT_FIELDS); + + int protocolVersion = integer(root, "protocolVersion"); + if (protocolVersion != ExchangeRateSnapshot.PROTOCOL_VERSION) { + throw failure(DecodeFailure.UNSUPPORTED_PROTOCOL); + } + int schemaVersion = integer(root, "schemaVersion"); + if (schemaVersion != ExchangeRateSnapshot.SCHEMA_VERSION) { + throw failure(DecodeFailure.UNSUPPORTED_SCHEMA); + } + + long generation = generation(root); + if (generation != expectedGeneration) { + throw failure(DecodeFailure.GENERATION_MISMATCH); + } + + return new ExchangeRateSnapshot( + protocolVersion, + schemaVersion, + generation, + text(root, "baseCurrency"), + integer(root, "decimalScale"), + rates(required(root, "rates"))); + } + catch (DecodeException e) { + throw e; + } + catch (StreamConstraintsException e) { + throw failure(DecodeFailure.LIMIT_EXCEEDED); + } + catch (IOException | RuntimeException e) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + } + + private ObjectNode parse(byte[] payload) + throws IOException + { + try (JsonParser parser = objectMapper.createParser(payload)) { + JsonNode root = objectMapper.readTree(parser); + if (parser.nextToken() != null) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return object(root); + } + } + + private static List rates(JsonNode node) + { + ArrayNode array = array(node); + if (array.size() > ExchangeRateSnapshot.MAXIMUM_RATES) { + throw failure(DecodeFailure.LIMIT_EXCEEDED); + } + List rates = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode rate = object(element); + validateFields(rate, RATE_FIELDS); + rates.add(new ExchangeRate( + text(rate, "currency"), + text(rate, "effectiveDate"), + text(rate, "unscaledRate"))); + } + return List.copyOf(rates); + } + + private static int integer(ObjectNode object, String field) + { + JsonNode node = required(object, field); + if (!node.isIntegralNumber() || !node.canConvertToInt()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return node.intValue(); + } + + private static long generation(ObjectNode object) + { + JsonNode node = required(object, "generation"); + if (!node.isIntegralNumber() || !node.canConvertToLong() || node.longValue() <= 0) { + throw failure(DecodeFailure.GENERATION_MISMATCH); + } + return node.longValue(); + } + + private static String text(ObjectNode object, String field) + { + JsonNode node = required(object, field); + if (!node.isTextual() || node.textValue().isEmpty()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return node.textValue(); + } + + private static JsonNode required(ObjectNode object, String field) + { + JsonNode node = object.get(field); + if (node == null || node.isNull()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return node; + } + + private static ObjectNode object(JsonNode node) + { + if (!(node instanceof ObjectNode object)) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return object; + } + + private static ArrayNode array(JsonNode node) + { + if (!(node instanceof ArrayNode array)) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return array; + } + + private static void validateFields(ObjectNode object, Set fields) + { + Iterator names = object.fieldNames(); + int count = 0; + while (names.hasNext()) { + if (!fields.contains(names.next())) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + count++; + } + if (count != fields.size()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + } + + private static DecodeException failure(DecodeFailure failure) + { + return new DecodeException(failure); + } + + public enum DecodeFailure + { + INVALID_PAYLOAD, + LIMIT_EXCEEDED, + UNSUPPORTED_PROTOCOL, + UNSUPPORTED_SCHEMA, + GENERATION_MISMATCH, + } + + public static final class DecodeException + extends IllegalArgumentException + { + private final DecodeFailure failure; + + private DecodeException(DecodeFailure failure) + { + super(requireNonNull(failure, "failure is null").name()); + this.failure = failure; + } + + public DecodeFailure failure() + { + return failure; + } + } +} diff --git a/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateConversionEngine.java b/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateConversionEngine.java new file mode 100644 index 000000000000..f2ee0bed4084 --- /dev/null +++ b/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateConversionEngine.java @@ -0,0 +1,108 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ducklake.function.currency; + +import io.trino.plugin.ducklake.function.currency.ExchangeRateSnapshot.ExchangeRate; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +final class TestExchangeRateConversionEngine +{ + private static final long GENERATION = 17; + private static final LocalDate FIRST_DATE = LocalDate.parse("2024-01-01"); + private static final LocalDate SECOND_DATE = LocalDate.parse("2024-01-10"); + + private final ExchangeRateConversionEngine engine = new ExchangeRateConversionEngine(new ExchangeRateSnapshot( + 1, + 1, + GENERATION, + "USD", + 10, + List.of( + rate("EUR", FIRST_DATE, "0.9049"), + rate("EUR", SECOND_DATE, "0.95"), + rate("JPY", FIRST_DATE, "108.1234"), + rate("USD", FIRST_DATE, "1"), + rate("ZZZ", FIRST_DATE, "0")))); + + @Test + void testFloorLookupUsesLatestRateNotAfterDate() + { + assertThat(engine.rate(GENERATION, "EUR", FIRST_DATE.minusDays(1))).isEmpty(); + assertThat(engine.rate(GENERATION, "EUR", FIRST_DATE)).contains(new BigDecimal("0.9049000000")); + assertThat(engine.rate(GENERATION, "EUR", SECOND_DATE.minusDays(1))).contains(new BigDecimal("0.9049000000")); + assertThat(engine.rate(GENERATION, "EUR", SECOND_DATE.plusYears(1))).contains(new BigDecimal("0.9500000000")); + } + + @Test + void testConversionMatchesPostHogDecimalSemantics() + { + assertDecimal(engine.convert(GENERATION, "USD", "EUR", new BigDecimal("100"), FIRST_DATE), "90.4900000000"); + assertDecimal(engine.convert(GENERATION, "EUR", "USD", new BigDecimal("90.49"), FIRST_DATE), "100.0000000000"); + assertDecimal(engine.convert(GENERATION, "AAA", "AAA", new BigDecimal("1.23456789019"), FIRST_DATE), "1.2345678901"); + assertDecimal(engine.convert(GENERATION, "USD", "JPY", new BigDecimal("-2.5"), FIRST_DATE), "-270.3085000000"); + } + + @Test + void testDivisionRoundsToScaleBeforeMultiplication() + { + ExchangeRateConversionEngine roundingEngine = new ExchangeRateConversionEngine(new ExchangeRateSnapshot( + 1, + 1, + GENERATION, + "USD", + 10, + List.of(rate("AAA", FIRST_DATE, "3"), rate("BBB", FIRST_DATE, "2"), rate("USD", FIRST_DATE, "1")))); + + assertDecimal(roundingEngine.convert(GENERATION, "AAA", "BBB", BigDecimal.ONE, FIRST_DATE), "0.6666666666"); + } + + @Test + void testMissingOrZeroRateReturnsZero() + { + assertDecimal(engine.convert(GENERATION, "MISSING", "EUR", BigDecimal.TEN, FIRST_DATE), "0.0000000000"); + assertDecimal(engine.convert(GENERATION, "USD", "MISSING", BigDecimal.TEN, FIRST_DATE), "0.0000000000"); + assertDecimal(engine.convert(GENERATION, "ZZZ", "EUR", BigDecimal.TEN, FIRST_DATE), "0.0000000000"); + assertDecimal(engine.convert(GENERATION, "USD", "ZZZ", BigDecimal.TEN, FIRST_DATE), "0.0000000000"); + assertDecimal(engine.convert(GENERATION, "EUR", "USD", BigDecimal.TEN, FIRST_DATE.minusDays(1)), "0.0000000000"); + } + + @Test + void testRejectsAnotherGeneration() + { + assertThatThrownBy(() -> engine.convert(GENERATION + 1, "USD", "EUR", BigDecimal.TEN, FIRST_DATE)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("exchange-rate snapshot generation mismatch"); + assertThatThrownBy(() -> engine.rate(GENERATION + 1, "EUR", FIRST_DATE)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("exchange-rate snapshot generation mismatch"); + } + + private static ExchangeRate rate(String currency, LocalDate date, String value) + { + return new ExchangeRate(currency, date.toString(), new BigDecimal(value).movePointRight(10).toBigIntegerExact().toString()); + } + + private static void assertDecimal(BigDecimal actual, String expected) + { + assertThat(actual.scale()).isEqualTo(10); + assertThat(actual).isEqualByComparingTo(expected); + } +} diff --git a/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateSnapshotJsonDecoder.java b/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateSnapshotJsonDecoder.java new file mode 100644 index 000000000000..33dd3ebd33eb --- /dev/null +++ b/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateSnapshotJsonDecoder.java @@ -0,0 +1,143 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ducklake.function.currency; + +import io.trino.plugin.ducklake.function.currency.ExchangeRateSnapshot.ExchangeRate; +import io.trino.plugin.ducklake.function.currency.ExchangeRateSnapshotJsonDecoder.DecodeFailure; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +final class TestExchangeRateSnapshotJsonDecoder +{ + @Test + void testDecodesExactGenerationSnapshot() + { + ExchangeRateSnapshot snapshot = new ExchangeRateSnapshotJsonDecoder().decode(bytes(validSnapshot()), 17); + + assertThat(snapshot.protocolVersion()).isEqualTo(1); + assertThat(snapshot.schemaVersion()).isEqualTo(1); + assertThat(snapshot.generation()).isEqualTo(17); + assertThat(snapshot.baseCurrency()).isEqualTo("USD"); + assertThat(snapshot.decimalScale()).isEqualTo(10); + assertThat(snapshot.rates()).containsExactly( + new ExchangeRate("EUR", "2024-01-01", "9049000000"), + new ExchangeRate("EUR", "2024-01-10", "9500000000"), + new ExchangeRate("USD", "2024-01-01", "10000000000")); + } + + @Test + void testSnapshotDefensivelyCopiesRates() + { + List rates = new ArrayList<>(); + rates.add(new ExchangeRate("USD", "2024-01-01", "10000000000")); + ExchangeRateSnapshot snapshot = new ExchangeRateSnapshot(1, 1, 17, "USD", 10, rates); + + rates.clear(); + + assertThat(snapshot.rates()).hasSize(1); + assertThatThrownBy(() -> snapshot.rates().clear()).isInstanceOf(UnsupportedOperationException.class); + } + + @Test + void testRejectsIncompatibleSnapshots() + { + compatibilityFailures().forEach(testCase -> assertDecodeFailure( + testCase.name(), + testCase.payload(), + testCase.expectedGeneration(), + testCase.failure())); + } + + private static List compatibilityFailures() + { + return List.of( + new DecodeFailureCase("protocol", validSnapshot().replace("\"protocolVersion\": 1", "\"protocolVersion\": 2"), 17, DecodeFailure.UNSUPPORTED_PROTOCOL), + new DecodeFailureCase("schema", validSnapshot().replace("\"schemaVersion\": 1", "\"schemaVersion\": 2"), 17, DecodeFailure.UNSUPPORTED_SCHEMA), + new DecodeFailureCase("generation", validSnapshot(), 18, DecodeFailure.GENERATION_MISMATCH), + new DecodeFailureCase("nonpositive expected generation", validSnapshot(), 0, DecodeFailure.GENERATION_MISMATCH), + new DecodeFailureCase("nonpositive snapshot generation", validSnapshot().replace("\"generation\": 17", "\"generation\": 0"), 17, DecodeFailure.GENERATION_MISMATCH)); + } + + @Test + void testRejectsMalformedOrExtendedPayloads() + { + strictFailures().forEach(testCase -> assertDecodeFailure( + testCase.name(), + testCase.payload(), + testCase.expectedGeneration(), + testCase.failure())); + } + + private static List strictFailures() + { + return List.of( + invalid("unknown root field", validSnapshot().replace("\"rates\":", "\"unknown\": true, \"rates\":")), + invalid("missing root field", validSnapshot().replace("\"schemaVersion\": 1,", "")), + invalid("duplicate root field", validSnapshot().replace("\"schemaVersion\": 1,", "\"schemaVersion\": 1, \"schemaVersion\": 1,")), + invalid("unsupported base", validSnapshot().replace("\"baseCurrency\": \"USD\"", "\"baseCurrency\": \"EUR\"")), + invalid("unsupported scale", validSnapshot().replace("\"decimalScale\": 10", "\"decimalScale\": 9")), + invalid("numeric unscaled rate", validSnapshot().replace("\"9049000000\"", "9049000000")), + invalid("negative unscaled rate", validSnapshot().replace("\"9049000000\"", "\"-9049000000\"")), + invalid("leading zero unscaled rate", validSnapshot().replace("\"9049000000\"", "\"09049000000\"")), + invalid("oversized unscaled rate", validSnapshot().replace("\"9049000000\"", "\"1234567890123456789\"")), + invalid("invalid currency", validSnapshot().replace("\"EUR\"", "\"eur\"")), + invalid("invalid date", validSnapshot().replace("2024-01-10", "2024-02-30")), + invalid("duplicate currency date", validSnapshot().replace("2024-01-10", "2024-01-01")), + invalid("unknown rate field", validSnapshot().replace("\"currency\": \"USD\"", "\"currency\": \"USD\", \"unknown\": true")), + invalid("trailing document", validSnapshot() + " {}")); + } + + private static DecodeFailureCase invalid(String name, String payload) + { + return new DecodeFailureCase(name, payload, 17, DecodeFailure.INVALID_PAYLOAD); + } + + private static void assertDecodeFailure(String name, String payload, long expectedGeneration, DecodeFailure failure) + { + assertThatThrownBy(() -> new ExchangeRateSnapshotJsonDecoder().decode(bytes(payload), expectedGeneration)) + .as(name) + .isInstanceOfSatisfying(ExchangeRateSnapshotJsonDecoder.DecodeException.class, exception -> assertThat(exception.failure()).isEqualTo(failure)); + } + + private static byte[] bytes(String value) + { + return value.getBytes(StandardCharsets.UTF_8); + } + + private static String validSnapshot() + { + return """ + { + "protocolVersion": 1, + "schemaVersion": 1, + "generation": 17, + "baseCurrency": "USD", + "decimalScale": 10, + "rates": [ + {"currency": "EUR", "effectiveDate": "2024-01-01", "unscaledRate": "9049000000"}, + {"currency": "EUR", "effectiveDate": "2024-01-10", "unscaledRate": "9500000000"}, + {"currency": "USD", "effectiveDate": "2024-01-01", "unscaledRate": "10000000000"} + ] + } + """; + } + + private record DecodeFailureCase(String name, String payload, long expectedGeneration, DecodeFailure failure) {} +} From 4ae035220e070bf74f8e9367317d31fa52c3d0e5 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 02:55:22 -0400 Subject: [PATCH 129/136] fix(hogql): allow unaliased derived table columns --- .../io/trino/hogql/parser/HogQlParser.java | 100 ++++++++++-------- .../trino/hogql/parser/TestHogQlParser.java | 11 ++ 2 files changed, 64 insertions(+), 47 deletions(-) diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 8f354f5fd044..7e8e99ed6e1f 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -2081,47 +2081,48 @@ private void validateQueryScope(HogQlQuery query, Set forbiddenOuterRela } Set localRelations = relationNames(query.from()); + boolean hasLocalRelation = query.from().isPresent(); query.projections().forEach(projection -> { if (projection instanceof ExpressionProjection expression) { - validateExpressionScope(expression.expression(), forbiddenOuterRelations, localRelations); + validateExpressionScope(expression.expression(), forbiddenOuterRelations, localRelations, hasLocalRelation); } }); - query.where().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); - query.groupBy().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); - query.having().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); - query.orderBy().forEach(item -> validateExpressionScope(item.expression(), forbiddenOuterRelations, localRelations)); - query.limit().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); - query.offset().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); - query.from().ifPresent(relation -> validateRelationExpressionScopes(relation, forbiddenOuterRelations, localRelations)); + query.where().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); + query.groupBy().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); + query.having().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); + query.orderBy().forEach(item -> validateExpressionScope(item.expression(), forbiddenOuterRelations, localRelations, hasLocalRelation)); + query.limit().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); + query.offset().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); + query.from().ifPresent(relation -> validateRelationExpressionScopes(relation, forbiddenOuterRelations, localRelations, hasLocalRelation)); Set nestedForbiddenRelations = new LinkedHashSet<>(forbiddenOuterRelations); nestedForbiddenRelations.addAll(localRelations); query.from().ifPresent(relation -> validateNestedRelationScopes(relation, Set.copyOf(nestedForbiddenRelations))); } - private void validateRelationExpressionScopes(Relation relation, Set forbiddenOuterRelations, Set localRelations) + private void validateRelationExpressionScopes(Relation relation, Set forbiddenOuterRelations, Set localRelations, boolean hasLocalRelation) { switch (relation) { - case AliasedRelation alias -> validateRelationExpressionScopes(alias.relation(), forbiddenOuterRelations, localRelations); + case AliasedRelation alias -> validateRelationExpressionScopes(alias.relation(), forbiddenOuterRelations, localRelations, hasLocalRelation); case JoinRelation join -> { - validateRelationExpressionScopes(join.left(), forbiddenOuterRelations, localRelations); - validateRelationExpressionScopes(join.right(), forbiddenOuterRelations, localRelations); + validateRelationExpressionScopes(join.left(), forbiddenOuterRelations, localRelations, hasLocalRelation); + validateRelationExpressionScopes(join.right(), forbiddenOuterRelations, localRelations, hasLocalRelation); join.criteria().ifPresent(criteria -> { if (criteria instanceof JoinOn on) { - validateExpressionScope(on.expression(), forbiddenOuterRelations, localRelations); + validateExpressionScope(on.expression(), forbiddenOuterRelations, localRelations, hasLocalRelation); } }); } case PivotRelation pivot -> { - validateRelationExpressionScopes(pivot.input(), forbiddenOuterRelations, localRelations); + validateRelationExpressionScopes(pivot.input(), forbiddenOuterRelations, localRelations, hasLocalRelation); pivot.aggregations().forEach(aggregation -> - validateExpressionScope(aggregation.expression(), forbiddenOuterRelations, localRelations)); - pivot.pivotColumns().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); + validateExpressionScope(aggregation.expression(), forbiddenOuterRelations, localRelations, hasLocalRelation)); + pivot.pivotColumns().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); pivot.valueGroups().forEach(group -> group.values().forEach(expression -> - validateExpressionScope(expression, forbiddenOuterRelations, localRelations))); - pivot.groupBy().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); + validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation))); + pivot.groupBy().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); } - case UnnestRelation unnest -> unnest.expressions().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations)); + case UnnestRelation unnest -> unnest.expressions().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); case CommonTableReference _, SubqueryRelation _, TablePlaceholder _, TableReference _, ValuesRelation _ -> {} } } @@ -2166,29 +2167,34 @@ private void collectRelationNames(Relation relation, Set names) } private void validateExpressionScope(Expression expression, Set forbiddenOuterRelations, Set localRelations) + { + validateExpressionScope(expression, forbiddenOuterRelations, localRelations, !localRelations.isEmpty()); + } + + private void validateExpressionScope(Expression expression, Set forbiddenOuterRelations, Set localRelations, boolean hasLocalRelation) { switch (expression) { - case ArrayExpression array -> array.values().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); + case ArrayExpression array -> array.values().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations, hasLocalRelation)); case BetweenExpression between -> { - validateExpressionScope(between.value(), forbiddenOuterRelations, localRelations); - validateExpressionScope(between.min(), forbiddenOuterRelations, localRelations); - validateExpressionScope(between.max(), forbiddenOuterRelations, localRelations); + validateExpressionScope(between.value(), forbiddenOuterRelations, localRelations, hasLocalRelation); + validateExpressionScope(between.min(), forbiddenOuterRelations, localRelations, hasLocalRelation); + validateExpressionScope(between.max(), forbiddenOuterRelations, localRelations, hasLocalRelation); } case BinaryExpression binary -> { - validateExpressionScope(binary.left(), forbiddenOuterRelations, localRelations); - validateExpressionScope(binary.right(), forbiddenOuterRelations, localRelations); + validateExpressionScope(binary.left(), forbiddenOuterRelations, localRelations, hasLocalRelation); + validateExpressionScope(binary.right(), forbiddenOuterRelations, localRelations, hasLocalRelation); } case CaseExpression caseExpression -> { - caseExpression.operand().ifPresent(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); + caseExpression.operand().ifPresent(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations, hasLocalRelation)); caseExpression.whenClauses().forEach(when -> { - validateExpressionScope(when.operand(), forbiddenOuterRelations, localRelations); - validateExpressionScope(when.result(), forbiddenOuterRelations, localRelations); + validateExpressionScope(when.operand(), forbiddenOuterRelations, localRelations, hasLocalRelation); + validateExpressionScope(when.result(), forbiddenOuterRelations, localRelations, hasLocalRelation); }); - caseExpression.defaultValue().ifPresent(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); + caseExpression.defaultValue().ifPresent(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations, hasLocalRelation)); } - case CastExpression cast -> validateExpressionScope(cast.value(), forbiddenOuterRelations, localRelations); + case CastExpression cast -> validateExpressionScope(cast.value(), forbiddenOuterRelations, localRelations, hasLocalRelation); case ColumnReference reference -> { - if (reference.parts().size() == 1 && !forbiddenOuterRelations.isEmpty() && localRelations.isEmpty()) { + if (reference.parts().size() == 1 && !forbiddenOuterRelations.isEmpty() && !hasLocalRelation) { throw unsupported(reference.span(), "correlated relation subquery"); } if (reference.parts().size() > 1) { @@ -2199,34 +2205,34 @@ private void validateExpressionScope(Expression expression, Set forbidde } } case FunctionCall function -> { - function.arguments().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); - function.orderBy().forEach(item -> validateExpressionScope(item.expression(), forbiddenOuterRelations, localRelations)); - function.filter().ifPresent(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); + function.arguments().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations, hasLocalRelation)); + function.orderBy().forEach(item -> validateExpressionScope(item.expression(), forbiddenOuterRelations, localRelations, hasLocalRelation)); + function.filter().ifPresent(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations, hasLocalRelation)); } case InExpression in -> { - validateExpressionScope(in.value(), forbiddenOuterRelations, localRelations); - in.values().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); + validateExpressionScope(in.value(), forbiddenOuterRelations, localRelations, hasLocalRelation); + in.values().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations, hasLocalRelation)); } case InCohortExpression in -> { - validateExpressionScope(in.value(), forbiddenOuterRelations, localRelations); - validateExpressionScope(in.cohort(), forbiddenOuterRelations, localRelations); + validateExpressionScope(in.value(), forbiddenOuterRelations, localRelations, hasLocalRelation); + validateExpressionScope(in.cohort(), forbiddenOuterRelations, localRelations, hasLocalRelation); } case InSubqueryExpression in -> { - validateExpressionScope(in.value(), forbiddenOuterRelations, localRelations); + validateExpressionScope(in.value(), forbiddenOuterRelations, localRelations, hasLocalRelation); validateQueryScope(in.query(), Set.of()); } - case IntervalExpression interval -> validateExpressionScope(interval.value(), forbiddenOuterRelations, localRelations); - case IsNullExpression isNull -> validateExpressionScope(isNull.value(), forbiddenOuterRelations, localRelations); - case LambdaExpression lambda -> validateExpressionScope(lambda.body(), forbiddenOuterRelations, localRelations); + case IntervalExpression interval -> validateExpressionScope(interval.value(), forbiddenOuterRelations, localRelations, hasLocalRelation); + case IsNullExpression isNull -> validateExpressionScope(isNull.value(), forbiddenOuterRelations, localRelations, hasLocalRelation); + case LambdaExpression lambda -> validateExpressionScope(lambda.body(), forbiddenOuterRelations, localRelations, hasLocalRelation); case Literal _, Placeholder _ -> {} - case MemberAccessExpression memberAccess -> validateExpressionScope(memberAccess.base(), forbiddenOuterRelations, localRelations); + case MemberAccessExpression memberAccess -> validateExpressionScope(memberAccess.base(), forbiddenOuterRelations, localRelations, hasLocalRelation); case ScalarSubqueryExpression subquery -> validateQueryScope(subquery.query(), Set.of()); case SubscriptExpression subscript -> { - validateExpressionScope(subscript.base(), forbiddenOuterRelations, localRelations); - validateExpressionScope(subscript.index(), forbiddenOuterRelations, localRelations); + validateExpressionScope(subscript.base(), forbiddenOuterRelations, localRelations, hasLocalRelation); + validateExpressionScope(subscript.index(), forbiddenOuterRelations, localRelations, hasLocalRelation); } - case TupleExpression tuple -> tuple.values().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations)); - case UnaryExpression unary -> validateExpressionScope(unary.operand(), forbiddenOuterRelations, localRelations); + case TupleExpression tuple -> tuple.values().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations, hasLocalRelation)); + case UnaryExpression unary -> validateExpressionScope(unary.operand(), forbiddenOuterRelations, localRelations, hasLocalRelation); } } diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java index 3833846b01a6..f96e144d3951 100644 --- a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -447,6 +447,17 @@ public void testBuildsCteAndDerivedTableAstWithSourceSpans() assertThat(subquery.span()).isEqualTo(new HogQlQuery.SourceSpan(80, 120, 2, 18, 2, 58)); } + @Test + public void testAllowsUnqualifiedColumnsFromUnaliasedNestedDerivedTables() + { + HogQlQuery query = parser.parseStatement( + "SELECT left_source.id FROM first_table left_source " + + "JOIN (SELECT id FROM (SELECT id FROM second_table)) right_source " + + "ON left_source.id = right_source.id"); + + assertThat(query.from()).get().isInstanceOf(JoinRelation.class); + } + @Test public void testBuildsCorrelatedInSubqueryWithSourceSpans() { From b4a9dabcea39627ed19c16fbac931e9421ce65f9 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 03:33:48 -0400 Subject: [PATCH 130/136] feat(hogql): lower currency conversions --- .../compiler/HogQlCompilationResult.java | 11 ++- .../trino/hogql/compiler/HogQlCompiler.java | 55 +++++++++--- .../hogql/compiler/HogQlFunctionResolver.java | 84 +++++++++++++++++-- .../compiler/HogQlV0FunctionRegistry.java | 3 + .../catalog/HogQlExchangeRateException.java | 6 ++ .../catalog/HogQlSemanticCatalogSnapshot.java | 6 +- .../compiler/TestHogQlCurrencyCompiler.java | 81 ++++++++++++++++++ .../compiler/TestHogQlFunctionResolver.java | 4 +- 8 files changed, 225 insertions(+), 25 deletions(-) create mode 100644 core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCurrencyCompiler.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java index 9aa9d2c5f5ca..487f17299d58 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java @@ -24,7 +24,8 @@ public record HogQlCompilationResult( Statement statement, List parameterNames, List modifierBindings, - OptionalLong catalogGeneration) + OptionalLong catalogGeneration, + OptionalLong exchangeRateGeneration) { public HogQlCompilationResult { @@ -32,10 +33,16 @@ public record HogQlCompilationResult( parameterNames = List.copyOf(requireNonNull(parameterNames, "parameterNames is null")); modifierBindings = List.copyOf(requireNonNull(modifierBindings, "modifierBindings is null")); catalogGeneration = requireNonNull(catalogGeneration, "catalogGeneration is null"); + exchangeRateGeneration = requireNonNull(exchangeRateGeneration, "exchangeRateGeneration is null"); } public HogQlCompilationResult(Statement statement, List parameterNames, OptionalLong catalogGeneration) { - this(statement, parameterNames, List.of(), catalogGeneration); + this(statement, parameterNames, List.of(), catalogGeneration, OptionalLong.empty()); + } + + public HogQlCompilationResult(Statement statement, List parameterNames, List modifierBindings, OptionalLong catalogGeneration) + { + this(statement, parameterNames, modifierBindings, catalogGeneration, OptionalLong.empty()); } } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index c5ae8c3c226d..d825baa7c5bb 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -13,6 +13,7 @@ */ package io.trino.hogql.compiler; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotProvider; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException.Failure; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinRequest; @@ -85,15 +86,27 @@ public final class HogQlCompiler { private final HogQlParser parser; + private final Optional exchangeRateSnapshotProvider; public HogQlCompiler() { - this(new HogQlParser()); + this(new HogQlParser(), Optional.empty()); } HogQlCompiler(HogQlParser parser) + { + this(parser, Optional.empty()); + } + + public HogQlCompiler(HogQlExchangeRateSnapshotProvider exchangeRateSnapshotProvider) + { + this(new HogQlParser(), Optional.of(requireNonNull(exchangeRateSnapshotProvider, "exchangeRateSnapshotProvider is null"))); + } + + private HogQlCompiler(HogQlParser parser, Optional exchangeRateSnapshotProvider) { this.parser = requireNonNull(parser, "parser is null"); + this.exchangeRateSnapshotProvider = requireNonNull(exchangeRateSnapshotProvider, "exchangeRateSnapshotProvider is null"); } public Statement compile(String hogql) @@ -117,7 +130,8 @@ public HogQlCompilationResult compile(HogQlCompileEnvelope envelope, Optional catalogContext) @@ -134,7 +148,8 @@ public HogQlCompilationResult compileV0(HogQlCompileEnvelope envelope, Optional< catalogContext, envelope.languageVersion(), envelope.catalogGeneration(), - true); + true, + exchangeRateSnapshotProvider); } public HogQlCompilationResult compile(String hogql, Map parameters) @@ -156,7 +171,8 @@ public HogQlCompilationResult compile(String hogql, Map Optional.empty(), HogQlLanguageContract.current().languageVersion(), OptionalLong.empty(), - false); + false, + exchangeRateSnapshotProvider); } private HogQlQuery parse(String hogql) @@ -181,7 +197,8 @@ private static HogQlCompilationResult compile( Optional catalogContext, HogQlLanguageVersion languageVersion, OptionalLong expectedCatalogGeneration, - boolean v0Profile) + boolean v0Profile, + Optional exchangeRateSnapshotProvider) { validateParameters(query, envelope.parameters()); validateQuery(query); @@ -223,7 +240,7 @@ private static HogQlCompilationResult compile( if (v0Profile) { HogQlV0ProfileValidator.validate(query, Optional.empty()); } - ResolvedQuery resolved = resolveQuery(query, catalogContext, languageVersion, expectedCatalogGeneration, !modifiers.isEmpty(), v0Profile); + ResolvedQuery resolved = resolveQuery(query, catalogContext, languageVersion, expectedCatalogGeneration, !modifiers.isEmpty(), v0Profile, exchangeRateSnapshotProvider); Statement statement = TrinoAstFactory.createStatement(resolved.query(), parameterIds); List modifierBindings = resolved.pinnedSnapshot() .map(snapshot -> HogQlModifierResolver.resolve(snapshot, modifiers, query.span())) @@ -234,7 +251,8 @@ private static HogQlCompilationResult compile( .map(Placeholder::name) .toList(), modifierBindings, - resolved.catalogGeneration()); + resolved.catalogGeneration(), + resolved.exchangeRateGeneration()); } private static void collectPlaceholders(HogQlQuery query, List placeholders) @@ -277,17 +295,22 @@ private static ResolvedQuery resolveQuery( HogQlLanguageVersion languageVersion, OptionalLong expectedCatalogGeneration, boolean modifiersRequireSnapshot, - boolean v0Profile) + boolean v0Profile, + Optional exchangeRateSnapshotProvider) { boolean semanticCandidate = containsSemanticCandidate(query); if (!semanticCandidate && !modifiersRequireSnapshot) { - return new ResolvedQuery(query, Optional.empty()); + return new ResolvedQuery(query, Optional.empty(), OptionalLong.empty()); } if (catalogContext.isEmpty()) { if (modifiersRequireSnapshot) { throw new HogQlSemanticCatalogException(Failure.UNAVAILABLE, "HogQL semantic catalog snapshot is required for modifiers"); } - return new ResolvedQuery(containsFunctionCall(query) ? HogQlFunctionResolver.resolve(query) : query, Optional.empty()); + if (!containsFunctionCall(query)) { + return new ResolvedQuery(query, Optional.empty(), OptionalLong.empty()); + } + HogQlFunctionResolver.Resolution resolution = HogQlFunctionResolver.resolve(query, exchangeRateSnapshotProvider); + return new ResolvedQuery(resolution.query(), Optional.empty(), resolution.exchangeRateGeneration()); } HogQlSemanticCatalogContext context = catalogContext.orElseThrow(); PinnedSnapshot pinned = context.snapshotProvider().pin(new PinRequest( @@ -298,8 +321,13 @@ private static ResolvedQuery resolveQuery( HogQlV0ProfileValidator.validate(query, Optional.of(pinned.snapshot())); } HogQlQuery resolved = query; + OptionalLong exchangeRateGeneration = OptionalLong.empty(); if (semanticCandidate) { - HogQlQuery functionsResolved = v0Profile ? HogQlFunctionResolver.resolveV0(query) : HogQlFunctionResolver.resolve(pinned, query); + HogQlFunctionResolver.Resolution functionResolution = v0Profile + ? HogQlFunctionResolver.resolveV0(query, exchangeRateSnapshotProvider) + : HogQlFunctionResolver.resolve(pinned, query, exchangeRateSnapshotProvider); + exchangeRateGeneration = functionResolution.exchangeRateGeneration(); + HogQlQuery functionsResolved = functionResolution.query(); if (hasSemanticDefinitions(pinned.snapshot())) { resolved = HogQlSemanticResolver.resolve(pinned, functionsResolved) .map(HogQlSemanticResolver.ResolvedQuery::query) @@ -309,7 +337,7 @@ private static ResolvedQuery resolveQuery( resolved = functionsResolved; } } - return new ResolvedQuery(resolved, Optional.of(pinned)); + return new ResolvedQuery(resolved, Optional.of(pinned), exchangeRateGeneration); } private static boolean hasSemanticDefinitions(io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot snapshot) @@ -450,12 +478,13 @@ private static boolean containsFunctionCall(Optional expression) return expression.map(HogQlCompiler::containsFunctionCall).orElse(false); } - private record ResolvedQuery(HogQlQuery query, Optional pinnedSnapshot) + private record ResolvedQuery(HogQlQuery query, Optional pinnedSnapshot, OptionalLong exchangeRateGeneration) { private ResolvedQuery { query = requireNonNull(query, "query is null"); pinnedSnapshot = requireNonNull(pinnedSnapshot, "pinnedSnapshot is null"); + exchangeRateGeneration = requireNonNull(exchangeRateGeneration, "exchangeRateGeneration is null"); } public OptionalLong catalogGeneration() diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 9ac8bccd0363..b8f496434737 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -13,6 +13,7 @@ */ package io.trino.hogql.compiler; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotProvider; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; @@ -83,6 +84,7 @@ import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; @@ -94,9 +96,13 @@ final class HogQlFunctionResolver private final Map functions; private final boolean semanticCatalogAvailable; + private final Optional exchangeRateSnapshotProvider; + private OptionalLong exchangeRateGeneration = OptionalLong.empty(); private ArrayList activeArrayJoins; - private HogQlFunctionResolver(Optional snapshot) + private HogQlFunctionResolver( + Optional snapshot, + Optional exchangeRateSnapshotProvider) { Map functions = new LinkedHashMap<>(HogQlV0FunctionRegistry.functions()); snapshot.stream() @@ -104,25 +110,54 @@ private HogQlFunctionResolver(Optional snapshot) .forEach(function -> functions.putIfAbsent(canonical(function.name()), function)); this.functions = Map.copyOf(functions); semanticCatalogAvailable = snapshot.isPresent(); + this.exchangeRateSnapshotProvider = requireNonNull(exchangeRateSnapshotProvider, "exchangeRateSnapshotProvider is null"); } public static HogQlQuery resolve(PinnedSnapshot snapshot, HogQlQuery query) { requireNonNull(snapshot, "snapshot is null"); requireNonNull(query, "query is null"); - return new HogQlFunctionResolver(Optional.of(snapshot)).resolveQuery(query); + return resolve(snapshot, query, Optional.empty()).query(); + } + + static Resolution resolve( + PinnedSnapshot snapshot, + HogQlQuery query, + Optional exchangeRateSnapshotProvider) + { + requireNonNull(snapshot, "snapshot is null"); + requireNonNull(query, "query is null"); + return new HogQlFunctionResolver(Optional.of(snapshot), exchangeRateSnapshotProvider).resolveWithMetadata(query); } public static HogQlQuery resolve(HogQlQuery query) { requireNonNull(query, "query is null"); - return new HogQlFunctionResolver(Optional.empty()).resolveQuery(query); + return resolve(query, Optional.empty()).query(); + } + + static Resolution resolve(HogQlQuery query, Optional exchangeRateSnapshotProvider) + { + requireNonNull(query, "query is null"); + return new HogQlFunctionResolver(Optional.empty(), exchangeRateSnapshotProvider).resolveWithMetadata(query); } public static HogQlQuery resolveV0(HogQlQuery query) { requireNonNull(query, "query is null"); - return new HogQlFunctionResolver(Optional.empty()).resolveQuery(query); + return resolveV0(query, Optional.empty()).query(); + } + + static Resolution resolveV0(HogQlQuery query, Optional exchangeRateSnapshotProvider) + { + requireNonNull(query, "query is null"); + return new HogQlFunctionResolver(Optional.empty(), exchangeRateSnapshotProvider).resolveWithMetadata(query); + } + + private Resolution resolveWithMetadata(HogQlQuery query) + { + HogQlQuery resolved = resolveQuery(query); + return new Resolution(resolved, exchangeRateGeneration); } private HogQlQuery resolveQuery(HogQlQuery query) @@ -499,6 +534,7 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< case UNIQ_EXACT_IF -> aggregate(function, "count", List.of(arguments.getFirst()), true, arguments.get(1), span); case GROUP_UNIQ_ARRAY_IF -> aggregate(function, "array_agg", List.of(arguments.getFirst()), true, arguments.get(1), span); case COUNT_DISTINCT -> aggregate(function, "count", List.of(arguments.getFirst()), true, null, span); + case CONVERT_CURRENCY -> convertCurrency(function, arguments); case MULTI_IF -> multiIf(function, arguments); case JSON_EXTRACT_STRING -> jsonExtractScalar(function, arguments, "varchar", new Literal(HogQlQuery.LiteralKind.STRING, "", span)); case JSON_EXTRACT_INT -> jsonExtractScalar(function, arguments, "bigint", new Literal(HogQlQuery.LiteralKind.INTEGER, "0", span)); @@ -531,7 +567,8 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< case TO_UNIX_TIMESTAMP -> cast(call("to_unixtime", arguments, span), "bigint", span); case PARSE_TIMESTAMP -> new CastExpression(arguments.getFirst(), new Identifier("timestamp(3)", false, span), true, span); case NOT -> new UnaryExpression(HogQlQuery.UnaryOperator.NOT, arguments.getFirst(), span); - case AND -> and(arguments, span); + case AND -> logical(arguments, HogQlQuery.BinaryOperator.AND, span); + case OR -> logical(arguments, HogQlQuery.BinaryOperator.OR, span); case GREATER -> new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN, arguments.getFirst(), arguments.get(1), span); case GREATER_OR_EQUAL -> new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN_OR_EQUAL, arguments.getFirst(), arguments.get(1), span); case LESS_OR_EQUAL -> new BinaryExpression(HogQlQuery.BinaryOperator.LESS_THAN_OR_EQUAL, arguments.getFirst(), arguments.get(1), span); @@ -596,11 +633,11 @@ private static Expression datePart(FunctionCall function, List argum return call(trinoFunction, List.of(arguments.get(1)), function.span()); } - private static Expression and(List arguments, HogQlQuery.SourceSpan span) + private static Expression logical(List arguments, HogQlQuery.BinaryOperator operator, HogQlQuery.SourceSpan span) { Expression result = arguments.getFirst(); for (int index = 1; index < arguments.size(); index++) { - result = new BinaryExpression(HogQlQuery.BinaryOperator.AND, result, arguments.get(index), span); + result = new BinaryExpression(operator, result, arguments.get(index), span); } return result; } @@ -932,6 +969,30 @@ private static FunctionCall coalesce(Expression value, Expression defaultValue, return call("coalesce", List.of(value, defaultValue), span); } + private Expression convertCurrency(FunctionCall function, List arguments) + { + if (exchangeRateGeneration.isEmpty()) { + HogQlExchangeRateSnapshotProvider provider = exchangeRateSnapshotProvider + .orElseThrow(() -> resolutionError(function, "HogQL function convertCurrency requires an exchange-rate snapshot")); + exchangeRateGeneration = OptionalLong.of(provider.pin(OptionalLong.empty()).generation()); + } + + HogQlQuery.SourceSpan span = function.span(); + Expression effectiveDate = cast( + arguments.size() == 4 ? arguments.get(3) : call("now", List.of(), span), + "date", + span); + return call( + "hogql_convert_currency", + List.of( + integerLiteral(Long.toString(exchangeRateGeneration.orElseThrow()), span), + cast(arguments.get(0), "varchar", span), + cast(arguments.get(1), "varchar", span), + cast(arguments.get(2), "decimal(38,10)", span), + effectiveDate), + span); + } + private static FunctionCall call(String name, List arguments, HogQlQuery.SourceSpan span) { return new FunctionCall( @@ -1269,4 +1330,13 @@ private static TrinoException error(HogQlErrorCode errorCode, FunctionCall funct message, null); } + + record Resolution(HogQlQuery query, OptionalLong exchangeRateGeneration) + { + Resolution + { + query = requireNonNull(query, "query is null"); + exchangeRateGeneration = requireNonNull(exchangeRateGeneration, "exchangeRateGeneration is null"); + } + } } diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java index d77693088d16..79afc295d0df 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -91,6 +91,7 @@ final class HogQlV0FunctionRegistry rewrite("splitByString", FunctionRewrite.SPLIT_STRING, "array(varchar)", signature(2)), rewrite("toString", FunctionRewrite.CAST_VARCHAR, "varchar", signature(1)), rewrite("toDate", FunctionRewrite.CAST_DATE, "date", signature(1)), + rewrite("_toDate", FunctionRewrite.CAST_DATE, "date", signature(1)), rewrite("toDateTime", FunctionRewrite.CAST_TIMESTAMP, "timestamp(0)", signature(1), signature(2)), rewrite("toStartOfMonth", FunctionRewrite.DATE_TRUNC_MONTH, "timestamp", signature(1)), rewrite("toStartOfDay", FunctionRewrite.DATE_TRUNC_DAY, "timestamp", signature(1)), @@ -113,10 +114,12 @@ final class HogQlV0FunctionRegistry rewrite("addDays", FunctionRewrite.ADD_DAYS, "any", signature(2)), rewrite("addMonths", FunctionRewrite.ADD_MONTHS, "any", signature(2)), rewrite("dateAdd", FunctionRewrite.DATE_ADD, "any", signature(2), signature(3)), + rewrite("convertCurrency", FunctionRewrite.CONVERT_CURRENCY, "decimal", signature(3), signature(4)), rewrite("toUnixTimestamp", FunctionRewrite.TO_UNIX_TIMESTAMP, "bigint", signature(1)), rewrite("parseDateTimeBestEffort", FunctionRewrite.PARSE_TIMESTAMP, "timestamp(3)", signature(1)), rewrite("not", FunctionRewrite.NOT, "boolean", signature(1)), rewrite("and", FunctionRewrite.AND, "boolean", signature(2), variadicSignature(2)), + rewrite("or", FunctionRewrite.OR, "boolean", signature(2), variadicSignature(2)), rewrite("greater", FunctionRewrite.GREATER, "boolean", signature(2)), rewrite("greaterOrEquals", FunctionRewrite.GREATER_OR_EQUAL, "boolean", signature(2)), rewrite("lessOrEquals", FunctionRewrite.LESS_OR_EQUAL, "boolean", signature(2)), diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateException.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateException.java index 4da3aa52047d..5ff60555464c 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateException.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateException.java @@ -31,6 +31,12 @@ public HogQlExchangeRateException(Failure failure, String message) this.failure = failure; } + public HogQlExchangeRateException(Failure failure, String message, Throwable cause) + { + super(errorCode(requireNonNull(failure, "failure is null")), message, cause); + this.failure = failure; + } + public Failure failure() { return failure; diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java index 3e29c0abf791..e42574c2c6e3 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -320,6 +320,8 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi !signature.variadic() && signature.argumentTypes().size() == 2; case ARG_MAX_IF, ARG_MIN_IF, ARRAY_SLICE, QUANTILE_IF, REGEX_REPLACE_ALL, REGEX_REPLACE_ONE -> !signature.variadic() && signature.argumentTypes().size() == 3; + case CONVERT_CURRENCY -> !signature.variadic() && + (signature.argumentTypes().size() == 3 || signature.argumentTypes().size() == 4); case GROUP_ARRAY_IF -> !signature.variadic() && (signature.argumentTypes().size() == 2 || signature.argumentTypes().size() == 3); case CAST_TIMESTAMP -> !signature.variadic() && @@ -331,7 +333,7 @@ private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSi case JSON_EXTRACT_ARRAY_RAW, JSON_EXTRACT_KEYS, JSON_EXTRACT_STRING, JSON_KEYS_AND_VALUES_RAW, JSON_LENGTH -> (!signature.variadic() && signature.argumentTypes().size() == 1) || (signature.variadic() && signature.argumentTypes().size() == 3); - case AND -> (!signature.variadic() && signature.argumentTypes().size() == 2) || + case AND, OR -> (!signature.variadic() && signature.argumentTypes().size() == 2) || (signature.variadic() && signature.argumentTypes().size() == 3); case ARRAY_SORT, RANGE -> !signature.variadic() && (signature.argumentTypes().size() == 1 || signature.argumentTypes().size() == 2); @@ -1356,6 +1358,7 @@ public enum FunctionRewrite DATE_TRUNC_WEEK, COUNT_IF, COUNT_DISTINCT, + CONVERT_CURRENCY, DECIMAL_CAST, DIVIDE_DECIMAL, EMPTY, @@ -1401,6 +1404,7 @@ public enum FunctionRewrite MULTIPLY_DECIMAL, MULTI_IF, NOT, + OR, NOT_EQUALS, NOT_EMPTY, PARSE_TIMESTAMP, diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCurrencyCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCurrencyCompiler.java new file mode 100644 index 000000000000..6dcddc58b7d8 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCurrencyCompiler.java @@ -0,0 +1,81 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot.ExchangeRate; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotProvider; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlCurrencyCompiler +{ + private final SqlParser sqlParser = new SqlParser(); + + @Test + public void testPinsOneGenerationAndLowersAllCurrencyCalls() + { + AtomicInteger pins = new AtomicInteger(); + HogQlExchangeRateSnapshotProvider provider = expectedGeneration -> { + assertThat(expectedGeneration).isEmpty(); + pins.incrementAndGet(); + return new HogQlExchangeRateSnapshotProvider.PinnedSnapshot(snapshot(42)); + }; + + HogQlCompilationResult result = new HogQlCompiler(provider).compile( + "SELECT convertCurrency('USD', 'EUR', 100), " + + "convertCurrency(source_currency, target_currency, amount, event_date), " + + "convertCurrency('USD', 'EUR', 100, _toDate('2024-01-01')) FROM events", + java.util.Map.of()); + + assertThat(pins).hasValue(1); + assertThat(result.exchangeRateGeneration()).hasValue(42); + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT " + + "hogql_convert_currency(42, CAST('USD' AS varchar), CAST('EUR' AS varchar), CAST(100 AS decimal(38, 10)), CAST(now() AS date)), " + + "hogql_convert_currency(42, CAST(source_currency AS varchar), CAST(target_currency AS varchar), CAST(amount AS decimal(38, 10)), CAST(event_date AS date)), " + + "hogql_convert_currency(42, CAST('USD' AS varchar), CAST('EUR' AS varchar), CAST(100 AS decimal(38, 10)), CAST(CAST('2024-01-01' AS date) AS date)) " + + "FROM events")); + } + + @Test + public void testFailsClosedWithoutExchangeRateProvider() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile("SELECT convertCurrency('USD', 'EUR', 100)")); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception).hasMessage("line 1:8: HogQL function convertCurrency requires an exchange-rate snapshot"); + } + + private static HogQlExchangeRateSnapshot snapshot(long generation) + { + return new HogQlExchangeRateSnapshot( + HogQlExchangeRateSnapshot.PROTOCOL_VERSION, + HogQlExchangeRateSnapshot.SCHEMA_VERSION, + generation, + HogQlExchangeRateSnapshot.BASE_CURRENCY, + HogQlExchangeRateSnapshot.DECIMAL_SCALE, + List.of(new ExchangeRate("USD", "1970-01-01", "10000000000"))); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index 164cda105231..b8f0973d0618 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -231,12 +231,12 @@ public void testCompilerLowersDateTimeCompatibilityFunctions() public void testCompilerLowersScalarCompatibilityFunctions() { HogQlCompilationResult result = new HogQlCompiler().compile(envelope( - "SELECT not(active), and(active, ready), greater(score, 10), like(name, 'pro%'), " + + "SELECT not(active), and(active, ready), or(active, ready, pending), greater(score, 10), like(name, 'pro%'), " + "least(first, second), greatest(first, second), position(name, 'needle'), " + "startsWith(name, 'prefix'), substring(name, 2, 4), log10(score) FROM records")); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( - "SELECT NOT active, active AND ready, score > 10, name LIKE 'pro%', " + + "SELECT NOT active, active AND ready, (active OR ready) OR pending, score > 10, name LIKE 'pro%', " + "least(first, second), greatest(first, second), strpos(name, 'needle'), " + "starts_with(name, 'prefix'), substring(name, 2, 4), log10(score) FROM records")); } From 34b2da0371e5fbfe36fff17c9e7d5b17b6bf840a Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 03:35:44 -0400 Subject: [PATCH 131/136] feat(hogql): execute pinned currency conversions --- .../io/trino/hogql/ForHogQlExchangeRate.java | 29 +++ .../HogQlExchangeRateConversionEngine.java | 28 +-- .../hogql/HogQlExchangeRateFunction.java | 114 +++++++++ .../hogql/HogQlExchangeRateHttpTransport.java | 36 +++ .../trino/hogql/HogQlExchangeRateManager.java | 151 +++++++++++ .../trino/hogql/HogQlExchangeRateModule.java | 63 +++++ .../hogql/HogQlSemanticCatalogConfig.java | 4 +- .../io/trino/server/CoordinatorModule.java | 15 +- .../io/trino/server/ServerMainModule.java | 9 + ...TestHogQlExchangeRateConversionEngine.java | 173 +++++++++++++ .../hogql/TestHogQlSemanticCatalogConfig.java | 2 +- plugin/trino-ducklake/pom.xml | 10 - .../currency/ExchangeRateSnapshot.java | 119 --------- .../ExchangeRateSnapshotJsonDecoder.java | 234 ------------------ .../TestExchangeRateConversionEngine.java | 108 -------- .../TestExchangeRateSnapshotJsonDecoder.java | 143 ----------- 16 files changed, 600 insertions(+), 638 deletions(-) create mode 100644 core/trino-main/src/main/java/io/trino/hogql/ForHogQlExchangeRate.java rename plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateConversionEngine.java => core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateConversionEngine.java (74%) create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateFunction.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateManager.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateModule.java create mode 100644 core/trino-main/src/test/java/io/trino/hogql/TestHogQlExchangeRateConversionEngine.java delete mode 100644 plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshot.java delete mode 100644 plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshotJsonDecoder.java delete mode 100644 plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateConversionEngine.java delete mode 100644 plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateSnapshotJsonDecoder.java diff --git a/core/trino-main/src/main/java/io/trino/hogql/ForHogQlExchangeRate.java b/core/trino-main/src/main/java/io/trino/hogql/ForHogQlExchangeRate.java new file mode 100644 index 000000000000..db82d8b5077e --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/ForHogQlExchangeRate.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@BindingAnnotation +@Target({FIELD, PARAMETER, METHOD}) +@Retention(RUNTIME) +public @interface ForHogQlExchangeRate {} diff --git a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateConversionEngine.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateConversionEngine.java similarity index 74% rename from plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateConversionEngine.java rename to core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateConversionEngine.java index b9395054f5d9..24b74bb19202 100644 --- a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateConversionEngine.java +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateConversionEngine.java @@ -11,10 +11,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.ducklake.function.currency; +package io.trino.hogql; import com.google.common.collect.ImmutableMap; -import io.trino.plugin.ducklake.function.currency.ExchangeRateSnapshot.ExchangeRate; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot.ExchangeRate; import java.math.BigDecimal; import java.math.BigInteger; @@ -27,17 +28,17 @@ import java.util.Optional; import java.util.TreeMap; -import static io.trino.plugin.ducklake.function.currency.ExchangeRateSnapshot.DECIMAL_SCALE; +import static io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot.DECIMAL_SCALE; import static java.util.Objects.requireNonNull; -public final class ExchangeRateConversionEngine +public final class HogQlExchangeRateConversionEngine { private static final BigDecimal ZERO = BigDecimal.ZERO.setScale(DECIMAL_SCALE); private final long generation; private final Map> ratesByCurrency; - public ExchangeRateConversionEngine(ExchangeRateSnapshot snapshot) + public HogQlExchangeRateConversionEngine(HogQlExchangeRateSnapshot snapshot) { requireNonNull(snapshot, "snapshot is null"); generation = snapshot.generation(); @@ -57,9 +58,8 @@ public long generation() return generation; } - public Optional rate(long expectedGeneration, String currency, LocalDate date) + public Optional rate(String currency, LocalDate date) { - verifyGeneration(expectedGeneration); requireNonNull(currency, "currency is null"); requireNonNull(date, "date is null"); NavigableMap currencyRates = ratesByCurrency.get(currency); @@ -70,9 +70,8 @@ public Optional rate(long expectedGeneration, String currency, Local return rate == null ? Optional.empty() : Optional.of(rate.getValue()); } - public BigDecimal convert(long expectedGeneration, String sourceCurrency, String targetCurrency, BigDecimal amount, LocalDate date) + public BigDecimal convert(String sourceCurrency, String targetCurrency, BigDecimal amount, LocalDate date) { - verifyGeneration(expectedGeneration); requireNonNull(sourceCurrency, "sourceCurrency is null"); requireNonNull(targetCurrency, "targetCurrency is null"); requireNonNull(amount, "amount is null"); @@ -83,8 +82,8 @@ public BigDecimal convert(long expectedGeneration, String sourceCurrency, String return decimalAmount; } - BigDecimal sourceRate = rate(expectedGeneration, sourceCurrency, date).orElse(ZERO); - BigDecimal targetRate = rate(expectedGeneration, targetCurrency, date).orElse(ZERO); + BigDecimal sourceRate = rate(sourceCurrency, date).orElse(ZERO); + BigDecimal targetRate = rate(targetCurrency, date).orElse(ZERO); if (sourceRate.signum() == 0 || targetRate.signum() == 0) { return ZERO; } @@ -94,11 +93,4 @@ public BigDecimal convert(long expectedGeneration, String sourceCurrency, String .multiply(targetRate) .setScale(DECIMAL_SCALE, RoundingMode.DOWN); } - - private void verifyGeneration(long expectedGeneration) - { - if (expectedGeneration != generation) { - throw new IllegalArgumentException("exchange-rate snapshot generation mismatch"); - } - } } diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateFunction.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateFunction.java new file mode 100644 index 000000000000..7cf727080a0c --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateFunction.java @@ -0,0 +1,114 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.common.collect.ImmutableList; +import io.airlift.slice.Slice; +import io.trino.annotation.UsedByGeneratedCode; +import io.trino.metadata.SqlScalarFunction; +import io.trino.operator.scalar.ChoicesSpecializedSqlScalarFunction; +import io.trino.operator.scalar.SpecializedSqlScalarFunction; +import io.trino.spi.function.BoundSignature; +import io.trino.spi.function.FunctionMetadata; +import io.trino.spi.function.Signature; +import io.trino.spi.type.Int128; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.util.Optional; + +import static io.trino.spi.function.InvocationConvention.InvocationArgumentConvention.NEVER_NULL; +import static io.trino.spi.function.InvocationConvention.InvocationReturnConvention.FAIL_ON_NULL; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.DecimalType.createDecimalType; +import static io.trino.spi.type.Decimals.encodeScaledValue; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static io.trino.util.Reflection.constructorMethodHandle; +import static io.trino.util.Reflection.methodHandle; +import static java.util.Objects.requireNonNull; + +public final class HogQlExchangeRateFunction + extends SqlScalarFunction +{ + public static final String NAME = "hogql_convert_currency"; + private static final int DECIMAL_SCALE = 10; + private static final java.lang.invoke.MethodHandle METHOD_HANDLE = methodHandle( + State.class, + "convert", + long.class, + Slice.class, + Slice.class, + Int128.class, + long.class); + + private final HogQlExchangeRateManager manager; + + public HogQlExchangeRateFunction(HogQlExchangeRateManager manager) + { + super(FunctionMetadata.scalarBuilder(NAME) + .signature(Signature.builder() + .returnType(createDecimalType(38, DECIMAL_SCALE)) + .argumentType(BIGINT) + .argumentType(VARCHAR) + .argumentType(VARCHAR) + .argumentType(createDecimalType(38, DECIMAL_SCALE)) + .argumentType(DATE) + .build()) + .hidden() + .description("Convert a decimal amount using a pinned HogQL exchange-rate generation") + .build()); + this.manager = requireNonNull(manager, "manager is null"); + } + + @Override + protected SpecializedSqlScalarFunction specialize(BoundSignature boundSignature) + { + java.lang.invoke.MethodHandle instanceFactory = constructorMethodHandle(State.class, HogQlExchangeRateManager.class).bindTo(manager); + return new ChoicesSpecializedSqlScalarFunction( + boundSignature, + FAIL_ON_NULL, + ImmutableList.of(NEVER_NULL, NEVER_NULL, NEVER_NULL, NEVER_NULL, NEVER_NULL), + METHOD_HANDLE, + Optional.of(instanceFactory)); + } + + public static final class State + { + private final HogQlExchangeRateManager manager; + private long generation = -1; + private HogQlExchangeRateConversionEngine engine; + + public State(HogQlExchangeRateManager manager) + { + this.manager = requireNonNull(manager, "manager is null"); + } + + @UsedByGeneratedCode + public Int128 convert(long generation, Slice sourceCurrency, Slice targetCurrency, Int128 amount, long effectiveDate) + { + if (engine == null || this.generation != generation) { + engine = manager.engine(generation); + this.generation = generation; + } + BigDecimal decimalAmount = new BigDecimal(amount.toBigInteger(), DECIMAL_SCALE); + BigDecimal converted = engine.convert( + sourceCurrency.toStringUtf8(), + targetCurrency.toStringUtf8(), + decimalAmount, + LocalDate.ofEpochDay(effectiveDate)); + return encodeScaledValue(converted, DECIMAL_SCALE); + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateHttpTransport.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateHttpTransport.java index 34aae458533c..187bb73603f0 100644 --- a/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateHttpTransport.java +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateHttpTransport.java @@ -14,6 +14,7 @@ package io.trino.hogql; import com.google.common.net.MediaType; +import com.google.inject.Inject; import io.airlift.http.client.HttpClient; import io.airlift.http.client.Request; import io.airlift.http.client.Response; @@ -27,6 +28,9 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.function.Supplier; @@ -45,6 +49,7 @@ public final class HogQlExchangeRateHttpTransport { private static final String METADATA_PATH = "v1/hogql/compatibility/exchange-rates"; private static final String AUTHENTICATION_HEADER = "X-Duckgres-Internal-Secret"; + private static final int MAXIMUM_AUTHENTICATION_TOKEN_BYTES = 4096; private static final MediaType JSON = JSON_UTF_8.withoutParameters(); private final URI baseUri; @@ -52,6 +57,17 @@ public final class HogQlExchangeRateHttpTransport private final int maximumResponseBytes; private final Supplier authenticationTokenSupplier; + @Inject + public HogQlExchangeRateHttpTransport( + HogQlSemanticCatalogConfig config, + @ForHogQlExchangeRate HttpClient httpClient) + { + this(requireNonNull(config, "config is null").getUri(), + httpClient, + Math.toIntExact(config.getMaximumResponseSize().toBytes()), + tokenSupplier(config.getAuthenticationTokenFile())); + } + HogQlExchangeRateHttpTransport(URI baseUri, HttpClient httpClient, Supplier authenticationTokenSupplier) { this(baseUri, httpClient, HogQlExchangeRateSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES, authenticationTokenSupplier); @@ -96,6 +112,26 @@ static URI buildUri(URI baseUri, LoadRequest request) return uriBuilder.build(); } + private static Supplier tokenSupplier(String authenticationTokenFile) + { + if (authenticationTokenFile == null || authenticationTokenFile.isBlank()) { + throw new IllegalArgumentException("HogQL exchange-rate authentication token file is not configured"); + } + Path path = Path.of(authenticationTokenFile); + return () -> { + try (InputStream input = Files.newInputStream(path)) { + byte[] token = input.readNBytes(MAXIMUM_AUTHENTICATION_TOKEN_BYTES + 1); + if (token.length > MAXIMUM_AUTHENTICATION_TOKEN_BYTES) { + throw new IllegalArgumentException("invalid HogQL exchange-rate authentication token"); + } + return new String(token, StandardCharsets.UTF_8).strip(); + } + catch (IOException e) { + throw new IllegalStateException("HogQL exchange-rate authentication token is unavailable", e); + } + }; + } + private static String validateAuthenticationToken(String token) { requireNonNull(token, "authentication token is null"); diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateManager.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateManager.java new file mode 100644 index 000000000000..5ceb580243bd --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateManager.java @@ -0,0 +1,151 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.Inject; +import io.trino.hogql.compiler.catalog.BoundedAsyncHogQlExchangeRateSnapshotCache; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException.Failure; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotJsonDecoder; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.LoadRequest; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotProvider; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotProvider.PinnedSnapshot; +import jakarta.annotation.PreDestroy; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +public final class HogQlExchangeRateManager + implements HogQlExchangeRateSnapshotProvider +{ + private final Object engineLock = new Object(); + private final int maximumEngines; + private final Duration requestTimeout; + private final ThreadPoolExecutor loaderExecutor; + private final BoundedAsyncHogQlExchangeRateSnapshotCache cache; + private final LinkedHashMap engines = new LinkedHashMap<>(16, 0.75f, true); + + @Inject + public HogQlExchangeRateManager( + HogQlSemanticCatalogConfig config, + HogQlExchangeRateHttpTransport transport, + HogQlExchangeRateSnapshotJsonDecoder decoder) + { + requireNonNull(config, "config is null"); + maximumEngines = config.getMaximumEntries(); + requestTimeout = config.getRequestTimeout().toJavaTime(); + loaderExecutor = new ThreadPoolExecutor( + config.getLoaderThreads(), + config.getLoaderThreads(), + 0, + MILLISECONDS, + new ArrayBlockingQueue<>(config.getLoaderQueueCapacity()), + daemonThreadsNamed("hogql-exchange-rate-loader-%s"), + new ThreadPoolExecutor.AbortPolicy()); + HogQlExchangeRateSnapshotLoader loader = HogQlExchangeRateSnapshotLoader.fromJsonTransport(transport, decoder); + cache = new BoundedAsyncHogQlExchangeRateSnapshotCache( + config.getMaximumEntries(), + config.getRefreshAfter().toJavaTime(), + config.getExpireAfter().toJavaTime(), + config.getFailureBackoff().toJavaTime(), + System::nanoTime, + loaderExecutor, + expectedGeneration -> loader.load(expectedGeneration.isPresent() + ? LoadRequest.pinned(expectedGeneration.orElseThrow()) + : LoadRequest.latest())); + cache.currentSnapshot(OptionalLong.empty()); + } + + @Override + public PinnedSnapshot pin(OptionalLong expectedGeneration) + { + return new PinnedSnapshot(loadSnapshot(expectedGeneration)); + } + + public HogQlExchangeRateConversionEngine engine(long generation) + { + synchronized (engineLock) { + HogQlExchangeRateConversionEngine engine = engines.get(generation); + if (engine != null) { + return engine; + } + } + + HogQlExchangeRateSnapshot snapshot = loadSnapshot(OptionalLong.of(generation)); + HogQlExchangeRateConversionEngine created = new HogQlExchangeRateConversionEngine(snapshot); + synchronized (engineLock) { + HogQlExchangeRateConversionEngine existing = engines.get(generation); + if (existing != null) { + return existing; + } + engines.put(generation, created); + while (engines.size() > maximumEngines) { + Long oldestGeneration = engines.keySet().iterator().next(); + engines.remove(oldestGeneration); + } + return created; + } + } + + private HogQlExchangeRateSnapshot loadSnapshot(OptionalLong expectedGeneration) + { + Optional current = cache.currentSnapshot(expectedGeneration); + if (current.isPresent()) { + return current.orElseThrow(); + } + try { + return cache.prewarm(expectedGeneration) + .toCompletableFuture() + .get(requestTimeout.toMillis(), TimeUnit.MILLISECONDS); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw unavailable(e); + } + catch (ExecutionException e) { + if (e.getCause() instanceof HogQlExchangeRateException exchangeRateException) { + throw exchangeRateException; + } + throw unavailable(e.getCause()); + } + catch (TimeoutException | RuntimeException e) { + throw unavailable(e); + } + } + + private static HogQlExchangeRateException unavailable(Throwable cause) + { + return new HogQlExchangeRateException(Failure.UNAVAILABLE, "HogQL exchange-rate snapshot is unavailable", cause); + } + + @PreDestroy + public void shutdown() + { + loaderExecutor.shutdownNow(); + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateModule.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateModule.java new file mode 100644 index 000000000000..e1bd0af7e281 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateModule.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.Binder; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import com.google.inject.Singleton; +import com.google.inject.multibindings.ProvidesIntoSet; +import com.google.inject.multibindings.OptionalBinder; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotJsonDecoder; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotProvider; +import io.trino.metadata.InternalFunctionBundle; +import io.trino.spi.function.FunctionBundle; + +import static io.airlift.http.client.HttpClientBinder.httpClientBinder; + +public final class HogQlExchangeRateModule + extends AbstractConfigurationAwareModule +{ + @Override + protected void setup(Binder binder) + { + HogQlSemanticCatalogConfig config = buildConfigObject(HogQlSemanticCatalogConfig.class); + httpClientBinder(binder) + .bindHttpClient("hogql-exchange-rate", ForHogQlExchangeRate.class) + .withConfigDefaults(httpClientConfig -> httpClientConfig + .setRequestTimeout(config.getRequestTimeout()) + .setMaxResponseContentLength(config.getMaximumResponseSize()) + .setMaxRequestsQueuedPerDestination(config.getLoaderQueueCapacity())); + binder.bind(HogQlExchangeRateHttpTransport.class).in(Scopes.SINGLETON); + binder.bind(HogQlExchangeRateManager.class).in(Scopes.SINGLETON); + OptionalBinder.newOptionalBinder(binder, HogQlExchangeRateSnapshotProvider.class) + .setBinding() + .to(HogQlExchangeRateManager.class); + } + + @Provides + @Singleton + public static HogQlExchangeRateSnapshotJsonDecoder provideDecoder() + { + return new HogQlExchangeRateSnapshotJsonDecoder(); + } + + @ProvidesIntoSet + @Singleton + public static FunctionBundle provideFunctionBundle(HogQlExchangeRateManager manager) + { + return new InternalFunctionBundle(new HogQlExchangeRateFunction(manager)); + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java index 639db487dce4..09a63366a40a 100644 --- a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java @@ -26,7 +26,7 @@ import java.net.URI; -import static io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES; +import static io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; @@ -172,7 +172,7 @@ public HogQlSemanticCatalogConfig setRequestTimeout(Duration requestTimeout) @NotNull @MinDataSize("1B") - @MaxDataSize("8MB") + @MaxDataSize("32MB") public DataSize getMaximumResponseSize() { return maximumResponseSize; diff --git a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java index 89e8bf635857..cdc11fcf00fc 100644 --- a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java +++ b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java @@ -107,6 +107,7 @@ import io.trino.hogql.HogQlSemanticCatalogConfig; import io.trino.hogql.HogQlSemanticCatalogModule; import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotProvider; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider; import io.trino.memory.ClusterMemoryManager; import io.trino.memory.ForMemoryManager; @@ -152,6 +153,7 @@ import io.trino.sql.rewrite.StatementRewrite.Rewrite; import java.util.List; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; @@ -186,9 +188,7 @@ protected void setup(Binder binder) jsonCodecBinder(binder).bindJsonCodec(TaskInfo.class); jaxrsBinder(binder).bind(QueuedStatementResource.class); jaxrsBinder(binder).bind(ExecutingStatementResource.class); - configBinder(binder).bindConfig(HogQlConfig.class); - configBinder(binder).bindConfig(HogQlSemanticCatalogConfig.class); - binder.bind(HogQlCompiler.class).in(Scopes.SINGLETON); + OptionalBinder.newOptionalBinder(binder, HogQlExchangeRateSnapshotProvider.class); binder.bind(HogQlPhysicalCatalogProvider.class).in(Scopes.SINGLETON); binder.bind(HogQlCompilationExecutor.class).in(Scopes.SINGLETON); binder.bind(HogQlCompilationStats.class).in(Scopes.SINGLETON); @@ -463,6 +463,15 @@ List getCompositeOutputDataSizeEstimatorDelegateFac closingBinder(binder).registerExecutor(Key.get(ScheduledExecutorService.class, ForScheduler.class)); } + @Provides + @Singleton + public static HogQlCompiler provideHogQlCompiler(Optional exchangeRateSnapshotProvider) + { + return exchangeRateSnapshotProvider + .map(HogQlCompiler::new) + .orElseGet(HogQlCompiler::new); + } + // working around circular dependency Metadata <-> PlannerContext private static class InitializeLanguageFunctionManager { diff --git a/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java b/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java index ddca3ad5a977..00577f6a7872 100644 --- a/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java +++ b/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java @@ -56,6 +56,9 @@ import io.trino.execution.executor.timesharing.MultilevelSplitQueue; import io.trino.execution.executor.timesharing.TimeSharingTaskExecutor; import io.trino.execution.scheduler.NodeSchedulerConfig; +import io.trino.hogql.HogQlConfig; +import io.trino.hogql.HogQlExchangeRateModule; +import io.trino.hogql.HogQlSemanticCatalogConfig; import io.trino.jsonpath.ir.IrJsonPath; import io.trino.memory.LocalMemoryManager; import io.trino.memory.LocalMemoryManagerExporter; @@ -200,6 +203,12 @@ protected void setup(Binder binder) { ServerConfig serverConfig = buildConfigObject(ServerConfig.class); + configBinder(binder).bindConfig(HogQlConfig.class); + configBinder(binder).bindConfig(HogQlSemanticCatalogConfig.class); + if (buildConfigObject(HogQlSemanticCatalogConfig.class).getUri() != null) { + install(new HogQlExchangeRateModule()); + } + if (serverConfig.isCoordinator()) { install(new CoordinatorModule()); } diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlExchangeRateConversionEngine.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlExchangeRateConversionEngine.java new file mode 100644 index 000000000000..0451fe30a05a --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlExchangeRateConversionEngine.java @@ -0,0 +1,173 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.http.client.testing.TestingHttpClient; +import io.airlift.http.client.testing.TestingResponse; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot.ExchangeRate; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotJsonDecoder; +import io.trino.metadata.InternalFunctionBundle; +import io.trino.spi.type.Int128; +import io.trino.sql.query.QueryAssertions; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.util.List; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.google.common.net.MediaType.JSON_UTF_8; +import static io.airlift.http.client.HttpStatus.OK; +import static io.airlift.http.client.testing.TestingResponse.contentType; +import static io.airlift.slice.Slices.utf8Slice; +import static io.trino.spi.type.Decimals.encodeScaledValue; +import static io.trino.spi.type.DecimalType.createDecimalType; +import static io.trino.spi.type.SqlDecimal.decimal; +import static org.assertj.core.api.Assertions.assertThat; + +final class TestHogQlExchangeRateConversionEngine +{ + private static final LocalDate FIRST_DATE = LocalDate.parse("2024-01-01"); + private static final LocalDate SECOND_DATE = LocalDate.parse("2024-01-10"); + + private final HogQlExchangeRateConversionEngine engine = new HogQlExchangeRateConversionEngine(new HogQlExchangeRateSnapshot( + 1, + 1, + 17, + "USD", + 10, + List.of( + rate("EUR", FIRST_DATE, "0.9049"), + rate("EUR", SECOND_DATE, "0.95"), + rate("JPY", FIRST_DATE, "108.1234"), + rate("USD", FIRST_DATE, "1"), + rate("ZZZ", FIRST_DATE, "0")))); + + @Test + void testFloorLookupUsesLatestRateNotAfterDate() + { + assertThat(engine.rate("EUR", FIRST_DATE.minusDays(1))).isEmpty(); + assertThat(engine.rate("EUR", FIRST_DATE)).contains(new BigDecimal("0.9049000000")); + assertThat(engine.rate("EUR", SECOND_DATE.minusDays(1))).contains(new BigDecimal("0.9049000000")); + assertThat(engine.rate("EUR", SECOND_DATE.plusYears(1))).contains(new BigDecimal("0.9500000000")); + } + + @Test + void testConversionMatchesPostHogDecimalSemantics() + { + assertDecimal(engine.convert("USD", "EUR", new BigDecimal("100"), FIRST_DATE), "90.4900000000"); + assertDecimal(engine.convert("EUR", "USD", new BigDecimal("90.49"), FIRST_DATE), "100.0000000000"); + assertDecimal(engine.convert("AAA", "AAA", new BigDecimal("1.23456789019"), FIRST_DATE), "1.2345678901"); + assertDecimal(engine.convert("USD", "JPY", new BigDecimal("-2.5"), FIRST_DATE), "-270.3085000000"); + } + + @Test + void testMissingOrZeroRateReturnsZero() + { + assertDecimal(engine.convert("MISSING", "EUR", BigDecimal.TEN, FIRST_DATE), "0.0000000000"); + assertDecimal(engine.convert("USD", "MISSING", BigDecimal.TEN, FIRST_DATE), "0.0000000000"); + assertDecimal(engine.convert("ZZZ", "EUR", BigDecimal.TEN, FIRST_DATE), "0.0000000000"); + assertDecimal(engine.convert("EUR", "USD", BigDecimal.TEN, FIRST_DATE.minusDays(1)), "0.0000000000"); + } + + @Test + void testRuntimeStateAndCompilerPinsReuseCachedGeneration() + { + String snapshot = """ + {"protocolVersion":1,"schemaVersion":1,"generation":17,"baseCurrency":"USD","decimalScale":10,"rates":[ + {"currency":"EUR","effectiveDate":"2024-01-01","unscaledRate":"9049000000"}, + {"currency":"USD","effectiveDate":"2024-01-01","unscaledRate":"10000000000"}]} + """; + AtomicInteger requests = new AtomicInteger(); + TestingHttpClient client = new TestingHttpClient(_ -> { + requests.incrementAndGet(); + return new TestingResponse( + OK, + contentType(JSON_UTF_8), + snapshot.getBytes(StandardCharsets.UTF_8)); + }); + HogQlExchangeRateHttpTransport transport = new HogQlExchangeRateHttpTransport( + URI.create("https://duckgres.example/"), + client, + () -> "test-token"); + HogQlExchangeRateManager manager = new HogQlExchangeRateManager( + new HogQlSemanticCatalogConfig(), + transport, + new HogQlExchangeRateSnapshotJsonDecoder()); + try { + manager.pin(OptionalLong.empty()); + manager.pin(OptionalLong.empty()); + HogQlExchangeRateFunction.State state = new HogQlExchangeRateFunction.State(manager); + Int128 result = state.convert( + 17, + utf8Slice("USD"), + utf8Slice("EUR"), + encodeScaledValue(new BigDecimal("100"), 10), + FIRST_DATE.toEpochDay()); + + assertDecimal(new BigDecimal(result.toBigInteger(), 10), "90.4900000000"); + assertThat(requests).hasValue(1); + } + finally { + manager.shutdown(); + } + } + + @Test + void testRegisteredScalarFunctionExecutesPinnedConversion() + { + String snapshot = """ + {"protocolVersion":1,"schemaVersion":1,"generation":17,"baseCurrency":"USD","decimalScale":10,"rates":[ + {"currency":"EUR","effectiveDate":"2024-01-01","unscaledRate":"9049000000"}, + {"currency":"USD","effectiveDate":"2024-01-01","unscaledRate":"10000000000"}]} + """; + TestingHttpClient client = new TestingHttpClient(_ -> new TestingResponse( + OK, + contentType(JSON_UTF_8), + snapshot.getBytes(StandardCharsets.UTF_8))); + HogQlExchangeRateManager manager = new HogQlExchangeRateManager( + new HogQlSemanticCatalogConfig(), + new HogQlExchangeRateHttpTransport(URI.create("https://duckgres.example/"), client, () -> "test-token"), + new HogQlExchangeRateSnapshotJsonDecoder()); + try { + manager.pin(OptionalLong.empty()); + try (QueryAssertions assertions = new QueryAssertions()) { + assertions.addFunctions(new InternalFunctionBundle(new HogQlExchangeRateFunction(manager))); + + assertThat(assertions.expression( + "hogql_convert_currency(17, 'USD', 'EUR', DECIMAL '100.0000000000', DATE '2024-01-01')")) + .hasType(createDecimalType(38, 10)) + .isEqualTo(decimal("90.4900000000", createDecimalType(38, 10))); + } + } + finally { + manager.shutdown(); + } + } + + private static ExchangeRate rate(String currency, LocalDate date, String value) + { + return new ExchangeRate(currency, date.toString(), new BigDecimal(value).movePointRight(10).toBigIntegerExact().toString()); + } + + private static void assertDecimal(BigDecimal actual, String expected) + { + assertThat(actual.scale()).isEqualTo(10); + assertThat(actual).isEqualByComparingTo(expected); + } +} diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java index 3d5e647394bf..9613fea8c08a 100644 --- a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java @@ -44,7 +44,7 @@ public void testDefaults() .setLoaderThreads(4) .setLoaderQueueCapacity(64) .setRequestTimeout(new Duration(10, SECONDS)) - .setMaximumResponseSize(DataSize.of(8, MEGABYTE))); + .setMaximumResponseSize(DataSize.of(32, MEGABYTE))); } @Test diff --git a/plugin/trino-ducklake/pom.xml b/plugin/trino-ducklake/pom.xml index 6217acfd0f20..bf4f9c2fc627 100644 --- a/plugin/trino-ducklake/pom.xml +++ b/plugin/trino-ducklake/pom.xml @@ -19,16 +19,6 @@ - - com.fasterxml.jackson.core - jackson-core - - - - com.fasterxml.jackson.core - jackson-databind - - com.google.errorprone error_prone_annotations diff --git a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshot.java b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshot.java deleted file mode 100644 index 7979e742558f..000000000000 --- a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshot.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.ducklake.function.currency; - -import java.time.LocalDate; -import java.time.format.DateTimeParseException; -import java.util.List; -import java.util.regex.Pattern; - -import static java.util.Objects.requireNonNull; - -public record ExchangeRateSnapshot( - int protocolVersion, - int schemaVersion, - long generation, - String baseCurrency, - int decimalScale, - List rates) -{ - public static final int PROTOCOL_VERSION = 1; - public static final int SCHEMA_VERSION = 1; - public static final String BASE_CURRENCY = "USD"; - public static final int DECIMAL_SCALE = 10; - public static final int MAXIMUM_RATES = 1_000_000; - - private static final Pattern CURRENCY_PATTERN = Pattern.compile("[A-Z]{3}"); - private static final Pattern UNSCALED_RATE_PATTERN = Pattern.compile("0|[1-9][0-9]{0,17}"); - private static final String UNSCALED_ONE = "10000000000"; - - public ExchangeRateSnapshot - { - if (protocolVersion != PROTOCOL_VERSION) { - throw new IllegalArgumentException("unsupported exchange-rate snapshot protocol"); - } - if (schemaVersion != SCHEMA_VERSION) { - throw new IllegalArgumentException("unsupported exchange-rate snapshot schema"); - } - if (generation <= 0) { - throw new IllegalArgumentException("exchange-rate snapshot generation must be positive"); - } - baseCurrency = requireNonNull(baseCurrency, "baseCurrency is null"); - if (!baseCurrency.equals(BASE_CURRENCY)) { - throw new IllegalArgumentException("unsupported exchange-rate base currency"); - } - if (decimalScale != DECIMAL_SCALE) { - throw new IllegalArgumentException("unsupported exchange-rate decimal scale"); - } - rates = List.copyOf(requireNonNull(rates, "rates is null")); - if (rates.isEmpty()) { - throw new IllegalArgumentException("exchange-rate snapshot has no rates"); - } - if (rates.size() > MAXIMUM_RATES) { - throw new IllegalArgumentException("exchange-rate snapshot exceeds rate limit"); - } - - boolean baseCurrencyPresent = false; - ExchangeRate previous = null; - for (ExchangeRate rate : rates) { - requireNonNull(rate, "rate is null"); - if (rate.currency().equals(BASE_CURRENCY)) { - baseCurrencyPresent = true; - if (!rate.unscaledRate().equals(UNSCALED_ONE)) { - throw new IllegalArgumentException("base-currency rate must equal one"); - } - } - if (previous != null && compare(previous, rate) >= 0) { - throw new IllegalArgumentException("exchange rates are not strictly sorted"); - } - previous = rate; - } - if (!baseCurrencyPresent) { - throw new IllegalArgumentException("exchange-rate snapshot has no base-currency rate"); - } - } - - private static int compare(ExchangeRate left, ExchangeRate right) - { - int currencyComparison = left.currency().compareTo(right.currency()); - if (currencyComparison != 0) { - return currencyComparison; - } - return left.effectiveDate().compareTo(right.effectiveDate()); - } - - public record ExchangeRate(String currency, String effectiveDate, String unscaledRate) - { - public ExchangeRate - { - currency = requireNonNull(currency, "currency is null"); - effectiveDate = requireNonNull(effectiveDate, "effectiveDate is null"); - unscaledRate = requireNonNull(unscaledRate, "unscaledRate is null"); - if (!CURRENCY_PATTERN.matcher(currency).matches()) { - throw new IllegalArgumentException("invalid exchange-rate currency"); - } - try { - if (!LocalDate.parse(effectiveDate).toString().equals(effectiveDate)) { - throw new IllegalArgumentException("noncanonical exchange-rate date"); - } - } - catch (DateTimeParseException e) { - throw new IllegalArgumentException("invalid exchange-rate date", e); - } - if (!UNSCALED_RATE_PATTERN.matcher(unscaledRate).matches()) { - throw new IllegalArgumentException("invalid exchange-rate value"); - } - } - } -} diff --git a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshotJsonDecoder.java b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshotJsonDecoder.java deleted file mode 100644 index 4d738bfcfe36..000000000000 --- a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/currency/ExchangeRateSnapshotJsonDecoder.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.ducklake.function.currency; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.StreamReadConstraints; -import com.fasterxml.jackson.core.StreamReadFeature; -import com.fasterxml.jackson.core.exc.StreamConstraintsException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.trino.plugin.ducklake.function.currency.ExchangeRateSnapshot.ExchangeRate; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Set; - -import static java.util.Objects.requireNonNull; - -public final class ExchangeRateSnapshotJsonDecoder -{ - public static final int MAXIMUM_PAYLOAD_BYTES = 32 * 1024 * 1024; - - private static final int MAXIMUM_NESTING_DEPTH = 16; - private static final Set SNAPSHOT_FIELDS = Set.of("protocolVersion", "schemaVersion", "generation", "baseCurrency", "decimalScale", "rates"); - private static final Set RATE_FIELDS = Set.of("currency", "effectiveDate", "unscaledRate"); - - private final ObjectMapper objectMapper; - - public ExchangeRateSnapshotJsonDecoder() - { - JsonFactory jsonFactory = JsonFactory.builder() - .streamReadConstraints(StreamReadConstraints.builder() - .maxNestingDepth(MAXIMUM_NESTING_DEPTH) - .maxStringLength(64) - .maxNumberLength(19) - .build()) - .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) - .build(); - objectMapper = new ObjectMapper(jsonFactory); - } - - public ExchangeRateSnapshot decode(byte[] payload, long expectedGeneration) - { - if (payload == null) { - throw failure(DecodeFailure.INVALID_PAYLOAD); - } - if (payload.length > MAXIMUM_PAYLOAD_BYTES) { - throw failure(DecodeFailure.LIMIT_EXCEEDED); - } - if (expectedGeneration <= 0) { - throw failure(DecodeFailure.GENERATION_MISMATCH); - } - - try { - ObjectNode root = parse(payload); - validateFields(root, SNAPSHOT_FIELDS); - - int protocolVersion = integer(root, "protocolVersion"); - if (protocolVersion != ExchangeRateSnapshot.PROTOCOL_VERSION) { - throw failure(DecodeFailure.UNSUPPORTED_PROTOCOL); - } - int schemaVersion = integer(root, "schemaVersion"); - if (schemaVersion != ExchangeRateSnapshot.SCHEMA_VERSION) { - throw failure(DecodeFailure.UNSUPPORTED_SCHEMA); - } - - long generation = generation(root); - if (generation != expectedGeneration) { - throw failure(DecodeFailure.GENERATION_MISMATCH); - } - - return new ExchangeRateSnapshot( - protocolVersion, - schemaVersion, - generation, - text(root, "baseCurrency"), - integer(root, "decimalScale"), - rates(required(root, "rates"))); - } - catch (DecodeException e) { - throw e; - } - catch (StreamConstraintsException e) { - throw failure(DecodeFailure.LIMIT_EXCEEDED); - } - catch (IOException | RuntimeException e) { - throw failure(DecodeFailure.INVALID_PAYLOAD); - } - } - - private ObjectNode parse(byte[] payload) - throws IOException - { - try (JsonParser parser = objectMapper.createParser(payload)) { - JsonNode root = objectMapper.readTree(parser); - if (parser.nextToken() != null) { - throw failure(DecodeFailure.INVALID_PAYLOAD); - } - return object(root); - } - } - - private static List rates(JsonNode node) - { - ArrayNode array = array(node); - if (array.size() > ExchangeRateSnapshot.MAXIMUM_RATES) { - throw failure(DecodeFailure.LIMIT_EXCEEDED); - } - List rates = new ArrayList<>(array.size()); - for (JsonNode element : array) { - ObjectNode rate = object(element); - validateFields(rate, RATE_FIELDS); - rates.add(new ExchangeRate( - text(rate, "currency"), - text(rate, "effectiveDate"), - text(rate, "unscaledRate"))); - } - return List.copyOf(rates); - } - - private static int integer(ObjectNode object, String field) - { - JsonNode node = required(object, field); - if (!node.isIntegralNumber() || !node.canConvertToInt()) { - throw failure(DecodeFailure.INVALID_PAYLOAD); - } - return node.intValue(); - } - - private static long generation(ObjectNode object) - { - JsonNode node = required(object, "generation"); - if (!node.isIntegralNumber() || !node.canConvertToLong() || node.longValue() <= 0) { - throw failure(DecodeFailure.GENERATION_MISMATCH); - } - return node.longValue(); - } - - private static String text(ObjectNode object, String field) - { - JsonNode node = required(object, field); - if (!node.isTextual() || node.textValue().isEmpty()) { - throw failure(DecodeFailure.INVALID_PAYLOAD); - } - return node.textValue(); - } - - private static JsonNode required(ObjectNode object, String field) - { - JsonNode node = object.get(field); - if (node == null || node.isNull()) { - throw failure(DecodeFailure.INVALID_PAYLOAD); - } - return node; - } - - private static ObjectNode object(JsonNode node) - { - if (!(node instanceof ObjectNode object)) { - throw failure(DecodeFailure.INVALID_PAYLOAD); - } - return object; - } - - private static ArrayNode array(JsonNode node) - { - if (!(node instanceof ArrayNode array)) { - throw failure(DecodeFailure.INVALID_PAYLOAD); - } - return array; - } - - private static void validateFields(ObjectNode object, Set fields) - { - Iterator names = object.fieldNames(); - int count = 0; - while (names.hasNext()) { - if (!fields.contains(names.next())) { - throw failure(DecodeFailure.INVALID_PAYLOAD); - } - count++; - } - if (count != fields.size()) { - throw failure(DecodeFailure.INVALID_PAYLOAD); - } - } - - private static DecodeException failure(DecodeFailure failure) - { - return new DecodeException(failure); - } - - public enum DecodeFailure - { - INVALID_PAYLOAD, - LIMIT_EXCEEDED, - UNSUPPORTED_PROTOCOL, - UNSUPPORTED_SCHEMA, - GENERATION_MISMATCH, - } - - public static final class DecodeException - extends IllegalArgumentException - { - private final DecodeFailure failure; - - private DecodeException(DecodeFailure failure) - { - super(requireNonNull(failure, "failure is null").name()); - this.failure = failure; - } - - public DecodeFailure failure() - { - return failure; - } - } -} diff --git a/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateConversionEngine.java b/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateConversionEngine.java deleted file mode 100644 index f2ee0bed4084..000000000000 --- a/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateConversionEngine.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.ducklake.function.currency; - -import io.trino.plugin.ducklake.function.currency.ExchangeRateSnapshot.ExchangeRate; -import org.junit.jupiter.api.Test; - -import java.math.BigDecimal; -import java.time.LocalDate; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -final class TestExchangeRateConversionEngine -{ - private static final long GENERATION = 17; - private static final LocalDate FIRST_DATE = LocalDate.parse("2024-01-01"); - private static final LocalDate SECOND_DATE = LocalDate.parse("2024-01-10"); - - private final ExchangeRateConversionEngine engine = new ExchangeRateConversionEngine(new ExchangeRateSnapshot( - 1, - 1, - GENERATION, - "USD", - 10, - List.of( - rate("EUR", FIRST_DATE, "0.9049"), - rate("EUR", SECOND_DATE, "0.95"), - rate("JPY", FIRST_DATE, "108.1234"), - rate("USD", FIRST_DATE, "1"), - rate("ZZZ", FIRST_DATE, "0")))); - - @Test - void testFloorLookupUsesLatestRateNotAfterDate() - { - assertThat(engine.rate(GENERATION, "EUR", FIRST_DATE.minusDays(1))).isEmpty(); - assertThat(engine.rate(GENERATION, "EUR", FIRST_DATE)).contains(new BigDecimal("0.9049000000")); - assertThat(engine.rate(GENERATION, "EUR", SECOND_DATE.minusDays(1))).contains(new BigDecimal("0.9049000000")); - assertThat(engine.rate(GENERATION, "EUR", SECOND_DATE.plusYears(1))).contains(new BigDecimal("0.9500000000")); - } - - @Test - void testConversionMatchesPostHogDecimalSemantics() - { - assertDecimal(engine.convert(GENERATION, "USD", "EUR", new BigDecimal("100"), FIRST_DATE), "90.4900000000"); - assertDecimal(engine.convert(GENERATION, "EUR", "USD", new BigDecimal("90.49"), FIRST_DATE), "100.0000000000"); - assertDecimal(engine.convert(GENERATION, "AAA", "AAA", new BigDecimal("1.23456789019"), FIRST_DATE), "1.2345678901"); - assertDecimal(engine.convert(GENERATION, "USD", "JPY", new BigDecimal("-2.5"), FIRST_DATE), "-270.3085000000"); - } - - @Test - void testDivisionRoundsToScaleBeforeMultiplication() - { - ExchangeRateConversionEngine roundingEngine = new ExchangeRateConversionEngine(new ExchangeRateSnapshot( - 1, - 1, - GENERATION, - "USD", - 10, - List.of(rate("AAA", FIRST_DATE, "3"), rate("BBB", FIRST_DATE, "2"), rate("USD", FIRST_DATE, "1")))); - - assertDecimal(roundingEngine.convert(GENERATION, "AAA", "BBB", BigDecimal.ONE, FIRST_DATE), "0.6666666666"); - } - - @Test - void testMissingOrZeroRateReturnsZero() - { - assertDecimal(engine.convert(GENERATION, "MISSING", "EUR", BigDecimal.TEN, FIRST_DATE), "0.0000000000"); - assertDecimal(engine.convert(GENERATION, "USD", "MISSING", BigDecimal.TEN, FIRST_DATE), "0.0000000000"); - assertDecimal(engine.convert(GENERATION, "ZZZ", "EUR", BigDecimal.TEN, FIRST_DATE), "0.0000000000"); - assertDecimal(engine.convert(GENERATION, "USD", "ZZZ", BigDecimal.TEN, FIRST_DATE), "0.0000000000"); - assertDecimal(engine.convert(GENERATION, "EUR", "USD", BigDecimal.TEN, FIRST_DATE.minusDays(1)), "0.0000000000"); - } - - @Test - void testRejectsAnotherGeneration() - { - assertThatThrownBy(() -> engine.convert(GENERATION + 1, "USD", "EUR", BigDecimal.TEN, FIRST_DATE)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("exchange-rate snapshot generation mismatch"); - assertThatThrownBy(() -> engine.rate(GENERATION + 1, "EUR", FIRST_DATE)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("exchange-rate snapshot generation mismatch"); - } - - private static ExchangeRate rate(String currency, LocalDate date, String value) - { - return new ExchangeRate(currency, date.toString(), new BigDecimal(value).movePointRight(10).toBigIntegerExact().toString()); - } - - private static void assertDecimal(BigDecimal actual, String expected) - { - assertThat(actual.scale()).isEqualTo(10); - assertThat(actual).isEqualByComparingTo(expected); - } -} diff --git a/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateSnapshotJsonDecoder.java b/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateSnapshotJsonDecoder.java deleted file mode 100644 index 33dd3ebd33eb..000000000000 --- a/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/currency/TestExchangeRateSnapshotJsonDecoder.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.ducklake.function.currency; - -import io.trino.plugin.ducklake.function.currency.ExchangeRateSnapshot.ExchangeRate; -import io.trino.plugin.ducklake.function.currency.ExchangeRateSnapshotJsonDecoder.DecodeFailure; -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -final class TestExchangeRateSnapshotJsonDecoder -{ - @Test - void testDecodesExactGenerationSnapshot() - { - ExchangeRateSnapshot snapshot = new ExchangeRateSnapshotJsonDecoder().decode(bytes(validSnapshot()), 17); - - assertThat(snapshot.protocolVersion()).isEqualTo(1); - assertThat(snapshot.schemaVersion()).isEqualTo(1); - assertThat(snapshot.generation()).isEqualTo(17); - assertThat(snapshot.baseCurrency()).isEqualTo("USD"); - assertThat(snapshot.decimalScale()).isEqualTo(10); - assertThat(snapshot.rates()).containsExactly( - new ExchangeRate("EUR", "2024-01-01", "9049000000"), - new ExchangeRate("EUR", "2024-01-10", "9500000000"), - new ExchangeRate("USD", "2024-01-01", "10000000000")); - } - - @Test - void testSnapshotDefensivelyCopiesRates() - { - List rates = new ArrayList<>(); - rates.add(new ExchangeRate("USD", "2024-01-01", "10000000000")); - ExchangeRateSnapshot snapshot = new ExchangeRateSnapshot(1, 1, 17, "USD", 10, rates); - - rates.clear(); - - assertThat(snapshot.rates()).hasSize(1); - assertThatThrownBy(() -> snapshot.rates().clear()).isInstanceOf(UnsupportedOperationException.class); - } - - @Test - void testRejectsIncompatibleSnapshots() - { - compatibilityFailures().forEach(testCase -> assertDecodeFailure( - testCase.name(), - testCase.payload(), - testCase.expectedGeneration(), - testCase.failure())); - } - - private static List compatibilityFailures() - { - return List.of( - new DecodeFailureCase("protocol", validSnapshot().replace("\"protocolVersion\": 1", "\"protocolVersion\": 2"), 17, DecodeFailure.UNSUPPORTED_PROTOCOL), - new DecodeFailureCase("schema", validSnapshot().replace("\"schemaVersion\": 1", "\"schemaVersion\": 2"), 17, DecodeFailure.UNSUPPORTED_SCHEMA), - new DecodeFailureCase("generation", validSnapshot(), 18, DecodeFailure.GENERATION_MISMATCH), - new DecodeFailureCase("nonpositive expected generation", validSnapshot(), 0, DecodeFailure.GENERATION_MISMATCH), - new DecodeFailureCase("nonpositive snapshot generation", validSnapshot().replace("\"generation\": 17", "\"generation\": 0"), 17, DecodeFailure.GENERATION_MISMATCH)); - } - - @Test - void testRejectsMalformedOrExtendedPayloads() - { - strictFailures().forEach(testCase -> assertDecodeFailure( - testCase.name(), - testCase.payload(), - testCase.expectedGeneration(), - testCase.failure())); - } - - private static List strictFailures() - { - return List.of( - invalid("unknown root field", validSnapshot().replace("\"rates\":", "\"unknown\": true, \"rates\":")), - invalid("missing root field", validSnapshot().replace("\"schemaVersion\": 1,", "")), - invalid("duplicate root field", validSnapshot().replace("\"schemaVersion\": 1,", "\"schemaVersion\": 1, \"schemaVersion\": 1,")), - invalid("unsupported base", validSnapshot().replace("\"baseCurrency\": \"USD\"", "\"baseCurrency\": \"EUR\"")), - invalid("unsupported scale", validSnapshot().replace("\"decimalScale\": 10", "\"decimalScale\": 9")), - invalid("numeric unscaled rate", validSnapshot().replace("\"9049000000\"", "9049000000")), - invalid("negative unscaled rate", validSnapshot().replace("\"9049000000\"", "\"-9049000000\"")), - invalid("leading zero unscaled rate", validSnapshot().replace("\"9049000000\"", "\"09049000000\"")), - invalid("oversized unscaled rate", validSnapshot().replace("\"9049000000\"", "\"1234567890123456789\"")), - invalid("invalid currency", validSnapshot().replace("\"EUR\"", "\"eur\"")), - invalid("invalid date", validSnapshot().replace("2024-01-10", "2024-02-30")), - invalid("duplicate currency date", validSnapshot().replace("2024-01-10", "2024-01-01")), - invalid("unknown rate field", validSnapshot().replace("\"currency\": \"USD\"", "\"currency\": \"USD\", \"unknown\": true")), - invalid("trailing document", validSnapshot() + " {}")); - } - - private static DecodeFailureCase invalid(String name, String payload) - { - return new DecodeFailureCase(name, payload, 17, DecodeFailure.INVALID_PAYLOAD); - } - - private static void assertDecodeFailure(String name, String payload, long expectedGeneration, DecodeFailure failure) - { - assertThatThrownBy(() -> new ExchangeRateSnapshotJsonDecoder().decode(bytes(payload), expectedGeneration)) - .as(name) - .isInstanceOfSatisfying(ExchangeRateSnapshotJsonDecoder.DecodeException.class, exception -> assertThat(exception.failure()).isEqualTo(failure)); - } - - private static byte[] bytes(String value) - { - return value.getBytes(StandardCharsets.UTF_8); - } - - private static String validSnapshot() - { - return """ - { - "protocolVersion": 1, - "schemaVersion": 1, - "generation": 17, - "baseCurrency": "USD", - "decimalScale": 10, - "rates": [ - {"currency": "EUR", "effectiveDate": "2024-01-01", "unscaledRate": "9049000000"}, - {"currency": "EUR", "effectiveDate": "2024-01-10", "unscaledRate": "9500000000"}, - {"currency": "USD", "effectiveDate": "2024-01-01", "unscaledRate": "10000000000"} - ] - } - """; - } - - private record DecodeFailureCase(String name, String payload, long expectedGeneration, DecodeFailure failure) {} -} From edd2a6b2300daebcf3e1527511613d7c3d3814d6 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 03:51:20 -0400 Subject: [PATCH 132/136] feat(hogql): compile catalog-backed corpus semantics --- .../trino/hogql/compiler/HogQlCompiler.java | 2 +- .../hogql/compiler/HogQlFunctionResolver.java | 19 ++++++-- .../hogql/compiler/HogQlProjectionDemand.java | 20 ++++++++ .../hogql/compiler/HogQlSemanticResolver.java | 43 +++++++++-------- .../compiler/HogQlV0ProfileValidator.java | 6 --- .../TestHogQlSemanticEntityExpansion.java | 23 ++++++++++ .../compiler/TestHogQlSemanticResolution.java | 46 +++++++++++++++++++ 7 files changed, 129 insertions(+), 30 deletions(-) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index d825baa7c5bb..c0919d4380ce 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -324,7 +324,7 @@ private static ResolvedQuery resolveQuery( OptionalLong exchangeRateGeneration = OptionalLong.empty(); if (semanticCandidate) { HogQlFunctionResolver.Resolution functionResolution = v0Profile - ? HogQlFunctionResolver.resolveV0(query, exchangeRateSnapshotProvider) + ? HogQlFunctionResolver.resolveV0(query, exchangeRateSnapshotProvider, true) : HogQlFunctionResolver.resolve(pinned, query, exchangeRateSnapshotProvider); exchangeRateGeneration = functionResolution.exchangeRateGeneration(); HogQlQuery functionsResolved = functionResolution.query(); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index b8f496434737..184cead24f17 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -102,14 +102,15 @@ final class HogQlFunctionResolver private HogQlFunctionResolver( Optional snapshot, - Optional exchangeRateSnapshotProvider) + Optional exchangeRateSnapshotProvider, + boolean semanticCatalogAvailable) { Map functions = new LinkedHashMap<>(HogQlV0FunctionRegistry.functions()); snapshot.stream() .flatMap(pinned -> pinned.snapshot().functions().stream()) .forEach(function -> functions.putIfAbsent(canonical(function.name()), function)); this.functions = Map.copyOf(functions); - semanticCatalogAvailable = snapshot.isPresent(); + this.semanticCatalogAvailable = semanticCatalogAvailable; this.exchangeRateSnapshotProvider = requireNonNull(exchangeRateSnapshotProvider, "exchangeRateSnapshotProvider is null"); } @@ -127,7 +128,7 @@ static Resolution resolve( { requireNonNull(snapshot, "snapshot is null"); requireNonNull(query, "query is null"); - return new HogQlFunctionResolver(Optional.of(snapshot), exchangeRateSnapshotProvider).resolveWithMetadata(query); + return new HogQlFunctionResolver(Optional.of(snapshot), exchangeRateSnapshotProvider, true).resolveWithMetadata(query); } public static HogQlQuery resolve(HogQlQuery query) @@ -139,7 +140,7 @@ public static HogQlQuery resolve(HogQlQuery query) static Resolution resolve(HogQlQuery query, Optional exchangeRateSnapshotProvider) { requireNonNull(query, "query is null"); - return new HogQlFunctionResolver(Optional.empty(), exchangeRateSnapshotProvider).resolveWithMetadata(query); + return new HogQlFunctionResolver(Optional.empty(), exchangeRateSnapshotProvider, false).resolveWithMetadata(query); } public static HogQlQuery resolveV0(HogQlQuery query) @@ -149,9 +150,17 @@ public static HogQlQuery resolveV0(HogQlQuery query) } static Resolution resolveV0(HogQlQuery query, Optional exchangeRateSnapshotProvider) + { + return resolveV0(query, exchangeRateSnapshotProvider, false); + } + + static Resolution resolveV0( + HogQlQuery query, + Optional exchangeRateSnapshotProvider, + boolean semanticCatalogAvailable) { requireNonNull(query, "query is null"); - return new HogQlFunctionResolver(Optional.empty(), exchangeRateSnapshotProvider).resolveWithMetadata(query); + return new HogQlFunctionResolver(Optional.empty(), exchangeRateSnapshotProvider, semanticCatalogAvailable).resolveWithMetadata(query); } private Resolution resolveWithMetadata(HogQlQuery query) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java index abfa5091285d..7d5b67ec7090 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java @@ -161,6 +161,16 @@ public static HogQlProjectionDemand collectNonProjection(HogQlQuery query) return builder.build(); } + public static RequiredOutputs collectOrderingOutputs(HogQlQuery query) + { + Builder builder = new Builder(); + if (query.body() instanceof SelectQueryBody select) { + select.limitBy().ifPresent(limitBy -> limitBy.partitionBy().forEach(expression -> collect(expression, builder))); + } + query.orderBy().forEach(sortItem -> collect(sortItem.expression(), builder)); + return builder.build().asRequiredOutputs(); + } + public static HogQlProjectionDemand collect(Expression expression) { Builder builder = new Builder(); @@ -204,6 +214,16 @@ public RequiredOutputs unqualified() return new RequiredOutputs(false, unqualified); } + private RequiredOutputs asRequiredOutputs() + { + if (all || !allQualifiers.isEmpty()) { + return RequiredOutputs.allOutputs(); + } + Set names = new HashSet<>(unqualified); + qualified.values().forEach(names::addAll); + return new RequiredOutputs(false, names); + } + public HogQlProjectionDemand merge(HogQlProjectionDemand other) { requireNonNull(other, "other is null"); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java index 7c1c591b5c36..dc151963eae0 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -465,7 +465,12 @@ yield inferRelationBindings(alias.relation(), availableBindings).stream() .map(definition -> resolveLogicalTable(definition, table.span()).bindings()) .orElse(List.of()) : List.of(); - case SubqueryRelation _, TablePlaceholder _, UnnestRelation _, ValuesRelation _ -> List.of(); + case SubqueryRelation subquery -> { + List outputs = inferQueryOutputs(subquery.query(), List.of(), subquery.span(), availableBindings); + Identifier anonymousName = new Identifier("__hogql_derived_" + subquery.span().startOffset(), true, subquery.span()); + yield List.of(inferredBinding(DERIVED_RELATION_PREFIX, anonymousName, outputs)); + } + case TablePlaceholder _, UnnestRelation _, ValuesRelation _ -> List.of(); }; } @@ -670,8 +675,10 @@ private static void addCommonTableDemand( private HogQlQuery resolveQuery(HogQlQuery query, List commonTables) { + Optional projectionOutputs = requiredOutputs + .map(outputs -> outputs.merge(HogQlProjectionDemand.collectOrderingOutputs(query))); List projections = new ArrayList<>(); - query.projections().forEach(projection -> projections.addAll(resolveProjection(projection))); + query.projections().forEach(projection -> projections.addAll(resolveProjection(projection, projectionOutputs))); if (projections.isEmpty() && requiredOutputs.isPresent()) { projections.add(new ExpressionProjection( new Literal(HogQlQuery.LiteralKind.INTEGER, "1", query.span()), @@ -707,18 +714,18 @@ private HogQlQuery resolveQuery(HogQlQuery query, List co query.span()); } - private List resolveProjection(Projection projection) + private List resolveProjection(Projection projection, Optional projectionOutputs) { return switch (projection) { - case ColumnsList columns -> resolveColumnsList(columns); - case ColumnsRegex columns -> resolveColumnsRegex(columns); - case Star star -> resolveStar(star); + case ColumnsList columns -> resolveColumnsList(columns, projectionOutputs); + case ColumnsRegex columns -> resolveColumnsRegex(columns, projectionOutputs); + case Star star -> resolveStar(star, projectionOutputs); case ExpressionProjection expressionProjection -> { Optional outputName = expressionProjection.alias().map(Identifier::value); if (outputName.isEmpty() && expressionProjection.expression() instanceof ColumnReference reference) { outputName = Optional.of(reference.parts().getLast().value()); } - if (outputName.isPresent() && !projectionDemanded(outputName.orElseThrow())) { + if (outputName.isPresent() && !projectionDemanded(outputName.orElseThrow(), projectionOutputs)) { yield List.of(); } Expression resolved = resolveExpression(expressionProjection.expression()); @@ -731,14 +738,14 @@ private List resolveProjection(Projection projection) }; } - private List resolveColumnsList(ColumnsList columns) + private List resolveColumnsList(ColumnsList columns, Optional projectionOutputs) { return columns.expressions().stream() - .flatMap(expression -> resolveProjection(new ExpressionProjection(expression, Optional.empty())).stream()) + .flatMap(expression -> resolveProjection(new ExpressionProjection(expression, Optional.empty()), projectionOutputs).stream()) .toList(); } - private List resolveColumnsRegex(ColumnsRegex columns) + private List resolveColumnsRegex(ColumnsRegex columns, Optional projectionOutputs) { if (!allRelationsLogical) { throw unsupportedColumns(columns.span()); @@ -754,7 +761,7 @@ private List resolveColumnsRegex(ColumnsRegex columns) .flatMap(binding -> binding.orderedFields().stream() .filter(BoundField::starVisible) .filter(field -> pattern.find(utf8Slice(field.name()))) - .filter(field -> projectionDemanded(field.name())) + .filter(field -> projectionDemanded(field.name(), projectionOutputs)) .map(field -> new ExpressionProjection( resolveBoundField(binding, field, binding.starQualifier(bindings.size()), columns.span(), expansionBudget), Optional.of(new Identifier(field.name(), true, columns.span()))))) @@ -766,9 +773,9 @@ private List resolveColumnsRegex(ColumnsRegex columns) return projections; } - private List resolveStar(Star star) + private List resolveStar(Star star, Optional projectionOutputs) { - Optional> lazyStar = resolveLazyStar(star); + Optional> lazyStar = resolveLazyStar(star, projectionOutputs); if (lazyStar.isPresent()) { return lazyStar.orElseThrow(); } @@ -802,7 +809,7 @@ private List resolveStar(Star star) .flatMap(binding -> binding.orderedFields().stream() .filter(BoundField::starVisible) .filter(field -> !exclusions.contains(new StarField(binding, field))) - .filter(field -> projectionDemanded(field.name())) + .filter(field -> projectionDemanded(field.name(), projectionOutputs)) .map(field -> { StarReplacement replacement = replacements.get(new StarField(binding, field)); Expression expression = replacement == null @@ -819,7 +826,7 @@ private List resolveStar(Star star) .toList(); } - private Optional> resolveLazyStar(Star star) + private Optional> resolveLazyStar(Star star, Optional projectionOutputs) { if (star.qualifier().isEmpty() || star.qualifier().size() > 2) { return Optional.empty(); @@ -880,7 +887,7 @@ private Optional> resolveLazyStar(Star star) return Optional.of(visible.stream() .filter(projection -> !exclusions.contains(canonical(projection.name()))) - .filter(projection -> projectionDemanded(projection.name())) + .filter(projection -> projectionDemanded(projection.name(), projectionOutputs)) .map(projection -> { StarReplacement replacement = replacements.get(canonical(projection.name())); Expression expression = replacement == null @@ -907,9 +914,9 @@ private Expression resolveLazyProjection( expansionBudget); } - private boolean projectionDemanded(String name) + private static boolean projectionDemanded(String name, Optional projectionOutputs) { - return requiredOutputs.map(outputs -> outputs.includes(name)).orElse(true); + return projectionOutputs.map(outputs -> outputs.includes(name)).orElse(true); } private static Map resolveStarReplacements(Star star, List starBindings, Set exclusions) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java index 6d248417f721..a90107fcb563 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java @@ -136,9 +136,6 @@ private static void validateTable(TableReference table, Optional canonical(tableDefinition.name()).equals(name)) || catalog.savedQueries().stream().anyMatch(query -> canonical(query.name()).equals(name)) || catalog.materializedViews().stream().anyMatch(view -> canonical(view.name()).equals(name)); @@ -174,9 +171,6 @@ private static void validate(Expression expression, Optional validate(argument, snapshot)); function.orderBy().forEach(sortItem -> validate(sortItem.expression(), snapshot)); function.filter().ifPresent(filter -> validate(filter, snapshot)); diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticEntityExpansion.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticEntityExpansion.java index 92f1bc60ee91..5745a8c39d1c 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticEntityExpansion.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticEntityExpansion.java @@ -93,6 +93,15 @@ public void testExpandsActionRelationMembership() "WHERE person_id IN (SELECT DISTINCT \"personId\" FROM analytics.data.active_people)")); } + @Test + public void testV0ExpandsActionPredicate() + { + assertThat(compileV0("SELECT event FROM events WHERE matchesAction(17)").statement()) + .isEqualTo(sqlParser.createStatement( + "SELECT event_name AS event FROM analytics.data.raw_events " + + "WHERE raw_events.event_name = CAST('purchase' AS varchar)")); + } + @Test public void testExpandsCohortPredicate() { @@ -150,6 +159,20 @@ private HogQlCompilationResult compile(String query) OptionalLong.of(11)), Optional.of(context)); } + private HogQlCompilationResult compileV0(String query) + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + return compiler.compileV0(new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(11)), Optional.of(context)); + } + private static HogQlSemanticCatalogSnapshot snapshot() { LogicalTableDefinition events = new LogicalTableDefinition( diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java index 5bf81f12b0b2..96091de667d4 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -134,6 +134,52 @@ public void testPinsAndResolvesLogicalTableFieldsAndStar() .containsExactly("event_name", "Person ID"); } + @Test + public void testV0ResolvesDeclaredLogicalTables() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compileV0( + envelope("SELECT * FROM persons", OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT person_id AS \"personId\", full_name AS \"name\" FROM analytics.\"Hog Data\".\"raw-persons\"")); + } + + @Test + public void testKeepsOrderingOutputsNeededByLimitByInPrunedSubquery() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compileV0( + envelope( + "SELECT personId FROM (SELECT * FROM events ORDER BY event LIMIT 1 BY personId) AS nested", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isInstanceOf(Query.class); + assertThat(result.catalogGeneration()).hasValue(7); + } + + @Test + public void testInfersStarsAcrossUnaliasedNestedDerivedTables() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compileV0( + envelope( + "WITH nested AS (SELECT * FROM (SELECT * FROM events)) " + + "SELECT event FROM nested", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "WITH nested AS (SELECT * FROM (" + + "SELECT event_name AS \"event\" FROM analytics.\"Hog Data\".\"raw-events\")) " + + "SELECT \"event\" AS event FROM nested")); + } + @Test public void testExpandsQualifiedLogicalStarsWithExclusionsInManifestOrder() { From 8593ae876da032a18b7e94ab212ef18429a214f1 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 04:18:06 -0400 Subject: [PATCH 133/136] feat(hogql): support typed emptiness checks --- .../hogql/compiler/HogQlFunctionResolver.java | 15 +++-- .../compiler/TestHogQlFunctionResolver.java | 2 +- .../io/trino/hogql/HogQlEmptyFunctions.java | 55 +++++++++++++++++++ .../io/trino/hogql/HogQlFunctionModule.java | 32 +++++++++++ .../io/trino/server/ServerMainModule.java | 4 ++ 5 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlEmptyFunctions.java create mode 100644 core/trino-main/src/main/java/io/trino/hogql/HogQlFunctionModule.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 184cead24f17..310e2b4996fb 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -1221,15 +1221,14 @@ private static Expression has(List arguments, HogQlQuery.SourceSpan private static Expression empty(Expression value, boolean expectedEmpty, HogQlQuery.SourceSpan span) { - Expression length = coalesce( - call("length", List.of(cast(value, "varchar", span)), span), - integerLiteral("0", span), - span); - return new BinaryExpression( - expectedEmpty ? HogQlQuery.BinaryOperator.EQUAL : HogQlQuery.BinaryOperator.GREATER_THAN, - length, - integerLiteral("0", span), + Expression isEmpty = coalesce( + call("hogql_empty", List.of(value), span), + new Literal(HogQlQuery.LiteralKind.BOOLEAN, "true", span), span); + if (expectedEmpty) { + return isEmpty; + } + return new UnaryExpression(HogQlQuery.UnaryOperator.NOT, isEmpty, span); } private static Expression startOfWeek(FunctionCall function, List arguments) diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index b8f0973d0618..f9fbf67679e4 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -458,7 +458,7 @@ public void testCompilerLowersExpressionAndDateAliases() "toLastDayOfMonth(timestamp), anyLast(name), lag(name, 2) OVER (), lagInFrame(name) OVER () FROM records")); assertThat(result.statement()).isEqualTo(sqlParser.createStatement( - "SELECT name, coalesce(length(CAST(name AS varchar)), 0) = 0, coalesce(length(CAST(name AS varchar)), 0) > 0, " + + "SELECT name, coalesce(hogql_empty(name), true), NOT coalesce(hogql_empty(name), true), " + "first = second, amount * 2, amount * rate, amount / rate, contains(ARRAY['a', 'b'], kind), ROW(name, amount), " + "date_add('month', -(2), timestamp), 3 * INTERVAL '1' MONTH, " + "date_add('day', -1, date_trunc('week', date_add('day', 1, timestamp))), split(name, '::'), " + diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlEmptyFunctions.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlEmptyFunctions.java new file mode 100644 index 000000000000..d134af00a5d6 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlEmptyFunctions.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.slice.Slice; +import io.trino.spi.block.Block; +import io.trino.spi.block.SqlMap; +import io.trino.spi.function.ScalarFunction; +import io.trino.spi.function.SqlType; +import io.trino.spi.function.TypeParameter; + +import static io.trino.spi.type.StandardTypes.BOOLEAN; +import static io.trino.spi.type.StandardTypes.VARCHAR; + +public final class HogQlEmptyFunctions +{ + public static final String NAME = "hogql_empty"; + + private HogQlEmptyFunctions() {} + + @ScalarFunction(value = NAME, hidden = true, neverFails = true) + @SqlType(BOOLEAN) + public static boolean varcharIsEmpty(@SqlType(VARCHAR) Slice value) + { + return value.length() == 0; + } + + @ScalarFunction(value = NAME, hidden = true, neverFails = true) + @TypeParameter("E") + @SqlType(BOOLEAN) + public static boolean arrayIsEmpty(@SqlType("array(E)") Block value) + { + return value.getPositionCount() == 0; + } + + @ScalarFunction(value = NAME, hidden = true, neverFails = true) + @TypeParameter("K") + @TypeParameter("V") + @SqlType(BOOLEAN) + public static boolean mapIsEmpty(@SqlType("map(K,V)") SqlMap value) + { + return value.getSize() == 0; + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlFunctionModule.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlFunctionModule.java new file mode 100644 index 000000000000..44098eccaadb --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlFunctionModule.java @@ -0,0 +1,32 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.AbstractModule; +import com.google.inject.Provides; +import com.google.inject.Singleton; +import com.google.inject.multibindings.ProvidesIntoSet; +import io.trino.metadata.InternalFunctionBundle; +import io.trino.spi.function.FunctionBundle; + +public final class HogQlFunctionModule + extends AbstractModule +{ + @ProvidesIntoSet + @Singleton + public static FunctionBundle provideFunctionBundle() + { + return InternalFunctionBundle.extractFunctions(HogQlEmptyFunctions.class); + } +} diff --git a/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java b/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java index 00577f6a7872..ba190e9a4e52 100644 --- a/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java +++ b/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java @@ -58,6 +58,7 @@ import io.trino.execution.scheduler.NodeSchedulerConfig; import io.trino.hogql.HogQlConfig; import io.trino.hogql.HogQlExchangeRateModule; +import io.trino.hogql.HogQlFunctionModule; import io.trino.hogql.HogQlSemanticCatalogConfig; import io.trino.jsonpath.ir.IrJsonPath; import io.trino.memory.LocalMemoryManager; @@ -205,6 +206,9 @@ protected void setup(Binder binder) configBinder(binder).bindConfig(HogQlConfig.class); configBinder(binder).bindConfig(HogQlSemanticCatalogConfig.class); + if (buildConfigObject(HogQlConfig.class).isEnabled()) { + install(new HogQlFunctionModule()); + } if (buildConfigObject(HogQlSemanticCatalogConfig.class).getUri() != null) { install(new HogQlExchangeRateModule()); } From 6fa2666d4047f70fc21fe9bd9c2db661c6175047 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 04:18:25 -0400 Subject: [PATCH 134/136] test(hogql): validate corpus execution compatibility --- core/trino-hogql-compiler/README.md | 11 +- .../TestHogQlStatementResource.java | 121 +++++++++++++++++- .../planner/TestHogQlProjectionPruning.java | 39 +++++- docs/src/main/sphinx/admin/hogql.md | 30 +++-- 4 files changed, 182 insertions(+), 19 deletions(-) diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md index 1955e486fbcf..b4ecdd2739bc 100644 --- a/core/trino-hogql-compiler/README.md +++ b/core/trino-hogql-compiler/README.md @@ -43,7 +43,7 @@ explicit error for valid constructs that do not yet have a stock Trino AST mapping. Adding syntax support therefore cannot silently claim semantic or execution parity. -## MVP executable contract +## Executable contract The MVP lowers a frozen, fail-closed subset directly to stock Trino AST nodes: @@ -113,9 +113,12 @@ Algorithm-level parity for `quantileExact` remains a compatibility-validation item even though its query shape has a stock AST lowering. Function aliases are resolved before Trino analysis. Unknown functions fail as -HogQL resolution errors. Actions, cohorts, saved queries, explicit modifiers, -HogQLX, PIVOT/UNPIVOT execution, ClickHouse-only clauses, advanced lambda and -table functions, and complete type/function parity remain outside the MVP. +HogQL resolution errors. The corpus compatibility extension also accepts every +logical table declared by the pinned manifest and expands declarative actions +by name or ID into stock predicates or relation membership. Cohorts, saved +queries, explicit modifiers without an enabled behavior, HogQLX, +PIVOT/UNPIVOT execution, unsupported ClickHouse clauses, and complete +type/function parity remain outside the endpoint profile. Set `hogql.enabled=true` on the coordinator to register `POST /v1/hogql`. The request body is the versioned JSON envelope containing the raw HogQL query, diff --git a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java index 3d4fdc15dc70..3f583dd6b695 100644 --- a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java +++ b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java @@ -237,7 +237,7 @@ private static Stream differentialQueries() "SELECT value FROM (VALUES (2), (1), (2)) AS numbers(value) UNION SELECT 3 ORDER BY value"), Arguments.of( "arrays, maps, and rows", - "SELECT ARRAY[1, 2, 3][2] AS array_value, MAP(ARRAY['k'], ARRAY[7])['k'] AS map_value, (11, 'x').1 AS row_value", + "SELECT ARRAY[1, 2, 3][2] AS array_value, mapFromArrays(['k'], [7])['k'] AS map_value, (11, 'x').1 AS row_value", "SELECT ARRAY[1, 2, 3][2] AS array_value, MAP(ARRAY['k'], ARRAY[7])['k'] AS map_value, ROW(11, 'x')[1] AS row_value"), Arguments.of( "timestamp precision", @@ -274,7 +274,122 @@ private static Stream differentialQueries() Arguments.of( "v0 value window function", "SELECT first_value(name) OVER (ORDER BY nationkey) FROM tpch.tiny.nation WHERE nationkey < 3 ORDER BY nationkey", - "SELECT first_value(name) OVER (ORDER BY nationkey) FROM tpch.tiny.nation WHERE nationkey < 3 ORDER BY nationkey")); + "SELECT first_value(name) OVER (ORDER BY nationkey) FROM tpch.tiny.nation WHERE nationkey < 3 ORDER BY nationkey"), + Arguments.of( + "corpus temporal rewrites", + "SELECT addDays(CAST('2024-01-15 12:34:56' AS Timestamp), 2), " + + "subtractDays(CAST('2024-01-15 12:34:56' AS Timestamp), 3), " + + "addMonths(CAST('2024-01-15 12:34:56' AS Timestamp), 1), " + + "subtractMonths(CAST('2024-01-15 12:34:56' AS Timestamp), 2), " + + "subtractYears(CAST('2024-01-15 12:34:56' AS Timestamp), 1), " + + "toStartOfDay(CAST('2024-01-15 12:34:56' AS Timestamp)), " + + "toStartOfHour(CAST('2024-01-15 12:34:56' AS Timestamp)), " + + "toStartOfMonth(CAST('2024-01-15 12:34:56' AS Timestamp)), " + + "toStartOfWeek(CAST('2024-01-15 12:34:56' AS Timestamp), 1), " + + "toDayOfMonth(CAST('2024-01-15' AS Date)), toDayOfWeek(CAST('2024-01-15' AS Date)), " + + "toMonth(CAST('2024-01-15' AS Date)), toYear(CAST('2024-01-15' AS Date)), " + + "toLastDayOfMonth(CAST('2024-01-15' AS Date)), " + + "formatDateTime(CAST('2024-01-15 12:34:56' AS Timestamp), '%Y-%m-%d'), " + + "parseDateTime('2024-01-15', '%Y-%m-%d'), parseDateTimeBestEffort('2024-01-15 12:34:56')", + "SELECT date_add('day', 2, CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_add('day', -3, CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_add('month', 1, CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_add('month', -2, CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_add('year', -1, CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_trunc('day', CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_trunc('hour', CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_trunc('month', CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_trunc('week', CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "day(CAST('2024-01-15' AS date)), day_of_week(CAST('2024-01-15' AS date)), " + + "month(CAST('2024-01-15' AS date)), year(CAST('2024-01-15' AS date)), " + + "last_day_of_month(CAST('2024-01-15' AS date)), " + + "date_format(CAST('2024-01-15 12:34:56' AS timestamp(0)), '%Y-%m-%d'), " + + "date_parse('2024-01-15', '%Y-%m-%d'), TRY_CAST('2024-01-15 12:34:56' AS timestamp(3))"), + Arguments.of( + "corpus JSON regex and string rewrites", + "SELECT JSONExtractString(payload, 'name'), JSONExtractInt(payload, 'items', 0), " + + "JSONExtractFloat(payload, 'score'), JSONExtractBool(payload, 'active'), " + + "JSONExtractUInt(payload, 'items', 1), JSONExtractRaw(payload, 'object'), " + + "JSONLength(payload, 'items'), JSONHas(payload, 'object'), JSONExtractKeys(payload, 'object'), " + + "extract(sample, '([a-z]+)'), extractAll(sample, '([a-z]+)'), match(sample, '^[a-z]+'), " + + "replaceRegexpAll(sample, '[0-9]', 'x'), replaceRegexpOne(sample, '[0-9]+', 'x'), " + + "splitByString('123', sample), substringUTF8(sample, 2, 3), position(sample, '123') " + + "FROM (VALUES ('{\"name\":\"Ada\",\"items\":[2,3],\"active\":true,\"score\":1.5,\"object\":{\"k\":7}}', 'abc123abc')) AS t(payload, sample)", + "SELECT coalesce(json_extract_scalar(payload, '$[\"name\"]'), ''), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"items\"][0]') AS bigint), 0), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"score\"]') AS double), 0E0), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"active\"]') AS boolean), false), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"items\"][1]') AS bigint), 0), " + + "coalesce(json_format(json_extract(payload, '$[\"object\"]')), ''), " + + "coalesce(json_size(payload, '$[\"items\"]'), 0), json_extract(payload, '$[\"object\"]') IS NOT NULL, " + + "map_keys(coalesce(TRY_CAST(json_extract(payload, '$[\"object\"]') AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))), " + + "coalesce(regexp_extract(sample, '([a-z]+)', 1), ''), regexp_extract_all(sample, '([a-z]+)', 1), " + + "regexp_like(sample, '^[a-z]+'), regexp_replace(sample, '[0-9]', 'x'), " + + "regexp_replace(sample, '(?s)^(.*?)(([0-9]+))', '$1x'), split(sample, '123'), substring(sample, 2, 3), strpos(sample, '123') " + + "FROM (VALUES ('{\"name\":\"Ada\",\"items\":[2,3],\"active\":true,\"score\":1.5,\"object\":{\"k\":7}}', 'abc123abc')) AS t(payload, sample)"), + Arguments.of( + "corpus collection and lambda rewrites", + "SELECT arrayElement([1, 2, 3], -1), arrayFilter(x -> x > 1, [1, 2, 3]), " + + "arrayFirst(x -> x > 1, [1, 2, 3]), arrayMap(x -> x + 1, [1, 2, 3]), " + + "arraySum([1, 2, 3]), arrayMin([3, 1, 2]), arraySlice([1, 2, 3, 4], 2, 2), " + + "arrayEnumerate(['a', 'b']), range(2, 5), tupleElement(tuple('x', 7), 2), " + + "splitByChar(',', 'a,b'), has([1, 2], 2), hasAny([1, 2], [2, 3]), " + + "map('a', 1, 'b', 2)['b'], mapUpdate(map('a', 1), map('a', 2))['a']", + "SELECT element_at(ARRAY[1, 2, 3], -1), filter(ARRAY[1, 2, 3], x -> x > 1), " + + "element_at(filter(ARRAY[1, 2, 3], x -> x > 1), 1), transform(ARRAY[1, 2, 3], x -> x + 1), " + + "reduce(ARRAY[1, 2, 3], 0, (total, item) -> total + item, total -> total), " + + "array_min(ARRAY[3, 1, 2]), slice(ARRAY[1, 2, 3, 4], 2, 2), sequence(1, cardinality(ARRAY['a', 'b'])), " + + "sequence(2, 5 - 1), ROW('x', 7)[2], split('a,b', ','), " + + "coalesce(contains(ARRAY[1, 2], 2), false), arrays_overlap(ARRAY[1, 2], ARRAY[2, 3]), " + + "map(ARRAY['a', 'b'], ARRAY[1, 2])['b'], map_concat(map(ARRAY['a'], ARRAY[1]), map(ARRAY['a'], ARRAY[2]))['a']"), + Arguments.of( + "corpus arrayJoin lowering", + "SELECT id, arrayJoin(values_array) AS value " + + "FROM (VALUES (2, [3, 1]), (1, [2])) AS t(id, values_array) ORDER BY id, value", + "SELECT id, value FROM (VALUES (2, ARRAY[3, 1]), (1, ARRAY[2])) AS t(id, values_array) " + + "CROSS JOIN UNNEST(values_array) AS u(value) ORDER BY id, value"), + Arguments.of( + "corpus nested LIMIT BY lowering", + "SELECT nested.* FROM (" + + "SELECT category, value FROM (VALUES ('a', 2), ('a', 1), ('b', 4), ('b', 3)) AS t(category, value) " + + "ORDER BY value DESC LIMIT 1 BY category) AS nested ORDER BY category", + "SELECT category, value FROM (" + + "SELECT category, value, row_number() OVER (PARTITION BY category ORDER BY value DESC) AS row_number " + + "FROM (VALUES ('a', 2), ('a', 1), ('b', 4), ('b', 3)) AS t(category, value)) " + + "WHERE row_number <= 1 ORDER BY category"), + Arguments.of( + "corpus aggregate rewrites", + "SELECT countIf(active), sumIf(value, active), minIf(value, active), maxIf(value, active), " + + "avgIf(value, active), uniqExactIf(value, active), countDistinct(value), " + + "argMaxIf(value, weight, active), argMinIf(value, weight, active), " + + "quantile(0.5)(value), quantileIf(0.5)(value, active) " + + "FROM (VALUES (1, true, 10), (1, false, 20), (2, true, 30), (3, true, 40)) AS t(value, active, weight)", + "SELECT count(*) FILTER (WHERE active), sum(value) FILTER (WHERE active), min(value) FILTER (WHERE active), " + + "max(value) FILTER (WHERE active), avg(value) FILTER (WHERE active), " + + "count(DISTINCT value) FILTER (WHERE active), count(DISTINCT value), " + + "max_by(value, weight) FILTER (WHERE active), min_by(value, weight) FILTER (WHERE active), " + + "approx_percentile(value, 0.5), approx_percentile(value, 0.5) FILTER (WHERE active) " + + "FROM (VALUES (1, true, 10), (1, false, 20), (2, true, 30), (3, true, 40)) AS t(value, active, weight)"), + Arguments.of( + "corpus numeric conversion and operator rewrites", + "SELECT toFloatOrZero('bad'), toFloatOrDefault('bad', 1), toDecimal('1.25', 2), intDiv(-5, 2), " + + "toInt('4'), toIntOrZero('bad'), _toInt16('3'), " + + "toUUID('00000000-0000-0000-0000-000000000001'), roundBankers(1.25, 1), " + + "plus(4, 2), minus(4, 2), multiply(4, 2), divide(5, 2), " + + "greater(4, 2), greaterOrEquals(4, 4), lessOrEquals(2, 4), notEquals(1, 2), " + + "and(true, true, false), or(false, false, true), not(false), empty(''), notEmpty([1]), " + + "empty(mapFromArrays(['key'], [1])), empty(CAST(NULL AS Array(Int64)))", + "SELECT coalesce(TRY_CAST('bad' AS double), 0E0), coalesce(TRY_CAST('bad' AS double), CAST(1 AS double)), " + + "TRY_CAST('1.25' AS decimal(18, 2)), " + + "CAST(-5 AS bigint) / CAST(2 AS bigint) - if(CAST(-5 AS bigint) % CAST(2 AS bigint) <> 0 AND " + + "(CAST(-5 AS bigint) < 0 AND CAST(2 AS bigint) > 0 OR CAST(-5 AS bigint) > 0 AND CAST(2 AS bigint) < 0), 1, 0), " + + "CAST('4' AS bigint), coalesce(TRY_CAST('bad' AS bigint), 0), CAST('3' AS smallint), " + + "TRY_CAST('00000000-0000-0000-0000-000000000001' AS uuid), round(DOUBLE '1.25', 1), " + + "4 + 2, 4 - 2, 4 * 2, 5 / 2, 4 > 2, 4 >= 4, 2 <= 4, 1 <> 2, " + + "(true AND true) AND false, (false OR false) OR true, NOT false, " + + "coalesce(length(CAST('' AS varchar)), 0) = 0, coalesce(cardinality(ARRAY[1]), 0) > 0, " + + "coalesce(cardinality(map(ARRAY['key'], ARRAY[1])), 0) = 0, " + + "coalesce(cardinality(CAST(NULL AS array(bigint))), 0) = 0")); } @Test @@ -393,7 +508,7 @@ public void testEmptyResultsAndCompilerDiagnostics() List invalidResults = runHogQlToCompletion(hogQlRequest("SELECT (")); assertThat(invalidResults.getLast().getError().getErrorName()).isEqualTo("HOGQL_SYNTAX_ERROR"); - List unsupportedFunction = runHogQlToCompletion(hogQlRequest("SELECT uniq(1)")); + List unsupportedFunction = runHogQlToCompletion(hogQlRequest("SELECT unsupportedHogQlFunction(1)")); assertThat(unsupportedFunction.getLast().getError().getErrorName()).isEqualTo("HOGQL_RESOLUTION_ERROR"); List pivot = runHogQlToCompletion(hogQlRequest( diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java index c67c473c669f..0dff9426030b 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java @@ -19,6 +19,7 @@ import io.trino.hogql.compiler.HogQlCompiler; import io.trino.hogql.compiler.HogQlSemanticCatalogContext; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ActionReference; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionArgument; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; @@ -28,14 +29,18 @@ import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PredicateRepresentation; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; import io.trino.hogql.parser.HogQlLanguageContract; import io.trino.sql.SqlFormatter; @@ -162,6 +167,15 @@ public void testVarcharJsonPropertyPredicateReachesStockPlanner() assertThat(nodes(plan, FilterNode.class)).hasSize(1); } + @Test + public void testV0ActionPredicateReachesStockPlanner() + { + Plan plan = plan(compileV0("SELECT event FROM events WHERE matchesAction(42)")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(1); + assertThat(nodes(plan, FilterNode.class)).hasSize(1); + } + @ParameterizedTest @ValueSource(strings = {"UNION ALL", "INTERSECT ALL", "EXCEPT ALL"}) public void testSetOperationDemandIsMappedByBranchPosition(String operator) @@ -199,6 +213,21 @@ private String compile(String query) return SqlFormatter.formatSql(result.statement()); } + private String compileV0(String query) + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + HogQlCompilationResult result = compiler.compileV0(new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(1)), Optional.of(context)); + return SqlFormatter.formatSql(result.statement()); + } + private static List nodes(Plan plan, Class type) { return nodes(plan.getRoot(), type); @@ -273,7 +302,15 @@ static HogQlSemanticCatalogSnapshot testingSnapshot() List.of(), List.of(), List.of(personProfile), - List.of(), + List.of(new ActionReference( + "Synthetic order", + "42", + "events", + new PredicateRepresentation(new OperatorRecipe( + SemanticOperator.EQUAL, + List.of( + new FieldReferenceRecipe("events", "event"), + new LiteralRecipe(new TypedLiteral("bigint", LiteralEncoding.INTEGER, "1"))))))), List.of()); } diff --git a/docs/src/main/sphinx/admin/hogql.md b/docs/src/main/sphinx/admin/hogql.md index 20c33693394b..f772ff7886e0 100644 --- a/docs/src/main/sphinx/admin/hogql.md +++ b/docs/src/main/sphinx/admin/hogql.md @@ -16,11 +16,11 @@ Compilation runs on a dedicated fixed-size executor. Configure `hogql.compilation-timeout`; saturation and timeout fail with retryable insufficient-resource errors without affecting standard SQL submission. -The MVP profile supports read-only queries over the logical `events` and -`persons` tables, scalar reads from their VARCHAR JSON `properties` columns, -and the lazy `events.person` relationship. It also supports the frozen -fail-closed function set documented below. Physical tables remain available -through normal Trino names. +The executable profile supports read-only queries over logical tables declared +by the pinned manifest, including ordered star expansion, properties, lazy +relationships, and declarative actions. Physical tables remain available +through normal Trino names. Unsupported functions and semantic definitions +fail before Trino analysis. The endpoint accepts a JSON request envelope. Identity, catalog, schema, time zone, source, tags, and allowed session properties continue to use the standard @@ -135,10 +135,9 @@ Clients can set `catalogGeneration` in the request envelope to require an exact immutable generation. Exact-generation cache entries never fall back to the latest manifest. A missing, expired, or mismatched generation fails closed. -## MVP function profile +## Function profile -The compiler accepts only the following unqualified function names. Unknown -names fail before Trino analysis. +The original MVP function set remains supported: | Family | HogQL names | | --- | --- | @@ -148,9 +147,18 @@ names fail before Trino analysis. | Aggregates | `count`, `sum`, `min`, `max`, `avg`, `any`, `argMin`, `argMax`, `array_agg` | | Windows | `first_value`, `rank`, `row_number` | -The MVP intentionally excludes actions, cohorts, saved queries, explicit -modifiers, HogQLX, PIVOT/UNPIVOT execution, ClickHouse-only clauses, advanced -lambda and table functions, and complete HogQL type/function parity. These +The Team 2 materialized-view compatibility extension adds the exact scalar, +aggregate, JSON, collection, date/time, regular-expression, conversion, and +window rewrites exercised by its frozen query corpus. The in-tree +`HogQlV0FunctionRegistry` is the source of truth for accepted names, arities, +determinism, and lowering strategy. Catalog function declarations do not +silently broaden this fail-closed profile. + +Declarative actions resolve by name or ID from the pinned manifest and lower +to optimizer-visible predicates or relation membership. Cohorts, saved +queries, explicit modifiers without an enabled behavior, HogQLX, +PIVOT/UNPIVOT execution, unsupported ClickHouse clauses, and complete +type/function parity remain outside this endpoint profile. Unsupported constructs return typed compatibility errors; they do not fall through as Trino functions. From 4669d750bcfc3214de22b4964e27639bd3d831a4 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 04:35:00 -0400 Subject: [PATCH 135/136] fix(hogql): align semantic catalog payload limit --- .../main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java | 4 ++-- .../java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java index 09a63366a40a..639db487dce4 100644 --- a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java @@ -26,7 +26,7 @@ import java.net.URI; -import static io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES; +import static io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; @@ -172,7 +172,7 @@ public HogQlSemanticCatalogConfig setRequestTimeout(Duration requestTimeout) @NotNull @MinDataSize("1B") - @MaxDataSize("32MB") + @MaxDataSize("8MB") public DataSize getMaximumResponseSize() { return maximumResponseSize; diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java index 9613fea8c08a..3d5e647394bf 100644 --- a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java @@ -44,7 +44,7 @@ public void testDefaults() .setLoaderThreads(4) .setLoaderQueueCapacity(64) .setRequestTimeout(new Duration(10, SECONDS)) - .setMaximumResponseSize(DataSize.of(32, MEGABYTE))); + .setMaximumResponseSize(DataSize.of(8, MEGABYTE))); } @Test From ae63d0292d6a6f64d2f0f26763561b209a06a11f Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Fri, 28 Aug 2026 06:46:06 -0400 Subject: [PATCH 136/136] fix(hogql): close corpus analyzer compatibility gaps --- .../trino/hogql/compiler/HogQlCompiler.java | 21 +- .../hogql/compiler/HogQlFunctionResolver.java | 16 +- .../compiler/HogQlSelectAliasRewriter.java | 360 ++++++++++++++++++ .../hogql/compiler/TestHogQlCompiler.java | 35 +- .../compiler/TestHogQlFunctionResolver.java | 10 + .../io/trino/hogql/parser/HogQlParser.java | 38 +- .../io/trino/execution/QueryPreparer.java | 6 + .../io/trino/sql/rewrite/ExplainRewrite.java | 9 +- .../TestHogQlStatementResource.java | 51 +++ 9 files changed, 528 insertions(+), 18 deletions(-) create mode 100644 core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSelectAliasRewriter.java diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java index c0919d4380ce..82b57b3f383c 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -233,21 +233,24 @@ private static HogQlCompilationResult compile( throw bindingError(query.span(), "Unused HogQL parameter bindings: " + String.join(", ", extra)); } - Map parameterIds = new HashMap<>(); - for (int index = 0; index < placeholders.size(); index++) { - parameterIds.put(placeholders.get(index).span(), index); - } if (v0Profile) { HogQlV0ProfileValidator.validate(query, Optional.empty()); } ResolvedQuery resolved = resolveQuery(query, catalogContext, languageVersion, expectedCatalogGeneration, !modifiers.isEmpty(), v0Profile, exchangeRateSnapshotProvider); + List resolvedPlaceholders = new ArrayList<>(); + collectPlaceholders(resolved.query(), resolvedPlaceholders); + resolvedPlaceholders.sort(Comparator.comparingInt(placeholder -> placeholder.span().startOffset())); + Map parameterIds = new HashMap<>(); + for (int index = 0; index < resolvedPlaceholders.size(); index++) { + parameterIds.put(resolvedPlaceholders.get(index).span(), index); + } Statement statement = TrinoAstFactory.createStatement(resolved.query(), parameterIds); List modifierBindings = resolved.pinnedSnapshot() .map(snapshot -> HogQlModifierResolver.resolve(snapshot, modifiers, query.span())) .orElseGet(List::of); return new HogQlCompilationResult( statement, - placeholders.stream() + resolvedPlaceholders.stream() .map(Placeholder::name) .toList(), modifierBindings, @@ -300,7 +303,7 @@ private static ResolvedQuery resolveQuery( { boolean semanticCandidate = containsSemanticCandidate(query); if (!semanticCandidate && !modifiersRequireSnapshot) { - return new ResolvedQuery(query, Optional.empty(), OptionalLong.empty()); + return new ResolvedQuery(HogQlSelectAliasRewriter.rewrite(query), Optional.empty(), OptionalLong.empty()); } if (catalogContext.isEmpty()) { if (modifiersRequireSnapshot) { @@ -310,7 +313,7 @@ private static ResolvedQuery resolveQuery( return new ResolvedQuery(query, Optional.empty(), OptionalLong.empty()); } HogQlFunctionResolver.Resolution resolution = HogQlFunctionResolver.resolve(query, exchangeRateSnapshotProvider); - return new ResolvedQuery(resolution.query(), Optional.empty(), resolution.exchangeRateGeneration()); + return new ResolvedQuery(HogQlSelectAliasRewriter.rewrite(resolution.query()), Optional.empty(), resolution.exchangeRateGeneration()); } HogQlSemanticCatalogContext context = catalogContext.orElseThrow(); PinnedSnapshot pinned = context.snapshotProvider().pin(new PinRequest( @@ -320,14 +323,14 @@ private static ResolvedQuery resolveQuery( if (v0Profile) { HogQlV0ProfileValidator.validate(query, Optional.of(pinned.snapshot())); } - HogQlQuery resolved = query; + HogQlQuery resolved = HogQlSelectAliasRewriter.rewrite(query); OptionalLong exchangeRateGeneration = OptionalLong.empty(); if (semanticCandidate) { HogQlFunctionResolver.Resolution functionResolution = v0Profile ? HogQlFunctionResolver.resolveV0(query, exchangeRateSnapshotProvider, true) : HogQlFunctionResolver.resolve(pinned, query, exchangeRateSnapshotProvider); exchangeRateGeneration = functionResolution.exchangeRateGeneration(); - HogQlQuery functionsResolved = functionResolution.query(); + HogQlQuery functionsResolved = HogQlSelectAliasRewriter.rewrite(functionResolution.query()); if (hasSemanticDefinitions(pinned.snapshot())) { resolved = HogQlSemanticResolver.resolve(pinned, functionsResolved) .map(HogQlSemanticResolver.ResolvedQuery::query) diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java index 310e2b4996fb..d337e978257d 100644 --- a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -511,7 +511,7 @@ private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List< case INTERVAL_MONTH -> new IntervalExpression(arguments.getFirst(), HogQlQuery.IntervalUnit.MONTH, span); case START_WEEK -> startOfWeek(function, arguments); case SPLIT_STRING -> call("split", List.of(arguments.get(1), arguments.getFirst()), span); - case CAST_BIGINT -> cast(arguments.getFirst(), "bigint", span); + case CAST_BIGINT -> castBigint(arguments.getFirst(), span); case CAST_TIMESTAMP -> arguments.size() == 1 ? cast(arguments.getFirst(), "timestamp(0)", span) : call("with_timezone", List.of(cast(arguments.getFirst(), "timestamp(0)", span), arguments.get(1)), span); @@ -1013,6 +1013,20 @@ private static FunctionCall call(String name, List arguments, HogQlQ span); } + private static Expression castBigint(Expression value, HogQlQuery.SourceSpan span) + { + if (value instanceof CastExpression cast && canonical(cast.type().value()).equals("date")) { + return call( + "date_diff", + List.of( + stringLiteral("day", span), + cast(stringLiteral("1970-01-01", span), "date", span), + value), + span); + } + return cast(value, "bigint", span); + } + private static CastExpression cast(Expression value, String type, HogQlQuery.SourceSpan span) { return new CastExpression(value, new Identifier(type, false, span), false, span); diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSelectAliasRewriter.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSelectAliasRewriter.java new file mode 100644 index 000000000000..1ce4ced56d71 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSelectAliasRewriter.java @@ -0,0 +1,360 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseWhen; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; +import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FrameBound; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; +import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; +import io.trino.hogql.parser.tree.HogQlQuery.LimitBy; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.PivotAggregation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotValueGroup; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; +import io.trino.hogql.parser.tree.HogQlQuery.SortItem; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.StarReplacement; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; +import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; +import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; +import io.trino.hogql.parser.tree.HogQlQuery.WindowFrame; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +final class HogQlSelectAliasRewriter +{ + private HogQlSelectAliasRewriter() {} + + public static HogQlQuery rewrite(HogQlQuery query) + { + List commonTables = query.with().stream() + .map(commonTable -> new CommonTableExpression( + commonTable.name(), + commonTable.columnAliases(), + rewrite(commonTable.query()), + commonTable.span())) + .toList(); + return switch (query.body()) { + case SelectQueryBody select -> rewriteSelect(query, commonTables, select); + case SetOperation set -> new HogQlQuery( + commonTables, + new SetOperation( + set.type(), + set.distinct(), + rewrite(set.left()), + rewrite(set.right()), + set.leftParenthesized(), + set.rightParenthesized(), + set.operatorSpan(), + set.span()), + query.orderBy().stream().map(item -> rewriteSortItem(item, Map.of())).toList(), + query.limit().map(expression -> rewriteExpression(expression, Map.of())), + query.offset().map(expression -> rewriteExpression(expression, Map.of())), + query.span()); + }; + } + + private static HogQlQuery rewriteSelect(HogQlQuery query, List commonTables, SelectQueryBody select) + { + Map aliases = new HashMap<>(); + Set ambiguous = new HashSet<>(); + List projections = select.projections().stream() + .map(projection -> rewriteProjection(projection, aliases, ambiguous)) + .toList(); + Map visibleAliases = Map.copyOf(aliases); + return new HogQlQuery( + commonTables, + new SelectQueryBody( + select.distinct(), + projections, + select.from().map(HogQlSelectAliasRewriter::rewriteRelation), + select.where().map(expression -> rewriteExpression(expression, visibleAliases)), + select.groupBy().stream().map(expression -> rewriteExpression(expression, visibleAliases)).toList(), + select.having().map(expression -> rewriteExpression(expression, visibleAliases)), + select.windows().stream().map(window -> rewriteWindowDefinition(window, visibleAliases)).toList(), + select.limitBy().map(limitBy -> new LimitBy( + rewriteExpression(limitBy.limit(), visibleAliases), + limitBy.offset().map(expression -> rewriteExpression(expression, visibleAliases)), + limitBy.partitionBy().stream().map(expression -> rewriteExpression(expression, visibleAliases)).toList(), + limitBy.span())), + select.span()), + query.orderBy().stream().map(item -> rewriteSortItem(item, visibleAliases)).toList(), + query.limit().map(expression -> rewriteExpression(expression, visibleAliases)), + query.offset().map(expression -> rewriteExpression(expression, visibleAliases)), + query.span()); + } + + private static Projection rewriteProjection(Projection projection, Map aliases, Set ambiguous) + { + return switch (projection) { + case ColumnsList columns -> new ColumnsList( + columns.expressions().stream().map(expression -> rewriteExpression(expression, aliases)).toList(), + columns.span()); + case ColumnsRegex columns -> columns; + case ExpressionProjection expression -> { + Expression rewritten = rewriteExpression(expression.expression(), aliases); + expression.alias().ifPresent(alias -> registerAlias(alias, rewritten, aliases, ambiguous)); + yield new ExpressionProjection(rewritten, expression.alias()); + } + case Star star -> new Star( + star.qualifier(), + star.exclusions(), + star.replacements().stream() + .map(replacement -> new StarReplacement( + rewriteExpression(replacement.expression(), aliases), + replacement.target(), + replacement.span())) + .toList(), + star.span()); + }; + } + + private static void registerAlias(Identifier alias, Expression expression, Map aliases, Set ambiguous) + { + AliasKey key = AliasKey.of(alias); + if (!ambiguous.add(key)) { + aliases.remove(key); + return; + } + aliases.put(key, expression); + } + + private static Relation rewriteRelation(Relation relation) + { + return switch (relation) { + case AliasedRelation alias -> new AliasedRelation(rewriteRelation(alias.relation()), alias.alias(), alias.columnAliases(), alias.span()); + case CommonTableReference commonTable -> commonTable; + case JoinRelation join -> new JoinRelation( + join.type(), + rewriteRelation(join.left()), + rewriteRelation(join.right()), + join.criteria().map(criteria -> switch (criteria) { + case JoinOn on -> new JoinOn(rewriteExpression(on.expression(), Map.of()), on.span()); + case JoinUsing using -> using; + }), + join.span()); + case PivotRelation pivot -> new PivotRelation( + rewriteRelation(pivot.input()), + pivot.aggregations().stream() + .map(aggregation -> new PivotAggregation( + rewriteExpression(aggregation.expression(), Map.of()), + aggregation.alias(), + aggregation.span())) + .toList(), + pivot.pivotColumns().stream().map(expression -> rewriteExpression(expression, Map.of())).toList(), + pivot.valueGroups().stream() + .map(group -> new PivotValueGroup( + group.values().stream().map(expression -> rewriteExpression(expression, Map.of())).toList(), + group.alias(), + group.span())) + .toList(), + pivot.groupBy().stream().map(expression -> rewriteExpression(expression, Map.of())).toList(), + pivot.span()); + case SubqueryRelation subquery -> new SubqueryRelation(rewrite(subquery.query()), subquery.span()); + case TablePlaceholder table -> table; + case TableReference table -> table; + case UnnestRelation unnest -> new UnnestRelation( + unnest.expressions().stream().map(expression -> rewriteExpression(expression, Map.of())).toList(), + unnest.alias(), + unnest.columnAliases(), + unnest.span()); + case ValuesRelation values -> new ValuesRelation( + values.rows().stream() + .map(row -> row.stream().map(expression -> rewriteExpression(expression, Map.of())).toList()) + .toList(), + values.span()); + }; + } + + private static Expression rewriteExpression(Expression expression, Map aliases) + { + return switch (expression) { + case ArrayExpression array -> new ArrayExpression(array.values().stream().map(value -> rewriteExpression(value, aliases)).toList(), array.span()); + case BetweenExpression between -> new BetweenExpression( + rewriteExpression(between.value(), aliases), + rewriteExpression(between.min(), aliases), + rewriteExpression(between.max(), aliases), + between.negated(), + between.predicateSpan(), + between.span()); + case BinaryExpression binary -> new BinaryExpression( + binary.operator(), + rewriteExpression(binary.left(), aliases), + rewriteExpression(binary.right(), aliases), + binary.span()); + case CaseExpression caseExpression -> new CaseExpression( + caseExpression.operand().map(value -> rewriteExpression(value, aliases)), + caseExpression.whenClauses().stream() + .map(when -> new CaseWhen( + rewriteExpression(when.operand(), aliases), + rewriteExpression(when.result(), aliases), + when.span())) + .toList(), + caseExpression.defaultValue().map(value -> rewriteExpression(value, aliases)), + caseExpression.span()); + case CastExpression cast -> new CastExpression(rewriteExpression(cast.value(), aliases), cast.type(), cast.safe(), cast.typeDialect(), cast.span()); + case ColumnReference reference -> alias(reference, aliases).orElse(reference); + case FunctionCall function -> new FunctionCall( + function.nameParts(), + function.arguments().stream().map(argument -> rewriteExpression(argument, aliases)).toList(), + function.distinct(), + function.orderBy().stream().map(item -> rewriteSortItem(item, aliases)).toList(), + function.filter().map(filter -> rewriteExpression(filter, aliases)), + function.nullTreatment(), + function.window().map(window -> rewriteWindow(window, aliases)), + function.span()); + case InCohortExpression in -> new InCohortExpression( + rewriteExpression(in.value(), aliases), + rewriteExpression(in.cohort(), aliases), + in.negated(), + in.predicateSpan(), + in.span()); + case InExpression in -> new InExpression( + rewriteExpression(in.value(), aliases), + in.values().stream().map(value -> rewriteExpression(value, aliases)).toList(), + in.negated(), + in.predicateSpan(), + in.span()); + case InSubqueryExpression in -> new InSubqueryExpression( + rewriteExpression(in.value(), aliases), + rewrite(in.query()), + in.negated(), + in.predicateSpan(), + in.span()); + case IntervalExpression interval -> new IntervalExpression(rewriteExpression(interval.value(), aliases), interval.unit(), interval.span()); + case IsNullExpression isNull -> new IsNullExpression( + rewriteExpression(isNull.value(), aliases), + isNull.negated(), + isNull.predicateSpan(), + isNull.span()); + case LambdaExpression lambda -> { + Map visibleAliases = new HashMap<>(aliases); + lambda.arguments().forEach(argument -> visibleAliases.remove(AliasKey.of(argument))); + yield new LambdaExpression(lambda.arguments(), rewriteExpression(lambda.body(), visibleAliases), lambda.span()); + } + case Literal literal -> literal; + case MemberAccessExpression member -> new MemberAccessExpression(rewriteExpression(member.base(), aliases), member.member(), member.span()); + case Placeholder placeholder -> placeholder; + case ScalarSubqueryExpression subquery -> new ScalarSubqueryExpression(rewrite(subquery.query()), subquery.span()); + case SubscriptExpression subscript -> new SubscriptExpression( + rewriteExpression(subscript.base(), aliases), + rewriteExpression(subscript.index(), aliases), + subscript.span()); + case TupleExpression tuple -> new TupleExpression(tuple.values().stream().map(value -> rewriteExpression(value, aliases)).toList(), tuple.span()); + case UnaryExpression unary -> new UnaryExpression(unary.operator(), rewriteExpression(unary.operand(), aliases), unary.span()); + }; + } + + private static Optional alias(ColumnReference reference, Map aliases) + { + if (reference.parts().size() != 1) { + return Optional.empty(); + } + return Optional.ofNullable(aliases.get(AliasKey.of(reference.parts().getFirst()))); + } + + private static SortItem rewriteSortItem(SortItem item, Map aliases) + { + return new SortItem(rewriteExpression(item.expression(), aliases), item.direction(), item.nullPlacement(), item.span()); + } + + private static WindowDefinition rewriteWindowDefinition(WindowDefinition definition, Map aliases) + { + return new WindowDefinition(definition.name(), rewriteWindowSpecification(definition.specification(), aliases), definition.span()); + } + + private static Window rewriteWindow(Window window, Map aliases) + { + return switch (window) { + case WindowReference reference -> reference; + case WindowSpecification specification -> rewriteWindowSpecification(specification, aliases); + }; + } + + private static WindowSpecification rewriteWindowSpecification(WindowSpecification window, Map aliases) + { + return new WindowSpecification( + window.partitionBy().stream().map(expression -> rewriteExpression(expression, aliases)).toList(), + window.orderBy().stream().map(item -> rewriteSortItem(item, aliases)).toList(), + window.frame().map(frame -> rewriteWindowFrame(frame, aliases)), + window.span()); + } + + private static WindowFrame rewriteWindowFrame(WindowFrame frame, Map aliases) + { + return new WindowFrame( + frame.type(), + rewriteFrameBound(frame.start(), aliases), + frame.end().map(bound -> rewriteFrameBound(bound, aliases)), + frame.span()); + } + + private static FrameBound rewriteFrameBound(FrameBound bound, Map aliases) + { + return new FrameBound(bound.type(), bound.value().map(value -> rewriteExpression(value, aliases)), bound.span()); + } + + private record AliasKey(String value, boolean delimited) + { + private static AliasKey of(Identifier identifier) + { + return new AliasKey(identifier.delimited() ? identifier.value() : identifier.value().toLowerCase(Locale.ENGLISH), identifier.delimited()); + } + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java index f93a4893c676..f327dd5885b7 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -94,6 +94,23 @@ public void testQuotesHogQlPropertyIdentifiersForTrino() .isEqualTo(sqlParser.createStatement("SELECT properties.\"$user_id\", properties.\"$host\" FROM events")); } + @Test + public void testExpandsSelectAliasesInClausesOutsideTrinoAliasScope() + { + assertThat(compiler.compile( + "SELECT number + 1 AS next FROM numbers(3) WHERE next > 1 GROUP BY next HAVING next < 4 ORDER BY next")) + .isEqualTo(compiler.compile( + "SELECT number + 1 AS next FROM numbers(3) WHERE number + 1 > 1 GROUP BY number + 1 HAVING number + 1 < 4 ORDER BY number + 1")); + } + + @Test + public void testDoesNotExpandQualifiedReferencesMatchingSelectAliases() + { + Statement statement = compiler.compile("SELECT number + 1 AS next FROM numbers(3) AS source WHERE source.next > 1"); + + assertThat(SqlFormatter.formatSql(statement)).contains("source.next > 1"); + } + @Test public void testLowersDistinctOrderingAndClickHousePaginationOrder() { @@ -530,18 +547,26 @@ public void testLowersLargeSetOperation() } Query query = (Query) compiler.compile(hogql.toString()); - Deque pending = new ArrayDeque<>(); - pending.add(query); + Deque pending = new ArrayDeque<>(); + pending.add(new NodeDepth(query, 0)); int querySpecificationCount = 0; + int maximumUnionDepth = 0; while (!pending.isEmpty()) { - Node node = pending.removeFirst(); + NodeDepth current = pending.removeFirst(); + Node node = current.node(); if (node instanceof QuerySpecification) { querySpecificationCount++; } - pending.addAll(node.getChildren()); + if (node instanceof Union) { + maximumUnionDepth = Math.max(maximumUnionDepth, current.depth()); + } + for (Node child : node.getChildren()) { + pending.add(new NodeDepth(child, current.depth() + 1)); + } } assertThat(querySpecificationCount).isEqualTo(operandCount); + assertThat(maximumUnionDepth).isLessThan(20); } @Test @@ -876,4 +901,6 @@ private static List parameters(Statement statement) .sorted(Comparator.comparingInt(Parameter::getId)) .toList(); } + + private record NodeDepth(Node node, int depth) {} } diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java index f9fbf67679e4..d00ec25238eb 100644 --- a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -142,6 +142,16 @@ public void testCompilerResolvesV0FunctionsWithoutCatalogContext() "CASE WHEN first THEN 'one' WHEN second THEN 'two' ELSE 'other' END FROM metrics")); } + @Test + public void testToIntConvertsDateToUnixDayNumber() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT toInt(toDate('2022-01-01')), toInt(value) FROM metrics")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT date_diff('day', CAST('1970-01-01' AS date), CAST('2022-01-01' AS date)), CAST(value AS bigint) FROM metrics")); + } + @Test public void testCompilerResolvesExponentialAndWeekStartModes() { diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java index 7e8e99ed6e1f..71b2622d801a 100644 --- a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -650,12 +650,46 @@ private QueryOperand applySetOperationPrecedence(List operands, Li unionAndExceptOperands.add(current); current = unionAndExceptOperands.getFirst(); - for (int index = 0; index < unionAndExceptOperators.size(); index++) { - current = combineSetOperation(current, unionAndExceptOperators.get(index), unionAndExceptOperands.get(index + 1)); + for (int index = 0; index < unionAndExceptOperators.size(); ) { + SetOperator operator = unionAndExceptOperators.get(index); + if (operator.type() != SetOperationType.UNION) { + current = combineSetOperation(current, operator, unionAndExceptOperands.get(index + 1)); + index++; + continue; + } + + int runEnd = index + 1; + while (runEnd < unionAndExceptOperators.size()) { + SetOperator next = unionAndExceptOperators.get(runEnd); + if (next.type() != SetOperationType.UNION || next.distinct() != operator.distinct()) { + break; + } + runEnd++; + } + List runOperands = new ArrayList<>(runEnd - index + 1); + runOperands.add(current); + runOperands.addAll(unionAndExceptOperands.subList(index + 1, runEnd + 1)); + current = combineAssociativeSetOperations( + runOperands, + unionAndExceptOperators.subList(index, runEnd), + 0, + runOperands.size()); + index = runEnd; } return current; } + private QueryOperand combineAssociativeSetOperations(List operands, List operators, int start, int end) + { + if (end - start == 1) { + return operands.get(start); + } + int split = (start + end) / 2; + QueryOperand left = combineAssociativeSetOperations(operands, operators, start, split); + QueryOperand right = combineAssociativeSetOperations(operands, operators, split, end); + return combineSetOperation(left, operators.get(split - 1), right); + } + private QueryOperand combineSetOperation(QueryOperand left, SetOperator operator, QueryOperand right) { SourceSpan span = enclosingSpan(left.span(), right.span()); diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java index bcde05a520d5..2508679a1cdb 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java @@ -194,6 +194,12 @@ public PreparedQuery prepareQuery(Session session, Statement wrappedStatement) return prepareQuery(session, wrappedStatement, Optional.empty()); } + public PreparedQuery prepareQuery(Session session, Statement wrappedStatement, List suppliedParameters) + throws ParsingException, TrinoException + { + return prepareQuery(session, wrappedStatement, Optional.of(ImmutableList.copyOf(suppliedParameters))); + } + private PreparedQuery prepareQuery(Session session, Statement wrappedStatement, Optional> suppliedParameters) throws ParsingException, TrinoException { diff --git a/core/trino-main/src/main/java/io/trino/sql/rewrite/ExplainRewrite.java b/core/trino-main/src/main/java/io/trino/sql/rewrite/ExplainRewrite.java index f09c477512f2..cc3ea775f7ca 100644 --- a/core/trino-main/src/main/java/io/trino/sql/rewrite/ExplainRewrite.java +++ b/core/trino-main/src/main/java/io/trino/sql/rewrite/ExplainRewrite.java @@ -71,7 +71,7 @@ public Statement rewrite( WarningCollector warningCollector, PlanOptimizersStatsCollector planOptimizersStatsCollector) { - return (Statement) new Visitor(session, sessionPropertyResolver, queryPreparer, queryExplainerFactory.createQueryExplainer(analyzerFactory), warningCollector, planOptimizersStatsCollector).process(node, null); + return (Statement) new Visitor(session, sessionPropertyResolver, queryPreparer, queryExplainerFactory.createQueryExplainer(analyzerFactory), parameter, warningCollector, planOptimizersStatsCollector).process(node, null); } private static final class Visitor @@ -81,6 +81,7 @@ private static final class Visitor private final SessionPropertyResolver sessionPropertyResolver; private final QueryPreparer queryPreparer; private final QueryExplainer queryExplainer; + private final List parameters; private final WarningCollector warningCollector; private final PlanOptimizersStatsCollector planOptimizersStatsCollector; @@ -89,6 +90,7 @@ public Visitor( SessionPropertyResolver sessionPropertyResolver, QueryPreparer queryPreparer, QueryExplainer queryExplainer, + List parameters, WarningCollector warningCollector, PlanOptimizersStatsCollector planOptimizersStatsCollector) { @@ -96,6 +98,7 @@ public Visitor( this.sessionPropertyResolver = requireNonNull(sessionPropertyResolver, "sessionPropertyResolver is null"); this.queryPreparer = requireNonNull(queryPreparer, "queryPreparer is null"); this.queryExplainer = requireNonNull(queryExplainer, "queryExplainer is null"); + this.parameters = List.copyOf(requireNonNull(parameters, "parameters is null")); this.warningCollector = requireNonNull(warningCollector, "warningCollector is null"); this.planOptimizersStatsCollector = planOptimizersStatsCollector; } @@ -138,7 +141,9 @@ protected Node visitExplain(Explain node, Void context) private Node getQueryPlan(Explain node, ExplainType.Type planType, ExplainFormat.Type planFormat) throws IllegalArgumentException { - PreparedQuery preparedQuery = queryPreparer.prepareQuery(session, node.getStatement()); + PreparedQuery preparedQuery = parameters.isEmpty() + ? queryPreparer.prepareQuery(session, node.getStatement()) + : queryPreparer.prepareQuery(session, node.getStatement(), parameters); Session resolvedSession = sessionPropertyResolver .getSessionPropertiesApplier(preparedQuery) .apply(session); diff --git a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java index 3f583dd6b695..96f74473cf39 100644 --- a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java +++ b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java @@ -563,6 +563,44 @@ public void testExplainUsesNativePlanner() assertThat(row).singleElement().asString().contains("TableScan")); } + @Test + public void testExplainBindsScopedVariables() + throws Exception + { + List results = runHogQlToCompletion(hogQlExplainRequestWithScopedBinding( + "SELECT nationkey FROM tpch.tiny.nation WHERE name = {variables.name}")); + + assertThat(results.getLast().getError()).isNull(); + assertThat(rows(results)).singleElement().satisfies(row -> + assertThat(row).singleElement().asString().contains("ALGERIA")); + } + + @Test + public void testExplainExpandsSelectAliasInWhere() + throws Exception + { + List results = runHogQlToCompletion(hogQlExplainRequest( + "SELECT nationkey + 1 AS next_key FROM tpch.tiny.nation WHERE next_key > 1", + "DISTRIBUTED", + "TEXT")); + + assertThat(results.getLast().getError()).isNull(); + assertThat(rows(results)).singleElement().satisfies(row -> + assertThat(row).singleElement().asString().contains("nationkey")); + } + + @Test + public void testExplainExpandsSelectAliasContainingScopedVariable() + throws Exception + { + List results = runHogQlToCompletion(hogQlExplainRequestWithScopedBinding( + "SELECT nationkey + length({variables.name}) AS next_key FROM tpch.tiny.nation WHERE next_key > 1")); + + assertThat(results.getLast().getError()).isNull(); + assertThat(rows(results)).singleElement().satisfies(row -> + assertThat(row).singleElement().asString().contains("bigint '7'")); + } + @Test public void testEndpointRequiresJsonContentType() { @@ -755,6 +793,19 @@ private static String hogQlRequestWithScopedBindings(String query) """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); } + private static String hogQlExplainRequestWithScopedBinding(String query) + { + return """ + { + "query": %s, + "protocolVersion": 1, + "languageVersion": "%s", + "variables": {"name": {"type": "varchar", "value": "ALGERIA"}}, + "explain": {"type": "LOGICAL", "format": "TEXT"} + } + """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); + } + private static String hogQlRequestWithModifier(String query) { return """